1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41#ifndef __ISCSI_ISER_H__
42#define __ISCSI_ISER_H__
43
44#include <linux/types.h>
45#include <linux/net.h>
46#include <linux/printk.h>
47#include <scsi/libiscsi.h>
48#include <scsi/scsi_transport_iscsi.h>
49#include <scsi/scsi_cmnd.h>
50#include <scsi/scsi_device.h>
51#include <scsi/iser.h>
52
53#include <linux/interrupt.h>
54#include <linux/wait.h>
55#include <linux/sched.h>
56#include <linux/list.h>
57#include <linux/slab.h>
58#include <linux/dma-mapping.h>
59#include <linux/mutex.h>
60#include <linux/mempool.h>
61#include <linux/uio.h>
62
63#include <linux/socket.h>
64#include <linux/in.h>
65#include <linux/in6.h>
66
67#include <rdma/ib_verbs.h>
68#include <rdma/ib_fmr_pool.h>
69#include <rdma/rdma_cm.h>
70
71#define DRV_NAME "iser"
72#define PFX DRV_NAME ": "
73#define DRV_VER "1.6"
74
75#define iser_dbg(fmt, arg...) \
76 do { \
77 if (unlikely(iser_debug_level > 2)) \
78 printk(KERN_DEBUG PFX "%s: " fmt,\
79 __func__ , ## arg); \
80 } while (0)
81
82#define iser_warn(fmt, arg...) \
83 do { \
84 if (unlikely(iser_debug_level > 0)) \
85 pr_warn(PFX "%s: " fmt, \
86 __func__ , ## arg); \
87 } while (0)
88
89#define iser_info(fmt, arg...) \
90 do { \
91 if (unlikely(iser_debug_level > 1)) \
92 pr_info(PFX "%s: " fmt, \
93 __func__ , ## arg); \
94 } while (0)
95
96#define iser_err(fmt, arg...) \
97 pr_err(PFX "%s: " fmt, __func__ , ## arg)
98
99#define SHIFT_4K 12
100#define SIZE_4K (1ULL << SHIFT_4K)
101#define MASK_4K (~(SIZE_4K-1))
102
103
104#define ISER_DEF_MAX_SECTORS 1024
105#define ISCSI_ISER_DEF_SG_TABLESIZE \
106 ((ISER_DEF_MAX_SECTORS * SECTOR_SIZE) >> SHIFT_4K)
107
108#define ISCSI_ISER_MAX_SG_TABLESIZE ((32768 * SECTOR_SIZE) >> SHIFT_4K)
109
110#define ISER_DEF_XMIT_CMDS_DEFAULT 512
111#if ISCSI_DEF_XMIT_CMDS_MAX > ISER_DEF_XMIT_CMDS_DEFAULT
112 #define ISER_DEF_XMIT_CMDS_MAX ISCSI_DEF_XMIT_CMDS_MAX
113#else
114 #define ISER_DEF_XMIT_CMDS_MAX ISER_DEF_XMIT_CMDS_DEFAULT
115#endif
116#define ISER_DEF_CMD_PER_LUN ISER_DEF_XMIT_CMDS_MAX
117
118
119
120#define ISER_MAX_RX_MISC_PDUS 4
121
122#define ISER_MAX_TX_MISC_PDUS 6
123
124
125#define ISER_QP_MAX_RECV_DTOS (ISER_DEF_XMIT_CMDS_MAX)
126
127#define ISER_MIN_POSTED_RX (ISER_DEF_XMIT_CMDS_MAX >> 2)
128
129
130
131
132
133
134
135#define ISER_INFLIGHT_DATAOUTS 8
136
137#define ISER_QP_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
138 (1 + ISER_INFLIGHT_DATAOUTS) + \
139 ISER_MAX_TX_MISC_PDUS + \
140 ISER_MAX_RX_MISC_PDUS)
141
142
143#define ISER_MAX_REG_WR_PER_CMD 5
144
145
146#define ISER_QP_SIG_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
147 (1 + ISER_MAX_REG_WR_PER_CMD) + \
148 ISER_MAX_TX_MISC_PDUS + \
149 ISER_MAX_RX_MISC_PDUS)
150
151#define ISER_GET_MAX_XMIT_CMDS(send_wr) ((send_wr \
152 - ISER_MAX_TX_MISC_PDUS \
153 - ISER_MAX_RX_MISC_PDUS) / \
154 (1 + ISER_INFLIGHT_DATAOUTS))
155
156#define ISER_SIGNAL_CMD_COUNT 32
157
158
159#define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + sizeof(struct iscsi_hdr))
160
161#define ISER_RECV_DATA_SEG_LEN 128
162#define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
163#define ISER_RX_LOGIN_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
164
165
166#define ISER_OBJECT_NAME_SIZE 64
167
168enum iser_conn_state {
169 ISER_CONN_INIT,
170 ISER_CONN_PENDING,
171 ISER_CONN_UP,
172 ISER_CONN_TERMINATING,
173 ISER_CONN_DOWN,
174 ISER_CONN_STATES_NUM
175};
176
177enum iser_task_status {
178 ISER_TASK_STATUS_INIT = 0,
179 ISER_TASK_STATUS_STARTED,
180 ISER_TASK_STATUS_COMPLETED
181};
182
183enum iser_data_dir {
184 ISER_DIR_IN = 0,
185 ISER_DIR_OUT,
186 ISER_DIRS_NUM
187};
188
189
190
191
192
193
194
195
196
197struct iser_data_buf {
198 struct scatterlist *sg;
199 int size;
200 unsigned long data_len;
201 int dma_nents;
202};
203
204
205struct iser_device;
206struct iscsi_iser_task;
207struct iscsi_endpoint;
208struct iser_reg_resources;
209
210
211
212
213
214
215
216
217struct iser_mem_reg {
218 struct ib_sge sge;
219 u32 rkey;
220 void *mem_h;
221};
222
223enum iser_desc_type {
224 ISCSI_TX_CONTROL ,
225 ISCSI_TX_SCSI_COMMAND,
226 ISCSI_TX_DATAOUT
227};
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245struct iser_tx_desc {
246 struct iser_ctrl iser_header;
247 struct iscsi_hdr iscsi_header;
248 enum iser_desc_type type;
249 u64 dma_addr;
250 struct ib_sge tx_sg[2];
251 int num_sge;
252 struct ib_cqe cqe;
253 bool mapped;
254 struct ib_reg_wr reg_wr;
255 struct ib_send_wr send_wr;
256 struct ib_send_wr inv_wr;
257};
258
259#define ISER_RX_PAD_SIZE (256 - (ISER_RX_PAYLOAD_SIZE + \
260 sizeof(u64) + sizeof(struct ib_sge) + \
261 sizeof(struct ib_cqe)))
262
263
264
265
266
267
268
269
270
271
272struct iser_rx_desc {
273 struct iser_ctrl iser_header;
274 struct iscsi_hdr iscsi_header;
275 char data[ISER_RECV_DATA_SEG_LEN];
276 u64 dma_addr;
277 struct ib_sge rx_sg;
278 struct ib_cqe cqe;
279 char pad[ISER_RX_PAD_SIZE];
280} __packed;
281
282
283
284
285
286
287
288
289
290
291
292struct iser_login_desc {
293 void *req;
294 void *rsp;
295 u64 req_dma;
296 u64 rsp_dma;
297 struct ib_sge sge;
298 struct ib_cqe cqe;
299} __packed;
300
301struct iser_conn;
302struct ib_conn;
303struct iscsi_iser_task;
304
305
306
307
308
309
310
311
312struct iser_comp {
313 struct ib_cq *cq;
314 int active_qps;
315};
316
317
318
319
320
321
322
323
324
325
326
327
328struct iser_reg_ops {
329 int (*alloc_reg_res)(struct ib_conn *ib_conn,
330 unsigned cmds_max,
331 unsigned int size);
332 void (*free_reg_res)(struct ib_conn *ib_conn);
333 int (*reg_mem)(struct iscsi_iser_task *iser_task,
334 struct iser_data_buf *mem,
335 struct iser_reg_resources *rsc,
336 struct iser_mem_reg *reg);
337 void (*unreg_mem)(struct iscsi_iser_task *iser_task,
338 enum iser_data_dir cmd_dir);
339 struct iser_fr_desc * (*reg_desc_get)(struct ib_conn *ib_conn);
340 void (*reg_desc_put)(struct ib_conn *ib_conn,
341 struct iser_fr_desc *desc);
342};
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359struct iser_device {
360 struct ib_device *ib_device;
361 struct ib_pd *pd;
362 struct ib_event_handler event_handler;
363 struct list_head ig_list;
364 int refcount;
365 int comps_used;
366 struct iser_comp *comps;
367 const struct iser_reg_ops *reg_ops;
368 bool remote_inv_sup;
369};
370
371
372
373
374
375
376
377
378
379
380struct iser_reg_resources {
381 union {
382 struct ib_mr *mr;
383 struct ib_fmr_pool *fmr_pool;
384 };
385 struct ib_mr *sig_mr;
386 struct iser_page_vec *page_vec;
387 u8 mr_valid:1;
388};
389
390
391
392
393
394
395
396
397struct iser_fr_desc {
398 struct list_head list;
399 struct iser_reg_resources rsc;
400 bool sig_protected;
401 struct list_head all_list;
402};
403
404
405
406
407
408
409
410
411struct iser_fr_pool {
412 struct list_head list;
413 spinlock_t lock;
414 int size;
415 struct list_head all_list;
416};
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431struct ib_conn {
432 struct rdma_cm_id *cma_id;
433 struct ib_qp *qp;
434 int post_recv_buf_count;
435 u8 sig_count;
436 struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX];
437 struct iser_device *device;
438 struct iser_comp *comp;
439 struct iser_fr_pool fr_pool;
440 bool pi_support;
441 struct ib_cqe reg_cqe;
442};
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471struct iser_conn {
472 struct ib_conn ib_conn;
473 struct iscsi_conn *iscsi_conn;
474 struct iscsi_endpoint *ep;
475 enum iser_conn_state state;
476 unsigned qp_max_recv_dtos;
477 unsigned qp_max_recv_dtos_mask;
478 unsigned min_posted_rx;
479 u16 max_cmds;
480 char name[ISER_OBJECT_NAME_SIZE];
481 struct work_struct release_work;
482 struct mutex state_mutex;
483 struct completion stop_completion;
484 struct completion ib_completion;
485 struct completion up_completion;
486 struct list_head conn_list;
487 struct iser_login_desc login_desc;
488 unsigned int rx_desc_head;
489 struct iser_rx_desc *rx_descs;
490 u32 num_rx_descs;
491 unsigned short scsi_sg_tablesize;
492 unsigned short pages_per_mr;
493 bool snd_w_inv;
494};
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509struct iscsi_iser_task {
510 struct iser_tx_desc desc;
511 struct iser_conn *iser_conn;
512 enum iser_task_status status;
513 struct scsi_cmnd *sc;
514 int command_sent;
515 int dir[ISER_DIRS_NUM];
516 struct iser_mem_reg rdma_reg[ISER_DIRS_NUM];
517 struct iser_data_buf data[ISER_DIRS_NUM];
518 struct iser_data_buf prot[ISER_DIRS_NUM];
519};
520
521struct iser_page_vec {
522 u64 *pages;
523 int npages;
524 struct ib_mr fake_mr;
525};
526
527
528
529
530
531
532
533
534
535
536struct iser_global {
537 struct mutex device_list_mutex;
538 struct list_head device_list;
539 struct mutex connlist_mutex;
540 struct list_head connlist;
541 struct kmem_cache *desc_cache;
542};
543
544extern struct iser_global ig;
545extern int iser_debug_level;
546extern bool iser_pi_enable;
547extern int iser_pi_guard;
548extern unsigned int iser_max_sectors;
549extern bool iser_always_reg;
550
551int iser_assign_reg_ops(struct iser_device *device);
552
553int iser_send_control(struct iscsi_conn *conn,
554 struct iscsi_task *task);
555
556int iser_send_command(struct iscsi_conn *conn,
557 struct iscsi_task *task);
558
559int iser_send_data_out(struct iscsi_conn *conn,
560 struct iscsi_task *task,
561 struct iscsi_data *hdr);
562
563void iscsi_iser_recv(struct iscsi_conn *conn,
564 struct iscsi_hdr *hdr,
565 char *rx_data,
566 int rx_data_len);
567
568void iser_conn_init(struct iser_conn *iser_conn);
569
570void iser_conn_release(struct iser_conn *iser_conn);
571
572int iser_conn_terminate(struct iser_conn *iser_conn);
573
574void iser_release_work(struct work_struct *work);
575
576void iser_err_comp(struct ib_wc *wc, const char *type);
577void iser_login_rsp(struct ib_cq *cq, struct ib_wc *wc);
578void iser_task_rsp(struct ib_cq *cq, struct ib_wc *wc);
579void iser_cmd_comp(struct ib_cq *cq, struct ib_wc *wc);
580void iser_ctrl_comp(struct ib_cq *cq, struct ib_wc *wc);
581void iser_dataout_comp(struct ib_cq *cq, struct ib_wc *wc);
582void iser_reg_comp(struct ib_cq *cq, struct ib_wc *wc);
583
584void iser_task_rdma_init(struct iscsi_iser_task *task);
585
586void iser_task_rdma_finalize(struct iscsi_iser_task *task);
587
588void iser_free_rx_descriptors(struct iser_conn *iser_conn);
589
590void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
591 struct iser_data_buf *mem,
592 enum iser_data_dir cmd_dir);
593
594int iser_reg_rdma_mem(struct iscsi_iser_task *task,
595 enum iser_data_dir dir,
596 bool all_imm);
597void iser_unreg_rdma_mem(struct iscsi_iser_task *task,
598 enum iser_data_dir dir);
599
600int iser_connect(struct iser_conn *iser_conn,
601 struct sockaddr *src_addr,
602 struct sockaddr *dst_addr,
603 int non_blocking);
604
605void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task,
606 enum iser_data_dir cmd_dir);
607void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task,
608 enum iser_data_dir cmd_dir);
609
610int iser_post_recvl(struct iser_conn *iser_conn);
611int iser_post_recvm(struct iser_conn *iser_conn, int count);
612int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
613 bool signal);
614
615int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
616 struct iser_data_buf *data,
617 enum iser_data_dir iser_dir,
618 enum dma_data_direction dma_dir);
619
620void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task,
621 struct iser_data_buf *data,
622 enum dma_data_direction dir);
623
624int iser_initialize_task_headers(struct iscsi_task *task,
625 struct iser_tx_desc *tx_desc);
626int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
627 struct iscsi_session *session);
628int iser_alloc_fmr_pool(struct ib_conn *ib_conn,
629 unsigned cmds_max,
630 unsigned int size);
631void iser_free_fmr_pool(struct ib_conn *ib_conn);
632int iser_alloc_fastreg_pool(struct ib_conn *ib_conn,
633 unsigned cmds_max,
634 unsigned int size);
635void iser_free_fastreg_pool(struct ib_conn *ib_conn);
636u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
637 enum iser_data_dir cmd_dir, sector_t *sector);
638struct iser_fr_desc *
639iser_reg_desc_get_fr(struct ib_conn *ib_conn);
640void
641iser_reg_desc_put_fr(struct ib_conn *ib_conn,
642 struct iser_fr_desc *desc);
643struct iser_fr_desc *
644iser_reg_desc_get_fmr(struct ib_conn *ib_conn);
645void
646iser_reg_desc_put_fmr(struct ib_conn *ib_conn,
647 struct iser_fr_desc *desc);
648
649static inline struct iser_conn *
650to_iser_conn(struct ib_conn *ib_conn)
651{
652 return container_of(ib_conn, struct iser_conn, ib_conn);
653}
654
655static inline struct iser_rx_desc *
656iser_rx(struct ib_cqe *cqe)
657{
658 return container_of(cqe, struct iser_rx_desc, cqe);
659}
660
661static inline struct iser_tx_desc *
662iser_tx(struct ib_cqe *cqe)
663{
664 return container_of(cqe, struct iser_tx_desc, cqe);
665}
666
667static inline struct iser_login_desc *
668iser_login(struct ib_cqe *cqe)
669{
670 return container_of(cqe, struct iser_login_desc, cqe);
671}
672
673#endif
674