1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41#ifndef __ISCSI_ISER_H__
42#define __ISCSI_ISER_H__
43
44#include <linux/types.h>
45#include <linux/net.h>
46#include <linux/printk.h>
47#include <scsi/libiscsi.h>
48#include <scsi/scsi_transport_iscsi.h>
49#include <scsi/scsi_cmnd.h>
50#include <scsi/scsi_device.h>
51#include <scsi/iser.h>
52
53#include <linux/interrupt.h>
54#include <linux/wait.h>
55#include <linux/sched.h>
56#include <linux/list.h>
57#include <linux/slab.h>
58#include <linux/dma-mapping.h>
59#include <linux/mutex.h>
60#include <linux/mempool.h>
61#include <linux/uio.h>
62
63#include <linux/socket.h>
64#include <linux/in.h>
65#include <linux/in6.h>
66
67#include <rdma/ib_verbs.h>
68#include <rdma/rdma_cm.h>
69
70#define DRV_NAME "iser"
71#define PFX DRV_NAME ": "
72#define DRV_VER "1.6"
73
74#define iser_dbg(fmt, arg...) \
75 do { \
76 if (unlikely(iser_debug_level > 2)) \
77 printk(KERN_DEBUG PFX "%s: " fmt,\
78 __func__ , ## arg); \
79 } while (0)
80
81#define iser_warn(fmt, arg...) \
82 do { \
83 if (unlikely(iser_debug_level > 0)) \
84 pr_warn(PFX "%s: " fmt, \
85 __func__ , ## arg); \
86 } while (0)
87
88#define iser_info(fmt, arg...) \
89 do { \
90 if (unlikely(iser_debug_level > 1)) \
91 pr_info(PFX "%s: " fmt, \
92 __func__ , ## arg); \
93 } while (0)
94
95#define iser_err(fmt, arg...) \
96 pr_err(PFX "%s: " fmt, __func__ , ## arg)
97
98
99#define ISER_DEF_MAX_SECTORS 1024
100#define ISCSI_ISER_DEF_SG_TABLESIZE \
101 ((ISER_DEF_MAX_SECTORS * SECTOR_SIZE) >> ilog2(SZ_4K))
102
103#define ISCSI_ISER_MAX_SG_TABLESIZE ((32768 * SECTOR_SIZE) >> ilog2(SZ_4K))
104
105#define ISER_DEF_XMIT_CMDS_DEFAULT 512
106#if ISCSI_DEF_XMIT_CMDS_MAX > ISER_DEF_XMIT_CMDS_DEFAULT
107 #define ISER_DEF_XMIT_CMDS_MAX ISCSI_DEF_XMIT_CMDS_MAX
108#else
109 #define ISER_DEF_XMIT_CMDS_MAX ISER_DEF_XMIT_CMDS_DEFAULT
110#endif
111#define ISER_DEF_CMD_PER_LUN ISER_DEF_XMIT_CMDS_MAX
112
113
114
115#define ISER_MAX_RX_MISC_PDUS 4
116
117#define ISER_MAX_TX_MISC_PDUS 6
118
119
120#define ISER_QP_MAX_RECV_DTOS (ISER_DEF_XMIT_CMDS_MAX)
121
122#define ISER_MIN_POSTED_RX (ISER_DEF_XMIT_CMDS_MAX >> 2)
123
124
125
126
127
128
129
130#define ISER_INFLIGHT_DATAOUTS 8
131
132#define ISER_QP_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
133 (1 + ISER_INFLIGHT_DATAOUTS) + \
134 ISER_MAX_TX_MISC_PDUS + \
135 ISER_MAX_RX_MISC_PDUS)
136
137
138#define ISER_MAX_REG_WR_PER_CMD 5
139
140
141#define ISER_QP_SIG_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
142 (1 + ISER_MAX_REG_WR_PER_CMD) + \
143 ISER_MAX_TX_MISC_PDUS + \
144 ISER_MAX_RX_MISC_PDUS)
145
146#define ISER_GET_MAX_XMIT_CMDS(send_wr) ((send_wr \
147 - ISER_MAX_TX_MISC_PDUS \
148 - ISER_MAX_RX_MISC_PDUS) / \
149 (1 + ISER_INFLIGHT_DATAOUTS))
150
151#define ISER_SIGNAL_CMD_COUNT 32
152
153
154#define ISER_HEADERS_LEN (sizeof(struct iser_ctrl) + sizeof(struct iscsi_hdr))
155
156#define ISER_RECV_DATA_SEG_LEN 128
157#define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
158#define ISER_RX_LOGIN_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
159
160
161#define ISER_OBJECT_NAME_SIZE 64
162
163enum iser_conn_state {
164 ISER_CONN_INIT,
165 ISER_CONN_PENDING,
166 ISER_CONN_UP,
167 ISER_CONN_TERMINATING,
168 ISER_CONN_DOWN,
169 ISER_CONN_STATES_NUM
170};
171
172enum iser_task_status {
173 ISER_TASK_STATUS_INIT = 0,
174 ISER_TASK_STATUS_STARTED,
175 ISER_TASK_STATUS_COMPLETED
176};
177
178enum iser_data_dir {
179 ISER_DIR_IN = 0,
180 ISER_DIR_OUT,
181 ISER_DIRS_NUM
182};
183
184
185
186
187
188
189
190
191
192struct iser_data_buf {
193 struct scatterlist *sg;
194 int size;
195 unsigned long data_len;
196 int dma_nents;
197};
198
199
200struct iser_device;
201struct iscsi_iser_task;
202struct iscsi_endpoint;
203struct iser_reg_resources;
204
205
206
207
208
209
210
211
212struct iser_mem_reg {
213 struct ib_sge sge;
214 u32 rkey;
215 void *mem_h;
216};
217
218enum iser_desc_type {
219 ISCSI_TX_CONTROL ,
220 ISCSI_TX_SCSI_COMMAND,
221 ISCSI_TX_DATAOUT
222};
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241struct iser_tx_desc {
242 struct iser_ctrl iser_header;
243 struct iscsi_hdr iscsi_header;
244 enum iser_desc_type type;
245 u64 dma_addr;
246 struct ib_sge tx_sg[2];
247 int num_sge;
248 struct ib_cqe cqe;
249 bool mapped;
250 struct ib_reg_wr reg_wr;
251 struct ib_send_wr send_wr;
252 struct ib_send_wr inv_wr;
253};
254
255#define ISER_RX_PAD_SIZE (256 - (ISER_RX_PAYLOAD_SIZE + \
256 sizeof(u64) + sizeof(struct ib_sge) + \
257 sizeof(struct ib_cqe)))
258
259
260
261
262
263
264
265
266
267
268
269struct iser_rx_desc {
270 struct iser_ctrl iser_header;
271 struct iscsi_hdr iscsi_header;
272 char data[ISER_RECV_DATA_SEG_LEN];
273 u64 dma_addr;
274 struct ib_sge rx_sg;
275 struct ib_cqe cqe;
276 char pad[ISER_RX_PAD_SIZE];
277} __packed;
278
279
280
281
282
283
284
285
286
287
288
289struct iser_login_desc {
290 void *req;
291 void *rsp;
292 u64 req_dma;
293 u64 rsp_dma;
294 struct ib_sge sge;
295 struct ib_cqe cqe;
296} __packed;
297
298struct iser_conn;
299struct ib_conn;
300struct iscsi_iser_task;
301
302
303
304
305
306
307
308
309
310
311
312struct iser_device {
313 struct ib_device *ib_device;
314 struct ib_pd *pd;
315 struct ib_event_handler event_handler;
316 struct list_head ig_list;
317 int refcount;
318};
319
320
321
322
323
324
325
326
327struct iser_reg_resources {
328 struct ib_mr *mr;
329 struct ib_mr *sig_mr;
330 u8 mr_valid:1;
331};
332
333
334
335
336
337
338
339
340
341struct iser_fr_desc {
342 struct list_head list;
343 struct iser_reg_resources rsc;
344 bool sig_protected;
345 struct list_head all_list;
346};
347
348
349
350
351
352
353
354
355
356struct iser_fr_pool {
357 struct list_head list;
358 spinlock_t lock;
359 int size;
360 struct list_head all_list;
361};
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378struct ib_conn {
379 struct rdma_cm_id *cma_id;
380 struct ib_qp *qp;
381 struct ib_cq *cq;
382 u32 cq_size;
383 int post_recv_buf_count;
384 u8 sig_count;
385 struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX];
386 struct iser_device *device;
387 struct iser_fr_pool fr_pool;
388 bool pi_support;
389 struct ib_cqe reg_cqe;
390};
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420struct iser_conn {
421 struct ib_conn ib_conn;
422 struct iscsi_conn *iscsi_conn;
423 struct iscsi_endpoint *ep;
424 enum iser_conn_state state;
425 unsigned qp_max_recv_dtos;
426 unsigned qp_max_recv_dtos_mask;
427 unsigned min_posted_rx;
428 u16 max_cmds;
429 char name[ISER_OBJECT_NAME_SIZE];
430 struct work_struct release_work;
431 struct mutex state_mutex;
432 struct completion stop_completion;
433 struct completion ib_completion;
434 struct completion up_completion;
435 struct list_head conn_list;
436 struct iser_login_desc login_desc;
437 unsigned int rx_desc_head;
438 struct iser_rx_desc *rx_descs;
439 u32 num_rx_descs;
440 unsigned short scsi_sg_tablesize;
441 unsigned short pages_per_mr;
442 bool snd_w_inv;
443};
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458struct iscsi_iser_task {
459 struct iser_tx_desc desc;
460 struct iser_conn *iser_conn;
461 enum iser_task_status status;
462 struct scsi_cmnd *sc;
463 int command_sent;
464 int dir[ISER_DIRS_NUM];
465 struct iser_mem_reg rdma_reg[ISER_DIRS_NUM];
466 struct iser_data_buf data[ISER_DIRS_NUM];
467 struct iser_data_buf prot[ISER_DIRS_NUM];
468};
469
470
471
472
473
474
475
476
477
478
479struct iser_global {
480 struct mutex device_list_mutex;
481 struct list_head device_list;
482 struct mutex connlist_mutex;
483 struct list_head connlist;
484 struct kmem_cache *desc_cache;
485};
486
487extern struct iser_global ig;
488extern int iser_debug_level;
489extern bool iser_pi_enable;
490extern int iser_pi_guard;
491extern unsigned int iser_max_sectors;
492extern bool iser_always_reg;
493
494int iser_send_control(struct iscsi_conn *conn,
495 struct iscsi_task *task);
496
497int iser_send_command(struct iscsi_conn *conn,
498 struct iscsi_task *task);
499
500int iser_send_data_out(struct iscsi_conn *conn,
501 struct iscsi_task *task,
502 struct iscsi_data *hdr);
503
504void iscsi_iser_recv(struct iscsi_conn *conn,
505 struct iscsi_hdr *hdr,
506 char *rx_data,
507 int rx_data_len);
508
509void iser_conn_init(struct iser_conn *iser_conn);
510
511void iser_conn_release(struct iser_conn *iser_conn);
512
513int iser_conn_terminate(struct iser_conn *iser_conn);
514
515void iser_release_work(struct work_struct *work);
516
517void iser_err_comp(struct ib_wc *wc, const char *type);
518void iser_login_rsp(struct ib_cq *cq, struct ib_wc *wc);
519void iser_task_rsp(struct ib_cq *cq, struct ib_wc *wc);
520void iser_cmd_comp(struct ib_cq *cq, struct ib_wc *wc);
521void iser_ctrl_comp(struct ib_cq *cq, struct ib_wc *wc);
522void iser_dataout_comp(struct ib_cq *cq, struct ib_wc *wc);
523void iser_reg_comp(struct ib_cq *cq, struct ib_wc *wc);
524
525void iser_task_rdma_init(struct iscsi_iser_task *task);
526
527void iser_task_rdma_finalize(struct iscsi_iser_task *task);
528
529void iser_free_rx_descriptors(struct iser_conn *iser_conn);
530
531void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
532 struct iser_data_buf *mem,
533 enum iser_data_dir cmd_dir);
534
535int iser_reg_mem_fastreg(struct iscsi_iser_task *task,
536 enum iser_data_dir dir,
537 bool all_imm);
538void iser_unreg_mem_fastreg(struct iscsi_iser_task *task,
539 enum iser_data_dir dir);
540
541int iser_connect(struct iser_conn *iser_conn,
542 struct sockaddr *src_addr,
543 struct sockaddr *dst_addr,
544 int non_blocking);
545
546int iser_post_recvl(struct iser_conn *iser_conn);
547int iser_post_recvm(struct iser_conn *iser_conn, int count);
548int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
549 bool signal);
550
551int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
552 struct iser_data_buf *data,
553 enum iser_data_dir iser_dir,
554 enum dma_data_direction dma_dir);
555
556void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task,
557 struct iser_data_buf *data,
558 enum dma_data_direction dir);
559
560int iser_initialize_task_headers(struct iscsi_task *task,
561 struct iser_tx_desc *tx_desc);
562int iser_alloc_rx_descriptors(struct iser_conn *iser_conn,
563 struct iscsi_session *session);
564int iser_alloc_fastreg_pool(struct ib_conn *ib_conn,
565 unsigned cmds_max,
566 unsigned int size);
567void iser_free_fastreg_pool(struct ib_conn *ib_conn);
568u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
569 enum iser_data_dir cmd_dir, sector_t *sector);
570
571static inline struct iser_conn *
572to_iser_conn(struct ib_conn *ib_conn)
573{
574 return container_of(ib_conn, struct iser_conn, ib_conn);
575}
576
577static inline struct iser_rx_desc *
578iser_rx(struct ib_cqe *cqe)
579{
580 return container_of(cqe, struct iser_rx_desc, cqe);
581}
582
583static inline struct iser_tx_desc *
584iser_tx(struct ib_cqe *cqe)
585{
586 return container_of(cqe, struct iser_tx_desc, cqe);
587}
588
589static inline struct iser_login_desc *
590iser_login(struct ib_cqe *cqe)
591{
592 return container_of(cqe, struct iser_login_desc, cqe);
593}
594
595#endif
596