1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/slab.h>
21#include <linux/random.h>
22
23#include "iscsi_target_core.h"
24#include "iscsi_target_util.h"
25#include "iscsi_target_tpg.h"
26#include "iscsi_target_seq_pdu_list.h"
27
28#define OFFLOAD_BUF_SIZE 32768
29
30#ifdef DEBUG
31static void iscsit_dump_seq_list(struct iscsi_cmd *cmd)
32{
33 int i;
34 struct iscsi_seq *seq;
35
36 pr_debug("Dumping Sequence List for ITT: 0x%08x:\n",
37 cmd->init_task_tag);
38
39 for (i = 0; i < cmd->seq_count; i++) {
40 seq = &cmd->seq_list[i];
41 pr_debug("i: %d, pdu_start: %d, pdu_count: %d,"
42 " offset: %d, xfer_len: %d, seq_send_order: %d,"
43 " seq_no: %d\n", i, seq->pdu_start, seq->pdu_count,
44 seq->offset, seq->xfer_len, seq->seq_send_order,
45 seq->seq_no);
46 }
47}
48
49static void iscsit_dump_pdu_list(struct iscsi_cmd *cmd)
50{
51 int i;
52 struct iscsi_pdu *pdu;
53
54 pr_debug("Dumping PDU List for ITT: 0x%08x:\n",
55 cmd->init_task_tag);
56
57 for (i = 0; i < cmd->pdu_count; i++) {
58 pdu = &cmd->pdu_list[i];
59 pr_debug("i: %d, offset: %d, length: %d,"
60 " pdu_send_order: %d, seq_no: %d\n", i, pdu->offset,
61 pdu->length, pdu->pdu_send_order, pdu->seq_no);
62 }
63}
64#else
65static void iscsit_dump_seq_list(struct iscsi_cmd *cmd) {}
66static void iscsit_dump_pdu_list(struct iscsi_cmd *cmd) {}
67#endif
68
69static void iscsit_ordered_seq_lists(
70 struct iscsi_cmd *cmd,
71 u8 type)
72{
73 u32 i, seq_count = 0;
74
75 for (i = 0; i < cmd->seq_count; i++) {
76 if (cmd->seq_list[i].type != SEQTYPE_NORMAL)
77 continue;
78 cmd->seq_list[i].seq_send_order = seq_count++;
79 }
80}
81
82static void iscsit_ordered_pdu_lists(
83 struct iscsi_cmd *cmd,
84 u8 type)
85{
86 u32 i, pdu_send_order = 0, seq_no = 0;
87
88 for (i = 0; i < cmd->pdu_count; i++) {
89redo:
90 if (cmd->pdu_list[i].seq_no == seq_no) {
91 cmd->pdu_list[i].pdu_send_order = pdu_send_order++;
92 continue;
93 }
94 seq_no++;
95 pdu_send_order = 0;
96 goto redo;
97 }
98}
99
100
101
102
103
104static void iscsit_create_random_array(u32 *array, u32 count)
105{
106 int i, j, k;
107
108 if (count == 1) {
109 array[0] = 0;
110 return;
111 }
112
113 for (i = 0; i < count; i++) {
114redo:
115 get_random_bytes(&j, sizeof(u32));
116 j = (1 + (int) (9999 + 1) - j) % count;
117 for (k = 0; k < i + 1; k++) {
118 j |= 0x80000000;
119 if ((array[k] & 0x80000000) && (array[k] == j))
120 goto redo;
121 }
122 array[i] = j;
123 }
124
125 for (i = 0; i < count; i++)
126 array[i] &= ~0x80000000;
127}
128
129static int iscsit_randomize_pdu_lists(
130 struct iscsi_cmd *cmd,
131 u8 type)
132{
133 int i = 0;
134 u32 *array, pdu_count, seq_count = 0, seq_no = 0, seq_offset = 0;
135
136 for (pdu_count = 0; pdu_count < cmd->pdu_count; pdu_count++) {
137redo:
138 if (cmd->pdu_list[pdu_count].seq_no == seq_no) {
139 seq_count++;
140 continue;
141 }
142 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
143 if (!array) {
144 pr_err("Unable to allocate memory"
145 " for random array.\n");
146 return -ENOMEM;
147 }
148 iscsit_create_random_array(array, seq_count);
149
150 for (i = 0; i < seq_count; i++)
151 cmd->pdu_list[seq_offset+i].pdu_send_order = array[i];
152
153 kfree(array);
154
155 seq_offset += seq_count;
156 seq_count = 0;
157 seq_no++;
158 goto redo;
159 }
160
161 if (seq_count) {
162 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
163 if (!array) {
164 pr_err("Unable to allocate memory for"
165 " random array.\n");
166 return -ENOMEM;
167 }
168 iscsit_create_random_array(array, seq_count);
169
170 for (i = 0; i < seq_count; i++)
171 cmd->pdu_list[seq_offset+i].pdu_send_order = array[i];
172
173 kfree(array);
174 }
175
176 return 0;
177}
178
179static int iscsit_randomize_seq_lists(
180 struct iscsi_cmd *cmd,
181 u8 type)
182{
183 int i, j = 0;
184 u32 *array, seq_count = cmd->seq_count;
185
186 if ((type == PDULIST_IMMEDIATE) || (type == PDULIST_UNSOLICITED))
187 seq_count--;
188 else if (type == PDULIST_IMMEDIATE_AND_UNSOLICITED)
189 seq_count -= 2;
190
191 if (!seq_count)
192 return 0;
193
194 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
195 if (!array) {
196 pr_err("Unable to allocate memory for random array.\n");
197 return -ENOMEM;
198 }
199 iscsit_create_random_array(array, seq_count);
200
201 for (i = 0; i < cmd->seq_count; i++) {
202 if (cmd->seq_list[i].type != SEQTYPE_NORMAL)
203 continue;
204 cmd->seq_list[i].seq_send_order = array[j++];
205 }
206
207 kfree(array);
208 return 0;
209}
210
211static void iscsit_determine_counts_for_list(
212 struct iscsi_cmd *cmd,
213 struct iscsi_build_list *bl,
214 u32 *seq_count,
215 u32 *pdu_count)
216{
217 int check_immediate = 0;
218 u32 burstlength = 0, offset = 0;
219 u32 unsolicited_data_length = 0;
220 u32 mdsl;
221 struct iscsi_conn *conn = cmd->conn;
222
223 if (cmd->se_cmd.data_direction == DMA_TO_DEVICE)
224 mdsl = cmd->conn->conn_ops->MaxXmitDataSegmentLength;
225 else
226 mdsl = cmd->conn->conn_ops->MaxRecvDataSegmentLength;
227
228 if ((bl->type == PDULIST_IMMEDIATE) ||
229 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
230 check_immediate = 1;
231
232 if ((bl->type == PDULIST_UNSOLICITED) ||
233 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
234 unsolicited_data_length = min(cmd->se_cmd.data_length,
235 conn->sess->sess_ops->FirstBurstLength);
236
237 while (offset < cmd->se_cmd.data_length) {
238 *pdu_count += 1;
239
240 if (check_immediate) {
241 check_immediate = 0;
242 offset += bl->immediate_data_length;
243 *seq_count += 1;
244 if (unsolicited_data_length)
245 unsolicited_data_length -=
246 bl->immediate_data_length;
247 continue;
248 }
249 if (unsolicited_data_length > 0) {
250 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
251 unsolicited_data_length -=
252 (cmd->se_cmd.data_length - offset);
253 offset += (cmd->se_cmd.data_length - offset);
254 continue;
255 }
256 if ((offset + mdsl)
257 >= conn->sess->sess_ops->FirstBurstLength) {
258 unsolicited_data_length -=
259 (conn->sess->sess_ops->FirstBurstLength -
260 offset);
261 offset += (conn->sess->sess_ops->FirstBurstLength -
262 offset);
263 burstlength = 0;
264 *seq_count += 1;
265 continue;
266 }
267
268 offset += mdsl;
269 unsolicited_data_length -= mdsl;
270 continue;
271 }
272 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
273 offset += (cmd->se_cmd.data_length - offset);
274 continue;
275 }
276 if ((burstlength + mdsl) >=
277 conn->sess->sess_ops->MaxBurstLength) {
278 offset += (conn->sess->sess_ops->MaxBurstLength -
279 burstlength);
280 burstlength = 0;
281 *seq_count += 1;
282 continue;
283 }
284
285 burstlength += mdsl;
286 offset += mdsl;
287 }
288}
289
290
291
292
293
294
295static int iscsit_do_build_pdu_and_seq_lists(
296 struct iscsi_cmd *cmd,
297 struct iscsi_build_list *bl)
298{
299 int check_immediate = 0, datapduinorder, datasequenceinorder;
300 u32 burstlength = 0, offset = 0, i = 0, mdsl;
301 u32 pdu_count = 0, seq_no = 0, unsolicited_data_length = 0;
302 struct iscsi_conn *conn = cmd->conn;
303 struct iscsi_pdu *pdu = cmd->pdu_list;
304 struct iscsi_seq *seq = cmd->seq_list;
305
306 if (cmd->se_cmd.data_direction == DMA_TO_DEVICE)
307 mdsl = cmd->conn->conn_ops->MaxXmitDataSegmentLength;
308 else
309 mdsl = cmd->conn->conn_ops->MaxRecvDataSegmentLength;
310
311 datapduinorder = conn->sess->sess_ops->DataPDUInOrder;
312 datasequenceinorder = conn->sess->sess_ops->DataSequenceInOrder;
313
314 if ((bl->type == PDULIST_IMMEDIATE) ||
315 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
316 check_immediate = 1;
317
318 if ((bl->type == PDULIST_UNSOLICITED) ||
319 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
320 unsolicited_data_length = min(cmd->se_cmd.data_length,
321 conn->sess->sess_ops->FirstBurstLength);
322
323 while (offset < cmd->se_cmd.data_length) {
324 pdu_count++;
325 if (!datapduinorder) {
326 pdu[i].offset = offset;
327 pdu[i].seq_no = seq_no;
328 }
329 if (!datasequenceinorder && (pdu_count == 1)) {
330 seq[seq_no].pdu_start = i;
331 seq[seq_no].seq_no = seq_no;
332 seq[seq_no].offset = offset;
333 seq[seq_no].orig_offset = offset;
334 }
335
336 if (check_immediate) {
337 check_immediate = 0;
338 if (!datapduinorder) {
339 pdu[i].type = PDUTYPE_IMMEDIATE;
340 pdu[i++].length = bl->immediate_data_length;
341 }
342 if (!datasequenceinorder) {
343 seq[seq_no].type = SEQTYPE_IMMEDIATE;
344 seq[seq_no].pdu_count = 1;
345 seq[seq_no].xfer_len =
346 bl->immediate_data_length;
347 }
348 offset += bl->immediate_data_length;
349 pdu_count = 0;
350 seq_no++;
351 if (unsolicited_data_length)
352 unsolicited_data_length -=
353 bl->immediate_data_length;
354 continue;
355 }
356 if (unsolicited_data_length > 0) {
357 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
358 if (!datapduinorder) {
359 pdu[i].type = PDUTYPE_UNSOLICITED;
360 pdu[i].length =
361 (cmd->se_cmd.data_length - offset);
362 }
363 if (!datasequenceinorder) {
364 seq[seq_no].type = SEQTYPE_UNSOLICITED;
365 seq[seq_no].pdu_count = pdu_count;
366 seq[seq_no].xfer_len = (burstlength +
367 (cmd->se_cmd.data_length - offset));
368 }
369 unsolicited_data_length -=
370 (cmd->se_cmd.data_length - offset);
371 offset += (cmd->se_cmd.data_length - offset);
372 continue;
373 }
374 if ((offset + mdsl) >=
375 conn->sess->sess_ops->FirstBurstLength) {
376 if (!datapduinorder) {
377 pdu[i].type = PDUTYPE_UNSOLICITED;
378 pdu[i++].length =
379 (conn->sess->sess_ops->FirstBurstLength -
380 offset);
381 }
382 if (!datasequenceinorder) {
383 seq[seq_no].type = SEQTYPE_UNSOLICITED;
384 seq[seq_no].pdu_count = pdu_count;
385 seq[seq_no].xfer_len = (burstlength +
386 (conn->sess->sess_ops->FirstBurstLength -
387 offset));
388 }
389 unsolicited_data_length -=
390 (conn->sess->sess_ops->FirstBurstLength -
391 offset);
392 offset += (conn->sess->sess_ops->FirstBurstLength -
393 offset);
394 burstlength = 0;
395 pdu_count = 0;
396 seq_no++;
397 continue;
398 }
399
400 if (!datapduinorder) {
401 pdu[i].type = PDUTYPE_UNSOLICITED;
402 pdu[i++].length = mdsl;
403 }
404 burstlength += mdsl;
405 offset += mdsl;
406 unsolicited_data_length -= mdsl;
407 continue;
408 }
409 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
410 if (!datapduinorder) {
411 pdu[i].type = PDUTYPE_NORMAL;
412 pdu[i].length = (cmd->se_cmd.data_length - offset);
413 }
414 if (!datasequenceinorder) {
415 seq[seq_no].type = SEQTYPE_NORMAL;
416 seq[seq_no].pdu_count = pdu_count;
417 seq[seq_no].xfer_len = (burstlength +
418 (cmd->se_cmd.data_length - offset));
419 }
420 offset += (cmd->se_cmd.data_length - offset);
421 continue;
422 }
423 if ((burstlength + mdsl) >=
424 conn->sess->sess_ops->MaxBurstLength) {
425 if (!datapduinorder) {
426 pdu[i].type = PDUTYPE_NORMAL;
427 pdu[i++].length =
428 (conn->sess->sess_ops->MaxBurstLength -
429 burstlength);
430 }
431 if (!datasequenceinorder) {
432 seq[seq_no].type = SEQTYPE_NORMAL;
433 seq[seq_no].pdu_count = pdu_count;
434 seq[seq_no].xfer_len = (burstlength +
435 (conn->sess->sess_ops->MaxBurstLength -
436 burstlength));
437 }
438 offset += (conn->sess->sess_ops->MaxBurstLength -
439 burstlength);
440 burstlength = 0;
441 pdu_count = 0;
442 seq_no++;
443 continue;
444 }
445
446 if (!datapduinorder) {
447 pdu[i].type = PDUTYPE_NORMAL;
448 pdu[i++].length = mdsl;
449 }
450 burstlength += mdsl;
451 offset += mdsl;
452 }
453
454 if (!datasequenceinorder) {
455 if (bl->data_direction & ISCSI_PDU_WRITE) {
456 if (bl->randomize & RANDOM_R2T_OFFSETS) {
457 if (iscsit_randomize_seq_lists(cmd, bl->type)
458 < 0)
459 return -1;
460 } else
461 iscsit_ordered_seq_lists(cmd, bl->type);
462 } else if (bl->data_direction & ISCSI_PDU_READ) {
463 if (bl->randomize & RANDOM_DATAIN_SEQ_OFFSETS) {
464 if (iscsit_randomize_seq_lists(cmd, bl->type)
465 < 0)
466 return -1;
467 } else
468 iscsit_ordered_seq_lists(cmd, bl->type);
469 }
470
471 iscsit_dump_seq_list(cmd);
472 }
473 if (!datapduinorder) {
474 if (bl->data_direction & ISCSI_PDU_WRITE) {
475 if (bl->randomize & RANDOM_DATAOUT_PDU_OFFSETS) {
476 if (iscsit_randomize_pdu_lists(cmd, bl->type)
477 < 0)
478 return -1;
479 } else
480 iscsit_ordered_pdu_lists(cmd, bl->type);
481 } else if (bl->data_direction & ISCSI_PDU_READ) {
482 if (bl->randomize & RANDOM_DATAIN_PDU_OFFSETS) {
483 if (iscsit_randomize_pdu_lists(cmd, bl->type)
484 < 0)
485 return -1;
486 } else
487 iscsit_ordered_pdu_lists(cmd, bl->type);
488 }
489
490 iscsit_dump_pdu_list(cmd);
491 }
492
493 return 0;
494}
495
496int iscsit_build_pdu_and_seq_lists(
497 struct iscsi_cmd *cmd,
498 u32 immediate_data_length)
499{
500 struct iscsi_build_list bl;
501 u32 pdu_count = 0, seq_count = 1;
502 struct iscsi_conn *conn = cmd->conn;
503 struct iscsi_pdu *pdu = NULL;
504 struct iscsi_seq *seq = NULL;
505
506 struct iscsi_session *sess = conn->sess;
507 struct iscsi_node_attrib *na;
508
509
510
511
512 if (sess->sess_ops->DataSequenceInOrder &&
513 sess->sess_ops->DataPDUInOrder)
514 return 0;
515
516 if (cmd->data_direction == DMA_NONE)
517 return 0;
518
519 na = iscsit_tpg_get_node_attrib(sess);
520 memset(&bl, 0, sizeof(struct iscsi_build_list));
521
522 if (cmd->data_direction == DMA_FROM_DEVICE) {
523 bl.data_direction = ISCSI_PDU_READ;
524 bl.type = PDULIST_NORMAL;
525 if (na->random_datain_pdu_offsets)
526 bl.randomize |= RANDOM_DATAIN_PDU_OFFSETS;
527 if (na->random_datain_seq_offsets)
528 bl.randomize |= RANDOM_DATAIN_SEQ_OFFSETS;
529 } else {
530 bl.data_direction = ISCSI_PDU_WRITE;
531 bl.immediate_data_length = immediate_data_length;
532 if (na->random_r2t_offsets)
533 bl.randomize |= RANDOM_R2T_OFFSETS;
534
535 if (!cmd->immediate_data && !cmd->unsolicited_data)
536 bl.type = PDULIST_NORMAL;
537 else if (cmd->immediate_data && !cmd->unsolicited_data)
538 bl.type = PDULIST_IMMEDIATE;
539 else if (!cmd->immediate_data && cmd->unsolicited_data)
540 bl.type = PDULIST_UNSOLICITED;
541 else if (cmd->immediate_data && cmd->unsolicited_data)
542 bl.type = PDULIST_IMMEDIATE_AND_UNSOLICITED;
543 }
544
545 iscsit_determine_counts_for_list(cmd, &bl, &seq_count, &pdu_count);
546
547 if (!conn->sess->sess_ops->DataSequenceInOrder) {
548 seq = kcalloc(seq_count, sizeof(struct iscsi_seq), GFP_ATOMIC);
549 if (!seq) {
550 pr_err("Unable to allocate struct iscsi_seq list\n");
551 return -ENOMEM;
552 }
553 cmd->seq_list = seq;
554 cmd->seq_count = seq_count;
555 }
556
557 if (!conn->sess->sess_ops->DataPDUInOrder) {
558 pdu = kcalloc(pdu_count, sizeof(struct iscsi_pdu), GFP_ATOMIC);
559 if (!pdu) {
560 pr_err("Unable to allocate struct iscsi_pdu list.\n");
561 kfree(seq);
562 return -ENOMEM;
563 }
564 cmd->pdu_list = pdu;
565 cmd->pdu_count = pdu_count;
566 }
567
568 return iscsit_do_build_pdu_and_seq_lists(cmd, &bl);
569}
570
571struct iscsi_pdu *iscsit_get_pdu_holder(
572 struct iscsi_cmd *cmd,
573 u32 offset,
574 u32 length)
575{
576 u32 i;
577 struct iscsi_pdu *pdu = NULL;
578
579 if (!cmd->pdu_list) {
580 pr_err("struct iscsi_cmd->pdu_list is NULL!\n");
581 return NULL;
582 }
583
584 pdu = &cmd->pdu_list[0];
585
586 for (i = 0; i < cmd->pdu_count; i++)
587 if ((pdu[i].offset == offset) && (pdu[i].length == length))
588 return &pdu[i];
589
590 pr_err("Unable to locate PDU holder for ITT: 0x%08x, Offset:"
591 " %u, Length: %u\n", cmd->init_task_tag, offset, length);
592 return NULL;
593}
594
595struct iscsi_pdu *iscsit_get_pdu_holder_for_seq(
596 struct iscsi_cmd *cmd,
597 struct iscsi_seq *seq)
598{
599 u32 i;
600 struct iscsi_conn *conn = cmd->conn;
601 struct iscsi_pdu *pdu = NULL;
602
603 if (!cmd->pdu_list) {
604 pr_err("struct iscsi_cmd->pdu_list is NULL!\n");
605 return NULL;
606 }
607
608 if (conn->sess->sess_ops->DataSequenceInOrder) {
609redo:
610 pdu = &cmd->pdu_list[cmd->pdu_start];
611
612 for (i = 0; pdu[i].seq_no != cmd->seq_no; i++) {
613 pr_debug("pdu[i].seq_no: %d, pdu[i].pdu"
614 "_send_order: %d, pdu[i].offset: %d,"
615 " pdu[i].length: %d\n", pdu[i].seq_no,
616 pdu[i].pdu_send_order, pdu[i].offset,
617 pdu[i].length);
618
619 if (pdu[i].pdu_send_order == cmd->pdu_send_order) {
620 cmd->pdu_send_order++;
621 return &pdu[i];
622 }
623 }
624
625 cmd->pdu_start += cmd->pdu_send_order;
626 cmd->pdu_send_order = 0;
627 cmd->seq_no++;
628
629 if (cmd->pdu_start < cmd->pdu_count)
630 goto redo;
631
632 pr_err("Command ITT: 0x%08x unable to locate"
633 " struct iscsi_pdu for cmd->pdu_send_order: %u.\n",
634 cmd->init_task_tag, cmd->pdu_send_order);
635 return NULL;
636 } else {
637 if (!seq) {
638 pr_err("struct iscsi_seq is NULL!\n");
639 return NULL;
640 }
641
642 pr_debug("seq->pdu_start: %d, seq->pdu_count: %d,"
643 " seq->seq_no: %d\n", seq->pdu_start, seq->pdu_count,
644 seq->seq_no);
645
646 pdu = &cmd->pdu_list[seq->pdu_start];
647
648 if (seq->pdu_send_order == seq->pdu_count) {
649 pr_err("Command ITT: 0x%08x seq->pdu_send"
650 "_order: %u equals seq->pdu_count: %u\n",
651 cmd->init_task_tag, seq->pdu_send_order,
652 seq->pdu_count);
653 return NULL;
654 }
655
656 for (i = 0; i < seq->pdu_count; i++) {
657 if (pdu[i].pdu_send_order == seq->pdu_send_order) {
658 seq->pdu_send_order++;
659 return &pdu[i];
660 }
661 }
662
663 pr_err("Command ITT: 0x%08x unable to locate iscsi"
664 "_pdu_t for seq->pdu_send_order: %u.\n",
665 cmd->init_task_tag, seq->pdu_send_order);
666 return NULL;
667 }
668
669 return NULL;
670}
671
672struct iscsi_seq *iscsit_get_seq_holder(
673 struct iscsi_cmd *cmd,
674 u32 offset,
675 u32 length)
676{
677 u32 i;
678
679 if (!cmd->seq_list) {
680 pr_err("struct iscsi_cmd->seq_list is NULL!\n");
681 return NULL;
682 }
683
684 for (i = 0; i < cmd->seq_count; i++) {
685 pr_debug("seq_list[i].orig_offset: %d, seq_list[i]."
686 "xfer_len: %d, seq_list[i].seq_no %u\n",
687 cmd->seq_list[i].orig_offset, cmd->seq_list[i].xfer_len,
688 cmd->seq_list[i].seq_no);
689
690 if ((cmd->seq_list[i].orig_offset +
691 cmd->seq_list[i].xfer_len) >=
692 (offset + length))
693 return &cmd->seq_list[i];
694 }
695
696 pr_err("Unable to locate Sequence holder for ITT: 0x%08x,"
697 " Offset: %u, Length: %u\n", cmd->init_task_tag, offset,
698 length);
699 return NULL;
700}
701
