1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22#include <linux/slab.h>
23#include <linux/random.h>
24
25#include "iscsi_target_core.h"
26#include "iscsi_target_util.h"
27#include "iscsi_target_tpg.h"
28#include "iscsi_target_seq_pdu_list.h"
29
30#define OFFLOAD_BUF_SIZE 32768
31
32#ifdef DEBUG
33static void iscsit_dump_seq_list(struct iscsi_cmd *cmd)
34{
35 int i;
36 struct iscsi_seq *seq;
37
38 pr_debug("Dumping Sequence List for ITT: 0x%08x:\n",
39 cmd->init_task_tag);
40
41 for (i = 0; i < cmd->seq_count; i++) {
42 seq = &cmd->seq_list[i];
43 pr_debug("i: %d, pdu_start: %d, pdu_count: %d,"
44 " offset: %d, xfer_len: %d, seq_send_order: %d,"
45 " seq_no: %d\n", i, seq->pdu_start, seq->pdu_count,
46 seq->offset, seq->xfer_len, seq->seq_send_order,
47 seq->seq_no);
48 }
49}
50
51static void iscsit_dump_pdu_list(struct iscsi_cmd *cmd)
52{
53 int i;
54 struct iscsi_pdu *pdu;
55
56 pr_debug("Dumping PDU List for ITT: 0x%08x:\n",
57 cmd->init_task_tag);
58
59 for (i = 0; i < cmd->pdu_count; i++) {
60 pdu = &cmd->pdu_list[i];
61 pr_debug("i: %d, offset: %d, length: %d,"
62 " pdu_send_order: %d, seq_no: %d\n", i, pdu->offset,
63 pdu->length, pdu->pdu_send_order, pdu->seq_no);
64 }
65}
66#else
67static void iscsit_dump_seq_list(struct iscsi_cmd *cmd) {}
68static void iscsit_dump_pdu_list(struct iscsi_cmd *cmd) {}
69#endif
70
71static void iscsit_ordered_seq_lists(
72 struct iscsi_cmd *cmd,
73 u8 type)
74{
75 u32 i, seq_count = 0;
76
77 for (i = 0; i < cmd->seq_count; i++) {
78 if (cmd->seq_list[i].type != SEQTYPE_NORMAL)
79 continue;
80 cmd->seq_list[i].seq_send_order = seq_count++;
81 }
82}
83
84static void iscsit_ordered_pdu_lists(
85 struct iscsi_cmd *cmd,
86 u8 type)
87{
88 u32 i, pdu_send_order = 0, seq_no = 0;
89
90 for (i = 0; i < cmd->pdu_count; i++) {
91redo:
92 if (cmd->pdu_list[i].seq_no == seq_no) {
93 cmd->pdu_list[i].pdu_send_order = pdu_send_order++;
94 continue;
95 }
96 seq_no++;
97 pdu_send_order = 0;
98 goto redo;
99 }
100}
101
102
103
104
105
106static void iscsit_create_random_array(u32 *array, u32 count)
107{
108 int i, j, k;
109
110 if (count == 1) {
111 array[0] = 0;
112 return;
113 }
114
115 for (i = 0; i < count; i++) {
116redo:
117 get_random_bytes(&j, sizeof(u32));
118 j = (1 + (int) (9999 + 1) - j) % count;
119 for (k = 0; k < i + 1; k++) {
120 j |= 0x80000000;
121 if ((array[k] & 0x80000000) && (array[k] == j))
122 goto redo;
123 }
124 array[i] = j;
125 }
126
127 for (i = 0; i < count; i++)
128 array[i] &= ~0x80000000;
129}
130
131static int iscsit_randomize_pdu_lists(
132 struct iscsi_cmd *cmd,
133 u8 type)
134{
135 int i = 0;
136 u32 *array, pdu_count, seq_count = 0, seq_no = 0, seq_offset = 0;
137
138 for (pdu_count = 0; pdu_count < cmd->pdu_count; pdu_count++) {
139redo:
140 if (cmd->pdu_list[pdu_count].seq_no == seq_no) {
141 seq_count++;
142 continue;
143 }
144 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
145 if (!array) {
146 pr_err("Unable to allocate memory"
147 " for random array.\n");
148 return -ENOMEM;
149 }
150 iscsit_create_random_array(array, seq_count);
151
152 for (i = 0; i < seq_count; i++)
153 cmd->pdu_list[seq_offset+i].pdu_send_order = array[i];
154
155 kfree(array);
156
157 seq_offset += seq_count;
158 seq_count = 0;
159 seq_no++;
160 goto redo;
161 }
162
163 if (seq_count) {
164 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
165 if (!array) {
166 pr_err("Unable to allocate memory for"
167 " random array.\n");
168 return -ENOMEM;
169 }
170 iscsit_create_random_array(array, seq_count);
171
172 for (i = 0; i < seq_count; i++)
173 cmd->pdu_list[seq_offset+i].pdu_send_order = array[i];
174
175 kfree(array);
176 }
177
178 return 0;
179}
180
181static int iscsit_randomize_seq_lists(
182 struct iscsi_cmd *cmd,
183 u8 type)
184{
185 int i, j = 0;
186 u32 *array, seq_count = cmd->seq_count;
187
188 if ((type == PDULIST_IMMEDIATE) || (type == PDULIST_UNSOLICITED))
189 seq_count--;
190 else if (type == PDULIST_IMMEDIATE_AND_UNSOLICITED)
191 seq_count -= 2;
192
193 if (!seq_count)
194 return 0;
195
196 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
197 if (!array) {
198 pr_err("Unable to allocate memory for random array.\n");
199 return -ENOMEM;
200 }
201 iscsit_create_random_array(array, seq_count);
202
203 for (i = 0; i < cmd->seq_count; i++) {
204 if (cmd->seq_list[i].type != SEQTYPE_NORMAL)
205 continue;
206 cmd->seq_list[i].seq_send_order = array[j++];
207 }
208
209 kfree(array);
210 return 0;
211}
212
213static void iscsit_determine_counts_for_list(
214 struct iscsi_cmd *cmd,
215 struct iscsi_build_list *bl,
216 u32 *seq_count,
217 u32 *pdu_count)
218{
219 int check_immediate = 0;
220 u32 burstlength = 0, offset = 0;
221 u32 unsolicited_data_length = 0;
222 u32 mdsl;
223 struct iscsi_conn *conn = cmd->conn;
224
225 if (cmd->se_cmd.data_direction == DMA_TO_DEVICE)
226 mdsl = cmd->conn->conn_ops->MaxXmitDataSegmentLength;
227 else
228 mdsl = cmd->conn->conn_ops->MaxRecvDataSegmentLength;
229
230 if ((bl->type == PDULIST_IMMEDIATE) ||
231 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
232 check_immediate = 1;
233
234 if ((bl->type == PDULIST_UNSOLICITED) ||
235 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
236 unsolicited_data_length = min(cmd->se_cmd.data_length,
237 conn->sess->sess_ops->FirstBurstLength);
238
239 while (offset < cmd->se_cmd.data_length) {
240 *pdu_count += 1;
241
242 if (check_immediate) {
243 check_immediate = 0;
244 offset += bl->immediate_data_length;
245 *seq_count += 1;
246 if (unsolicited_data_length)
247 unsolicited_data_length -=
248 bl->immediate_data_length;
249 continue;
250 }
251 if (unsolicited_data_length > 0) {
252 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
253 unsolicited_data_length -=
254 (cmd->se_cmd.data_length - offset);
255 offset += (cmd->se_cmd.data_length - offset);
256 continue;
257 }
258 if ((offset + mdsl)
259 >= conn->sess->sess_ops->FirstBurstLength) {
260 unsolicited_data_length -=
261 (conn->sess->sess_ops->FirstBurstLength -
262 offset);
263 offset += (conn->sess->sess_ops->FirstBurstLength -
264 offset);
265 burstlength = 0;
266 *seq_count += 1;
267 continue;
268 }
269
270 offset += mdsl;
271 unsolicited_data_length -= mdsl;
272 continue;
273 }
274 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
275 offset += (cmd->se_cmd.data_length - offset);
276 continue;
277 }
278 if ((burstlength + mdsl) >=
279 conn->sess->sess_ops->MaxBurstLength) {
280 offset += (conn->sess->sess_ops->MaxBurstLength -
281 burstlength);
282 burstlength = 0;
283 *seq_count += 1;
284 continue;
285 }
286
287 burstlength += mdsl;
288 offset += mdsl;
289 }
290}
291
292
293
294
295
296
297static int iscsit_do_build_pdu_and_seq_lists(
298 struct iscsi_cmd *cmd,
299 struct iscsi_build_list *bl)
300{
301 int check_immediate = 0, datapduinorder, datasequenceinorder;
302 u32 burstlength = 0, offset = 0, i = 0, mdsl;
303 u32 pdu_count = 0, seq_no = 0, unsolicited_data_length = 0;
304 struct iscsi_conn *conn = cmd->conn;
305 struct iscsi_pdu *pdu = cmd->pdu_list;
306 struct iscsi_seq *seq = cmd->seq_list;
307
308 if (cmd->se_cmd.data_direction == DMA_TO_DEVICE)
309 mdsl = cmd->conn->conn_ops->MaxXmitDataSegmentLength;
310 else
311 mdsl = cmd->conn->conn_ops->MaxRecvDataSegmentLength;
312
313 datapduinorder = conn->sess->sess_ops->DataPDUInOrder;
314 datasequenceinorder = conn->sess->sess_ops->DataSequenceInOrder;
315
316 if ((bl->type == PDULIST_IMMEDIATE) ||
317 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
318 check_immediate = 1;
319
320 if ((bl->type == PDULIST_UNSOLICITED) ||
321 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
322 unsolicited_data_length = min(cmd->se_cmd.data_length,
323 conn->sess->sess_ops->FirstBurstLength);
324
325 while (offset < cmd->se_cmd.data_length) {
326 pdu_count++;
327 if (!datapduinorder) {
328 pdu[i].offset = offset;
329 pdu[i].seq_no = seq_no;
330 }
331 if (!datasequenceinorder && (pdu_count == 1)) {
332 seq[seq_no].pdu_start = i;
333 seq[seq_no].seq_no = seq_no;
334 seq[seq_no].offset = offset;
335 seq[seq_no].orig_offset = offset;
336 }
337
338 if (check_immediate) {
339 check_immediate = 0;
340 if (!datapduinorder) {
341 pdu[i].type = PDUTYPE_IMMEDIATE;
342 pdu[i++].length = bl->immediate_data_length;
343 }
344 if (!datasequenceinorder) {
345 seq[seq_no].type = SEQTYPE_IMMEDIATE;
346 seq[seq_no].pdu_count = 1;
347 seq[seq_no].xfer_len =
348 bl->immediate_data_length;
349 }
350 offset += bl->immediate_data_length;
351 pdu_count = 0;
352 seq_no++;
353 if (unsolicited_data_length)
354 unsolicited_data_length -=
355 bl->immediate_data_length;
356 continue;
357 }
358 if (unsolicited_data_length > 0) {
359 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
360 if (!datapduinorder) {
361 pdu[i].type = PDUTYPE_UNSOLICITED;
362 pdu[i].length =
363 (cmd->se_cmd.data_length - offset);
364 }
365 if (!datasequenceinorder) {
366 seq[seq_no].type = SEQTYPE_UNSOLICITED;
367 seq[seq_no].pdu_count = pdu_count;
368 seq[seq_no].xfer_len = (burstlength +
369 (cmd->se_cmd.data_length - offset));
370 }
371 unsolicited_data_length -=
372 (cmd->se_cmd.data_length - offset);
373 offset += (cmd->se_cmd.data_length - offset);
374 continue;
375 }
376 if ((offset + mdsl) >=
377 conn->sess->sess_ops->FirstBurstLength) {
378 if (!datapduinorder) {
379 pdu[i].type = PDUTYPE_UNSOLICITED;
380 pdu[i++].length =
381 (conn->sess->sess_ops->FirstBurstLength -
382 offset);
383 }
384 if (!datasequenceinorder) {
385 seq[seq_no].type = SEQTYPE_UNSOLICITED;
386 seq[seq_no].pdu_count = pdu_count;
387 seq[seq_no].xfer_len = (burstlength +
388 (conn->sess->sess_ops->FirstBurstLength -
389 offset));
390 }
391 unsolicited_data_length -=
392 (conn->sess->sess_ops->FirstBurstLength -
393 offset);
394 offset += (conn->sess->sess_ops->FirstBurstLength -
395 offset);
396 burstlength = 0;
397 pdu_count = 0;
398 seq_no++;
399 continue;
400 }
401
402 if (!datapduinorder) {
403 pdu[i].type = PDUTYPE_UNSOLICITED;
404 pdu[i++].length = mdsl;
405 }
406 burstlength += mdsl;
407 offset += mdsl;
408 unsolicited_data_length -= mdsl;
409 continue;
410 }
411 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
412 if (!datapduinorder) {
413 pdu[i].type = PDUTYPE_NORMAL;
414 pdu[i].length = (cmd->se_cmd.data_length - offset);
415 }
416 if (!datasequenceinorder) {
417 seq[seq_no].type = SEQTYPE_NORMAL;
418 seq[seq_no].pdu_count = pdu_count;
419 seq[seq_no].xfer_len = (burstlength +
420 (cmd->se_cmd.data_length - offset));
421 }
422 offset += (cmd->se_cmd.data_length - offset);
423 continue;
424 }
425 if ((burstlength + mdsl) >=
426 conn->sess->sess_ops->MaxBurstLength) {
427 if (!datapduinorder) {
428 pdu[i].type = PDUTYPE_NORMAL;
429 pdu[i++].length =
430 (conn->sess->sess_ops->MaxBurstLength -
431 burstlength);
432 }
433 if (!datasequenceinorder) {
434 seq[seq_no].type = SEQTYPE_NORMAL;
435 seq[seq_no].pdu_count = pdu_count;
436 seq[seq_no].xfer_len = (burstlength +
437 (conn->sess->sess_ops->MaxBurstLength -
438 burstlength));
439 }
440 offset += (conn->sess->sess_ops->MaxBurstLength -
441 burstlength);
442 burstlength = 0;
443 pdu_count = 0;
444 seq_no++;
445 continue;
446 }
447
448 if (!datapduinorder) {
449 pdu[i].type = PDUTYPE_NORMAL;
450 pdu[i++].length = mdsl;
451 }
452 burstlength += mdsl;
453 offset += mdsl;
454 }
455
456 if (!datasequenceinorder) {
457 if (bl->data_direction & ISCSI_PDU_WRITE) {
458 if (bl->randomize & RANDOM_R2T_OFFSETS) {
459 if (iscsit_randomize_seq_lists(cmd, bl->type)
460 < 0)
461 return -1;
462 } else
463 iscsit_ordered_seq_lists(cmd, bl->type);
464 } else if (bl->data_direction & ISCSI_PDU_READ) {
465 if (bl->randomize & RANDOM_DATAIN_SEQ_OFFSETS) {
466 if (iscsit_randomize_seq_lists(cmd, bl->type)
467 < 0)
468 return -1;
469 } else
470 iscsit_ordered_seq_lists(cmd, bl->type);
471 }
472
473 iscsit_dump_seq_list(cmd);
474 }
475 if (!datapduinorder) {
476 if (bl->data_direction & ISCSI_PDU_WRITE) {
477 if (bl->randomize & RANDOM_DATAOUT_PDU_OFFSETS) {
478 if (iscsit_randomize_pdu_lists(cmd, bl->type)
479 < 0)
480 return -1;
481 } else
482 iscsit_ordered_pdu_lists(cmd, bl->type);
483 } else if (bl->data_direction & ISCSI_PDU_READ) {
484 if (bl->randomize & RANDOM_DATAIN_PDU_OFFSETS) {
485 if (iscsit_randomize_pdu_lists(cmd, bl->type)
486 < 0)
487 return -1;
488 } else
489 iscsit_ordered_pdu_lists(cmd, bl->type);
490 }
491
492 iscsit_dump_pdu_list(cmd);
493 }
494
495 return 0;
496}
497
498int iscsit_build_pdu_and_seq_lists(
499 struct iscsi_cmd *cmd,
500 u32 immediate_data_length)
501{
502 struct iscsi_build_list bl;
503 u32 pdu_count = 0, seq_count = 1;
504 struct iscsi_conn *conn = cmd->conn;
505 struct iscsi_pdu *pdu = NULL;
506 struct iscsi_seq *seq = NULL;
507
508 struct iscsi_session *sess = conn->sess;
509 struct iscsi_node_attrib *na;
510
511
512
513
514 if (sess->sess_ops->DataSequenceInOrder &&
515 sess->sess_ops->DataPDUInOrder)
516 return 0;
517
518 if (cmd->data_direction == DMA_NONE)
519 return 0;
520
521 na = iscsit_tpg_get_node_attrib(sess);
522 memset(&bl, 0, sizeof(struct iscsi_build_list));
523
524 if (cmd->data_direction == DMA_FROM_DEVICE) {
525 bl.data_direction = ISCSI_PDU_READ;
526 bl.type = PDULIST_NORMAL;
527 if (na->random_datain_pdu_offsets)
528 bl.randomize |= RANDOM_DATAIN_PDU_OFFSETS;
529 if (na->random_datain_seq_offsets)
530 bl.randomize |= RANDOM_DATAIN_SEQ_OFFSETS;
531 } else {
532 bl.data_direction = ISCSI_PDU_WRITE;
533 bl.immediate_data_length = immediate_data_length;
534 if (na->random_r2t_offsets)
535 bl.randomize |= RANDOM_R2T_OFFSETS;
536
537 if (!cmd->immediate_data && !cmd->unsolicited_data)
538 bl.type = PDULIST_NORMAL;
539 else if (cmd->immediate_data && !cmd->unsolicited_data)
540 bl.type = PDULIST_IMMEDIATE;
541 else if (!cmd->immediate_data && cmd->unsolicited_data)
542 bl.type = PDULIST_UNSOLICITED;
543 else if (cmd->immediate_data && cmd->unsolicited_data)
544 bl.type = PDULIST_IMMEDIATE_AND_UNSOLICITED;
545 }
546
547 iscsit_determine_counts_for_list(cmd, &bl, &seq_count, &pdu_count);
548
549 if (!conn->sess->sess_ops->DataSequenceInOrder) {
550 seq = kcalloc(seq_count, sizeof(struct iscsi_seq), GFP_ATOMIC);
551 if (!seq) {
552 pr_err("Unable to allocate struct iscsi_seq list\n");
553 return -ENOMEM;
554 }
555 cmd->seq_list = seq;
556 cmd->seq_count = seq_count;
557 }
558
559 if (!conn->sess->sess_ops->DataPDUInOrder) {
560 pdu = kcalloc(pdu_count, sizeof(struct iscsi_pdu), GFP_ATOMIC);
561 if (!pdu) {
562 pr_err("Unable to allocate struct iscsi_pdu list.\n");
563 kfree(seq);
564 return -ENOMEM;
565 }
566 cmd->pdu_list = pdu;
567 cmd->pdu_count = pdu_count;
568 }
569
570 return iscsit_do_build_pdu_and_seq_lists(cmd, &bl);
571}
572
573struct iscsi_pdu *iscsit_get_pdu_holder(
574 struct iscsi_cmd *cmd,
575 u32 offset,
576 u32 length)
577{
578 u32 i;
579 struct iscsi_pdu *pdu = NULL;
580
581 if (!cmd->pdu_list) {
582 pr_err("struct iscsi_cmd->pdu_list is NULL!\n");
583 return NULL;
584 }
585
586 pdu = &cmd->pdu_list[0];
587
588 for (i = 0; i < cmd->pdu_count; i++)
589 if ((pdu[i].offset == offset) && (pdu[i].length == length))
590 return &pdu[i];
591
592 pr_err("Unable to locate PDU holder for ITT: 0x%08x, Offset:"
593 " %u, Length: %u\n", cmd->init_task_tag, offset, length);
594 return NULL;
595}
596
597struct iscsi_pdu *iscsit_get_pdu_holder_for_seq(
598 struct iscsi_cmd *cmd,
599 struct iscsi_seq *seq)
600{
601 u32 i;
602 struct iscsi_conn *conn = cmd->conn;
603 struct iscsi_pdu *pdu = NULL;
604
605 if (!cmd->pdu_list) {
606 pr_err("struct iscsi_cmd->pdu_list is NULL!\n");
607 return NULL;
608 }
609
610 if (conn->sess->sess_ops->DataSequenceInOrder) {
611redo:
612 pdu = &cmd->pdu_list[cmd->pdu_start];
613
614 for (i = 0; pdu[i].seq_no != cmd->seq_no; i++) {
615 pr_debug("pdu[i].seq_no: %d, pdu[i].pdu"
616 "_send_order: %d, pdu[i].offset: %d,"
617 " pdu[i].length: %d\n", pdu[i].seq_no,
618 pdu[i].pdu_send_order, pdu[i].offset,
619 pdu[i].length);
620
621 if (pdu[i].pdu_send_order == cmd->pdu_send_order) {
622 cmd->pdu_send_order++;
623 return &pdu[i];
624 }
625 }
626
627 cmd->pdu_start += cmd->pdu_send_order;
628 cmd->pdu_send_order = 0;
629 cmd->seq_no++;
630
631 if (cmd->pdu_start < cmd->pdu_count)
632 goto redo;
633
634 pr_err("Command ITT: 0x%08x unable to locate"
635 " struct iscsi_pdu for cmd->pdu_send_order: %u.\n",
636 cmd->init_task_tag, cmd->pdu_send_order);
637 return NULL;
638 } else {
639 if (!seq) {
640 pr_err("struct iscsi_seq is NULL!\n");
641 return NULL;
642 }
643
644 pr_debug("seq->pdu_start: %d, seq->pdu_count: %d,"
645 " seq->seq_no: %d\n", seq->pdu_start, seq->pdu_count,
646 seq->seq_no);
647
648 pdu = &cmd->pdu_list[seq->pdu_start];
649
650 if (seq->pdu_send_order == seq->pdu_count) {
651 pr_err("Command ITT: 0x%08x seq->pdu_send"
652 "_order: %u equals seq->pdu_count: %u\n",
653 cmd->init_task_tag, seq->pdu_send_order,
654 seq->pdu_count);
655 return NULL;
656 }
657
658 for (i = 0; i < seq->pdu_count; i++) {
659 if (pdu[i].pdu_send_order == seq->pdu_send_order) {
660 seq->pdu_send_order++;
661 return &pdu[i];
662 }
663 }
664
665 pr_err("Command ITT: 0x%08x unable to locate iscsi"
666 "_pdu_t for seq->pdu_send_order: %u.\n",
667 cmd->init_task_tag, seq->pdu_send_order);
668 return NULL;
669 }
670
671 return NULL;
672}
673
674struct iscsi_seq *iscsit_get_seq_holder(
675 struct iscsi_cmd *cmd,
676 u32 offset,
677 u32 length)
678{
679 u32 i;
680
681 if (!cmd->seq_list) {
682 pr_err("struct iscsi_cmd->seq_list is NULL!\n");
683 return NULL;
684 }
685
686 for (i = 0; i < cmd->seq_count; i++) {
687 pr_debug("seq_list[i].orig_offset: %d, seq_list[i]."
688 "xfer_len: %d, seq_list[i].seq_no %u\n",
689 cmd->seq_list[i].orig_offset, cmd->seq_list[i].xfer_len,
690 cmd->seq_list[i].seq_no);
691
692 if ((cmd->seq_list[i].orig_offset +
693 cmd->seq_list[i].xfer_len) >=
694 (offset + length))
695 return &cmd->seq_list[i];
696 }
697
698 pr_err("Unable to locate Sequence holder for ITT: 0x%08x,"
699 " Offset: %u, Length: %u\n", cmd->init_task_tag, offset,
700 length);
701 return NULL;
702}
703
