1
2
3
4
5
6
7
8
9
10
11
12#include <linux/slab.h>
13#include <linux/random.h>
14
15#include <target/iscsi/iscsi_target_core.h>
16#include "iscsi_target_util.h"
17#include "iscsi_target_tpg.h"
18#include "iscsi_target_seq_pdu_list.h"
19
20#ifdef DEBUG
21static void iscsit_dump_seq_list(struct iscsi_cmd *cmd)
22{
23 int i;
24 struct iscsi_seq *seq;
25
26 pr_debug("Dumping Sequence List for ITT: 0x%08x:\n",
27 cmd->init_task_tag);
28
29 for (i = 0; i < cmd->seq_count; i++) {
30 seq = &cmd->seq_list[i];
31 pr_debug("i: %d, pdu_start: %d, pdu_count: %d,"
32 " offset: %d, xfer_len: %d, seq_send_order: %d,"
33 " seq_no: %d\n", i, seq->pdu_start, seq->pdu_count,
34 seq->offset, seq->xfer_len, seq->seq_send_order,
35 seq->seq_no);
36 }
37}
38
39static void iscsit_dump_pdu_list(struct iscsi_cmd *cmd)
40{
41 int i;
42 struct iscsi_pdu *pdu;
43
44 pr_debug("Dumping PDU List for ITT: 0x%08x:\n",
45 cmd->init_task_tag);
46
47 for (i = 0; i < cmd->pdu_count; i++) {
48 pdu = &cmd->pdu_list[i];
49 pr_debug("i: %d, offset: %d, length: %d,"
50 " pdu_send_order: %d, seq_no: %d\n", i, pdu->offset,
51 pdu->length, pdu->pdu_send_order, pdu->seq_no);
52 }
53}
54#else
55static void iscsit_dump_seq_list(struct iscsi_cmd *cmd) {}
56static void iscsit_dump_pdu_list(struct iscsi_cmd *cmd) {}
57#endif
58
59static void iscsit_ordered_seq_lists(
60 struct iscsi_cmd *cmd,
61 u8 type)
62{
63 u32 i, seq_count = 0;
64
65 for (i = 0; i < cmd->seq_count; i++) {
66 if (cmd->seq_list[i].type != SEQTYPE_NORMAL)
67 continue;
68 cmd->seq_list[i].seq_send_order = seq_count++;
69 }
70}
71
72static void iscsit_ordered_pdu_lists(
73 struct iscsi_cmd *cmd,
74 u8 type)
75{
76 u32 i, pdu_send_order = 0, seq_no = 0;
77
78 for (i = 0; i < cmd->pdu_count; i++) {
79redo:
80 if (cmd->pdu_list[i].seq_no == seq_no) {
81 cmd->pdu_list[i].pdu_send_order = pdu_send_order++;
82 continue;
83 }
84 seq_no++;
85 pdu_send_order = 0;
86 goto redo;
87 }
88}
89
90
91
92
93
94static void iscsit_create_random_array(u32 *array, u32 count)
95{
96 int i, j, k;
97
98 if (count == 1) {
99 array[0] = 0;
100 return;
101 }
102
103 for (i = 0; i < count; i++) {
104redo:
105 get_random_bytes(&j, sizeof(u32));
106 j = (1 + (int) (9999 + 1) - j) % count;
107 for (k = 0; k < i + 1; k++) {
108 j |= 0x80000000;
109 if ((array[k] & 0x80000000) && (array[k] == j))
110 goto redo;
111 }
112 array[i] = j;
113 }
114
115 for (i = 0; i < count; i++)
116 array[i] &= ~0x80000000;
117}
118
119static int iscsit_randomize_pdu_lists(
120 struct iscsi_cmd *cmd,
121 u8 type)
122{
123 int i = 0;
124 u32 *array, pdu_count, seq_count = 0, seq_no = 0, seq_offset = 0;
125
126 for (pdu_count = 0; pdu_count < cmd->pdu_count; pdu_count++) {
127redo:
128 if (cmd->pdu_list[pdu_count].seq_no == seq_no) {
129 seq_count++;
130 continue;
131 }
132 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
133 if (!array) {
134 pr_err("Unable to allocate memory"
135 " for random array.\n");
136 return -ENOMEM;
137 }
138 iscsit_create_random_array(array, seq_count);
139
140 for (i = 0; i < seq_count; i++)
141 cmd->pdu_list[seq_offset+i].pdu_send_order = array[i];
142
143 kfree(array);
144
145 seq_offset += seq_count;
146 seq_count = 0;
147 seq_no++;
148 goto redo;
149 }
150
151 if (seq_count) {
152 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
153 if (!array) {
154 pr_err("Unable to allocate memory for"
155 " random array.\n");
156 return -ENOMEM;
157 }
158 iscsit_create_random_array(array, seq_count);
159
160 for (i = 0; i < seq_count; i++)
161 cmd->pdu_list[seq_offset+i].pdu_send_order = array[i];
162
163 kfree(array);
164 }
165
166 return 0;
167}
168
169static int iscsit_randomize_seq_lists(
170 struct iscsi_cmd *cmd,
171 u8 type)
172{
173 int i, j = 0;
174 u32 *array, seq_count = cmd->seq_count;
175
176 if ((type == PDULIST_IMMEDIATE) || (type == PDULIST_UNSOLICITED))
177 seq_count--;
178 else if (type == PDULIST_IMMEDIATE_AND_UNSOLICITED)
179 seq_count -= 2;
180
181 if (!seq_count)
182 return 0;
183
184 array = kcalloc(seq_count, sizeof(u32), GFP_KERNEL);
185 if (!array) {
186 pr_err("Unable to allocate memory for random array.\n");
187 return -ENOMEM;
188 }
189 iscsit_create_random_array(array, seq_count);
190
191 for (i = 0; i < cmd->seq_count; i++) {
192 if (cmd->seq_list[i].type != SEQTYPE_NORMAL)
193 continue;
194 cmd->seq_list[i].seq_send_order = array[j++];
195 }
196
197 kfree(array);
198 return 0;
199}
200
201static void iscsit_determine_counts_for_list(
202 struct iscsi_cmd *cmd,
203 struct iscsi_build_list *bl,
204 u32 *seq_count,
205 u32 *pdu_count)
206{
207 int check_immediate = 0;
208 u32 burstlength = 0, offset = 0;
209 u32 unsolicited_data_length = 0;
210 u32 mdsl;
211 struct iscsi_conn *conn = cmd->conn;
212
213 if (cmd->se_cmd.data_direction == DMA_TO_DEVICE)
214 mdsl = cmd->conn->conn_ops->MaxXmitDataSegmentLength;
215 else
216 mdsl = cmd->conn->conn_ops->MaxRecvDataSegmentLength;
217
218 if ((bl->type == PDULIST_IMMEDIATE) ||
219 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
220 check_immediate = 1;
221
222 if ((bl->type == PDULIST_UNSOLICITED) ||
223 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
224 unsolicited_data_length = min(cmd->se_cmd.data_length,
225 conn->sess->sess_ops->FirstBurstLength);
226
227 while (offset < cmd->se_cmd.data_length) {
228 *pdu_count += 1;
229
230 if (check_immediate) {
231 check_immediate = 0;
232 offset += bl->immediate_data_length;
233 *seq_count += 1;
234 if (unsolicited_data_length)
235 unsolicited_data_length -=
236 bl->immediate_data_length;
237 continue;
238 }
239 if (unsolicited_data_length > 0) {
240 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
241 unsolicited_data_length -=
242 (cmd->se_cmd.data_length - offset);
243 offset += (cmd->se_cmd.data_length - offset);
244 continue;
245 }
246 if ((offset + mdsl)
247 >= conn->sess->sess_ops->FirstBurstLength) {
248 unsolicited_data_length -=
249 (conn->sess->sess_ops->FirstBurstLength -
250 offset);
251 offset += (conn->sess->sess_ops->FirstBurstLength -
252 offset);
253 burstlength = 0;
254 *seq_count += 1;
255 continue;
256 }
257
258 offset += mdsl;
259 unsolicited_data_length -= mdsl;
260 continue;
261 }
262 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
263 offset += (cmd->se_cmd.data_length - offset);
264 continue;
265 }
266 if ((burstlength + mdsl) >=
267 conn->sess->sess_ops->MaxBurstLength) {
268 offset += (conn->sess->sess_ops->MaxBurstLength -
269 burstlength);
270 burstlength = 0;
271 *seq_count += 1;
272 continue;
273 }
274
275 burstlength += mdsl;
276 offset += mdsl;
277 }
278}
279
280
281
282
283
284
285static int iscsit_do_build_pdu_and_seq_lists(
286 struct iscsi_cmd *cmd,
287 struct iscsi_build_list *bl)
288{
289 int check_immediate = 0, datapduinorder, datasequenceinorder;
290 u32 burstlength = 0, offset = 0, i = 0, mdsl;
291 u32 pdu_count = 0, seq_no = 0, unsolicited_data_length = 0;
292 struct iscsi_conn *conn = cmd->conn;
293 struct iscsi_pdu *pdu = cmd->pdu_list;
294 struct iscsi_seq *seq = cmd->seq_list;
295
296 if (cmd->se_cmd.data_direction == DMA_TO_DEVICE)
297 mdsl = cmd->conn->conn_ops->MaxXmitDataSegmentLength;
298 else
299 mdsl = cmd->conn->conn_ops->MaxRecvDataSegmentLength;
300
301 datapduinorder = conn->sess->sess_ops->DataPDUInOrder;
302 datasequenceinorder = conn->sess->sess_ops->DataSequenceInOrder;
303
304 if ((bl->type == PDULIST_IMMEDIATE) ||
305 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
306 check_immediate = 1;
307
308 if ((bl->type == PDULIST_UNSOLICITED) ||
309 (bl->type == PDULIST_IMMEDIATE_AND_UNSOLICITED))
310 unsolicited_data_length = min(cmd->se_cmd.data_length,
311 conn->sess->sess_ops->FirstBurstLength);
312
313 while (offset < cmd->se_cmd.data_length) {
314 pdu_count++;
315 if (!datapduinorder) {
316 pdu[i].offset = offset;
317 pdu[i].seq_no = seq_no;
318 }
319 if (!datasequenceinorder && (pdu_count == 1)) {
320 seq[seq_no].pdu_start = i;
321 seq[seq_no].seq_no = seq_no;
322 seq[seq_no].offset = offset;
323 seq[seq_no].orig_offset = offset;
324 }
325
326 if (check_immediate) {
327 check_immediate = 0;
328 if (!datapduinorder) {
329 pdu[i].type = PDUTYPE_IMMEDIATE;
330 pdu[i++].length = bl->immediate_data_length;
331 }
332 if (!datasequenceinorder) {
333 seq[seq_no].type = SEQTYPE_IMMEDIATE;
334 seq[seq_no].pdu_count = 1;
335 seq[seq_no].xfer_len =
336 bl->immediate_data_length;
337 }
338 offset += bl->immediate_data_length;
339 pdu_count = 0;
340 seq_no++;
341 if (unsolicited_data_length)
342 unsolicited_data_length -=
343 bl->immediate_data_length;
344 continue;
345 }
346 if (unsolicited_data_length > 0) {
347 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
348 if (!datapduinorder) {
349 pdu[i].type = PDUTYPE_UNSOLICITED;
350 pdu[i].length =
351 (cmd->se_cmd.data_length - offset);
352 }
353 if (!datasequenceinorder) {
354 seq[seq_no].type = SEQTYPE_UNSOLICITED;
355 seq[seq_no].pdu_count = pdu_count;
356 seq[seq_no].xfer_len = (burstlength +
357 (cmd->se_cmd.data_length - offset));
358 }
359 unsolicited_data_length -=
360 (cmd->se_cmd.data_length - offset);
361 offset += (cmd->se_cmd.data_length - offset);
362 continue;
363 }
364 if ((offset + mdsl) >=
365 conn->sess->sess_ops->FirstBurstLength) {
366 if (!datapduinorder) {
367 pdu[i].type = PDUTYPE_UNSOLICITED;
368 pdu[i++].length =
369 (conn->sess->sess_ops->FirstBurstLength -
370 offset);
371 }
372 if (!datasequenceinorder) {
373 seq[seq_no].type = SEQTYPE_UNSOLICITED;
374 seq[seq_no].pdu_count = pdu_count;
375 seq[seq_no].xfer_len = (burstlength +
376 (conn->sess->sess_ops->FirstBurstLength -
377 offset));
378 }
379 unsolicited_data_length -=
380 (conn->sess->sess_ops->FirstBurstLength -
381 offset);
382 offset += (conn->sess->sess_ops->FirstBurstLength -
383 offset);
384 burstlength = 0;
385 pdu_count = 0;
386 seq_no++;
387 continue;
388 }
389
390 if (!datapduinorder) {
391 pdu[i].type = PDUTYPE_UNSOLICITED;
392 pdu[i++].length = mdsl;
393 }
394 burstlength += mdsl;
395 offset += mdsl;
396 unsolicited_data_length -= mdsl;
397 continue;
398 }
399 if ((offset + mdsl) >= cmd->se_cmd.data_length) {
400 if (!datapduinorder) {
401 pdu[i].type = PDUTYPE_NORMAL;
402 pdu[i].length = (cmd->se_cmd.data_length - offset);
403 }
404 if (!datasequenceinorder) {
405 seq[seq_no].type = SEQTYPE_NORMAL;
406 seq[seq_no].pdu_count = pdu_count;
407 seq[seq_no].xfer_len = (burstlength +
408 (cmd->se_cmd.data_length - offset));
409 }
410 offset += (cmd->se_cmd.data_length - offset);
411 continue;
412 }
413 if ((burstlength + mdsl) >=
414 conn->sess->sess_ops->MaxBurstLength) {
415 if (!datapduinorder) {
416 pdu[i].type = PDUTYPE_NORMAL;
417 pdu[i++].length =
418 (conn->sess->sess_ops->MaxBurstLength -
419 burstlength);
420 }
421 if (!datasequenceinorder) {
422 seq[seq_no].type = SEQTYPE_NORMAL;
423 seq[seq_no].pdu_count = pdu_count;
424 seq[seq_no].xfer_len = (burstlength +
425 (conn->sess->sess_ops->MaxBurstLength -
426 burstlength));
427 }
428 offset += (conn->sess->sess_ops->MaxBurstLength -
429 burstlength);
430 burstlength = 0;
431 pdu_count = 0;
432 seq_no++;
433 continue;
434 }
435
436 if (!datapduinorder) {
437 pdu[i].type = PDUTYPE_NORMAL;
438 pdu[i++].length = mdsl;
439 }
440 burstlength += mdsl;
441 offset += mdsl;
442 }
443
444 if (!datasequenceinorder) {
445 if (bl->data_direction & ISCSI_PDU_WRITE) {
446 if (bl->randomize & RANDOM_R2T_OFFSETS) {
447 if (iscsit_randomize_seq_lists(cmd, bl->type)
448 < 0)
449 return -1;
450 } else
451 iscsit_ordered_seq_lists(cmd, bl->type);
452 } else if (bl->data_direction & ISCSI_PDU_READ) {
453 if (bl->randomize & RANDOM_DATAIN_SEQ_OFFSETS) {
454 if (iscsit_randomize_seq_lists(cmd, bl->type)
455 < 0)
456 return -1;
457 } else
458 iscsit_ordered_seq_lists(cmd, bl->type);
459 }
460
461 iscsit_dump_seq_list(cmd);
462 }
463 if (!datapduinorder) {
464 if (bl->data_direction & ISCSI_PDU_WRITE) {
465 if (bl->randomize & RANDOM_DATAOUT_PDU_OFFSETS) {
466 if (iscsit_randomize_pdu_lists(cmd, bl->type)
467 < 0)
468 return -1;
469 } else
470 iscsit_ordered_pdu_lists(cmd, bl->type);
471 } else if (bl->data_direction & ISCSI_PDU_READ) {
472 if (bl->randomize & RANDOM_DATAIN_PDU_OFFSETS) {
473 if (iscsit_randomize_pdu_lists(cmd, bl->type)
474 < 0)
475 return -1;
476 } else
477 iscsit_ordered_pdu_lists(cmd, bl->type);
478 }
479
480 iscsit_dump_pdu_list(cmd);
481 }
482
483 return 0;
484}
485
486int iscsit_build_pdu_and_seq_lists(
487 struct iscsi_cmd *cmd,
488 u32 immediate_data_length)
489{
490 struct iscsi_build_list bl;
491 u32 pdu_count = 0, seq_count = 1;
492 struct iscsi_conn *conn = cmd->conn;
493 struct iscsi_pdu *pdu = NULL;
494 struct iscsi_seq *seq = NULL;
495
496 struct iscsi_session *sess = conn->sess;
497 struct iscsi_node_attrib *na;
498
499
500
501
502 if (sess->sess_ops->DataSequenceInOrder &&
503 sess->sess_ops->DataPDUInOrder)
504 return 0;
505
506 if (cmd->data_direction == DMA_NONE)
507 return 0;
508
509 na = iscsit_tpg_get_node_attrib(sess);
510 memset(&bl, 0, sizeof(struct iscsi_build_list));
511
512 if (cmd->data_direction == DMA_FROM_DEVICE) {
513 bl.data_direction = ISCSI_PDU_READ;
514 bl.type = PDULIST_NORMAL;
515 if (na->random_datain_pdu_offsets)
516 bl.randomize |= RANDOM_DATAIN_PDU_OFFSETS;
517 if (na->random_datain_seq_offsets)
518 bl.randomize |= RANDOM_DATAIN_SEQ_OFFSETS;
519 } else {
520 bl.data_direction = ISCSI_PDU_WRITE;
521 bl.immediate_data_length = immediate_data_length;
522 if (na->random_r2t_offsets)
523 bl.randomize |= RANDOM_R2T_OFFSETS;
524
525 if (!cmd->immediate_data && !cmd->unsolicited_data)
526 bl.type = PDULIST_NORMAL;
527 else if (cmd->immediate_data && !cmd->unsolicited_data)
528 bl.type = PDULIST_IMMEDIATE;
529 else if (!cmd->immediate_data && cmd->unsolicited_data)
530 bl.type = PDULIST_UNSOLICITED;
531 else if (cmd->immediate_data && cmd->unsolicited_data)
532 bl.type = PDULIST_IMMEDIATE_AND_UNSOLICITED;
533 }
534
535 iscsit_determine_counts_for_list(cmd, &bl, &seq_count, &pdu_count);
536
537 if (!conn->sess->sess_ops->DataSequenceInOrder) {
538 seq = kcalloc(seq_count, sizeof(struct iscsi_seq), GFP_ATOMIC);
539 if (!seq) {
540 pr_err("Unable to allocate struct iscsi_seq list\n");
541 return -ENOMEM;
542 }
543 cmd->seq_list = seq;
544 cmd->seq_count = seq_count;
545 }
546
547 if (!conn->sess->sess_ops->DataPDUInOrder) {
548 pdu = kcalloc(pdu_count, sizeof(struct iscsi_pdu), GFP_ATOMIC);
549 if (!pdu) {
550 pr_err("Unable to allocate struct iscsi_pdu list.\n");
551 kfree(seq);
552 return -ENOMEM;
553 }
554 cmd->pdu_list = pdu;
555 cmd->pdu_count = pdu_count;
556 }
557
558 return iscsit_do_build_pdu_and_seq_lists(cmd, &bl);
559}
560
561struct iscsi_pdu *iscsit_get_pdu_holder(
562 struct iscsi_cmd *cmd,
563 u32 offset,
564 u32 length)
565{
566 u32 i;
567 struct iscsi_pdu *pdu = NULL;
568
569 if (!cmd->pdu_list) {
570 pr_err("struct iscsi_cmd->pdu_list is NULL!\n");
571 return NULL;
572 }
573
574 pdu = &cmd->pdu_list[0];
575
576 for (i = 0; i < cmd->pdu_count; i++)
577 if ((pdu[i].offset == offset) && (pdu[i].length == length))
578 return &pdu[i];
579
580 pr_err("Unable to locate PDU holder for ITT: 0x%08x, Offset:"
581 " %u, Length: %u\n", cmd->init_task_tag, offset, length);
582 return NULL;
583}
584
585struct iscsi_pdu *iscsit_get_pdu_holder_for_seq(
586 struct iscsi_cmd *cmd,
587 struct iscsi_seq *seq)
588{
589 u32 i;
590 struct iscsi_conn *conn = cmd->conn;
591 struct iscsi_pdu *pdu = NULL;
592
593 if (!cmd->pdu_list) {
594 pr_err("struct iscsi_cmd->pdu_list is NULL!\n");
595 return NULL;
596 }
597
598 if (conn->sess->sess_ops->DataSequenceInOrder) {
599redo:
600 pdu = &cmd->pdu_list[cmd->pdu_start];
601
602 for (i = 0; pdu[i].seq_no != cmd->seq_no; i++) {
603 pr_debug("pdu[i].seq_no: %d, pdu[i].pdu"
604 "_send_order: %d, pdu[i].offset: %d,"
605 " pdu[i].length: %d\n", pdu[i].seq_no,
606 pdu[i].pdu_send_order, pdu[i].offset,
607 pdu[i].length);
608
609 if (pdu[i].pdu_send_order == cmd->pdu_send_order) {
610 cmd->pdu_send_order++;
611 return &pdu[i];
612 }
613 }
614
615 cmd->pdu_start += cmd->pdu_send_order;
616 cmd->pdu_send_order = 0;
617 cmd->seq_no++;
618
619 if (cmd->pdu_start < cmd->pdu_count)
620 goto redo;
621
622 pr_err("Command ITT: 0x%08x unable to locate"
623 " struct iscsi_pdu for cmd->pdu_send_order: %u.\n",
624 cmd->init_task_tag, cmd->pdu_send_order);
625 return NULL;
626 } else {
627 if (!seq) {
628 pr_err("struct iscsi_seq is NULL!\n");
629 return NULL;
630 }
631
632 pr_debug("seq->pdu_start: %d, seq->pdu_count: %d,"
633 " seq->seq_no: %d\n", seq->pdu_start, seq->pdu_count,
634 seq->seq_no);
635
636 pdu = &cmd->pdu_list[seq->pdu_start];
637
638 if (seq->pdu_send_order == seq->pdu_count) {
639 pr_err("Command ITT: 0x%08x seq->pdu_send"
640 "_order: %u equals seq->pdu_count: %u\n",
641 cmd->init_task_tag, seq->pdu_send_order,
642 seq->pdu_count);
643 return NULL;
644 }
645
646 for (i = 0; i < seq->pdu_count; i++) {
647 if (pdu[i].pdu_send_order == seq->pdu_send_order) {
648 seq->pdu_send_order++;
649 return &pdu[i];
650 }
651 }
652
653 pr_err("Command ITT: 0x%08x unable to locate iscsi"
654 "_pdu_t for seq->pdu_send_order: %u.\n",
655 cmd->init_task_tag, seq->pdu_send_order);
656 return NULL;
657 }
658
659 return NULL;
660}
661
662struct iscsi_seq *iscsit_get_seq_holder(
663 struct iscsi_cmd *cmd,
664 u32 offset,
665 u32 length)
666{
667 u32 i;
668
669 if (!cmd->seq_list) {
670 pr_err("struct iscsi_cmd->seq_list is NULL!\n");
671 return NULL;
672 }
673
674 for (i = 0; i < cmd->seq_count; i++) {
675 pr_debug("seq_list[i].orig_offset: %d, seq_list[i]."
676 "xfer_len: %d, seq_list[i].seq_no %u\n",
677 cmd->seq_list[i].orig_offset, cmd->seq_list[i].xfer_len,
678 cmd->seq_list[i].seq_no);
679
680 if ((cmd->seq_list[i].orig_offset +
681 cmd->seq_list[i].xfer_len) >=
682 (offset + length))
683 return &cmd->seq_list[i];
684 }
685
686 pr_err("Unable to locate Sequence holder for ITT: 0x%08x,"
687 " Offset: %u, Length: %u\n", cmd->init_task_tag, offset,
688 length);
689 return NULL;
690}
691
