1
2
3
4
5
6
7
8
9#include <linux/workqueue.h>
10#include <linux/kthread.h>
11#include <linux/sched/signal.h>
12
13#include <asm/unaligned.h>
14#include <net/tcp.h>
15#include <target/target_core_base.h>
16#include <target/target_core_fabric.h>
17#include "cxgbit.h"
18
19struct sge_opaque_hdr {
20 void *dev;
21 dma_addr_t addr[MAX_SKB_FRAGS + 1];
22};
23
24static const u8 cxgbit_digest_len[] = {0, 4, 4, 8};
25
26#define TX_HDR_LEN (sizeof(struct sge_opaque_hdr) + \
27 sizeof(struct fw_ofld_tx_data_wr))
28
29static struct sk_buff *
30__cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len, bool iso)
31{
32 struct sk_buff *skb = NULL;
33 u8 submode = 0;
34 int errcode;
35 static const u32 hdr_len = TX_HDR_LEN + ISCSI_HDR_LEN;
36
37 if (len) {
38 skb = alloc_skb_with_frags(hdr_len, len,
39 0, &errcode,
40 GFP_KERNEL);
41 if (!skb)
42 return NULL;
43
44 skb_reserve(skb, TX_HDR_LEN);
45 skb_reset_transport_header(skb);
46 __skb_put(skb, ISCSI_HDR_LEN);
47 skb->data_len = len;
48 skb->len += len;
49 submode |= (csk->submode & CXGBIT_SUBMODE_DCRC);
50
51 } else {
52 u32 iso_len = iso ? sizeof(struct cpl_tx_data_iso) : 0;
53
54 skb = alloc_skb(hdr_len + iso_len, GFP_KERNEL);
55 if (!skb)
56 return NULL;
57
58 skb_reserve(skb, TX_HDR_LEN + iso_len);
59 skb_reset_transport_header(skb);
60 __skb_put(skb, ISCSI_HDR_LEN);
61 }
62
63 submode |= (csk->submode & CXGBIT_SUBMODE_HCRC);
64 cxgbit_skcb_submode(skb) = submode;
65 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[submode];
66 cxgbit_skcb_flags(skb) |= SKCBF_TX_NEED_HDR;
67 return skb;
68}
69
70static struct sk_buff *cxgbit_alloc_skb(struct cxgbit_sock *csk, u32 len)
71{
72 return __cxgbit_alloc_skb(csk, len, false);
73}
74
75
76
77
78
79
80
81
82static int cxgbit_is_ofld_imm(const struct sk_buff *skb)
83{
84 int length = skb->len;
85
86 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
87 length += sizeof(struct fw_ofld_tx_data_wr);
88
89 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_ISO))
90 length += sizeof(struct cpl_tx_data_iso);
91
92#define MAX_IMM_TX_PKT_LEN 256
93 return length <= MAX_IMM_TX_PKT_LEN;
94}
95
96
97
98
99
100
101
102static inline unsigned int cxgbit_sgl_len(unsigned int n)
103{
104 n--;
105 return (3 * n) / 2 + (n & 1) + 2;
106}
107
108
109
110
111
112
113
114
115
116static unsigned int cxgbit_calc_tx_flits_ofld(const struct sk_buff *skb)
117{
118 unsigned int flits, cnt;
119
120 if (cxgbit_is_ofld_imm(skb))
121 return DIV_ROUND_UP(skb->len, 8);
122 flits = skb_transport_offset(skb) / 8;
123 cnt = skb_shinfo(skb)->nr_frags;
124 if (skb_tail_pointer(skb) != skb_transport_header(skb))
125 cnt++;
126 return flits + cxgbit_sgl_len(cnt);
127}
128
129#define CXGBIT_ISO_FSLICE 0x1
130#define CXGBIT_ISO_LSLICE 0x2
131static void
132cxgbit_cpl_tx_data_iso(struct sk_buff *skb, struct cxgbit_iso_info *iso_info)
133{
134 struct cpl_tx_data_iso *cpl;
135 unsigned int submode = cxgbit_skcb_submode(skb);
136 unsigned int fslice = !!(iso_info->flags & CXGBIT_ISO_FSLICE);
137 unsigned int lslice = !!(iso_info->flags & CXGBIT_ISO_LSLICE);
138
139 cpl = __skb_push(skb, sizeof(*cpl));
140
141 cpl->op_to_scsi = htonl(CPL_TX_DATA_ISO_OP_V(CPL_TX_DATA_ISO) |
142 CPL_TX_DATA_ISO_FIRST_V(fslice) |
143 CPL_TX_DATA_ISO_LAST_V(lslice) |
144 CPL_TX_DATA_ISO_CPLHDRLEN_V(0) |
145 CPL_TX_DATA_ISO_HDRCRC_V(submode & 1) |
146 CPL_TX_DATA_ISO_PLDCRC_V(((submode >> 1) & 1)) |
147 CPL_TX_DATA_ISO_IMMEDIATE_V(0) |
148 CPL_TX_DATA_ISO_SCSI_V(2));
149
150 cpl->ahs_len = 0;
151 cpl->mpdu = htons(DIV_ROUND_UP(iso_info->mpdu, 4));
152 cpl->burst_size = htonl(DIV_ROUND_UP(iso_info->burst_len, 4));
153 cpl->len = htonl(iso_info->len);
154 cpl->reserved2_seglen_offset = htonl(0);
155 cpl->datasn_offset = htonl(0);
156 cpl->buffer_offset = htonl(0);
157 cpl->reserved3 = 0;
158
159 __skb_pull(skb, sizeof(*cpl));
160}
161
162static void
163cxgbit_tx_data_wr(struct cxgbit_sock *csk, struct sk_buff *skb, u32 dlen,
164 u32 len, u32 credits, u32 compl)
165{
166 struct fw_ofld_tx_data_wr *req;
167 const struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
168 u32 submode = cxgbit_skcb_submode(skb);
169 u32 wr_ulp_mode = 0;
170 u32 hdr_size = sizeof(*req);
171 u32 opcode = FW_OFLD_TX_DATA_WR;
172 u32 immlen = 0;
173 u32 force = is_t5(lldi->adapter_type) ? TX_FORCE_V(!submode) :
174 T6_TX_FORCE_F;
175
176 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO) {
177 opcode = FW_ISCSI_TX_DATA_WR;
178 immlen += sizeof(struct cpl_tx_data_iso);
179 hdr_size += sizeof(struct cpl_tx_data_iso);
180 submode |= 8;
181 }
182
183 if (cxgbit_is_ofld_imm(skb))
184 immlen += dlen;
185
186 req = __skb_push(skb, hdr_size);
187 req->op_to_immdlen = cpu_to_be32(FW_WR_OP_V(opcode) |
188 FW_WR_COMPL_V(compl) |
189 FW_WR_IMMDLEN_V(immlen));
190 req->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(csk->tid) |
191 FW_WR_LEN16_V(credits));
192 req->plen = htonl(len);
193 wr_ulp_mode = FW_OFLD_TX_DATA_WR_ULPMODE_V(ULP_MODE_ISCSI) |
194 FW_OFLD_TX_DATA_WR_ULPSUBMODE_V(submode);
195
196 req->tunnel_to_proxy = htonl((wr_ulp_mode) | force |
197 FW_OFLD_TX_DATA_WR_SHOVE_V(skb_peek(&csk->txq) ? 0 : 1));
198}
199
200static void cxgbit_arp_failure_skb_discard(void *handle, struct sk_buff *skb)
201{
202 kfree_skb(skb);
203}
204
205void cxgbit_push_tx_frames(struct cxgbit_sock *csk)
206{
207 struct sk_buff *skb;
208
209 while (csk->wr_cred && ((skb = skb_peek(&csk->txq)) != NULL)) {
210 u32 dlen = skb->len;
211 u32 len = skb->len;
212 u32 credits_needed;
213 u32 compl = 0;
214 u32 flowclen16 = 0;
215 u32 iso_cpl_len = 0;
216
217 if (cxgbit_skcb_flags(skb) & SKCBF_TX_ISO)
218 iso_cpl_len = sizeof(struct cpl_tx_data_iso);
219
220 if (cxgbit_is_ofld_imm(skb))
221 credits_needed = DIV_ROUND_UP(dlen + iso_cpl_len, 16);
222 else
223 credits_needed = DIV_ROUND_UP((8 *
224 cxgbit_calc_tx_flits_ofld(skb)) +
225 iso_cpl_len, 16);
226
227 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR))
228 credits_needed += DIV_ROUND_UP(
229 sizeof(struct fw_ofld_tx_data_wr), 16);
230
231
232
233
234
235 if (!test_and_set_bit(CSK_TX_DATA_SENT, &csk->com.flags)) {
236 flowclen16 = cxgbit_send_tx_flowc_wr(csk);
237 csk->wr_cred -= flowclen16;
238 csk->wr_una_cred += flowclen16;
239 }
240
241 if (csk->wr_cred < credits_needed) {
242 pr_debug("csk 0x%p, skb %u/%u, wr %d < %u.\n",
243 csk, skb->len, skb->data_len,
244 credits_needed, csk->wr_cred);
245 break;
246 }
247 __skb_unlink(skb, &csk->txq);
248 set_wr_txq(skb, CPL_PRIORITY_DATA, csk->txq_idx);
249 skb->csum = (__force __wsum)(credits_needed + flowclen16);
250 csk->wr_cred -= credits_needed;
251 csk->wr_una_cred += credits_needed;
252
253 pr_debug("csk 0x%p, skb %u/%u, wr %d, left %u, unack %u.\n",
254 csk, skb->len, skb->data_len, credits_needed,
255 csk->wr_cred, csk->wr_una_cred);
256
257 if (likely(cxgbit_skcb_flags(skb) & SKCBF_TX_NEED_HDR)) {
258 len += cxgbit_skcb_tx_extralen(skb);
259
260 if ((csk->wr_una_cred >= (csk->wr_max_cred / 2)) ||
261 (!before(csk->write_seq,
262 csk->snd_una + csk->snd_win))) {
263 compl = 1;
264 csk->wr_una_cred = 0;
265 }
266
267 cxgbit_tx_data_wr(csk, skb, dlen, len, credits_needed,
268 compl);
269 csk->snd_nxt += len;
270
271 } else if ((cxgbit_skcb_flags(skb) & SKCBF_TX_FLAG_COMPL) ||
272 (csk->wr_una_cred >= (csk->wr_max_cred / 2))) {
273 struct cpl_close_con_req *req =
274 (struct cpl_close_con_req *)skb->data;
275 req->wr.wr_hi |= htonl(FW_WR_COMPL_F);
276 csk->wr_una_cred = 0;
277 }
278
279 cxgbit_sock_enqueue_wr(csk, skb);
280 t4_set_arp_err_handler(skb, csk,
281 cxgbit_arp_failure_skb_discard);
282
283 pr_debug("csk 0x%p,%u, skb 0x%p, %u.\n",
284 csk, csk->tid, skb, len);
285
286 cxgbit_l2t_send(csk->com.cdev, skb, csk->l2t);
287 }
288}
289
290static bool cxgbit_lock_sock(struct cxgbit_sock *csk)
291{
292 spin_lock_bh(&csk->lock);
293
294 if (before(csk->write_seq, csk->snd_una + csk->snd_win))
295 csk->lock_owner = true;
296
297 spin_unlock_bh(&csk->lock);
298
299 return csk->lock_owner;
300}
301
302static void cxgbit_unlock_sock(struct cxgbit_sock *csk)
303{
304 struct sk_buff_head backlogq;
305 struct sk_buff *skb;
306 void (*fn)(struct cxgbit_sock *, struct sk_buff *);
307
308 skb_queue_head_init(&backlogq);
309
310 spin_lock_bh(&csk->lock);
311 while (skb_queue_len(&csk->backlogq)) {
312 skb_queue_splice_init(&csk->backlogq, &backlogq);
313 spin_unlock_bh(&csk->lock);
314
315 while ((skb = __skb_dequeue(&backlogq))) {
316 fn = cxgbit_skcb_rx_backlog_fn(skb);
317 fn(csk, skb);
318 }
319
320 spin_lock_bh(&csk->lock);
321 }
322
323 csk->lock_owner = false;
324 spin_unlock_bh(&csk->lock);
325}
326
327static int cxgbit_queue_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
328{
329 int ret = 0;
330
331 wait_event_interruptible(csk->ack_waitq, cxgbit_lock_sock(csk));
332
333 if (unlikely((csk->com.state != CSK_STATE_ESTABLISHED) ||
334 signal_pending(current))) {
335 __kfree_skb(skb);
336 __skb_queue_purge(&csk->ppodq);
337 ret = -1;
338 spin_lock_bh(&csk->lock);
339 if (csk->lock_owner) {
340 spin_unlock_bh(&csk->lock);
341 goto unlock;
342 }
343 spin_unlock_bh(&csk->lock);
344 return ret;
345 }
346
347 csk->write_seq += skb->len +
348 cxgbit_skcb_tx_extralen(skb);
349
350 skb_queue_splice_tail_init(&csk->ppodq, &csk->txq);
351 __skb_queue_tail(&csk->txq, skb);
352 cxgbit_push_tx_frames(csk);
353
354unlock:
355 cxgbit_unlock_sock(csk);
356 return ret;
357}
358
359static int
360cxgbit_map_skb(struct iscsi_cmd *cmd, struct sk_buff *skb, u32 data_offset,
361 u32 data_length)
362{
363 u32 i = 0, nr_frags = MAX_SKB_FRAGS;
364 u32 padding = ((-data_length) & 3);
365 struct scatterlist *sg;
366 struct page *page;
367 unsigned int page_off;
368
369 if (padding)
370 nr_frags--;
371
372
373
374
375 sg = &cmd->se_cmd.t_data_sg[data_offset / PAGE_SIZE];
376 page_off = (data_offset % PAGE_SIZE);
377
378 while (data_length && (i < nr_frags)) {
379 u32 cur_len = min_t(u32, data_length, sg->length - page_off);
380
381 page = sg_page(sg);
382
383 get_page(page);
384 skb_fill_page_desc(skb, i, page, sg->offset + page_off,
385 cur_len);
386 skb->data_len += cur_len;
387 skb->len += cur_len;
388 skb->truesize += cur_len;
389
390 data_length -= cur_len;
391 page_off = 0;
392 sg = sg_next(sg);
393 i++;
394 }
395
396 if (data_length)
397 return -1;
398
399 if (padding) {
400 page = alloc_page(GFP_KERNEL | __GFP_ZERO);
401 if (!page)
402 return -1;
403 skb_fill_page_desc(skb, i, page, 0, padding);
404 skb->data_len += padding;
405 skb->len += padding;
406 skb->truesize += padding;
407 }
408
409 return 0;
410}
411
412static int
413cxgbit_tx_datain_iso(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
414 struct iscsi_datain_req *dr)
415{
416 struct iscsi_conn *conn = csk->conn;
417 struct sk_buff *skb;
418 struct iscsi_datain datain;
419 struct cxgbit_iso_info iso_info;
420 u32 data_length = cmd->se_cmd.data_length;
421 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
422 u32 num_pdu, plen, tx_data = 0;
423 bool task_sense = !!(cmd->se_cmd.se_cmd_flags &
424 SCF_TRANSPORT_TASK_SENSE);
425 bool set_statsn = false;
426 int ret = -1;
427
428 while (data_length) {
429 num_pdu = (data_length + mrdsl - 1) / mrdsl;
430 if (num_pdu > csk->max_iso_npdu)
431 num_pdu = csk->max_iso_npdu;
432
433 plen = num_pdu * mrdsl;
434 if (plen > data_length)
435 plen = data_length;
436
437 skb = __cxgbit_alloc_skb(csk, 0, true);
438 if (unlikely(!skb))
439 return -ENOMEM;
440
441 memset(skb->data, 0, ISCSI_HDR_LEN);
442 cxgbit_skcb_flags(skb) |= SKCBF_TX_ISO;
443 cxgbit_skcb_submode(skb) |= (csk->submode &
444 CXGBIT_SUBMODE_DCRC);
445 cxgbit_skcb_tx_extralen(skb) = (num_pdu *
446 cxgbit_digest_len[cxgbit_skcb_submode(skb)]) +
447 ((num_pdu - 1) * ISCSI_HDR_LEN);
448
449 memset(&datain, 0, sizeof(struct iscsi_datain));
450 memset(&iso_info, 0, sizeof(iso_info));
451
452 if (!tx_data)
453 iso_info.flags |= CXGBIT_ISO_FSLICE;
454
455 if (!(data_length - plen)) {
456 iso_info.flags |= CXGBIT_ISO_LSLICE;
457 if (!task_sense) {
458 datain.flags = ISCSI_FLAG_DATA_STATUS;
459 iscsit_increment_maxcmdsn(cmd, conn->sess);
460 cmd->stat_sn = conn->stat_sn++;
461 set_statsn = true;
462 }
463 }
464
465 iso_info.burst_len = num_pdu * mrdsl;
466 iso_info.mpdu = mrdsl;
467 iso_info.len = ISCSI_HDR_LEN + plen;
468
469 cxgbit_cpl_tx_data_iso(skb, &iso_info);
470
471 datain.offset = tx_data;
472 datain.data_sn = cmd->data_sn - 1;
473
474 iscsit_build_datain_pdu(cmd, conn, &datain,
475 (struct iscsi_data_rsp *)skb->data,
476 set_statsn);
477
478 ret = cxgbit_map_skb(cmd, skb, tx_data, plen);
479 if (unlikely(ret)) {
480 __kfree_skb(skb);
481 goto out;
482 }
483
484 ret = cxgbit_queue_skb(csk, skb);
485 if (unlikely(ret))
486 goto out;
487
488 tx_data += plen;
489 data_length -= plen;
490
491 cmd->read_data_done += plen;
492 cmd->data_sn += num_pdu;
493 }
494
495 dr->dr_complete = DATAIN_COMPLETE_NORMAL;
496
497 return 0;
498
499out:
500 return ret;
501}
502
503static int
504cxgbit_tx_datain(struct cxgbit_sock *csk, struct iscsi_cmd *cmd,
505 const struct iscsi_datain *datain)
506{
507 struct sk_buff *skb;
508 int ret = 0;
509
510 skb = cxgbit_alloc_skb(csk, 0);
511 if (unlikely(!skb))
512 return -ENOMEM;
513
514 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
515
516 if (datain->length) {
517 cxgbit_skcb_submode(skb) |= (csk->submode &
518 CXGBIT_SUBMODE_DCRC);
519 cxgbit_skcb_tx_extralen(skb) =
520 cxgbit_digest_len[cxgbit_skcb_submode(skb)];
521 }
522
523 ret = cxgbit_map_skb(cmd, skb, datain->offset, datain->length);
524 if (ret < 0) {
525 __kfree_skb(skb);
526 return ret;
527 }
528
529 return cxgbit_queue_skb(csk, skb);
530}
531
532static int
533cxgbit_xmit_datain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
534 struct iscsi_datain_req *dr,
535 const struct iscsi_datain *datain)
536{
537 struct cxgbit_sock *csk = conn->context;
538 u32 data_length = cmd->se_cmd.data_length;
539 u32 padding = ((-data_length) & 3);
540 u32 mrdsl = conn->conn_ops->MaxRecvDataSegmentLength;
541
542 if ((data_length > mrdsl) && (!dr->recovery) &&
543 (!padding) && (!datain->offset) && csk->max_iso_npdu) {
544 atomic_long_add(data_length - datain->length,
545 &conn->sess->tx_data_octets);
546 return cxgbit_tx_datain_iso(csk, cmd, dr);
547 }
548
549 return cxgbit_tx_datain(csk, cmd, datain);
550}
551
552static int
553cxgbit_xmit_nondatain_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
554 const void *data_buf, u32 data_buf_len)
555{
556 struct cxgbit_sock *csk = conn->context;
557 struct sk_buff *skb;
558 u32 padding = ((-data_buf_len) & 3);
559
560 skb = cxgbit_alloc_skb(csk, data_buf_len + padding);
561 if (unlikely(!skb))
562 return -ENOMEM;
563
564 memcpy(skb->data, cmd->pdu, ISCSI_HDR_LEN);
565
566 if (data_buf_len) {
567 u32 pad_bytes = 0;
568
569 skb_store_bits(skb, ISCSI_HDR_LEN, data_buf, data_buf_len);
570
571 if (padding)
572 skb_store_bits(skb, ISCSI_HDR_LEN + data_buf_len,
573 &pad_bytes, padding);
574 }
575
576 cxgbit_skcb_tx_extralen(skb) = cxgbit_digest_len[
577 cxgbit_skcb_submode(skb)];
578
579 return cxgbit_queue_skb(csk, skb);
580}
581
582int
583cxgbit_xmit_pdu(struct iscsi_conn *conn, struct iscsi_cmd *cmd,
584 struct iscsi_datain_req *dr, const void *buf, u32 buf_len)
585{
586 if (dr)
587 return cxgbit_xmit_datain_pdu(conn, cmd, dr, buf);
588 else
589 return cxgbit_xmit_nondatain_pdu(conn, cmd, buf, buf_len);
590}
591
592int cxgbit_validate_params(struct iscsi_conn *conn)
593{
594 struct cxgbit_sock *csk = conn->context;
595 struct cxgbit_device *cdev = csk->com.cdev;
596 struct iscsi_param *param;
597 u32 max_xmitdsl;
598
599 param = iscsi_find_param_from_key(MAXXMITDATASEGMENTLENGTH,
600 conn->param_list);
601 if (!param)
602 return -1;
603
604 if (kstrtou32(param->value, 0, &max_xmitdsl) < 0)
605 return -1;
606
607 if (max_xmitdsl > cdev->mdsl) {
608 if (iscsi_change_param_sprintf(
609 conn, "MaxXmitDataSegmentLength=%u", cdev->mdsl))
610 return -1;
611 }
612
613 return 0;
614}
615
616static int cxgbit_set_digest(struct cxgbit_sock *csk)
617{
618 struct iscsi_conn *conn = csk->conn;
619 struct iscsi_param *param;
620
621 param = iscsi_find_param_from_key(HEADERDIGEST, conn->param_list);
622 if (!param) {
623 pr_err("param not found key %s\n", HEADERDIGEST);
624 return -1;
625 }
626
627 if (!strcmp(param->value, CRC32C))
628 csk->submode |= CXGBIT_SUBMODE_HCRC;
629
630 param = iscsi_find_param_from_key(DATADIGEST, conn->param_list);
631 if (!param) {
632 csk->submode = 0;
633 pr_err("param not found key %s\n", DATADIGEST);
634 return -1;
635 }
636
637 if (!strcmp(param->value, CRC32C))
638 csk->submode |= CXGBIT_SUBMODE_DCRC;
639
640 if (cxgbit_setup_conn_digest(csk)) {
641 csk->submode = 0;
642 return -1;
643 }
644
645 return 0;
646}
647
648static int cxgbit_set_iso_npdu(struct cxgbit_sock *csk)
649{
650 struct iscsi_conn *conn = csk->conn;
651 struct iscsi_conn_ops *conn_ops = conn->conn_ops;
652 struct iscsi_param *param;
653 u32 mrdsl, mbl;
654 u32 max_npdu, max_iso_npdu;
655 u32 max_iso_payload;
656
657 if (conn->login->leading_connection) {
658 param = iscsi_find_param_from_key(MAXBURSTLENGTH,
659 conn->param_list);
660 if (!param) {
661 pr_err("param not found key %s\n", MAXBURSTLENGTH);
662 return -1;
663 }
664
665 if (kstrtou32(param->value, 0, &mbl) < 0)
666 return -1;
667 } else {
668 mbl = conn->sess->sess_ops->MaxBurstLength;
669 }
670
671 mrdsl = conn_ops->MaxRecvDataSegmentLength;
672 max_npdu = mbl / mrdsl;
673
674 max_iso_payload = rounddown(CXGBIT_MAX_ISO_PAYLOAD, csk->emss);
675
676 max_iso_npdu = max_iso_payload /
677 (ISCSI_HDR_LEN + mrdsl +
678 cxgbit_digest_len[csk->submode]);
679
680 csk->max_iso_npdu = min(max_npdu, max_iso_npdu);
681
682 if (csk->max_iso_npdu <= 1)
683 csk->max_iso_npdu = 0;
684
685 return 0;
686}
687
688
689
690
691
692
693
694
695
696
697
698
699static int cxgbit_seq_pdu_inorder(struct cxgbit_sock *csk)
700{
701 struct iscsi_conn *conn = csk->conn;
702 struct iscsi_param *param;
703
704 if (conn->login->leading_connection) {
705 param = iscsi_find_param_from_key(DATASEQUENCEINORDER,
706 conn->param_list);
707 if (!param) {
708 pr_err("param not found key %s\n", DATASEQUENCEINORDER);
709 return -1;
710 }
711
712 if (strcmp(param->value, YES))
713 return 1;
714
715 param = iscsi_find_param_from_key(DATAPDUINORDER,
716 conn->param_list);
717 if (!param) {
718 pr_err("param not found key %s\n", DATAPDUINORDER);
719 return -1;
720 }
721
722 if (strcmp(param->value, YES))
723 return 1;
724
725 } else {
726 if (!conn->sess->sess_ops->DataSequenceInOrder)
727 return 1;
728 if (!conn->sess->sess_ops->DataPDUInOrder)
729 return 1;
730 }
731
732 return 0;
733}
734
735static int cxgbit_set_params(struct iscsi_conn *conn)
736{
737 struct cxgbit_sock *csk = conn->context;
738 struct cxgbit_device *cdev = csk->com.cdev;
739 struct cxgbi_ppm *ppm = *csk->com.cdev->lldi.iscsi_ppm;
740 struct iscsi_conn_ops *conn_ops = conn->conn_ops;
741 struct iscsi_param *param;
742 u8 erl;
743
744 if (conn_ops->MaxRecvDataSegmentLength > cdev->mdsl)
745 conn_ops->MaxRecvDataSegmentLength = cdev->mdsl;
746
747 if (cxgbit_set_digest(csk))
748 return -1;
749
750 if (conn->login->leading_connection) {
751 param = iscsi_find_param_from_key(ERRORRECOVERYLEVEL,
752 conn->param_list);
753 if (!param) {
754 pr_err("param not found key %s\n", ERRORRECOVERYLEVEL);
755 return -1;
756 }
757 if (kstrtou8(param->value, 0, &erl) < 0)
758 return -1;
759 } else {
760 erl = conn->sess->sess_ops->ErrorRecoveryLevel;
761 }
762
763 if (!erl) {
764 int ret;
765
766 ret = cxgbit_seq_pdu_inorder(csk);
767 if (ret < 0) {
768 return -1;
769 } else if (ret > 0) {
770 if (is_t5(cdev->lldi.adapter_type))
771 goto enable_ddp;
772 else
773 return 0;
774 }
775
776 if (test_bit(CDEV_ISO_ENABLE, &cdev->flags)) {
777 if (cxgbit_set_iso_npdu(csk))
778 return -1;
779 }
780
781enable_ddp:
782 if (test_bit(CDEV_DDP_ENABLE, &cdev->flags)) {
783 if (cxgbit_setup_conn_pgidx(csk,
784 ppm->tformat.pgsz_idx_dflt))
785 return -1;
786 set_bit(CSK_DDP_ENABLE, &csk->com.flags);
787 }
788 }
789
790 return 0;
791}
792
793int
794cxgbit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
795 u32 length)
796{
797 struct cxgbit_sock *csk = conn->context;
798 struct sk_buff *skb;
799 u32 padding_buf = 0;
800 u8 padding = ((-length) & 3);
801
802 skb = cxgbit_alloc_skb(csk, length + padding);
803 if (!skb)
804 return -ENOMEM;
805 skb_store_bits(skb, 0, login->rsp, ISCSI_HDR_LEN);
806 skb_store_bits(skb, ISCSI_HDR_LEN, login->rsp_buf, length);
807
808 if (padding)
809 skb_store_bits(skb, ISCSI_HDR_LEN + length,
810 &padding_buf, padding);
811
812 if (login->login_complete) {
813 if (cxgbit_set_params(conn)) {
814 kfree_skb(skb);
815 return -1;
816 }
817
818 set_bit(CSK_LOGIN_DONE, &csk->com.flags);
819 }
820
821 if (cxgbit_queue_skb(csk, skb))
822 return -1;
823
824 if ((!login->login_complete) && (!login->login_failed))
825 schedule_delayed_work(&conn->login_work, 0);
826
827 return 0;
828}
829
830static void
831cxgbit_skb_copy_to_sg(struct sk_buff *skb, struct scatterlist *sg,
832 unsigned int nents, u32 skip)
833{
834 struct skb_seq_state st;
835 const u8 *buf;
836 unsigned int consumed = 0, buf_len;
837 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(skb);
838
839 skb_prepare_seq_read(skb, pdu_cb->doffset,
840 pdu_cb->doffset + pdu_cb->dlen,
841 &st);
842
843 while (true) {
844 buf_len = skb_seq_read(consumed, &buf, &st);
845 if (!buf_len) {
846 skb_abort_seq_read(&st);
847 break;
848 }
849
850 consumed += sg_pcopy_from_buffer(sg, nents, (void *)buf,
851 buf_len, skip + consumed);
852 }
853}
854
855static struct iscsi_cmd *cxgbit_allocate_cmd(struct cxgbit_sock *csk)
856{
857 struct iscsi_conn *conn = csk->conn;
858 struct cxgbi_ppm *ppm = cdev2ppm(csk->com.cdev);
859 struct cxgbit_cmd *ccmd;
860 struct iscsi_cmd *cmd;
861
862 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
863 if (!cmd) {
864 pr_err("Unable to allocate iscsi_cmd + cxgbit_cmd\n");
865 return NULL;
866 }
867
868 ccmd = iscsit_priv_cmd(cmd);
869 ccmd->ttinfo.tag = ppm->tformat.no_ddp_mask;
870 ccmd->setup_ddp = true;
871
872 return cmd;
873}
874
875static int
876cxgbit_handle_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
877 u32 length)
878{
879 struct iscsi_conn *conn = cmd->conn;
880 struct cxgbit_sock *csk = conn->context;
881 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
882
883 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
884 pr_err("ImmediateData CRC32C DataDigest error\n");
885 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
886 pr_err("Unable to recover from"
887 " Immediate Data digest failure while"
888 " in ERL=0.\n");
889 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
890 (unsigned char *)hdr);
891 return IMMEDIATE_DATA_CANNOT_RECOVER;
892 }
893
894 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
895 (unsigned char *)hdr);
896 return IMMEDIATE_DATA_ERL1_CRC_FAILURE;
897 }
898
899 if (cmd->se_cmd.se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC) {
900 struct cxgbit_cmd *ccmd = iscsit_priv_cmd(cmd);
901 struct skb_shared_info *ssi = skb_shinfo(csk->skb);
902 skb_frag_t *dfrag = &ssi->frags[pdu_cb->dfrag_idx];
903
904 sg_init_table(&ccmd->sg, 1);
905 sg_set_page(&ccmd->sg, dfrag->page.p, skb_frag_size(dfrag),
906 dfrag->page_offset);
907 get_page(dfrag->page.p);
908
909 cmd->se_cmd.t_data_sg = &ccmd->sg;
910 cmd->se_cmd.t_data_nents = 1;
911
912 ccmd->release = true;
913 } else {
914 struct scatterlist *sg = &cmd->se_cmd.t_data_sg[0];
915 u32 sg_nents = max(1UL, DIV_ROUND_UP(pdu_cb->dlen, PAGE_SIZE));
916
917 cxgbit_skb_copy_to_sg(csk->skb, sg, sg_nents, 0);
918 }
919
920 cmd->write_data_done += pdu_cb->dlen;
921
922 if (cmd->write_data_done == cmd->se_cmd.data_length) {
923 spin_lock_bh(&cmd->istate_lock);
924 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
925 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
926 spin_unlock_bh(&cmd->istate_lock);
927 }
928
929 return IMMEDIATE_DATA_NORMAL_OPERATION;
930}
931
932static int
933cxgbit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr,
934 bool dump_payload)
935{
936 struct iscsi_conn *conn = cmd->conn;
937 int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION;
938
939
940
941 if (dump_payload)
942 goto after_immediate_data;
943
944 immed_ret = cxgbit_handle_immediate_data(cmd, hdr,
945 cmd->first_burst_len);
946after_immediate_data:
947 if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) {
948
949
950
951
952
953 cmdsn_ret = iscsit_sequence_cmd(conn, cmd,
954 (unsigned char *)hdr,
955 hdr->cmdsn);
956 if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER)
957 return -1;
958
959 if (cmd->sense_reason || cmdsn_ret == CMDSN_LOWER_THAN_EXP) {
960 target_put_sess_cmd(&cmd->se_cmd);
961 return 0;
962 } else if (cmd->unsolicited_data) {
963 iscsit_set_unsoliticed_dataout(cmd);
964 }
965
966 } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) {
967
968
969
970
971
972
973
974
975
976
977
978 cmd->i_state = ISTATE_REMOVE;
979 iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state);
980 } else
981 return -1;
982
983 return 0;
984}
985
986static int
987cxgbit_handle_scsi_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
988{
989 struct iscsi_conn *conn = csk->conn;
990 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
991 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)pdu_cb->hdr;
992 int rc;
993 bool dump_payload = false;
994
995 rc = iscsit_setup_scsi_cmd(conn, cmd, (unsigned char *)hdr);
996 if (rc < 0)
997 return rc;
998
999 if (pdu_cb->dlen && (pdu_cb->dlen == cmd->se_cmd.data_length) &&
1000 (pdu_cb->nr_dfrags == 1))
1001 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
1002
1003 rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1004 if (rc < 0)
1005 return 0;
1006 else if (rc > 0)
1007 dump_payload = true;
1008
1009 if (!pdu_cb->dlen)
1010 return 0;
1011
1012 return cxgbit_get_immediate_data(cmd, hdr, dump_payload);
1013}
1014
1015static int cxgbit_handle_iscsi_dataout(struct cxgbit_sock *csk)
1016{
1017 struct scatterlist *sg_start;
1018 struct iscsi_conn *conn = csk->conn;
1019 struct iscsi_cmd *cmd = NULL;
1020 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1021 struct iscsi_data *hdr = (struct iscsi_data *)pdu_cb->hdr;
1022 u32 data_offset = be32_to_cpu(hdr->offset);
1023 u32 data_len = pdu_cb->dlen;
1024 int rc, sg_nents, sg_off;
1025 bool dcrc_err = false;
1026
1027 if (pdu_cb->flags & PDUCBF_RX_DDP_CMP) {
1028 u32 offset = be32_to_cpu(hdr->offset);
1029 u32 ddp_data_len;
1030 u32 payload_length = ntoh24(hdr->dlength);
1031 bool success = false;
1032
1033 cmd = iscsit_find_cmd_from_itt_or_dump(conn, hdr->itt, 0);
1034 if (!cmd)
1035 return 0;
1036
1037 ddp_data_len = offset - cmd->write_data_done;
1038 atomic_long_add(ddp_data_len, &conn->sess->rx_data_octets);
1039
1040 cmd->write_data_done = offset;
1041 cmd->next_burst_len = ddp_data_len;
1042 cmd->data_sn = be32_to_cpu(hdr->datasn);
1043
1044 rc = __iscsit_check_dataout_hdr(conn, (unsigned char *)hdr,
1045 cmd, payload_length, &success);
1046 if (rc < 0)
1047 return rc;
1048 else if (!success)
1049 return 0;
1050 } else {
1051 rc = iscsit_check_dataout_hdr(conn, (unsigned char *)hdr, &cmd);
1052 if (rc < 0)
1053 return rc;
1054 else if (!cmd)
1055 return 0;
1056 }
1057
1058 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1059 pr_err("ITT: 0x%08x, Offset: %u, Length: %u,"
1060 " DataSN: 0x%08x\n",
1061 hdr->itt, hdr->offset, data_len,
1062 hdr->datasn);
1063
1064 dcrc_err = true;
1065 goto check_payload;
1066 }
1067
1068 pr_debug("DataOut data_len: %u, "
1069 "write_data_done: %u, data_length: %u\n",
1070 data_len, cmd->write_data_done,
1071 cmd->se_cmd.data_length);
1072
1073 if (!(pdu_cb->flags & PDUCBF_RX_DATA_DDPD)) {
1074 u32 skip = data_offset % PAGE_SIZE;
1075
1076 sg_off = data_offset / PAGE_SIZE;
1077 sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1078 sg_nents = max(1UL, DIV_ROUND_UP(skip + data_len, PAGE_SIZE));
1079
1080 cxgbit_skb_copy_to_sg(csk->skb, sg_start, sg_nents, skip);
1081 }
1082
1083check_payload:
1084
1085 rc = iscsit_check_dataout_payload(cmd, hdr, dcrc_err);
1086 if (rc < 0)
1087 return rc;
1088
1089 return 0;
1090}
1091
1092static int cxgbit_handle_nop_out(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1093{
1094 struct iscsi_conn *conn = csk->conn;
1095 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1096 struct iscsi_nopout *hdr = (struct iscsi_nopout *)pdu_cb->hdr;
1097 unsigned char *ping_data = NULL;
1098 u32 payload_length = pdu_cb->dlen;
1099 int ret;
1100
1101 ret = iscsit_setup_nop_out(conn, cmd, hdr);
1102 if (ret < 0)
1103 return 0;
1104
1105 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1106 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1107 pr_err("Unable to recover from"
1108 " NOPOUT Ping DataCRC failure while in"
1109 " ERL=0.\n");
1110 ret = -1;
1111 goto out;
1112 } else {
1113
1114
1115
1116
1117 pr_info("Dropping NOPOUT"
1118 " Command CmdSN: 0x%08x due to"
1119 " DataCRC error.\n", hdr->cmdsn);
1120 ret = 0;
1121 goto out;
1122 }
1123 }
1124
1125
1126
1127
1128 if (payload_length && hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1129 ping_data = kzalloc(payload_length + 1, GFP_KERNEL);
1130 if (!ping_data) {
1131 pr_err("Unable to allocate memory for"
1132 " NOPOUT ping data.\n");
1133 ret = -1;
1134 goto out;
1135 }
1136
1137 skb_copy_bits(csk->skb, pdu_cb->doffset,
1138 ping_data, payload_length);
1139
1140 ping_data[payload_length] = '\0';
1141
1142
1143
1144 cmd->buf_ptr = ping_data;
1145 cmd->buf_ptr_size = payload_length;
1146
1147 pr_debug("Got %u bytes of NOPOUT ping"
1148 " data.\n", payload_length);
1149 pr_debug("Ping Data: \"%s\"\n", ping_data);
1150 }
1151
1152 return iscsit_process_nop_out(conn, cmd, hdr);
1153out:
1154 if (cmd)
1155 iscsit_free_cmd(cmd, false);
1156 return ret;
1157}
1158
1159static int
1160cxgbit_handle_text_cmd(struct cxgbit_sock *csk, struct iscsi_cmd *cmd)
1161{
1162 struct iscsi_conn *conn = csk->conn;
1163 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1164 struct iscsi_text *hdr = (struct iscsi_text *)pdu_cb->hdr;
1165 u32 payload_length = pdu_cb->dlen;
1166 int rc;
1167 unsigned char *text_in = NULL;
1168
1169 rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1170 if (rc < 0)
1171 return rc;
1172
1173 if (pdu_cb->flags & PDUCBF_RX_DCRC_ERR) {
1174 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
1175 pr_err("Unable to recover from"
1176 " Text Data digest failure while in"
1177 " ERL=0.\n");
1178 goto reject;
1179 } else {
1180
1181
1182
1183
1184 pr_info("Dropping Text"
1185 " Command CmdSN: 0x%08x due to"
1186 " DataCRC error.\n", hdr->cmdsn);
1187 return 0;
1188 }
1189 }
1190
1191 if (payload_length) {
1192 text_in = kzalloc(payload_length, GFP_KERNEL);
1193 if (!text_in) {
1194 pr_err("Unable to allocate text_in of payload_length: %u\n",
1195 payload_length);
1196 return -ENOMEM;
1197 }
1198 skb_copy_bits(csk->skb, pdu_cb->doffset,
1199 text_in, payload_length);
1200
1201 text_in[payload_length - 1] = '\0';
1202
1203 cmd->text_in_ptr = text_in;
1204 }
1205
1206 return iscsit_process_text_cmd(conn, cmd, hdr);
1207
1208reject:
1209 return iscsit_reject_cmd(cmd, ISCSI_REASON_PROTOCOL_ERROR,
1210 pdu_cb->hdr);
1211}
1212
1213static int cxgbit_target_rx_opcode(struct cxgbit_sock *csk)
1214{
1215 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1216 struct iscsi_hdr *hdr = (struct iscsi_hdr *)pdu_cb->hdr;
1217 struct iscsi_conn *conn = csk->conn;
1218 struct iscsi_cmd *cmd = NULL;
1219 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1220 int ret = -EINVAL;
1221
1222 switch (opcode) {
1223 case ISCSI_OP_SCSI_CMD:
1224 cmd = cxgbit_allocate_cmd(csk);
1225 if (!cmd)
1226 goto reject;
1227
1228 ret = cxgbit_handle_scsi_cmd(csk, cmd);
1229 break;
1230 case ISCSI_OP_SCSI_DATA_OUT:
1231 ret = cxgbit_handle_iscsi_dataout(csk);
1232 break;
1233 case ISCSI_OP_NOOP_OUT:
1234 if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) {
1235 cmd = cxgbit_allocate_cmd(csk);
1236 if (!cmd)
1237 goto reject;
1238 }
1239
1240 ret = cxgbit_handle_nop_out(csk, cmd);
1241 break;
1242 case ISCSI_OP_SCSI_TMFUNC:
1243 cmd = cxgbit_allocate_cmd(csk);
1244 if (!cmd)
1245 goto reject;
1246
1247 ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1248 (unsigned char *)hdr);
1249 break;
1250 case ISCSI_OP_TEXT:
1251 if (hdr->ttt != cpu_to_be32(0xFFFFFFFF)) {
1252 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1253 if (!cmd)
1254 goto reject;
1255 } else {
1256 cmd = cxgbit_allocate_cmd(csk);
1257 if (!cmd)
1258 goto reject;
1259 }
1260
1261 ret = cxgbit_handle_text_cmd(csk, cmd);
1262 break;
1263 case ISCSI_OP_LOGOUT:
1264 cmd = cxgbit_allocate_cmd(csk);
1265 if (!cmd)
1266 goto reject;
1267
1268 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1269 if (ret > 0)
1270 wait_for_completion_timeout(&conn->conn_logout_comp,
1271 SECONDS_FOR_LOGOUT_COMP
1272 * HZ);
1273 break;
1274 case ISCSI_OP_SNACK:
1275 ret = iscsit_handle_snack(conn, (unsigned char *)hdr);
1276 break;
1277 default:
1278 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1279 dump_stack();
1280 break;
1281 }
1282
1283 return ret;
1284
1285reject:
1286 return iscsit_add_reject(conn, ISCSI_REASON_BOOKMARK_NO_RESOURCES,
1287 (unsigned char *)hdr);
1288 return ret;
1289}
1290
1291static int cxgbit_rx_opcode(struct cxgbit_sock *csk)
1292{
1293 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1294 struct iscsi_conn *conn = csk->conn;
1295 struct iscsi_hdr *hdr = pdu_cb->hdr;
1296 u8 opcode;
1297
1298 if (pdu_cb->flags & PDUCBF_RX_HCRC_ERR) {
1299 atomic_long_inc(&conn->sess->conn_digest_errors);
1300 goto transport_err;
1301 }
1302
1303 if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT)
1304 goto transport_err;
1305
1306 opcode = hdr->opcode & ISCSI_OPCODE_MASK;
1307
1308 if (conn->sess->sess_ops->SessionType &&
1309 ((!(opcode & ISCSI_OP_TEXT)) ||
1310 (!(opcode & ISCSI_OP_LOGOUT)))) {
1311 pr_err("Received illegal iSCSI Opcode: 0x%02x"
1312 " while in Discovery Session, rejecting.\n", opcode);
1313 iscsit_add_reject(conn, ISCSI_REASON_PROTOCOL_ERROR,
1314 (unsigned char *)hdr);
1315 goto transport_err;
1316 }
1317
1318 if (cxgbit_target_rx_opcode(csk) < 0)
1319 goto transport_err;
1320
1321 return 0;
1322
1323transport_err:
1324 return -1;
1325}
1326
1327static int cxgbit_rx_login_pdu(struct cxgbit_sock *csk)
1328{
1329 struct iscsi_conn *conn = csk->conn;
1330 struct iscsi_login *login = conn->login;
1331 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_rx_pdu_cb(csk->skb);
1332 struct iscsi_login_req *login_req;
1333
1334 login_req = (struct iscsi_login_req *)login->req;
1335 memcpy(login_req, pdu_cb->hdr, sizeof(*login_req));
1336
1337 pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x,"
1338 " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n",
1339 login_req->flags, login_req->itt, login_req->cmdsn,
1340 login_req->exp_statsn, login_req->cid, pdu_cb->dlen);
1341
1342
1343
1344
1345 if (login->first_request) {
1346 login_req = (struct iscsi_login_req *)login->req;
1347 login->leading_connection = (!login_req->tsih) ? 1 : 0;
1348 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE(
1349 login_req->flags);
1350 login->version_min = login_req->min_version;
1351 login->version_max = login_req->max_version;
1352 memcpy(login->isid, login_req->isid, 6);
1353 login->cmd_sn = be32_to_cpu(login_req->cmdsn);
1354 login->init_task_tag = login_req->itt;
1355 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1356 login->cid = be16_to_cpu(login_req->cid);
1357 login->tsih = be16_to_cpu(login_req->tsih);
1358 }
1359
1360 if (iscsi_target_check_login_request(conn, login) < 0)
1361 return -1;
1362
1363 memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS);
1364 skb_copy_bits(csk->skb, pdu_cb->doffset, login->req_buf, pdu_cb->dlen);
1365
1366 return 0;
1367}
1368
1369static int
1370cxgbit_process_iscsi_pdu(struct cxgbit_sock *csk, struct sk_buff *skb, int idx)
1371{
1372 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, idx);
1373 int ret;
1374
1375 cxgbit_rx_pdu_cb(skb) = pdu_cb;
1376
1377 csk->skb = skb;
1378
1379 if (!test_bit(CSK_LOGIN_DONE, &csk->com.flags)) {
1380 ret = cxgbit_rx_login_pdu(csk);
1381 set_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1382 } else {
1383 ret = cxgbit_rx_opcode(csk);
1384 }
1385
1386 return ret;
1387}
1388
1389static void cxgbit_lro_skb_dump(struct sk_buff *skb)
1390{
1391 struct skb_shared_info *ssi = skb_shinfo(skb);
1392 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1393 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1394 u8 i;
1395
1396 pr_info("skb 0x%p, head 0x%p, 0x%p, len %u,%u, frags %u.\n",
1397 skb, skb->head, skb->data, skb->len, skb->data_len,
1398 ssi->nr_frags);
1399 pr_info("skb 0x%p, lro_cb, csk 0x%p, pdu %u, %u.\n",
1400 skb, lro_cb->csk, lro_cb->pdu_idx, lro_cb->pdu_totallen);
1401
1402 for (i = 0; i < lro_cb->pdu_idx; i++, pdu_cb++)
1403 pr_info("skb 0x%p, pdu %d, %u, f 0x%x, seq 0x%x, dcrc 0x%x, "
1404 "frags %u.\n",
1405 skb, i, pdu_cb->pdulen, pdu_cb->flags, pdu_cb->seq,
1406 pdu_cb->ddigest, pdu_cb->frags);
1407 for (i = 0; i < ssi->nr_frags; i++)
1408 pr_info("skb 0x%p, frag %d, off %u, sz %u.\n",
1409 skb, i, ssi->frags[i].page_offset, ssi->frags[i].size);
1410}
1411
1412static void cxgbit_lro_hskb_reset(struct cxgbit_sock *csk)
1413{
1414 struct sk_buff *skb = csk->lro_hskb;
1415 struct skb_shared_info *ssi = skb_shinfo(skb);
1416 u8 i;
1417
1418 memset(skb->data, 0, LRO_SKB_MIN_HEADROOM);
1419 for (i = 0; i < ssi->nr_frags; i++)
1420 put_page(skb_frag_page(&ssi->frags[i]));
1421 ssi->nr_frags = 0;
1422 skb->data_len = 0;
1423 skb->truesize -= skb->len;
1424 skb->len = 0;
1425}
1426
1427static void
1428cxgbit_lro_skb_merge(struct cxgbit_sock *csk, struct sk_buff *skb, u8 pdu_idx)
1429{
1430 struct sk_buff *hskb = csk->lro_hskb;
1431 struct cxgbit_lro_pdu_cb *hpdu_cb = cxgbit_skb_lro_pdu_cb(hskb, 0);
1432 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, pdu_idx);
1433 struct skb_shared_info *hssi = skb_shinfo(hskb);
1434 struct skb_shared_info *ssi = skb_shinfo(skb);
1435 unsigned int len = 0;
1436
1437 if (pdu_cb->flags & PDUCBF_RX_HDR) {
1438 u8 hfrag_idx = hssi->nr_frags;
1439
1440 hpdu_cb->flags |= pdu_cb->flags;
1441 hpdu_cb->seq = pdu_cb->seq;
1442 hpdu_cb->hdr = pdu_cb->hdr;
1443 hpdu_cb->hlen = pdu_cb->hlen;
1444
1445 memcpy(&hssi->frags[hfrag_idx], &ssi->frags[pdu_cb->hfrag_idx],
1446 sizeof(skb_frag_t));
1447
1448 get_page(skb_frag_page(&hssi->frags[hfrag_idx]));
1449 hssi->nr_frags++;
1450 hpdu_cb->frags++;
1451 hpdu_cb->hfrag_idx = hfrag_idx;
1452
1453 len = hssi->frags[hfrag_idx].size;
1454 hskb->len += len;
1455 hskb->data_len += len;
1456 hskb->truesize += len;
1457 }
1458
1459 if (pdu_cb->flags & PDUCBF_RX_DATA) {
1460 u8 dfrag_idx = hssi->nr_frags, i;
1461
1462 hpdu_cb->flags |= pdu_cb->flags;
1463 hpdu_cb->dfrag_idx = dfrag_idx;
1464
1465 len = 0;
1466 for (i = 0; i < pdu_cb->nr_dfrags; dfrag_idx++, i++) {
1467 memcpy(&hssi->frags[dfrag_idx],
1468 &ssi->frags[pdu_cb->dfrag_idx + i],
1469 sizeof(skb_frag_t));
1470
1471 get_page(skb_frag_page(&hssi->frags[dfrag_idx]));
1472
1473 len += hssi->frags[dfrag_idx].size;
1474
1475 hssi->nr_frags++;
1476 hpdu_cb->frags++;
1477 }
1478
1479 hpdu_cb->dlen = pdu_cb->dlen;
1480 hpdu_cb->doffset = hpdu_cb->hlen;
1481 hpdu_cb->nr_dfrags = pdu_cb->nr_dfrags;
1482 hskb->len += len;
1483 hskb->data_len += len;
1484 hskb->truesize += len;
1485 }
1486
1487 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1488 hpdu_cb->flags |= pdu_cb->flags;
1489
1490 if (hpdu_cb->flags & PDUCBF_RX_DATA)
1491 hpdu_cb->flags &= ~PDUCBF_RX_DATA_DDPD;
1492
1493 hpdu_cb->ddigest = pdu_cb->ddigest;
1494 hpdu_cb->pdulen = pdu_cb->pdulen;
1495 }
1496}
1497
1498static int cxgbit_process_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1499{
1500 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1501 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1502 u8 pdu_idx = 0, last_idx = 0;
1503 int ret = 0;
1504
1505 if (!pdu_cb->complete) {
1506 cxgbit_lro_skb_merge(csk, skb, 0);
1507
1508 if (pdu_cb->flags & PDUCBF_RX_STATUS) {
1509 struct sk_buff *hskb = csk->lro_hskb;
1510
1511 ret = cxgbit_process_iscsi_pdu(csk, hskb, 0);
1512
1513 cxgbit_lro_hskb_reset(csk);
1514
1515 if (ret < 0)
1516 goto out;
1517 }
1518
1519 pdu_idx = 1;
1520 }
1521
1522 if (lro_cb->pdu_idx)
1523 last_idx = lro_cb->pdu_idx - 1;
1524
1525 for (; pdu_idx <= last_idx; pdu_idx++) {
1526 ret = cxgbit_process_iscsi_pdu(csk, skb, pdu_idx);
1527 if (ret < 0)
1528 goto out;
1529 }
1530
1531 if ((!lro_cb->complete) && lro_cb->pdu_idx)
1532 cxgbit_lro_skb_merge(csk, skb, lro_cb->pdu_idx);
1533
1534out:
1535 return ret;
1536}
1537
1538static int cxgbit_rx_lro_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1539{
1540 struct cxgbit_lro_cb *lro_cb = cxgbit_skb_lro_cb(skb);
1541 struct cxgbit_lro_pdu_cb *pdu_cb = cxgbit_skb_lro_pdu_cb(skb, 0);
1542 int ret = -1;
1543
1544 if ((pdu_cb->flags & PDUCBF_RX_HDR) &&
1545 (pdu_cb->seq != csk->rcv_nxt)) {
1546 pr_info("csk 0x%p, tid 0x%x, seq 0x%x != 0x%x.\n",
1547 csk, csk->tid, pdu_cb->seq, csk->rcv_nxt);
1548 cxgbit_lro_skb_dump(skb);
1549 return ret;
1550 }
1551
1552 csk->rcv_nxt += lro_cb->pdu_totallen;
1553
1554 ret = cxgbit_process_lro_skb(csk, skb);
1555
1556 csk->rx_credits += lro_cb->pdu_totallen;
1557
1558 if (csk->rx_credits >= (csk->rcv_win / 4))
1559 cxgbit_rx_data_ack(csk);
1560
1561 return ret;
1562}
1563
1564static int cxgbit_rx_skb(struct cxgbit_sock *csk, struct sk_buff *skb)
1565{
1566 struct cxgb4_lld_info *lldi = &csk->com.cdev->lldi;
1567 int ret = -1;
1568
1569 if (likely(cxgbit_skcb_flags(skb) & SKCBF_RX_LRO)) {
1570 if (is_t5(lldi->adapter_type))
1571 ret = cxgbit_rx_lro_skb(csk, skb);
1572 else
1573 ret = cxgbit_process_lro_skb(csk, skb);
1574 }
1575
1576 __kfree_skb(skb);
1577 return ret;
1578}
1579
1580static bool cxgbit_rxq_len(struct cxgbit_sock *csk, struct sk_buff_head *rxq)
1581{
1582 spin_lock_bh(&csk->rxq.lock);
1583 if (skb_queue_len(&csk->rxq)) {
1584 skb_queue_splice_init(&csk->rxq, rxq);
1585 spin_unlock_bh(&csk->rxq.lock);
1586 return true;
1587 }
1588 spin_unlock_bh(&csk->rxq.lock);
1589 return false;
1590}
1591
1592static int cxgbit_wait_rxq(struct cxgbit_sock *csk)
1593{
1594 struct sk_buff *skb;
1595 struct sk_buff_head rxq;
1596
1597 skb_queue_head_init(&rxq);
1598
1599 wait_event_interruptible(csk->waitq, cxgbit_rxq_len(csk, &rxq));
1600
1601 if (signal_pending(current))
1602 goto out;
1603
1604 while ((skb = __skb_dequeue(&rxq))) {
1605 if (cxgbit_rx_skb(csk, skb))
1606 goto out;
1607 }
1608
1609 return 0;
1610out:
1611 __skb_queue_purge(&rxq);
1612 return -1;
1613}
1614
1615int cxgbit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
1616{
1617 struct cxgbit_sock *csk = conn->context;
1618 int ret = -1;
1619
1620 while (!test_and_clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags)) {
1621 ret = cxgbit_wait_rxq(csk);
1622 if (ret) {
1623 clear_bit(CSK_LOGIN_PDU_DONE, &csk->com.flags);
1624 break;
1625 }
1626 }
1627
1628 return ret;
1629}
1630
1631void cxgbit_get_rx_pdu(struct iscsi_conn *conn)
1632{
1633 struct cxgbit_sock *csk = conn->context;
1634
1635 while (!kthread_should_stop()) {
1636 iscsit_thread_check_cpumask(conn, current, 0);
1637 if (cxgbit_wait_rxq(csk))
1638 return;
1639 }
1640}
1641
