1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/tcp.h>
17#include <linux/ip.h>
18#include <linux/ipv6.h>
19#include <linux/crc32.h>
20#include <linux/if_vlan.h>
21#include <linux/jiffies.h>
22#include <linux/phy.h>
23#include <linux/of.h>
24#include <net/ip6_checksum.h>
25#include "emac.h"
26#include "emac-sgmii.h"
27
28
29#define SINGLE_PAUSE_MODE 0x10000000
30#define DEBUG_MODE 0x08000000
31#define BROAD_EN 0x04000000
32#define MULTI_ALL 0x02000000
33#define RX_CHKSUM_EN 0x01000000
34#define HUGE 0x00800000
35#define SPEED(x) (((x) & 0x3) << 20)
36#define SPEED_MASK SPEED(0x3)
37#define SIMR 0x00080000
38#define TPAUSE 0x00010000
39#define PROM_MODE 0x00008000
40#define VLAN_STRIP 0x00004000
41#define PRLEN_BMSK 0x00003c00
42#define PRLEN_SHFT 10
43#define HUGEN 0x00000200
44#define FLCHK 0x00000100
45#define PCRCE 0x00000080
46#define CRCE 0x00000040
47#define FULLD 0x00000020
48#define MAC_LP_EN 0x00000010
49#define RXFC 0x00000008
50#define TXFC 0x00000004
51#define RXEN 0x00000002
52#define TXEN 0x00000001
53
54
55#define RFD_RING_SIZE_BMSK 0xfff
56
57
58#define RX_BUFFER_SIZE_BMSK 0xffff
59
60
61#define RRD_RING_SIZE_BMSK 0xfff
62
63
64#define TPD_RING_SIZE_BMSK 0xffff
65
66
67#define NUM_TXF_BURST_PREF_BMSK 0xffff0000
68#define NUM_TXF_BURST_PREF_SHFT 16
69#define LS_8023_SP 0x80
70#define TXQ_MODE 0x40
71#define TXQ_EN 0x20
72#define IP_OP_SP 0x10
73#define NUM_TPD_BURST_PREF_BMSK 0xf
74#define NUM_TPD_BURST_PREF_SHFT 0
75
76
77#define JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK 0x7ff
78
79
80#define TXF_HWM_BMSK 0xfff0000
81#define TXF_LWM_BMSK 0xfff
82
83
84#define RXQ_EN BIT(31)
85#define CUT_THRU_EN BIT(30)
86#define RSS_HASH_EN BIT(29)
87#define NUM_RFD_BURST_PREF_BMSK 0x3f00000
88#define NUM_RFD_BURST_PREF_SHFT 20
89#define IDT_TABLE_SIZE_BMSK 0x1ff00
90#define IDT_TABLE_SIZE_SHFT 8
91#define SP_IPV6 0x80
92
93
94#define JUMBO_1KAH_BMSK 0xf000
95#define JUMBO_1KAH_SHFT 12
96#define RFD_PREF_LOW_TH 0x10
97#define RFD_PREF_LOW_THRESHOLD_BMSK 0xfc0
98#define RFD_PREF_LOW_THRESHOLD_SHFT 6
99#define RFD_PREF_UP_TH 0x10
100#define RFD_PREF_UP_THRESHOLD_BMSK 0x3f
101#define RFD_PREF_UP_THRESHOLD_SHFT 0
102
103
104#define RXF_DOF_THRESFHOLD 0x1a0
105#define RXF_DOF_THRESHOLD_BMSK 0xfff0000
106#define RXF_DOF_THRESHOLD_SHFT 16
107#define RXF_UOF_THRESFHOLD 0xbe
108#define RXF_UOF_THRESHOLD_BMSK 0xfff
109#define RXF_UOF_THRESHOLD_SHFT 0
110
111
112#define RXD_TIMER_BMSK 0xffff0000
113#define RXD_THRESHOLD_BMSK 0xfff
114#define RXD_THRESHOLD_SHFT 0
115
116
117#define DMAW_DLY_CNT_BMSK 0xf0000
118#define DMAW_DLY_CNT_SHFT 16
119#define DMAR_DLY_CNT_BMSK 0xf800
120#define DMAR_DLY_CNT_SHFT 11
121#define DMAR_REQ_PRI 0x400
122#define REGWRBLEN_BMSK 0x380
123#define REGWRBLEN_SHFT 7
124#define REGRDBLEN_BMSK 0x70
125#define REGRDBLEN_SHFT 4
126#define OUT_ORDER_MODE 0x4
127#define ENH_ORDER_MODE 0x2
128#define IN_ORDER_MODE 0x1
129
130
131#define RFD3_PROC_IDX_BMSK 0xfff0000
132#define RFD3_PROC_IDX_SHFT 16
133#define RFD3_PROD_IDX_BMSK 0xfff
134#define RFD3_PROD_IDX_SHFT 0
135
136
137#define NTPD_CONS_IDX_BMSK 0xffff0000
138#define NTPD_CONS_IDX_SHFT 16
139
140
141#define RFD0_CONS_IDX_BMSK 0xfff
142#define RFD0_CONS_IDX_SHFT 0
143
144
145#define H3TPD_PROD_IDX_BMSK 0xffff0000
146#define H3TPD_PROD_IDX_SHFT 16
147
148
149#define DATA_BYTE_SWAP 0x8
150#define MAX_BOUND 0x2
151#define MAX_BTYPE 0x1
152
153
154#define H3TPD_CONS_IDX_BMSK 0xffff0000
155#define H3TPD_CONS_IDX_SHFT 16
156
157
158#define H2TPD_PROD_IDX_BMSK 0xffff
159#define H2TPD_PROD_IDX_SHFT 0
160
161
162#define H1TPD_CONS_IDX_BMSK 0xffff0000
163#define H1TPD_CONS_IDX_SHFT 16
164#define H2TPD_CONS_IDX_BMSK 0xffff
165#define H2TPD_CONS_IDX_SHFT 0
166
167
168#define HEADER_CNT_EN 0x2
169#define HEADER_ENABLE 0x1
170
171
172#define RFD0_PROC_IDX_BMSK 0xfff0000
173#define RFD0_PROC_IDX_SHFT 16
174#define RFD0_PROD_IDX_BMSK 0xfff
175#define RFD0_PROD_IDX_SHFT 0
176
177
178#define RFD1_PROC_IDX_BMSK 0xfff0000
179#define RFD1_PROC_IDX_SHFT 16
180#define RFD1_PROD_IDX_BMSK 0xfff
181#define RFD1_PROD_IDX_SHFT 0
182
183
184#define RX_UNCPL_INT_EN 0x1
185
186
187#define RFD2_CONS_IDX_BMSK 0xfff0000
188#define RFD2_CONS_IDX_SHFT 16
189#define RFD1_CONS_IDX_BMSK 0xfff
190#define RFD1_CONS_IDX_SHFT 0
191
192
193#define RFD3_CONS_IDX_BMSK 0xfff
194#define RFD3_CONS_IDX_SHFT 0
195
196
197#define NTPD_PROD_IDX_BMSK 0xffff
198#define NTPD_PROD_IDX_SHFT 0
199
200
201#define H1TPD_PROD_IDX_BMSK 0xffff
202#define H1TPD_PROD_IDX_SHFT 0
203
204#define RXQ0_RSS_HSTYP_IPV6_TCP_EN 0x20
205#define RXQ0_RSS_HSTYP_IPV6_EN 0x10
206#define RXQ0_RSS_HSTYP_IPV4_TCP_EN 0x8
207#define RXQ0_RSS_HSTYP_IPV4_EN 0x4
208
209
210#define EMAC_WRAPPER_TX_TS_EMPTY BIT(31)
211#define EMAC_WRAPPER_TX_TS_INX_BMSK 0xffff
212
213struct emac_skb_cb {
214 u32 tpd_idx;
215 unsigned long jiffies;
216};
217
218#define EMAC_SKB_CB(skb) ((struct emac_skb_cb *)(skb)->cb)
219#define EMAC_RSS_IDT_SIZE 256
220#define JUMBO_1KAH 0x4
221#define RXD_TH 0x100
222#define EMAC_TPD_LAST_FRAGMENT 0x80000000
223#define EMAC_TPD_TSTAMP_SAVE 0x80000000
224
225
226#define EMAC_RRD_L4F BIT(14)
227#define EMAC_RRD_IPF BIT(15)
228#define EMAC_RRD_CRC BIT(21)
229#define EMAC_RRD_FAE BIT(22)
230#define EMAC_RRD_TRN BIT(23)
231#define EMAC_RRD_RNT BIT(24)
232#define EMAC_RRD_INC BIT(25)
233#define EMAC_RRD_FOV BIT(29)
234#define EMAC_RRD_LEN BIT(30)
235
236
237#define EMAC_RRD_ERROR (EMAC_RRD_IPF | EMAC_RRD_CRC | EMAC_RRD_FAE | \
238 EMAC_RRD_TRN | EMAC_RRD_RNT | EMAC_RRD_INC | \
239 EMAC_RRD_FOV | EMAC_RRD_LEN)
240#define EMAC_RRD_STATS_DW_IDX 3
241
242#define EMAC_RRD(RXQ, SIZE, IDX) ((RXQ)->rrd.v_addr + (SIZE * (IDX)))
243#define EMAC_RFD(RXQ, SIZE, IDX) ((RXQ)->rfd.v_addr + (SIZE * (IDX)))
244#define EMAC_TPD(TXQ, SIZE, IDX) ((TXQ)->tpd.v_addr + (SIZE * (IDX)))
245
246#define GET_RFD_BUFFER(RXQ, IDX) (&((RXQ)->rfd.rfbuff[(IDX)]))
247#define GET_TPD_BUFFER(RTQ, IDX) (&((RTQ)->tpd.tpbuff[(IDX)]))
248
249#define EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD 8
250
251#define ISR_RX_PKT (\
252 RX_PKT_INT0 |\
253 RX_PKT_INT1 |\
254 RX_PKT_INT2 |\
255 RX_PKT_INT3)
256
257void emac_mac_multicast_addr_set(struct emac_adapter *adpt, u8 *addr)
258{
259 u32 crc32, bit, reg, mta;
260
261
262 crc32 = ether_crc(ETH_ALEN, addr);
263
264
265
266
267
268 reg = (crc32 >> 31) & 0x1;
269 bit = (crc32 >> 26) & 0x1F;
270
271 mta = readl(adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
272 mta |= BIT(bit);
273 writel(mta, adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2));
274}
275
276void emac_mac_multicast_addr_clear(struct emac_adapter *adpt)
277{
278 writel(0, adpt->base + EMAC_HASH_TAB_REG0);
279 writel(0, adpt->base + EMAC_HASH_TAB_REG1);
280}
281
282
283#define EMAC_RSS_KEY(_i, _type) \
284 (EMAC_RSS_KEY0 + ((_i) * sizeof(_type)))
285#define EMAC_RSS_TBL(_i, _type) \
286 (EMAC_IDT_TABLE0 + ((_i) * sizeof(_type)))
287
288
289void emac_mac_mode_config(struct emac_adapter *adpt)
290{
291 struct net_device *netdev = adpt->netdev;
292 u32 mac;
293
294 mac = readl(adpt->base + EMAC_MAC_CTRL);
295 mac &= ~(VLAN_STRIP | PROM_MODE | MULTI_ALL | MAC_LP_EN);
296
297 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
298 mac |= VLAN_STRIP;
299
300 if (netdev->flags & IFF_PROMISC)
301 mac |= PROM_MODE;
302
303 if (netdev->flags & IFF_ALLMULTI)
304 mac |= MULTI_ALL;
305
306 writel(mac, adpt->base + EMAC_MAC_CTRL);
307}
308
309
310static void emac_mac_dma_rings_config(struct emac_adapter *adpt)
311{
312 static const unsigned short tpd_q_offset[] = {
313 EMAC_DESC_CTRL_8, EMAC_H1TPD_BASE_ADDR_LO,
314 EMAC_H2TPD_BASE_ADDR_LO, EMAC_H3TPD_BASE_ADDR_LO};
315 static const unsigned short rfd_q_offset[] = {
316 EMAC_DESC_CTRL_2, EMAC_DESC_CTRL_10,
317 EMAC_DESC_CTRL_12, EMAC_DESC_CTRL_13};
318 static const unsigned short rrd_q_offset[] = {
319 EMAC_DESC_CTRL_5, EMAC_DESC_CTRL_14,
320 EMAC_DESC_CTRL_15, EMAC_DESC_CTRL_16};
321
322
323 writel(upper_32_bits(adpt->tx_q.tpd.dma_addr),
324 adpt->base + EMAC_DESC_CTRL_1);
325
326 writel(lower_32_bits(adpt->tx_q.tpd.dma_addr),
327 adpt->base + tpd_q_offset[0]);
328
329 writel(adpt->tx_q.tpd.count & TPD_RING_SIZE_BMSK,
330 adpt->base + EMAC_DESC_CTRL_9);
331
332
333 writel(upper_32_bits(adpt->rx_q.rfd.dma_addr),
334 adpt->base + EMAC_DESC_CTRL_0);
335
336 writel(lower_32_bits(adpt->rx_q.rfd.dma_addr),
337 adpt->base + rfd_q_offset[0]);
338 writel(lower_32_bits(adpt->rx_q.rrd.dma_addr),
339 adpt->base + rrd_q_offset[0]);
340
341 writel(adpt->rx_q.rfd.count & RFD_RING_SIZE_BMSK,
342 adpt->base + EMAC_DESC_CTRL_3);
343 writel(adpt->rx_q.rrd.count & RRD_RING_SIZE_BMSK,
344 adpt->base + EMAC_DESC_CTRL_6);
345
346 writel(adpt->rxbuf_size & RX_BUFFER_SIZE_BMSK,
347 adpt->base + EMAC_DESC_CTRL_4);
348
349 writel(0, adpt->base + EMAC_DESC_CTRL_11);
350
351
352
353
354 writel(1, adpt->base + EMAC_INTER_SRAM_PART9);
355}
356
357
358static void emac_mac_tx_config(struct emac_adapter *adpt)
359{
360 u32 val;
361
362 writel((EMAC_MAX_TX_OFFLOAD_THRESH >> 3) &
363 JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK, adpt->base + EMAC_TXQ_CTRL_1);
364
365 val = (adpt->tpd_burst << NUM_TPD_BURST_PREF_SHFT) &
366 NUM_TPD_BURST_PREF_BMSK;
367
368 val |= TXQ_MODE | LS_8023_SP;
369 val |= (0x0100 << NUM_TXF_BURST_PREF_SHFT) &
370 NUM_TXF_BURST_PREF_BMSK;
371
372 writel(val, adpt->base + EMAC_TXQ_CTRL_0);
373 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_2,
374 (TXF_HWM_BMSK | TXF_LWM_BMSK), 0);
375}
376
377
378static void emac_mac_rx_config(struct emac_adapter *adpt)
379{
380 u32 val;
381
382 val = (adpt->rfd_burst << NUM_RFD_BURST_PREF_SHFT) &
383 NUM_RFD_BURST_PREF_BMSK;
384 val |= (SP_IPV6 | CUT_THRU_EN);
385
386 writel(val, adpt->base + EMAC_RXQ_CTRL_0);
387
388 val = readl(adpt->base + EMAC_RXQ_CTRL_1);
389 val &= ~(JUMBO_1KAH_BMSK | RFD_PREF_LOW_THRESHOLD_BMSK |
390 RFD_PREF_UP_THRESHOLD_BMSK);
391 val |= (JUMBO_1KAH << JUMBO_1KAH_SHFT) |
392 (RFD_PREF_LOW_TH << RFD_PREF_LOW_THRESHOLD_SHFT) |
393 (RFD_PREF_UP_TH << RFD_PREF_UP_THRESHOLD_SHFT);
394 writel(val, adpt->base + EMAC_RXQ_CTRL_1);
395
396 val = readl(adpt->base + EMAC_RXQ_CTRL_2);
397 val &= ~(RXF_DOF_THRESHOLD_BMSK | RXF_UOF_THRESHOLD_BMSK);
398 val |= (RXF_DOF_THRESFHOLD << RXF_DOF_THRESHOLD_SHFT) |
399 (RXF_UOF_THRESFHOLD << RXF_UOF_THRESHOLD_SHFT);
400 writel(val, adpt->base + EMAC_RXQ_CTRL_2);
401
402 val = readl(adpt->base + EMAC_RXQ_CTRL_3);
403 val &= ~(RXD_TIMER_BMSK | RXD_THRESHOLD_BMSK);
404 val |= RXD_TH << RXD_THRESHOLD_SHFT;
405 writel(val, adpt->base + EMAC_RXQ_CTRL_3);
406}
407
408
409static void emac_mac_dma_config(struct emac_adapter *adpt)
410{
411 u32 dma_ctrl = DMAR_REQ_PRI;
412
413 switch (adpt->dma_order) {
414 case emac_dma_ord_in:
415 dma_ctrl |= IN_ORDER_MODE;
416 break;
417 case emac_dma_ord_enh:
418 dma_ctrl |= ENH_ORDER_MODE;
419 break;
420 case emac_dma_ord_out:
421 dma_ctrl |= OUT_ORDER_MODE;
422 break;
423 default:
424 break;
425 }
426
427 dma_ctrl |= (((u32)adpt->dmar_block) << REGRDBLEN_SHFT) &
428 REGRDBLEN_BMSK;
429 dma_ctrl |= (((u32)adpt->dmaw_block) << REGWRBLEN_SHFT) &
430 REGWRBLEN_BMSK;
431 dma_ctrl |= (((u32)adpt->dmar_dly_cnt) << DMAR_DLY_CNT_SHFT) &
432 DMAR_DLY_CNT_BMSK;
433 dma_ctrl |= (((u32)adpt->dmaw_dly_cnt) << DMAW_DLY_CNT_SHFT) &
434 DMAW_DLY_CNT_BMSK;
435
436
437 writel(dma_ctrl, adpt->base + EMAC_DMA_CTRL);
438}
439
440
441static void emac_set_mac_address(struct emac_adapter *adpt, u8 *addr)
442{
443 u32 sta;
444
445
446
447
448
449
450 sta = (((u32)addr[2]) << 24) | (((u32)addr[3]) << 16) |
451 (((u32)addr[4]) << 8) | (((u32)addr[5]));
452 writel(sta, adpt->base + EMAC_MAC_STA_ADDR0);
453
454
455 sta = (((u32)addr[0]) << 8) | (u32)addr[1];
456 writel(sta, adpt->base + EMAC_MAC_STA_ADDR1);
457}
458
459static void emac_mac_config(struct emac_adapter *adpt)
460{
461 struct net_device *netdev = adpt->netdev;
462 unsigned int max_frame;
463 u32 val;
464
465 emac_set_mac_address(adpt, netdev->dev_addr);
466
467 max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
468 adpt->rxbuf_size = netdev->mtu > EMAC_DEF_RX_BUF_SIZE ?
469 ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE;
470
471 emac_mac_dma_rings_config(adpt);
472
473 writel(netdev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN,
474 adpt->base + EMAC_MAX_FRAM_LEN_CTRL);
475
476 emac_mac_tx_config(adpt);
477 emac_mac_rx_config(adpt);
478 emac_mac_dma_config(adpt);
479
480 val = readl(adpt->base + EMAC_AXI_MAST_CTRL);
481 val &= ~(DATA_BYTE_SWAP | MAX_BOUND);
482 val |= MAX_BTYPE;
483 writel(val, adpt->base + EMAC_AXI_MAST_CTRL);
484 writel(0, adpt->base + EMAC_CLK_GATE_CTRL);
485 writel(RX_UNCPL_INT_EN, adpt->base + EMAC_MISC_CTRL);
486}
487
488void emac_mac_reset(struct emac_adapter *adpt)
489{
490 emac_mac_stop(adpt);
491
492 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, SOFT_RST);
493 usleep_range(100, 150);
494
495
496 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, INT_RD_CLR_EN);
497}
498
499static void emac_mac_start(struct emac_adapter *adpt)
500{
501 struct phy_device *phydev = adpt->phydev;
502 u32 mac, csr1;
503
504
505 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, 0, TXQ_EN);
506
507
508 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, 0, RXQ_EN);
509
510
511 mac = readl(adpt->base + EMAC_MAC_CTRL);
512 csr1 = readl(adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
513
514 mac |= TXEN | RXEN;
515
516
517
518
519
520 mac &= ~(RXFC | TXFC);
521
522 if (adpt->automatic) {
523
524 adpt->rx_flow_control = phydev->pause;
525 adpt->tx_flow_control = phydev->pause != phydev->asym_pause;
526 }
527 mac |= adpt->rx_flow_control ? RXFC : 0;
528 mac |= adpt->tx_flow_control ? TXFC : 0;
529
530
531 mac &= ~SPEED_MASK;
532 if (phydev->speed == SPEED_1000) {
533 mac |= SPEED(2);
534 csr1 |= FREQ_MODE;
535 } else {
536 mac |= SPEED(1);
537 csr1 &= ~FREQ_MODE;
538 }
539
540 if (phydev->duplex == DUPLEX_FULL)
541 mac |= FULLD;
542 else
543 mac &= ~FULLD;
544
545
546 mac |= (CRCE | PCRCE);
547 mac |= ((adpt->preamble << PRLEN_SHFT) & PRLEN_BMSK);
548 mac |= BROAD_EN;
549 mac |= FLCHK;
550 mac &= ~RX_CHKSUM_EN;
551 mac &= ~(HUGEN | VLAN_STRIP | TPAUSE | SIMR | HUGE | MULTI_ALL |
552 DEBUG_MODE | SINGLE_PAUSE_MODE);
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574 mac |= adpt->single_pause_mode ? SINGLE_PAUSE_MODE : 0;
575
576 writel_relaxed(csr1, adpt->csr + EMAC_EMAC_WRAPPER_CSR1);
577
578 writel_relaxed(mac, adpt->base + EMAC_MAC_CTRL);
579
580
581
582
583
584 writel_relaxed(adpt->irq_mod, adpt->base + EMAC_IRQ_MOD_TIM_INIT);
585 writel_relaxed(INT_RD_CLR_EN | LPW_MODE | IRQ_MODERATOR_EN |
586 IRQ_MODERATOR2_EN, adpt->base + EMAC_DMA_MAS_CTRL);
587
588 emac_mac_mode_config(adpt);
589
590 emac_reg_update32(adpt->base + EMAC_ATHR_HEADER_CTRL,
591 (HEADER_ENABLE | HEADER_CNT_EN), 0);
592}
593
594void emac_mac_stop(struct emac_adapter *adpt)
595{
596 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, RXQ_EN, 0);
597 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, TXQ_EN, 0);
598 emac_reg_update32(adpt->base + EMAC_MAC_CTRL, TXEN | RXEN, 0);
599 usleep_range(1000, 1050);
600}
601
602
603static void emac_tx_q_descs_free(struct emac_adapter *adpt)
604{
605 struct emac_tx_queue *tx_q = &adpt->tx_q;
606 unsigned int i;
607 size_t size;
608
609
610 if (!tx_q->tpd.tpbuff)
611 return;
612
613 for (i = 0; i < tx_q->tpd.count; i++) {
614 struct emac_buffer *tpbuf = GET_TPD_BUFFER(tx_q, i);
615
616 if (tpbuf->dma_addr) {
617 dma_unmap_single(adpt->netdev->dev.parent,
618 tpbuf->dma_addr, tpbuf->length,
619 DMA_TO_DEVICE);
620 tpbuf->dma_addr = 0;
621 }
622 if (tpbuf->skb) {
623 dev_kfree_skb_any(tpbuf->skb);
624 tpbuf->skb = NULL;
625 }
626 }
627
628 size = sizeof(struct emac_buffer) * tx_q->tpd.count;
629 memset(tx_q->tpd.tpbuff, 0, size);
630
631
632 memset(tx_q->tpd.v_addr, 0, tx_q->tpd.size);
633
634 tx_q->tpd.consume_idx = 0;
635 tx_q->tpd.produce_idx = 0;
636}
637
638
639static void emac_rx_q_free_descs(struct emac_adapter *adpt)
640{
641 struct device *dev = adpt->netdev->dev.parent;
642 struct emac_rx_queue *rx_q = &adpt->rx_q;
643 unsigned int i;
644 size_t size;
645
646
647 if (!rx_q->rfd.rfbuff)
648 return;
649
650 for (i = 0; i < rx_q->rfd.count; i++) {
651 struct emac_buffer *rfbuf = GET_RFD_BUFFER(rx_q, i);
652
653 if (rfbuf->dma_addr) {
654 dma_unmap_single(dev, rfbuf->dma_addr, rfbuf->length,
655 DMA_FROM_DEVICE);
656 rfbuf->dma_addr = 0;
657 }
658 if (rfbuf->skb) {
659 dev_kfree_skb(rfbuf->skb);
660 rfbuf->skb = NULL;
661 }
662 }
663
664 size = sizeof(struct emac_buffer) * rx_q->rfd.count;
665 memset(rx_q->rfd.rfbuff, 0, size);
666
667
668 memset(rx_q->rrd.v_addr, 0, rx_q->rrd.size);
669 rx_q->rrd.produce_idx = 0;
670 rx_q->rrd.consume_idx = 0;
671
672 memset(rx_q->rfd.v_addr, 0, rx_q->rfd.size);
673 rx_q->rfd.produce_idx = 0;
674 rx_q->rfd.consume_idx = 0;
675}
676
677
678static void emac_tx_q_bufs_free(struct emac_adapter *adpt)
679{
680 struct emac_tx_queue *tx_q = &adpt->tx_q;
681
682 emac_tx_q_descs_free(adpt);
683
684 kfree(tx_q->tpd.tpbuff);
685 tx_q->tpd.tpbuff = NULL;
686 tx_q->tpd.v_addr = NULL;
687 tx_q->tpd.dma_addr = 0;
688 tx_q->tpd.size = 0;
689}
690
691
692static int emac_tx_q_desc_alloc(struct emac_adapter *adpt,
693 struct emac_tx_queue *tx_q)
694{
695 struct emac_ring_header *ring_header = &adpt->ring_header;
696 size_t size;
697
698 size = sizeof(struct emac_buffer) * tx_q->tpd.count;
699 tx_q->tpd.tpbuff = kzalloc(size, GFP_KERNEL);
700 if (!tx_q->tpd.tpbuff)
701 return -ENOMEM;
702
703 tx_q->tpd.size = tx_q->tpd.count * (adpt->tpd_size * 4);
704 tx_q->tpd.dma_addr = ring_header->dma_addr + ring_header->used;
705 tx_q->tpd.v_addr = ring_header->v_addr + ring_header->used;
706 ring_header->used += ALIGN(tx_q->tpd.size, 8);
707 tx_q->tpd.produce_idx = 0;
708 tx_q->tpd.consume_idx = 0;
709
710 return 0;
711}
712
713
714static void emac_rx_q_bufs_free(struct emac_adapter *adpt)
715{
716 struct emac_rx_queue *rx_q = &adpt->rx_q;
717
718 emac_rx_q_free_descs(adpt);
719
720 kfree(rx_q->rfd.rfbuff);
721 rx_q->rfd.rfbuff = NULL;
722
723 rx_q->rfd.v_addr = NULL;
724 rx_q->rfd.dma_addr = 0;
725 rx_q->rfd.size = 0;
726
727 rx_q->rrd.v_addr = NULL;
728 rx_q->rrd.dma_addr = 0;
729 rx_q->rrd.size = 0;
730}
731
732
733static int emac_rx_descs_alloc(struct emac_adapter *adpt)
734{
735 struct emac_ring_header *ring_header = &adpt->ring_header;
736 struct emac_rx_queue *rx_q = &adpt->rx_q;
737 size_t size;
738
739 size = sizeof(struct emac_buffer) * rx_q->rfd.count;
740 rx_q->rfd.rfbuff = kzalloc(size, GFP_KERNEL);
741 if (!rx_q->rfd.rfbuff)
742 return -ENOMEM;
743
744 rx_q->rrd.size = rx_q->rrd.count * (adpt->rrd_size * 4);
745 rx_q->rfd.size = rx_q->rfd.count * (adpt->rfd_size * 4);
746
747 rx_q->rrd.dma_addr = ring_header->dma_addr + ring_header->used;
748 rx_q->rrd.v_addr = ring_header->v_addr + ring_header->used;
749 ring_header->used += ALIGN(rx_q->rrd.size, 8);
750
751 rx_q->rfd.dma_addr = ring_header->dma_addr + ring_header->used;
752 rx_q->rfd.v_addr = ring_header->v_addr + ring_header->used;
753 ring_header->used += ALIGN(rx_q->rfd.size, 8);
754
755 rx_q->rrd.produce_idx = 0;
756 rx_q->rrd.consume_idx = 0;
757
758 rx_q->rfd.produce_idx = 0;
759 rx_q->rfd.consume_idx = 0;
760
761 return 0;
762}
763
764
765int emac_mac_rx_tx_rings_alloc_all(struct emac_adapter *adpt)
766{
767 struct emac_ring_header *ring_header = &adpt->ring_header;
768 struct device *dev = adpt->netdev->dev.parent;
769 unsigned int num_tx_descs = adpt->tx_desc_cnt;
770 unsigned int num_rx_descs = adpt->rx_desc_cnt;
771 int ret;
772
773 adpt->tx_q.tpd.count = adpt->tx_desc_cnt;
774
775 adpt->rx_q.rrd.count = adpt->rx_desc_cnt;
776 adpt->rx_q.rfd.count = adpt->rx_desc_cnt;
777
778
779
780
781 ring_header->size = num_tx_descs * (adpt->tpd_size * 4) +
782 num_rx_descs * (adpt->rfd_size * 4) +
783 num_rx_descs * (adpt->rrd_size * 4) +
784 8 + 2 * 8;
785
786 ring_header->used = 0;
787 ring_header->v_addr = dma_zalloc_coherent(dev, ring_header->size,
788 &ring_header->dma_addr,
789 GFP_KERNEL);
790 if (!ring_header->v_addr)
791 return -ENOMEM;
792
793 ring_header->used = ALIGN(ring_header->dma_addr, 8) -
794 ring_header->dma_addr;
795
796 ret = emac_tx_q_desc_alloc(adpt, &adpt->tx_q);
797 if (ret) {
798 netdev_err(adpt->netdev, "error: Tx Queue alloc failed\n");
799 goto err_alloc_tx;
800 }
801
802 ret = emac_rx_descs_alloc(adpt);
803 if (ret) {
804 netdev_err(adpt->netdev, "error: Rx Queue alloc failed\n");
805 goto err_alloc_rx;
806 }
807
808 return 0;
809
810err_alloc_rx:
811 emac_tx_q_bufs_free(adpt);
812err_alloc_tx:
813 dma_free_coherent(dev, ring_header->size,
814 ring_header->v_addr, ring_header->dma_addr);
815
816 ring_header->v_addr = NULL;
817 ring_header->dma_addr = 0;
818 ring_header->size = 0;
819 ring_header->used = 0;
820
821 return ret;
822}
823
824
825void emac_mac_rx_tx_rings_free_all(struct emac_adapter *adpt)
826{
827 struct emac_ring_header *ring_header = &adpt->ring_header;
828 struct device *dev = adpt->netdev->dev.parent;
829
830 emac_tx_q_bufs_free(adpt);
831 emac_rx_q_bufs_free(adpt);
832
833 dma_free_coherent(dev, ring_header->size,
834 ring_header->v_addr, ring_header->dma_addr);
835
836 ring_header->v_addr = NULL;
837 ring_header->dma_addr = 0;
838 ring_header->size = 0;
839 ring_header->used = 0;
840}
841
842
843static void emac_mac_rx_tx_ring_reset_all(struct emac_adapter *adpt)
844{
845 unsigned int i;
846
847 adpt->tx_q.tpd.produce_idx = 0;
848 adpt->tx_q.tpd.consume_idx = 0;
849 for (i = 0; i < adpt->tx_q.tpd.count; i++)
850 adpt->tx_q.tpd.tpbuff[i].dma_addr = 0;
851
852 adpt->rx_q.rrd.produce_idx = 0;
853 adpt->rx_q.rrd.consume_idx = 0;
854 adpt->rx_q.rfd.produce_idx = 0;
855 adpt->rx_q.rfd.consume_idx = 0;
856 for (i = 0; i < adpt->rx_q.rfd.count; i++)
857 adpt->rx_q.rfd.rfbuff[i].dma_addr = 0;
858}
859
860
861static void emac_mac_rx_rfd_create(struct emac_adapter *adpt,
862 struct emac_rx_queue *rx_q,
863 dma_addr_t addr)
864{
865 u32 *hw_rfd = EMAC_RFD(rx_q, adpt->rfd_size, rx_q->rfd.produce_idx);
866
867 *(hw_rfd++) = lower_32_bits(addr);
868 *hw_rfd = upper_32_bits(addr);
869
870 if (++rx_q->rfd.produce_idx == rx_q->rfd.count)
871 rx_q->rfd.produce_idx = 0;
872}
873
874
875static void emac_mac_rx_descs_refill(struct emac_adapter *adpt,
876 struct emac_rx_queue *rx_q)
877{
878 struct emac_buffer *curr_rxbuf;
879 struct emac_buffer *next_rxbuf;
880 unsigned int count = 0;
881 u32 next_produce_idx;
882
883 next_produce_idx = rx_q->rfd.produce_idx + 1;
884 if (next_produce_idx == rx_q->rfd.count)
885 next_produce_idx = 0;
886
887 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
888 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
889
890
891 while (!next_rxbuf->dma_addr) {
892 struct sk_buff *skb;
893 int ret;
894
895 skb = netdev_alloc_skb_ip_align(adpt->netdev, adpt->rxbuf_size);
896 if (!skb)
897 break;
898
899 curr_rxbuf->dma_addr =
900 dma_map_single(adpt->netdev->dev.parent, skb->data,
901 adpt->rxbuf_size, DMA_FROM_DEVICE);
902
903 ret = dma_mapping_error(adpt->netdev->dev.parent,
904 curr_rxbuf->dma_addr);
905 if (ret) {
906 dev_kfree_skb(skb);
907 break;
908 }
909 curr_rxbuf->skb = skb;
910 curr_rxbuf->length = adpt->rxbuf_size;
911
912 emac_mac_rx_rfd_create(adpt, rx_q, curr_rxbuf->dma_addr);
913 next_produce_idx = rx_q->rfd.produce_idx + 1;
914 if (next_produce_idx == rx_q->rfd.count)
915 next_produce_idx = 0;
916
917 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx);
918 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx);
919 count++;
920 }
921
922 if (count) {
923 u32 prod_idx = (rx_q->rfd.produce_idx << rx_q->produce_shift) &
924 rx_q->produce_mask;
925 emac_reg_update32(adpt->base + rx_q->produce_reg,
926 rx_q->produce_mask, prod_idx);
927 }
928}
929
930static void emac_adjust_link(struct net_device *netdev)
931{
932 struct emac_adapter *adpt = netdev_priv(netdev);
933 struct emac_sgmii *sgmii = &adpt->phy;
934 struct phy_device *phydev = netdev->phydev;
935
936 if (phydev->link) {
937 emac_mac_start(adpt);
938 sgmii->link_up(adpt);
939 } else {
940 sgmii->link_down(adpt);
941 emac_mac_stop(adpt);
942 }
943
944 phy_print_status(phydev);
945}
946
947
948int emac_mac_up(struct emac_adapter *adpt)
949{
950 struct net_device *netdev = adpt->netdev;
951 int ret;
952
953 emac_mac_rx_tx_ring_reset_all(adpt);
954 emac_mac_config(adpt);
955 emac_mac_rx_descs_refill(adpt, &adpt->rx_q);
956
957 adpt->phydev->irq = PHY_POLL;
958 ret = phy_connect_direct(netdev, adpt->phydev, emac_adjust_link,
959 PHY_INTERFACE_MODE_SGMII);
960 if (ret) {
961 netdev_err(adpt->netdev, "could not connect phy\n");
962 return ret;
963 }
964
965 phy_attached_print(adpt->phydev, NULL);
966
967
968 writel((u32)~DIS_INT, adpt->base + EMAC_INT_STATUS);
969 writel(adpt->irq.mask, adpt->base + EMAC_INT_MASK);
970
971 phy_start(adpt->phydev);
972
973 napi_enable(&adpt->rx_q.napi);
974 netif_start_queue(netdev);
975
976 return 0;
977}
978
979
980void emac_mac_down(struct emac_adapter *adpt)
981{
982 struct net_device *netdev = adpt->netdev;
983
984 netif_stop_queue(netdev);
985 napi_disable(&adpt->rx_q.napi);
986
987 phy_stop(adpt->phydev);
988
989
990
991
992
993 writel(DIS_INT, adpt->base + EMAC_INT_STATUS);
994 writel(0, adpt->base + EMAC_INT_MASK);
995 synchronize_irq(adpt->irq.irq);
996
997 phy_disconnect(adpt->phydev);
998
999 emac_mac_reset(adpt);
1000
1001 emac_tx_q_descs_free(adpt);
1002 netdev_reset_queue(adpt->netdev);
1003 emac_rx_q_free_descs(adpt);
1004}
1005
1006
1007static bool emac_rx_process_rrd(struct emac_adapter *adpt,
1008 struct emac_rx_queue *rx_q,
1009 struct emac_rrd *rrd)
1010{
1011 u32 *hw_rrd = EMAC_RRD(rx_q, adpt->rrd_size, rx_q->rrd.consume_idx);
1012
1013 rrd->word[3] = *(hw_rrd + 3);
1014
1015 if (!RRD_UPDT(rrd))
1016 return false;
1017
1018 rrd->word[4] = 0;
1019 rrd->word[5] = 0;
1020
1021 rrd->word[0] = *(hw_rrd++);
1022 rrd->word[1] = *(hw_rrd++);
1023 rrd->word[2] = *(hw_rrd++);
1024
1025 if (unlikely(RRD_NOR(rrd) != 1)) {
1026 netdev_err(adpt->netdev,
1027 "error: multi-RFD not support yet! nor:%lu\n",
1028 RRD_NOR(rrd));
1029 }
1030
1031
1032 RRD_UPDT_SET(rrd, 0);
1033 *hw_rrd = rrd->word[3];
1034
1035 if (++rx_q->rrd.consume_idx == rx_q->rrd.count)
1036 rx_q->rrd.consume_idx = 0;
1037
1038 return true;
1039}
1040
1041
1042static void emac_tx_tpd_create(struct emac_adapter *adpt,
1043 struct emac_tx_queue *tx_q, struct emac_tpd *tpd)
1044{
1045 u32 *hw_tpd;
1046
1047 tx_q->tpd.last_produce_idx = tx_q->tpd.produce_idx;
1048 hw_tpd = EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.produce_idx);
1049
1050 if (++tx_q->tpd.produce_idx == tx_q->tpd.count)
1051 tx_q->tpd.produce_idx = 0;
1052
1053 *(hw_tpd++) = tpd->word[0];
1054 *(hw_tpd++) = tpd->word[1];
1055 *(hw_tpd++) = tpd->word[2];
1056 *hw_tpd = tpd->word[3];
1057}
1058
1059
1060static void emac_tx_tpd_mark_last(struct emac_adapter *adpt,
1061 struct emac_tx_queue *tx_q)
1062{
1063 u32 *hw_tpd =
1064 EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.last_produce_idx);
1065 u32 tmp_tpd;
1066
1067 tmp_tpd = *(hw_tpd + 1);
1068 tmp_tpd |= EMAC_TPD_LAST_FRAGMENT;
1069 *(hw_tpd + 1) = tmp_tpd;
1070}
1071
1072static void emac_rx_rfd_clean(struct emac_rx_queue *rx_q, struct emac_rrd *rrd)
1073{
1074 struct emac_buffer *rfbuf = rx_q->rfd.rfbuff;
1075 u32 consume_idx = RRD_SI(rrd);
1076 unsigned int i;
1077
1078 for (i = 0; i < RRD_NOR(rrd); i++) {
1079 rfbuf[consume_idx].skb = NULL;
1080 if (++consume_idx == rx_q->rfd.count)
1081 consume_idx = 0;
1082 }
1083
1084 rx_q->rfd.consume_idx = consume_idx;
1085 rx_q->rfd.process_idx = consume_idx;
1086}
1087
1088
1089static void emac_receive_skb(struct emac_rx_queue *rx_q,
1090 struct sk_buff *skb,
1091 u16 vlan_tag, bool vlan_flag)
1092{
1093 if (vlan_flag) {
1094 u16 vlan;
1095
1096 EMAC_TAG_TO_VLAN(vlan_tag, vlan);
1097 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);
1098 }
1099
1100 napi_gro_receive(&rx_q->napi, skb);
1101}
1102
1103
1104void emac_mac_rx_process(struct emac_adapter *adpt, struct emac_rx_queue *rx_q,
1105 int *num_pkts, int max_pkts)
1106{
1107 u32 proc_idx, hw_consume_idx, num_consume_pkts;
1108 struct net_device *netdev = adpt->netdev;
1109 struct emac_buffer *rfbuf;
1110 unsigned int count = 0;
1111 struct emac_rrd rrd;
1112 struct sk_buff *skb;
1113 u32 reg;
1114
1115 reg = readl_relaxed(adpt->base + rx_q->consume_reg);
1116
1117 hw_consume_idx = (reg & rx_q->consume_mask) >> rx_q->consume_shift;
1118 num_consume_pkts = (hw_consume_idx >= rx_q->rrd.consume_idx) ?
1119 (hw_consume_idx - rx_q->rrd.consume_idx) :
1120 (hw_consume_idx + rx_q->rrd.count - rx_q->rrd.consume_idx);
1121
1122 do {
1123 if (!num_consume_pkts)
1124 break;
1125
1126 if (!emac_rx_process_rrd(adpt, rx_q, &rrd))
1127 break;
1128
1129 if (likely(RRD_NOR(&rrd) == 1)) {
1130
1131 rfbuf = GET_RFD_BUFFER(rx_q, RRD_SI(&rrd));
1132 dma_unmap_single(adpt->netdev->dev.parent,
1133 rfbuf->dma_addr, rfbuf->length,
1134 DMA_FROM_DEVICE);
1135 rfbuf->dma_addr = 0;
1136 skb = rfbuf->skb;
1137 } else {
1138 netdev_err(adpt->netdev,
1139 "error: multi-RFD not support yet!\n");
1140 break;
1141 }
1142 emac_rx_rfd_clean(rx_q, &rrd);
1143 num_consume_pkts--;
1144 count++;
1145
1146
1147
1148
1149
1150 if (rrd.word[EMAC_RRD_STATS_DW_IDX] & EMAC_RRD_ERROR) {
1151 netif_dbg(adpt, rx_status, adpt->netdev,
1152 "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n",
1153 rrd.word[0], rrd.word[1],
1154 rrd.word[2], rrd.word[3]);
1155
1156 dev_kfree_skb(skb);
1157 continue;
1158 }
1159
1160 skb_put(skb, RRD_PKT_SIZE(&rrd) - ETH_FCS_LEN);
1161 skb->dev = netdev;
1162 skb->protocol = eth_type_trans(skb, skb->dev);
1163 if (netdev->features & NETIF_F_RXCSUM)
1164 skb->ip_summed = RRD_L4F(&rrd) ?
1165 CHECKSUM_NONE : CHECKSUM_UNNECESSARY;
1166 else
1167 skb_checksum_none_assert(skb);
1168
1169 emac_receive_skb(rx_q, skb, (u16)RRD_CVALN_TAG(&rrd),
1170 (bool)RRD_CVTAG(&rrd));
1171
1172 (*num_pkts)++;
1173 } while (*num_pkts < max_pkts);
1174
1175 if (count) {
1176 proc_idx = (rx_q->rfd.process_idx << rx_q->process_shft) &
1177 rx_q->process_mask;
1178 emac_reg_update32(adpt->base + rx_q->process_reg,
1179 rx_q->process_mask, proc_idx);
1180 emac_mac_rx_descs_refill(adpt, rx_q);
1181 }
1182}
1183
1184
1185static unsigned int emac_tpd_num_free_descs(struct emac_tx_queue *tx_q)
1186{
1187 u32 produce_idx = tx_q->tpd.produce_idx;
1188 u32 consume_idx = tx_q->tpd.consume_idx;
1189
1190 return (consume_idx > produce_idx) ?
1191 (consume_idx - produce_idx - 1) :
1192 (tx_q->tpd.count + consume_idx - produce_idx - 1);
1193}
1194
1195
1196void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q)
1197{
1198 u32 reg = readl_relaxed(adpt->base + tx_q->consume_reg);
1199 u32 hw_consume_idx, pkts_compl = 0, bytes_compl = 0;
1200 struct emac_buffer *tpbuf;
1201
1202 hw_consume_idx = (reg & tx_q->consume_mask) >> tx_q->consume_shift;
1203
1204 while (tx_q->tpd.consume_idx != hw_consume_idx) {
1205 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx);
1206 if (tpbuf->dma_addr) {
1207 dma_unmap_single(adpt->netdev->dev.parent,
1208 tpbuf->dma_addr, tpbuf->length,
1209 DMA_TO_DEVICE);
1210 tpbuf->dma_addr = 0;
1211 }
1212
1213 if (tpbuf->skb) {
1214 pkts_compl++;
1215 bytes_compl += tpbuf->skb->len;
1216 dev_kfree_skb_irq(tpbuf->skb);
1217 tpbuf->skb = NULL;
1218 }
1219
1220 if (++tx_q->tpd.consume_idx == tx_q->tpd.count)
1221 tx_q->tpd.consume_idx = 0;
1222 }
1223
1224 netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl);
1225
1226 if (netif_queue_stopped(adpt->netdev))
1227 if (emac_tpd_num_free_descs(tx_q) > (MAX_SKB_FRAGS + 1))
1228 netif_wake_queue(adpt->netdev);
1229}
1230
1231
1232void emac_mac_rx_tx_ring_init_all(struct platform_device *pdev,
1233 struct emac_adapter *adpt)
1234{
1235 adpt->rx_q.netdev = adpt->netdev;
1236
1237 adpt->rx_q.produce_reg = EMAC_MAILBOX_0;
1238 adpt->rx_q.produce_mask = RFD0_PROD_IDX_BMSK;
1239 adpt->rx_q.produce_shift = RFD0_PROD_IDX_SHFT;
1240
1241 adpt->rx_q.process_reg = EMAC_MAILBOX_0;
1242 adpt->rx_q.process_mask = RFD0_PROC_IDX_BMSK;
1243 adpt->rx_q.process_shft = RFD0_PROC_IDX_SHFT;
1244
1245 adpt->rx_q.consume_reg = EMAC_MAILBOX_3;
1246 adpt->rx_q.consume_mask = RFD0_CONS_IDX_BMSK;
1247 adpt->rx_q.consume_shift = RFD0_CONS_IDX_SHFT;
1248
1249 adpt->rx_q.irq = &adpt->irq;
1250 adpt->rx_q.intr = adpt->irq.mask & ISR_RX_PKT;
1251
1252 adpt->tx_q.produce_reg = EMAC_MAILBOX_15;
1253 adpt->tx_q.produce_mask = NTPD_PROD_IDX_BMSK;
1254 adpt->tx_q.produce_shift = NTPD_PROD_IDX_SHFT;
1255
1256 adpt->tx_q.consume_reg = EMAC_MAILBOX_2;
1257 adpt->tx_q.consume_mask = NTPD_CONS_IDX_BMSK;
1258 adpt->tx_q.consume_shift = NTPD_CONS_IDX_SHFT;
1259}
1260
1261
1262static int emac_tso_csum(struct emac_adapter *adpt,
1263 struct emac_tx_queue *tx_q,
1264 struct sk_buff *skb,
1265 struct emac_tpd *tpd)
1266{
1267 unsigned int hdr_len;
1268 int ret;
1269
1270 if (skb_is_gso(skb)) {
1271 if (skb_header_cloned(skb)) {
1272 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
1273 if (unlikely(ret))
1274 return ret;
1275 }
1276
1277 if (skb->protocol == htons(ETH_P_IP)) {
1278 u32 pkt_len = ((unsigned char *)ip_hdr(skb) - skb->data)
1279 + ntohs(ip_hdr(skb)->tot_len);
1280 if (skb->len > pkt_len)
1281 pskb_trim(skb, pkt_len);
1282 }
1283
1284 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1285 if (unlikely(skb->len == hdr_len)) {
1286
1287 netif_warn(adpt, tx_err, adpt->netdev,
1288 "tso not needed for packet with 0 data\n");
1289 goto do_csum;
1290 }
1291
1292 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
1293 ip_hdr(skb)->check = 0;
1294 tcp_hdr(skb)->check =
1295 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1296 ip_hdr(skb)->daddr,
1297 0, IPPROTO_TCP, 0);
1298 TPD_IPV4_SET(tpd, 1);
1299 }
1300
1301 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
1302
1303 struct emac_tpd extra_tpd;
1304
1305 memset(tpd, 0, sizeof(*tpd));
1306 memset(&extra_tpd, 0, sizeof(extra_tpd));
1307
1308 ipv6_hdr(skb)->payload_len = 0;
1309 tcp_hdr(skb)->check =
1310 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1311 &ipv6_hdr(skb)->daddr,
1312 0, IPPROTO_TCP, 0);
1313 TPD_PKT_LEN_SET(&extra_tpd, skb->len);
1314 TPD_LSO_SET(&extra_tpd, 1);
1315 TPD_LSOV_SET(&extra_tpd, 1);
1316 emac_tx_tpd_create(adpt, tx_q, &extra_tpd);
1317 TPD_LSOV_SET(tpd, 1);
1318 }
1319
1320 TPD_LSO_SET(tpd, 1);
1321 TPD_TCPHDR_OFFSET_SET(tpd, skb_transport_offset(skb));
1322 TPD_MSS_SET(tpd, skb_shinfo(skb)->gso_size);
1323 return 0;
1324 }
1325
1326do_csum:
1327 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
1328 unsigned int css, cso;
1329
1330 cso = skb_transport_offset(skb);
1331 if (unlikely(cso & 0x1)) {
1332 netdev_err(adpt->netdev,
1333 "error: payload offset should be even\n");
1334 return -EINVAL;
1335 }
1336 css = cso + skb->csum_offset;
1337
1338 TPD_PAYLOAD_OFFSET_SET(tpd, cso >> 1);
1339 TPD_CXSUM_OFFSET_SET(tpd, css >> 1);
1340 TPD_CSX_SET(tpd, 1);
1341 }
1342
1343 return 0;
1344}
1345
1346
1347static void emac_tx_fill_tpd(struct emac_adapter *adpt,
1348 struct emac_tx_queue *tx_q, struct sk_buff *skb,
1349 struct emac_tpd *tpd)
1350{
1351 unsigned int nr_frags = skb_shinfo(skb)->nr_frags;
1352 unsigned int first = tx_q->tpd.produce_idx;
1353 unsigned int len = skb_headlen(skb);
1354 struct emac_buffer *tpbuf = NULL;
1355 unsigned int mapped_len = 0;
1356 unsigned int i;
1357 int count = 0;
1358 int ret;
1359
1360
1361 if (TPD_LSO(tpd)) {
1362 mapped_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
1363
1364 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1365 tpbuf->length = mapped_len;
1366 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
1367 skb->data, tpbuf->length,
1368 DMA_TO_DEVICE);
1369 ret = dma_mapping_error(adpt->netdev->dev.parent,
1370 tpbuf->dma_addr);
1371 if (ret)
1372 goto error;
1373
1374 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1375 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1376 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1377 emac_tx_tpd_create(adpt, tx_q, tpd);
1378 count++;
1379 }
1380
1381 if (mapped_len < len) {
1382 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1383 tpbuf->length = len - mapped_len;
1384 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent,
1385 skb->data + mapped_len,
1386 tpbuf->length, DMA_TO_DEVICE);
1387 ret = dma_mapping_error(adpt->netdev->dev.parent,
1388 tpbuf->dma_addr);
1389 if (ret)
1390 goto error;
1391
1392 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1393 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1394 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1395 emac_tx_tpd_create(adpt, tx_q, tpd);
1396 count++;
1397 }
1398
1399 for (i = 0; i < nr_frags; i++) {
1400 struct skb_frag_struct *frag;
1401
1402 frag = &skb_shinfo(skb)->frags[i];
1403
1404 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx);
1405 tpbuf->length = frag->size;
1406 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent,
1407 frag->page.p, frag->page_offset,
1408 tpbuf->length, DMA_TO_DEVICE);
1409 ret = dma_mapping_error(adpt->netdev->dev.parent,
1410 tpbuf->dma_addr);
1411 if (ret)
1412 goto error;
1413
1414 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr));
1415 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr));
1416 TPD_BUF_LEN_SET(tpd, tpbuf->length);
1417 emac_tx_tpd_create(adpt, tx_q, tpd);
1418 count++;
1419 }
1420
1421
1422 wmb();
1423 emac_tx_tpd_mark_last(adpt, tx_q);
1424
1425
1426
1427
1428 tpbuf->skb = skb;
1429
1430 return;
1431
1432error:
1433
1434 tx_q->tpd.produce_idx = first;
1435
1436 while (count--) {
1437 tpbuf = GET_TPD_BUFFER(tx_q, first);
1438 dma_unmap_page(adpt->netdev->dev.parent, tpbuf->dma_addr,
1439 tpbuf->length, DMA_TO_DEVICE);
1440 tpbuf->dma_addr = 0;
1441 tpbuf->length = 0;
1442
1443 if (++first == tx_q->tpd.count)
1444 first = 0;
1445 }
1446
1447 dev_kfree_skb(skb);
1448}
1449
1450
1451int emac_mac_tx_buf_send(struct emac_adapter *adpt, struct emac_tx_queue *tx_q,
1452 struct sk_buff *skb)
1453{
1454 struct emac_tpd tpd;
1455 u32 prod_idx;
1456
1457 memset(&tpd, 0, sizeof(tpd));
1458
1459 if (emac_tso_csum(adpt, tx_q, skb, &tpd) != 0) {
1460 dev_kfree_skb_any(skb);
1461 return NETDEV_TX_OK;
1462 }
1463
1464 if (skb_vlan_tag_present(skb)) {
1465 u16 tag;
1466
1467 EMAC_VLAN_TO_TAG(skb_vlan_tag_get(skb), tag);
1468 TPD_CVLAN_TAG_SET(&tpd, tag);
1469 TPD_INSTC_SET(&tpd, 1);
1470 }
1471
1472 if (skb_network_offset(skb) != ETH_HLEN)
1473 TPD_TYP_SET(&tpd, 1);
1474
1475 emac_tx_fill_tpd(adpt, tx_q, skb, &tpd);
1476
1477 netdev_sent_queue(adpt->netdev, skb->len);
1478
1479
1480
1481
1482
1483
1484 if (emac_tpd_num_free_descs(tx_q) < (MAX_SKB_FRAGS + 3))
1485 netif_stop_queue(adpt->netdev);
1486
1487
1488 prod_idx = (tx_q->tpd.produce_idx << tx_q->produce_shift) &
1489 tx_q->produce_mask;
1490 emac_reg_update32(adpt->base + tx_q->produce_reg,
1491 tx_q->produce_mask, prod_idx);
1492
1493 return NETDEV_TX_OK;
1494}
1495
