1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/kernel.h>
17#include <linux/types.h>
18#include <linux/errno.h>
19#include <linux/pci.h>
20#include <linux/device.h>
21#include <linux/netdevice.h>
22#include <linux/etherdevice.h>
23#include <linux/u64_stats_sync.h>
24#include <linux/slab.h>
25#include <linux/interrupt.h>
26#include <linux/skbuff.h>
27#include <linux/dma-mapping.h>
28#include <linux/prefetch.h>
29#include <linux/cpumask.h>
30#include <asm/barrier.h>
31
32#include "hinic_common.h"
33#include "hinic_hw_if.h"
34#include "hinic_hw_wqe.h"
35#include "hinic_hw_wq.h"
36#include "hinic_hw_qp.h"
37#include "hinic_hw_dev.h"
38#include "hinic_rx.h"
39#include "hinic_dev.h"
40
41#define RX_IRQ_NO_PENDING 0
42#define RX_IRQ_NO_COALESC 0
43#define RX_IRQ_NO_LLI_TIMER 0
44#define RX_IRQ_NO_CREDIT 0
45#define RX_IRQ_NO_RESEND_TIMER 0
46
47
48
49
50
51void hinic_rxq_clean_stats(struct hinic_rxq *rxq)
52{
53 struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
54
55 u64_stats_update_begin(&rxq_stats->syncp);
56 rxq_stats->pkts = 0;
57 rxq_stats->bytes = 0;
58 u64_stats_update_end(&rxq_stats->syncp);
59}
60
61
62
63
64
65
66void hinic_rxq_get_stats(struct hinic_rxq *rxq, struct hinic_rxq_stats *stats)
67{
68 struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
69 unsigned int start;
70
71 u64_stats_update_begin(&stats->syncp);
72 do {
73 start = u64_stats_fetch_begin(&rxq_stats->syncp);
74 stats->pkts = rxq_stats->pkts;
75 stats->bytes = rxq_stats->bytes;
76 } while (u64_stats_fetch_retry(&rxq_stats->syncp, start));
77 u64_stats_update_end(&stats->syncp);
78}
79
80
81
82
83
84static void rxq_stats_init(struct hinic_rxq *rxq)
85{
86 struct hinic_rxq_stats *rxq_stats = &rxq->rxq_stats;
87
88 u64_stats_init(&rxq_stats->syncp);
89 hinic_rxq_clean_stats(rxq);
90}
91
92
93
94
95
96
97
98
99static struct sk_buff *rx_alloc_skb(struct hinic_rxq *rxq,
100 dma_addr_t *dma_addr)
101{
102 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
103 struct hinic_hwdev *hwdev = nic_dev->hwdev;
104 struct hinic_hwif *hwif = hwdev->hwif;
105 struct pci_dev *pdev = hwif->pdev;
106 struct sk_buff *skb;
107 dma_addr_t addr;
108 int err;
109
110 skb = netdev_alloc_skb_ip_align(rxq->netdev, rxq->rq->buf_sz);
111 if (!skb) {
112 netdev_err(rxq->netdev, "Failed to allocate Rx SKB\n");
113 return NULL;
114 }
115
116 addr = dma_map_single(&pdev->dev, skb->data, rxq->rq->buf_sz,
117 DMA_FROM_DEVICE);
118 err = dma_mapping_error(&pdev->dev, addr);
119 if (err) {
120 dev_err(&pdev->dev, "Failed to map Rx DMA, err = %d\n", err);
121 goto err_rx_map;
122 }
123
124 *dma_addr = addr;
125 return skb;
126
127err_rx_map:
128 dev_kfree_skb_any(skb);
129 return NULL;
130}
131
132
133
134
135
136
137static void rx_unmap_skb(struct hinic_rxq *rxq, dma_addr_t dma_addr)
138{
139 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
140 struct hinic_hwdev *hwdev = nic_dev->hwdev;
141 struct hinic_hwif *hwif = hwdev->hwif;
142 struct pci_dev *pdev = hwif->pdev;
143
144 dma_unmap_single(&pdev->dev, dma_addr, rxq->rq->buf_sz,
145 DMA_FROM_DEVICE);
146}
147
148
149
150
151
152
153
154static void rx_free_skb(struct hinic_rxq *rxq, struct sk_buff *skb,
155 dma_addr_t dma_addr)
156{
157 rx_unmap_skb(rxq, dma_addr);
158 dev_kfree_skb_any(skb);
159}
160
161
162
163
164
165
166
167static int rx_alloc_pkts(struct hinic_rxq *rxq)
168{
169 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
170 struct hinic_rq_wqe *rq_wqe;
171 unsigned int free_wqebbs;
172 struct hinic_sge sge;
173 dma_addr_t dma_addr;
174 struct sk_buff *skb;
175 u16 prod_idx;
176 int i;
177
178 free_wqebbs = hinic_get_rq_free_wqebbs(rxq->rq);
179
180
181 if (free_wqebbs > nic_dev->rx_weight)
182 free_wqebbs = nic_dev->rx_weight;
183
184 for (i = 0; i < free_wqebbs; i++) {
185 skb = rx_alloc_skb(rxq, &dma_addr);
186 if (!skb) {
187 netdev_err(rxq->netdev, "Failed to alloc Rx skb\n");
188 goto skb_out;
189 }
190
191 hinic_set_sge(&sge, dma_addr, skb->len);
192
193 rq_wqe = hinic_rq_get_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,
194 &prod_idx);
195 if (!rq_wqe) {
196 rx_free_skb(rxq, skb, dma_addr);
197 goto skb_out;
198 }
199
200 hinic_rq_prepare_wqe(rxq->rq, prod_idx, rq_wqe, &sge);
201
202 hinic_rq_write_wqe(rxq->rq, prod_idx, rq_wqe, skb);
203 }
204
205skb_out:
206 if (i) {
207 wmb();
208
209 hinic_rq_update(rxq->rq, prod_idx);
210 }
211
212 tasklet_schedule(&rxq->rx_task);
213 return i;
214}
215
216
217
218
219
220static void free_all_rx_skbs(struct hinic_rxq *rxq)
221{
222 struct hinic_rq *rq = rxq->rq;
223 struct hinic_hw_wqe *hw_wqe;
224 struct hinic_sge sge;
225 u16 ci;
226
227 while ((hw_wqe = hinic_read_wqe(rq->wq, HINIC_RQ_WQE_SIZE, &ci))) {
228 if (IS_ERR(hw_wqe))
229 break;
230
231 hinic_rq_get_sge(rq, &hw_wqe->rq_wqe, ci, &sge);
232
233 hinic_put_wqe(rq->wq, HINIC_RQ_WQE_SIZE);
234
235 rx_free_skb(rxq, rq->saved_skb[ci], hinic_sge_to_dma(&sge));
236 }
237}
238
239
240
241
242
243static void rx_alloc_task(unsigned long data)
244{
245 struct hinic_rxq *rxq = (struct hinic_rxq *)data;
246
247 (void)rx_alloc_pkts(rxq);
248}
249
250
251
252
253
254
255
256
257
258
259static int rx_recv_jumbo_pkt(struct hinic_rxq *rxq, struct sk_buff *head_skb,
260 unsigned int left_pkt_len, u16 ci)
261{
262 struct sk_buff *skb, *curr_skb = head_skb;
263 struct hinic_rq_wqe *rq_wqe;
264 unsigned int curr_len;
265 struct hinic_sge sge;
266 int num_wqes = 0;
267
268 while (left_pkt_len > 0) {
269 rq_wqe = hinic_rq_read_next_wqe(rxq->rq, HINIC_RQ_WQE_SIZE,
270 &skb, &ci);
271
272 num_wqes++;
273
274 hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);
275
276 rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));
277
278 prefetch(skb->data);
279
280 curr_len = (left_pkt_len > HINIC_RX_BUF_SZ) ? HINIC_RX_BUF_SZ :
281 left_pkt_len;
282
283 left_pkt_len -= curr_len;
284
285 __skb_put(skb, curr_len);
286
287 if (curr_skb == head_skb)
288 skb_shinfo(head_skb)->frag_list = skb;
289 else
290 curr_skb->next = skb;
291
292 head_skb->len += skb->len;
293 head_skb->data_len += skb->len;
294 head_skb->truesize += skb->truesize;
295
296 curr_skb = skb;
297 }
298
299 return num_wqes;
300}
301
302
303
304
305
306
307
308
309static int rxq_recv(struct hinic_rxq *rxq, int budget)
310{
311 struct hinic_qp *qp = container_of(rxq->rq, struct hinic_qp, rq);
312 u64 pkt_len = 0, rx_bytes = 0;
313 struct hinic_rq_wqe *rq_wqe;
314 int num_wqes, pkts = 0;
315 struct hinic_sge sge;
316 struct sk_buff *skb;
317 u16 ci;
318
319 while (pkts < budget) {
320 num_wqes = 0;
321
322 rq_wqe = hinic_rq_read_wqe(rxq->rq, HINIC_RQ_WQE_SIZE, &skb,
323 &ci);
324 if (!rq_wqe)
325 break;
326
327 hinic_rq_get_sge(rxq->rq, rq_wqe, ci, &sge);
328
329 rx_unmap_skb(rxq, hinic_sge_to_dma(&sge));
330
331 prefetch(skb->data);
332
333 pkt_len = sge.len;
334
335 if (pkt_len <= HINIC_RX_BUF_SZ) {
336 __skb_put(skb, pkt_len);
337 } else {
338 __skb_put(skb, HINIC_RX_BUF_SZ);
339 num_wqes = rx_recv_jumbo_pkt(rxq, skb, pkt_len -
340 HINIC_RX_BUF_SZ, ci);
341 }
342
343 hinic_rq_put_wqe(rxq->rq, ci,
344 (num_wqes + 1) * HINIC_RQ_WQE_SIZE);
345
346 skb_record_rx_queue(skb, qp->q_id);
347 skb->protocol = eth_type_trans(skb, rxq->netdev);
348
349 napi_gro_receive(&rxq->napi, skb);
350
351 pkts++;
352 rx_bytes += pkt_len;
353 }
354
355 if (pkts)
356 tasklet_schedule(&rxq->rx_task);
357
358 u64_stats_update_begin(&rxq->rxq_stats.syncp);
359 rxq->rxq_stats.pkts += pkts;
360 rxq->rxq_stats.bytes += rx_bytes;
361 u64_stats_update_end(&rxq->rxq_stats.syncp);
362
363 return pkts;
364}
365
366static int rx_poll(struct napi_struct *napi, int budget)
367{
368 struct hinic_rxq *rxq = container_of(napi, struct hinic_rxq, napi);
369 struct hinic_rq *rq = rxq->rq;
370 int pkts;
371
372 pkts = rxq_recv(rxq, budget);
373 if (pkts >= budget)
374 return budget;
375
376 napi_complete(napi);
377 enable_irq(rq->irq);
378 return pkts;
379}
380
381static void rx_add_napi(struct hinic_rxq *rxq)
382{
383 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
384
385 netif_napi_add(rxq->netdev, &rxq->napi, rx_poll, nic_dev->rx_weight);
386 napi_enable(&rxq->napi);
387}
388
389static void rx_del_napi(struct hinic_rxq *rxq)
390{
391 napi_disable(&rxq->napi);
392 netif_napi_del(&rxq->napi);
393}
394
395static irqreturn_t rx_irq(int irq, void *data)
396{
397 struct hinic_rxq *rxq = (struct hinic_rxq *)data;
398 struct hinic_rq *rq = rxq->rq;
399 struct hinic_dev *nic_dev;
400
401
402 disable_irq_nosync(rq->irq);
403
404 nic_dev = netdev_priv(rxq->netdev);
405 hinic_hwdev_msix_cnt_set(nic_dev->hwdev, rq->msix_entry);
406
407 napi_schedule(&rxq->napi);
408 return IRQ_HANDLED;
409}
410
411static int rx_request_irq(struct hinic_rxq *rxq)
412{
413 struct hinic_dev *nic_dev = netdev_priv(rxq->netdev);
414 struct hinic_hwdev *hwdev = nic_dev->hwdev;
415 struct hinic_rq *rq = rxq->rq;
416 struct hinic_qp *qp;
417 struct cpumask mask;
418 int err;
419
420 rx_add_napi(rxq);
421
422 hinic_hwdev_msix_set(hwdev, rq->msix_entry,
423 RX_IRQ_NO_PENDING, RX_IRQ_NO_COALESC,
424 RX_IRQ_NO_LLI_TIMER, RX_IRQ_NO_CREDIT,
425 RX_IRQ_NO_RESEND_TIMER);
426
427 err = request_irq(rq->irq, rx_irq, 0, rxq->irq_name, rxq);
428 if (err) {
429 rx_del_napi(rxq);
430 return err;
431 }
432
433 qp = container_of(rq, struct hinic_qp, rq);
434 cpumask_set_cpu(qp->q_id % num_online_cpus(), &mask);
435 return irq_set_affinity_hint(rq->irq, &mask);
436}
437
438static void rx_free_irq(struct hinic_rxq *rxq)
439{
440 struct hinic_rq *rq = rxq->rq;
441
442 irq_set_affinity_hint(rq->irq, NULL);
443 free_irq(rq->irq, rxq);
444 rx_del_napi(rxq);
445}
446
447
448
449
450
451
452
453
454
455int hinic_init_rxq(struct hinic_rxq *rxq, struct hinic_rq *rq,
456 struct net_device *netdev)
457{
458 struct hinic_qp *qp = container_of(rq, struct hinic_qp, rq);
459 int err, pkts, irqname_len;
460
461 rxq->netdev = netdev;
462 rxq->rq = rq;
463
464 rxq_stats_init(rxq);
465
466 irqname_len = snprintf(NULL, 0, "hinic_rxq%d", qp->q_id) + 1;
467 rxq->irq_name = devm_kzalloc(&netdev->dev, irqname_len, GFP_KERNEL);
468 if (!rxq->irq_name)
469 return -ENOMEM;
470
471 sprintf(rxq->irq_name, "hinic_rxq%d", qp->q_id);
472
473 tasklet_init(&rxq->rx_task, rx_alloc_task, (unsigned long)rxq);
474
475 pkts = rx_alloc_pkts(rxq);
476 if (!pkts) {
477 err = -ENOMEM;
478 goto err_rx_pkts;
479 }
480
481 err = rx_request_irq(rxq);
482 if (err) {
483 netdev_err(netdev, "Failed to request Rx irq\n");
484 goto err_req_rx_irq;
485 }
486
487 return 0;
488
489err_req_rx_irq:
490err_rx_pkts:
491 tasklet_kill(&rxq->rx_task);
492 free_all_rx_skbs(rxq);
493 devm_kfree(&netdev->dev, rxq->irq_name);
494 return err;
495}
496
497
498
499
500
501void hinic_clean_rxq(struct hinic_rxq *rxq)
502{
503 struct net_device *netdev = rxq->netdev;
504
505 rx_free_irq(rxq);
506
507 tasklet_kill(&rxq->rx_task);
508 free_all_rx_skbs(rxq);
509 devm_kfree(&netdev->dev, rxq->irq_name);
510}
511