1
2
3
4#include <linux/etherdevice.h>
5#include <linux/of_net.h>
6#include <linux/pci.h>
7#include <linux/bpf.h>
8
9
10#include "i40e.h"
11#include "i40e_diag.h"
12#include "i40e_xsk.h"
13#include <net/udp_tunnel.h>
14#include <net/xdp_sock.h>
15
16
17
18
19#define CREATE_TRACE_POINTS
20#include "i40e_trace.h"
21
22const char i40e_driver_name[] = "i40e";
23static const char i40e_driver_string[] =
24 "Intel(R) Ethernet Connection XL710 Network Driver";
25
26#define DRV_KERN "-k"
27
28#define DRV_VERSION_MAJOR 2
29#define DRV_VERSION_MINOR 8
30#define DRV_VERSION_BUILD 20
31#define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \
32 __stringify(DRV_VERSION_MINOR) "." \
33 __stringify(DRV_VERSION_BUILD) DRV_KERN
34const char i40e_driver_version_str[] = DRV_VERSION;
35static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation.";
36
37
38static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi);
39static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired);
40static int i40e_add_vsi(struct i40e_vsi *vsi);
41static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi);
42static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit);
43static int i40e_setup_misc_vector(struct i40e_pf *pf);
44static void i40e_determine_queue_usage(struct i40e_pf *pf);
45static int i40e_setup_pf_filter_control(struct i40e_pf *pf);
46static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired);
47static int i40e_reset(struct i40e_pf *pf);
48static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired);
49static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf);
50static int i40e_restore_interrupt_scheme(struct i40e_pf *pf);
51static bool i40e_check_recovery_mode(struct i40e_pf *pf);
52static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw);
53static void i40e_fdir_sb_setup(struct i40e_pf *pf);
54static int i40e_veb_get_bw_info(struct i40e_veb *veb);
55static int i40e_get_capabilities(struct i40e_pf *pf,
56 enum i40e_admin_queue_opc list_type);
57
58
59
60
61
62
63
64
65
66static const struct pci_device_id i40e_pci_tbl[] = {
67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0},
68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0},
69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0},
70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0},
71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0},
72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0},
73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0},
74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0},
75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0},
76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0},
77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0},
78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0},
79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0},
80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0},
81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0},
82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0},
83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0},
84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0},
85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0},
86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0},
87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0},
88 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0},
89 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0},
90
91 {0, }
92};
93MODULE_DEVICE_TABLE(pci, i40e_pci_tbl);
94
95#define I40E_MAX_VF_COUNT 128
96static int debug = -1;
97module_param(debug, uint, 0);
98MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)");
99
100MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
101MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver");
102MODULE_LICENSE("GPL v2");
103MODULE_VERSION(DRV_VERSION);
104
105static struct workqueue_struct *i40e_wq;
106
107
108
109
110
111
112
113
114int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem,
115 u64 size, u32 alignment)
116{
117 struct i40e_pf *pf = (struct i40e_pf *)hw->back;
118
119 mem->size = ALIGN(size, alignment);
120 mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa,
121 GFP_KERNEL);
122 if (!mem->va)
123 return -ENOMEM;
124
125 return 0;
126}
127
128
129
130
131
132
133int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem)
134{
135 struct i40e_pf *pf = (struct i40e_pf *)hw->back;
136
137 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa);
138 mem->va = NULL;
139 mem->pa = 0;
140 mem->size = 0;
141
142 return 0;
143}
144
145
146
147
148
149
150
151int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem,
152 u32 size)
153{
154 mem->size = size;
155 mem->va = kzalloc(size, GFP_KERNEL);
156
157 if (!mem->va)
158 return -ENOMEM;
159
160 return 0;
161}
162
163
164
165
166
167
168int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem)
169{
170
171 kfree(mem->va);
172 mem->va = NULL;
173 mem->size = 0;
174
175 return 0;
176}
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile,
192 u16 needed, u16 id)
193{
194 int ret = -ENOMEM;
195 int i, j;
196
197 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) {
198 dev_info(&pf->pdev->dev,
199 "param err: pile=%s needed=%d id=0x%04x\n",
200 pile ? "<valid>" : "<null>", needed, id);
201 return -EINVAL;
202 }
203
204
205 i = pile->search_hint;
206 while (i < pile->num_entries) {
207
208 if (pile->list[i] & I40E_PILE_VALID_BIT) {
209 i++;
210 continue;
211 }
212
213
214 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) {
215 if (pile->list[i+j] & I40E_PILE_VALID_BIT)
216 break;
217 }
218
219 if (j == needed) {
220
221 for (j = 0; j < needed; j++)
222 pile->list[i+j] = id | I40E_PILE_VALID_BIT;
223 ret = i;
224 pile->search_hint = i + j;
225 break;
226 }
227
228
229 i += j;
230 }
231
232 return ret;
233}
234
235
236
237
238
239
240
241
242
243static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id)
244{
245 int valid_id = (id | I40E_PILE_VALID_BIT);
246 int count = 0;
247 int i;
248
249 if (!pile || index >= pile->num_entries)
250 return -EINVAL;
251
252 for (i = index;
253 i < pile->num_entries && pile->list[i] == valid_id;
254 i++) {
255 pile->list[i] = 0;
256 count++;
257 }
258
259 if (count && index < pile->search_hint)
260 pile->search_hint = index;
261
262 return count;
263}
264
265
266
267
268
269
270struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id)
271{
272 int i;
273
274 for (i = 0; i < pf->num_alloc_vsi; i++)
275 if (pf->vsi[i] && (pf->vsi[i]->id == id))
276 return pf->vsi[i];
277
278 return NULL;
279}
280
281
282
283
284
285
286
287void i40e_service_event_schedule(struct i40e_pf *pf)
288{
289 if ((!test_bit(__I40E_DOWN, pf->state) &&
290 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) ||
291 test_bit(__I40E_RECOVERY_MODE, pf->state))
292 queue_work(i40e_wq, &pf->service_task);
293}
294
295
296
297
298
299
300
301
302
303static void i40e_tx_timeout(struct net_device *netdev)
304{
305 struct i40e_netdev_priv *np = netdev_priv(netdev);
306 struct i40e_vsi *vsi = np->vsi;
307 struct i40e_pf *pf = vsi->back;
308 struct i40e_ring *tx_ring = NULL;
309 unsigned int i, hung_queue = 0;
310 u32 head, val;
311
312 pf->tx_timeout_count++;
313
314
315 for (i = 0; i < netdev->num_tx_queues; i++) {
316 struct netdev_queue *q;
317 unsigned long trans_start;
318
319 q = netdev_get_tx_queue(netdev, i);
320 trans_start = q->trans_start;
321 if (netif_xmit_stopped(q) &&
322 time_after(jiffies,
323 (trans_start + netdev->watchdog_timeo))) {
324 hung_queue = i;
325 break;
326 }
327 }
328
329 if (i == netdev->num_tx_queues) {
330 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n");
331 } else {
332
333 for (i = 0; i < vsi->num_queue_pairs; i++) {
334 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) {
335 if (hung_queue ==
336 vsi->tx_rings[i]->queue_index) {
337 tx_ring = vsi->tx_rings[i];
338 break;
339 }
340 }
341 }
342 }
343
344 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20)))
345 pf->tx_timeout_recovery_level = 1;
346 else if (time_before(jiffies,
347 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo)))
348 return;
349
350
351 if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state))
352 return;
353
354 if (tx_ring) {
355 head = i40e_get_head(tx_ring);
356
357 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
358 val = rd32(&pf->hw,
359 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx +
360 tx_ring->vsi->base_vector - 1));
361 else
362 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0);
363
364 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n",
365 vsi->seid, hung_queue, tx_ring->next_to_clean,
366 head, tx_ring->next_to_use,
367 readl(tx_ring->tail), val);
368 }
369
370 pf->tx_timeout_last_recovery = jiffies;
371 netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n",
372 pf->tx_timeout_recovery_level, hung_queue);
373
374 switch (pf->tx_timeout_recovery_level) {
375 case 1:
376 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
377 break;
378 case 2:
379 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
380 break;
381 case 3:
382 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
383 break;
384 default:
385 netdev_err(netdev, "tx_timeout recovery unsuccessful\n");
386 break;
387 }
388
389 i40e_service_event_schedule(pf);
390 pf->tx_timeout_recovery_level++;
391}
392
393
394
395
396
397
398
399
400struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi)
401{
402 return &vsi->net_stats;
403}
404
405
406
407
408
409
410static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring,
411 struct rtnl_link_stats64 *stats)
412{
413 u64 bytes, packets;
414 unsigned int start;
415
416 do {
417 start = u64_stats_fetch_begin_irq(&ring->syncp);
418 packets = ring->stats.packets;
419 bytes = ring->stats.bytes;
420 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
421
422 stats->tx_packets += packets;
423 stats->tx_bytes += bytes;
424}
425
426
427
428
429
430
431
432
433
434static void i40e_get_netdev_stats_struct(struct net_device *netdev,
435 struct rtnl_link_stats64 *stats)
436{
437 struct i40e_netdev_priv *np = netdev_priv(netdev);
438 struct i40e_vsi *vsi = np->vsi;
439 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi);
440 struct i40e_ring *ring;
441 int i;
442
443 if (test_bit(__I40E_VSI_DOWN, vsi->state))
444 return;
445
446 if (!vsi->tx_rings)
447 return;
448
449 rcu_read_lock();
450 for (i = 0; i < vsi->num_queue_pairs; i++) {
451 u64 bytes, packets;
452 unsigned int start;
453
454 ring = READ_ONCE(vsi->tx_rings[i]);
455 if (!ring)
456 continue;
457 i40e_get_netdev_stats_struct_tx(ring, stats);
458
459 if (i40e_enabled_xdp_vsi(vsi)) {
460 ring++;
461 i40e_get_netdev_stats_struct_tx(ring, stats);
462 }
463
464 ring++;
465 do {
466 start = u64_stats_fetch_begin_irq(&ring->syncp);
467 packets = ring->stats.packets;
468 bytes = ring->stats.bytes;
469 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
470
471 stats->rx_packets += packets;
472 stats->rx_bytes += bytes;
473
474 }
475 rcu_read_unlock();
476
477
478 stats->multicast = vsi_stats->multicast;
479 stats->tx_errors = vsi_stats->tx_errors;
480 stats->tx_dropped = vsi_stats->tx_dropped;
481 stats->rx_errors = vsi_stats->rx_errors;
482 stats->rx_dropped = vsi_stats->rx_dropped;
483 stats->rx_crc_errors = vsi_stats->rx_crc_errors;
484 stats->rx_length_errors = vsi_stats->rx_length_errors;
485}
486
487
488
489
490
491void i40e_vsi_reset_stats(struct i40e_vsi *vsi)
492{
493 struct rtnl_link_stats64 *ns;
494 int i;
495
496 if (!vsi)
497 return;
498
499 ns = i40e_get_vsi_stats_struct(vsi);
500 memset(ns, 0, sizeof(*ns));
501 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets));
502 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats));
503 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets));
504 if (vsi->rx_rings && vsi->rx_rings[0]) {
505 for (i = 0; i < vsi->num_queue_pairs; i++) {
506 memset(&vsi->rx_rings[i]->stats, 0,
507 sizeof(vsi->rx_rings[i]->stats));
508 memset(&vsi->rx_rings[i]->rx_stats, 0,
509 sizeof(vsi->rx_rings[i]->rx_stats));
510 memset(&vsi->tx_rings[i]->stats, 0,
511 sizeof(vsi->tx_rings[i]->stats));
512 memset(&vsi->tx_rings[i]->tx_stats, 0,
513 sizeof(vsi->tx_rings[i]->tx_stats));
514 }
515 }
516 vsi->stat_offsets_loaded = false;
517}
518
519
520
521
522
523void i40e_pf_reset_stats(struct i40e_pf *pf)
524{
525 int i;
526
527 memset(&pf->stats, 0, sizeof(pf->stats));
528 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets));
529 pf->stat_offsets_loaded = false;
530
531 for (i = 0; i < I40E_MAX_VEB; i++) {
532 if (pf->veb[i]) {
533 memset(&pf->veb[i]->stats, 0,
534 sizeof(pf->veb[i]->stats));
535 memset(&pf->veb[i]->stats_offsets, 0,
536 sizeof(pf->veb[i]->stats_offsets));
537 pf->veb[i]->stat_offsets_loaded = false;
538 }
539 }
540 pf->hw_csum_rx_error = 0;
541}
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg,
559 bool offset_loaded, u64 *offset, u64 *stat)
560{
561 u64 new_data;
562
563 if (hw->device_id == I40E_DEV_ID_QEMU) {
564 new_data = rd32(hw, loreg);
565 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
566 } else {
567 new_data = rd64(hw, loreg);
568 }
569 if (!offset_loaded)
570 *offset = new_data;
571 if (likely(new_data >= *offset))
572 *stat = new_data - *offset;
573 else
574 *stat = (new_data + BIT_ULL(48)) - *offset;
575 *stat &= 0xFFFFFFFFFFFFULL;
576}
577
578
579
580
581
582
583
584
585
586static void i40e_stat_update32(struct i40e_hw *hw, u32 reg,
587 bool offset_loaded, u64 *offset, u64 *stat)
588{
589 u32 new_data;
590
591 new_data = rd32(hw, reg);
592 if (!offset_loaded)
593 *offset = new_data;
594 if (likely(new_data >= *offset))
595 *stat = (u32)(new_data - *offset);
596 else
597 *stat = (u32)((new_data + BIT_ULL(32)) - *offset);
598}
599
600
601
602
603
604
605
606static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat)
607{
608 u32 new_data = rd32(hw, reg);
609
610 wr32(hw, reg, 1);
611 *stat += new_data;
612}
613
614
615
616
617
618void i40e_update_eth_stats(struct i40e_vsi *vsi)
619{
620 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx);
621 struct i40e_pf *pf = vsi->back;
622 struct i40e_hw *hw = &pf->hw;
623 struct i40e_eth_stats *oes;
624 struct i40e_eth_stats *es;
625
626 es = &vsi->eth_stats;
627 oes = &vsi->eth_stats_offsets;
628
629
630 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx),
631 vsi->stat_offsets_loaded,
632 &oes->tx_errors, &es->tx_errors);
633 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx),
634 vsi->stat_offsets_loaded,
635 &oes->rx_discards, &es->rx_discards);
636 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx),
637 vsi->stat_offsets_loaded,
638 &oes->rx_unknown_protocol, &es->rx_unknown_protocol);
639
640 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx),
641 I40E_GLV_GORCL(stat_idx),
642 vsi->stat_offsets_loaded,
643 &oes->rx_bytes, &es->rx_bytes);
644 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx),
645 I40E_GLV_UPRCL(stat_idx),
646 vsi->stat_offsets_loaded,
647 &oes->rx_unicast, &es->rx_unicast);
648 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx),
649 I40E_GLV_MPRCL(stat_idx),
650 vsi->stat_offsets_loaded,
651 &oes->rx_multicast, &es->rx_multicast);
652 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx),
653 I40E_GLV_BPRCL(stat_idx),
654 vsi->stat_offsets_loaded,
655 &oes->rx_broadcast, &es->rx_broadcast);
656
657 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx),
658 I40E_GLV_GOTCL(stat_idx),
659 vsi->stat_offsets_loaded,
660 &oes->tx_bytes, &es->tx_bytes);
661 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx),
662 I40E_GLV_UPTCL(stat_idx),
663 vsi->stat_offsets_loaded,
664 &oes->tx_unicast, &es->tx_unicast);
665 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx),
666 I40E_GLV_MPTCL(stat_idx),
667 vsi->stat_offsets_loaded,
668 &oes->tx_multicast, &es->tx_multicast);
669 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx),
670 I40E_GLV_BPTCL(stat_idx),
671 vsi->stat_offsets_loaded,
672 &oes->tx_broadcast, &es->tx_broadcast);
673 vsi->stat_offsets_loaded = true;
674}
675
676
677
678
679
680static void i40e_update_veb_stats(struct i40e_veb *veb)
681{
682 struct i40e_pf *pf = veb->pf;
683 struct i40e_hw *hw = &pf->hw;
684 struct i40e_eth_stats *oes;
685 struct i40e_eth_stats *es;
686 struct i40e_veb_tc_stats *veb_oes;
687 struct i40e_veb_tc_stats *veb_es;
688 int i, idx = 0;
689
690 idx = veb->stats_idx;
691 es = &veb->stats;
692 oes = &veb->stats_offsets;
693 veb_es = &veb->tc_stats;
694 veb_oes = &veb->tc_stats_offsets;
695
696
697 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx),
698 veb->stat_offsets_loaded,
699 &oes->tx_discards, &es->tx_discards);
700 if (hw->revision_id > 0)
701 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx),
702 veb->stat_offsets_loaded,
703 &oes->rx_unknown_protocol,
704 &es->rx_unknown_protocol);
705 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx),
706 veb->stat_offsets_loaded,
707 &oes->rx_bytes, &es->rx_bytes);
708 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx),
709 veb->stat_offsets_loaded,
710 &oes->rx_unicast, &es->rx_unicast);
711 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx),
712 veb->stat_offsets_loaded,
713 &oes->rx_multicast, &es->rx_multicast);
714 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx),
715 veb->stat_offsets_loaded,
716 &oes->rx_broadcast, &es->rx_broadcast);
717
718 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx),
719 veb->stat_offsets_loaded,
720 &oes->tx_bytes, &es->tx_bytes);
721 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx),
722 veb->stat_offsets_loaded,
723 &oes->tx_unicast, &es->tx_unicast);
724 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx),
725 veb->stat_offsets_loaded,
726 &oes->tx_multicast, &es->tx_multicast);
727 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx),
728 veb->stat_offsets_loaded,
729 &oes->tx_broadcast, &es->tx_broadcast);
730 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
731 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx),
732 I40E_GLVEBTC_RPCL(i, idx),
733 veb->stat_offsets_loaded,
734 &veb_oes->tc_rx_packets[i],
735 &veb_es->tc_rx_packets[i]);
736 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx),
737 I40E_GLVEBTC_RBCL(i, idx),
738 veb->stat_offsets_loaded,
739 &veb_oes->tc_rx_bytes[i],
740 &veb_es->tc_rx_bytes[i]);
741 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx),
742 I40E_GLVEBTC_TPCL(i, idx),
743 veb->stat_offsets_loaded,
744 &veb_oes->tc_tx_packets[i],
745 &veb_es->tc_tx_packets[i]);
746 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx),
747 I40E_GLVEBTC_TBCL(i, idx),
748 veb->stat_offsets_loaded,
749 &veb_oes->tc_tx_bytes[i],
750 &veb_es->tc_tx_bytes[i]);
751 }
752 veb->stat_offsets_loaded = true;
753}
754
755
756
757
758
759
760
761
762
763
764
765static void i40e_update_vsi_stats(struct i40e_vsi *vsi)
766{
767 struct i40e_pf *pf = vsi->back;
768 struct rtnl_link_stats64 *ons;
769 struct rtnl_link_stats64 *ns;
770 struct i40e_eth_stats *oes;
771 struct i40e_eth_stats *es;
772 u32 tx_restart, tx_busy;
773 struct i40e_ring *p;
774 u32 rx_page, rx_buf;
775 u64 bytes, packets;
776 unsigned int start;
777 u64 tx_linearize;
778 u64 tx_force_wb;
779 u64 rx_p, rx_b;
780 u64 tx_p, tx_b;
781 u16 q;
782
783 if (test_bit(__I40E_VSI_DOWN, vsi->state) ||
784 test_bit(__I40E_CONFIG_BUSY, pf->state))
785 return;
786
787 ns = i40e_get_vsi_stats_struct(vsi);
788 ons = &vsi->net_stats_offsets;
789 es = &vsi->eth_stats;
790 oes = &vsi->eth_stats_offsets;
791
792
793
794
795 rx_b = rx_p = 0;
796 tx_b = tx_p = 0;
797 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0;
798 rx_page = 0;
799 rx_buf = 0;
800 rcu_read_lock();
801 for (q = 0; q < vsi->num_queue_pairs; q++) {
802
803 p = READ_ONCE(vsi->tx_rings[q]);
804
805 do {
806 start = u64_stats_fetch_begin_irq(&p->syncp);
807 packets = p->stats.packets;
808 bytes = p->stats.bytes;
809 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
810 tx_b += bytes;
811 tx_p += packets;
812 tx_restart += p->tx_stats.restart_queue;
813 tx_busy += p->tx_stats.tx_busy;
814 tx_linearize += p->tx_stats.tx_linearize;
815 tx_force_wb += p->tx_stats.tx_force_wb;
816
817
818 p = &p[1];
819 do {
820 start = u64_stats_fetch_begin_irq(&p->syncp);
821 packets = p->stats.packets;
822 bytes = p->stats.bytes;
823 } while (u64_stats_fetch_retry_irq(&p->syncp, start));
824 rx_b += bytes;
825 rx_p += packets;
826 rx_buf += p->rx_stats.alloc_buff_failed;
827 rx_page += p->rx_stats.alloc_page_failed;
828 }
829 rcu_read_unlock();
830 vsi->tx_restart = tx_restart;
831 vsi->tx_busy = tx_busy;
832 vsi->tx_linearize = tx_linearize;
833 vsi->tx_force_wb = tx_force_wb;
834 vsi->rx_page_failed = rx_page;
835 vsi->rx_buf_failed = rx_buf;
836
837 ns->rx_packets = rx_p;
838 ns->rx_bytes = rx_b;
839 ns->tx_packets = tx_p;
840 ns->tx_bytes = tx_b;
841
842
843 i40e_update_eth_stats(vsi);
844 ons->tx_errors = oes->tx_errors;
845 ns->tx_errors = es->tx_errors;
846 ons->multicast = oes->rx_multicast;
847 ns->multicast = es->rx_multicast;
848 ons->rx_dropped = oes->rx_discards;
849 ns->rx_dropped = es->rx_discards;
850 ons->tx_dropped = oes->tx_discards;
851 ns->tx_dropped = es->tx_discards;
852
853
854 if (vsi == pf->vsi[pf->lan_vsi]) {
855 ns->rx_crc_errors = pf->stats.crc_errors;
856 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes;
857 ns->rx_length_errors = pf->stats.rx_length_errors;
858 }
859}
860
861
862
863
864
865static void i40e_update_pf_stats(struct i40e_pf *pf)
866{
867 struct i40e_hw_port_stats *osd = &pf->stats_offsets;
868 struct i40e_hw_port_stats *nsd = &pf->stats;
869 struct i40e_hw *hw = &pf->hw;
870 u32 val;
871 int i;
872
873 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port),
874 I40E_GLPRT_GORCL(hw->port),
875 pf->stat_offsets_loaded,
876 &osd->eth.rx_bytes, &nsd->eth.rx_bytes);
877 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port),
878 I40E_GLPRT_GOTCL(hw->port),
879 pf->stat_offsets_loaded,
880 &osd->eth.tx_bytes, &nsd->eth.tx_bytes);
881 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port),
882 pf->stat_offsets_loaded,
883 &osd->eth.rx_discards,
884 &nsd->eth.rx_discards);
885 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port),
886 I40E_GLPRT_UPRCL(hw->port),
887 pf->stat_offsets_loaded,
888 &osd->eth.rx_unicast,
889 &nsd->eth.rx_unicast);
890 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port),
891 I40E_GLPRT_MPRCL(hw->port),
892 pf->stat_offsets_loaded,
893 &osd->eth.rx_multicast,
894 &nsd->eth.rx_multicast);
895 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port),
896 I40E_GLPRT_BPRCL(hw->port),
897 pf->stat_offsets_loaded,
898 &osd->eth.rx_broadcast,
899 &nsd->eth.rx_broadcast);
900 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port),
901 I40E_GLPRT_UPTCL(hw->port),
902 pf->stat_offsets_loaded,
903 &osd->eth.tx_unicast,
904 &nsd->eth.tx_unicast);
905 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port),
906 I40E_GLPRT_MPTCL(hw->port),
907 pf->stat_offsets_loaded,
908 &osd->eth.tx_multicast,
909 &nsd->eth.tx_multicast);
910 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port),
911 I40E_GLPRT_BPTCL(hw->port),
912 pf->stat_offsets_loaded,
913 &osd->eth.tx_broadcast,
914 &nsd->eth.tx_broadcast);
915
916 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port),
917 pf->stat_offsets_loaded,
918 &osd->tx_dropped_link_down,
919 &nsd->tx_dropped_link_down);
920
921 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port),
922 pf->stat_offsets_loaded,
923 &osd->crc_errors, &nsd->crc_errors);
924
925 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port),
926 pf->stat_offsets_loaded,
927 &osd->illegal_bytes, &nsd->illegal_bytes);
928
929 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port),
930 pf->stat_offsets_loaded,
931 &osd->mac_local_faults,
932 &nsd->mac_local_faults);
933 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port),
934 pf->stat_offsets_loaded,
935 &osd->mac_remote_faults,
936 &nsd->mac_remote_faults);
937
938 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port),
939 pf->stat_offsets_loaded,
940 &osd->rx_length_errors,
941 &nsd->rx_length_errors);
942
943 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port),
944 pf->stat_offsets_loaded,
945 &osd->link_xon_rx, &nsd->link_xon_rx);
946 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port),
947 pf->stat_offsets_loaded,
948 &osd->link_xon_tx, &nsd->link_xon_tx);
949 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port),
950 pf->stat_offsets_loaded,
951 &osd->link_xoff_rx, &nsd->link_xoff_rx);
952 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port),
953 pf->stat_offsets_loaded,
954 &osd->link_xoff_tx, &nsd->link_xoff_tx);
955
956 for (i = 0; i < 8; i++) {
957 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i),
958 pf->stat_offsets_loaded,
959 &osd->priority_xoff_rx[i],
960 &nsd->priority_xoff_rx[i]);
961 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i),
962 pf->stat_offsets_loaded,
963 &osd->priority_xon_rx[i],
964 &nsd->priority_xon_rx[i]);
965 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i),
966 pf->stat_offsets_loaded,
967 &osd->priority_xon_tx[i],
968 &nsd->priority_xon_tx[i]);
969 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i),
970 pf->stat_offsets_loaded,
971 &osd->priority_xoff_tx[i],
972 &nsd->priority_xoff_tx[i]);
973 i40e_stat_update32(hw,
974 I40E_GLPRT_RXON2OFFCNT(hw->port, i),
975 pf->stat_offsets_loaded,
976 &osd->priority_xon_2_xoff[i],
977 &nsd->priority_xon_2_xoff[i]);
978 }
979
980 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port),
981 I40E_GLPRT_PRC64L(hw->port),
982 pf->stat_offsets_loaded,
983 &osd->rx_size_64, &nsd->rx_size_64);
984 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port),
985 I40E_GLPRT_PRC127L(hw->port),
986 pf->stat_offsets_loaded,
987 &osd->rx_size_127, &nsd->rx_size_127);
988 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port),
989 I40E_GLPRT_PRC255L(hw->port),
990 pf->stat_offsets_loaded,
991 &osd->rx_size_255, &nsd->rx_size_255);
992 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port),
993 I40E_GLPRT_PRC511L(hw->port),
994 pf->stat_offsets_loaded,
995 &osd->rx_size_511, &nsd->rx_size_511);
996 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port),
997 I40E_GLPRT_PRC1023L(hw->port),
998 pf->stat_offsets_loaded,
999 &osd->rx_size_1023, &nsd->rx_size_1023);
1000 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port),
1001 I40E_GLPRT_PRC1522L(hw->port),
1002 pf->stat_offsets_loaded,
1003 &osd->rx_size_1522, &nsd->rx_size_1522);
1004 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port),
1005 I40E_GLPRT_PRC9522L(hw->port),
1006 pf->stat_offsets_loaded,
1007 &osd->rx_size_big, &nsd->rx_size_big);
1008
1009 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port),
1010 I40E_GLPRT_PTC64L(hw->port),
1011 pf->stat_offsets_loaded,
1012 &osd->tx_size_64, &nsd->tx_size_64);
1013 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port),
1014 I40E_GLPRT_PTC127L(hw->port),
1015 pf->stat_offsets_loaded,
1016 &osd->tx_size_127, &nsd->tx_size_127);
1017 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port),
1018 I40E_GLPRT_PTC255L(hw->port),
1019 pf->stat_offsets_loaded,
1020 &osd->tx_size_255, &nsd->tx_size_255);
1021 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port),
1022 I40E_GLPRT_PTC511L(hw->port),
1023 pf->stat_offsets_loaded,
1024 &osd->tx_size_511, &nsd->tx_size_511);
1025 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port),
1026 I40E_GLPRT_PTC1023L(hw->port),
1027 pf->stat_offsets_loaded,
1028 &osd->tx_size_1023, &nsd->tx_size_1023);
1029 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port),
1030 I40E_GLPRT_PTC1522L(hw->port),
1031 pf->stat_offsets_loaded,
1032 &osd->tx_size_1522, &nsd->tx_size_1522);
1033 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port),
1034 I40E_GLPRT_PTC9522L(hw->port),
1035 pf->stat_offsets_loaded,
1036 &osd->tx_size_big, &nsd->tx_size_big);
1037
1038 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port),
1039 pf->stat_offsets_loaded,
1040 &osd->rx_undersize, &nsd->rx_undersize);
1041 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port),
1042 pf->stat_offsets_loaded,
1043 &osd->rx_fragments, &nsd->rx_fragments);
1044 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port),
1045 pf->stat_offsets_loaded,
1046 &osd->rx_oversize, &nsd->rx_oversize);
1047 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port),
1048 pf->stat_offsets_loaded,
1049 &osd->rx_jabber, &nsd->rx_jabber);
1050
1051
1052 i40e_stat_update_and_clear32(hw,
1053 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)),
1054 &nsd->fd_atr_match);
1055 i40e_stat_update_and_clear32(hw,
1056 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)),
1057 &nsd->fd_sb_match);
1058 i40e_stat_update_and_clear32(hw,
1059 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)),
1060 &nsd->fd_atr_tunnel_match);
1061
1062 val = rd32(hw, I40E_PRTPM_EEE_STAT);
1063 nsd->tx_lpi_status =
1064 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >>
1065 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT;
1066 nsd->rx_lpi_status =
1067 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >>
1068 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT;
1069 i40e_stat_update32(hw, I40E_PRTPM_TLPIC,
1070 pf->stat_offsets_loaded,
1071 &osd->tx_lpi_count, &nsd->tx_lpi_count);
1072 i40e_stat_update32(hw, I40E_PRTPM_RLPIC,
1073 pf->stat_offsets_loaded,
1074 &osd->rx_lpi_count, &nsd->rx_lpi_count);
1075
1076 if (pf->flags & I40E_FLAG_FD_SB_ENABLED &&
1077 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
1078 nsd->fd_sb_status = true;
1079 else
1080 nsd->fd_sb_status = false;
1081
1082 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED &&
1083 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
1084 nsd->fd_atr_status = true;
1085 else
1086 nsd->fd_atr_status = false;
1087
1088 pf->stat_offsets_loaded = true;
1089}
1090
1091
1092
1093
1094
1095
1096
1097void i40e_update_stats(struct i40e_vsi *vsi)
1098{
1099 struct i40e_pf *pf = vsi->back;
1100
1101 if (vsi == pf->vsi[pf->lan_vsi])
1102 i40e_update_pf_stats(pf);
1103
1104 i40e_update_vsi_stats(vsi);
1105}
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi,
1116 const u8 *macaddr, s16 vlan)
1117{
1118 struct i40e_mac_filter *f;
1119 u64 key;
1120
1121 if (!vsi || !macaddr)
1122 return NULL;
1123
1124 key = i40e_addr_to_hkey(macaddr);
1125 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1126 if ((ether_addr_equal(macaddr, f->macaddr)) &&
1127 (vlan == f->vlan))
1128 return f;
1129 }
1130 return NULL;
1131}
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr)
1142{
1143 struct i40e_mac_filter *f;
1144 u64 key;
1145
1146 if (!vsi || !macaddr)
1147 return NULL;
1148
1149 key = i40e_addr_to_hkey(macaddr);
1150 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) {
1151 if ((ether_addr_equal(macaddr, f->macaddr)))
1152 return f;
1153 }
1154 return NULL;
1155}
1156
1157
1158
1159
1160
1161
1162
1163bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi)
1164{
1165
1166 if (vsi->info.pvid)
1167 return true;
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189 return vsi->has_vlan_filter;
1190}
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi,
1222 struct hlist_head *tmp_add_list,
1223 struct hlist_head *tmp_del_list,
1224 int vlan_filters)
1225{
1226 s16 pvid = le16_to_cpu(vsi->info.pvid);
1227 struct i40e_mac_filter *f, *add_head;
1228 struct i40e_new_mac_filter *new;
1229 struct hlist_node *h;
1230 int bkt, new_vlan;
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247 hlist_for_each_entry(new, tmp_add_list, hlist) {
1248 if (pvid && new->f->vlan != pvid)
1249 new->f->vlan = pvid;
1250 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY)
1251 new->f->vlan = 0;
1252 else if (!vlan_filters && new->f->vlan == 0)
1253 new->f->vlan = I40E_VLAN_ANY;
1254 }
1255
1256
1257 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1258
1259
1260
1261
1262
1263 if ((pvid && f->vlan != pvid) ||
1264 (vlan_filters && f->vlan == I40E_VLAN_ANY) ||
1265 (!vlan_filters && f->vlan == 0)) {
1266
1267 if (pvid)
1268 new_vlan = pvid;
1269 else if (vlan_filters)
1270 new_vlan = 0;
1271 else
1272 new_vlan = I40E_VLAN_ANY;
1273
1274
1275 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan);
1276 if (!add_head)
1277 return -ENOMEM;
1278
1279
1280 new = kzalloc(sizeof(*new), GFP_ATOMIC);
1281 if (!new)
1282 return -ENOMEM;
1283
1284 new->f = add_head;
1285 new->state = add_head->state;
1286
1287
1288 hlist_add_head(&new->hlist, tmp_add_list);
1289
1290
1291 f->state = I40E_FILTER_REMOVE;
1292 hash_del(&f->hlist);
1293 hlist_add_head(&f->hlist, tmp_del_list);
1294 }
1295 }
1296
1297 vsi->has_vlan_filter = !!vlan_filters;
1298
1299 return 0;
1300}
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr)
1311{
1312 struct i40e_aqc_remove_macvlan_element_data element;
1313 struct i40e_pf *pf = vsi->back;
1314
1315
1316 if (vsi->type != I40E_VSI_MAIN)
1317 return;
1318
1319 memset(&element, 0, sizeof(element));
1320 ether_addr_copy(element.mac_addr, macaddr);
1321 element.vlan_tag = 0;
1322
1323 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
1324 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1325
1326 memset(&element, 0, sizeof(element));
1327 ether_addr_copy(element.mac_addr, macaddr);
1328 element.vlan_tag = 0;
1329
1330 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH |
1331 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
1332 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL);
1333}
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi,
1347 const u8 *macaddr, s16 vlan)
1348{
1349 struct i40e_mac_filter *f;
1350 u64 key;
1351
1352 if (!vsi || !macaddr)
1353 return NULL;
1354
1355 f = i40e_find_filter(vsi, macaddr, vlan);
1356 if (!f) {
1357 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1358 if (!f)
1359 return NULL;
1360
1361
1362
1363
1364 if (vlan >= 0)
1365 vsi->has_vlan_filter = true;
1366
1367 ether_addr_copy(f->macaddr, macaddr);
1368 f->vlan = vlan;
1369 f->state = I40E_FILTER_NEW;
1370 INIT_HLIST_NODE(&f->hlist);
1371
1372 key = i40e_addr_to_hkey(macaddr);
1373 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1374
1375 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1376 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1377 }
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387 if (f->state == I40E_FILTER_REMOVE)
1388 f->state = I40E_FILTER_ACTIVE;
1389
1390 return f;
1391}
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f)
1409{
1410 if (!f)
1411 return;
1412
1413
1414
1415
1416
1417 if ((f->state == I40E_FILTER_FAILED) ||
1418 (f->state == I40E_FILTER_NEW)) {
1419 hash_del(&f->hlist);
1420 kfree(f);
1421 } else {
1422 f->state = I40E_FILTER_REMOVE;
1423 }
1424
1425 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1426 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1427}
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan)
1442{
1443 struct i40e_mac_filter *f;
1444
1445 if (!vsi || !macaddr)
1446 return;
1447
1448 f = i40e_find_filter(vsi, macaddr, vlan);
1449 __i40e_del_filter(vsi, f);
1450}
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi,
1465 const u8 *macaddr)
1466{
1467 struct i40e_mac_filter *f, *add = NULL;
1468 struct hlist_node *h;
1469 int bkt;
1470
1471 if (vsi->info.pvid)
1472 return i40e_add_filter(vsi, macaddr,
1473 le16_to_cpu(vsi->info.pvid));
1474
1475 if (!i40e_is_vsi_in_vlan(vsi))
1476 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY);
1477
1478 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1479 if (f->state == I40E_FILTER_REMOVE)
1480 continue;
1481 add = i40e_add_filter(vsi, macaddr, f->vlan);
1482 if (!add)
1483 return NULL;
1484 }
1485
1486 return add;
1487}
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr)
1500{
1501 struct i40e_mac_filter *f;
1502 struct hlist_node *h;
1503 bool found = false;
1504 int bkt;
1505
1506 lockdep_assert_held(&vsi->mac_filter_hash_lock);
1507 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
1508 if (ether_addr_equal(macaddr, f->macaddr)) {
1509 __i40e_del_filter(vsi, f);
1510 found = true;
1511 }
1512 }
1513
1514 if (found)
1515 return 0;
1516 else
1517 return -ENOENT;
1518}
1519
1520
1521
1522
1523
1524
1525
1526
1527static int i40e_set_mac(struct net_device *netdev, void *p)
1528{
1529 struct i40e_netdev_priv *np = netdev_priv(netdev);
1530 struct i40e_vsi *vsi = np->vsi;
1531 struct i40e_pf *pf = vsi->back;
1532 struct i40e_hw *hw = &pf->hw;
1533 struct sockaddr *addr = p;
1534
1535 if (!is_valid_ether_addr(addr->sa_data))
1536 return -EADDRNOTAVAIL;
1537
1538 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) {
1539 netdev_info(netdev, "already using mac address %pM\n",
1540 addr->sa_data);
1541 return 0;
1542 }
1543
1544 if (test_bit(__I40E_DOWN, pf->state) ||
1545 test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
1546 return -EADDRNOTAVAIL;
1547
1548 if (ether_addr_equal(hw->mac.addr, addr->sa_data))
1549 netdev_info(netdev, "returning to hw mac address %pM\n",
1550 hw->mac.addr);
1551 else
1552 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data);
1553
1554
1555
1556
1557
1558
1559
1560 spin_lock_bh(&vsi->mac_filter_hash_lock);
1561 i40e_del_mac_filter(vsi, netdev->dev_addr);
1562 ether_addr_copy(netdev->dev_addr, addr->sa_data);
1563 i40e_add_mac_filter(vsi, netdev->dev_addr);
1564 spin_unlock_bh(&vsi->mac_filter_hash_lock);
1565
1566 if (vsi->type == I40E_VSI_MAIN) {
1567 i40e_status ret;
1568
1569 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL,
1570 addr->sa_data, NULL);
1571 if (ret)
1572 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n",
1573 i40e_stat_str(hw, ret),
1574 i40e_aq_str(hw, hw->aq.asq_last_status));
1575 }
1576
1577
1578
1579
1580 i40e_service_event_schedule(pf);
1581 return 0;
1582}
1583
1584
1585
1586
1587
1588
1589static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
1590 u8 *lut, u16 lut_size)
1591{
1592 struct i40e_pf *pf = vsi->back;
1593 struct i40e_hw *hw = &pf->hw;
1594 int ret = 0;
1595
1596 if (seed) {
1597 struct i40e_aqc_get_set_rss_key_data *seed_dw =
1598 (struct i40e_aqc_get_set_rss_key_data *)seed;
1599 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw);
1600 if (ret) {
1601 dev_info(&pf->pdev->dev,
1602 "Cannot set RSS key, err %s aq_err %s\n",
1603 i40e_stat_str(hw, ret),
1604 i40e_aq_str(hw, hw->aq.asq_last_status));
1605 return ret;
1606 }
1607 }
1608 if (lut) {
1609 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
1610
1611 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
1612 if (ret) {
1613 dev_info(&pf->pdev->dev,
1614 "Cannot set RSS lut, err %s aq_err %s\n",
1615 i40e_stat_str(hw, ret),
1616 i40e_aq_str(hw, hw->aq.asq_last_status));
1617 return ret;
1618 }
1619 }
1620 return ret;
1621}
1622
1623
1624
1625
1626
1627static int i40e_vsi_config_rss(struct i40e_vsi *vsi)
1628{
1629 struct i40e_pf *pf = vsi->back;
1630 u8 seed[I40E_HKEY_ARRAY_SIZE];
1631 u8 *lut;
1632 int ret;
1633
1634 if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE))
1635 return 0;
1636 if (!vsi->rss_size)
1637 vsi->rss_size = min_t(int, pf->alloc_rss_size,
1638 vsi->num_queue_pairs);
1639 if (!vsi->rss_size)
1640 return -EINVAL;
1641 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
1642 if (!lut)
1643 return -ENOMEM;
1644
1645
1646
1647
1648 if (vsi->rss_lut_user)
1649 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
1650 else
1651 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
1652 if (vsi->rss_hkey_user)
1653 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
1654 else
1655 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
1656 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size);
1657 kfree(lut);
1658 return ret;
1659}
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi,
1670 struct i40e_vsi_context *ctxt,
1671 u8 enabled_tc)
1672{
1673 u16 qcount = 0, max_qcount, qmap, sections = 0;
1674 int i, override_q, pow, num_qps, ret;
1675 u8 netdev_tc = 0, offset = 0;
1676
1677 if (vsi->type != I40E_VSI_MAIN)
1678 return -EINVAL;
1679 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1680 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1681 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc;
1682 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1683 num_qps = vsi->mqprio_qopt.qopt.count[0];
1684
1685
1686 pow = ilog2(num_qps);
1687 if (!is_power_of_2(num_qps))
1688 pow++;
1689 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1690 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1691
1692
1693 max_qcount = vsi->mqprio_qopt.qopt.count[0];
1694 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1695
1696 if (vsi->tc_config.enabled_tc & BIT(i)) {
1697 offset = vsi->mqprio_qopt.qopt.offset[i];
1698 qcount = vsi->mqprio_qopt.qopt.count[i];
1699 if (qcount > max_qcount)
1700 max_qcount = qcount;
1701 vsi->tc_config.tc_info[i].qoffset = offset;
1702 vsi->tc_config.tc_info[i].qcount = qcount;
1703 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1704 } else {
1705
1706
1707
1708
1709 vsi->tc_config.tc_info[i].qoffset = 0;
1710 vsi->tc_config.tc_info[i].qcount = 1;
1711 vsi->tc_config.tc_info[i].netdev_tc = 0;
1712 }
1713 }
1714
1715
1716 vsi->num_queue_pairs = offset + qcount;
1717
1718
1719 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
1720 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1721 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1722 ctxt->info.valid_sections |= cpu_to_le16(sections);
1723
1724
1725 vsi->rss_size = max_qcount;
1726 ret = i40e_vsi_config_rss(vsi);
1727 if (ret) {
1728 dev_info(&vsi->back->pdev->dev,
1729 "Failed to reconfig rss for num_queues (%u)\n",
1730 max_qcount);
1731 return ret;
1732 }
1733 vsi->reconfig_rss = true;
1734 dev_dbg(&vsi->back->pdev->dev,
1735 "Reconfigured rss with num_queues (%u)\n", max_qcount);
1736
1737
1738
1739
1740 override_q = vsi->mqprio_qopt.qopt.count[0];
1741 if (override_q && override_q < vsi->num_queue_pairs) {
1742 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q;
1743 vsi->next_base_queue = override_q;
1744 }
1745 return 0;
1746}
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi,
1758 struct i40e_vsi_context *ctxt,
1759 u8 enabled_tc,
1760 bool is_add)
1761{
1762 struct i40e_pf *pf = vsi->back;
1763 u16 sections = 0;
1764 u8 netdev_tc = 0;
1765 u16 numtc = 1;
1766 u16 qcount;
1767 u8 offset;
1768 u16 qmap;
1769 int i;
1770 u16 num_tc_qps = 0;
1771
1772 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
1773 offset = 0;
1774
1775
1776 num_tc_qps = vsi->alloc_queue_pairs;
1777 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
1778
1779 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1780 if (enabled_tc & BIT(i))
1781 numtc++;
1782 }
1783 if (!numtc) {
1784 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n");
1785 numtc = 1;
1786 }
1787 num_tc_qps = num_tc_qps / numtc;
1788 num_tc_qps = min_t(int, num_tc_qps,
1789 i40e_pf_get_max_q_per_tc(pf));
1790 }
1791
1792 vsi->tc_config.numtc = numtc;
1793 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1;
1794
1795
1796 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1797 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix);
1798
1799
1800 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1801
1802 if (vsi->tc_config.enabled_tc & BIT(i)) {
1803
1804 int pow, num_qps;
1805
1806 switch (vsi->type) {
1807 case I40E_VSI_MAIN:
1808 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED |
1809 I40E_FLAG_FD_ATR_ENABLED)) ||
1810 vsi->tc_config.enabled_tc != 1) {
1811 qcount = min_t(int, pf->alloc_rss_size,
1812 num_tc_qps);
1813 break;
1814 }
1815
1816 case I40E_VSI_FDIR:
1817 case I40E_VSI_SRIOV:
1818 case I40E_VSI_VMDQ2:
1819 default:
1820 qcount = num_tc_qps;
1821 WARN_ON(i != 0);
1822 break;
1823 }
1824 vsi->tc_config.tc_info[i].qoffset = offset;
1825 vsi->tc_config.tc_info[i].qcount = qcount;
1826
1827
1828 num_qps = qcount;
1829 pow = 0;
1830 while (num_qps && (BIT_ULL(pow) < qcount)) {
1831 pow++;
1832 num_qps >>= 1;
1833 }
1834
1835 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++;
1836 qmap =
1837 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
1838 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
1839
1840 offset += qcount;
1841 } else {
1842
1843
1844
1845
1846 vsi->tc_config.tc_info[i].qoffset = 0;
1847 vsi->tc_config.tc_info[i].qcount = 1;
1848 vsi->tc_config.tc_info[i].netdev_tc = 0;
1849
1850 qmap = 0;
1851 }
1852 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1853 }
1854
1855
1856 vsi->num_queue_pairs = offset;
1857 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) {
1858 if (vsi->req_queue_pairs > 0)
1859 vsi->num_queue_pairs = vsi->req_queue_pairs;
1860 else if (pf->flags & I40E_FLAG_MSIX_ENABLED)
1861 vsi->num_queue_pairs = pf->num_lan_msix;
1862 }
1863
1864
1865 if (is_add) {
1866 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
1867
1868 ctxt->info.up_enable_bits = enabled_tc;
1869 }
1870 if (vsi->type == I40E_VSI_SRIOV) {
1871 ctxt->info.mapping_flags |=
1872 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG);
1873 for (i = 0; i < vsi->num_queue_pairs; i++)
1874 ctxt->info.queue_mapping[i] =
1875 cpu_to_le16(vsi->base_queue + i);
1876 } else {
1877 ctxt->info.mapping_flags |=
1878 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
1879 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
1880 }
1881 ctxt->info.valid_sections |= cpu_to_le16(sections);
1882}
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892static int i40e_addr_sync(struct net_device *netdev, const u8 *addr)
1893{
1894 struct i40e_netdev_priv *np = netdev_priv(netdev);
1895 struct i40e_vsi *vsi = np->vsi;
1896
1897 if (i40e_add_mac_filter(vsi, addr))
1898 return 0;
1899 else
1900 return -ENOMEM;
1901}
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr)
1912{
1913 struct i40e_netdev_priv *np = netdev_priv(netdev);
1914 struct i40e_vsi *vsi = np->vsi;
1915
1916
1917
1918
1919
1920
1921 if (ether_addr_equal(addr, netdev->dev_addr))
1922 return 0;
1923
1924 i40e_del_mac_filter(vsi, addr);
1925
1926 return 0;
1927}
1928
1929
1930
1931
1932
1933static void i40e_set_rx_mode(struct net_device *netdev)
1934{
1935 struct i40e_netdev_priv *np = netdev_priv(netdev);
1936 struct i40e_vsi *vsi = np->vsi;
1937
1938 spin_lock_bh(&vsi->mac_filter_hash_lock);
1939
1940 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1941 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync);
1942
1943 spin_unlock_bh(&vsi->mac_filter_hash_lock);
1944
1945
1946 if (vsi->current_netdev_flags != vsi->netdev->flags) {
1947 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
1948 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state);
1949 }
1950}
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi,
1961 struct hlist_head *from)
1962{
1963 struct i40e_mac_filter *f;
1964 struct hlist_node *h;
1965
1966 hlist_for_each_entry_safe(f, h, from, hlist) {
1967 u64 key = i40e_addr_to_hkey(f->macaddr);
1968
1969
1970 hlist_del(&f->hlist);
1971 hash_add(vsi->mac_filter_hash, &f->hlist, key);
1972 }
1973}
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi,
1984 struct hlist_head *from)
1985{
1986 struct i40e_new_mac_filter *new;
1987 struct hlist_node *h;
1988
1989 hlist_for_each_entry_safe(new, h, from, hlist) {
1990
1991 hlist_del(&new->hlist);
1992 kfree(new);
1993 }
1994}
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004static
2005struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next)
2006{
2007 hlist_for_each_entry_continue(next, hlist) {
2008 if (!is_broadcast_ether_addr(next->f->macaddr))
2009 return next;
2010 }
2011
2012 return NULL;
2013}
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025static int
2026i40e_update_filter_state(int count,
2027 struct i40e_aqc_add_macvlan_element_data *add_list,
2028 struct i40e_new_mac_filter *add_head)
2029{
2030 int retval = 0;
2031 int i;
2032
2033 for (i = 0; i < count; i++) {
2034
2035
2036
2037
2038
2039
2040 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) {
2041 add_head->state = I40E_FILTER_FAILED;
2042 } else {
2043 add_head->state = I40E_FILTER_ACTIVE;
2044 retval++;
2045 }
2046
2047 add_head = i40e_next_filter(add_head);
2048 if (!add_head)
2049 break;
2050 }
2051
2052 return retval;
2053}
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068static
2069void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name,
2070 struct i40e_aqc_remove_macvlan_element_data *list,
2071 int num_del, int *retval)
2072{
2073 struct i40e_hw *hw = &vsi->back->hw;
2074 i40e_status aq_ret;
2075 int aq_err;
2076
2077 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL);
2078 aq_err = hw->aq.asq_last_status;
2079
2080
2081 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) {
2082 *retval = -EIO;
2083 dev_info(&vsi->back->pdev->dev,
2084 "ignoring delete macvlan error on %s, err %s, aq_err %s\n",
2085 vsi_name, i40e_stat_str(hw, aq_ret),
2086 i40e_aq_str(hw, aq_err));
2087 }
2088}
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102static
2103void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name,
2104 struct i40e_aqc_add_macvlan_element_data *list,
2105 struct i40e_new_mac_filter *add_head,
2106 int num_add)
2107{
2108 struct i40e_hw *hw = &vsi->back->hw;
2109 int aq_err, fcnt;
2110
2111 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL);
2112 aq_err = hw->aq.asq_last_status;
2113 fcnt = i40e_update_filter_state(num_add, list, add_head);
2114
2115 if (fcnt != num_add) {
2116 if (vsi->type == I40E_VSI_MAIN) {
2117 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2118 dev_warn(&vsi->back->pdev->dev,
2119 "Error %s adding RX filters on %s, promiscuous mode forced on\n",
2120 i40e_aq_str(hw, aq_err), vsi_name);
2121 } else if (vsi->type == I40E_VSI_SRIOV ||
2122 vsi->type == I40E_VSI_VMDQ1 ||
2123 vsi->type == I40E_VSI_VMDQ2) {
2124 dev_warn(&vsi->back->pdev->dev,
2125 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n",
2126 i40e_aq_str(hw, aq_err), vsi_name, vsi_name);
2127 } else {
2128 dev_warn(&vsi->back->pdev->dev,
2129 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n",
2130 i40e_aq_str(hw, aq_err), vsi_name, vsi->type);
2131 }
2132 }
2133}
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147static i40e_status
2148i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name,
2149 struct i40e_mac_filter *f)
2150{
2151 bool enable = f->state == I40E_FILTER_NEW;
2152 struct i40e_hw *hw = &vsi->back->hw;
2153 i40e_status aq_ret;
2154
2155 if (f->vlan == I40E_VLAN_ANY) {
2156 aq_ret = i40e_aq_set_vsi_broadcast(hw,
2157 vsi->seid,
2158 enable,
2159 NULL);
2160 } else {
2161 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw,
2162 vsi->seid,
2163 enable,
2164 f->vlan,
2165 NULL);
2166 }
2167
2168 if (aq_ret) {
2169 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2170 dev_warn(&vsi->back->pdev->dev,
2171 "Error %s, forcing overflow promiscuous on %s\n",
2172 i40e_aq_str(hw, hw->aq.asq_last_status),
2173 vsi_name);
2174 }
2175
2176 return aq_ret;
2177}
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc)
2189{
2190 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
2191 struct i40e_hw *hw = &pf->hw;
2192 i40e_status aq_ret;
2193
2194 if (vsi->type == I40E_VSI_MAIN &&
2195 pf->lan_veb != I40E_NO_VEB &&
2196 !(pf->flags & I40E_FLAG_MFP_ENABLED)) {
2197
2198
2199
2200
2201
2202 if (promisc)
2203 aq_ret = i40e_aq_set_default_vsi(hw,
2204 vsi->seid,
2205 NULL);
2206 else
2207 aq_ret = i40e_aq_clear_default_vsi(hw,
2208 vsi->seid,
2209 NULL);
2210 if (aq_ret) {
2211 dev_info(&pf->pdev->dev,
2212 "Set default VSI failed, err %s, aq_err %s\n",
2213 i40e_stat_str(hw, aq_ret),
2214 i40e_aq_str(hw, hw->aq.asq_last_status));
2215 }
2216 } else {
2217 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(
2218 hw,
2219 vsi->seid,
2220 promisc, NULL,
2221 true);
2222 if (aq_ret) {
2223 dev_info(&pf->pdev->dev,
2224 "set unicast promisc failed, err %s, aq_err %s\n",
2225 i40e_stat_str(hw, aq_ret),
2226 i40e_aq_str(hw, hw->aq.asq_last_status));
2227 }
2228 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(
2229 hw,
2230 vsi->seid,
2231 promisc, NULL);
2232 if (aq_ret) {
2233 dev_info(&pf->pdev->dev,
2234 "set multicast promisc failed, err %s, aq_err %s\n",
2235 i40e_stat_str(hw, aq_ret),
2236 i40e_aq_str(hw, hw->aq.asq_last_status));
2237 }
2238 }
2239
2240 if (!aq_ret)
2241 pf->cur_promisc = promisc;
2242
2243 return aq_ret;
2244}
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254int i40e_sync_vsi_filters(struct i40e_vsi *vsi)
2255{
2256 struct hlist_head tmp_add_list, tmp_del_list;
2257 struct i40e_mac_filter *f;
2258 struct i40e_new_mac_filter *new, *add_head = NULL;
2259 struct i40e_hw *hw = &vsi->back->hw;
2260 bool old_overflow, new_overflow;
2261 unsigned int failed_filters = 0;
2262 unsigned int vlan_filters = 0;
2263 char vsi_name[16] = "PF";
2264 int filter_list_len = 0;
2265 i40e_status aq_ret = 0;
2266 u32 changed_flags = 0;
2267 struct hlist_node *h;
2268 struct i40e_pf *pf;
2269 int num_add = 0;
2270 int num_del = 0;
2271 int retval = 0;
2272 u16 cmd_flags;
2273 int list_size;
2274 int bkt;
2275
2276
2277 struct i40e_aqc_add_macvlan_element_data *add_list;
2278 struct i40e_aqc_remove_macvlan_element_data *del_list;
2279
2280 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state))
2281 usleep_range(1000, 2000);
2282 pf = vsi->back;
2283
2284 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2285
2286 if (vsi->netdev) {
2287 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
2288 vsi->current_netdev_flags = vsi->netdev->flags;
2289 }
2290
2291 INIT_HLIST_HEAD(&tmp_add_list);
2292 INIT_HLIST_HEAD(&tmp_del_list);
2293
2294 if (vsi->type == I40E_VSI_SRIOV)
2295 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id);
2296 else if (vsi->type != I40E_VSI_MAIN)
2297 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid);
2298
2299 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) {
2300 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED;
2301
2302 spin_lock_bh(&vsi->mac_filter_hash_lock);
2303
2304 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2305 if (f->state == I40E_FILTER_REMOVE) {
2306
2307 hash_del(&f->hlist);
2308 hlist_add_head(&f->hlist, &tmp_del_list);
2309
2310
2311 continue;
2312 }
2313 if (f->state == I40E_FILTER_NEW) {
2314
2315 new = kzalloc(sizeof(*new), GFP_ATOMIC);
2316 if (!new)
2317 goto err_no_memory_locked;
2318
2319
2320 new->f = f;
2321 new->state = f->state;
2322
2323
2324 hlist_add_head(&new->hlist, &tmp_add_list);
2325 }
2326
2327
2328
2329
2330
2331 if (f->vlan > 0)
2332 vlan_filters++;
2333 }
2334
2335 retval = i40e_correct_mac_vlan_filters(vsi,
2336 &tmp_add_list,
2337 &tmp_del_list,
2338 vlan_filters);
2339 if (retval)
2340 goto err_no_memory_locked;
2341
2342 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2343 }
2344
2345
2346 if (!hlist_empty(&tmp_del_list)) {
2347 filter_list_len = hw->aq.asq_buf_size /
2348 sizeof(struct i40e_aqc_remove_macvlan_element_data);
2349 list_size = filter_list_len *
2350 sizeof(struct i40e_aqc_remove_macvlan_element_data);
2351 del_list = kzalloc(list_size, GFP_ATOMIC);
2352 if (!del_list)
2353 goto err_no_memory;
2354
2355 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) {
2356 cmd_flags = 0;
2357
2358
2359
2360
2361 if (is_broadcast_ether_addr(f->macaddr)) {
2362 i40e_aqc_broadcast_filter(vsi, vsi_name, f);
2363
2364 hlist_del(&f->hlist);
2365 kfree(f);
2366 continue;
2367 }
2368
2369
2370 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr);
2371 if (f->vlan == I40E_VLAN_ANY) {
2372 del_list[num_del].vlan_tag = 0;
2373 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN;
2374 } else {
2375 del_list[num_del].vlan_tag =
2376 cpu_to_le16((u16)(f->vlan));
2377 }
2378
2379 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
2380 del_list[num_del].flags = cmd_flags;
2381 num_del++;
2382
2383
2384 if (num_del == filter_list_len) {
2385 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2386 num_del, &retval);
2387 memset(del_list, 0, list_size);
2388 num_del = 0;
2389 }
2390
2391
2392
2393 hlist_del(&f->hlist);
2394 kfree(f);
2395 }
2396
2397 if (num_del) {
2398 i40e_aqc_del_filters(vsi, vsi_name, del_list,
2399 num_del, &retval);
2400 }
2401
2402 kfree(del_list);
2403 del_list = NULL;
2404 }
2405
2406 if (!hlist_empty(&tmp_add_list)) {
2407
2408 filter_list_len = hw->aq.asq_buf_size /
2409 sizeof(struct i40e_aqc_add_macvlan_element_data);
2410 list_size = filter_list_len *
2411 sizeof(struct i40e_aqc_add_macvlan_element_data);
2412 add_list = kzalloc(list_size, GFP_ATOMIC);
2413 if (!add_list)
2414 goto err_no_memory;
2415
2416 num_add = 0;
2417 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2418
2419
2420
2421 if (is_broadcast_ether_addr(new->f->macaddr)) {
2422 if (i40e_aqc_broadcast_filter(vsi, vsi_name,
2423 new->f))
2424 new->state = I40E_FILTER_FAILED;
2425 else
2426 new->state = I40E_FILTER_ACTIVE;
2427 continue;
2428 }
2429
2430
2431 if (num_add == 0)
2432 add_head = new;
2433 cmd_flags = 0;
2434 ether_addr_copy(add_list[num_add].mac_addr,
2435 new->f->macaddr);
2436 if (new->f->vlan == I40E_VLAN_ANY) {
2437 add_list[num_add].vlan_tag = 0;
2438 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN;
2439 } else {
2440 add_list[num_add].vlan_tag =
2441 cpu_to_le16((u16)(new->f->vlan));
2442 }
2443 add_list[num_add].queue_number = 0;
2444
2445 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES;
2446 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
2447 add_list[num_add].flags = cpu_to_le16(cmd_flags);
2448 num_add++;
2449
2450
2451 if (num_add == filter_list_len) {
2452 i40e_aqc_add_filters(vsi, vsi_name, add_list,
2453 add_head, num_add);
2454 memset(add_list, 0, list_size);
2455 num_add = 0;
2456 }
2457 }
2458 if (num_add) {
2459 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head,
2460 num_add);
2461 }
2462
2463
2464
2465 spin_lock_bh(&vsi->mac_filter_hash_lock);
2466 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) {
2467
2468 if (new->f->state == I40E_FILTER_NEW)
2469 new->f->state = new->state;
2470 hlist_del(&new->hlist);
2471 kfree(new);
2472 }
2473 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2474 kfree(add_list);
2475 add_list = NULL;
2476 }
2477
2478
2479 spin_lock_bh(&vsi->mac_filter_hash_lock);
2480 vsi->active_filters = 0;
2481 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) {
2482 if (f->state == I40E_FILTER_ACTIVE)
2483 vsi->active_filters++;
2484 else if (f->state == I40E_FILTER_FAILED)
2485 failed_filters++;
2486 }
2487 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2488
2489
2490
2491
2492
2493 if (old_overflow && !failed_filters &&
2494 vsi->active_filters < vsi->promisc_threshold) {
2495 dev_info(&pf->pdev->dev,
2496 "filter logjam cleared on %s, leaving overflow promiscuous mode\n",
2497 vsi_name);
2498 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2499 vsi->promisc_threshold = 0;
2500 }
2501
2502
2503 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) {
2504 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2505 goto out;
2506 }
2507
2508 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
2509
2510
2511
2512
2513 if (!old_overflow && new_overflow)
2514 vsi->promisc_threshold = (vsi->active_filters * 3) / 4;
2515
2516
2517 if (changed_flags & IFF_ALLMULTI) {
2518 bool cur_multipromisc;
2519
2520 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI);
2521 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw,
2522 vsi->seid,
2523 cur_multipromisc,
2524 NULL);
2525 if (aq_ret) {
2526 retval = i40e_aq_rc_to_posix(aq_ret,
2527 hw->aq.asq_last_status);
2528 dev_info(&pf->pdev->dev,
2529 "set multi promisc failed on %s, err %s aq_err %s\n",
2530 vsi_name,
2531 i40e_stat_str(hw, aq_ret),
2532 i40e_aq_str(hw, hw->aq.asq_last_status));
2533 }
2534 }
2535
2536 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) {
2537 bool cur_promisc;
2538
2539 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) ||
2540 new_overflow);
2541 aq_ret = i40e_set_promiscuous(pf, cur_promisc);
2542 if (aq_ret) {
2543 retval = i40e_aq_rc_to_posix(aq_ret,
2544 hw->aq.asq_last_status);
2545 dev_info(&pf->pdev->dev,
2546 "Setting promiscuous %s failed on %s, err %s aq_err %s\n",
2547 cur_promisc ? "on" : "off",
2548 vsi_name,
2549 i40e_stat_str(hw, aq_ret),
2550 i40e_aq_str(hw, hw->aq.asq_last_status));
2551 }
2552 }
2553out:
2554
2555 if (retval)
2556 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2557
2558 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2559 return retval;
2560
2561err_no_memory:
2562
2563 spin_lock_bh(&vsi->mac_filter_hash_lock);
2564err_no_memory_locked:
2565 i40e_undo_del_filter_entries(vsi, &tmp_del_list);
2566 i40e_undo_add_filter_entries(vsi, &tmp_add_list);
2567 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2568
2569 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
2570 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state);
2571 return -ENOMEM;
2572}
2573
2574
2575
2576
2577
2578static void i40e_sync_filters_subtask(struct i40e_pf *pf)
2579{
2580 int v;
2581
2582 if (!pf)
2583 return;
2584 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state))
2585 return;
2586
2587 for (v = 0; v < pf->num_alloc_vsi; v++) {
2588 if (pf->vsi[v] &&
2589 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) {
2590 int ret = i40e_sync_vsi_filters(pf->vsi[v]);
2591
2592 if (ret) {
2593
2594 set_bit(__I40E_MACVLAN_SYNC_PENDING,
2595 pf->state);
2596 break;
2597 }
2598 }
2599 }
2600}
2601
2602
2603
2604
2605
2606static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi)
2607{
2608 if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
2609 return I40E_RXBUFFER_2048;
2610 else
2611 return I40E_RXBUFFER_3072;
2612}
2613
2614
2615
2616
2617
2618
2619
2620
2621static int i40e_change_mtu(struct net_device *netdev, int new_mtu)
2622{
2623 struct i40e_netdev_priv *np = netdev_priv(netdev);
2624 struct i40e_vsi *vsi = np->vsi;
2625 struct i40e_pf *pf = vsi->back;
2626
2627 if (i40e_enabled_xdp_vsi(vsi)) {
2628 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
2629
2630 if (frame_size > i40e_max_xdp_frame_size(vsi))
2631 return -EINVAL;
2632 }
2633
2634 netdev_info(netdev, "changing MTU from %d to %d\n",
2635 netdev->mtu, new_mtu);
2636 netdev->mtu = new_mtu;
2637 if (netif_running(netdev))
2638 i40e_vsi_reinit_locked(vsi);
2639 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
2640 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
2641 return 0;
2642}
2643
2644
2645
2646
2647
2648
2649
2650int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2651{
2652 struct i40e_netdev_priv *np = netdev_priv(netdev);
2653 struct i40e_pf *pf = np->vsi->back;
2654
2655 switch (cmd) {
2656 case SIOCGHWTSTAMP:
2657 return i40e_ptp_get_ts_config(pf, ifr);
2658 case SIOCSHWTSTAMP:
2659 return i40e_ptp_set_ts_config(pf, ifr);
2660 default:
2661 return -EOPNOTSUPP;
2662 }
2663}
2664
2665
2666
2667
2668
2669void i40e_vlan_stripping_enable(struct i40e_vsi *vsi)
2670{
2671 struct i40e_vsi_context ctxt;
2672 i40e_status ret;
2673
2674
2675 if (vsi->info.pvid)
2676 return;
2677
2678 if ((vsi->info.valid_sections &
2679 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2680 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0))
2681 return;
2682
2683 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2684 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2685 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH;
2686
2687 ctxt.seid = vsi->seid;
2688 ctxt.info = vsi->info;
2689 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2690 if (ret) {
2691 dev_info(&vsi->back->pdev->dev,
2692 "update vlan stripping failed, err %s aq_err %s\n",
2693 i40e_stat_str(&vsi->back->hw, ret),
2694 i40e_aq_str(&vsi->back->hw,
2695 vsi->back->hw.aq.asq_last_status));
2696 }
2697}
2698
2699
2700
2701
2702
2703void i40e_vlan_stripping_disable(struct i40e_vsi *vsi)
2704{
2705 struct i40e_vsi_context ctxt;
2706 i40e_status ret;
2707
2708
2709 if (vsi->info.pvid)
2710 return;
2711
2712 if ((vsi->info.valid_sections &
2713 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) &&
2714 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) ==
2715 I40E_AQ_VSI_PVLAN_EMOD_MASK))
2716 return;
2717
2718 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2719 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL |
2720 I40E_AQ_VSI_PVLAN_EMOD_NOTHING;
2721
2722 ctxt.seid = vsi->seid;
2723 ctxt.info = vsi->info;
2724 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2725 if (ret) {
2726 dev_info(&vsi->back->pdev->dev,
2727 "update vlan stripping failed, err %s aq_err %s\n",
2728 i40e_stat_str(&vsi->back->hw, ret),
2729 i40e_aq_str(&vsi->back->hw,
2730 vsi->back->hw.aq.asq_last_status));
2731 }
2732}
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2748{
2749 struct i40e_mac_filter *f, *add_f;
2750 struct hlist_node *h;
2751 int bkt;
2752
2753 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2754 if (f->state == I40E_FILTER_REMOVE)
2755 continue;
2756 add_f = i40e_add_filter(vsi, f->macaddr, vid);
2757 if (!add_f) {
2758 dev_info(&vsi->back->pdev->dev,
2759 "Could not add vlan filter %d for %pM\n",
2760 vid, f->macaddr);
2761 return -ENOMEM;
2762 }
2763 }
2764
2765 return 0;
2766}
2767
2768
2769
2770
2771
2772
2773int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid)
2774{
2775 int err;
2776
2777 if (vsi->info.pvid)
2778 return -EINVAL;
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788 if (!vid)
2789 return 0;
2790
2791
2792 spin_lock_bh(&vsi->mac_filter_hash_lock);
2793 err = i40e_add_vlan_all_mac(vsi, vid);
2794 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2795 if (err)
2796 return err;
2797
2798
2799
2800
2801 i40e_service_event_schedule(vsi->back);
2802 return 0;
2803}
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid)
2819{
2820 struct i40e_mac_filter *f;
2821 struct hlist_node *h;
2822 int bkt;
2823
2824 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
2825 if (f->vlan == vid)
2826 __i40e_del_filter(vsi, f);
2827 }
2828}
2829
2830
2831
2832
2833
2834
2835void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid)
2836{
2837 if (!vid || vsi->info.pvid)
2838 return;
2839
2840 spin_lock_bh(&vsi->mac_filter_hash_lock);
2841 i40e_rm_vlan_all_mac(vsi, vid);
2842 spin_unlock_bh(&vsi->mac_filter_hash_lock);
2843
2844
2845
2846
2847 i40e_service_event_schedule(vsi->back);
2848}
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858static int i40e_vlan_rx_add_vid(struct net_device *netdev,
2859 __always_unused __be16 proto, u16 vid)
2860{
2861 struct i40e_netdev_priv *np = netdev_priv(netdev);
2862 struct i40e_vsi *vsi = np->vsi;
2863 int ret = 0;
2864
2865 if (vid >= VLAN_N_VID)
2866 return -EINVAL;
2867
2868 ret = i40e_vsi_add_vlan(vsi, vid);
2869 if (!ret)
2870 set_bit(vid, vsi->active_vlans);
2871
2872 return ret;
2873}
2874
2875
2876
2877
2878
2879
2880
2881static void i40e_vlan_rx_add_vid_up(struct net_device *netdev,
2882 __always_unused __be16 proto, u16 vid)
2883{
2884 struct i40e_netdev_priv *np = netdev_priv(netdev);
2885 struct i40e_vsi *vsi = np->vsi;
2886
2887 if (vid >= VLAN_N_VID)
2888 return;
2889 set_bit(vid, vsi->active_vlans);
2890}
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900static int i40e_vlan_rx_kill_vid(struct net_device *netdev,
2901 __always_unused __be16 proto, u16 vid)
2902{
2903 struct i40e_netdev_priv *np = netdev_priv(netdev);
2904 struct i40e_vsi *vsi = np->vsi;
2905
2906
2907
2908
2909
2910 i40e_vsi_kill_vlan(vsi, vid);
2911
2912 clear_bit(vid, vsi->active_vlans);
2913
2914 return 0;
2915}
2916
2917
2918
2919
2920
2921static void i40e_restore_vlan(struct i40e_vsi *vsi)
2922{
2923 u16 vid;
2924
2925 if (!vsi->netdev)
2926 return;
2927
2928 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2929 i40e_vlan_stripping_enable(vsi);
2930 else
2931 i40e_vlan_stripping_disable(vsi);
2932
2933 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID)
2934 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q),
2935 vid);
2936}
2937
2938
2939
2940
2941
2942
2943int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid)
2944{
2945 struct i40e_vsi_context ctxt;
2946 i40e_status ret;
2947
2948 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
2949 vsi->info.pvid = cpu_to_le16(vid);
2950 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED |
2951 I40E_AQ_VSI_PVLAN_INSERT_PVID |
2952 I40E_AQ_VSI_PVLAN_EMOD_STR;
2953
2954 ctxt.seid = vsi->seid;
2955 ctxt.info = vsi->info;
2956 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
2957 if (ret) {
2958 dev_info(&vsi->back->pdev->dev,
2959 "add pvid failed, err %s aq_err %s\n",
2960 i40e_stat_str(&vsi->back->hw, ret),
2961 i40e_aq_str(&vsi->back->hw,
2962 vsi->back->hw.aq.asq_last_status));
2963 return -ENOENT;
2964 }
2965
2966 return 0;
2967}
2968
2969
2970
2971
2972
2973
2974
2975void i40e_vsi_remove_pvid(struct i40e_vsi *vsi)
2976{
2977 vsi->info.pvid = 0;
2978
2979 i40e_vlan_stripping_disable(vsi);
2980}
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi)
2993{
2994 int i, err = 0;
2995
2996 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
2997 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]);
2998
2999 if (!i40e_enabled_xdp_vsi(vsi))
3000 return err;
3001
3002 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3003 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]);
3004
3005 return err;
3006}
3007
3008
3009
3010
3011
3012
3013
3014static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi)
3015{
3016 int i;
3017
3018 if (vsi->tx_rings) {
3019 for (i = 0; i < vsi->num_queue_pairs; i++)
3020 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
3021 i40e_free_tx_resources(vsi->tx_rings[i]);
3022 }
3023
3024 if (vsi->xdp_rings) {
3025 for (i = 0; i < vsi->num_queue_pairs; i++)
3026 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc)
3027 i40e_free_tx_resources(vsi->xdp_rings[i]);
3028 }
3029}
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi)
3042{
3043 int i, err = 0;
3044
3045 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3046 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]);
3047 return err;
3048}
3049
3050
3051
3052
3053
3054
3055
3056static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi)
3057{
3058 int i;
3059
3060 if (!vsi->rx_rings)
3061 return;
3062
3063 for (i = 0; i < vsi->num_queue_pairs; i++)
3064 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
3065 i40e_free_rx_resources(vsi->rx_rings[i]);
3066}
3067
3068
3069
3070
3071
3072
3073
3074
3075static void i40e_config_xps_tx_ring(struct i40e_ring *ring)
3076{
3077 int cpu;
3078
3079 if (!ring->q_vector || !ring->netdev || ring->ch)
3080 return;
3081
3082
3083 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state))
3084 return;
3085
3086 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1);
3087 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu),
3088 ring->queue_index);
3089}
3090
3091
3092
3093
3094
3095
3096
3097static struct xdp_umem *i40e_xsk_umem(struct i40e_ring *ring)
3098{
3099 bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi);
3100 int qid = ring->queue_index;
3101
3102 if (ring_is_xdp(ring))
3103 qid -= ring->vsi->alloc_queue_pairs;
3104
3105 if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps))
3106 return NULL;
3107
3108 return xdp_get_umem_from_qid(ring->vsi->netdev, qid);
3109}
3110
3111
3112
3113
3114
3115
3116
3117static int i40e_configure_tx_ring(struct i40e_ring *ring)
3118{
3119 struct i40e_vsi *vsi = ring->vsi;
3120 u16 pf_q = vsi->base_queue + ring->queue_index;
3121 struct i40e_hw *hw = &vsi->back->hw;
3122 struct i40e_hmc_obj_txq tx_ctx;
3123 i40e_status err = 0;
3124 u32 qtx_ctl = 0;
3125
3126 if (ring_is_xdp(ring))
3127 ring->xsk_umem = i40e_xsk_umem(ring);
3128
3129
3130 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) {
3131 ring->atr_sample_rate = vsi->back->atr_sample_rate;
3132 ring->atr_count = 0;
3133 } else {
3134 ring->atr_sample_rate = 0;
3135 }
3136
3137
3138 i40e_config_xps_tx_ring(ring);
3139
3140
3141 memset(&tx_ctx, 0, sizeof(tx_ctx));
3142
3143 tx_ctx.new_context = 1;
3144 tx_ctx.base = (ring->dma / 128);
3145 tx_ctx.qlen = ring->count;
3146 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED |
3147 I40E_FLAG_FD_ATR_ENABLED));
3148 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP);
3149
3150 if (vsi->type != I40E_VSI_FDIR)
3151 tx_ctx.head_wb_ena = 1;
3152 tx_ctx.head_wb_addr = ring->dma +
3153 (ring->count * sizeof(struct i40e_tx_desc));
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166 if (ring->ch)
3167 tx_ctx.rdylist =
3168 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]);
3169
3170 else
3171 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]);
3172
3173 tx_ctx.rdylist_act = 0;
3174
3175
3176 err = i40e_clear_lan_tx_queue_context(hw, pf_q);
3177 if (err) {
3178 dev_info(&vsi->back->pdev->dev,
3179 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n",
3180 ring->queue_index, pf_q, err);
3181 return -ENOMEM;
3182 }
3183
3184
3185 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
3186 if (err) {
3187 dev_info(&vsi->back->pdev->dev,
3188 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n",
3189 ring->queue_index, pf_q, err);
3190 return -ENOMEM;
3191 }
3192
3193
3194 if (ring->ch) {
3195 if (ring->ch->type == I40E_VSI_VMDQ2)
3196 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3197 else
3198 return -EINVAL;
3199
3200 qtx_ctl |= (ring->ch->vsi_number <<
3201 I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3202 I40E_QTX_CTL_VFVM_INDX_MASK;
3203 } else {
3204 if (vsi->type == I40E_VSI_VMDQ2) {
3205 qtx_ctl = I40E_QTX_CTL_VM_QUEUE;
3206 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) &
3207 I40E_QTX_CTL_VFVM_INDX_MASK;
3208 } else {
3209 qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
3210 }
3211 }
3212
3213 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
3214 I40E_QTX_CTL_PF_INDX_MASK);
3215 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
3216 i40e_flush(hw);
3217
3218
3219 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
3220
3221 return 0;
3222}
3223
3224
3225
3226
3227
3228
3229
3230static int i40e_configure_rx_ring(struct i40e_ring *ring)
3231{
3232 struct i40e_vsi *vsi = ring->vsi;
3233 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len;
3234 u16 pf_q = vsi->base_queue + ring->queue_index;
3235 struct i40e_hw *hw = &vsi->back->hw;
3236 struct i40e_hmc_obj_rxq rx_ctx;
3237 i40e_status err = 0;
3238 bool ok;
3239 int ret;
3240
3241 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS);
3242
3243
3244 memset(&rx_ctx, 0, sizeof(rx_ctx));
3245
3246 if (ring->vsi->type == I40E_VSI_MAIN)
3247 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
3248
3249 ring->xsk_umem = i40e_xsk_umem(ring);
3250 if (ring->xsk_umem) {
3251 ring->rx_buf_len = ring->xsk_umem->chunk_size_nohr -
3252 XDP_PACKET_HEADROOM;
3253
3254
3255
3256
3257 chain_len = 1;
3258 ring->zca.free = i40e_zca_free;
3259 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3260 MEM_TYPE_ZERO_COPY,
3261 &ring->zca);
3262 if (ret)
3263 return ret;
3264 dev_info(&vsi->back->pdev->dev,
3265 "Registered XDP mem model MEM_TYPE_ZERO_COPY on Rx ring %d\n",
3266 ring->queue_index);
3267
3268 } else {
3269 ring->rx_buf_len = vsi->rx_buf_len;
3270 if (ring->vsi->type == I40E_VSI_MAIN) {
3271 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
3272 MEM_TYPE_PAGE_SHARED,
3273 NULL);
3274 if (ret)
3275 return ret;
3276 }
3277 }
3278
3279 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len,
3280 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT));
3281
3282 rx_ctx.base = (ring->dma / 128);
3283 rx_ctx.qlen = ring->count;
3284
3285
3286 rx_ctx.dsize = 1;
3287
3288
3289
3290
3291 rx_ctx.hsplit_0 = 0;
3292
3293 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len);
3294 if (hw->revision_id == 0)
3295 rx_ctx.lrxqthresh = 0;
3296 else
3297 rx_ctx.lrxqthresh = 1;
3298 rx_ctx.crcstrip = 1;
3299 rx_ctx.l2tsel = 1;
3300
3301 rx_ctx.showiv = 0;
3302
3303 rx_ctx.prefena = 1;
3304
3305
3306 err = i40e_clear_lan_rx_queue_context(hw, pf_q);
3307 if (err) {
3308 dev_info(&vsi->back->pdev->dev,
3309 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3310 ring->queue_index, pf_q, err);
3311 return -ENOMEM;
3312 }
3313
3314
3315 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
3316 if (err) {
3317 dev_info(&vsi->back->pdev->dev,
3318 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n",
3319 ring->queue_index, pf_q, err);
3320 return -ENOMEM;
3321 }
3322
3323
3324 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX))
3325 clear_ring_build_skb_enabled(ring);
3326 else
3327 set_ring_build_skb_enabled(ring);
3328
3329
3330 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
3331 writel(0, ring->tail);
3332
3333 ok = ring->xsk_umem ?
3334 i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)) :
3335 !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring));
3336 if (!ok) {
3337
3338
3339
3340 dev_info(&vsi->back->pdev->dev,
3341 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n",
3342 ring->xsk_umem ? "UMEM enabled " : "",
3343 ring->queue_index, pf_q);
3344 }
3345
3346 return 0;
3347}
3348
3349
3350
3351
3352
3353
3354
3355static int i40e_vsi_configure_tx(struct i40e_vsi *vsi)
3356{
3357 int err = 0;
3358 u16 i;
3359
3360 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3361 err = i40e_configure_tx_ring(vsi->tx_rings[i]);
3362
3363 if (!i40e_enabled_xdp_vsi(vsi))
3364 return err;
3365
3366 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++)
3367 err = i40e_configure_tx_ring(vsi->xdp_rings[i]);
3368
3369 return err;
3370}
3371
3372
3373
3374
3375
3376
3377
3378static int i40e_vsi_configure_rx(struct i40e_vsi *vsi)
3379{
3380 int err = 0;
3381 u16 i;
3382
3383 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) {
3384 vsi->max_frame = I40E_MAX_RXBUFFER;
3385 vsi->rx_buf_len = I40E_RXBUFFER_2048;
3386#if (PAGE_SIZE < 8192)
3387 } else if (!I40E_2K_TOO_SMALL_WITH_PADDING &&
3388 (vsi->netdev->mtu <= ETH_DATA_LEN)) {
3389 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3390 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN;
3391#endif
3392 } else {
3393 vsi->max_frame = I40E_MAX_RXBUFFER;
3394 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 :
3395 I40E_RXBUFFER_2048;
3396 }
3397
3398
3399 for (i = 0; i < vsi->num_queue_pairs && !err; i++)
3400 err = i40e_configure_rx_ring(vsi->rx_rings[i]);
3401
3402 return err;
3403}
3404
3405
3406
3407
3408
3409static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi)
3410{
3411 struct i40e_ring *tx_ring, *rx_ring;
3412 u16 qoffset, qcount;
3413 int i, n;
3414
3415 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) {
3416
3417 for (i = 0; i < vsi->num_queue_pairs; i++) {
3418 rx_ring = vsi->rx_rings[i];
3419 tx_ring = vsi->tx_rings[i];
3420 rx_ring->dcb_tc = 0;
3421 tx_ring->dcb_tc = 0;
3422 }
3423 return;
3424 }
3425
3426 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) {
3427 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n)))
3428 continue;
3429
3430 qoffset = vsi->tc_config.tc_info[n].qoffset;
3431 qcount = vsi->tc_config.tc_info[n].qcount;
3432 for (i = qoffset; i < (qoffset + qcount); i++) {
3433 rx_ring = vsi->rx_rings[i];
3434 tx_ring = vsi->tx_rings[i];
3435 rx_ring->dcb_tc = n;
3436 tx_ring->dcb_tc = n;
3437 }
3438 }
3439}
3440
3441
3442
3443
3444
3445static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi)
3446{
3447 if (vsi->netdev)
3448 i40e_set_rx_mode(vsi->netdev);
3449}
3450
3451
3452
3453
3454
3455
3456
3457
3458static void i40e_fdir_filter_restore(struct i40e_vsi *vsi)
3459{
3460 struct i40e_fdir_filter *filter;
3461 struct i40e_pf *pf = vsi->back;
3462 struct hlist_node *node;
3463
3464 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
3465 return;
3466
3467
3468 pf->fd_tcp4_filter_cnt = 0;
3469 pf->fd_udp4_filter_cnt = 0;
3470 pf->fd_sctp4_filter_cnt = 0;
3471 pf->fd_ip4_filter_cnt = 0;
3472
3473 hlist_for_each_entry_safe(filter, node,
3474 &pf->fdir_filter_list, fdir_node) {
3475 i40e_add_del_fdir(vsi, filter, true);
3476 }
3477}
3478
3479
3480
3481
3482
3483static int i40e_vsi_configure(struct i40e_vsi *vsi)
3484{
3485 int err;
3486
3487 i40e_set_vsi_rx_mode(vsi);
3488 i40e_restore_vlan(vsi);
3489 i40e_vsi_config_dcb_rings(vsi);
3490 err = i40e_vsi_configure_tx(vsi);
3491 if (!err)
3492 err = i40e_vsi_configure_rx(vsi);
3493
3494 return err;
3495}
3496
3497
3498
3499
3500
3501static void i40e_vsi_configure_msix(struct i40e_vsi *vsi)
3502{
3503 bool has_xdp = i40e_enabled_xdp_vsi(vsi);
3504 struct i40e_pf *pf = vsi->back;
3505 struct i40e_hw *hw = &pf->hw;
3506 u16 vector;
3507 int i, q;
3508 u32 qp;
3509
3510
3511
3512
3513
3514 qp = vsi->base_queue;
3515 vector = vsi->base_vector;
3516 for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
3517 struct i40e_q_vector *q_vector = vsi->q_vectors[i];
3518
3519 q_vector->rx.next_update = jiffies + 1;
3520 q_vector->rx.target_itr =
3521 ITR_TO_REG(vsi->rx_rings[i]->itr_setting);
3522 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1),
3523 q_vector->rx.target_itr);
3524 q_vector->rx.current_itr = q_vector->rx.target_itr;
3525
3526 q_vector->tx.next_update = jiffies + 1;
3527 q_vector->tx.target_itr =
3528 ITR_TO_REG(vsi->tx_rings[i]->itr_setting);
3529 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1),
3530 q_vector->tx.target_itr);
3531 q_vector->tx.current_itr = q_vector->tx.target_itr;
3532
3533 wr32(hw, I40E_PFINT_RATEN(vector - 1),
3534 i40e_intrl_usec_to_reg(vsi->int_rate_limit));
3535
3536
3537 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp);
3538 for (q = 0; q < q_vector->num_ringpairs; q++) {
3539 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp;
3540 u32 val;
3541
3542 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3543 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3544 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) |
3545 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) |
3546 (I40E_QUEUE_TYPE_TX <<
3547 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT);
3548
3549 wr32(hw, I40E_QINT_RQCTL(qp), val);
3550
3551 if (has_xdp) {
3552 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3553 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3554 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3555 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3556 (I40E_QUEUE_TYPE_TX <<
3557 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3558
3559 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3560 }
3561
3562 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3563 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3564 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) |
3565 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) |
3566 (I40E_QUEUE_TYPE_RX <<
3567 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3568
3569
3570 if (q == (q_vector->num_ringpairs - 1))
3571 val |= (I40E_QUEUE_END_OF_LIST <<
3572 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3573
3574 wr32(hw, I40E_QINT_TQCTL(qp), val);
3575 qp++;
3576 }
3577 }
3578
3579 i40e_flush(hw);
3580}
3581
3582
3583
3584
3585
3586static void i40e_enable_misc_int_causes(struct i40e_pf *pf)
3587{
3588 struct i40e_hw *hw = &pf->hw;
3589 u32 val;
3590
3591
3592 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3593 rd32(hw, I40E_PFINT_ICR0);
3594
3595 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK |
3596 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK |
3597 I40E_PFINT_ICR0_ENA_GRST_MASK |
3598 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK |
3599 I40E_PFINT_ICR0_ENA_GPIO_MASK |
3600 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK |
3601 I40E_PFINT_ICR0_ENA_VFLR_MASK |
3602 I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3603
3604 if (pf->flags & I40E_FLAG_IWARP_ENABLED)
3605 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3606
3607 if (pf->flags & I40E_FLAG_PTP)
3608 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
3609
3610 wr32(hw, I40E_PFINT_ICR0_ENA, val);
3611
3612
3613 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK |
3614 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK);
3615
3616
3617 wr32(hw, I40E_PFINT_STAT_CTL0, 0);
3618}
3619
3620
3621
3622
3623
3624static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi)
3625{
3626 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0;
3627 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3628 struct i40e_pf *pf = vsi->back;
3629 struct i40e_hw *hw = &pf->hw;
3630 u32 val;
3631
3632
3633 q_vector->rx.next_update = jiffies + 1;
3634 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting);
3635 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr);
3636 q_vector->rx.current_itr = q_vector->rx.target_itr;
3637 q_vector->tx.next_update = jiffies + 1;
3638 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting);
3639 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr);
3640 q_vector->tx.current_itr = q_vector->tx.target_itr;
3641
3642 i40e_enable_misc_int_causes(pf);
3643
3644
3645 wr32(hw, I40E_PFINT_LNKLST0, 0);
3646
3647
3648 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK |
3649 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) |
3650 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)|
3651 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3652
3653 wr32(hw, I40E_QINT_RQCTL(0), val);
3654
3655 if (i40e_enabled_xdp_vsi(vsi)) {
3656 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3657 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)|
3658 (I40E_QUEUE_TYPE_TX
3659 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT);
3660
3661 wr32(hw, I40E_QINT_TQCTL(nextqp), val);
3662 }
3663
3664 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK |
3665 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) |
3666 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT);
3667
3668 wr32(hw, I40E_QINT_TQCTL(0), val);
3669 i40e_flush(hw);
3670}
3671
3672
3673
3674
3675
3676void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf)
3677{
3678 struct i40e_hw *hw = &pf->hw;
3679
3680 wr32(hw, I40E_PFINT_DYN_CTL0,
3681 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT);
3682 i40e_flush(hw);
3683}
3684
3685
3686
3687
3688
3689void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf)
3690{
3691 struct i40e_hw *hw = &pf->hw;
3692 u32 val;
3693
3694 val = I40E_PFINT_DYN_CTL0_INTENA_MASK |
3695 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK |
3696 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT);
3697
3698 wr32(hw, I40E_PFINT_DYN_CTL0, val);
3699 i40e_flush(hw);
3700}
3701
3702
3703
3704
3705
3706
3707static irqreturn_t i40e_msix_clean_rings(int irq, void *data)
3708{
3709 struct i40e_q_vector *q_vector = data;
3710
3711 if (!q_vector->tx.ring && !q_vector->rx.ring)
3712 return IRQ_HANDLED;
3713
3714 napi_schedule_irqoff(&q_vector->napi);
3715
3716 return IRQ_HANDLED;
3717}
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify,
3728 const cpumask_t *mask)
3729{
3730 struct i40e_q_vector *q_vector =
3731 container_of(notify, struct i40e_q_vector, affinity_notify);
3732
3733 cpumask_copy(&q_vector->affinity_mask, mask);
3734}
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744static void i40e_irq_affinity_release(struct kref *ref) {}
3745
3746
3747
3748
3749
3750
3751
3752
3753static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename)
3754{
3755 int q_vectors = vsi->num_q_vectors;
3756 struct i40e_pf *pf = vsi->back;
3757 int base = vsi->base_vector;
3758 int rx_int_idx = 0;
3759 int tx_int_idx = 0;
3760 int vector, err;
3761 int irq_num;
3762 int cpu;
3763
3764 for (vector = 0; vector < q_vectors; vector++) {
3765 struct i40e_q_vector *q_vector = vsi->q_vectors[vector];
3766
3767 irq_num = pf->msix_entries[base + vector].vector;
3768
3769 if (q_vector->tx.ring && q_vector->rx.ring) {
3770 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3771 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
3772 tx_int_idx++;
3773 } else if (q_vector->rx.ring) {
3774 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3775 "%s-%s-%d", basename, "rx", rx_int_idx++);
3776 } else if (q_vector->tx.ring) {
3777 snprintf(q_vector->name, sizeof(q_vector->name) - 1,
3778 "%s-%s-%d", basename, "tx", tx_int_idx++);
3779 } else {
3780
3781 continue;
3782 }
3783 err = request_irq(irq_num,
3784 vsi->irq_handler,
3785 0,
3786 q_vector->name,
3787 q_vector);
3788 if (err) {
3789 dev_info(&pf->pdev->dev,
3790 "MSIX request_irq failed, error: %d\n", err);
3791 goto free_queue_irqs;
3792 }
3793
3794
3795 q_vector->affinity_notify.notify = i40e_irq_affinity_notify;
3796 q_vector->affinity_notify.release = i40e_irq_affinity_release;
3797 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
3798
3799
3800
3801
3802
3803
3804 cpu = cpumask_local_spread(q_vector->v_idx, -1);
3805 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu));
3806 }
3807
3808 vsi->irqs_ready = true;
3809 return 0;
3810
3811free_queue_irqs:
3812 while (vector) {
3813 vector--;
3814 irq_num = pf->msix_entries[base + vector].vector;
3815 irq_set_affinity_notifier(irq_num, NULL);
3816 irq_set_affinity_hint(irq_num, NULL);
3817 free_irq(irq_num, &vsi->q_vectors[vector]);
3818 }
3819 return err;
3820}
3821
3822
3823
3824
3825
3826static void i40e_vsi_disable_irq(struct i40e_vsi *vsi)
3827{
3828 struct i40e_pf *pf = vsi->back;
3829 struct i40e_hw *hw = &pf->hw;
3830 int base = vsi->base_vector;
3831 int i;
3832
3833
3834 for (i = 0; i < vsi->num_queue_pairs; i++) {
3835 u32 val;
3836
3837 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
3838 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK;
3839 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
3840
3841 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx));
3842 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK;
3843 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val);
3844
3845 if (!i40e_enabled_xdp_vsi(vsi))
3846 continue;
3847 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0);
3848 }
3849
3850
3851 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3852 for (i = vsi->base_vector;
3853 i < (vsi->num_q_vectors + vsi->base_vector); i++)
3854 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0);
3855
3856 i40e_flush(hw);
3857 for (i = 0; i < vsi->num_q_vectors; i++)
3858 synchronize_irq(pf->msix_entries[i + base].vector);
3859 } else {
3860
3861 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
3862 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
3863 i40e_flush(hw);
3864 synchronize_irq(pf->pdev->irq);
3865 }
3866}
3867
3868
3869
3870
3871
3872static int i40e_vsi_enable_irq(struct i40e_vsi *vsi)
3873{
3874 struct i40e_pf *pf = vsi->back;
3875 int i;
3876
3877 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
3878 for (i = 0; i < vsi->num_q_vectors; i++)
3879 i40e_irq_dynamic_enable(vsi, i);
3880 } else {
3881 i40e_irq_dynamic_enable_icr0(pf);
3882 }
3883
3884 i40e_flush(&pf->hw);
3885 return 0;
3886}
3887
3888
3889
3890
3891
3892static void i40e_free_misc_vector(struct i40e_pf *pf)
3893{
3894
3895 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0);
3896 i40e_flush(&pf->hw);
3897
3898 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) {
3899 synchronize_irq(pf->msix_entries[0].vector);
3900 free_irq(pf->msix_entries[0].vector, pf);
3901 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
3902 }
3903}
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914static irqreturn_t i40e_intr(int irq, void *data)
3915{
3916 struct i40e_pf *pf = (struct i40e_pf *)data;
3917 struct i40e_hw *hw = &pf->hw;
3918 irqreturn_t ret = IRQ_NONE;
3919 u32 icr0, icr0_remaining;
3920 u32 val, ena_mask;
3921
3922 icr0 = rd32(hw, I40E_PFINT_ICR0);
3923 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA);
3924
3925
3926 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0)
3927 goto enable_intr;
3928
3929
3930 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) ||
3931 (icr0 & I40E_PFINT_ICR0_SWINT_MASK))
3932 pf->sw_int_count++;
3933
3934 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
3935 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) {
3936 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK;
3937 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n");
3938 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
3939 }
3940
3941
3942 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) {
3943 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
3944 struct i40e_q_vector *q_vector = vsi->q_vectors[0];
3945
3946
3947
3948
3949
3950
3951
3952 if (!test_bit(__I40E_DOWN, pf->state))
3953 napi_schedule_irqoff(&q_vector->napi);
3954 }
3955
3956 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) {
3957 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
3958 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
3959 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n");
3960 }
3961
3962 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) {
3963 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
3964 set_bit(__I40E_MDD_EVENT_PENDING, pf->state);
3965 }
3966
3967 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) {
3968 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK;
3969 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state);
3970 }
3971
3972 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) {
3973 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
3974 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
3975 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK;
3976 val = rd32(hw, I40E_GLGEN_RSTAT);
3977 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK)
3978 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT;
3979 if (val == I40E_RESET_CORER) {
3980 pf->corer_count++;
3981 } else if (val == I40E_RESET_GLOBR) {
3982 pf->globr_count++;
3983 } else if (val == I40E_RESET_EMPR) {
3984 pf->empr_count++;
3985 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state);
3986 }
3987 }
3988
3989 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) {
3990 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK;
3991 dev_info(&pf->pdev->dev, "HMC error interrupt\n");
3992 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n",
3993 rd32(hw, I40E_PFHMC_ERRORINFO),
3994 rd32(hw, I40E_PFHMC_ERRORDATA));
3995 }
3996
3997 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) {
3998 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0);
3999
4000 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) {
4001 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK;
4002 i40e_ptp_tx_hwtstamp(pf);
4003 }
4004 }
4005
4006
4007
4008
4009
4010 icr0_remaining = icr0 & ena_mask;
4011 if (icr0_remaining) {
4012 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n",
4013 icr0_remaining);
4014 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) ||
4015 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) ||
4016 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) {
4017 dev_info(&pf->pdev->dev, "device will be reset\n");
4018 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
4019 i40e_service_event_schedule(pf);
4020 }
4021 ena_mask &= ~icr0_remaining;
4022 }
4023 ret = IRQ_HANDLED;
4024
4025enable_intr:
4026
4027 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask);
4028 if (!test_bit(__I40E_DOWN, pf->state) ||
4029 test_bit(__I40E_RECOVERY_MODE, pf->state)) {
4030 i40e_service_event_schedule(pf);
4031 i40e_irq_dynamic_enable_icr0(pf);
4032 }
4033
4034 return ret;
4035}
4036
4037
4038
4039
4040
4041
4042
4043
4044static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget)
4045{
4046 struct i40e_vsi *vsi = tx_ring->vsi;
4047 u16 i = tx_ring->next_to_clean;
4048 struct i40e_tx_buffer *tx_buf;
4049 struct i40e_tx_desc *tx_desc;
4050
4051 tx_buf = &tx_ring->tx_bi[i];
4052 tx_desc = I40E_TX_DESC(tx_ring, i);
4053 i -= tx_ring->count;
4054
4055 do {
4056 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
4057
4058
4059 if (!eop_desc)
4060 break;
4061
4062
4063 smp_rmb();
4064
4065
4066 if (!(eop_desc->cmd_type_offset_bsz &
4067 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
4068 break;
4069
4070
4071 tx_buf->next_to_watch = NULL;
4072
4073 tx_desc->buffer_addr = 0;
4074 tx_desc->cmd_type_offset_bsz = 0;
4075
4076 tx_buf++;
4077 tx_desc++;
4078 i++;
4079 if (unlikely(!i)) {
4080 i -= tx_ring->count;
4081 tx_buf = tx_ring->tx_bi;
4082 tx_desc = I40E_TX_DESC(tx_ring, 0);
4083 }
4084
4085 dma_unmap_single(tx_ring->dev,
4086 dma_unmap_addr(tx_buf, dma),
4087 dma_unmap_len(tx_buf, len),
4088 DMA_TO_DEVICE);
4089 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB)
4090 kfree(tx_buf->raw_buf);
4091
4092 tx_buf->raw_buf = NULL;
4093 tx_buf->tx_flags = 0;
4094 tx_buf->next_to_watch = NULL;
4095 dma_unmap_len_set(tx_buf, len, 0);
4096 tx_desc->buffer_addr = 0;
4097 tx_desc->cmd_type_offset_bsz = 0;
4098
4099
4100 tx_buf++;
4101 tx_desc++;
4102 i++;
4103 if (unlikely(!i)) {
4104 i -= tx_ring->count;
4105 tx_buf = tx_ring->tx_bi;
4106 tx_desc = I40E_TX_DESC(tx_ring, 0);
4107 }
4108
4109
4110 budget--;
4111 } while (likely(budget));
4112
4113 i += tx_ring->count;
4114 tx_ring->next_to_clean = i;
4115
4116 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED)
4117 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx);
4118
4119 return budget > 0;
4120}
4121
4122
4123
4124
4125
4126
4127static irqreturn_t i40e_fdir_clean_ring(int irq, void *data)
4128{
4129 struct i40e_q_vector *q_vector = data;
4130 struct i40e_vsi *vsi;
4131
4132 if (!q_vector->tx.ring)
4133 return IRQ_HANDLED;
4134
4135 vsi = q_vector->tx.ring->vsi;
4136 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit);
4137
4138 return IRQ_HANDLED;
4139}
4140
4141
4142
4143
4144
4145
4146
4147static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx)
4148{
4149 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4150 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx];
4151 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx];
4152
4153 tx_ring->q_vector = q_vector;
4154 tx_ring->next = q_vector->tx.ring;
4155 q_vector->tx.ring = tx_ring;
4156 q_vector->tx.count++;
4157
4158
4159 if (i40e_enabled_xdp_vsi(vsi)) {
4160 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx];
4161
4162 xdp_ring->q_vector = q_vector;
4163 xdp_ring->next = q_vector->tx.ring;
4164 q_vector->tx.ring = xdp_ring;
4165 q_vector->tx.count++;
4166 }
4167
4168 rx_ring->q_vector = q_vector;
4169 rx_ring->next = q_vector->rx.ring;
4170 q_vector->rx.ring = rx_ring;
4171 q_vector->rx.count++;
4172}
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi)
4184{
4185 int qp_remaining = vsi->num_queue_pairs;
4186 int q_vectors = vsi->num_q_vectors;
4187 int num_ringpairs;
4188 int v_start = 0;
4189 int qp_idx = 0;
4190
4191
4192
4193
4194
4195
4196
4197
4198 for (; v_start < q_vectors; v_start++) {
4199 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start];
4200
4201 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start);
4202
4203 q_vector->num_ringpairs = num_ringpairs;
4204 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1;
4205
4206 q_vector->rx.count = 0;
4207 q_vector->tx.count = 0;
4208 q_vector->rx.ring = NULL;
4209 q_vector->tx.ring = NULL;
4210
4211 while (num_ringpairs--) {
4212 i40e_map_vector_to_qp(vsi, v_start, qp_idx);
4213 qp_idx++;
4214 qp_remaining--;
4215 }
4216 }
4217}
4218
4219
4220
4221
4222
4223
4224static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename)
4225{
4226 struct i40e_pf *pf = vsi->back;
4227 int err;
4228
4229 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
4230 err = i40e_vsi_request_irq_msix(vsi, basename);
4231 else if (pf->flags & I40E_FLAG_MSI_ENABLED)
4232 err = request_irq(pf->pdev->irq, i40e_intr, 0,
4233 pf->int_name, pf);
4234 else
4235 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED,
4236 pf->int_name, pf);
4237
4238 if (err)
4239 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err);
4240
4241 return err;
4242}
4243
4244#ifdef CONFIG_NET_POLL_CONTROLLER
4245
4246
4247
4248
4249
4250
4251
4252static void i40e_netpoll(struct net_device *netdev)
4253{
4254 struct i40e_netdev_priv *np = netdev_priv(netdev);
4255 struct i40e_vsi *vsi = np->vsi;
4256 struct i40e_pf *pf = vsi->back;
4257 int i;
4258
4259
4260 if (test_bit(__I40E_VSI_DOWN, vsi->state))
4261 return;
4262
4263 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4264 for (i = 0; i < vsi->num_q_vectors; i++)
4265 i40e_msix_clean_rings(0, vsi->q_vectors[i]);
4266 } else {
4267 i40e_intr(pf->pdev->irq, netdev);
4268 }
4269}
4270#endif
4271
4272#define I40E_QTX_ENA_WAIT_COUNT 50
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4286{
4287 int i;
4288 u32 tx_reg;
4289
4290 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4291 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q));
4292 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4293 break;
4294
4295 usleep_range(10, 20);
4296 }
4297 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4298 return -ETIMEDOUT;
4299
4300 return 0;
4301}
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable)
4314{
4315 struct i40e_hw *hw = &pf->hw;
4316 u32 tx_reg;
4317 int i;
4318
4319
4320 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable);
4321 if (!enable)
4322 usleep_range(10, 20);
4323
4324 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4325 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q));
4326 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) ==
4327 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1))
4328 break;
4329 usleep_range(1000, 2000);
4330 }
4331
4332
4333 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK))
4334 return;
4335
4336
4337 if (enable) {
4338 wr32(hw, I40E_QTX_HEAD(pf_q), 0);
4339 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK;
4340 } else {
4341 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK;
4342 }
4343
4344 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg);
4345}
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q,
4356 bool is_xdp, bool enable)
4357{
4358 int ret;
4359
4360 i40e_control_tx_q(pf, pf_q, enable);
4361
4362
4363 ret = i40e_pf_txq_wait(pf, pf_q, enable);
4364 if (ret) {
4365 dev_info(&pf->pdev->dev,
4366 "VSI seid %d %sTx ring %d %sable timeout\n",
4367 seid, (is_xdp ? "XDP " : ""), pf_q,
4368 (enable ? "en" : "dis"));
4369 }
4370
4371 return ret;
4372}
4373
4374
4375
4376
4377
4378
4379static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable)
4380{
4381 struct i40e_pf *pf = vsi->back;
4382 int i, pf_q, ret = 0;
4383
4384 pf_q = vsi->base_queue;
4385 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4386 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4387 pf_q,
4388 false , enable);
4389 if (ret)
4390 break;
4391
4392 if (!i40e_enabled_xdp_vsi(vsi))
4393 continue;
4394
4395 ret = i40e_control_wait_tx_q(vsi->seid, pf,
4396 pf_q + vsi->alloc_queue_pairs,
4397 true , enable);
4398 if (ret)
4399 break;
4400 }
4401 return ret;
4402}
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable)
4416{
4417 int i;
4418 u32 rx_reg;
4419
4420 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) {
4421 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q));
4422 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4423 break;
4424
4425 usleep_range(10, 20);
4426 }
4427 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT)
4428 return -ETIMEDOUT;
4429
4430 return 0;
4431}
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4444{
4445 struct i40e_hw *hw = &pf->hw;
4446 u32 rx_reg;
4447 int i;
4448
4449 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) {
4450 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q));
4451 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) ==
4452 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1))
4453 break;
4454 usleep_range(1000, 2000);
4455 }
4456
4457
4458 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK))
4459 return;
4460
4461
4462 if (enable)
4463 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK;
4464 else
4465 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK;
4466
4467 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg);
4468}
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable)
4481{
4482 int ret = 0;
4483
4484 i40e_control_rx_q(pf, pf_q, enable);
4485
4486
4487 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
4488 if (ret)
4489 return ret;
4490
4491 return ret;
4492}
4493
4494
4495
4496
4497
4498
4499static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable)
4500{
4501 struct i40e_pf *pf = vsi->back;
4502 int i, pf_q, ret = 0;
4503
4504 pf_q = vsi->base_queue;
4505 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4506 ret = i40e_control_wait_rx_q(pf, pf_q, enable);
4507 if (ret) {
4508 dev_info(&pf->pdev->dev,
4509 "VSI seid %d Rx ring %d %sable timeout\n",
4510 vsi->seid, pf_q, (enable ? "en" : "dis"));
4511 break;
4512 }
4513 }
4514
4515
4516
4517
4518 if (!enable)
4519 mdelay(50);
4520
4521 return ret;
4522}
4523
4524
4525
4526
4527
4528int i40e_vsi_start_rings(struct i40e_vsi *vsi)
4529{
4530 int ret = 0;
4531
4532
4533 ret = i40e_vsi_control_rx(vsi, true);
4534 if (ret)
4535 return ret;
4536 ret = i40e_vsi_control_tx(vsi, true);
4537
4538 return ret;
4539}
4540
4541
4542
4543
4544
4545void i40e_vsi_stop_rings(struct i40e_vsi *vsi)
4546{
4547
4548 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state))
4549 return i40e_vsi_stop_rings_no_wait(vsi);
4550
4551
4552
4553
4554 i40e_vsi_control_tx(vsi, false);
4555 i40e_vsi_control_rx(vsi, false);
4556}
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi)
4570{
4571 struct i40e_pf *pf = vsi->back;
4572 int i, pf_q;
4573
4574 pf_q = vsi->base_queue;
4575 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4576 i40e_control_tx_q(pf, pf_q, false);
4577 i40e_control_rx_q(pf, pf_q, false);
4578 }
4579}
4580
4581
4582
4583
4584
4585static void i40e_vsi_free_irq(struct i40e_vsi *vsi)
4586{
4587 struct i40e_pf *pf = vsi->back;
4588 struct i40e_hw *hw = &pf->hw;
4589 int base = vsi->base_vector;
4590 u32 val, qp;
4591 int i;
4592
4593 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4594 if (!vsi->q_vectors)
4595 return;
4596
4597 if (!vsi->irqs_ready)
4598 return;
4599
4600 vsi->irqs_ready = false;
4601 for (i = 0; i < vsi->num_q_vectors; i++) {
4602 int irq_num;
4603 u16 vector;
4604
4605 vector = i + base;
4606 irq_num = pf->msix_entries[vector].vector;
4607
4608
4609 if (!vsi->q_vectors[i] ||
4610 !vsi->q_vectors[i]->num_ringpairs)
4611 continue;
4612
4613
4614 irq_set_affinity_notifier(irq_num, NULL);
4615
4616 irq_set_affinity_hint(irq_num, NULL);
4617 synchronize_irq(irq_num);
4618 free_irq(irq_num, vsi->q_vectors[i]);
4619
4620
4621
4622
4623
4624
4625
4626
4627 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1));
4628 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4629 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4630 val |= I40E_QUEUE_END_OF_LIST
4631 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4632 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val);
4633
4634 while (qp != I40E_QUEUE_END_OF_LIST) {
4635 u32 next;
4636
4637 val = rd32(hw, I40E_QINT_RQCTL(qp));
4638
4639 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
4640 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4641 I40E_QINT_RQCTL_CAUSE_ENA_MASK |
4642 I40E_QINT_RQCTL_INTEVENT_MASK);
4643
4644 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4645 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4646
4647 wr32(hw, I40E_QINT_RQCTL(qp), val);
4648
4649 val = rd32(hw, I40E_QINT_TQCTL(qp));
4650
4651 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK)
4652 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT;
4653
4654 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
4655 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4656 I40E_QINT_TQCTL_CAUSE_ENA_MASK |
4657 I40E_QINT_TQCTL_INTEVENT_MASK);
4658
4659 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4660 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4661
4662 wr32(hw, I40E_QINT_TQCTL(qp), val);
4663 qp = next;
4664 }
4665 }
4666 } else {
4667 free_irq(pf->pdev->irq, pf);
4668
4669 val = rd32(hw, I40E_PFINT_LNKLST0);
4670 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK)
4671 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT;
4672 val |= I40E_QUEUE_END_OF_LIST
4673 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT;
4674 wr32(hw, I40E_PFINT_LNKLST0, val);
4675
4676 val = rd32(hw, I40E_QINT_RQCTL(qp));
4677 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK |
4678 I40E_QINT_RQCTL_MSIX0_INDX_MASK |
4679 I40E_QINT_RQCTL_CAUSE_ENA_MASK |
4680 I40E_QINT_RQCTL_INTEVENT_MASK);
4681
4682 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK |
4683 I40E_QINT_RQCTL_NEXTQ_INDX_MASK);
4684
4685 wr32(hw, I40E_QINT_RQCTL(qp), val);
4686
4687 val = rd32(hw, I40E_QINT_TQCTL(qp));
4688
4689 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK |
4690 I40E_QINT_TQCTL_MSIX0_INDX_MASK |
4691 I40E_QINT_TQCTL_CAUSE_ENA_MASK |
4692 I40E_QINT_TQCTL_INTEVENT_MASK);
4693
4694 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK |
4695 I40E_QINT_TQCTL_NEXTQ_INDX_MASK);
4696
4697 wr32(hw, I40E_QINT_TQCTL(qp), val);
4698 }
4699}
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx)
4711{
4712 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx];
4713 struct i40e_ring *ring;
4714
4715 if (!q_vector)
4716 return;
4717
4718
4719 i40e_for_each_ring(ring, q_vector->tx)
4720 ring->q_vector = NULL;
4721
4722 i40e_for_each_ring(ring, q_vector->rx)
4723 ring->q_vector = NULL;
4724
4725
4726 if (vsi->netdev)
4727 netif_napi_del(&q_vector->napi);
4728
4729 vsi->q_vectors[v_idx] = NULL;
4730
4731 kfree_rcu(q_vector, rcu);
4732}
4733
4734
4735
4736
4737
4738
4739
4740
4741static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi)
4742{
4743 int v_idx;
4744
4745 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
4746 i40e_free_q_vector(vsi, v_idx);
4747}
4748
4749
4750
4751
4752
4753static void i40e_reset_interrupt_capability(struct i40e_pf *pf)
4754{
4755
4756 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
4757 pci_disable_msix(pf->pdev);
4758 kfree(pf->msix_entries);
4759 pf->msix_entries = NULL;
4760 kfree(pf->irq_pile);
4761 pf->irq_pile = NULL;
4762 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) {
4763 pci_disable_msi(pf->pdev);
4764 }
4765 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
4766}
4767
4768
4769
4770
4771
4772
4773
4774
4775static void i40e_clear_interrupt_scheme(struct i40e_pf *pf)
4776{
4777 int i;
4778
4779 i40e_free_misc_vector(pf);
4780
4781 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector,
4782 I40E_IWARP_IRQ_PILE_ID);
4783
4784 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1);
4785 for (i = 0; i < pf->num_alloc_vsi; i++)
4786 if (pf->vsi[i])
4787 i40e_vsi_free_q_vectors(pf->vsi[i]);
4788 i40e_reset_interrupt_capability(pf);
4789}
4790
4791
4792
4793
4794
4795static void i40e_napi_enable_all(struct i40e_vsi *vsi)
4796{
4797 int q_idx;
4798
4799 if (!vsi->netdev)
4800 return;
4801
4802 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4803 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4804
4805 if (q_vector->rx.ring || q_vector->tx.ring)
4806 napi_enable(&q_vector->napi);
4807 }
4808}
4809
4810
4811
4812
4813
4814static void i40e_napi_disable_all(struct i40e_vsi *vsi)
4815{
4816 int q_idx;
4817
4818 if (!vsi->netdev)
4819 return;
4820
4821 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) {
4822 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx];
4823
4824 if (q_vector->rx.ring || q_vector->tx.ring)
4825 napi_disable(&q_vector->napi);
4826 }
4827}
4828
4829
4830
4831
4832
4833static void i40e_vsi_close(struct i40e_vsi *vsi)
4834{
4835 struct i40e_pf *pf = vsi->back;
4836 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state))
4837 i40e_down(vsi);
4838 i40e_vsi_free_irq(vsi);
4839 i40e_vsi_free_tx_resources(vsi);
4840 i40e_vsi_free_rx_resources(vsi);
4841 vsi->current_netdev_flags = 0;
4842 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
4843 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
4844 set_bit(__I40E_CLIENT_RESET, pf->state);
4845}
4846
4847
4848
4849
4850
4851static void i40e_quiesce_vsi(struct i40e_vsi *vsi)
4852{
4853 if (test_bit(__I40E_VSI_DOWN, vsi->state))
4854 return;
4855
4856 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state);
4857 if (vsi->netdev && netif_running(vsi->netdev))
4858 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
4859 else
4860 i40e_vsi_close(vsi);
4861}
4862
4863
4864
4865
4866
4867static void i40e_unquiesce_vsi(struct i40e_vsi *vsi)
4868{
4869 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state))
4870 return;
4871
4872 if (vsi->netdev && netif_running(vsi->netdev))
4873 vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
4874 else
4875 i40e_vsi_open(vsi);
4876}
4877
4878
4879
4880
4881
4882static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf)
4883{
4884 int v;
4885
4886 for (v = 0; v < pf->num_alloc_vsi; v++) {
4887 if (pf->vsi[v])
4888 i40e_quiesce_vsi(pf->vsi[v]);
4889 }
4890}
4891
4892
4893
4894
4895
4896static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf)
4897{
4898 int v;
4899
4900 for (v = 0; v < pf->num_alloc_vsi; v++) {
4901 if (pf->vsi[v])
4902 i40e_unquiesce_vsi(pf->vsi[v]);
4903 }
4904}
4905
4906
4907
4908
4909
4910
4911
4912int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi)
4913{
4914 struct i40e_pf *pf = vsi->back;
4915 int i, pf_q, ret;
4916
4917 pf_q = vsi->base_queue;
4918 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) {
4919
4920 ret = i40e_pf_txq_wait(pf, pf_q, false);
4921 if (ret) {
4922 dev_info(&pf->pdev->dev,
4923 "VSI seid %d Tx ring %d disable timeout\n",
4924 vsi->seid, pf_q);
4925 return ret;
4926 }
4927
4928 if (!i40e_enabled_xdp_vsi(vsi))
4929 goto wait_rx;
4930
4931
4932 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs,
4933 false);
4934 if (ret) {
4935 dev_info(&pf->pdev->dev,
4936 "VSI seid %d XDP Tx ring %d disable timeout\n",
4937 vsi->seid, pf_q);
4938 return ret;
4939 }
4940wait_rx:
4941
4942 ret = i40e_pf_rxq_wait(pf, pf_q, false);
4943 if (ret) {
4944 dev_info(&pf->pdev->dev,
4945 "VSI seid %d Rx ring %d disable timeout\n",
4946 vsi->seid, pf_q);
4947 return ret;
4948 }
4949 }
4950
4951 return 0;
4952}
4953
4954#ifdef CONFIG_I40E_DCB
4955
4956
4957
4958
4959
4960
4961
4962static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf)
4963{
4964 int v, ret = 0;
4965
4966 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) {
4967 if (pf->vsi[v]) {
4968 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]);
4969 if (ret)
4970 break;
4971 }
4972 }
4973
4974 return ret;
4975}
4976
4977#endif
4978
4979
4980
4981
4982
4983
4984
4985
4986static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf)
4987{
4988 struct i40e_dcb_app_priority_table app;
4989 struct i40e_hw *hw = &pf->hw;
4990 u8 enabled_tc = 1;
4991 u8 tc, i;
4992
4993 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
4994
4995 for (i = 0; i < dcbcfg->numapps; i++) {
4996 app = dcbcfg->app[i];
4997 if (app.selector == I40E_APP_SEL_TCPIP &&
4998 app.protocolid == I40E_APP_PROTOID_ISCSI) {
4999 tc = dcbcfg->etscfg.prioritytable[app.priority];
5000 enabled_tc |= BIT(tc);
5001 break;
5002 }
5003 }
5004
5005 return enabled_tc;
5006}
5007
5008
5009
5010
5011
5012
5013
5014static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg)
5015{
5016 int i, tc_unused = 0;
5017 u8 num_tc = 0;
5018 u8 ret = 0;
5019
5020
5021
5022
5023
5024 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++)
5025 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]);
5026
5027
5028
5029
5030 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5031 if (num_tc & BIT(i)) {
5032 if (!tc_unused) {
5033 ret++;
5034 } else {
5035 pr_err("Non-contiguous TC - Disabling DCB\n");
5036 return 1;
5037 }
5038 } else {
5039 tc_unused = 1;
5040 }
5041 }
5042
5043
5044 if (!ret)
5045 ret = 1;
5046
5047 return ret;
5048}
5049
5050
5051
5052
5053
5054
5055
5056
5057static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg)
5058{
5059 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg);
5060 u8 enabled_tc = 1;
5061 u8 i;
5062
5063 for (i = 0; i < num_tc; i++)
5064 enabled_tc |= BIT(i);
5065
5066 return enabled_tc;
5067}
5068
5069
5070
5071
5072
5073
5074
5075
5076static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf)
5077{
5078 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
5079 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc;
5080 u8 enabled_tc = 1, i;
5081
5082 for (i = 1; i < num_tc; i++)
5083 enabled_tc |= BIT(i);
5084 return enabled_tc;
5085}
5086
5087
5088
5089
5090
5091
5092
5093static u8 i40e_pf_get_num_tc(struct i40e_pf *pf)
5094{
5095 struct i40e_hw *hw = &pf->hw;
5096 u8 i, enabled_tc = 1;
5097 u8 num_tc = 0;
5098 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5099
5100 if (pf->flags & I40E_FLAG_TC_MQPRIO)
5101 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc;
5102
5103
5104 if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5105 return 1;
5106
5107
5108 if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5109 return i40e_dcb_get_num_tc(dcbcfg);
5110
5111
5112 if (pf->hw.func_caps.iscsi)
5113 enabled_tc = i40e_get_iscsi_tc_map(pf);
5114 else
5115 return 1;
5116
5117 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5118 if (enabled_tc & BIT(i))
5119 num_tc++;
5120 }
5121 return num_tc;
5122}
5123
5124
5125
5126
5127
5128
5129
5130static u8 i40e_pf_get_tc_map(struct i40e_pf *pf)
5131{
5132 if (pf->flags & I40E_FLAG_TC_MQPRIO)
5133 return i40e_mqprio_get_enabled_tc(pf);
5134
5135
5136
5137
5138 if (!(pf->flags & I40E_FLAG_DCB_ENABLED))
5139 return I40E_DEFAULT_TRAFFIC_CLASS;
5140
5141
5142 if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
5143 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config);
5144
5145
5146 if (pf->hw.func_caps.iscsi)
5147 return i40e_get_iscsi_tc_map(pf);
5148 else
5149 return I40E_DEFAULT_TRAFFIC_CLASS;
5150}
5151
5152
5153
5154
5155
5156
5157
5158static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi)
5159{
5160 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0};
5161 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5162 struct i40e_pf *pf = vsi->back;
5163 struct i40e_hw *hw = &pf->hw;
5164 i40e_status ret;
5165 u32 tc_bw_max;
5166 int i;
5167
5168
5169 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL);
5170 if (ret) {
5171 dev_info(&pf->pdev->dev,
5172 "couldn't get PF vsi bw config, err %s aq_err %s\n",
5173 i40e_stat_str(&pf->hw, ret),
5174 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5175 return -EINVAL;
5176 }
5177
5178
5179 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config,
5180 NULL);
5181 if (ret) {
5182 dev_info(&pf->pdev->dev,
5183 "couldn't get PF vsi ets bw config, err %s aq_err %s\n",
5184 i40e_stat_str(&pf->hw, ret),
5185 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5186 return -EINVAL;
5187 }
5188
5189 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) {
5190 dev_info(&pf->pdev->dev,
5191 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n",
5192 bw_config.tc_valid_bits,
5193 bw_ets_config.tc_valid_bits);
5194
5195 }
5196
5197 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit);
5198 vsi->bw_max_quanta = bw_config.max_bw;
5199 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) |
5200 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16);
5201 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5202 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i];
5203 vsi->bw_ets_limit_credits[i] =
5204 le16_to_cpu(bw_ets_config.credits[i]);
5205
5206 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7);
5207 }
5208
5209 return 0;
5210}
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc,
5221 u8 *bw_share)
5222{
5223 struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5224 struct i40e_pf *pf = vsi->back;
5225 i40e_status ret;
5226 int i;
5227
5228
5229 if (pf->flags & I40E_FLAG_TC_MQPRIO)
5230 return 0;
5231 if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) {
5232 ret = i40e_set_bw_limit(vsi, vsi->seid, 0);
5233 if (ret)
5234 dev_info(&pf->pdev->dev,
5235 "Failed to reset tx rate for vsi->seid %u\n",
5236 vsi->seid);
5237 return ret;
5238 }
5239 bw_data.tc_valid_bits = enabled_tc;
5240 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5241 bw_data.tc_bw_credits[i] = bw_share[i];
5242
5243 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL);
5244 if (ret) {
5245 dev_info(&pf->pdev->dev,
5246 "AQ command Config VSI BW allocation per TC failed = %d\n",
5247 pf->hw.aq.asq_last_status);
5248 return -EINVAL;
5249 }
5250
5251 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5252 vsi->info.qs_handle[i] = bw_data.qs_handles[i];
5253
5254 return 0;
5255}
5256
5257
5258
5259
5260
5261
5262
5263static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5264{
5265 struct net_device *netdev = vsi->netdev;
5266 struct i40e_pf *pf = vsi->back;
5267 struct i40e_hw *hw = &pf->hw;
5268 u8 netdev_tc = 0;
5269 int i;
5270 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config;
5271
5272 if (!netdev)
5273 return;
5274
5275 if (!enabled_tc) {
5276 netdev_reset_tc(netdev);
5277 return;
5278 }
5279
5280
5281 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc))
5282 return;
5283
5284
5285 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5286
5287
5288
5289
5290
5291
5292
5293 if (vsi->tc_config.enabled_tc & BIT(i))
5294 netdev_set_tc_queue(netdev,
5295 vsi->tc_config.tc_info[i].netdev_tc,
5296 vsi->tc_config.tc_info[i].qcount,
5297 vsi->tc_config.tc_info[i].qoffset);
5298 }
5299
5300 if (pf->flags & I40E_FLAG_TC_MQPRIO)
5301 return;
5302
5303
5304 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) {
5305
5306 u8 ets_tc = dcbcfg->etscfg.prioritytable[i];
5307
5308 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc;
5309 netdev_set_prio_tc_map(netdev, i, netdev_tc);
5310 }
5311}
5312
5313
5314
5315
5316
5317
5318static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi,
5319 struct i40e_vsi_context *ctxt)
5320{
5321
5322
5323
5324
5325 vsi->info.mapping_flags = ctxt->info.mapping_flags;
5326 memcpy(&vsi->info.queue_mapping,
5327 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping));
5328 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping,
5329 sizeof(vsi->info.tc_mapping));
5330}
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc)
5346{
5347 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5348 struct i40e_pf *pf = vsi->back;
5349 struct i40e_hw *hw = &pf->hw;
5350 struct i40e_vsi_context ctxt;
5351 int ret = 0;
5352 int i;
5353
5354
5355 if (vsi->tc_config.enabled_tc == enabled_tc &&
5356 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL)
5357 return ret;
5358
5359
5360 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5361 if (enabled_tc & BIT(i))
5362 bw_share[i] = 1;
5363 }
5364
5365 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5366 if (ret) {
5367 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0};
5368
5369 dev_info(&pf->pdev->dev,
5370 "Failed configuring TC map %d for VSI %d\n",
5371 enabled_tc, vsi->seid);
5372 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid,
5373 &bw_config, NULL);
5374 if (ret) {
5375 dev_info(&pf->pdev->dev,
5376 "Failed querying vsi bw info, err %s aq_err %s\n",
5377 i40e_stat_str(hw, ret),
5378 i40e_aq_str(hw, hw->aq.asq_last_status));
5379 goto out;
5380 }
5381 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) {
5382 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc;
5383
5384 if (!valid_tc)
5385 valid_tc = bw_config.tc_valid_bits;
5386
5387 valid_tc |= 1;
5388 dev_info(&pf->pdev->dev,
5389 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n",
5390 enabled_tc, bw_config.tc_valid_bits, valid_tc);
5391 enabled_tc = valid_tc;
5392 }
5393
5394 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share);
5395 if (ret) {
5396 dev_err(&pf->pdev->dev,
5397 "Unable to configure TC map %d for VSI %d\n",
5398 enabled_tc, vsi->seid);
5399 goto out;
5400 }
5401 }
5402
5403
5404 ctxt.seid = vsi->seid;
5405 ctxt.pf_num = vsi->back->hw.pf_id;
5406 ctxt.vf_num = 0;
5407 ctxt.uplink_seid = vsi->uplink_seid;
5408 ctxt.info = vsi->info;
5409 if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) {
5410 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc);
5411 if (ret)
5412 goto out;
5413 } else {
5414 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
5415 }
5416
5417
5418
5419
5420 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) {
5421 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size,
5422 vsi->num_queue_pairs);
5423 ret = i40e_vsi_config_rss(vsi);
5424 if (ret) {
5425 dev_info(&vsi->back->pdev->dev,
5426 "Failed to reconfig rss for num_queues\n");
5427 return ret;
5428 }
5429 vsi->reconfig_rss = false;
5430 }
5431 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
5432 ctxt.info.valid_sections |=
5433 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
5434 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA;
5435 }
5436
5437
5438
5439
5440 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
5441 if (ret) {
5442 dev_info(&pf->pdev->dev,
5443 "Update vsi tc config failed, err %s aq_err %s\n",
5444 i40e_stat_str(hw, ret),
5445 i40e_aq_str(hw, hw->aq.asq_last_status));
5446 goto out;
5447 }
5448
5449 i40e_vsi_update_queue_map(vsi, &ctxt);
5450 vsi->info.valid_sections = 0;
5451
5452
5453 ret = i40e_vsi_get_bw_info(vsi);
5454 if (ret) {
5455 dev_info(&pf->pdev->dev,
5456 "Failed updating vsi bw info, err %s aq_err %s\n",
5457 i40e_stat_str(hw, ret),
5458 i40e_aq_str(hw, hw->aq.asq_last_status));
5459 goto out;
5460 }
5461
5462
5463 i40e_vsi_config_netdev_tc(vsi, enabled_tc);
5464out:
5465 return ret;
5466}
5467
5468
5469
5470
5471
5472
5473static int i40e_get_link_speed(struct i40e_vsi *vsi)
5474{
5475 struct i40e_pf *pf = vsi->back;
5476
5477 switch (pf->hw.phy.link_info.link_speed) {
5478 case I40E_LINK_SPEED_40GB:
5479 return 40000;
5480 case I40E_LINK_SPEED_25GB:
5481 return 25000;
5482 case I40E_LINK_SPEED_20GB:
5483 return 20000;
5484 case I40E_LINK_SPEED_10GB:
5485 return 10000;
5486 case I40E_LINK_SPEED_1GB:
5487 return 1000;
5488 default:
5489 return -EINVAL;
5490 }
5491}
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate)
5502{
5503 struct i40e_pf *pf = vsi->back;
5504 u64 credits = 0;
5505 int speed = 0;
5506 int ret = 0;
5507
5508 speed = i40e_get_link_speed(vsi);
5509 if (max_tx_rate > speed) {
5510 dev_err(&pf->pdev->dev,
5511 "Invalid max tx rate %llu specified for VSI seid %d.",
5512 max_tx_rate, seid);
5513 return -EINVAL;
5514 }
5515 if (max_tx_rate && max_tx_rate < 50) {
5516 dev_warn(&pf->pdev->dev,
5517 "Setting max tx rate to minimum usable value of 50Mbps.\n");
5518 max_tx_rate = 50;
5519 }
5520
5521
5522 credits = max_tx_rate;
5523 do_div(credits, I40E_BW_CREDIT_DIVISOR);
5524 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits,
5525 I40E_MAX_BW_INACTIVE_ACCUM, NULL);
5526 if (ret)
5527 dev_err(&pf->pdev->dev,
5528 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n",
5529 max_tx_rate, seid, i40e_stat_str(&pf->hw, ret),
5530 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
5531 return ret;
5532}
5533
5534
5535
5536
5537
5538
5539
5540static void i40e_remove_queue_channels(struct i40e_vsi *vsi)
5541{
5542 enum i40e_admin_queue_err last_aq_status;
5543 struct i40e_cloud_filter *cfilter;
5544 struct i40e_channel *ch, *ch_tmp;
5545 struct i40e_pf *pf = vsi->back;
5546 struct hlist_node *node;
5547 int ret, i;
5548
5549
5550
5551
5552 vsi->current_rss_size = 0;
5553
5554
5555 if (list_empty(&vsi->ch_list))
5556 return;
5557
5558 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5559 struct i40e_vsi *p_vsi;
5560
5561 list_del(&ch->list);
5562 p_vsi = ch->parent_vsi;
5563 if (!p_vsi || !ch->initialized) {
5564 kfree(ch);
5565 continue;
5566 }
5567
5568 for (i = 0; i < ch->num_queue_pairs; i++) {
5569 struct i40e_ring *tx_ring, *rx_ring;
5570 u16 pf_q;
5571
5572 pf_q = ch->base_queue + i;
5573 tx_ring = vsi->tx_rings[pf_q];
5574 tx_ring->ch = NULL;
5575
5576 rx_ring = vsi->rx_rings[pf_q];
5577 rx_ring->ch = NULL;
5578 }
5579
5580
5581 ret = i40e_set_bw_limit(vsi, ch->seid, 0);
5582 if (ret)
5583 dev_info(&vsi->back->pdev->dev,
5584 "Failed to reset tx rate for ch->seid %u\n",
5585 ch->seid);
5586
5587
5588 hlist_for_each_entry_safe(cfilter, node,
5589 &pf->cloud_filter_list, cloud_node) {
5590 if (cfilter->seid != ch->seid)
5591 continue;
5592
5593 hash_del(&cfilter->cloud_node);
5594 if (cfilter->dst_port)
5595 ret = i40e_add_del_cloud_filter_big_buf(vsi,
5596 cfilter,
5597 false);
5598 else
5599 ret = i40e_add_del_cloud_filter(vsi, cfilter,
5600 false);
5601 last_aq_status = pf->hw.aq.asq_last_status;
5602 if (ret)
5603 dev_info(&pf->pdev->dev,
5604 "Failed to delete cloud filter, err %s aq_err %s\n",
5605 i40e_stat_str(&pf->hw, ret),
5606 i40e_aq_str(&pf->hw, last_aq_status));
5607 kfree(cfilter);
5608 }
5609
5610
5611 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
5612 NULL);
5613 if (ret)
5614 dev_err(&vsi->back->pdev->dev,
5615 "unable to remove channel (%d) for parent VSI(%d)\n",
5616 ch->seid, p_vsi->seid);
5617 kfree(ch);
5618 }
5619 INIT_LIST_HEAD(&vsi->ch_list);
5620}
5621
5622
5623
5624
5625
5626
5627
5628static bool i40e_is_any_channel(struct i40e_vsi *vsi)
5629{
5630 struct i40e_channel *ch, *ch_tmp;
5631
5632 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5633 if (ch->initialized)
5634 return true;
5635 }
5636
5637 return false;
5638}
5639
5640
5641
5642
5643
5644
5645
5646
5647static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi)
5648{
5649 struct i40e_channel *ch, *ch_tmp;
5650 int max = 0;
5651
5652 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
5653 if (!ch->initialized)
5654 continue;
5655 if (ch->num_queue_pairs > max)
5656 max = ch->num_queue_pairs;
5657 }
5658
5659 return max;
5660}
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues,
5674 struct i40e_vsi *vsi, bool *reconfig_rss)
5675{
5676 int max_ch_queues;
5677
5678 if (!reconfig_rss)
5679 return -EINVAL;
5680
5681 *reconfig_rss = false;
5682 if (vsi->current_rss_size) {
5683 if (num_queues > vsi->current_rss_size) {
5684 dev_dbg(&pf->pdev->dev,
5685 "Error: num_queues (%d) > vsi's current_size(%d)\n",
5686 num_queues, vsi->current_rss_size);
5687 return -EINVAL;
5688 } else if ((num_queues < vsi->current_rss_size) &&
5689 (!is_power_of_2(num_queues))) {
5690 dev_dbg(&pf->pdev->dev,
5691 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n",
5692 num_queues, vsi->current_rss_size);
5693 return -EINVAL;
5694 }
5695 }
5696
5697 if (!is_power_of_2(num_queues)) {
5698
5699
5700
5701
5702
5703 max_ch_queues = i40e_get_max_queues_for_channel(vsi);
5704 if (num_queues < max_ch_queues) {
5705 dev_dbg(&pf->pdev->dev,
5706 "Error: num_queues (%d) < max queues configured for channel(%d)\n",
5707 num_queues, max_ch_queues);
5708 return -EINVAL;
5709 }
5710 *reconfig_rss = true;
5711 }
5712
5713 return 0;
5714}
5715
5716
5717
5718
5719
5720
5721
5722
5723static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size)
5724{
5725 struct i40e_pf *pf = vsi->back;
5726 u8 seed[I40E_HKEY_ARRAY_SIZE];
5727 struct i40e_hw *hw = &pf->hw;
5728 int local_rss_size;
5729 u8 *lut;
5730 int ret;
5731
5732 if (!vsi->rss_size)
5733 return -EINVAL;
5734
5735 if (rss_size > vsi->rss_size)
5736 return -EINVAL;
5737
5738 local_rss_size = min_t(int, vsi->rss_size, rss_size);
5739 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
5740 if (!lut)
5741 return -ENOMEM;
5742
5743
5744 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size);
5745
5746
5747
5748
5749 if (vsi->rss_hkey_user)
5750 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
5751 else
5752 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
5753
5754 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
5755 if (ret) {
5756 dev_info(&pf->pdev->dev,
5757 "Cannot set RSS lut, err %s aq_err %s\n",
5758 i40e_stat_str(hw, ret),
5759 i40e_aq_str(hw, hw->aq.asq_last_status));
5760 kfree(lut);
5761 return ret;
5762 }
5763 kfree(lut);
5764
5765
5766 if (!vsi->orig_rss_size)
5767 vsi->orig_rss_size = vsi->rss_size;
5768 vsi->current_rss_size = local_rss_size;
5769
5770 return ret;
5771}
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782static void i40e_channel_setup_queue_map(struct i40e_pf *pf,
5783 struct i40e_vsi_context *ctxt,
5784 struct i40e_channel *ch)
5785{
5786 u16 qcount, qmap, sections = 0;
5787 u8 offset = 0;
5788 int pow;
5789
5790 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
5791 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
5792
5793 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix);
5794 ch->num_queue_pairs = qcount;
5795
5796
5797 pow = ilog2(qcount);
5798 if (!is_power_of_2(qcount))
5799 pow++;
5800
5801 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
5802 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
5803
5804
5805 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap);
5806
5807 ctxt->info.up_enable_bits = 0x1;
5808 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
5809 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue);
5810 ctxt->info.valid_sections |= cpu_to_le16(sections);
5811}
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid,
5822 struct i40e_channel *ch)
5823{
5824 struct i40e_hw *hw = &pf->hw;
5825 struct i40e_vsi_context ctxt;
5826 u8 enabled_tc = 0x1;
5827 int ret;
5828
5829 if (ch->type != I40E_VSI_VMDQ2) {
5830 dev_info(&pf->pdev->dev,
5831 "add new vsi failed, ch->type %d\n", ch->type);
5832 return -EINVAL;
5833 }
5834
5835 memset(&ctxt, 0, sizeof(ctxt));
5836 ctxt.pf_num = hw->pf_id;
5837 ctxt.vf_num = 0;
5838 ctxt.uplink_seid = uplink_seid;
5839 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
5840 if (ch->type == I40E_VSI_VMDQ2)
5841 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
5842
5843 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) {
5844 ctxt.info.valid_sections |=
5845 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
5846 ctxt.info.switch_id =
5847 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
5848 }
5849
5850
5851 i40e_channel_setup_queue_map(pf, &ctxt, ch);
5852
5853
5854 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
5855 if (ret) {
5856 dev_info(&pf->pdev->dev,
5857 "add new vsi failed, err %s aq_err %s\n",
5858 i40e_stat_str(&pf->hw, ret),
5859 i40e_aq_str(&pf->hw,
5860 pf->hw.aq.asq_last_status));
5861 return -ENOENT;
5862 }
5863
5864
5865
5866
5867 ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc;
5868 ch->seid = ctxt.seid;
5869 ch->vsi_number = ctxt.vsi_number;
5870 ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx);
5871
5872
5873
5874
5875
5876 ch->info.mapping_flags = ctxt.info.mapping_flags;
5877 memcpy(&ch->info.queue_mapping,
5878 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping));
5879 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping,
5880 sizeof(ctxt.info.tc_mapping));
5881
5882 return 0;
5883}
5884
5885static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch,
5886 u8 *bw_share)
5887{
5888 struct i40e_aqc_configure_vsi_tc_bw_data bw_data;
5889 i40e_status ret;
5890 int i;
5891
5892 bw_data.tc_valid_bits = ch->enabled_tc;
5893 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5894 bw_data.tc_bw_credits[i] = bw_share[i];
5895
5896 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid,
5897 &bw_data, NULL);
5898 if (ret) {
5899 dev_info(&vsi->back->pdev->dev,
5900 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n",
5901 vsi->back->hw.aq.asq_last_status, ch->seid);
5902 return -EINVAL;
5903 }
5904
5905 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++)
5906 ch->info.qs_handle[i] = bw_data.qs_handles[i];
5907
5908 return 0;
5909}
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920static int i40e_channel_config_tx_ring(struct i40e_pf *pf,
5921 struct i40e_vsi *vsi,
5922 struct i40e_channel *ch)
5923{
5924 i40e_status ret;
5925 int i;
5926 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0};
5927
5928
5929 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
5930 if (ch->enabled_tc & BIT(i))
5931 bw_share[i] = 1;
5932 }
5933
5934
5935 ret = i40e_channel_config_bw(vsi, ch, bw_share);
5936 if (ret) {
5937 dev_info(&vsi->back->pdev->dev,
5938 "Failed configuring TC map %d for channel (seid %u)\n",
5939 ch->enabled_tc, ch->seid);
5940 return ret;
5941 }
5942
5943 for (i = 0; i < ch->num_queue_pairs; i++) {
5944 struct i40e_ring *tx_ring, *rx_ring;
5945 u16 pf_q;
5946
5947 pf_q = ch->base_queue + i;
5948
5949
5950
5951
5952 tx_ring = vsi->tx_rings[pf_q];
5953 tx_ring->ch = ch;
5954
5955
5956 rx_ring = vsi->rx_rings[pf_q];
5957 rx_ring->ch = ch;
5958 }
5959
5960 return 0;
5961}
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974static inline int i40e_setup_hw_channel(struct i40e_pf *pf,
5975 struct i40e_vsi *vsi,
5976 struct i40e_channel *ch,
5977 u16 uplink_seid, u8 type)
5978{
5979 int ret;
5980
5981 ch->initialized = false;
5982 ch->base_queue = vsi->next_base_queue;
5983 ch->type = type;
5984
5985
5986 ret = i40e_add_channel(pf, uplink_seid, ch);
5987 if (ret) {
5988 dev_info(&pf->pdev->dev,
5989 "failed to add_channel using uplink_seid %u\n",
5990 uplink_seid);
5991 return ret;
5992 }
5993
5994
5995 ch->initialized = true;
5996
5997
5998 ret = i40e_channel_config_tx_ring(pf, vsi, ch);
5999 if (ret) {
6000 dev_info(&pf->pdev->dev,
6001 "failed to configure TX rings for channel %u\n",
6002 ch->seid);
6003 return ret;
6004 }
6005
6006
6007 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs;
6008 dev_dbg(&pf->pdev->dev,
6009 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n",
6010 ch->seid, ch->vsi_number, ch->stat_counter_idx,
6011 ch->num_queue_pairs,
6012 vsi->next_base_queue);
6013 return ret;
6014}
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi,
6027 struct i40e_channel *ch)
6028{
6029 u8 vsi_type;
6030 u16 seid;
6031 int ret;
6032
6033 if (vsi->type == I40E_VSI_MAIN) {
6034 vsi_type = I40E_VSI_VMDQ2;
6035 } else {
6036 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n",
6037 vsi->type);
6038 return false;
6039 }
6040
6041
6042 seid = pf->vsi[pf->lan_vsi]->uplink_seid;
6043
6044
6045 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type);
6046 if (ret) {
6047 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n");
6048 return false;
6049 }
6050
6051 return ch->initialized ? true : false;
6052}
6053
6054
6055
6056
6057
6058
6059
6060
6061static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi)
6062{
6063 u8 mode;
6064 struct i40e_pf *pf = vsi->back;
6065 struct i40e_hw *hw = &pf->hw;
6066 int ret;
6067
6068 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities);
6069 if (ret)
6070 return -EINVAL;
6071
6072 if (hw->dev_caps.switch_mode) {
6073
6074
6075
6076 u32 switch_mode = hw->dev_caps.switch_mode &
6077 I40E_SWITCH_MODE_MASK;
6078 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) {
6079 if (switch_mode == I40E_CLOUD_FILTER_MODE2)
6080 return 0;
6081 dev_err(&pf->pdev->dev,
6082 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n",
6083 hw->dev_caps.switch_mode);
6084 return -EINVAL;
6085 }
6086 }
6087
6088
6089 mode = I40E_AQ_SET_SWITCH_BIT7_VALID;
6090
6091
6092 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP;
6093
6094
6095 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL;
6096
6097
6098 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags,
6099 pf->last_sw_conf_valid_flags,
6100 mode, NULL);
6101 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH)
6102 dev_err(&pf->pdev->dev,
6103 "couldn't set switch config bits, err %s aq_err %s\n",
6104 i40e_stat_str(hw, ret),
6105 i40e_aq_str(hw,
6106 hw->aq.asq_last_status));
6107
6108 return ret;
6109}
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119int i40e_create_queue_channel(struct i40e_vsi *vsi,
6120 struct i40e_channel *ch)
6121{
6122 struct i40e_pf *pf = vsi->back;
6123 bool reconfig_rss;
6124 int err;
6125
6126 if (!ch)
6127 return -EINVAL;
6128
6129 if (!ch->num_queue_pairs) {
6130 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n",
6131 ch->num_queue_pairs);
6132 return -EINVAL;
6133 }
6134
6135
6136 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi,
6137 &reconfig_rss);
6138 if (err) {
6139 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n",
6140 ch->num_queue_pairs);
6141 return -EINVAL;
6142 }
6143
6144
6145
6146
6147 if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) ||
6148 (!i40e_is_any_channel(vsi))) {
6149 if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) {
6150 dev_dbg(&pf->pdev->dev,
6151 "Failed to create channel. Override queues (%u) not power of 2\n",
6152 vsi->tc_config.tc_info[0].qcount);
6153 return -EINVAL;
6154 }
6155
6156 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
6157 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
6158
6159 if (vsi->type == I40E_VSI_MAIN) {
6160 if (pf->flags & I40E_FLAG_TC_MQPRIO)
6161 i40e_do_reset(pf, I40E_PF_RESET_FLAG,
6162 true);
6163 else
6164 i40e_do_reset_safe(pf,
6165 I40E_PF_RESET_FLAG);
6166 }
6167 }
6168
6169
6170
6171 }
6172
6173
6174
6175
6176 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) {
6177 dev_dbg(&pf->pdev->dev,
6178 "Error: cnt_q_avail (%u) less than num_queues %d\n",
6179 vsi->cnt_q_avail, ch->num_queue_pairs);
6180 return -EINVAL;
6181 }
6182
6183
6184 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) {
6185 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs);
6186 if (err) {
6187 dev_info(&pf->pdev->dev,
6188 "Error: unable to reconfig rss for num_queues (%u)\n",
6189 ch->num_queue_pairs);
6190 return -EINVAL;
6191 }
6192 }
6193
6194 if (!i40e_setup_channel(pf, vsi, ch)) {
6195 dev_info(&pf->pdev->dev, "Failed to setup channel\n");
6196 return -EINVAL;
6197 }
6198
6199 dev_info(&pf->pdev->dev,
6200 "Setup channel (id:%u) utilizing num_queues %d\n",
6201 ch->seid, ch->num_queue_pairs);
6202
6203
6204 if (ch->max_tx_rate) {
6205 u64 credits = ch->max_tx_rate;
6206
6207 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate))
6208 return -EINVAL;
6209
6210 do_div(credits, I40E_BW_CREDIT_DIVISOR);
6211 dev_dbg(&pf->pdev->dev,
6212 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
6213 ch->max_tx_rate,
6214 credits,
6215 ch->seid);
6216 }
6217
6218
6219 ch->parent_vsi = vsi;
6220
6221
6222 vsi->cnt_q_avail -= ch->num_queue_pairs;
6223
6224 return 0;
6225}
6226
6227
6228
6229
6230
6231
6232
6233static int i40e_configure_queue_channels(struct i40e_vsi *vsi)
6234{
6235 struct i40e_channel *ch;
6236 u64 max_rate = 0;
6237 int ret = 0, i;
6238
6239
6240 vsi->tc_seid_map[0] = vsi->seid;
6241 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6242 if (vsi->tc_config.enabled_tc & BIT(i)) {
6243 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
6244 if (!ch) {
6245 ret = -ENOMEM;
6246 goto err_free;
6247 }
6248
6249 INIT_LIST_HEAD(&ch->list);
6250 ch->num_queue_pairs =
6251 vsi->tc_config.tc_info[i].qcount;
6252 ch->base_queue =
6253 vsi->tc_config.tc_info[i].qoffset;
6254
6255
6256
6257
6258 max_rate = vsi->mqprio_qopt.max_rate[i];
6259 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6260 ch->max_tx_rate = max_rate;
6261
6262 list_add_tail(&ch->list, &vsi->ch_list);
6263
6264 ret = i40e_create_queue_channel(vsi, ch);
6265 if (ret) {
6266 dev_err(&vsi->back->pdev->dev,
6267 "Failed creating queue channel with TC%d: queues %d\n",
6268 i, ch->num_queue_pairs);
6269 goto err_free;
6270 }
6271 vsi->tc_seid_map[i] = ch->seid;
6272 }
6273 }
6274 return ret;
6275
6276err_free:
6277 i40e_remove_queue_channels(vsi);
6278 return ret;
6279}
6280
6281
6282
6283
6284
6285
6286
6287
6288int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc)
6289{
6290 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0};
6291 struct i40e_pf *pf = veb->pf;
6292 int ret = 0;
6293 int i;
6294
6295
6296 if (!enabled_tc || veb->enabled_tc == enabled_tc)
6297 return ret;
6298
6299 bw_data.tc_valid_bits = enabled_tc;
6300
6301
6302
6303 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6304 if (enabled_tc & BIT(i))
6305 bw_data.tc_bw_share_credits[i] = 1;
6306 }
6307
6308 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid,
6309 &bw_data, NULL);
6310 if (ret) {
6311 dev_info(&pf->pdev->dev,
6312 "VEB bw config failed, err %s aq_err %s\n",
6313 i40e_stat_str(&pf->hw, ret),
6314 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6315 goto out;
6316 }
6317
6318
6319 ret = i40e_veb_get_bw_info(veb);
6320 if (ret) {
6321 dev_info(&pf->pdev->dev,
6322 "Failed getting veb bw config, err %s aq_err %s\n",
6323 i40e_stat_str(&pf->hw, ret),
6324 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6325 }
6326
6327out:
6328 return ret;
6329}
6330
6331#ifdef CONFIG_I40E_DCB
6332
6333
6334
6335
6336
6337
6338
6339
6340static void i40e_dcb_reconfigure(struct i40e_pf *pf)
6341{
6342 u8 tc_map = 0;
6343 int ret;
6344 u8 v;
6345
6346
6347 tc_map = i40e_pf_get_tc_map(pf);
6348 for (v = 0; v < I40E_MAX_VEB; v++) {
6349 if (!pf->veb[v])
6350 continue;
6351 ret = i40e_veb_config_tc(pf->veb[v], tc_map);
6352 if (ret) {
6353 dev_info(&pf->pdev->dev,
6354 "Failed configuring TC for VEB seid=%d\n",
6355 pf->veb[v]->seid);
6356
6357 }
6358 }
6359
6360
6361 for (v = 0; v < pf->num_alloc_vsi; v++) {
6362 if (!pf->vsi[v])
6363 continue;
6364
6365
6366
6367
6368 if (v == pf->lan_vsi)
6369 tc_map = i40e_pf_get_tc_map(pf);
6370 else
6371 tc_map = I40E_DEFAULT_TRAFFIC_CLASS;
6372
6373 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map);
6374 if (ret) {
6375 dev_info(&pf->pdev->dev,
6376 "Failed configuring TC for VSI seid=%d\n",
6377 pf->vsi[v]->seid);
6378
6379 } else {
6380
6381 i40e_vsi_map_rings_to_vectors(pf->vsi[v]);
6382 if (pf->vsi[v]->netdev)
6383 i40e_dcbnl_set_all(pf->vsi[v]);
6384 }
6385 }
6386}
6387
6388
6389
6390
6391
6392
6393
6394
6395static int i40e_resume_port_tx(struct i40e_pf *pf)
6396{
6397 struct i40e_hw *hw = &pf->hw;
6398 int ret;
6399
6400 ret = i40e_aq_resume_port_tx(hw, NULL);
6401 if (ret) {
6402 dev_info(&pf->pdev->dev,
6403 "Resume Port Tx failed, err %s aq_err %s\n",
6404 i40e_stat_str(&pf->hw, ret),
6405 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6406
6407 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
6408 i40e_service_event_schedule(pf);
6409 }
6410
6411 return ret;
6412}
6413
6414
6415
6416
6417
6418
6419
6420
6421static enum i40e_status_code
6422i40e_update_dcb_config(struct i40e_hw *hw, bool enable_mib_change)
6423{
6424 struct i40e_lldp_variables lldp_cfg;
6425 i40e_status ret;
6426
6427 if (!hw->func_caps.dcb)
6428 return I40E_NOT_SUPPORTED;
6429
6430
6431 ret = i40e_read_lldp_cfg(hw, &lldp_cfg);
6432 if (ret)
6433 return I40E_ERR_NOT_READY;
6434
6435
6436 ret = i40e_get_dcbx_status(hw, &hw->dcbx_status);
6437 if (ret)
6438 return ret;
6439
6440
6441 if (hw->dcbx_status == I40E_DCBX_STATUS_DONE ||
6442 hw->dcbx_status == I40E_DCBX_STATUS_IN_PROGRESS) {
6443
6444 ret = i40e_get_dcb_config(hw);
6445 if (ret)
6446 return ret;
6447 } else if (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED) {
6448 return I40E_ERR_NOT_READY;
6449 }
6450
6451
6452 if (enable_mib_change)
6453 ret = i40e_aq_cfg_lldp_mib_change_event(hw, true, NULL);
6454
6455 return ret;
6456}
6457
6458
6459
6460
6461
6462
6463
6464
6465static int i40e_init_pf_dcb(struct i40e_pf *pf)
6466{
6467 struct i40e_hw *hw = &pf->hw;
6468 int err = 0;
6469
6470
6471
6472
6473 if ((pf->hw_features & I40E_HW_NO_DCB_SUPPORT) ||
6474 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP)) {
6475 dev_info(&pf->pdev->dev, "DCB is not supported or FW LLDP is disabled\n");
6476 err = I40E_NOT_SUPPORTED;
6477 goto out;
6478 }
6479
6480 err = i40e_update_dcb_config(hw, true);
6481 if (!err) {
6482
6483 if ((!hw->func_caps.dcb) ||
6484 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) {
6485 dev_info(&pf->pdev->dev,
6486 "DCBX offload is not supported or is disabled for this PF.\n");
6487 } else {
6488
6489 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED |
6490 DCB_CAP_DCBX_VER_IEEE;
6491
6492 pf->flags |= I40E_FLAG_DCB_CAPABLE;
6493
6494
6495
6496 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
6497 pf->flags |= I40E_FLAG_DCB_ENABLED;
6498 else
6499 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
6500 dev_dbg(&pf->pdev->dev,
6501 "DCBX offload is supported for this PF.\n");
6502 }
6503 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) {
6504 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n");
6505 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP;
6506 } else {
6507 dev_info(&pf->pdev->dev,
6508 "Query for DCB configuration failed, err %s aq_err %s\n",
6509 i40e_stat_str(&pf->hw, err),
6510 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6511 }
6512
6513out:
6514 return err;
6515}
6516#endif
6517#define SPEED_SIZE 14
6518#define FC_SIZE 8
6519
6520
6521
6522
6523
6524void i40e_print_link_message(struct i40e_vsi *vsi, bool isup)
6525{
6526 enum i40e_aq_link_speed new_speed;
6527 struct i40e_pf *pf = vsi->back;
6528 char *speed = "Unknown";
6529 char *fc = "Unknown";
6530 char *fec = "";
6531 char *req_fec = "";
6532 char *an = "";
6533
6534 if (isup)
6535 new_speed = pf->hw.phy.link_info.link_speed;
6536 else
6537 new_speed = I40E_LINK_SPEED_UNKNOWN;
6538
6539 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed))
6540 return;
6541 vsi->current_isup = isup;
6542 vsi->current_speed = new_speed;
6543 if (!isup) {
6544 netdev_info(vsi->netdev, "NIC Link is Down\n");
6545 return;
6546 }
6547
6548
6549
6550
6551 if (pf->hw.func_caps.npar_enable &&
6552 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB ||
6553 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB))
6554 netdev_warn(vsi->netdev,
6555 "The partition detected link speed that is less than 10Gbps\n");
6556
6557 switch (pf->hw.phy.link_info.link_speed) {
6558 case I40E_LINK_SPEED_40GB:
6559 speed = "40 G";
6560 break;
6561 case I40E_LINK_SPEED_20GB:
6562 speed = "20 G";
6563 break;
6564 case I40E_LINK_SPEED_25GB:
6565 speed = "25 G";
6566 break;
6567 case I40E_LINK_SPEED_10GB:
6568 speed = "10 G";
6569 break;
6570 case I40E_LINK_SPEED_5GB:
6571 speed = "5 G";
6572 break;
6573 case I40E_LINK_SPEED_2_5GB:
6574 speed = "2.5 G";
6575 break;
6576 case I40E_LINK_SPEED_1GB:
6577 speed = "1000 M";
6578 break;
6579 case I40E_LINK_SPEED_100MB:
6580 speed = "100 M";
6581 break;
6582 default:
6583 break;
6584 }
6585
6586 switch (pf->hw.fc.current_mode) {
6587 case I40E_FC_FULL:
6588 fc = "RX/TX";
6589 break;
6590 case I40E_FC_TX_PAUSE:
6591 fc = "TX";
6592 break;
6593 case I40E_FC_RX_PAUSE:
6594 fc = "RX";
6595 break;
6596 default:
6597 fc = "None";
6598 break;
6599 }
6600
6601 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) {
6602 req_fec = ", Requested FEC: None";
6603 fec = ", FEC: None";
6604 an = ", Autoneg: False";
6605
6606 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED)
6607 an = ", Autoneg: True";
6608
6609 if (pf->hw.phy.link_info.fec_info &
6610 I40E_AQ_CONFIG_FEC_KR_ENA)
6611 fec = ", FEC: CL74 FC-FEC/BASE-R";
6612 else if (pf->hw.phy.link_info.fec_info &
6613 I40E_AQ_CONFIG_FEC_RS_ENA)
6614 fec = ", FEC: CL108 RS-FEC";
6615
6616
6617
6618
6619 if (vsi->back->hw.phy.link_info.req_fec_info &
6620 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) {
6621 if (vsi->back->hw.phy.link_info.req_fec_info &
6622 I40E_AQ_REQUEST_FEC_RS)
6623 req_fec = ", Requested FEC: CL108 RS-FEC";
6624 else
6625 req_fec = ", Requested FEC: CL74 FC-FEC/BASE-R";
6626 }
6627 }
6628
6629 netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s%s, Flow Control: %s\n",
6630 speed, req_fec, fec, an, fc);
6631}
6632
6633
6634
6635
6636
6637static int i40e_up_complete(struct i40e_vsi *vsi)
6638{
6639 struct i40e_pf *pf = vsi->back;
6640 int err;
6641
6642 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
6643 i40e_vsi_configure_msix(vsi);
6644 else
6645 i40e_configure_msi_and_legacy(vsi);
6646
6647
6648 err = i40e_vsi_start_rings(vsi);
6649 if (err)
6650 return err;
6651
6652 clear_bit(__I40E_VSI_DOWN, vsi->state);
6653 i40e_napi_enable_all(vsi);
6654 i40e_vsi_enable_irq(vsi);
6655
6656 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) &&
6657 (vsi->netdev)) {
6658 i40e_print_link_message(vsi, true);
6659 netif_tx_start_all_queues(vsi->netdev);
6660 netif_carrier_on(vsi->netdev);
6661 }
6662
6663
6664 if (vsi->type == I40E_VSI_FDIR) {
6665
6666 pf->fd_add_err = 0;
6667 pf->fd_atr_cnt = 0;
6668 i40e_fdir_filter_restore(vsi);
6669 }
6670
6671
6672
6673
6674 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
6675 i40e_service_event_schedule(pf);
6676
6677 return 0;
6678}
6679
6680
6681
6682
6683
6684
6685
6686
6687static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi)
6688{
6689 struct i40e_pf *pf = vsi->back;
6690
6691 WARN_ON(in_interrupt());
6692 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state))
6693 usleep_range(1000, 2000);
6694 i40e_down(vsi);
6695
6696 i40e_up(vsi);
6697 clear_bit(__I40E_CONFIG_BUSY, pf->state);
6698}
6699
6700
6701
6702
6703
6704int i40e_up(struct i40e_vsi *vsi)
6705{
6706 int err;
6707
6708 err = i40e_vsi_configure(vsi);
6709 if (!err)
6710 err = i40e_up_complete(vsi);
6711
6712 return err;
6713}
6714
6715
6716
6717
6718
6719
6720static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up)
6721{
6722 struct i40e_aq_get_phy_abilities_resp abilities;
6723 struct i40e_aq_set_phy_config config = {0};
6724 struct i40e_hw *hw = &pf->hw;
6725 i40e_status err;
6726 u64 mask;
6727 u8 speed;
6728
6729
6730
6731
6732
6733
6734
6735 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities,
6736 NULL);
6737 if (err) {
6738 dev_err(&pf->pdev->dev,
6739 "failed to get phy cap., ret = %s last_status = %s\n",
6740 i40e_stat_str(hw, err),
6741 i40e_aq_str(hw, hw->aq.asq_last_status));
6742 return err;
6743 }
6744 speed = abilities.link_speed;
6745
6746
6747 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities,
6748 NULL);
6749 if (err) {
6750 dev_err(&pf->pdev->dev,
6751 "failed to get phy cap., ret = %s last_status = %s\n",
6752 i40e_stat_str(hw, err),
6753 i40e_aq_str(hw, hw->aq.asq_last_status));
6754 return err;
6755 }
6756
6757
6758
6759
6760 if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0)
6761 return I40E_SUCCESS;
6762
6763
6764
6765
6766
6767 mask = I40E_PHY_TYPES_BITMASK;
6768 config.phy_type = is_up ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0;
6769 config.phy_type_ext = is_up ? (u8)((mask >> 32) & 0xff) : 0;
6770
6771 config.abilities = abilities.abilities;
6772 if (abilities.link_speed != 0)
6773 config.link_speed = abilities.link_speed;
6774 else
6775 config.link_speed = speed;
6776 config.eee_capability = abilities.eee_capability;
6777 config.eeer = abilities.eeer_val;
6778 config.low_power_ctrl = abilities.d3_lpan;
6779 config.fec_config = abilities.fec_cfg_curr_mod_ext_info &
6780 I40E_AQ_PHY_FEC_CONFIG_MASK;
6781 err = i40e_aq_set_phy_config(hw, &config, NULL);
6782
6783 if (err) {
6784 dev_err(&pf->pdev->dev,
6785 "set phy config ret = %s last_status = %s\n",
6786 i40e_stat_str(&pf->hw, err),
6787 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
6788 return err;
6789 }
6790
6791
6792 err = i40e_update_link_info(hw);
6793 if (err) {
6794
6795
6796
6797
6798 msleep(1000);
6799 i40e_update_link_info(hw);
6800 }
6801
6802 i40e_aq_set_link_restart_an(hw, true, NULL);
6803
6804 return I40E_SUCCESS;
6805}
6806
6807
6808
6809
6810
6811void i40e_down(struct i40e_vsi *vsi)
6812{
6813 int i;
6814
6815
6816
6817
6818 if (vsi->netdev) {
6819 netif_carrier_off(vsi->netdev);
6820 netif_tx_disable(vsi->netdev);
6821 }
6822 i40e_vsi_disable_irq(vsi);
6823 i40e_vsi_stop_rings(vsi);
6824 if (vsi->type == I40E_VSI_MAIN &&
6825 vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED)
6826 i40e_force_link_state(vsi->back, false);
6827 i40e_napi_disable_all(vsi);
6828
6829 for (i = 0; i < vsi->num_queue_pairs; i++) {
6830 i40e_clean_tx_ring(vsi->tx_rings[i]);
6831 if (i40e_enabled_xdp_vsi(vsi)) {
6832
6833
6834
6835 synchronize_rcu();
6836 i40e_clean_tx_ring(vsi->xdp_rings[i]);
6837 }
6838 i40e_clean_rx_ring(vsi->rx_rings[i]);
6839 }
6840
6841}
6842
6843
6844
6845
6846
6847
6848static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi,
6849 struct tc_mqprio_qopt_offload *mqprio_qopt)
6850{
6851 u64 sum_max_rate = 0;
6852 u64 max_rate = 0;
6853 int i;
6854
6855 if (mqprio_qopt->qopt.offset[0] != 0 ||
6856 mqprio_qopt->qopt.num_tc < 1 ||
6857 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS)
6858 return -EINVAL;
6859 for (i = 0; ; i++) {
6860 if (!mqprio_qopt->qopt.count[i])
6861 return -EINVAL;
6862 if (mqprio_qopt->min_rate[i]) {
6863 dev_err(&vsi->back->pdev->dev,
6864 "Invalid min tx rate (greater than 0) specified\n");
6865 return -EINVAL;
6866 }
6867 max_rate = mqprio_qopt->max_rate[i];
6868 do_div(max_rate, I40E_BW_MBPS_DIVISOR);
6869 sum_max_rate += max_rate;
6870
6871 if (i >= mqprio_qopt->qopt.num_tc - 1)
6872 break;
6873 if (mqprio_qopt->qopt.offset[i + 1] !=
6874 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i]))
6875 return -EINVAL;
6876 }
6877 if (vsi->num_queue_pairs <
6878 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) {
6879 return -EINVAL;
6880 }
6881 if (sum_max_rate > i40e_get_link_speed(vsi)) {
6882 dev_err(&vsi->back->pdev->dev,
6883 "Invalid max tx rate specified\n");
6884 return -EINVAL;
6885 }
6886 return 0;
6887}
6888
6889
6890
6891
6892
6893static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi)
6894{
6895 u16 qcount;
6896 int i;
6897
6898
6899 vsi->tc_config.numtc = 1;
6900 vsi->tc_config.enabled_tc = 1;
6901 qcount = min_t(int, vsi->alloc_queue_pairs,
6902 i40e_pf_get_max_q_per_tc(vsi->back));
6903 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
6904
6905
6906
6907 vsi->tc_config.tc_info[i].qoffset = 0;
6908 if (i == 0)
6909 vsi->tc_config.tc_info[i].qcount = qcount;
6910 else
6911 vsi->tc_config.tc_info[i].qcount = 1;
6912 vsi->tc_config.tc_info[i].netdev_tc = 0;
6913 }
6914}
6915
6916
6917
6918
6919
6920
6921
6922
6923
6924
6925
6926static i40e_status i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid,
6927 const u8 *macaddr, int *aq_err)
6928{
6929 struct i40e_aqc_remove_macvlan_element_data element;
6930 i40e_status status;
6931
6932 memset(&element, 0, sizeof(element));
6933 ether_addr_copy(element.mac_addr, macaddr);
6934 element.vlan_tag = 0;
6935 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH;
6936 status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL);
6937 *aq_err = hw->aq.asq_last_status;
6938
6939 return status;
6940}
6941
6942
6943
6944
6945
6946
6947
6948
6949
6950
6951
6952static i40e_status i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid,
6953 const u8 *macaddr, int *aq_err)
6954{
6955 struct i40e_aqc_add_macvlan_element_data element;
6956 i40e_status status;
6957 u16 cmd_flags = 0;
6958
6959 ether_addr_copy(element.mac_addr, macaddr);
6960 element.vlan_tag = 0;
6961 element.queue_number = 0;
6962 element.match_method = I40E_AQC_MM_ERR_NO_RES;
6963 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH;
6964 element.flags = cpu_to_le16(cmd_flags);
6965 status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL);
6966 *aq_err = hw->aq.asq_last_status;
6967
6968 return status;
6969}
6970
6971
6972
6973
6974
6975
6976static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch)
6977{
6978 struct i40e_ring *tx_ring, *rx_ring;
6979 u16 pf_q;
6980 int i;
6981
6982 for (i = 0; i < ch->num_queue_pairs; i++) {
6983 pf_q = ch->base_queue + i;
6984 tx_ring = vsi->tx_rings[pf_q];
6985 tx_ring->ch = NULL;
6986 rx_ring = vsi->rx_rings[pf_q];
6987 rx_ring->ch = NULL;
6988 }
6989}
6990
6991
6992
6993
6994
6995
6996
6997
6998
6999static void i40e_free_macvlan_channels(struct i40e_vsi *vsi)
7000{
7001 struct i40e_channel *ch, *ch_tmp;
7002 int ret;
7003
7004 if (list_empty(&vsi->macvlan_list))
7005 return;
7006
7007 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7008 struct i40e_vsi *parent_vsi;
7009
7010 if (i40e_is_channel_macvlan(ch)) {
7011 i40e_reset_ch_rings(vsi, ch);
7012 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7013 netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev);
7014 netdev_set_sb_channel(ch->fwd->netdev, 0);
7015 kfree(ch->fwd);
7016 ch->fwd = NULL;
7017 }
7018
7019 list_del(&ch->list);
7020 parent_vsi = ch->parent_vsi;
7021 if (!parent_vsi || !ch->initialized) {
7022 kfree(ch);
7023 continue;
7024 }
7025
7026
7027 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid,
7028 NULL);
7029 if (ret)
7030 dev_err(&vsi->back->pdev->dev,
7031 "unable to remove channel (%d) for parent VSI(%d)\n",
7032 ch->seid, parent_vsi->seid);
7033 kfree(ch);
7034 }
7035 vsi->macvlan_cnt = 0;
7036}
7037
7038
7039
7040
7041
7042
7043
7044static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev,
7045 struct i40e_fwd_adapter *fwd)
7046{
7047 int ret = 0, num_tc = 1, i, aq_err;
7048 struct i40e_channel *ch, *ch_tmp;
7049 struct i40e_pf *pf = vsi->back;
7050 struct i40e_hw *hw = &pf->hw;
7051
7052 if (list_empty(&vsi->macvlan_list))
7053 return -EINVAL;
7054
7055
7056 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7057 if (!i40e_is_channel_macvlan(ch)) {
7058 ch->fwd = fwd;
7059
7060 for (i = 0; i < num_tc; i++)
7061 netdev_bind_sb_channel_queue(vsi->netdev, vdev,
7062 i,
7063 ch->num_queue_pairs,
7064 ch->base_queue);
7065 for (i = 0; i < ch->num_queue_pairs; i++) {
7066 struct i40e_ring *tx_ring, *rx_ring;
7067 u16 pf_q;
7068
7069 pf_q = ch->base_queue + i;
7070
7071
7072 tx_ring = vsi->tx_rings[pf_q];
7073 tx_ring->ch = ch;
7074
7075
7076 rx_ring = vsi->rx_rings[pf_q];
7077 rx_ring->ch = ch;
7078 }
7079 break;
7080 }
7081 }
7082
7083
7084
7085
7086 wmb();
7087
7088
7089 ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err);
7090 if (ret) {
7091
7092 macvlan_release_l2fw_offload(vdev);
7093 for (i = 0; i < ch->num_queue_pairs; i++) {
7094 struct i40e_ring *rx_ring;
7095 u16 pf_q;
7096
7097 pf_q = ch->base_queue + i;
7098 rx_ring = vsi->rx_rings[pf_q];
7099 rx_ring->netdev = NULL;
7100 }
7101 dev_info(&pf->pdev->dev,
7102 "Error adding mac filter on macvlan err %s, aq_err %s\n",
7103 i40e_stat_str(hw, ret),
7104 i40e_aq_str(hw, aq_err));
7105 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n");
7106 }
7107
7108 return ret;
7109}
7110
7111
7112
7113
7114
7115
7116
7117
7118static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt,
7119 struct net_device *vdev)
7120{
7121 struct i40e_pf *pf = vsi->back;
7122 struct i40e_hw *hw = &pf->hw;
7123 struct i40e_vsi_context ctxt;
7124 u16 sections, qmap, num_qps;
7125 struct i40e_channel *ch;
7126 int i, pow, ret = 0;
7127 u8 offset = 0;
7128
7129 if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt)
7130 return -EINVAL;
7131
7132 num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt);
7133
7134
7135 pow = fls(roundup_pow_of_two(num_qps) - 1);
7136
7137 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) |
7138 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT);
7139
7140
7141 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID;
7142 sections |= I40E_AQ_VSI_PROP_SCHED_VALID;
7143 memset(&ctxt, 0, sizeof(ctxt));
7144 ctxt.seid = vsi->seid;
7145 ctxt.pf_num = vsi->back->hw.pf_id;
7146 ctxt.vf_num = 0;
7147 ctxt.uplink_seid = vsi->uplink_seid;
7148 ctxt.info = vsi->info;
7149 ctxt.info.tc_mapping[0] = cpu_to_le16(qmap);
7150 ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG);
7151 ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue);
7152 ctxt.info.valid_sections |= cpu_to_le16(sections);
7153
7154
7155 vsi->rss_size = max_t(u16, num_qps, qcnt);
7156 ret = i40e_vsi_config_rss(vsi);
7157 if (ret) {
7158 dev_info(&pf->pdev->dev,
7159 "Failed to reconfig RSS for num_queues (%u)\n",
7160 vsi->rss_size);
7161 return ret;
7162 }
7163 vsi->reconfig_rss = true;
7164 dev_dbg(&vsi->back->pdev->dev,
7165 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size);
7166 vsi->next_base_queue = num_qps;
7167 vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps;
7168
7169
7170
7171
7172 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
7173 if (ret) {
7174 dev_info(&pf->pdev->dev,
7175 "Update vsi tc config failed, err %s aq_err %s\n",
7176 i40e_stat_str(hw, ret),
7177 i40e_aq_str(hw, hw->aq.asq_last_status));
7178 return ret;
7179 }
7180
7181 i40e_vsi_update_queue_map(vsi, &ctxt);
7182 vsi->info.valid_sections = 0;
7183
7184
7185 INIT_LIST_HEAD(&vsi->macvlan_list);
7186 for (i = 0; i < macvlan_cnt; i++) {
7187 ch = kzalloc(sizeof(*ch), GFP_KERNEL);
7188 if (!ch) {
7189 ret = -ENOMEM;
7190 goto err_free;
7191 }
7192 INIT_LIST_HEAD(&ch->list);
7193 ch->num_queue_pairs = qcnt;
7194 if (!i40e_setup_channel(pf, vsi, ch)) {
7195 ret = -EINVAL;
7196 goto err_free;
7197 }
7198 ch->parent_vsi = vsi;
7199 vsi->cnt_q_avail -= ch->num_queue_pairs;
7200 vsi->macvlan_cnt++;
7201 list_add_tail(&ch->list, &vsi->macvlan_list);
7202 }
7203
7204 return ret;
7205
7206err_free:
7207 dev_info(&pf->pdev->dev, "Failed to setup macvlans\n");
7208 i40e_free_macvlan_channels(vsi);
7209
7210 return ret;
7211}
7212
7213
7214
7215
7216
7217
7218static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev)
7219{
7220 struct i40e_netdev_priv *np = netdev_priv(netdev);
7221 u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors;
7222 struct i40e_vsi *vsi = np->vsi;
7223 struct i40e_pf *pf = vsi->back;
7224 struct i40e_fwd_adapter *fwd;
7225 int avail_macvlan, ret;
7226
7227 if ((pf->flags & I40E_FLAG_DCB_ENABLED)) {
7228 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n");
7229 return ERR_PTR(-EINVAL);
7230 }
7231 if ((pf->flags & I40E_FLAG_TC_MQPRIO)) {
7232 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n");
7233 return ERR_PTR(-EINVAL);
7234 }
7235 if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) {
7236 netdev_info(netdev, "Not enough vectors available to support macvlans\n");
7237 return ERR_PTR(-EINVAL);
7238 }
7239
7240
7241
7242
7243 if (netif_is_multiqueue(vdev))
7244 return ERR_PTR(-ERANGE);
7245
7246 if (!vsi->macvlan_cnt) {
7247
7248 set_bit(0, vsi->fwd_bitmask);
7249
7250
7251
7252
7253
7254 vectors = pf->num_lan_msix;
7255 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) {
7256
7257 q_per_macvlan = 4;
7258 macvlan_cnt = (vectors - 32) / 4;
7259 } else if (vectors <= 64 && vectors > 32) {
7260
7261 q_per_macvlan = 2;
7262 macvlan_cnt = (vectors - 16) / 2;
7263 } else if (vectors <= 32 && vectors > 16) {
7264
7265 q_per_macvlan = 1;
7266 macvlan_cnt = vectors - 16;
7267 } else if (vectors <= 16 && vectors > 8) {
7268
7269 q_per_macvlan = 1;
7270 macvlan_cnt = vectors - 8;
7271 } else {
7272
7273 q_per_macvlan = 1;
7274 macvlan_cnt = vectors - 1;
7275 }
7276
7277 if (macvlan_cnt == 0)
7278 return ERR_PTR(-EBUSY);
7279
7280
7281 i40e_quiesce_vsi(vsi);
7282
7283
7284 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan,
7285 vdev);
7286 if (ret)
7287 return ERR_PTR(ret);
7288
7289
7290 i40e_unquiesce_vsi(vsi);
7291 }
7292 avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask,
7293 vsi->macvlan_cnt);
7294 if (avail_macvlan >= I40E_MAX_MACVLANS)
7295 return ERR_PTR(-EBUSY);
7296
7297
7298 fwd = kzalloc(sizeof(*fwd), GFP_KERNEL);
7299 if (!fwd)
7300 return ERR_PTR(-ENOMEM);
7301
7302 set_bit(avail_macvlan, vsi->fwd_bitmask);
7303 fwd->bit_no = avail_macvlan;
7304 netdev_set_sb_channel(vdev, avail_macvlan);
7305 fwd->netdev = vdev;
7306
7307 if (!netif_running(netdev))
7308 return fwd;
7309
7310
7311 ret = i40e_fwd_ring_up(vsi, vdev, fwd);
7312 if (ret) {
7313
7314 netdev_unbind_sb_channel(netdev, vdev);
7315 netdev_set_sb_channel(vdev, 0);
7316
7317 kfree(fwd);
7318 return ERR_PTR(-EINVAL);
7319 }
7320
7321 return fwd;
7322}
7323
7324
7325
7326
7327
7328static void i40e_del_all_macvlans(struct i40e_vsi *vsi)
7329{
7330 struct i40e_channel *ch, *ch_tmp;
7331 struct i40e_pf *pf = vsi->back;
7332 struct i40e_hw *hw = &pf->hw;
7333 int aq_err, ret = 0;
7334
7335 if (list_empty(&vsi->macvlan_list))
7336 return;
7337
7338 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7339 if (i40e_is_channel_macvlan(ch)) {
7340 ret = i40e_del_macvlan_filter(hw, ch->seid,
7341 i40e_channel_mac(ch),
7342 &aq_err);
7343 if (!ret) {
7344
7345 i40e_reset_ch_rings(vsi, ch);
7346 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7347 netdev_unbind_sb_channel(vsi->netdev,
7348 ch->fwd->netdev);
7349 netdev_set_sb_channel(ch->fwd->netdev, 0);
7350 kfree(ch->fwd);
7351 ch->fwd = NULL;
7352 }
7353 }
7354 }
7355}
7356
7357
7358
7359
7360
7361
7362static void i40e_fwd_del(struct net_device *netdev, void *vdev)
7363{
7364 struct i40e_netdev_priv *np = netdev_priv(netdev);
7365 struct i40e_fwd_adapter *fwd = vdev;
7366 struct i40e_channel *ch, *ch_tmp;
7367 struct i40e_vsi *vsi = np->vsi;
7368 struct i40e_pf *pf = vsi->back;
7369 struct i40e_hw *hw = &pf->hw;
7370 int aq_err, ret = 0;
7371
7372
7373 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) {
7374 if (i40e_is_channel_macvlan(ch) &&
7375 ether_addr_equal(i40e_channel_mac(ch),
7376 fwd->netdev->dev_addr)) {
7377 ret = i40e_del_macvlan_filter(hw, ch->seid,
7378 i40e_channel_mac(ch),
7379 &aq_err);
7380 if (!ret) {
7381
7382 i40e_reset_ch_rings(vsi, ch);
7383 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask);
7384 netdev_unbind_sb_channel(netdev, fwd->netdev);
7385 netdev_set_sb_channel(fwd->netdev, 0);
7386 kfree(ch->fwd);
7387 ch->fwd = NULL;
7388 } else {
7389 dev_info(&pf->pdev->dev,
7390 "Error deleting mac filter on macvlan err %s, aq_err %s\n",
7391 i40e_stat_str(hw, ret),
7392 i40e_aq_str(hw, aq_err));
7393 }
7394 break;
7395 }
7396 }
7397}
7398
7399
7400
7401
7402
7403
7404static int i40e_setup_tc(struct net_device *netdev, void *type_data)
7405{
7406 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
7407 struct i40e_netdev_priv *np = netdev_priv(netdev);
7408 struct i40e_vsi *vsi = np->vsi;
7409 struct i40e_pf *pf = vsi->back;
7410 u8 enabled_tc = 0, num_tc, hw;
7411 bool need_reset = false;
7412 int old_queue_pairs;
7413 int ret = -EINVAL;
7414 u16 mode;
7415 int i;
7416
7417 old_queue_pairs = vsi->num_queue_pairs;
7418 num_tc = mqprio_qopt->qopt.num_tc;
7419 hw = mqprio_qopt->qopt.hw;
7420 mode = mqprio_qopt->mode;
7421 if (!hw) {
7422 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
7423 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt));
7424 goto config_tc;
7425 }
7426
7427
7428 if (pf->flags & I40E_FLAG_MFP_ENABLED) {
7429 netdev_info(netdev,
7430 "Configuring TC not supported in MFP mode\n");
7431 return ret;
7432 }
7433 switch (mode) {
7434 case TC_MQPRIO_MODE_DCB:
7435 pf->flags &= ~I40E_FLAG_TC_MQPRIO;
7436
7437
7438 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) {
7439 netdev_info(netdev,
7440 "DCB is not enabled for adapter\n");
7441 return ret;
7442 }
7443
7444
7445 if (num_tc > i40e_pf_get_num_tc(pf)) {
7446 netdev_info(netdev,
7447 "TC count greater than enabled on link for adapter\n");
7448 return ret;
7449 }
7450 break;
7451 case TC_MQPRIO_MODE_CHANNEL:
7452 if (pf->flags & I40E_FLAG_DCB_ENABLED) {
7453 netdev_info(netdev,
7454 "Full offload of TC Mqprio options is not supported when DCB is enabled\n");
7455 return ret;
7456 }
7457 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
7458 return ret;
7459 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt);
7460 if (ret)
7461 return ret;
7462 memcpy(&vsi->mqprio_qopt, mqprio_qopt,
7463 sizeof(*mqprio_qopt));
7464 pf->flags |= I40E_FLAG_TC_MQPRIO;
7465 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
7466 break;
7467 default:
7468 return -EINVAL;
7469 }
7470
7471config_tc:
7472
7473 for (i = 0; i < num_tc; i++)
7474 enabled_tc |= BIT(i);
7475
7476
7477 if (enabled_tc == vsi->tc_config.enabled_tc &&
7478 mode != TC_MQPRIO_MODE_CHANNEL)
7479 return 0;
7480
7481
7482 i40e_quiesce_vsi(vsi);
7483
7484 if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO))
7485 i40e_remove_queue_channels(vsi);
7486
7487
7488 ret = i40e_vsi_config_tc(vsi, enabled_tc);
7489 if (ret) {
7490 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n",
7491 vsi->seid);
7492 need_reset = true;
7493 goto exit;
7494 } else {
7495 dev_info(&vsi->back->pdev->dev,
7496 "Setup channel (id:%u) utilizing num_queues %d\n",
7497 vsi->seid, vsi->tc_config.tc_info[0].qcount);
7498 }
7499
7500 if (pf->flags & I40E_FLAG_TC_MQPRIO) {
7501 if (vsi->mqprio_qopt.max_rate[0]) {
7502 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
7503
7504 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
7505 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
7506 if (!ret) {
7507 u64 credits = max_tx_rate;
7508
7509 do_div(credits, I40E_BW_CREDIT_DIVISOR);
7510 dev_dbg(&vsi->back->pdev->dev,
7511 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
7512 max_tx_rate,
7513 credits,
7514 vsi->seid);
7515 } else {
7516 need_reset = true;
7517 goto exit;
7518 }
7519 }
7520 ret = i40e_configure_queue_channels(vsi);
7521 if (ret) {
7522 vsi->num_queue_pairs = old_queue_pairs;
7523 netdev_info(netdev,
7524 "Failed configuring queue channels\n");
7525 need_reset = true;
7526 goto exit;
7527 }
7528 }
7529
7530exit:
7531
7532 if (need_reset) {
7533 i40e_vsi_set_default_tc_config(vsi);
7534 need_reset = false;
7535 }
7536
7537
7538 i40e_unquiesce_vsi(vsi);
7539 return ret;
7540}
7541
7542
7543
7544
7545
7546
7547
7548
7549static inline void
7550i40e_set_cld_element(struct i40e_cloud_filter *filter,
7551 struct i40e_aqc_cloud_filters_element_data *cld)
7552{
7553 int i, j;
7554 u32 ipa;
7555
7556 memset(cld, 0, sizeof(*cld));
7557 ether_addr_copy(cld->outer_mac, filter->dst_mac);
7558 ether_addr_copy(cld->inner_mac, filter->src_mac);
7559
7560 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6)
7561 return;
7562
7563 if (filter->n_proto == ETH_P_IPV6) {
7564#define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1)
7565 for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6);
7566 i++, j += 2) {
7567 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]);
7568 ipa = cpu_to_le32(ipa);
7569 memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa));
7570 }
7571 } else {
7572 ipa = be32_to_cpu(filter->dst_ipv4);
7573 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa));
7574 }
7575
7576 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id));
7577
7578
7579
7580
7581 if (filter->tenant_id)
7582 return;
7583}
7584
7585
7586
7587
7588
7589
7590
7591
7592
7593
7594int i40e_add_del_cloud_filter(struct i40e_vsi *vsi,
7595 struct i40e_cloud_filter *filter, bool add)
7596{
7597 struct i40e_aqc_cloud_filters_element_data cld_filter;
7598 struct i40e_pf *pf = vsi->back;
7599 int ret;
7600 static const u16 flag_table[128] = {
7601 [I40E_CLOUD_FILTER_FLAGS_OMAC] =
7602 I40E_AQC_ADD_CLOUD_FILTER_OMAC,
7603 [I40E_CLOUD_FILTER_FLAGS_IMAC] =
7604 I40E_AQC_ADD_CLOUD_FILTER_IMAC,
7605 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] =
7606 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN,
7607 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] =
7608 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID,
7609 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] =
7610 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC,
7611 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] =
7612 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID,
7613 [I40E_CLOUD_FILTER_FLAGS_IIP] =
7614 I40E_AQC_ADD_CLOUD_FILTER_IIP,
7615 };
7616
7617 if (filter->flags >= ARRAY_SIZE(flag_table))
7618 return I40E_ERR_CONFIG;
7619
7620
7621 i40e_set_cld_element(filter, &cld_filter);
7622
7623 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE)
7624 cld_filter.flags = cpu_to_le16(filter->tunnel_type <<
7625 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT);
7626
7627 if (filter->n_proto == ETH_P_IPV6)
7628 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
7629 I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
7630 else
7631 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] |
7632 I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
7633
7634 if (add)
7635 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid,
7636 &cld_filter, 1);
7637 else
7638 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid,
7639 &cld_filter, 1);
7640 if (ret)
7641 dev_dbg(&pf->pdev->dev,
7642 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n",
7643 add ? "add" : "delete", filter->dst_port, ret,
7644 pf->hw.aq.asq_last_status);
7645 else
7646 dev_info(&pf->pdev->dev,
7647 "%s cloud filter for VSI: %d\n",
7648 add ? "Added" : "Deleted", filter->seid);
7649 return ret;
7650}
7651
7652
7653
7654
7655
7656
7657
7658
7659
7660
7661int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi,
7662 struct i40e_cloud_filter *filter,
7663 bool add)
7664{
7665 struct i40e_aqc_cloud_filters_element_bb cld_filter;
7666 struct i40e_pf *pf = vsi->back;
7667 int ret;
7668
7669
7670 if ((is_valid_ether_addr(filter->dst_mac) &&
7671 is_valid_ether_addr(filter->src_mac)) ||
7672 (is_multicast_ether_addr(filter->dst_mac) &&
7673 is_multicast_ether_addr(filter->src_mac)))
7674 return -EOPNOTSUPP;
7675
7676
7677
7678
7679 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP)
7680 return -EOPNOTSUPP;
7681
7682
7683 if (filter->src_port || filter->src_ipv4 ||
7684 !ipv6_addr_any(&filter->ip.v6.src_ip6))
7685 return -EOPNOTSUPP;
7686
7687
7688 i40e_set_cld_element(filter, &cld_filter.element);
7689
7690 if (is_valid_ether_addr(filter->dst_mac) ||
7691 is_valid_ether_addr(filter->src_mac) ||
7692 is_multicast_ether_addr(filter->dst_mac) ||
7693 is_multicast_ether_addr(filter->src_mac)) {
7694
7695 if (filter->dst_ipv4)
7696 return -EOPNOTSUPP;
7697
7698
7699
7700
7701
7702 cld_filter.element.flags =
7703 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT);
7704
7705 if (filter->vlan_id) {
7706 cld_filter.element.flags =
7707 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT);
7708 }
7709
7710 } else if (filter->dst_ipv4 ||
7711 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) {
7712 cld_filter.element.flags =
7713 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT);
7714 if (filter->n_proto == ETH_P_IPV6)
7715 cld_filter.element.flags |=
7716 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6);
7717 else
7718 cld_filter.element.flags |=
7719 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4);
7720 } else {
7721 dev_err(&pf->pdev->dev,
7722 "either mac or ip has to be valid for cloud filter\n");
7723 return -EINVAL;
7724 }
7725
7726
7727 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] =
7728 be16_to_cpu(filter->dst_port);
7729
7730 if (add) {
7731
7732 ret = i40e_validate_and_set_switch_mode(vsi);
7733 if (ret) {
7734 dev_err(&pf->pdev->dev,
7735 "failed to set switch mode, ret %d\n",
7736 ret);
7737 return ret;
7738 }
7739
7740 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid,
7741 &cld_filter, 1);
7742 } else {
7743 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid,
7744 &cld_filter, 1);
7745 }
7746
7747 if (ret)
7748 dev_dbg(&pf->pdev->dev,
7749 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n",
7750 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status);
7751 else
7752 dev_info(&pf->pdev->dev,
7753 "%s cloud filter for VSI: %d, L4 port: %d\n",
7754 add ? "add" : "delete", filter->seid,
7755 ntohs(filter->dst_port));
7756 return ret;
7757}
7758
7759
7760
7761
7762
7763
7764
7765
7766static int i40e_parse_cls_flower(struct i40e_vsi *vsi,
7767 struct flow_cls_offload *f,
7768 struct i40e_cloud_filter *filter)
7769{
7770 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
7771 struct flow_dissector *dissector = rule->match.dissector;
7772 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0;
7773 struct i40e_pf *pf = vsi->back;
7774 u8 field_flags = 0;
7775
7776 if (dissector->used_keys &
7777 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
7778 BIT(FLOW_DISSECTOR_KEY_BASIC) |
7779 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
7780 BIT(FLOW_DISSECTOR_KEY_VLAN) |
7781 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
7782 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
7783 BIT(FLOW_DISSECTOR_KEY_PORTS) |
7784 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
7785 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n",
7786 dissector->used_keys);
7787 return -EOPNOTSUPP;
7788 }
7789
7790 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
7791 struct flow_match_enc_keyid match;
7792
7793 flow_rule_match_enc_keyid(rule, &match);
7794 if (match.mask->keyid != 0)
7795 field_flags |= I40E_CLOUD_FIELD_TEN_ID;
7796
7797 filter->tenant_id = be32_to_cpu(match.key->keyid);
7798 }
7799
7800 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
7801 struct flow_match_basic match;
7802
7803 flow_rule_match_basic(rule, &match);
7804 n_proto_key = ntohs(match.key->n_proto);
7805 n_proto_mask = ntohs(match.mask->n_proto);
7806
7807 if (n_proto_key == ETH_P_ALL) {
7808 n_proto_key = 0;
7809 n_proto_mask = 0;
7810 }
7811 filter->n_proto = n_proto_key & n_proto_mask;
7812 filter->ip_proto = match.key->ip_proto;
7813 }
7814
7815 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
7816 struct flow_match_eth_addrs match;
7817
7818 flow_rule_match_eth_addrs(rule, &match);
7819
7820
7821 if (!is_zero_ether_addr(match.mask->dst)) {
7822 if (is_broadcast_ether_addr(match.mask->dst)) {
7823 field_flags |= I40E_CLOUD_FIELD_OMAC;
7824 } else {
7825 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n",
7826 match.mask->dst);
7827 return I40E_ERR_CONFIG;
7828 }
7829 }
7830
7831 if (!is_zero_ether_addr(match.mask->src)) {
7832 if (is_broadcast_ether_addr(match.mask->src)) {
7833 field_flags |= I40E_CLOUD_FIELD_IMAC;
7834 } else {
7835 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n",
7836 match.mask->src);
7837 return I40E_ERR_CONFIG;
7838 }
7839 }
7840 ether_addr_copy(filter->dst_mac, match.key->dst);
7841 ether_addr_copy(filter->src_mac, match.key->src);
7842 }
7843
7844 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
7845 struct flow_match_vlan match;
7846
7847 flow_rule_match_vlan(rule, &match);
7848 if (match.mask->vlan_id) {
7849 if (match.mask->vlan_id == VLAN_VID_MASK) {
7850 field_flags |= I40E_CLOUD_FIELD_IVLAN;
7851
7852 } else {
7853 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n",
7854 match.mask->vlan_id);
7855 return I40E_ERR_CONFIG;
7856 }
7857 }
7858
7859 filter->vlan_id = cpu_to_be16(match.key->vlan_id);
7860 }
7861
7862 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
7863 struct flow_match_control match;
7864
7865 flow_rule_match_control(rule, &match);
7866 addr_type = match.key->addr_type;
7867 }
7868
7869 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
7870 struct flow_match_ipv4_addrs match;
7871
7872 flow_rule_match_ipv4_addrs(rule, &match);
7873 if (match.mask->dst) {
7874 if (match.mask->dst == cpu_to_be32(0xffffffff)) {
7875 field_flags |= I40E_CLOUD_FIELD_IIP;
7876 } else {
7877 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n",
7878 &match.mask->dst);
7879 return I40E_ERR_CONFIG;
7880 }
7881 }
7882
7883 if (match.mask->src) {
7884 if (match.mask->src == cpu_to_be32(0xffffffff)) {
7885 field_flags |= I40E_CLOUD_FIELD_IIP;
7886 } else {
7887 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n",
7888 &match.mask->src);
7889 return I40E_ERR_CONFIG;
7890 }
7891 }
7892
7893 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) {
7894 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n");
7895 return I40E_ERR_CONFIG;
7896 }
7897 filter->dst_ipv4 = match.key->dst;
7898 filter->src_ipv4 = match.key->src;
7899 }
7900
7901 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
7902 struct flow_match_ipv6_addrs match;
7903
7904 flow_rule_match_ipv6_addrs(rule, &match);
7905
7906
7907
7908
7909 if (ipv6_addr_loopback(&match.key->dst) ||
7910 ipv6_addr_loopback(&match.key->src)) {
7911 dev_err(&pf->pdev->dev,
7912 "Bad ipv6, addr is LOOPBACK\n");
7913 return I40E_ERR_CONFIG;
7914 }
7915 if (!ipv6_addr_any(&match.mask->dst) ||
7916 !ipv6_addr_any(&match.mask->src))
7917 field_flags |= I40E_CLOUD_FIELD_IIP;
7918
7919 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32,
7920 sizeof(filter->src_ipv6));
7921 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32,
7922 sizeof(filter->dst_ipv6));
7923 }
7924
7925 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
7926 struct flow_match_ports match;
7927
7928 flow_rule_match_ports(rule, &match);
7929 if (match.mask->src) {
7930 if (match.mask->src == cpu_to_be16(0xffff)) {
7931 field_flags |= I40E_CLOUD_FIELD_IIP;
7932 } else {
7933 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n",
7934 be16_to_cpu(match.mask->src));
7935 return I40E_ERR_CONFIG;
7936 }
7937 }
7938
7939 if (match.mask->dst) {
7940 if (match.mask->dst == cpu_to_be16(0xffff)) {
7941 field_flags |= I40E_CLOUD_FIELD_IIP;
7942 } else {
7943 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n",
7944 be16_to_cpu(match.mask->dst));
7945 return I40E_ERR_CONFIG;
7946 }
7947 }
7948
7949 filter->dst_port = match.key->dst;
7950 filter->src_port = match.key->src;
7951
7952 switch (filter->ip_proto) {
7953 case IPPROTO_TCP:
7954 case IPPROTO_UDP:
7955 break;
7956 default:
7957 dev_err(&pf->pdev->dev,
7958 "Only UDP and TCP transport are supported\n");
7959 return -EINVAL;
7960 }
7961 }
7962 filter->flags = field_flags;
7963 return 0;
7964}
7965
7966
7967
7968
7969
7970
7971
7972
7973static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc,
7974 struct i40e_cloud_filter *filter)
7975{
7976 struct i40e_channel *ch, *ch_tmp;
7977
7978
7979 if (tc == 0) {
7980 filter->seid = vsi->seid;
7981 return 0;
7982 } else if (vsi->tc_config.enabled_tc & BIT(tc)) {
7983 if (!filter->dst_port) {
7984 dev_err(&vsi->back->pdev->dev,
7985 "Specify destination port to direct to traffic class that is not default\n");
7986 return -EINVAL;
7987 }
7988 if (list_empty(&vsi->ch_list))
7989 return -EINVAL;
7990 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list,
7991 list) {
7992 if (ch->seid == vsi->tc_seid_map[tc])
7993 filter->seid = ch->seid;
7994 }
7995 return 0;
7996 }
7997 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n");
7998 return -EINVAL;
7999}
8000
8001
8002
8003
8004
8005
8006
8007static int i40e_configure_clsflower(struct i40e_vsi *vsi,
8008 struct flow_cls_offload *cls_flower)
8009{
8010 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid);
8011 struct i40e_cloud_filter *filter = NULL;
8012 struct i40e_pf *pf = vsi->back;
8013 int err = 0;
8014
8015 if (tc < 0) {
8016 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n");
8017 return -EOPNOTSUPP;
8018 }
8019
8020 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
8021 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state))
8022 return -EBUSY;
8023
8024 if (pf->fdir_pf_active_filters ||
8025 (!hlist_empty(&pf->fdir_filter_list))) {
8026 dev_err(&vsi->back->pdev->dev,
8027 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n");
8028 return -EINVAL;
8029 }
8030
8031 if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) {
8032 dev_err(&vsi->back->pdev->dev,
8033 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n");
8034 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED;
8035 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8036 }
8037
8038 filter = kzalloc(sizeof(*filter), GFP_KERNEL);
8039 if (!filter)
8040 return -ENOMEM;
8041
8042 filter->cookie = cls_flower->cookie;
8043
8044 err = i40e_parse_cls_flower(vsi, cls_flower, filter);
8045 if (err < 0)
8046 goto err;
8047
8048 err = i40e_handle_tclass(vsi, tc, filter);
8049 if (err < 0)
8050 goto err;
8051
8052
8053 if (filter->dst_port)
8054 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true);
8055 else
8056 err = i40e_add_del_cloud_filter(vsi, filter, true);
8057
8058 if (err) {
8059 dev_err(&pf->pdev->dev,
8060 "Failed to add cloud filter, err %s\n",
8061 i40e_stat_str(&pf->hw, err));
8062 goto err;
8063 }
8064
8065
8066 INIT_HLIST_NODE(&filter->cloud_node);
8067
8068 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list);
8069
8070 pf->num_cloud_filters++;
8071
8072 return err;
8073err:
8074 kfree(filter);
8075 return err;
8076}
8077
8078
8079
8080
8081
8082
8083
8084static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi,
8085 unsigned long *cookie)
8086{
8087 struct i40e_cloud_filter *filter = NULL;
8088 struct hlist_node *node2;
8089
8090 hlist_for_each_entry_safe(filter, node2,
8091 &vsi->back->cloud_filter_list, cloud_node)
8092 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
8093 return filter;
8094 return NULL;
8095}
8096
8097
8098
8099
8100
8101
8102
8103static int i40e_delete_clsflower(struct i40e_vsi *vsi,
8104 struct flow_cls_offload *cls_flower)
8105{
8106 struct i40e_cloud_filter *filter = NULL;
8107 struct i40e_pf *pf = vsi->back;
8108 int err = 0;
8109
8110 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie);
8111
8112 if (!filter)
8113 return -EINVAL;
8114
8115 hash_del(&filter->cloud_node);
8116
8117 if (filter->dst_port)
8118 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false);
8119 else
8120 err = i40e_add_del_cloud_filter(vsi, filter, false);
8121
8122 kfree(filter);
8123 if (err) {
8124 dev_err(&pf->pdev->dev,
8125 "Failed to delete cloud filter, err %s\n",
8126 i40e_stat_str(&pf->hw, err));
8127 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status);
8128 }
8129
8130 pf->num_cloud_filters--;
8131 if (!pf->num_cloud_filters)
8132 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8133 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8134 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8135 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8136 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8137 }
8138 return 0;
8139}
8140
8141
8142
8143
8144
8145
8146static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np,
8147 struct flow_cls_offload *cls_flower)
8148{
8149 struct i40e_vsi *vsi = np->vsi;
8150
8151 switch (cls_flower->command) {
8152 case FLOW_CLS_REPLACE:
8153 return i40e_configure_clsflower(vsi, cls_flower);
8154 case FLOW_CLS_DESTROY:
8155 return i40e_delete_clsflower(vsi, cls_flower);
8156 case FLOW_CLS_STATS:
8157 return -EOPNOTSUPP;
8158 default:
8159 return -EOPNOTSUPP;
8160 }
8161}
8162
8163static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
8164 void *cb_priv)
8165{
8166 struct i40e_netdev_priv *np = cb_priv;
8167
8168 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data))
8169 return -EOPNOTSUPP;
8170
8171 switch (type) {
8172 case TC_SETUP_CLSFLOWER:
8173 return i40e_setup_tc_cls_flower(np, type_data);
8174
8175 default:
8176 return -EOPNOTSUPP;
8177 }
8178}
8179
8180static LIST_HEAD(i40e_block_cb_list);
8181
8182static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type,
8183 void *type_data)
8184{
8185 struct i40e_netdev_priv *np = netdev_priv(netdev);
8186
8187 switch (type) {
8188 case TC_SETUP_QDISC_MQPRIO:
8189 return i40e_setup_tc(netdev, type_data);
8190 case TC_SETUP_BLOCK:
8191 return flow_block_cb_setup_simple(type_data,
8192 &i40e_block_cb_list,
8193 i40e_setup_tc_block_cb,
8194 np, np, true);
8195 default:
8196 return -EOPNOTSUPP;
8197 }
8198}
8199
8200
8201
8202
8203
8204
8205
8206
8207
8208
8209
8210
8211
8212int i40e_open(struct net_device *netdev)
8213{
8214 struct i40e_netdev_priv *np = netdev_priv(netdev);
8215 struct i40e_vsi *vsi = np->vsi;
8216 struct i40e_pf *pf = vsi->back;
8217 int err;
8218
8219
8220 if (test_bit(__I40E_TESTING, pf->state) ||
8221 test_bit(__I40E_BAD_EEPROM, pf->state))
8222 return -EBUSY;
8223
8224 netif_carrier_off(netdev);
8225
8226 if (i40e_force_link_state(pf, true))
8227 return -EAGAIN;
8228
8229 err = i40e_vsi_open(vsi);
8230 if (err)
8231 return err;
8232
8233
8234 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH |
8235 TCP_FLAG_FIN) >> 16);
8236 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH |
8237 TCP_FLAG_FIN |
8238 TCP_FLAG_CWR) >> 16);
8239 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16);
8240
8241 udp_tunnel_get_rx_info(netdev);
8242
8243 return 0;
8244}
8245
8246
8247
8248
8249
8250
8251
8252
8253
8254
8255
8256int i40e_vsi_open(struct i40e_vsi *vsi)
8257{
8258 struct i40e_pf *pf = vsi->back;
8259 char int_name[I40E_INT_NAME_STR_LEN];
8260 int err;
8261
8262
8263 err = i40e_vsi_setup_tx_resources(vsi);
8264 if (err)
8265 goto err_setup_tx;
8266 err = i40e_vsi_setup_rx_resources(vsi);
8267 if (err)
8268 goto err_setup_rx;
8269
8270 err = i40e_vsi_configure(vsi);
8271 if (err)
8272 goto err_setup_rx;
8273
8274 if (vsi->netdev) {
8275 snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
8276 dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
8277 err = i40e_vsi_request_irq(vsi, int_name);
8278 if (err)
8279 goto err_setup_rx;
8280
8281
8282 err = netif_set_real_num_tx_queues(vsi->netdev,
8283 vsi->num_queue_pairs);
8284 if (err)
8285 goto err_set_queues;
8286
8287 err = netif_set_real_num_rx_queues(vsi->netdev,
8288 vsi->num_queue_pairs);
8289 if (err)
8290 goto err_set_queues;
8291
8292 } else if (vsi->type == I40E_VSI_FDIR) {
8293 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir",
8294 dev_driver_string(&pf->pdev->dev),
8295 dev_name(&pf->pdev->dev));
8296 err = i40e_vsi_request_irq(vsi, int_name);
8297
8298 } else {
8299 err = -EINVAL;
8300 goto err_setup_rx;
8301 }
8302
8303 err = i40e_up_complete(vsi);
8304 if (err)
8305 goto err_up_complete;
8306
8307 return 0;
8308
8309err_up_complete:
8310 i40e_down(vsi);
8311err_set_queues:
8312 i40e_vsi_free_irq(vsi);
8313err_setup_rx:
8314 i40e_vsi_free_rx_resources(vsi);
8315err_setup_tx:
8316 i40e_vsi_free_tx_resources(vsi);
8317 if (vsi == pf->vsi[pf->lan_vsi])
8318 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
8319
8320 return err;
8321}
8322
8323
8324
8325
8326
8327
8328
8329
8330static void i40e_fdir_filter_exit(struct i40e_pf *pf)
8331{
8332 struct i40e_fdir_filter *filter;
8333 struct i40e_flex_pit *pit_entry, *tmp;
8334 struct hlist_node *node2;
8335
8336 hlist_for_each_entry_safe(filter, node2,
8337 &pf->fdir_filter_list, fdir_node) {
8338 hlist_del(&filter->fdir_node);
8339 kfree(filter);
8340 }
8341
8342 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) {
8343 list_del(&pit_entry->list);
8344 kfree(pit_entry);
8345 }
8346 INIT_LIST_HEAD(&pf->l3_flex_pit_list);
8347
8348 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) {
8349 list_del(&pit_entry->list);
8350 kfree(pit_entry);
8351 }
8352 INIT_LIST_HEAD(&pf->l4_flex_pit_list);
8353
8354 pf->fdir_pf_active_filters = 0;
8355 pf->fd_tcp4_filter_cnt = 0;
8356 pf->fd_udp4_filter_cnt = 0;
8357 pf->fd_sctp4_filter_cnt = 0;
8358 pf->fd_ip4_filter_cnt = 0;
8359
8360
8361 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
8362 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8363 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8364
8365
8366 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP,
8367 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8368 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8369
8370
8371 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP,
8372 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8373 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8374
8375
8376 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER,
8377 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8378
8379 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4,
8380 I40E_L3_SRC_MASK | I40E_L3_DST_MASK);
8381}
8382
8383
8384
8385
8386
8387
8388
8389
8390static void i40e_cloud_filter_exit(struct i40e_pf *pf)
8391{
8392 struct i40e_cloud_filter *cfilter;
8393 struct hlist_node *node;
8394
8395 hlist_for_each_entry_safe(cfilter, node,
8396 &pf->cloud_filter_list, cloud_node) {
8397 hlist_del(&cfilter->cloud_node);
8398 kfree(cfilter);
8399 }
8400 pf->num_cloud_filters = 0;
8401
8402 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) &&
8403 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) {
8404 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
8405 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER;
8406 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
8407 }
8408}
8409
8410
8411
8412
8413
8414
8415
8416
8417
8418
8419
8420int i40e_close(struct net_device *netdev)
8421{
8422 struct i40e_netdev_priv *np = netdev_priv(netdev);
8423 struct i40e_vsi *vsi = np->vsi;
8424
8425 i40e_vsi_close(vsi);
8426
8427 return 0;
8428}
8429
8430
8431
8432
8433
8434
8435
8436
8437
8438
8439
8440
8441void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired)
8442{
8443 u32 val;
8444
8445 WARN_ON(in_interrupt());
8446
8447
8448
8449 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) {
8450
8451
8452
8453
8454
8455
8456
8457
8458
8459 dev_dbg(&pf->pdev->dev, "GlobalR requested\n");
8460 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8461 val |= I40E_GLGEN_RTRIG_GLOBR_MASK;
8462 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
8463
8464 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) {
8465
8466
8467
8468
8469
8470 dev_dbg(&pf->pdev->dev, "CoreR requested\n");
8471 val = rd32(&pf->hw, I40E_GLGEN_RTRIG);
8472 val |= I40E_GLGEN_RTRIG_CORER_MASK;
8473 wr32(&pf->hw, I40E_GLGEN_RTRIG, val);
8474 i40e_flush(&pf->hw);
8475
8476 } else if (reset_flags & I40E_PF_RESET_FLAG) {
8477
8478
8479
8480
8481
8482
8483
8484
8485
8486 dev_dbg(&pf->pdev->dev, "PFR requested\n");
8487 i40e_handle_reset_warning(pf, lock_acquired);
8488
8489 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) {
8490 int v;
8491
8492
8493 dev_info(&pf->pdev->dev,
8494 "VSI reinit requested\n");
8495 for (v = 0; v < pf->num_alloc_vsi; v++) {
8496 struct i40e_vsi *vsi = pf->vsi[v];
8497
8498 if (vsi != NULL &&
8499 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED,
8500 vsi->state))
8501 i40e_vsi_reinit_locked(pf->vsi[v]);
8502 }
8503 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) {
8504 int v;
8505
8506
8507 dev_info(&pf->pdev->dev, "VSI down requested\n");
8508 for (v = 0; v < pf->num_alloc_vsi; v++) {
8509 struct i40e_vsi *vsi = pf->vsi[v];
8510
8511 if (vsi != NULL &&
8512 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED,
8513 vsi->state)) {
8514 set_bit(__I40E_VSI_DOWN, vsi->state);
8515 i40e_down(vsi);
8516 }
8517 }
8518 } else {
8519 dev_info(&pf->pdev->dev,
8520 "bad reset request 0x%08x\n", reset_flags);
8521 }
8522}
8523
8524#ifdef CONFIG_I40E_DCB
8525
8526
8527
8528
8529
8530
8531bool i40e_dcb_need_reconfig(struct i40e_pf *pf,
8532 struct i40e_dcbx_config *old_cfg,
8533 struct i40e_dcbx_config *new_cfg)
8534{
8535 bool need_reconfig = false;
8536
8537
8538 if (memcmp(&new_cfg->etscfg,
8539 &old_cfg->etscfg,
8540 sizeof(new_cfg->etscfg))) {
8541
8542 if (memcmp(&new_cfg->etscfg.prioritytable,
8543 &old_cfg->etscfg.prioritytable,
8544 sizeof(new_cfg->etscfg.prioritytable))) {
8545 need_reconfig = true;
8546 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n");
8547 }
8548
8549 if (memcmp(&new_cfg->etscfg.tcbwtable,
8550 &old_cfg->etscfg.tcbwtable,
8551 sizeof(new_cfg->etscfg.tcbwtable)))
8552 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n");
8553
8554 if (memcmp(&new_cfg->etscfg.tsatable,
8555 &old_cfg->etscfg.tsatable,
8556 sizeof(new_cfg->etscfg.tsatable)))
8557 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n");
8558 }
8559
8560
8561 if (memcmp(&new_cfg->pfc,
8562 &old_cfg->pfc,
8563 sizeof(new_cfg->pfc))) {
8564 need_reconfig = true;
8565 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n");
8566 }
8567
8568
8569 if (memcmp(&new_cfg->app,
8570 &old_cfg->app,
8571 sizeof(new_cfg->app))) {
8572 need_reconfig = true;
8573 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n");
8574 }
8575
8576 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig);
8577 return need_reconfig;
8578}
8579
8580
8581
8582
8583
8584
8585static int i40e_handle_lldp_event(struct i40e_pf *pf,
8586 struct i40e_arq_event_info *e)
8587{
8588 struct i40e_aqc_lldp_get_mib *mib =
8589 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw;
8590 struct i40e_hw *hw = &pf->hw;
8591 struct i40e_dcbx_config tmp_dcbx_cfg;
8592 bool need_reconfig = false;
8593 int ret = 0;
8594 u8 type;
8595
8596
8597 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE))
8598 return ret;
8599
8600
8601 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT)
8602 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK);
8603 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type);
8604 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE)
8605 return ret;
8606
8607
8608 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK;
8609 dev_dbg(&pf->pdev->dev,
8610 "LLDP event mib type %s\n", type ? "remote" : "local");
8611 if (type == I40E_AQ_LLDP_MIB_REMOTE) {
8612
8613 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE,
8614 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE,
8615 &hw->remote_dcbx_config);
8616 goto exit;
8617 }
8618
8619
8620 tmp_dcbx_cfg = hw->local_dcbx_config;
8621
8622
8623 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config));
8624
8625 ret = i40e_get_dcb_config(&pf->hw);
8626 if (ret) {
8627 dev_info(&pf->pdev->dev,
8628 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n",
8629 i40e_stat_str(&pf->hw, ret),
8630 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
8631 goto exit;
8632 }
8633
8634
8635 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config,
8636 sizeof(tmp_dcbx_cfg))) {
8637 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n");
8638 goto exit;
8639 }
8640
8641 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg,
8642 &hw->local_dcbx_config);
8643
8644 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config);
8645
8646 if (!need_reconfig)
8647 goto exit;
8648
8649
8650 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1)
8651 pf->flags |= I40E_FLAG_DCB_ENABLED;
8652 else
8653 pf->flags &= ~I40E_FLAG_DCB_ENABLED;
8654
8655 set_bit(__I40E_PORT_SUSPENDED, pf->state);
8656
8657 i40e_pf_quiesce_all_vsi(pf);
8658
8659
8660 i40e_dcb_reconfigure(pf);
8661
8662 ret = i40e_resume_port_tx(pf);
8663
8664 clear_bit(__I40E_PORT_SUSPENDED, pf->state);
8665
8666 if (ret)
8667 goto exit;
8668
8669
8670 ret = i40e_pf_wait_queues_disabled(pf);
8671 if (ret) {
8672
8673 set_bit(__I40E_PF_RESET_REQUESTED, pf->state);
8674 i40e_service_event_schedule(pf);
8675 } else {
8676 i40e_pf_unquiesce_all_vsi(pf);
8677 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state);
8678 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state);
8679 }
8680
8681exit:
8682 return ret;
8683}
8684#endif
8685
8686
8687
8688
8689
8690
8691
8692void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags)
8693{
8694 rtnl_lock();
8695 i40e_do_reset(pf, reset_flags, true);
8696 rtnl_unlock();
8697}
8698
8699
8700
8701
8702
8703
8704
8705
8706
8707static void i40e_handle_lan_overflow_event(struct i40e_pf *pf,
8708 struct i40e_arq_event_info *e)
8709{
8710 struct i40e_aqc_lan_overflow *data =
8711 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw;
8712 u32 queue = le32_to_cpu(data->prtdcb_rupto);
8713 u32 qtx_ctl = le32_to_cpu(data->otx_ctl);
8714 struct i40e_hw *hw = &pf->hw;
8715 struct i40e_vf *vf;
8716 u16 vf_id;
8717
8718 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n",
8719 queue, qtx_ctl);
8720
8721
8722 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK)
8723 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) {
8724 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK)
8725 >> I40E_QTX_CTL_VFVM_INDX_SHIFT);
8726 vf_id -= hw->func_caps.vf_base_id;
8727 vf = &pf->vf[vf_id];
8728 i40e_vc_notify_vf_reset(vf);
8729
8730 msleep(20);
8731 i40e_reset_vf(vf, false);
8732 }
8733}
8734
8735
8736
8737
8738
8739u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf)
8740{
8741 u32 val, fcnt_prog;
8742
8743 val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
8744 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK);
8745 return fcnt_prog;
8746}
8747
8748
8749
8750
8751
8752u32 i40e_get_current_fd_count(struct i40e_pf *pf)
8753{
8754 u32 val, fcnt_prog;
8755
8756 val = rd32(&pf->hw, I40E_PFQF_FDSTAT);
8757 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) +
8758 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >>
8759 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT);
8760 return fcnt_prog;
8761}
8762
8763
8764
8765
8766
8767u32 i40e_get_global_fd_count(struct i40e_pf *pf)
8768{
8769 u32 val, fcnt_prog;
8770
8771 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0);
8772 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) +
8773 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >>
8774 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT);
8775 return fcnt_prog;
8776}
8777
8778
8779
8780
8781
8782static void i40e_reenable_fdir_sb(struct i40e_pf *pf)
8783{
8784 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state))
8785 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
8786 (I40E_DEBUG_FD & pf->hw.debug_mask))
8787 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n");
8788}
8789
8790
8791
8792
8793
8794static void i40e_reenable_fdir_atr(struct i40e_pf *pf)
8795{
8796 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) {
8797
8798
8799
8800
8801
8802 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP,
8803 I40E_L3_SRC_MASK | I40E_L3_DST_MASK |
8804 I40E_L4_SRC_MASK | I40E_L4_DST_MASK);
8805
8806 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
8807 (I40E_DEBUG_FD & pf->hw.debug_mask))
8808 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n");
8809 }
8810}
8811
8812
8813
8814
8815
8816
8817static void i40e_delete_invalid_filter(struct i40e_pf *pf,
8818 struct i40e_fdir_filter *filter)
8819{
8820
8821 pf->fdir_pf_active_filters--;
8822 pf->fd_inv = 0;
8823
8824 switch (filter->flow_type) {
8825 case TCP_V4_FLOW:
8826 pf->fd_tcp4_filter_cnt--;
8827 break;
8828 case UDP_V4_FLOW:
8829 pf->fd_udp4_filter_cnt--;
8830 break;
8831 case SCTP_V4_FLOW:
8832 pf->fd_sctp4_filter_cnt--;
8833 break;
8834 case IP_USER_FLOW:
8835 switch (filter->ip4_proto) {
8836 case IPPROTO_TCP:
8837 pf->fd_tcp4_filter_cnt--;
8838 break;
8839 case IPPROTO_UDP:
8840 pf->fd_udp4_filter_cnt--;
8841 break;
8842 case IPPROTO_SCTP:
8843 pf->fd_sctp4_filter_cnt--;
8844 break;
8845 case IPPROTO_IP:
8846 pf->fd_ip4_filter_cnt--;
8847 break;
8848 }
8849 break;
8850 }
8851
8852
8853 hlist_del(&filter->fdir_node);
8854 kfree(filter);
8855}
8856
8857
8858
8859
8860
8861void i40e_fdir_check_and_reenable(struct i40e_pf *pf)
8862{
8863 struct i40e_fdir_filter *filter;
8864 u32 fcnt_prog, fcnt_avail;
8865 struct hlist_node *node;
8866
8867 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
8868 return;
8869
8870
8871 fcnt_prog = i40e_get_global_fd_count(pf);
8872 fcnt_avail = pf->fdir_pf_filter_count;
8873 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) ||
8874 (pf->fd_add_err == 0) ||
8875 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt))
8876 i40e_reenable_fdir_sb(pf);
8877
8878
8879
8880
8881
8882 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) &&
8883 (pf->fd_tcp4_filter_cnt == 0))
8884 i40e_reenable_fdir_atr(pf);
8885
8886
8887 if (pf->fd_inv > 0) {
8888 hlist_for_each_entry_safe(filter, node,
8889 &pf->fdir_filter_list, fdir_node)
8890 if (filter->fd_id == pf->fd_inv)
8891 i40e_delete_invalid_filter(pf, filter);
8892 }
8893}
8894
8895#define I40E_MIN_FD_FLUSH_INTERVAL 10
8896#define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30
8897
8898
8899
8900
8901static void i40e_fdir_flush_and_replay(struct i40e_pf *pf)
8902{
8903 unsigned long min_flush_time;
8904 int flush_wait_retry = 50;
8905 bool disable_atr = false;
8906 int fd_room;
8907 int reg;
8908
8909 if (!time_after(jiffies, pf->fd_flush_timestamp +
8910 (I40E_MIN_FD_FLUSH_INTERVAL * HZ)))
8911 return;
8912
8913
8914
8915
8916 min_flush_time = pf->fd_flush_timestamp +
8917 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ);
8918 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters;
8919
8920 if (!(time_after(jiffies, min_flush_time)) &&
8921 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) {
8922 if (I40E_DEBUG_FD & pf->hw.debug_mask)
8923 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n");
8924 disable_atr = true;
8925 }
8926
8927 pf->fd_flush_timestamp = jiffies;
8928 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
8929
8930 wr32(&pf->hw, I40E_PFQF_CTL_1,
8931 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK);
8932 i40e_flush(&pf->hw);
8933 pf->fd_flush_cnt++;
8934 pf->fd_add_err = 0;
8935 do {
8936
8937 usleep_range(5000, 6000);
8938 reg = rd32(&pf->hw, I40E_PFQF_CTL_1);
8939 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK))
8940 break;
8941 } while (flush_wait_retry--);
8942 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) {
8943 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n");
8944 } else {
8945
8946 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]);
8947 if (!disable_atr && !pf->fd_tcp4_filter_cnt)
8948 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state);
8949 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state);
8950 if (I40E_DEBUG_FD & pf->hw.debug_mask)
8951 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n");
8952 }
8953}
8954
8955
8956
8957
8958
8959u32 i40e_get_current_atr_cnt(struct i40e_pf *pf)
8960{
8961 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters;
8962}
8963
8964
8965
8966
8967
8968
8969#define I40E_MAX_FD_PROGRAM_ERROR 256
8970
8971
8972
8973
8974
8975static void i40e_fdir_reinit_subtask(struct i40e_pf *pf)
8976{
8977
8978
8979 if (test_bit(__I40E_DOWN, pf->state))
8980 return;
8981
8982 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state))
8983 i40e_fdir_flush_and_replay(pf);
8984
8985 i40e_fdir_check_and_reenable(pf);
8986
8987}
8988
8989
8990
8991
8992
8993
8994static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up)
8995{
8996 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state))
8997 return;
8998
8999 switch (vsi->type) {
9000 case I40E_VSI_MAIN:
9001 if (!vsi->netdev || !vsi->netdev_registered)
9002 break;
9003
9004 if (link_up) {
9005 netif_carrier_on(vsi->netdev);
9006 netif_tx_wake_all_queues(vsi->netdev);
9007 } else {
9008 netif_carrier_off(vsi->netdev);
9009 netif_tx_stop_all_queues(vsi->netdev);
9010 }
9011 break;
9012
9013 case I40E_VSI_SRIOV:
9014 case I40E_VSI_VMDQ2:
9015 case I40E_VSI_CTRL:
9016 case I40E_VSI_IWARP:
9017 case I40E_VSI_MIRROR:
9018 default:
9019
9020 break;
9021 }
9022}
9023
9024
9025
9026
9027
9028
9029static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up)
9030{
9031 struct i40e_pf *pf;
9032 int i;
9033
9034 if (!veb || !veb->pf)
9035 return;
9036 pf = veb->pf;
9037
9038
9039 for (i = 0; i < I40E_MAX_VEB; i++)
9040 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid))
9041 i40e_veb_link_event(pf->veb[i], link_up);
9042
9043
9044 for (i = 0; i < pf->num_alloc_vsi; i++)
9045 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid))
9046 i40e_vsi_link_event(pf->vsi[i], link_up);
9047}
9048
9049
9050
9051
9052
9053static void i40e_link_event(struct i40e_pf *pf)
9054{
9055 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9056 u8 new_link_speed, old_link_speed;
9057 i40e_status status;
9058 bool new_link, old_link;
9059
9060
9061 pf->hw.phy.get_link_info = true;
9062 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP);
9063 status = i40e_get_link_status(&pf->hw, &new_link);
9064
9065
9066 if (status == I40E_SUCCESS) {
9067 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9068 } else {
9069
9070
9071
9072 set_bit(__I40E_TEMP_LINK_POLLING, pf->state);
9073 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n",
9074 status);
9075 return;
9076 }
9077
9078 old_link_speed = pf->hw.phy.link_info_old.link_speed;
9079 new_link_speed = pf->hw.phy.link_info.link_speed;
9080
9081 if (new_link == old_link &&
9082 new_link_speed == old_link_speed &&
9083 (test_bit(__I40E_VSI_DOWN, vsi->state) ||
9084 new_link == netif_carrier_ok(vsi->netdev)))
9085 return;
9086
9087 i40e_print_link_message(vsi, new_link);
9088
9089
9090
9091
9092 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
9093 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link);
9094 else
9095 i40e_vsi_link_event(vsi, new_link);
9096
9097 if (pf->vf)
9098 i40e_vc_notify_link_state(pf);
9099
9100 if (pf->flags & I40E_FLAG_PTP)
9101 i40e_ptp_set_increment(pf);
9102}
9103
9104
9105
9106
9107
9108static void i40e_watchdog_subtask(struct i40e_pf *pf)
9109{
9110 int i;
9111
9112
9113 if (test_bit(__I40E_DOWN, pf->state) ||
9114 test_bit(__I40E_CONFIG_BUSY, pf->state))
9115 return;
9116
9117
9118 if (time_before(jiffies, (pf->service_timer_previous +
9119 pf->service_timer_period)))
9120 return;
9121 pf->service_timer_previous = jiffies;
9122
9123 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) ||
9124 test_bit(__I40E_TEMP_LINK_POLLING, pf->state))
9125 i40e_link_event(pf);
9126
9127
9128
9129
9130 for (i = 0; i < pf->num_alloc_vsi; i++)
9131 if (pf->vsi[i] && pf->vsi[i]->netdev)
9132 i40e_update_stats(pf->vsi[i]);
9133
9134 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) {
9135
9136 for (i = 0; i < I40E_MAX_VEB; i++)
9137 if (pf->veb[i])
9138 i40e_update_veb_stats(pf->veb[i]);
9139 }
9140
9141 i40e_ptp_rx_hang(pf);
9142 i40e_ptp_tx_hang(pf);
9143}
9144
9145
9146
9147
9148
9149static void i40e_reset_subtask(struct i40e_pf *pf)
9150{
9151 u32 reset_flags = 0;
9152
9153 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) {
9154 reset_flags |= BIT(__I40E_REINIT_REQUESTED);
9155 clear_bit(__I40E_REINIT_REQUESTED, pf->state);
9156 }
9157 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) {
9158 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED);
9159 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state);
9160 }
9161 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) {
9162 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED);
9163 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state);
9164 }
9165 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) {
9166 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED);
9167 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state);
9168 }
9169 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) {
9170 reset_flags |= BIT(__I40E_DOWN_REQUESTED);
9171 clear_bit(__I40E_DOWN_REQUESTED, pf->state);
9172 }
9173
9174
9175
9176
9177 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) {
9178 i40e_prep_for_reset(pf, false);
9179 i40e_reset(pf);
9180 i40e_rebuild(pf, false, false);
9181 }
9182
9183
9184 if (reset_flags &&
9185 !test_bit(__I40E_DOWN, pf->state) &&
9186 !test_bit(__I40E_CONFIG_BUSY, pf->state)) {
9187 i40e_do_reset(pf, reset_flags, false);
9188 }
9189}
9190
9191
9192
9193
9194
9195
9196static void i40e_handle_link_event(struct i40e_pf *pf,
9197 struct i40e_arq_event_info *e)
9198{
9199 struct i40e_aqc_get_link_status *status =
9200 (struct i40e_aqc_get_link_status *)&e->desc.params.raw;
9201
9202
9203
9204
9205
9206
9207
9208 i40e_link_event(pf);
9209
9210
9211 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) {
9212 dev_err(&pf->pdev->dev,
9213 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n");
9214 dev_err(&pf->pdev->dev,
9215 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
9216 } else {
9217
9218
9219
9220 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) &&
9221 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) &&
9222 (!(status->link_info & I40E_AQ_LINK_UP)) &&
9223 (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) {
9224 dev_err(&pf->pdev->dev,
9225 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n");
9226 dev_err(&pf->pdev->dev,
9227 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
9228 }
9229 }
9230}
9231
9232
9233
9234
9235
9236static void i40e_clean_adminq_subtask(struct i40e_pf *pf)
9237{
9238 struct i40e_arq_event_info event;
9239 struct i40e_hw *hw = &pf->hw;
9240 u16 pending, i = 0;
9241 i40e_status ret;
9242 u16 opcode;
9243 u32 oldval;
9244 u32 val;
9245
9246
9247 if (test_bit(__I40E_RESET_FAILED, pf->state))
9248 return;
9249
9250
9251 val = rd32(&pf->hw, pf->hw.aq.arq.len);
9252 oldval = val;
9253 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) {
9254 if (hw->debug_mask & I40E_DEBUG_AQ)
9255 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n");
9256 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK;
9257 }
9258 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) {
9259 if (hw->debug_mask & I40E_DEBUG_AQ)
9260 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n");
9261 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK;
9262 pf->arq_overflows++;
9263 }
9264 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) {
9265 if (hw->debug_mask & I40E_DEBUG_AQ)
9266 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n");
9267 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK;
9268 }
9269 if (oldval != val)
9270 wr32(&pf->hw, pf->hw.aq.arq.len, val);
9271
9272 val = rd32(&pf->hw, pf->hw.aq.asq.len);
9273 oldval = val;
9274 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) {
9275 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9276 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n");
9277 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK;
9278 }
9279 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) {
9280 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9281 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n");
9282 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK;
9283 }
9284 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) {
9285 if (pf->hw.debug_mask & I40E_DEBUG_AQ)
9286 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n");
9287 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK;
9288 }
9289 if (oldval != val)
9290 wr32(&pf->hw, pf->hw.aq.asq.len, val);
9291
9292 event.buf_len = I40E_MAX_AQ_BUF_SIZE;
9293 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
9294 if (!event.msg_buf)
9295 return;
9296
9297 do {
9298 ret = i40e_clean_arq_element(hw, &event, &pending);
9299 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK)
9300 break;
9301 else if (ret) {
9302 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret);
9303 break;
9304 }
9305
9306 opcode = le16_to_cpu(event.desc.opcode);
9307 switch (opcode) {
9308
9309 case i40e_aqc_opc_get_link_status:
9310 i40e_handle_link_event(pf, &event);
9311 break;
9312 case i40e_aqc_opc_send_msg_to_pf:
9313 ret = i40e_vc_process_vf_msg(pf,
9314 le16_to_cpu(event.desc.retval),
9315 le32_to_cpu(event.desc.cookie_high),
9316 le32_to_cpu(event.desc.cookie_low),
9317 event.msg_buf,
9318 event.msg_len);
9319 break;
9320 case i40e_aqc_opc_lldp_update_mib:
9321 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n");
9322#ifdef CONFIG_I40E_DCB
9323 rtnl_lock();
9324 ret = i40e_handle_lldp_event(pf, &event);
9325 rtnl_unlock();
9326#endif
9327 break;
9328 case i40e_aqc_opc_event_lan_overflow:
9329 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n");
9330 i40e_handle_lan_overflow_event(pf, &event);
9331 break;
9332 case i40e_aqc_opc_send_msg_to_peer:
9333 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n");
9334 break;
9335 case i40e_aqc_opc_nvm_erase:
9336 case i40e_aqc_opc_nvm_update:
9337 case i40e_aqc_opc_oem_post_update:
9338 i40e_debug(&pf->hw, I40E_DEBUG_NVM,
9339 "ARQ NVM operation 0x%04x completed\n",
9340 opcode);
9341 break;
9342 default:
9343 dev_info(&pf->pdev->dev,
9344 "ARQ: Unknown event 0x%04x ignored\n",
9345 opcode);
9346 break;
9347 }
9348 } while (i++ < pf->adminq_work_limit);
9349
9350 if (i < pf->adminq_work_limit)
9351 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state);
9352
9353
9354 val = rd32(hw, I40E_PFINT_ICR0_ENA);
9355 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK;
9356 wr32(hw, I40E_PFINT_ICR0_ENA, val);
9357 i40e_flush(hw);
9358
9359 kfree(event.msg_buf);
9360}
9361
9362
9363
9364
9365
9366static void i40e_verify_eeprom(struct i40e_pf *pf)
9367{
9368 int err;
9369
9370 err = i40e_diag_eeprom_test(&pf->hw);
9371 if (err) {
9372
9373 err = i40e_diag_eeprom_test(&pf->hw);
9374 if (err) {
9375 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n",
9376 err);
9377 set_bit(__I40E_BAD_EEPROM, pf->state);
9378 }
9379 }
9380
9381 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) {
9382 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n");
9383 clear_bit(__I40E_BAD_EEPROM, pf->state);
9384 }
9385}
9386
9387
9388
9389
9390
9391
9392
9393static void i40e_enable_pf_switch_lb(struct i40e_pf *pf)
9394{
9395 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9396 struct i40e_vsi_context ctxt;
9397 int ret;
9398
9399 ctxt.seid = pf->main_vsi_seid;
9400 ctxt.pf_num = pf->hw.pf_id;
9401 ctxt.vf_num = 0;
9402 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9403 if (ret) {
9404 dev_info(&pf->pdev->dev,
9405 "couldn't get PF vsi config, err %s aq_err %s\n",
9406 i40e_stat_str(&pf->hw, ret),
9407 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9408 return;
9409 }
9410 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9411 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9412 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9413
9414 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
9415 if (ret) {
9416 dev_info(&pf->pdev->dev,
9417 "update vsi switch failed, err %s aq_err %s\n",
9418 i40e_stat_str(&pf->hw, ret),
9419 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9420 }
9421}
9422
9423
9424
9425
9426
9427
9428
9429static void i40e_disable_pf_switch_lb(struct i40e_pf *pf)
9430{
9431 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9432 struct i40e_vsi_context ctxt;
9433 int ret;
9434
9435 ctxt.seid = pf->main_vsi_seid;
9436 ctxt.pf_num = pf->hw.pf_id;
9437 ctxt.vf_num = 0;
9438 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
9439 if (ret) {
9440 dev_info(&pf->pdev->dev,
9441 "couldn't get PF vsi config, err %s aq_err %s\n",
9442 i40e_stat_str(&pf->hw, ret),
9443 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9444 return;
9445 }
9446 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
9447 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
9448 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
9449
9450 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL);
9451 if (ret) {
9452 dev_info(&pf->pdev->dev,
9453 "update vsi switch failed, err %s aq_err %s\n",
9454 i40e_stat_str(&pf->hw, ret),
9455 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9456 }
9457}
9458
9459
9460
9461
9462
9463
9464
9465
9466
9467static void i40e_config_bridge_mode(struct i40e_veb *veb)
9468{
9469 struct i40e_pf *pf = veb->pf;
9470
9471 if (pf->hw.debug_mask & I40E_DEBUG_LAN)
9472 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n",
9473 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
9474 if (veb->bridge_mode & BRIDGE_MODE_VEPA)
9475 i40e_disable_pf_switch_lb(pf);
9476 else
9477 i40e_enable_pf_switch_lb(pf);
9478}
9479
9480
9481
9482
9483
9484
9485
9486
9487
9488
9489static int i40e_reconstitute_veb(struct i40e_veb *veb)
9490{
9491 struct i40e_vsi *ctl_vsi = NULL;
9492 struct i40e_pf *pf = veb->pf;
9493 int v, veb_idx;
9494 int ret;
9495
9496
9497 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) {
9498 if (pf->vsi[v] &&
9499 pf->vsi[v]->veb_idx == veb->idx &&
9500 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) {
9501 ctl_vsi = pf->vsi[v];
9502 break;
9503 }
9504 }
9505 if (!ctl_vsi) {
9506 dev_info(&pf->pdev->dev,
9507 "missing owner VSI for veb_idx %d\n", veb->idx);
9508 ret = -ENOENT;
9509 goto end_reconstitute;
9510 }
9511 if (ctl_vsi != pf->vsi[pf->lan_vsi])
9512 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
9513 ret = i40e_add_vsi(ctl_vsi);
9514 if (ret) {
9515 dev_info(&pf->pdev->dev,
9516 "rebuild of veb_idx %d owner VSI failed: %d\n",
9517 veb->idx, ret);
9518 goto end_reconstitute;
9519 }
9520 i40e_vsi_reset_stats(ctl_vsi);
9521
9522
9523 ret = i40e_add_veb(veb, ctl_vsi);
9524 if (ret)
9525 goto end_reconstitute;
9526
9527 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
9528 veb->bridge_mode = BRIDGE_MODE_VEB;
9529 else
9530 veb->bridge_mode = BRIDGE_MODE_VEPA;
9531 i40e_config_bridge_mode(veb);
9532
9533
9534 for (v = 0; v < pf->num_alloc_vsi; v++) {
9535 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi)
9536 continue;
9537
9538 if (pf->vsi[v]->veb_idx == veb->idx) {
9539 struct i40e_vsi *vsi = pf->vsi[v];
9540
9541 vsi->uplink_seid = veb->seid;
9542 ret = i40e_add_vsi(vsi);
9543 if (ret) {
9544 dev_info(&pf->pdev->dev,
9545 "rebuild of vsi_idx %d failed: %d\n",
9546 v, ret);
9547 goto end_reconstitute;
9548 }
9549 i40e_vsi_reset_stats(vsi);
9550 }
9551 }
9552
9553
9554 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
9555 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) {
9556 pf->veb[veb_idx]->uplink_seid = veb->seid;
9557 ret = i40e_reconstitute_veb(pf->veb[veb_idx]);
9558 if (ret)
9559 break;
9560 }
9561 }
9562
9563end_reconstitute:
9564 return ret;
9565}
9566
9567
9568
9569
9570
9571static int i40e_get_capabilities(struct i40e_pf *pf,
9572 enum i40e_admin_queue_opc list_type)
9573{
9574 struct i40e_aqc_list_capabilities_element_resp *cap_buf;
9575 u16 data_size;
9576 int buf_len;
9577 int err;
9578
9579 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp);
9580 do {
9581 cap_buf = kzalloc(buf_len, GFP_KERNEL);
9582 if (!cap_buf)
9583 return -ENOMEM;
9584
9585
9586 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len,
9587 &data_size, list_type,
9588 NULL);
9589
9590 kfree(cap_buf);
9591
9592 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) {
9593
9594 buf_len = data_size;
9595 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) {
9596 dev_info(&pf->pdev->dev,
9597 "capability discovery failed, err %s aq_err %s\n",
9598 i40e_stat_str(&pf->hw, err),
9599 i40e_aq_str(&pf->hw,
9600 pf->hw.aq.asq_last_status));
9601 return -ENODEV;
9602 }
9603 } while (err);
9604
9605 if (pf->hw.debug_mask & I40E_DEBUG_USER) {
9606 if (list_type == i40e_aqc_opc_list_func_capabilities) {
9607 dev_info(&pf->pdev->dev,
9608 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n",
9609 pf->hw.pf_id, pf->hw.func_caps.num_vfs,
9610 pf->hw.func_caps.num_msix_vectors,
9611 pf->hw.func_caps.num_msix_vectors_vf,
9612 pf->hw.func_caps.fd_filters_guaranteed,
9613 pf->hw.func_caps.fd_filters_best_effort,
9614 pf->hw.func_caps.num_tx_qp,
9615 pf->hw.func_caps.num_vsis);
9616 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) {
9617 dev_info(&pf->pdev->dev,
9618 "switch_mode=0x%04x, function_valid=0x%08x\n",
9619 pf->hw.dev_caps.switch_mode,
9620 pf->hw.dev_caps.valid_functions);
9621 dev_info(&pf->pdev->dev,
9622 "SR-IOV=%d, num_vfs for all function=%u\n",
9623 pf->hw.dev_caps.sr_iov_1_1,
9624 pf->hw.dev_caps.num_vfs);
9625 dev_info(&pf->pdev->dev,
9626 "num_vsis=%u, num_rx:%u, num_tx=%u\n",
9627 pf->hw.dev_caps.num_vsis,
9628 pf->hw.dev_caps.num_rx_qp,
9629 pf->hw.dev_caps.num_tx_qp);
9630 }
9631 }
9632 if (list_type == i40e_aqc_opc_list_func_capabilities) {
9633#define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \
9634 + pf->hw.func_caps.num_vfs)
9635 if (pf->hw.revision_id == 0 &&
9636 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) {
9637 dev_info(&pf->pdev->dev,
9638 "got num_vsis %d, setting num_vsis to %d\n",
9639 pf->hw.func_caps.num_vsis, DEF_NUM_VSI);
9640 pf->hw.func_caps.num_vsis = DEF_NUM_VSI;
9641 }
9642 }
9643 return 0;
9644}
9645
9646static int i40e_vsi_clear(struct i40e_vsi *vsi);
9647
9648
9649
9650
9651
9652static void i40e_fdir_sb_setup(struct i40e_pf *pf)
9653{
9654 struct i40e_vsi *vsi;
9655
9656
9657
9658
9659 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) {
9660 static const u32 hkey[] = {
9661 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36,
9662 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb,
9663 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21,
9664 0x95b3a76d};
9665 int i;
9666
9667 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++)
9668 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]);
9669 }
9670
9671 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
9672 return;
9673
9674
9675 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
9676
9677
9678 if (!vsi) {
9679 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR,
9680 pf->vsi[pf->lan_vsi]->seid, 0);
9681 if (!vsi) {
9682 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n");
9683 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
9684 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
9685 return;
9686 }
9687 }
9688
9689 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring);
9690}
9691
9692
9693
9694
9695
9696static void i40e_fdir_teardown(struct i40e_pf *pf)
9697{
9698 struct i40e_vsi *vsi;
9699
9700 i40e_fdir_filter_exit(pf);
9701 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR);
9702 if (vsi)
9703 i40e_vsi_release(vsi);
9704}
9705
9706
9707
9708
9709
9710
9711
9712
9713
9714static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid)
9715{
9716 struct i40e_cloud_filter *cfilter;
9717 struct i40e_pf *pf = vsi->back;
9718 struct hlist_node *node;
9719 i40e_status ret;
9720
9721
9722 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list,
9723 cloud_node) {
9724 if (cfilter->seid != seid)
9725 continue;
9726
9727 if (cfilter->dst_port)
9728 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter,
9729 true);
9730 else
9731 ret = i40e_add_del_cloud_filter(vsi, cfilter, true);
9732
9733 if (ret) {
9734 dev_dbg(&pf->pdev->dev,
9735 "Failed to rebuild cloud filter, err %s aq_err %s\n",
9736 i40e_stat_str(&pf->hw, ret),
9737 i40e_aq_str(&pf->hw,
9738 pf->hw.aq.asq_last_status));
9739 return ret;
9740 }
9741 }
9742 return 0;
9743}
9744
9745
9746
9747
9748
9749
9750
9751static int i40e_rebuild_channels(struct i40e_vsi *vsi)
9752{
9753 struct i40e_channel *ch, *ch_tmp;
9754 i40e_status ret;
9755
9756 if (list_empty(&vsi->ch_list))
9757 return 0;
9758
9759 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) {
9760 if (!ch->initialized)
9761 break;
9762
9763 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch);
9764 if (ret) {
9765 dev_info(&vsi->back->pdev->dev,
9766 "failed to rebuild channels using uplink_seid %u\n",
9767 vsi->uplink_seid);
9768 return ret;
9769 }
9770
9771 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch);
9772 if (ret) {
9773 dev_info(&vsi->back->pdev->dev,
9774 "failed to configure TX rings for channel %u\n",
9775 ch->seid);
9776 return ret;
9777 }
9778
9779 vsi->next_base_queue = vsi->next_base_queue +
9780 ch->num_queue_pairs;
9781 if (ch->max_tx_rate) {
9782 u64 credits = ch->max_tx_rate;
9783
9784 if (i40e_set_bw_limit(vsi, ch->seid,
9785 ch->max_tx_rate))
9786 return -EINVAL;
9787
9788 do_div(credits, I40E_BW_CREDIT_DIVISOR);
9789 dev_dbg(&vsi->back->pdev->dev,
9790 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
9791 ch->max_tx_rate,
9792 credits,
9793 ch->seid);
9794 }
9795 ret = i40e_rebuild_cloud_filters(vsi, ch->seid);
9796 if (ret) {
9797 dev_dbg(&vsi->back->pdev->dev,
9798 "Failed to rebuild cloud filters for channel VSI %u\n",
9799 ch->seid);
9800 return ret;
9801 }
9802 }
9803 return 0;
9804}
9805
9806
9807
9808
9809
9810
9811
9812
9813
9814static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired)
9815{
9816 struct i40e_hw *hw = &pf->hw;
9817 i40e_status ret = 0;
9818 u32 v;
9819
9820 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state);
9821 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state))
9822 return;
9823 if (i40e_check_asq_alive(&pf->hw))
9824 i40e_vc_notify_reset(pf);
9825
9826 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
9827
9828
9829
9830 if (!lock_acquired)
9831 rtnl_lock();
9832 i40e_pf_quiesce_all_vsi(pf);
9833 if (!lock_acquired)
9834 rtnl_unlock();
9835
9836 for (v = 0; v < pf->num_alloc_vsi; v++) {
9837 if (pf->vsi[v])
9838 pf->vsi[v]->seid = 0;
9839 }
9840
9841 i40e_shutdown_adminq(&pf->hw);
9842
9843
9844 if (hw->hmc.hmc_obj) {
9845 ret = i40e_shutdown_lan_hmc(hw);
9846 if (ret)
9847 dev_warn(&pf->pdev->dev,
9848 "shutdown_lan_hmc failed: %d\n", ret);
9849 }
9850
9851
9852
9853
9854 i40e_ptp_save_hw_time(pf);
9855}
9856
9857
9858
9859
9860
9861static void i40e_send_version(struct i40e_pf *pf)
9862{
9863 struct i40e_driver_version dv;
9864
9865 dv.major_version = DRV_VERSION_MAJOR;
9866 dv.minor_version = DRV_VERSION_MINOR;
9867 dv.build_version = DRV_VERSION_BUILD;
9868 dv.subbuild_version = 0;
9869 strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string));
9870 i40e_aq_send_driver_version(&pf->hw, &dv, NULL);
9871}
9872
9873
9874
9875
9876
9877static void i40e_get_oem_version(struct i40e_hw *hw)
9878{
9879 u16 block_offset = 0xffff;
9880 u16 block_length = 0;
9881 u16 capabilities = 0;
9882 u16 gen_snap = 0;
9883 u16 release = 0;
9884
9885#define I40E_SR_NVM_OEM_VERSION_PTR 0x1B
9886#define I40E_NVM_OEM_LENGTH_OFFSET 0x00
9887#define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01
9888#define I40E_NVM_OEM_GEN_OFFSET 0x02
9889#define I40E_NVM_OEM_RELEASE_OFFSET 0x03
9890#define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F
9891#define I40E_NVM_OEM_LENGTH 3
9892
9893
9894 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset);
9895 if (block_offset == 0xffff)
9896 return;
9897
9898
9899 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET,
9900 &block_length);
9901 if (block_length < I40E_NVM_OEM_LENGTH)
9902 return;
9903
9904
9905 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET,
9906 &capabilities);
9907 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0)
9908 return;
9909
9910 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET,
9911 &gen_snap);
9912 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET,
9913 &release);
9914 hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release;
9915 hw->nvm.eetrack = I40E_OEM_EETRACK_ID;
9916}
9917
9918
9919
9920
9921
9922static int i40e_reset(struct i40e_pf *pf)
9923{
9924 struct i40e_hw *hw = &pf->hw;
9925 i40e_status ret;
9926
9927 ret = i40e_pf_reset(hw);
9928 if (ret) {
9929 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret);
9930 set_bit(__I40E_RESET_FAILED, pf->state);
9931 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
9932 } else {
9933 pf->pfr_count++;
9934 }
9935 return ret;
9936}
9937
9938
9939
9940
9941
9942
9943
9944
9945static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired)
9946{
9947 int old_recovery_mode_bit = test_bit(__I40E_RECOVERY_MODE, pf->state);
9948 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
9949 struct i40e_hw *hw = &pf->hw;
9950 u8 set_fc_aq_fail = 0;
9951 i40e_status ret;
9952 u32 val;
9953 int v;
9954
9955 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
9956 i40e_check_recovery_mode(pf)) {
9957 i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev);
9958 }
9959
9960 if (test_bit(__I40E_DOWN, pf->state) &&
9961 !test_bit(__I40E_RECOVERY_MODE, pf->state) &&
9962 !old_recovery_mode_bit)
9963 goto clear_recovery;
9964 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n");
9965
9966
9967 ret = i40e_init_adminq(&pf->hw);
9968 if (ret) {
9969 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n",
9970 i40e_stat_str(&pf->hw, ret),
9971 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
9972 goto clear_recovery;
9973 }
9974 i40e_get_oem_version(&pf->hw);
9975
9976 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) &&
9977 ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) ||
9978 hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) {
9979
9980
9981
9982
9983
9984 mdelay(300);
9985 }
9986
9987
9988 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state))
9989 i40e_verify_eeprom(pf);
9990
9991
9992
9993
9994
9995 if (test_bit(__I40E_RECOVERY_MODE, pf->state) ||
9996 old_recovery_mode_bit) {
9997 if (i40e_get_capabilities(pf,
9998 i40e_aqc_opc_list_func_capabilities))
9999 goto end_unlock;
10000
10001 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10002
10003
10004
10005 if (i40e_setup_misc_vector_for_recovery_mode(pf))
10006 goto end_unlock;
10007 } else {
10008 if (!lock_acquired)
10009 rtnl_lock();
10010
10011
10012
10013
10014 free_irq(pf->pdev->irq, pf);
10015 i40e_clear_interrupt_scheme(pf);
10016 if (i40e_restore_interrupt_scheme(pf))
10017 goto end_unlock;
10018 }
10019
10020
10021 i40e_send_version(pf);
10022
10023
10024
10025
10026 goto end_unlock;
10027 }
10028
10029 i40e_clear_pxe_mode(hw);
10030 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
10031 if (ret)
10032 goto end_core_reset;
10033
10034 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
10035 hw->func_caps.num_rx_qp, 0, 0);
10036 if (ret) {
10037 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret);
10038 goto end_core_reset;
10039 }
10040 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
10041 if (ret) {
10042 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret);
10043 goto end_core_reset;
10044 }
10045
10046
10047 i40e_aq_set_dcb_parameters(hw, true, NULL);
10048
10049#ifdef CONFIG_I40E_DCB
10050 ret = i40e_init_pf_dcb(pf);
10051 if (ret) {
10052 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret);
10053 pf->flags &= ~I40E_FLAG_DCB_CAPABLE;
10054
10055 }
10056#endif
10057
10058 if (!lock_acquired)
10059 rtnl_lock();
10060 ret = i40e_setup_pf_switch(pf, reinit);
10061 if (ret)
10062 goto end_unlock;
10063
10064
10065
10066
10067 ret = i40e_aq_set_phy_int_mask(&pf->hw,
10068 ~(I40E_AQ_EVENT_LINK_UPDOWN |
10069 I40E_AQ_EVENT_MEDIA_NA |
10070 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
10071 if (ret)
10072 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
10073 i40e_stat_str(&pf->hw, ret),
10074 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10075
10076
10077 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true);
10078 if (ret)
10079 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n",
10080 i40e_stat_str(&pf->hw, ret),
10081 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10082
10083
10084
10085
10086
10087
10088
10089
10090 if (vsi->uplink_seid != pf->mac_seid) {
10091 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n");
10092
10093 for (v = 0; v < I40E_MAX_VEB; v++) {
10094 if (!pf->veb[v])
10095 continue;
10096
10097 if (pf->veb[v]->uplink_seid == pf->mac_seid ||
10098 pf->veb[v]->uplink_seid == 0) {
10099 ret = i40e_reconstitute_veb(pf->veb[v]);
10100
10101 if (!ret)
10102 continue;
10103
10104
10105
10106
10107
10108
10109
10110 if (pf->veb[v]->uplink_seid == pf->mac_seid) {
10111 dev_info(&pf->pdev->dev,
10112 "rebuild of switch failed: %d, will try to set up simple PF connection\n",
10113 ret);
10114 vsi->uplink_seid = pf->mac_seid;
10115 break;
10116 } else if (pf->veb[v]->uplink_seid == 0) {
10117 dev_info(&pf->pdev->dev,
10118 "rebuild of orphan VEB failed: %d\n",
10119 ret);
10120 }
10121 }
10122 }
10123 }
10124
10125 if (vsi->uplink_seid == pf->mac_seid) {
10126 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n");
10127
10128 ret = i40e_add_vsi(vsi);
10129 if (ret) {
10130 dev_info(&pf->pdev->dev,
10131 "rebuild of Main VSI failed: %d\n", ret);
10132 goto end_unlock;
10133 }
10134 }
10135
10136 if (vsi->mqprio_qopt.max_rate[0]) {
10137 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0];
10138 u64 credits = 0;
10139
10140 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR);
10141 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate);
10142 if (ret)
10143 goto end_unlock;
10144
10145 credits = max_tx_rate;
10146 do_div(credits, I40E_BW_CREDIT_DIVISOR);
10147 dev_dbg(&vsi->back->pdev->dev,
10148 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n",
10149 max_tx_rate,
10150 credits,
10151 vsi->seid);
10152 }
10153
10154 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid);
10155 if (ret)
10156 goto end_unlock;
10157
10158
10159
10160
10161 ret = i40e_rebuild_channels(vsi);
10162 if (ret)
10163 goto end_unlock;
10164
10165
10166
10167
10168
10169#define I40E_REG_MSS 0x000E64DC
10170#define I40E_REG_MSS_MIN_MASK 0x3FF0000
10171#define I40E_64BYTE_MSS 0x400000
10172 val = rd32(hw, I40E_REG_MSS);
10173 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
10174 val &= ~I40E_REG_MSS_MIN_MASK;
10175 val |= I40E_64BYTE_MSS;
10176 wr32(hw, I40E_REG_MSS, val);
10177 }
10178
10179 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
10180 msleep(75);
10181 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
10182 if (ret)
10183 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
10184 i40e_stat_str(&pf->hw, ret),
10185 i40e_aq_str(&pf->hw,
10186 pf->hw.aq.asq_last_status));
10187 }
10188
10189 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
10190 ret = i40e_setup_misc_vector(pf);
10191
10192
10193
10194
10195
10196
10197
10198 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
10199 pf->main_vsi_seid);
10200
10201
10202 i40e_pf_unquiesce_all_vsi(pf);
10203
10204
10205 if (!lock_acquired)
10206 rtnl_unlock();
10207
10208
10209 ret = i40e_set_promiscuous(pf, pf->cur_promisc);
10210 if (ret)
10211 dev_warn(&pf->pdev->dev,
10212 "Failed to restore promiscuous setting: %s, err %s aq_err %s\n",
10213 pf->cur_promisc ? "on" : "off",
10214 i40e_stat_str(&pf->hw, ret),
10215 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
10216
10217 i40e_reset_all_vfs(pf, true);
10218
10219
10220 i40e_send_version(pf);
10221
10222
10223 goto end_core_reset;
10224
10225end_unlock:
10226 if (!lock_acquired)
10227 rtnl_unlock();
10228end_core_reset:
10229 clear_bit(__I40E_RESET_FAILED, pf->state);
10230clear_recovery:
10231 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state);
10232 clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state);
10233}
10234
10235
10236
10237
10238
10239
10240
10241
10242static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit,
10243 bool lock_acquired)
10244{
10245 int ret;
10246
10247
10248
10249
10250 ret = i40e_reset(pf);
10251 if (!ret)
10252 i40e_rebuild(pf, reinit, lock_acquired);
10253}
10254
10255
10256
10257
10258
10259
10260
10261
10262
10263
10264static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired)
10265{
10266 i40e_prep_for_reset(pf, lock_acquired);
10267 i40e_reset_and_rebuild(pf, false, lock_acquired);
10268}
10269
10270
10271
10272
10273
10274
10275
10276static void i40e_handle_mdd_event(struct i40e_pf *pf)
10277{
10278 struct i40e_hw *hw = &pf->hw;
10279 bool mdd_detected = false;
10280 struct i40e_vf *vf;
10281 u32 reg;
10282 int i;
10283
10284 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state))
10285 return;
10286
10287
10288 reg = rd32(hw, I40E_GL_MDET_TX);
10289 if (reg & I40E_GL_MDET_TX_VALID_MASK) {
10290 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >>
10291 I40E_GL_MDET_TX_PF_NUM_SHIFT;
10292 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >>
10293 I40E_GL_MDET_TX_VF_NUM_SHIFT;
10294 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >>
10295 I40E_GL_MDET_TX_EVENT_SHIFT;
10296 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >>
10297 I40E_GL_MDET_TX_QUEUE_SHIFT) -
10298 pf->hw.func_caps.base_queue;
10299 if (netif_msg_tx_err(pf))
10300 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n",
10301 event, queue, pf_num, vf_num);
10302 wr32(hw, I40E_GL_MDET_TX, 0xffffffff);
10303 mdd_detected = true;
10304 }
10305 reg = rd32(hw, I40E_GL_MDET_RX);
10306 if (reg & I40E_GL_MDET_RX_VALID_MASK) {
10307 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >>
10308 I40E_GL_MDET_RX_FUNCTION_SHIFT;
10309 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >>
10310 I40E_GL_MDET_RX_EVENT_SHIFT;
10311 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >>
10312 I40E_GL_MDET_RX_QUEUE_SHIFT) -
10313 pf->hw.func_caps.base_queue;
10314 if (netif_msg_rx_err(pf))
10315 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n",
10316 event, queue, func);
10317 wr32(hw, I40E_GL_MDET_RX, 0xffffffff);
10318 mdd_detected = true;
10319 }
10320
10321 if (mdd_detected) {
10322 reg = rd32(hw, I40E_PF_MDET_TX);
10323 if (reg & I40E_PF_MDET_TX_VALID_MASK) {
10324 wr32(hw, I40E_PF_MDET_TX, 0xFFFF);
10325 dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n");
10326 }
10327 reg = rd32(hw, I40E_PF_MDET_RX);
10328 if (reg & I40E_PF_MDET_RX_VALID_MASK) {
10329 wr32(hw, I40E_PF_MDET_RX, 0xFFFF);
10330 dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n");
10331 }
10332 }
10333
10334
10335 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) {
10336 vf = &(pf->vf[i]);
10337 reg = rd32(hw, I40E_VP_MDET_TX(i));
10338 if (reg & I40E_VP_MDET_TX_VALID_MASK) {
10339 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF);
10340 vf->num_mdd_events++;
10341 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n",
10342 i);
10343 dev_info(&pf->pdev->dev,
10344 "Use PF Control I/F to re-enable the VF\n");
10345 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
10346 }
10347
10348 reg = rd32(hw, I40E_VP_MDET_RX(i));
10349 if (reg & I40E_VP_MDET_RX_VALID_MASK) {
10350 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF);
10351 vf->num_mdd_events++;
10352 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n",
10353 i);
10354 dev_info(&pf->pdev->dev,
10355 "Use PF Control I/F to re-enable the VF\n");
10356 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states);
10357 }
10358 }
10359
10360
10361 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state);
10362 reg = rd32(hw, I40E_PFINT_ICR0_ENA);
10363 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK;
10364 wr32(hw, I40E_PFINT_ICR0_ENA, reg);
10365 i40e_flush(hw);
10366}
10367
10368static const char *i40e_tunnel_name(u8 type)
10369{
10370 switch (type) {
10371 case UDP_TUNNEL_TYPE_VXLAN:
10372 return "vxlan";
10373 case UDP_TUNNEL_TYPE_GENEVE:
10374 return "geneve";
10375 default:
10376 return "unknown";
10377 }
10378}
10379
10380
10381
10382
10383
10384static void i40e_sync_udp_filters(struct i40e_pf *pf)
10385{
10386 int i;
10387
10388
10389 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
10390 if (pf->udp_ports[i].port)
10391 pf->pending_udp_bitmap |= BIT_ULL(i);
10392 }
10393
10394 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
10395}
10396
10397
10398
10399
10400
10401static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf)
10402{
10403 struct i40e_hw *hw = &pf->hw;
10404 u8 filter_index, type;
10405 u16 port;
10406 int i;
10407
10408 if (!test_and_clear_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state))
10409 return;
10410
10411
10412 rtnl_lock();
10413
10414 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
10415 if (pf->pending_udp_bitmap & BIT_ULL(i)) {
10416 struct i40e_udp_port_config *udp_port;
10417 i40e_status ret = 0;
10418
10419 udp_port = &pf->udp_ports[i];
10420 pf->pending_udp_bitmap &= ~BIT_ULL(i);
10421
10422 port = READ_ONCE(udp_port->port);
10423 type = READ_ONCE(udp_port->type);
10424 filter_index = READ_ONCE(udp_port->filter_index);
10425
10426
10427 rtnl_unlock();
10428
10429 if (port)
10430 ret = i40e_aq_add_udp_tunnel(hw, port,
10431 type,
10432 &filter_index,
10433 NULL);
10434 else if (filter_index != I40E_UDP_PORT_INDEX_UNUSED)
10435 ret = i40e_aq_del_udp_tunnel(hw, filter_index,
10436 NULL);
10437
10438
10439 rtnl_lock();
10440
10441 if (ret) {
10442 dev_info(&pf->pdev->dev,
10443 "%s %s port %d, index %d failed, err %s aq_err %s\n",
10444 i40e_tunnel_name(type),
10445 port ? "add" : "delete",
10446 port,
10447 filter_index,
10448 i40e_stat_str(&pf->hw, ret),
10449 i40e_aq_str(&pf->hw,
10450 pf->hw.aq.asq_last_status));
10451 if (port) {
10452
10453
10454
10455 udp_port->port = 0;
10456 pf->pending_udp_bitmap &= ~BIT_ULL(i);
10457 }
10458 } else if (port) {
10459
10460 udp_port->filter_index = filter_index;
10461 }
10462 }
10463 }
10464
10465 rtnl_unlock();
10466}
10467
10468
10469
10470
10471
10472static void i40e_service_task(struct work_struct *work)
10473{
10474 struct i40e_pf *pf = container_of(work,
10475 struct i40e_pf,
10476 service_task);
10477 unsigned long start_time = jiffies;
10478
10479
10480 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) ||
10481 test_bit(__I40E_SUSPENDED, pf->state))
10482 return;
10483
10484 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state))
10485 return;
10486
10487 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) {
10488 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]);
10489 i40e_sync_filters_subtask(pf);
10490 i40e_reset_subtask(pf);
10491 i40e_handle_mdd_event(pf);
10492 i40e_vc_process_vflr_event(pf);
10493 i40e_watchdog_subtask(pf);
10494 i40e_fdir_reinit_subtask(pf);
10495 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) {
10496
10497 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi],
10498 true);
10499 } else {
10500 i40e_client_subtask(pf);
10501 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE,
10502 pf->state))
10503 i40e_notify_client_of_l2_param_changes(
10504 pf->vsi[pf->lan_vsi]);
10505 }
10506 i40e_sync_filters_subtask(pf);
10507 i40e_sync_udp_filters_subtask(pf);
10508 } else {
10509 i40e_reset_subtask(pf);
10510 }
10511
10512 i40e_clean_adminq_subtask(pf);
10513
10514
10515 smp_mb__before_atomic();
10516 clear_bit(__I40E_SERVICE_SCHED, pf->state);
10517
10518
10519
10520
10521
10522 if (time_after(jiffies, (start_time + pf->service_timer_period)) ||
10523 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) ||
10524 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) ||
10525 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state))
10526 i40e_service_event_schedule(pf);
10527}
10528
10529
10530
10531
10532
10533static void i40e_service_timer(struct timer_list *t)
10534{
10535 struct i40e_pf *pf = from_timer(pf, t, service_timer);
10536
10537 mod_timer(&pf->service_timer,
10538 round_jiffies(jiffies + pf->service_timer_period));
10539 i40e_service_event_schedule(pf);
10540}
10541
10542
10543
10544
10545
10546static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi)
10547{
10548 struct i40e_pf *pf = vsi->back;
10549
10550 switch (vsi->type) {
10551 case I40E_VSI_MAIN:
10552 vsi->alloc_queue_pairs = pf->num_lan_qps;
10553 if (!vsi->num_tx_desc)
10554 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10555 I40E_REQ_DESCRIPTOR_MULTIPLE);
10556 if (!vsi->num_rx_desc)
10557 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10558 I40E_REQ_DESCRIPTOR_MULTIPLE);
10559 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
10560 vsi->num_q_vectors = pf->num_lan_msix;
10561 else
10562 vsi->num_q_vectors = 1;
10563
10564 break;
10565
10566 case I40E_VSI_FDIR:
10567 vsi->alloc_queue_pairs = 1;
10568 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT,
10569 I40E_REQ_DESCRIPTOR_MULTIPLE);
10570 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT,
10571 I40E_REQ_DESCRIPTOR_MULTIPLE);
10572 vsi->num_q_vectors = pf->num_fdsb_msix;
10573 break;
10574
10575 case I40E_VSI_VMDQ2:
10576 vsi->alloc_queue_pairs = pf->num_vmdq_qps;
10577 if (!vsi->num_tx_desc)
10578 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10579 I40E_REQ_DESCRIPTOR_MULTIPLE);
10580 if (!vsi->num_rx_desc)
10581 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10582 I40E_REQ_DESCRIPTOR_MULTIPLE);
10583 vsi->num_q_vectors = pf->num_vmdq_msix;
10584 break;
10585
10586 case I40E_VSI_SRIOV:
10587 vsi->alloc_queue_pairs = pf->num_vf_qps;
10588 if (!vsi->num_tx_desc)
10589 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10590 I40E_REQ_DESCRIPTOR_MULTIPLE);
10591 if (!vsi->num_rx_desc)
10592 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS,
10593 I40E_REQ_DESCRIPTOR_MULTIPLE);
10594 break;
10595
10596 default:
10597 WARN_ON(1);
10598 return -ENODATA;
10599 }
10600
10601 return 0;
10602}
10603
10604
10605
10606
10607
10608
10609
10610
10611
10612static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors)
10613{
10614 struct i40e_ring **next_rings;
10615 int size;
10616 int ret = 0;
10617
10618
10619 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs *
10620 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2);
10621 vsi->tx_rings = kzalloc(size, GFP_KERNEL);
10622 if (!vsi->tx_rings)
10623 return -ENOMEM;
10624 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs;
10625 if (i40e_enabled_xdp_vsi(vsi)) {
10626 vsi->xdp_rings = next_rings;
10627 next_rings += vsi->alloc_queue_pairs;
10628 }
10629 vsi->rx_rings = next_rings;
10630
10631 if (alloc_qvectors) {
10632
10633 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors;
10634 vsi->q_vectors = kzalloc(size, GFP_KERNEL);
10635 if (!vsi->q_vectors) {
10636 ret = -ENOMEM;
10637 goto err_vectors;
10638 }
10639 }
10640 return ret;
10641
10642err_vectors:
10643 kfree(vsi->tx_rings);
10644 return ret;
10645}
10646
10647
10648
10649
10650
10651
10652
10653
10654
10655static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type)
10656{
10657 int ret = -ENODEV;
10658 struct i40e_vsi *vsi;
10659 int vsi_idx;
10660 int i;
10661
10662
10663 mutex_lock(&pf->switch_mutex);
10664
10665
10666
10667
10668
10669
10670
10671 i = pf->next_vsi;
10672 while (i < pf->num_alloc_vsi && pf->vsi[i])
10673 i++;
10674 if (i >= pf->num_alloc_vsi) {
10675 i = 0;
10676 while (i < pf->next_vsi && pf->vsi[i])
10677 i++;
10678 }
10679
10680 if (i < pf->num_alloc_vsi && !pf->vsi[i]) {
10681 vsi_idx = i;
10682 } else {
10683 ret = -ENODEV;
10684 goto unlock_pf;
10685 }
10686 pf->next_vsi = ++i;
10687
10688 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL);
10689 if (!vsi) {
10690 ret = -ENOMEM;
10691 goto unlock_pf;
10692 }
10693 vsi->type = type;
10694 vsi->back = pf;
10695 set_bit(__I40E_VSI_DOWN, vsi->state);
10696 vsi->flags = 0;
10697 vsi->idx = vsi_idx;
10698 vsi->int_rate_limit = 0;
10699 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ?
10700 pf->rss_table_size : 64;
10701 vsi->netdev_registered = false;
10702 vsi->work_limit = I40E_DEFAULT_IRQ_WORK;
10703 hash_init(vsi->mac_filter_hash);
10704 vsi->irqs_ready = false;
10705
10706 if (type == I40E_VSI_MAIN) {
10707 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL);
10708 if (!vsi->af_xdp_zc_qps)
10709 goto err_rings;
10710 }
10711
10712 ret = i40e_set_num_rings_in_vsi(vsi);
10713 if (ret)
10714 goto err_rings;
10715
10716 ret = i40e_vsi_alloc_arrays(vsi, true);
10717 if (ret)
10718 goto err_rings;
10719
10720
10721 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings);
10722
10723
10724 spin_lock_init(&vsi->mac_filter_hash_lock);
10725 pf->vsi[vsi_idx] = vsi;
10726 ret = vsi_idx;
10727 goto unlock_pf;
10728
10729err_rings:
10730 bitmap_free(vsi->af_xdp_zc_qps);
10731 pf->next_vsi = i - 1;
10732 kfree(vsi);
10733unlock_pf:
10734 mutex_unlock(&pf->switch_mutex);
10735 return ret;
10736}
10737
10738
10739
10740
10741
10742
10743
10744
10745
10746static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors)
10747{
10748
10749 if (free_qvectors) {
10750 kfree(vsi->q_vectors);
10751 vsi->q_vectors = NULL;
10752 }
10753 kfree(vsi->tx_rings);
10754 vsi->tx_rings = NULL;
10755 vsi->rx_rings = NULL;
10756 vsi->xdp_rings = NULL;
10757}
10758
10759
10760
10761
10762
10763
10764static void i40e_clear_rss_config_user(struct i40e_vsi *vsi)
10765{
10766 if (!vsi)
10767 return;
10768
10769 kfree(vsi->rss_hkey_user);
10770 vsi->rss_hkey_user = NULL;
10771
10772 kfree(vsi->rss_lut_user);
10773 vsi->rss_lut_user = NULL;
10774}
10775
10776
10777
10778
10779
10780static int i40e_vsi_clear(struct i40e_vsi *vsi)
10781{
10782 struct i40e_pf *pf;
10783
10784 if (!vsi)
10785 return 0;
10786
10787 if (!vsi->back)
10788 goto free_vsi;
10789 pf = vsi->back;
10790
10791 mutex_lock(&pf->switch_mutex);
10792 if (!pf->vsi[vsi->idx]) {
10793 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n",
10794 vsi->idx, vsi->idx, vsi->type);
10795 goto unlock_vsi;
10796 }
10797
10798 if (pf->vsi[vsi->idx] != vsi) {
10799 dev_err(&pf->pdev->dev,
10800 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n",
10801 pf->vsi[vsi->idx]->idx,
10802 pf->vsi[vsi->idx]->type,
10803 vsi->idx, vsi->type);
10804 goto unlock_vsi;
10805 }
10806
10807
10808 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
10809 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx);
10810
10811 bitmap_free(vsi->af_xdp_zc_qps);
10812 i40e_vsi_free_arrays(vsi, true);
10813 i40e_clear_rss_config_user(vsi);
10814
10815 pf->vsi[vsi->idx] = NULL;
10816 if (vsi->idx < pf->next_vsi)
10817 pf->next_vsi = vsi->idx;
10818
10819unlock_vsi:
10820 mutex_unlock(&pf->switch_mutex);
10821free_vsi:
10822 kfree(vsi);
10823
10824 return 0;
10825}
10826
10827
10828
10829
10830
10831static void i40e_vsi_clear_rings(struct i40e_vsi *vsi)
10832{
10833 int i;
10834
10835 if (vsi->tx_rings && vsi->tx_rings[0]) {
10836 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
10837 kfree_rcu(vsi->tx_rings[i], rcu);
10838 vsi->tx_rings[i] = NULL;
10839 vsi->rx_rings[i] = NULL;
10840 if (vsi->xdp_rings)
10841 vsi->xdp_rings[i] = NULL;
10842 }
10843 }
10844}
10845
10846
10847
10848
10849
10850static int i40e_alloc_rings(struct i40e_vsi *vsi)
10851{
10852 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2;
10853 struct i40e_pf *pf = vsi->back;
10854 struct i40e_ring *ring;
10855
10856
10857 for (i = 0; i < vsi->alloc_queue_pairs; i++) {
10858
10859 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL);
10860 if (!ring)
10861 goto err_out;
10862
10863 ring->queue_index = i;
10864 ring->reg_idx = vsi->base_queue + i;
10865 ring->ring_active = false;
10866 ring->vsi = vsi;
10867 ring->netdev = vsi->netdev;
10868 ring->dev = &pf->pdev->dev;
10869 ring->count = vsi->num_tx_desc;
10870 ring->size = 0;
10871 ring->dcb_tc = 0;
10872 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
10873 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
10874 ring->itr_setting = pf->tx_itr_default;
10875 vsi->tx_rings[i] = ring++;
10876
10877 if (!i40e_enabled_xdp_vsi(vsi))
10878 goto setup_rx;
10879
10880 ring->queue_index = vsi->alloc_queue_pairs + i;
10881 ring->reg_idx = vsi->base_queue + ring->queue_index;
10882 ring->ring_active = false;
10883 ring->vsi = vsi;
10884 ring->netdev = NULL;
10885 ring->dev = &pf->pdev->dev;
10886 ring->count = vsi->num_tx_desc;
10887 ring->size = 0;
10888 ring->dcb_tc = 0;
10889 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE)
10890 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR;
10891 set_ring_xdp(ring);
10892 ring->itr_setting = pf->tx_itr_default;
10893 vsi->xdp_rings[i] = ring++;
10894
10895setup_rx:
10896 ring->queue_index = i;
10897 ring->reg_idx = vsi->base_queue + i;
10898 ring->ring_active = false;
10899 ring->vsi = vsi;
10900 ring->netdev = vsi->netdev;
10901 ring->dev = &pf->pdev->dev;
10902 ring->count = vsi->num_rx_desc;
10903 ring->size = 0;
10904 ring->dcb_tc = 0;
10905 ring->itr_setting = pf->rx_itr_default;
10906 vsi->rx_rings[i] = ring;
10907 }
10908
10909 return 0;
10910
10911err_out:
10912 i40e_vsi_clear_rings(vsi);
10913 return -ENOMEM;
10914}
10915
10916
10917
10918
10919
10920
10921
10922
10923static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors)
10924{
10925 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries,
10926 I40E_MIN_MSIX, vectors);
10927 if (vectors < 0) {
10928 dev_info(&pf->pdev->dev,
10929 "MSI-X vector reservation failed: %d\n", vectors);
10930 vectors = 0;
10931 }
10932
10933 return vectors;
10934}
10935
10936
10937
10938
10939
10940
10941
10942
10943
10944static int i40e_init_msix(struct i40e_pf *pf)
10945{
10946 struct i40e_hw *hw = &pf->hw;
10947 int cpus, extra_vectors;
10948 int vectors_left;
10949 int v_budget, i;
10950 int v_actual;
10951 int iwarp_requested = 0;
10952
10953 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
10954 return -ENODEV;
10955
10956
10957
10958
10959
10960
10961
10962
10963
10964
10965
10966
10967
10968
10969
10970
10971 vectors_left = hw->func_caps.num_msix_vectors;
10972 v_budget = 0;
10973
10974
10975 if (vectors_left) {
10976 v_budget++;
10977 vectors_left--;
10978 }
10979
10980
10981
10982
10983
10984
10985
10986
10987 cpus = num_online_cpus();
10988 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2);
10989 vectors_left -= pf->num_lan_msix;
10990
10991
10992 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
10993 if (vectors_left) {
10994 pf->num_fdsb_msix = 1;
10995 v_budget++;
10996 vectors_left--;
10997 } else {
10998 pf->num_fdsb_msix = 0;
10999 }
11000 }
11001
11002
11003 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11004 iwarp_requested = pf->num_iwarp_msix;
11005
11006 if (!vectors_left)
11007 pf->num_iwarp_msix = 0;
11008 else if (vectors_left < pf->num_iwarp_msix)
11009 pf->num_iwarp_msix = 1;
11010 v_budget += pf->num_iwarp_msix;
11011 vectors_left -= pf->num_iwarp_msix;
11012 }
11013
11014
11015 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) {
11016 if (!vectors_left) {
11017 pf->num_vmdq_msix = 0;
11018 pf->num_vmdq_qps = 0;
11019 } else {
11020 int vmdq_vecs_wanted =
11021 pf->num_vmdq_vsis * pf->num_vmdq_qps;
11022 int vmdq_vecs =
11023 min_t(int, vectors_left, vmdq_vecs_wanted);
11024
11025
11026
11027
11028
11029
11030
11031 if (vectors_left < vmdq_vecs_wanted) {
11032 pf->num_vmdq_qps = 1;
11033 vmdq_vecs_wanted = pf->num_vmdq_vsis;
11034 vmdq_vecs = min_t(int,
11035 vectors_left,
11036 vmdq_vecs_wanted);
11037 }
11038 pf->num_vmdq_msix = pf->num_vmdq_qps;
11039
11040 v_budget += vmdq_vecs;
11041 vectors_left -= vmdq_vecs;
11042 }
11043 }
11044
11045
11046
11047
11048
11049
11050
11051
11052
11053
11054 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left);
11055 pf->num_lan_msix += extra_vectors;
11056 vectors_left -= extra_vectors;
11057
11058 WARN(vectors_left < 0,
11059 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n");
11060
11061 v_budget += pf->num_lan_msix;
11062 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
11063 GFP_KERNEL);
11064 if (!pf->msix_entries)
11065 return -ENOMEM;
11066
11067 for (i = 0; i < v_budget; i++)
11068 pf->msix_entries[i].entry = i;
11069 v_actual = i40e_reserve_msix_vectors(pf, v_budget);
11070
11071 if (v_actual < I40E_MIN_MSIX) {
11072 pf->flags &= ~I40E_FLAG_MSIX_ENABLED;
11073 kfree(pf->msix_entries);
11074 pf->msix_entries = NULL;
11075 pci_disable_msix(pf->pdev);
11076 return -ENODEV;
11077
11078 } else if (v_actual == I40E_MIN_MSIX) {
11079
11080 pf->num_vmdq_vsis = 0;
11081 pf->num_vmdq_qps = 0;
11082 pf->num_lan_qps = 1;
11083 pf->num_lan_msix = 1;
11084
11085 } else if (v_actual != v_budget) {
11086
11087
11088
11089
11090
11091 int vec;
11092
11093 dev_info(&pf->pdev->dev,
11094 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n",
11095 v_actual, v_budget);
11096
11097 vec = v_actual - 1;
11098
11099
11100 pf->num_vmdq_msix = 1;
11101 pf->num_vmdq_vsis = 1;
11102 pf->num_vmdq_qps = 1;
11103
11104
11105 switch (vec) {
11106 case 2:
11107 pf->num_lan_msix = 1;
11108 break;
11109 case 3:
11110 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11111 pf->num_lan_msix = 1;
11112 pf->num_iwarp_msix = 1;
11113 } else {
11114 pf->num_lan_msix = 2;
11115 }
11116 break;
11117 default:
11118 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
11119 pf->num_iwarp_msix = min_t(int, (vec / 3),
11120 iwarp_requested);
11121 pf->num_vmdq_vsis = min_t(int, (vec / 3),
11122 I40E_DEFAULT_NUM_VMDQ_VSI);
11123 } else {
11124 pf->num_vmdq_vsis = min_t(int, (vec / 2),
11125 I40E_DEFAULT_NUM_VMDQ_VSI);
11126 }
11127 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
11128 pf->num_fdsb_msix = 1;
11129 vec--;
11130 }
11131 pf->num_lan_msix = min_t(int,
11132 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)),
11133 pf->num_lan_msix);
11134 pf->num_lan_qps = pf->num_lan_msix;
11135 break;
11136 }
11137 }
11138
11139 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) &&
11140 (pf->num_fdsb_msix == 0)) {
11141 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n");
11142 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
11143 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11144 }
11145 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
11146 (pf->num_vmdq_msix == 0)) {
11147 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n");
11148 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED;
11149 }
11150
11151 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) &&
11152 (pf->num_iwarp_msix == 0)) {
11153 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n");
11154 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
11155 }
11156 i40e_debug(&pf->hw, I40E_DEBUG_INIT,
11157 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n",
11158 pf->num_lan_msix,
11159 pf->num_vmdq_msix * pf->num_vmdq_vsis,
11160 pf->num_fdsb_msix,
11161 pf->num_iwarp_msix);
11162
11163 return v_actual;
11164}
11165
11166
11167
11168
11169
11170
11171
11172
11173
11174static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu)
11175{
11176 struct i40e_q_vector *q_vector;
11177
11178
11179 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL);
11180 if (!q_vector)
11181 return -ENOMEM;
11182
11183 q_vector->vsi = vsi;
11184 q_vector->v_idx = v_idx;
11185 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
11186
11187 if (vsi->netdev)
11188 netif_napi_add(vsi->netdev, &q_vector->napi,
11189 i40e_napi_poll, NAPI_POLL_WEIGHT);
11190
11191
11192 vsi->q_vectors[v_idx] = q_vector;
11193
11194 return 0;
11195}
11196
11197
11198
11199
11200
11201
11202
11203
11204static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi)
11205{
11206 struct i40e_pf *pf = vsi->back;
11207 int err, v_idx, num_q_vectors, current_cpu;
11208
11209
11210 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
11211 num_q_vectors = vsi->num_q_vectors;
11212 else if (vsi == pf->vsi[pf->lan_vsi])
11213 num_q_vectors = 1;
11214 else
11215 return -EINVAL;
11216
11217 current_cpu = cpumask_first(cpu_online_mask);
11218
11219 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
11220 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu);
11221 if (err)
11222 goto err_out;
11223 current_cpu = cpumask_next(current_cpu, cpu_online_mask);
11224 if (unlikely(current_cpu >= nr_cpu_ids))
11225 current_cpu = cpumask_first(cpu_online_mask);
11226 }
11227
11228 return 0;
11229
11230err_out:
11231 while (v_idx--)
11232 i40e_free_q_vector(vsi, v_idx);
11233
11234 return err;
11235}
11236
11237
11238
11239
11240
11241static int i40e_init_interrupt_scheme(struct i40e_pf *pf)
11242{
11243 int vectors = 0;
11244 ssize_t size;
11245
11246 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11247 vectors = i40e_init_msix(pf);
11248 if (vectors < 0) {
11249 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED |
11250 I40E_FLAG_IWARP_ENABLED |
11251 I40E_FLAG_RSS_ENABLED |
11252 I40E_FLAG_DCB_CAPABLE |
11253 I40E_FLAG_DCB_ENABLED |
11254 I40E_FLAG_SRIOV_ENABLED |
11255 I40E_FLAG_FD_SB_ENABLED |
11256 I40E_FLAG_FD_ATR_ENABLED |
11257 I40E_FLAG_VMDQ_ENABLED);
11258 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
11259
11260
11261 i40e_determine_queue_usage(pf);
11262 }
11263 }
11264
11265 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) &&
11266 (pf->flags & I40E_FLAG_MSI_ENABLED)) {
11267 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n");
11268 vectors = pci_enable_msi(pf->pdev);
11269 if (vectors < 0) {
11270 dev_info(&pf->pdev->dev, "MSI init failed - %d\n",
11271 vectors);
11272 pf->flags &= ~I40E_FLAG_MSI_ENABLED;
11273 }
11274 vectors = 1;
11275 }
11276
11277 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED)))
11278 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n");
11279
11280
11281 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors);
11282 pf->irq_pile = kzalloc(size, GFP_KERNEL);
11283 if (!pf->irq_pile)
11284 return -ENOMEM;
11285
11286 pf->irq_pile->num_entries = vectors;
11287 pf->irq_pile->search_hint = 0;
11288
11289
11290 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1);
11291
11292 return 0;
11293}
11294
11295
11296
11297
11298
11299
11300
11301
11302
11303static int i40e_restore_interrupt_scheme(struct i40e_pf *pf)
11304{
11305 int err, i;
11306
11307
11308
11309
11310
11311 pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED);
11312
11313 err = i40e_init_interrupt_scheme(pf);
11314 if (err)
11315 return err;
11316
11317
11318
11319
11320 for (i = 0; i < pf->num_alloc_vsi; i++) {
11321 if (pf->vsi[i]) {
11322 err = i40e_vsi_alloc_q_vectors(pf->vsi[i]);
11323 if (err)
11324 goto err_unwind;
11325 i40e_vsi_map_rings_to_vectors(pf->vsi[i]);
11326 }
11327 }
11328
11329 err = i40e_setup_misc_vector(pf);
11330 if (err)
11331 goto err_unwind;
11332
11333 if (pf->flags & I40E_FLAG_IWARP_ENABLED)
11334 i40e_client_update_msix_info(pf);
11335
11336 return 0;
11337
11338err_unwind:
11339 while (i--) {
11340 if (pf->vsi[i])
11341 i40e_vsi_free_q_vectors(pf->vsi[i]);
11342 }
11343
11344 return err;
11345}
11346
11347
11348
11349
11350
11351
11352
11353
11354
11355
11356
11357static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf)
11358{
11359 int err;
11360
11361 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
11362 err = i40e_setup_misc_vector(pf);
11363
11364 if (err) {
11365 dev_info(&pf->pdev->dev,
11366 "MSI-X misc vector request failed, error %d\n",
11367 err);
11368 return err;
11369 }
11370 } else {
11371 u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED;
11372
11373 err = request_irq(pf->pdev->irq, i40e_intr, flags,
11374 pf->int_name, pf);
11375
11376 if (err) {
11377 dev_info(&pf->pdev->dev,
11378 "MSI/legacy misc vector request failed, error %d\n",
11379 err);
11380 return err;
11381 }
11382 i40e_enable_misc_int_causes(pf);
11383 i40e_irq_dynamic_enable_icr0(pf);
11384 }
11385
11386 return 0;
11387}
11388
11389
11390
11391
11392
11393
11394
11395
11396
11397static int i40e_setup_misc_vector(struct i40e_pf *pf)
11398{
11399 struct i40e_hw *hw = &pf->hw;
11400 int err = 0;
11401
11402
11403 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) {
11404 err = request_irq(pf->msix_entries[0].vector,
11405 i40e_intr, 0, pf->int_name, pf);
11406 if (err) {
11407 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state);
11408 dev_info(&pf->pdev->dev,
11409 "request_irq for %s failed: %d\n",
11410 pf->int_name, err);
11411 return -EFAULT;
11412 }
11413 }
11414
11415 i40e_enable_misc_int_causes(pf);
11416
11417
11418 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST);
11419 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K);
11420
11421 i40e_flush(hw);
11422
11423 i40e_irq_dynamic_enable_icr0(pf);
11424
11425 return err;
11426}
11427
11428
11429
11430
11431
11432
11433
11434
11435
11436
11437static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed,
11438 u8 *lut, u16 lut_size)
11439{
11440 struct i40e_pf *pf = vsi->back;
11441 struct i40e_hw *hw = &pf->hw;
11442 int ret = 0;
11443
11444 if (seed) {
11445 ret = i40e_aq_get_rss_key(hw, vsi->id,
11446 (struct i40e_aqc_get_set_rss_key_data *)seed);
11447 if (ret) {
11448 dev_info(&pf->pdev->dev,
11449 "Cannot get RSS key, err %s aq_err %s\n",
11450 i40e_stat_str(&pf->hw, ret),
11451 i40e_aq_str(&pf->hw,
11452 pf->hw.aq.asq_last_status));
11453 return ret;
11454 }
11455 }
11456
11457 if (lut) {
11458 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false;
11459
11460 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size);
11461 if (ret) {
11462 dev_info(&pf->pdev->dev,
11463 "Cannot get RSS lut, err %s aq_err %s\n",
11464 i40e_stat_str(&pf->hw, ret),
11465 i40e_aq_str(&pf->hw,
11466 pf->hw.aq.asq_last_status));
11467 return ret;
11468 }
11469 }
11470
11471 return ret;
11472}
11473
11474
11475
11476
11477
11478
11479
11480
11481
11482
11483static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed,
11484 const u8 *lut, u16 lut_size)
11485{
11486 struct i40e_pf *pf = vsi->back;
11487 struct i40e_hw *hw = &pf->hw;
11488 u16 vf_id = vsi->vf_id;
11489 u8 i;
11490
11491
11492 if (seed) {
11493 u32 *seed_dw = (u32 *)seed;
11494
11495 if (vsi->type == I40E_VSI_MAIN) {
11496 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
11497 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]);
11498 } else if (vsi->type == I40E_VSI_SRIOV) {
11499 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++)
11500 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]);
11501 } else {
11502 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n");
11503 }
11504 }
11505
11506 if (lut) {
11507 u32 *lut_dw = (u32 *)lut;
11508
11509 if (vsi->type == I40E_VSI_MAIN) {
11510 if (lut_size != I40E_HLUT_ARRAY_SIZE)
11511 return -EINVAL;
11512 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
11513 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]);
11514 } else if (vsi->type == I40E_VSI_SRIOV) {
11515 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE)
11516 return -EINVAL;
11517 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
11518 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]);
11519 } else {
11520 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
11521 }
11522 }
11523 i40e_flush(hw);
11524
11525 return 0;
11526}
11527
11528
11529
11530
11531
11532
11533
11534
11535
11536
11537static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed,
11538 u8 *lut, u16 lut_size)
11539{
11540 struct i40e_pf *pf = vsi->back;
11541 struct i40e_hw *hw = &pf->hw;
11542 u16 i;
11543
11544 if (seed) {
11545 u32 *seed_dw = (u32 *)seed;
11546
11547 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++)
11548 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i));
11549 }
11550 if (lut) {
11551 u32 *lut_dw = (u32 *)lut;
11552
11553 if (lut_size != I40E_HLUT_ARRAY_SIZE)
11554 return -EINVAL;
11555 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
11556 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i));
11557 }
11558
11559 return 0;
11560}
11561
11562
11563
11564
11565
11566
11567
11568
11569
11570
11571int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
11572{
11573 struct i40e_pf *pf = vsi->back;
11574
11575 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
11576 return i40e_config_rss_aq(vsi, seed, lut, lut_size);
11577 else
11578 return i40e_config_rss_reg(vsi, seed, lut, lut_size);
11579}
11580
11581
11582
11583
11584
11585
11586
11587
11588
11589
11590int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
11591{
11592 struct i40e_pf *pf = vsi->back;
11593
11594 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)
11595 return i40e_get_rss_aq(vsi, seed, lut, lut_size);
11596 else
11597 return i40e_get_rss_reg(vsi, seed, lut, lut_size);
11598}
11599
11600
11601
11602
11603
11604
11605
11606
11607void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut,
11608 u16 rss_table_size, u16 rss_size)
11609{
11610 u16 i;
11611
11612 for (i = 0; i < rss_table_size; i++)
11613 lut[i] = i % rss_size;
11614}
11615
11616
11617
11618
11619
11620static int i40e_pf_config_rss(struct i40e_pf *pf)
11621{
11622 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
11623 u8 seed[I40E_HKEY_ARRAY_SIZE];
11624 u8 *lut;
11625 struct i40e_hw *hw = &pf->hw;
11626 u32 reg_val;
11627 u64 hena;
11628 int ret;
11629
11630
11631 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) |
11632 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32);
11633 hena |= i40e_pf_get_default_rss_hena(pf);
11634
11635 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena);
11636 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32));
11637
11638
11639 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0);
11640 reg_val = (pf->rss_table_size == 512) ?
11641 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) :
11642 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512);
11643 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val);
11644
11645
11646 if (!vsi->rss_size) {
11647 u16 qcount;
11648
11649
11650
11651
11652
11653 qcount = vsi->num_queue_pairs /
11654 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1);
11655 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
11656 }
11657 if (!vsi->rss_size)
11658 return -EINVAL;
11659
11660 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL);
11661 if (!lut)
11662 return -ENOMEM;
11663
11664
11665 if (vsi->rss_lut_user)
11666 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
11667 else
11668 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size);
11669
11670
11671
11672
11673 if (vsi->rss_hkey_user)
11674 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE);
11675 else
11676 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE);
11677 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size);
11678 kfree(lut);
11679
11680 return ret;
11681}
11682
11683
11684
11685
11686
11687
11688
11689
11690
11691
11692int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count)
11693{
11694 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi];
11695 int new_rss_size;
11696
11697 if (!(pf->flags & I40E_FLAG_RSS_ENABLED))
11698 return 0;
11699
11700 queue_count = min_t(int, queue_count, num_online_cpus());
11701 new_rss_size = min_t(int, queue_count, pf->rss_size_max);
11702
11703 if (queue_count != vsi->num_queue_pairs) {
11704 u16 qcount;
11705
11706 vsi->req_queue_pairs = queue_count;
11707 i40e_prep_for_reset(pf, true);
11708
11709 pf->alloc_rss_size = new_rss_size;
11710
11711 i40e_reset_and_rebuild(pf, true, true);
11712
11713
11714
11715
11716 if (queue_count < vsi->rss_size) {
11717 i40e_clear_rss_config_user(vsi);
11718 dev_dbg(&pf->pdev->dev,
11719 "discard user configured hash keys and lut\n");
11720 }
11721
11722
11723 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc;
11724 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount);
11725
11726 i40e_pf_config_rss(pf);
11727 }
11728 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n",
11729 vsi->req_queue_pairs, pf->rss_size_max);
11730 return pf->alloc_rss_size;
11731}
11732
11733
11734
11735
11736
11737i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf)
11738{
11739 i40e_status status;
11740 bool min_valid, max_valid;
11741 u32 max_bw, min_bw;
11742
11743 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw,
11744 &min_valid, &max_valid);
11745
11746 if (!status) {
11747 if (min_valid)
11748 pf->min_bw = min_bw;
11749 if (max_valid)
11750 pf->max_bw = max_bw;
11751 }
11752
11753 return status;
11754}
11755
11756
11757
11758
11759
11760i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf)
11761{
11762 struct i40e_aqc_configure_partition_bw_data bw_data;
11763 i40e_status status;
11764
11765
11766 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id));
11767 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK;
11768 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK;
11769
11770
11771 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL);
11772
11773 return status;
11774}
11775
11776
11777
11778
11779
11780i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf)
11781{
11782
11783 enum i40e_admin_queue_err last_aq_status;
11784 i40e_status ret;
11785 u16 nvm_word;
11786
11787 if (pf->hw.partition_id != 1) {
11788 dev_info(&pf->pdev->dev,
11789 "Commit BW only works on partition 1! This is partition %d",
11790 pf->hw.partition_id);
11791 ret = I40E_NOT_SUPPORTED;
11792 goto bw_commit_out;
11793 }
11794
11795
11796 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ);
11797 last_aq_status = pf->hw.aq.asq_last_status;
11798 if (ret) {
11799 dev_info(&pf->pdev->dev,
11800 "Cannot acquire NVM for read access, err %s aq_err %s\n",
11801 i40e_stat_str(&pf->hw, ret),
11802 i40e_aq_str(&pf->hw, last_aq_status));
11803 goto bw_commit_out;
11804 }
11805
11806
11807 ret = i40e_aq_read_nvm(&pf->hw,
11808 I40E_SR_NVM_CONTROL_WORD,
11809 0x10, sizeof(nvm_word), &nvm_word,
11810 false, NULL);
11811
11812
11813
11814 last_aq_status = pf->hw.aq.asq_last_status;
11815 i40e_release_nvm(&pf->hw);
11816 if (ret) {
11817 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n",
11818 i40e_stat_str(&pf->hw, ret),
11819 i40e_aq_str(&pf->hw, last_aq_status));
11820 goto bw_commit_out;
11821 }
11822
11823
11824 msleep(50);
11825
11826
11827 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE);
11828 last_aq_status = pf->hw.aq.asq_last_status;
11829 if (ret) {
11830 dev_info(&pf->pdev->dev,
11831 "Cannot acquire NVM for write access, err %s aq_err %s\n",
11832 i40e_stat_str(&pf->hw, ret),
11833 i40e_aq_str(&pf->hw, last_aq_status));
11834 goto bw_commit_out;
11835 }
11836
11837
11838
11839
11840 ret = i40e_aq_update_nvm(&pf->hw,
11841 I40E_SR_NVM_CONTROL_WORD,
11842 0x10, sizeof(nvm_word),
11843 &nvm_word, true, 0, NULL);
11844
11845
11846
11847 last_aq_status = pf->hw.aq.asq_last_status;
11848 i40e_release_nvm(&pf->hw);
11849 if (ret)
11850 dev_info(&pf->pdev->dev,
11851 "BW settings NOT SAVED, err %s aq_err %s\n",
11852 i40e_stat_str(&pf->hw, ret),
11853 i40e_aq_str(&pf->hw, last_aq_status));
11854bw_commit_out:
11855
11856 return ret;
11857}
11858
11859
11860
11861
11862
11863
11864
11865
11866
11867static int i40e_sw_init(struct i40e_pf *pf)
11868{
11869 int err = 0;
11870 int size;
11871
11872
11873 pf->flags = I40E_FLAG_RX_CSUM_ENABLED |
11874 I40E_FLAG_MSI_ENABLED |
11875 I40E_FLAG_MSIX_ENABLED;
11876
11877
11878 pf->rx_itr_default = I40E_ITR_RX_DEF;
11879 pf->tx_itr_default = I40E_ITR_TX_DEF;
11880
11881
11882
11883
11884 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width);
11885 pf->alloc_rss_size = 1;
11886 pf->rss_table_size = pf->hw.func_caps.rss_table_size;
11887 pf->rss_size_max = min_t(int, pf->rss_size_max,
11888 pf->hw.func_caps.num_tx_qp);
11889 if (pf->hw.func_caps.rss) {
11890 pf->flags |= I40E_FLAG_RSS_ENABLED;
11891 pf->alloc_rss_size = min_t(int, pf->rss_size_max,
11892 num_online_cpus());
11893 }
11894
11895
11896 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) {
11897 pf->flags |= I40E_FLAG_MFP_ENABLED;
11898 dev_info(&pf->pdev->dev, "MFP mode Enabled\n");
11899 if (i40e_get_partition_bw_setting(pf)) {
11900 dev_warn(&pf->pdev->dev,
11901 "Could not get partition bw settings\n");
11902 } else {
11903 dev_info(&pf->pdev->dev,
11904 "Partition BW Min = %8.8x, Max = %8.8x\n",
11905 pf->min_bw, pf->max_bw);
11906
11907
11908 i40e_set_partition_bw_setting(pf);
11909 }
11910 }
11911
11912 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) ||
11913 (pf->hw.func_caps.fd_filters_best_effort > 0)) {
11914 pf->flags |= I40E_FLAG_FD_ATR_ENABLED;
11915 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE;
11916 if (pf->flags & I40E_FLAG_MFP_ENABLED &&
11917 pf->hw.num_partitions > 1)
11918 dev_info(&pf->pdev->dev,
11919 "Flow Director Sideband mode Disabled in MFP mode\n");
11920 else
11921 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
11922 pf->fdir_pf_filter_count =
11923 pf->hw.func_caps.fd_filters_guaranteed;
11924 pf->hw.fdir_shared_filter_count =
11925 pf->hw.func_caps.fd_filters_best_effort;
11926 }
11927
11928 if (pf->hw.mac.type == I40E_MAC_X722) {
11929 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE |
11930 I40E_HW_128_QP_RSS_CAPABLE |
11931 I40E_HW_ATR_EVICT_CAPABLE |
11932 I40E_HW_WB_ON_ITR_CAPABLE |
11933 I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE |
11934 I40E_HW_NO_PCI_LINK_CHECK |
11935 I40E_HW_USE_SET_LLDP_MIB |
11936 I40E_HW_GENEVE_OFFLOAD_CAPABLE |
11937 I40E_HW_PTP_L4_CAPABLE |
11938 I40E_HW_WOL_MC_MAGIC_PKT_WAKE |
11939 I40E_HW_OUTER_UDP_CSUM_CAPABLE);
11940
11941#define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03
11942 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) !=
11943 I40E_FDEVICT_PCTYPE_DEFAULT) {
11944 dev_warn(&pf->pdev->dev,
11945 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n");
11946 pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE;
11947 }
11948 } else if ((pf->hw.aq.api_maj_ver > 1) ||
11949 ((pf->hw.aq.api_maj_ver == 1) &&
11950 (pf->hw.aq.api_min_ver > 4))) {
11951
11952 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE;
11953 }
11954
11955
11956 if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE)
11957 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED;
11958
11959 if ((pf->hw.mac.type == I40E_MAC_XL710) &&
11960 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) ||
11961 (pf->hw.aq.fw_maj_ver < 4))) {
11962 pf->hw_features |= I40E_HW_RESTART_AUTONEG;
11963
11964 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT;
11965 }
11966
11967
11968 if ((pf->hw.mac.type == I40E_MAC_XL710) &&
11969 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) ||
11970 (pf->hw.aq.fw_maj_ver < 4)))
11971 pf->hw_features |= I40E_HW_STOP_FW_LLDP;
11972
11973
11974 if ((pf->hw.mac.type == I40E_MAC_XL710) &&
11975 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) ||
11976 (pf->hw.aq.fw_maj_ver >= 5)))
11977 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB;
11978
11979
11980 if (pf->hw.mac.type == I40E_MAC_XL710 &&
11981 pf->hw.aq.fw_maj_ver >= 6)
11982 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE;
11983
11984 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) {
11985 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI;
11986 pf->flags |= I40E_FLAG_VMDQ_ENABLED;
11987 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf);
11988 }
11989
11990 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) {
11991 pf->flags |= I40E_FLAG_IWARP_ENABLED;
11992
11993 pf->num_iwarp_msix = (int)num_online_cpus() + 1;
11994 }
11995
11996
11997
11998
11999
12000 if (pf->hw.mac.type == I40E_MAC_XL710 &&
12001 pf->hw.func_caps.npar_enable &&
12002 (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE))
12003 pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE;
12004
12005#ifdef CONFIG_PCI_IOV
12006 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) {
12007 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF;
12008 pf->flags |= I40E_FLAG_SRIOV_ENABLED;
12009 pf->num_req_vfs = min_t(int,
12010 pf->hw.func_caps.num_vfs,
12011 I40E_MAX_VF_COUNT);
12012 }
12013#endif
12014 pf->eeprom_version = 0xDEAD;
12015 pf->lan_veb = I40E_NO_VEB;
12016 pf->lan_vsi = I40E_NO_VSI;
12017
12018
12019 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED;
12020
12021
12022 size = sizeof(struct i40e_lump_tracking)
12023 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp);
12024 pf->qp_pile = kzalloc(size, GFP_KERNEL);
12025 if (!pf->qp_pile) {
12026 err = -ENOMEM;
12027 goto sw_init_done;
12028 }
12029 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp;
12030 pf->qp_pile->search_hint = 0;
12031
12032 pf->tx_timeout_recovery_level = 1;
12033
12034 mutex_init(&pf->switch_mutex);
12035
12036sw_init_done:
12037 return err;
12038}
12039
12040
12041
12042
12043
12044
12045
12046
12047bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features)
12048{
12049 bool need_reset = false;
12050
12051
12052
12053
12054 if (features & NETIF_F_NTUPLE) {
12055
12056 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED))
12057 need_reset = true;
12058
12059
12060
12061 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) {
12062 pf->flags |= I40E_FLAG_FD_SB_ENABLED;
12063 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE;
12064 }
12065 } else {
12066
12067 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
12068 need_reset = true;
12069 i40e_fdir_filter_exit(pf);
12070 }
12071 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
12072 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state);
12073 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
12074
12075
12076 pf->fd_add_err = 0;
12077 pf->fd_atr_cnt = 0;
12078
12079 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state))
12080 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) &&
12081 (I40E_DEBUG_FD & pf->hw.debug_mask))
12082 dev_info(&pf->pdev->dev, "ATR re-enabled.\n");
12083 }
12084 return need_reset;
12085}
12086
12087
12088
12089
12090
12091static void i40e_clear_rss_lut(struct i40e_vsi *vsi)
12092{
12093 struct i40e_pf *pf = vsi->back;
12094 struct i40e_hw *hw = &pf->hw;
12095 u16 vf_id = vsi->vf_id;
12096 u8 i;
12097
12098 if (vsi->type == I40E_VSI_MAIN) {
12099 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++)
12100 wr32(hw, I40E_PFQF_HLUT(i), 0);
12101 } else if (vsi->type == I40E_VSI_SRIOV) {
12102 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++)
12103 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0);
12104 } else {
12105 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n");
12106 }
12107}
12108
12109
12110
12111
12112
12113
12114
12115static int i40e_set_features(struct net_device *netdev,
12116 netdev_features_t features)
12117{
12118 struct i40e_netdev_priv *np = netdev_priv(netdev);
12119 struct i40e_vsi *vsi = np->vsi;
12120 struct i40e_pf *pf = vsi->back;
12121 bool need_reset;
12122
12123 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH))
12124 i40e_pf_config_rss(pf);
12125 else if (!(features & NETIF_F_RXHASH) &&
12126 netdev->features & NETIF_F_RXHASH)
12127 i40e_clear_rss_lut(vsi);
12128
12129 if (features & NETIF_F_HW_VLAN_CTAG_RX)
12130 i40e_vlan_stripping_enable(vsi);
12131 else
12132 i40e_vlan_stripping_disable(vsi);
12133
12134 if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) {
12135 dev_err(&pf->pdev->dev,
12136 "Offloaded tc filters active, can't turn hw_tc_offload off");
12137 return -EINVAL;
12138 }
12139
12140 if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt)
12141 i40e_del_all_macvlans(vsi);
12142
12143 need_reset = i40e_set_ntuple(pf, features);
12144
12145 if (need_reset)
12146 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12147
12148 return 0;
12149}
12150
12151
12152
12153
12154
12155
12156
12157
12158static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port)
12159{
12160 u8 i;
12161
12162 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) {
12163
12164
12165
12166 if (!port && (pf->pending_udp_bitmap & BIT_ULL(i)))
12167 continue;
12168 if (pf->udp_ports[i].port == port)
12169 return i;
12170 }
12171
12172 return i;
12173}
12174
12175
12176
12177
12178
12179
12180static void i40e_udp_tunnel_add(struct net_device *netdev,
12181 struct udp_tunnel_info *ti)
12182{
12183 struct i40e_netdev_priv *np = netdev_priv(netdev);
12184 struct i40e_vsi *vsi = np->vsi;
12185 struct i40e_pf *pf = vsi->back;
12186 u16 port = ntohs(ti->port);
12187 u8 next_idx;
12188 u8 idx;
12189
12190 idx = i40e_get_udp_port_idx(pf, port);
12191
12192
12193 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
12194 netdev_info(netdev, "port %d already offloaded\n", port);
12195 return;
12196 }
12197
12198
12199 next_idx = i40e_get_udp_port_idx(pf, 0);
12200
12201 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) {
12202 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n",
12203 port);
12204 return;
12205 }
12206
12207 switch (ti->type) {
12208 case UDP_TUNNEL_TYPE_VXLAN:
12209 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN;
12210 break;
12211 case UDP_TUNNEL_TYPE_GENEVE:
12212 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE))
12213 return;
12214 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE;
12215 break;
12216 default:
12217 return;
12218 }
12219
12220
12221 pf->udp_ports[next_idx].port = port;
12222 pf->udp_ports[next_idx].filter_index = I40E_UDP_PORT_INDEX_UNUSED;
12223 pf->pending_udp_bitmap |= BIT_ULL(next_idx);
12224 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
12225}
12226
12227
12228
12229
12230
12231
12232static void i40e_udp_tunnel_del(struct net_device *netdev,
12233 struct udp_tunnel_info *ti)
12234{
12235 struct i40e_netdev_priv *np = netdev_priv(netdev);
12236 struct i40e_vsi *vsi = np->vsi;
12237 struct i40e_pf *pf = vsi->back;
12238 u16 port = ntohs(ti->port);
12239 u8 idx;
12240
12241 idx = i40e_get_udp_port_idx(pf, port);
12242
12243
12244 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS)
12245 goto not_found;
12246
12247 switch (ti->type) {
12248 case UDP_TUNNEL_TYPE_VXLAN:
12249 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN)
12250 goto not_found;
12251 break;
12252 case UDP_TUNNEL_TYPE_GENEVE:
12253 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE)
12254 goto not_found;
12255 break;
12256 default:
12257 goto not_found;
12258 }
12259
12260
12261
12262
12263 pf->udp_ports[idx].port = 0;
12264
12265
12266
12267
12268
12269 pf->pending_udp_bitmap ^= BIT_ULL(idx);
12270 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state);
12271
12272 return;
12273not_found:
12274 netdev_warn(netdev, "UDP port %d was not found, not deleting\n",
12275 port);
12276}
12277
12278static int i40e_get_phys_port_id(struct net_device *netdev,
12279 struct netdev_phys_item_id *ppid)
12280{
12281 struct i40e_netdev_priv *np = netdev_priv(netdev);
12282 struct i40e_pf *pf = np->vsi->back;
12283 struct i40e_hw *hw = &pf->hw;
12284
12285 if (!(pf->hw_features & I40E_HW_PORT_ID_VALID))
12286 return -EOPNOTSUPP;
12287
12288 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id));
12289 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len);
12290
12291 return 0;
12292}
12293
12294
12295
12296
12297
12298
12299
12300
12301
12302
12303static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
12304 struct net_device *dev,
12305 const unsigned char *addr, u16 vid,
12306 u16 flags,
12307 struct netlink_ext_ack *extack)
12308{
12309 struct i40e_netdev_priv *np = netdev_priv(dev);
12310 struct i40e_pf *pf = np->vsi->back;
12311 int err = 0;
12312
12313 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED))
12314 return -EOPNOTSUPP;
12315
12316 if (vid) {
12317 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name);
12318 return -EINVAL;
12319 }
12320
12321
12322
12323
12324 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
12325 netdev_info(dev, "FDB only supports static addresses\n");
12326 return -EINVAL;
12327 }
12328
12329 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
12330 err = dev_uc_add_excl(dev, addr);
12331 else if (is_multicast_ether_addr(addr))
12332 err = dev_mc_add_excl(dev, addr);
12333 else
12334 err = -EINVAL;
12335
12336
12337 if (err == -EEXIST && !(flags & NLM_F_EXCL))
12338 err = 0;
12339
12340 return err;
12341}
12342
12343
12344
12345
12346
12347
12348
12349
12350
12351
12352
12353
12354
12355
12356
12357
12358
12359static int i40e_ndo_bridge_setlink(struct net_device *dev,
12360 struct nlmsghdr *nlh,
12361 u16 flags,
12362 struct netlink_ext_ack *extack)
12363{
12364 struct i40e_netdev_priv *np = netdev_priv(dev);
12365 struct i40e_vsi *vsi = np->vsi;
12366 struct i40e_pf *pf = vsi->back;
12367 struct i40e_veb *veb = NULL;
12368 struct nlattr *attr, *br_spec;
12369 int i, rem;
12370
12371
12372 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
12373 return -EOPNOTSUPP;
12374
12375
12376 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12377 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12378 veb = pf->veb[i];
12379 }
12380
12381 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
12382
12383 nla_for_each_nested(attr, br_spec, rem) {
12384 __u16 mode;
12385
12386 if (nla_type(attr) != IFLA_BRIDGE_MODE)
12387 continue;
12388
12389 mode = nla_get_u16(attr);
12390 if ((mode != BRIDGE_MODE_VEPA) &&
12391 (mode != BRIDGE_MODE_VEB))
12392 return -EINVAL;
12393
12394
12395 if (!veb) {
12396 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
12397 vsi->tc_config.enabled_tc);
12398 if (veb) {
12399 veb->bridge_mode = mode;
12400 i40e_config_bridge_mode(veb);
12401 } else {
12402
12403 return -ENOENT;
12404 }
12405 break;
12406 } else if (mode != veb->bridge_mode) {
12407
12408 veb->bridge_mode = mode;
12409
12410 if (mode == BRIDGE_MODE_VEB)
12411 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
12412 else
12413 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
12414 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true);
12415 break;
12416 }
12417 }
12418
12419 return 0;
12420}
12421
12422
12423
12424
12425
12426
12427
12428
12429
12430
12431
12432
12433
12434static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
12435 struct net_device *dev,
12436 u32 __always_unused filter_mask,
12437 int nlflags)
12438{
12439 struct i40e_netdev_priv *np = netdev_priv(dev);
12440 struct i40e_vsi *vsi = np->vsi;
12441 struct i40e_pf *pf = vsi->back;
12442 struct i40e_veb *veb = NULL;
12443 int i;
12444
12445
12446 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid)
12447 return -EOPNOTSUPP;
12448
12449
12450 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
12451 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
12452 veb = pf->veb[i];
12453 }
12454
12455 if (!veb)
12456 return 0;
12457
12458 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode,
12459 0, 0, nlflags, filter_mask, NULL);
12460}
12461
12462
12463
12464
12465
12466
12467
12468static netdev_features_t i40e_features_check(struct sk_buff *skb,
12469 struct net_device *dev,
12470 netdev_features_t features)
12471{
12472 size_t len;
12473
12474
12475
12476
12477
12478 if (skb->ip_summed != CHECKSUM_PARTIAL)
12479 return features;
12480
12481
12482
12483
12484 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
12485 features &= ~NETIF_F_GSO_MASK;
12486
12487
12488 len = skb_network_header(skb) - skb->data;
12489 if (len & ~(63 * 2))
12490 goto out_err;
12491
12492
12493 len = skb_transport_header(skb) - skb_network_header(skb);
12494 if (len & ~(127 * 4))
12495 goto out_err;
12496
12497 if (skb->encapsulation) {
12498
12499 len = skb_inner_network_header(skb) - skb_transport_header(skb);
12500 if (len & ~(127 * 2))
12501 goto out_err;
12502
12503
12504 len = skb_inner_transport_header(skb) -
12505 skb_inner_network_header(skb);
12506 if (len & ~(127 * 4))
12507 goto out_err;
12508 }
12509
12510
12511
12512
12513
12514
12515 return features;
12516out_err:
12517 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
12518}
12519
12520
12521
12522
12523
12524
12525static int i40e_xdp_setup(struct i40e_vsi *vsi,
12526 struct bpf_prog *prog)
12527{
12528 int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
12529 struct i40e_pf *pf = vsi->back;
12530 struct bpf_prog *old_prog;
12531 bool need_reset;
12532 int i;
12533
12534
12535 if (frame_size > vsi->rx_buf_len)
12536 return -EINVAL;
12537
12538 if (!i40e_enabled_xdp_vsi(vsi) && !prog)
12539 return 0;
12540
12541
12542 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog);
12543
12544 if (need_reset)
12545 i40e_prep_for_reset(pf, true);
12546
12547 old_prog = xchg(&vsi->xdp_prog, prog);
12548
12549 if (need_reset)
12550 i40e_reset_and_rebuild(pf, true, true);
12551
12552 for (i = 0; i < vsi->num_queue_pairs; i++)
12553 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog);
12554
12555 if (old_prog)
12556 bpf_prog_put(old_prog);
12557
12558
12559
12560
12561 if (need_reset && prog)
12562 for (i = 0; i < vsi->num_queue_pairs; i++)
12563 if (vsi->xdp_rings[i]->xsk_umem)
12564 (void)i40e_xsk_async_xmit(vsi->netdev, i);
12565
12566 return 0;
12567}
12568
12569
12570
12571
12572
12573
12574
12575static int i40e_enter_busy_conf(struct i40e_vsi *vsi)
12576{
12577 struct i40e_pf *pf = vsi->back;
12578 int timeout = 50;
12579
12580 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) {
12581 timeout--;
12582 if (!timeout)
12583 return -EBUSY;
12584 usleep_range(1000, 2000);
12585 }
12586
12587 return 0;
12588}
12589
12590
12591
12592
12593
12594static void i40e_exit_busy_conf(struct i40e_vsi *vsi)
12595{
12596 struct i40e_pf *pf = vsi->back;
12597
12598 clear_bit(__I40E_CONFIG_BUSY, pf->state);
12599}
12600
12601
12602
12603
12604
12605
12606static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair)
12607{
12608 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0,
12609 sizeof(vsi->rx_rings[queue_pair]->rx_stats));
12610 memset(&vsi->tx_rings[queue_pair]->stats, 0,
12611 sizeof(vsi->tx_rings[queue_pair]->stats));
12612 if (i40e_enabled_xdp_vsi(vsi)) {
12613 memset(&vsi->xdp_rings[queue_pair]->stats, 0,
12614 sizeof(vsi->xdp_rings[queue_pair]->stats));
12615 }
12616}
12617
12618
12619
12620
12621
12622
12623static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair)
12624{
12625 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]);
12626 if (i40e_enabled_xdp_vsi(vsi)) {
12627
12628
12629
12630 synchronize_rcu();
12631 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]);
12632 }
12633 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]);
12634}
12635
12636
12637
12638
12639
12640
12641
12642static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair,
12643 bool enable)
12644{
12645 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
12646 struct i40e_q_vector *q_vector = rxr->q_vector;
12647
12648 if (!vsi->netdev)
12649 return;
12650
12651
12652 if (q_vector->rx.ring || q_vector->tx.ring) {
12653 if (enable)
12654 napi_enable(&q_vector->napi);
12655 else
12656 napi_disable(&q_vector->napi);
12657 }
12658}
12659
12660
12661
12662
12663
12664
12665
12666
12667
12668static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair,
12669 bool enable)
12670{
12671 struct i40e_pf *pf = vsi->back;
12672 int pf_q, ret = 0;
12673
12674 pf_q = vsi->base_queue + queue_pair;
12675 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q,
12676 false , enable);
12677 if (ret) {
12678 dev_info(&pf->pdev->dev,
12679 "VSI seid %d Tx ring %d %sable timeout\n",
12680 vsi->seid, pf_q, (enable ? "en" : "dis"));
12681 return ret;
12682 }
12683
12684 i40e_control_rx_q(pf, pf_q, enable);
12685 ret = i40e_pf_rxq_wait(pf, pf_q, enable);
12686 if (ret) {
12687 dev_info(&pf->pdev->dev,
12688 "VSI seid %d Rx ring %d %sable timeout\n",
12689 vsi->seid, pf_q, (enable ? "en" : "dis"));
12690 return ret;
12691 }
12692
12693
12694
12695
12696 if (!enable)
12697 mdelay(50);
12698
12699 if (!i40e_enabled_xdp_vsi(vsi))
12700 return ret;
12701
12702 ret = i40e_control_wait_tx_q(vsi->seid, pf,
12703 pf_q + vsi->alloc_queue_pairs,
12704 true , enable);
12705 if (ret) {
12706 dev_info(&pf->pdev->dev,
12707 "VSI seid %d XDP Tx ring %d %sable timeout\n",
12708 vsi->seid, pf_q, (enable ? "en" : "dis"));
12709 }
12710
12711 return ret;
12712}
12713
12714
12715
12716
12717
12718
12719static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair)
12720{
12721 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
12722 struct i40e_pf *pf = vsi->back;
12723 struct i40e_hw *hw = &pf->hw;
12724
12725
12726 if (pf->flags & I40E_FLAG_MSIX_ENABLED)
12727 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx);
12728 else
12729 i40e_irq_dynamic_enable_icr0(pf);
12730
12731 i40e_flush(hw);
12732}
12733
12734
12735
12736
12737
12738
12739static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair)
12740{
12741 struct i40e_ring *rxr = vsi->rx_rings[queue_pair];
12742 struct i40e_pf *pf = vsi->back;
12743 struct i40e_hw *hw = &pf->hw;
12744
12745
12746
12747
12748
12749
12750
12751 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
12752 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx;
12753
12754 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0);
12755 i40e_flush(hw);
12756 synchronize_irq(pf->msix_entries[intpf].vector);
12757 } else {
12758
12759 wr32(hw, I40E_PFINT_ICR0_ENA, 0);
12760 wr32(hw, I40E_PFINT_DYN_CTL0, 0);
12761 i40e_flush(hw);
12762 synchronize_irq(pf->pdev->irq);
12763 }
12764}
12765
12766
12767
12768
12769
12770
12771
12772
12773int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair)
12774{
12775 int err;
12776
12777 err = i40e_enter_busy_conf(vsi);
12778 if (err)
12779 return err;
12780
12781 i40e_queue_pair_disable_irq(vsi, queue_pair);
12782 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false );
12783 i40e_queue_pair_toggle_napi(vsi, queue_pair, false );
12784 i40e_queue_pair_clean_rings(vsi, queue_pair);
12785 i40e_queue_pair_reset_stats(vsi, queue_pair);
12786
12787 return err;
12788}
12789
12790
12791
12792
12793
12794
12795
12796
12797int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair)
12798{
12799 int err;
12800
12801 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]);
12802 if (err)
12803 return err;
12804
12805 if (i40e_enabled_xdp_vsi(vsi)) {
12806 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]);
12807 if (err)
12808 return err;
12809 }
12810
12811 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]);
12812 if (err)
12813 return err;
12814
12815 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true );
12816 i40e_queue_pair_toggle_napi(vsi, queue_pair, true );
12817 i40e_queue_pair_enable_irq(vsi, queue_pair);
12818
12819 i40e_exit_busy_conf(vsi);
12820
12821 return err;
12822}
12823
12824
12825
12826
12827
12828
12829static int i40e_xdp(struct net_device *dev,
12830 struct netdev_bpf *xdp)
12831{
12832 struct i40e_netdev_priv *np = netdev_priv(dev);
12833 struct i40e_vsi *vsi = np->vsi;
12834
12835 if (vsi->type != I40E_VSI_MAIN)
12836 return -EINVAL;
12837
12838 switch (xdp->command) {
12839 case XDP_SETUP_PROG:
12840 return i40e_xdp_setup(vsi, xdp->prog);
12841 case XDP_QUERY_PROG:
12842 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0;
12843 return 0;
12844 case XDP_SETUP_XSK_UMEM:
12845 return i40e_xsk_umem_setup(vsi, xdp->xsk.umem,
12846 xdp->xsk.queue_id);
12847 default:
12848 return -EINVAL;
12849 }
12850}
12851
12852static const struct net_device_ops i40e_netdev_ops = {
12853 .ndo_open = i40e_open,
12854 .ndo_stop = i40e_close,
12855 .ndo_start_xmit = i40e_lan_xmit_frame,
12856 .ndo_get_stats64 = i40e_get_netdev_stats_struct,
12857 .ndo_set_rx_mode = i40e_set_rx_mode,
12858 .ndo_validate_addr = eth_validate_addr,
12859 .ndo_set_mac_address = i40e_set_mac,
12860 .ndo_change_mtu = i40e_change_mtu,
12861 .ndo_do_ioctl = i40e_ioctl,
12862 .ndo_tx_timeout = i40e_tx_timeout,
12863 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid,
12864 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid,
12865#ifdef CONFIG_NET_POLL_CONTROLLER
12866 .ndo_poll_controller = i40e_netpoll,
12867#endif
12868 .ndo_setup_tc = __i40e_setup_tc,
12869 .ndo_set_features = i40e_set_features,
12870 .ndo_set_vf_mac = i40e_ndo_set_vf_mac,
12871 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan,
12872 .ndo_set_vf_rate = i40e_ndo_set_vf_bw,
12873 .ndo_get_vf_config = i40e_ndo_get_vf_config,
12874 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state,
12875 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk,
12876 .ndo_set_vf_trust = i40e_ndo_set_vf_trust,
12877 .ndo_udp_tunnel_add = i40e_udp_tunnel_add,
12878 .ndo_udp_tunnel_del = i40e_udp_tunnel_del,
12879 .ndo_get_phys_port_id = i40e_get_phys_port_id,
12880 .ndo_fdb_add = i40e_ndo_fdb_add,
12881 .ndo_features_check = i40e_features_check,
12882 .ndo_bridge_getlink = i40e_ndo_bridge_getlink,
12883 .ndo_bridge_setlink = i40e_ndo_bridge_setlink,
12884 .ndo_bpf = i40e_xdp,
12885 .ndo_xdp_xmit = i40e_xdp_xmit,
12886 .ndo_xsk_async_xmit = i40e_xsk_async_xmit,
12887 .ndo_dfwd_add_station = i40e_fwd_add,
12888 .ndo_dfwd_del_station = i40e_fwd_del,
12889};
12890
12891
12892
12893
12894
12895
12896
12897static int i40e_config_netdev(struct i40e_vsi *vsi)
12898{
12899 struct i40e_pf *pf = vsi->back;
12900 struct i40e_hw *hw = &pf->hw;
12901 struct i40e_netdev_priv *np;
12902 struct net_device *netdev;
12903 u8 broadcast[ETH_ALEN];
12904 u8 mac_addr[ETH_ALEN];
12905 int etherdev_size;
12906 netdev_features_t hw_enc_features;
12907 netdev_features_t hw_features;
12908
12909 etherdev_size = sizeof(struct i40e_netdev_priv);
12910 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs);
12911 if (!netdev)
12912 return -ENOMEM;
12913
12914 vsi->netdev = netdev;
12915 np = netdev_priv(netdev);
12916 np->vsi = vsi;
12917
12918 hw_enc_features = NETIF_F_SG |
12919 NETIF_F_IP_CSUM |
12920 NETIF_F_IPV6_CSUM |
12921 NETIF_F_HIGHDMA |
12922 NETIF_F_SOFT_FEATURES |
12923 NETIF_F_TSO |
12924 NETIF_F_TSO_ECN |
12925 NETIF_F_TSO6 |
12926 NETIF_F_GSO_GRE |
12927 NETIF_F_GSO_GRE_CSUM |
12928 NETIF_F_GSO_PARTIAL |
12929 NETIF_F_GSO_IPXIP4 |
12930 NETIF_F_GSO_IPXIP6 |
12931 NETIF_F_GSO_UDP_TUNNEL |
12932 NETIF_F_GSO_UDP_TUNNEL_CSUM |
12933 NETIF_F_SCTP_CRC |
12934 NETIF_F_RXHASH |
12935 NETIF_F_RXCSUM |
12936 0;
12937
12938 if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE))
12939 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
12940
12941 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
12942
12943 netdev->hw_enc_features |= hw_enc_features;
12944
12945
12946 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
12947
12948
12949 netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD;
12950
12951 hw_features = hw_enc_features |
12952 NETIF_F_HW_VLAN_CTAG_TX |
12953 NETIF_F_HW_VLAN_CTAG_RX;
12954
12955 if (!(pf->flags & I40E_FLAG_MFP_ENABLED))
12956 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC;
12957
12958 netdev->hw_features |= hw_features;
12959
12960 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
12961 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
12962
12963 if (vsi->type == I40E_VSI_MAIN) {
12964 SET_NETDEV_DEV(netdev, &pf->pdev->dev);
12965 ether_addr_copy(mac_addr, hw->mac.perm_addr);
12966
12967
12968
12969
12970
12971
12972
12973
12974
12975
12976 i40e_rm_default_mac_filter(vsi, mac_addr);
12977 spin_lock_bh(&vsi->mac_filter_hash_lock);
12978 i40e_add_mac_filter(vsi, mac_addr);
12979 spin_unlock_bh(&vsi->mac_filter_hash_lock);
12980 } else {
12981
12982
12983
12984
12985
12986 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d",
12987 IFNAMSIZ - 4,
12988 pf->vsi[pf->lan_vsi]->netdev->name);
12989 eth_random_addr(mac_addr);
12990
12991 spin_lock_bh(&vsi->mac_filter_hash_lock);
12992 i40e_add_mac_filter(vsi, mac_addr);
12993 spin_unlock_bh(&vsi->mac_filter_hash_lock);
12994 }
12995
12996
12997
12998
12999
13000
13001
13002
13003
13004
13005
13006
13007
13008
13009 eth_broadcast_addr(broadcast);
13010 spin_lock_bh(&vsi->mac_filter_hash_lock);
13011 i40e_add_mac_filter(vsi, broadcast);
13012 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13013
13014 ether_addr_copy(netdev->dev_addr, mac_addr);
13015 ether_addr_copy(netdev->perm_addr, mac_addr);
13016
13017
13018 netdev->neigh_priv_len = sizeof(u32) * 4;
13019
13020 netdev->priv_flags |= IFF_UNICAST_FLT;
13021 netdev->priv_flags |= IFF_SUPP_NOFCS;
13022
13023 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc);
13024
13025 netdev->netdev_ops = &i40e_netdev_ops;
13026 netdev->watchdog_timeo = 5 * HZ;
13027 i40e_set_ethtool_ops(netdev);
13028
13029
13030 netdev->min_mtu = ETH_MIN_MTU;
13031 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD;
13032
13033 return 0;
13034}
13035
13036
13037
13038
13039
13040
13041
13042static void i40e_vsi_delete(struct i40e_vsi *vsi)
13043{
13044
13045 if (vsi == vsi->back->vsi[vsi->back->lan_vsi])
13046 return;
13047
13048 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL);
13049}
13050
13051
13052
13053
13054
13055
13056
13057int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi)
13058{
13059 struct i40e_veb *veb;
13060 struct i40e_pf *pf = vsi->back;
13061
13062
13063 if (vsi->veb_idx >= I40E_MAX_VEB)
13064 return 1;
13065
13066 veb = pf->veb[vsi->veb_idx];
13067 if (!veb) {
13068 dev_info(&pf->pdev->dev,
13069 "There is no veb associated with the bridge\n");
13070 return -ENOENT;
13071 }
13072
13073
13074 if (veb->bridge_mode & BRIDGE_MODE_VEPA) {
13075 return 0;
13076 } else {
13077
13078 return 1;
13079 }
13080
13081
13082 return 0;
13083}
13084
13085
13086
13087
13088
13089
13090
13091
13092static int i40e_add_vsi(struct i40e_vsi *vsi)
13093{
13094 int ret = -ENODEV;
13095 struct i40e_pf *pf = vsi->back;
13096 struct i40e_hw *hw = &pf->hw;
13097 struct i40e_vsi_context ctxt;
13098 struct i40e_mac_filter *f;
13099 struct hlist_node *h;
13100 int bkt;
13101
13102 u8 enabled_tc = 0x1;
13103 int f_count = 0;
13104
13105 memset(&ctxt, 0, sizeof(ctxt));
13106 switch (vsi->type) {
13107 case I40E_VSI_MAIN:
13108
13109
13110
13111
13112
13113 ctxt.seid = pf->main_vsi_seid;
13114 ctxt.pf_num = pf->hw.pf_id;
13115 ctxt.vf_num = 0;
13116 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL);
13117 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13118 if (ret) {
13119 dev_info(&pf->pdev->dev,
13120 "couldn't get PF vsi config, err %s aq_err %s\n",
13121 i40e_stat_str(&pf->hw, ret),
13122 i40e_aq_str(&pf->hw,
13123 pf->hw.aq.asq_last_status));
13124 return -ENOENT;
13125 }
13126 vsi->info = ctxt.info;
13127 vsi->info.valid_sections = 0;
13128
13129 vsi->seid = ctxt.seid;
13130 vsi->id = ctxt.vsi_number;
13131
13132 enabled_tc = i40e_pf_get_tc_map(pf);
13133
13134
13135
13136
13137
13138 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) {
13139 memset(&ctxt, 0, sizeof(ctxt));
13140 ctxt.seid = pf->main_vsi_seid;
13141 ctxt.pf_num = pf->hw.pf_id;
13142 ctxt.vf_num = 0;
13143 ctxt.info.valid_sections |=
13144 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13145 ctxt.info.switch_id =
13146 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB);
13147 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13148 if (ret) {
13149 dev_info(&pf->pdev->dev,
13150 "update vsi failed, err %s aq_err %s\n",
13151 i40e_stat_str(&pf->hw, ret),
13152 i40e_aq_str(&pf->hw,
13153 pf->hw.aq.asq_last_status));
13154 ret = -ENOENT;
13155 goto err;
13156 }
13157 }
13158
13159
13160 if ((pf->flags & I40E_FLAG_MFP_ENABLED) &&
13161 !(pf->hw.func_caps.iscsi)) {
13162 memset(&ctxt, 0, sizeof(ctxt));
13163 ctxt.seid = pf->main_vsi_seid;
13164 ctxt.pf_num = pf->hw.pf_id;
13165 ctxt.vf_num = 0;
13166 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false);
13167 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL);
13168 if (ret) {
13169 dev_info(&pf->pdev->dev,
13170 "update vsi failed, err %s aq_err %s\n",
13171 i40e_stat_str(&pf->hw, ret),
13172 i40e_aq_str(&pf->hw,
13173 pf->hw.aq.asq_last_status));
13174 ret = -ENOENT;
13175 goto err;
13176 }
13177
13178 i40e_vsi_update_queue_map(vsi, &ctxt);
13179 vsi->info.valid_sections = 0;
13180 } else {
13181
13182
13183
13184
13185
13186
13187 ret = i40e_vsi_config_tc(vsi, enabled_tc);
13188 if (ret) {
13189
13190
13191
13192 dev_info(&pf->pdev->dev,
13193 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n",
13194 enabled_tc,
13195 i40e_stat_str(&pf->hw, ret),
13196 i40e_aq_str(&pf->hw,
13197 pf->hw.aq.asq_last_status));
13198 }
13199 }
13200 break;
13201
13202 case I40E_VSI_FDIR:
13203 ctxt.pf_num = hw->pf_id;
13204 ctxt.vf_num = 0;
13205 ctxt.uplink_seid = vsi->uplink_seid;
13206 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13207 ctxt.flags = I40E_AQ_VSI_TYPE_PF;
13208 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) &&
13209 (i40e_is_vsi_uplink_mode_veb(vsi))) {
13210 ctxt.info.valid_sections |=
13211 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13212 ctxt.info.switch_id =
13213 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13214 }
13215 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13216 break;
13217
13218 case I40E_VSI_VMDQ2:
13219 ctxt.pf_num = hw->pf_id;
13220 ctxt.vf_num = 0;
13221 ctxt.uplink_seid = vsi->uplink_seid;
13222 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13223 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2;
13224
13225
13226
13227
13228 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
13229 ctxt.info.valid_sections |=
13230 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13231 ctxt.info.switch_id =
13232 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13233 }
13234
13235
13236 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13237 break;
13238
13239 case I40E_VSI_SRIOV:
13240 ctxt.pf_num = hw->pf_id;
13241 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id;
13242 ctxt.uplink_seid = vsi->uplink_seid;
13243 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL;
13244 ctxt.flags = I40E_AQ_VSI_TYPE_VF;
13245
13246
13247
13248
13249 if (i40e_is_vsi_uplink_mode_veb(vsi)) {
13250 ctxt.info.valid_sections |=
13251 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID);
13252 ctxt.info.switch_id =
13253 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB);
13254 }
13255
13256 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) {
13257 ctxt.info.valid_sections |=
13258 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID);
13259 ctxt.info.queueing_opt_flags |=
13260 (I40E_AQ_VSI_QUE_OPT_TCP_ENA |
13261 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI);
13262 }
13263
13264 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID);
13265 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL;
13266 if (pf->vf[vsi->vf_id].spoofchk) {
13267 ctxt.info.valid_sections |=
13268 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID);
13269 ctxt.info.sec_flags |=
13270 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK |
13271 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK);
13272 }
13273
13274 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true);
13275 break;
13276
13277 case I40E_VSI_IWARP:
13278
13279 break;
13280
13281 default:
13282 return -ENODEV;
13283 }
13284
13285 if (vsi->type != I40E_VSI_MAIN) {
13286 ret = i40e_aq_add_vsi(hw, &ctxt, NULL);
13287 if (ret) {
13288 dev_info(&vsi->back->pdev->dev,
13289 "add vsi failed, err %s aq_err %s\n",
13290 i40e_stat_str(&pf->hw, ret),
13291 i40e_aq_str(&pf->hw,
13292 pf->hw.aq.asq_last_status));
13293 ret = -ENOENT;
13294 goto err;
13295 }
13296 vsi->info = ctxt.info;
13297 vsi->info.valid_sections = 0;
13298 vsi->seid = ctxt.seid;
13299 vsi->id = ctxt.vsi_number;
13300 }
13301
13302 vsi->active_filters = 0;
13303 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state);
13304 spin_lock_bh(&vsi->mac_filter_hash_lock);
13305
13306 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) {
13307 f->state = I40E_FILTER_NEW;
13308 f_count++;
13309 }
13310 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13311
13312 if (f_count) {
13313 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED;
13314 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state);
13315 }
13316
13317
13318 ret = i40e_vsi_get_bw_info(vsi);
13319 if (ret) {
13320 dev_info(&pf->pdev->dev,
13321 "couldn't get vsi bw info, err %s aq_err %s\n",
13322 i40e_stat_str(&pf->hw, ret),
13323 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13324
13325 ret = 0;
13326 }
13327
13328err:
13329 return ret;
13330}
13331
13332
13333
13334
13335
13336
13337
13338int i40e_vsi_release(struct i40e_vsi *vsi)
13339{
13340 struct i40e_mac_filter *f;
13341 struct hlist_node *h;
13342 struct i40e_veb *veb = NULL;
13343 struct i40e_pf *pf;
13344 u16 uplink_seid;
13345 int i, n, bkt;
13346
13347 pf = vsi->back;
13348
13349
13350 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) {
13351 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n",
13352 vsi->seid, vsi->uplink_seid);
13353 return -ENODEV;
13354 }
13355 if (vsi == pf->vsi[pf->lan_vsi] &&
13356 !test_bit(__I40E_DOWN, pf->state)) {
13357 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n");
13358 return -ENODEV;
13359 }
13360
13361 uplink_seid = vsi->uplink_seid;
13362 if (vsi->type != I40E_VSI_SRIOV) {
13363 if (vsi->netdev_registered) {
13364 vsi->netdev_registered = false;
13365 if (vsi->netdev) {
13366
13367 unregister_netdev(vsi->netdev);
13368 }
13369 } else {
13370 i40e_vsi_close(vsi);
13371 }
13372 i40e_vsi_disable_irq(vsi);
13373 }
13374
13375 spin_lock_bh(&vsi->mac_filter_hash_lock);
13376
13377
13378 if (vsi->netdev) {
13379 __dev_uc_unsync(vsi->netdev, NULL);
13380 __dev_mc_unsync(vsi->netdev, NULL);
13381 }
13382
13383
13384 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist)
13385 __i40e_del_filter(vsi, f);
13386
13387 spin_unlock_bh(&vsi->mac_filter_hash_lock);
13388
13389 i40e_sync_vsi_filters(vsi);
13390
13391 i40e_vsi_delete(vsi);
13392 i40e_vsi_free_q_vectors(vsi);
13393 if (vsi->netdev) {
13394 free_netdev(vsi->netdev);
13395 vsi->netdev = NULL;
13396 }
13397 i40e_vsi_clear_rings(vsi);
13398 i40e_vsi_clear(vsi);
13399
13400
13401
13402
13403
13404
13405
13406
13407
13408 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) {
13409 if (pf->vsi[i] &&
13410 pf->vsi[i]->uplink_seid == uplink_seid &&
13411 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
13412 n++;
13413 }
13414 }
13415 for (i = 0; i < I40E_MAX_VEB; i++) {
13416 if (!pf->veb[i])
13417 continue;
13418 if (pf->veb[i]->uplink_seid == uplink_seid)
13419 n++;
13420 if (pf->veb[i]->seid == uplink_seid)
13421 veb = pf->veb[i];
13422 }
13423 if (n == 0 && veb && veb->uplink_seid != 0)
13424 i40e_veb_release(veb);
13425
13426 return 0;
13427}
13428
13429
13430
13431
13432
13433
13434
13435
13436
13437
13438
13439static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi)
13440{
13441 int ret = -ENOENT;
13442 struct i40e_pf *pf = vsi->back;
13443
13444 if (vsi->q_vectors[0]) {
13445 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
13446 vsi->seid);
13447 return -EEXIST;
13448 }
13449
13450 if (vsi->base_vector) {
13451 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
13452 vsi->seid, vsi->base_vector);
13453 return -EEXIST;
13454 }
13455
13456 ret = i40e_vsi_alloc_q_vectors(vsi);
13457 if (ret) {
13458 dev_info(&pf->pdev->dev,
13459 "failed to allocate %d q_vector for VSI %d, ret=%d\n",
13460 vsi->num_q_vectors, vsi->seid, ret);
13461 vsi->num_q_vectors = 0;
13462 goto vector_setup_out;
13463 }
13464
13465
13466
13467
13468 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED))
13469 return ret;
13470 if (vsi->num_q_vectors)
13471 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile,
13472 vsi->num_q_vectors, vsi->idx);
13473 if (vsi->base_vector < 0) {
13474 dev_info(&pf->pdev->dev,
13475 "failed to get tracking for %d vectors for VSI %d, err=%d\n",
13476 vsi->num_q_vectors, vsi->seid, vsi->base_vector);
13477 i40e_vsi_free_q_vectors(vsi);
13478 ret = -ENOENT;
13479 goto vector_setup_out;
13480 }
13481
13482vector_setup_out:
13483 return ret;
13484}
13485
13486
13487
13488
13489
13490
13491
13492
13493
13494
13495static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi)
13496{
13497 u16 alloc_queue_pairs;
13498 struct i40e_pf *pf;
13499 u8 enabled_tc;
13500 int ret;
13501
13502 if (!vsi)
13503 return NULL;
13504
13505 pf = vsi->back;
13506
13507 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx);
13508 i40e_vsi_clear_rings(vsi);
13509
13510 i40e_vsi_free_arrays(vsi, false);
13511 i40e_set_num_rings_in_vsi(vsi);
13512 ret = i40e_vsi_alloc_arrays(vsi, false);
13513 if (ret)
13514 goto err_vsi;
13515
13516 alloc_queue_pairs = vsi->alloc_queue_pairs *
13517 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
13518
13519 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
13520 if (ret < 0) {
13521 dev_info(&pf->pdev->dev,
13522 "failed to get tracking for %d queues for VSI %d err %d\n",
13523 alloc_queue_pairs, vsi->seid, ret);
13524 goto err_vsi;
13525 }
13526 vsi->base_queue = ret;
13527
13528
13529
13530
13531 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
13532 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
13533 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
13534 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
13535 if (vsi->type == I40E_VSI_MAIN)
13536 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr);
13537
13538
13539 ret = i40e_alloc_rings(vsi);
13540 if (ret)
13541 goto err_rings;
13542
13543
13544 i40e_vsi_map_rings_to_vectors(vsi);
13545 return vsi;
13546
13547err_rings:
13548 i40e_vsi_free_q_vectors(vsi);
13549 if (vsi->netdev_registered) {
13550 vsi->netdev_registered = false;
13551 unregister_netdev(vsi->netdev);
13552 free_netdev(vsi->netdev);
13553 vsi->netdev = NULL;
13554 }
13555 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
13556err_vsi:
13557 i40e_vsi_clear(vsi);
13558 return NULL;
13559}
13560
13561
13562
13563
13564
13565
13566
13567
13568
13569
13570
13571
13572
13573
13574struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type,
13575 u16 uplink_seid, u32 param1)
13576{
13577 struct i40e_vsi *vsi = NULL;
13578 struct i40e_veb *veb = NULL;
13579 u16 alloc_queue_pairs;
13580 int ret, i;
13581 int v_idx;
13582
13583
13584
13585
13586
13587
13588
13589
13590
13591
13592
13593
13594
13595
13596 for (i = 0; i < I40E_MAX_VEB; i++) {
13597 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) {
13598 veb = pf->veb[i];
13599 break;
13600 }
13601 }
13602
13603 if (!veb && uplink_seid != pf->mac_seid) {
13604
13605 for (i = 0; i < pf->num_alloc_vsi; i++) {
13606 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) {
13607 vsi = pf->vsi[i];
13608 break;
13609 }
13610 }
13611 if (!vsi) {
13612 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n",
13613 uplink_seid);
13614 return NULL;
13615 }
13616
13617 if (vsi->uplink_seid == pf->mac_seid)
13618 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid,
13619 vsi->tc_config.enabled_tc);
13620 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0)
13621 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid,
13622 vsi->tc_config.enabled_tc);
13623 if (veb) {
13624 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) {
13625 dev_info(&vsi->back->pdev->dev,
13626 "New VSI creation error, uplink seid of LAN VSI expected.\n");
13627 return NULL;
13628 }
13629
13630
13631
13632
13633 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) {
13634 veb->bridge_mode = BRIDGE_MODE_VEPA;
13635 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED;
13636 }
13637 i40e_config_bridge_mode(veb);
13638 }
13639 for (i = 0; i < I40E_MAX_VEB && !veb; i++) {
13640 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid)
13641 veb = pf->veb[i];
13642 }
13643 if (!veb) {
13644 dev_info(&pf->pdev->dev, "couldn't add VEB\n");
13645 return NULL;
13646 }
13647
13648 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
13649 uplink_seid = veb->seid;
13650 }
13651
13652
13653 v_idx = i40e_vsi_mem_alloc(pf, type);
13654 if (v_idx < 0)
13655 goto err_alloc;
13656 vsi = pf->vsi[v_idx];
13657 if (!vsi)
13658 goto err_alloc;
13659 vsi->type = type;
13660 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB);
13661
13662 if (type == I40E_VSI_MAIN)
13663 pf->lan_vsi = v_idx;
13664 else if (type == I40E_VSI_SRIOV)
13665 vsi->vf_id = param1;
13666
13667 alloc_queue_pairs = vsi->alloc_queue_pairs *
13668 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1);
13669
13670 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx);
13671 if (ret < 0) {
13672 dev_info(&pf->pdev->dev,
13673 "failed to get tracking for %d queues for VSI %d err=%d\n",
13674 alloc_queue_pairs, vsi->seid, ret);
13675 goto err_vsi;
13676 }
13677 vsi->base_queue = ret;
13678
13679
13680 vsi->uplink_seid = uplink_seid;
13681 ret = i40e_add_vsi(vsi);
13682 if (ret)
13683 goto err_vsi;
13684
13685 switch (vsi->type) {
13686
13687 case I40E_VSI_MAIN:
13688 case I40E_VSI_VMDQ2:
13689 ret = i40e_config_netdev(vsi);
13690 if (ret)
13691 goto err_netdev;
13692 ret = register_netdev(vsi->netdev);
13693 if (ret)
13694 goto err_netdev;
13695 vsi->netdev_registered = true;
13696 netif_carrier_off(vsi->netdev);
13697#ifdef CONFIG_I40E_DCB
13698
13699 i40e_dcbnl_setup(vsi);
13700#endif
13701
13702
13703 case I40E_VSI_FDIR:
13704
13705 ret = i40e_vsi_setup_vectors(vsi);
13706 if (ret)
13707 goto err_msix;
13708
13709 ret = i40e_alloc_rings(vsi);
13710 if (ret)
13711 goto err_rings;
13712
13713
13714 i40e_vsi_map_rings_to_vectors(vsi);
13715
13716 i40e_vsi_reset_stats(vsi);
13717 break;
13718
13719 default:
13720
13721 break;
13722 }
13723
13724 if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) &&
13725 (vsi->type == I40E_VSI_VMDQ2)) {
13726 ret = i40e_vsi_config_rss(vsi);
13727 }
13728 return vsi;
13729
13730err_rings:
13731 i40e_vsi_free_q_vectors(vsi);
13732err_msix:
13733 if (vsi->netdev_registered) {
13734 vsi->netdev_registered = false;
13735 unregister_netdev(vsi->netdev);
13736 free_netdev(vsi->netdev);
13737 vsi->netdev = NULL;
13738 }
13739err_netdev:
13740 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL);
13741err_vsi:
13742 i40e_vsi_clear(vsi);
13743err_alloc:
13744 return NULL;
13745}
13746
13747
13748
13749
13750
13751
13752
13753static int i40e_veb_get_bw_info(struct i40e_veb *veb)
13754{
13755 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data;
13756 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data;
13757 struct i40e_pf *pf = veb->pf;
13758 struct i40e_hw *hw = &pf->hw;
13759 u32 tc_bw_max;
13760 int ret = 0;
13761 int i;
13762
13763 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid,
13764 &bw_data, NULL);
13765 if (ret) {
13766 dev_info(&pf->pdev->dev,
13767 "query veb bw config failed, err %s aq_err %s\n",
13768 i40e_stat_str(&pf->hw, ret),
13769 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
13770 goto out;
13771 }
13772
13773 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid,
13774 &ets_data, NULL);
13775 if (ret) {
13776 dev_info(&pf->pdev->dev,
13777 "query veb bw ets config failed, err %s aq_err %s\n",
13778 i40e_stat_str(&pf->hw, ret),
13779 i40e_aq_str(&pf->hw, hw->aq.asq_last_status));
13780 goto out;
13781 }
13782
13783 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit);
13784 veb->bw_max_quanta = ets_data.tc_bw_max;
13785 veb->is_abs_credits = bw_data.absolute_credits_enable;
13786 veb->enabled_tc = ets_data.tc_valid_bits;
13787 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) |
13788 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16);
13789 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
13790 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i];
13791 veb->bw_tc_limit_credits[i] =
13792 le16_to_cpu(bw_data.tc_bw_limits[i]);
13793 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7);
13794 }
13795
13796out:
13797 return ret;
13798}
13799
13800
13801
13802
13803
13804
13805
13806
13807static int i40e_veb_mem_alloc(struct i40e_pf *pf)
13808{
13809 int ret = -ENOENT;
13810 struct i40e_veb *veb;
13811 int i;
13812
13813
13814 mutex_lock(&pf->switch_mutex);
13815
13816
13817
13818
13819
13820
13821
13822 i = 0;
13823 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL))
13824 i++;
13825 if (i >= I40E_MAX_VEB) {
13826 ret = -ENOMEM;
13827 goto err_alloc_veb;
13828 }
13829
13830 veb = kzalloc(sizeof(*veb), GFP_KERNEL);
13831 if (!veb) {
13832 ret = -ENOMEM;
13833 goto err_alloc_veb;
13834 }
13835 veb->pf = pf;
13836 veb->idx = i;
13837 veb->enabled_tc = 1;
13838
13839 pf->veb[i] = veb;
13840 ret = i;
13841err_alloc_veb:
13842 mutex_unlock(&pf->switch_mutex);
13843 return ret;
13844}
13845
13846
13847
13848
13849
13850
13851
13852
13853static void i40e_switch_branch_release(struct i40e_veb *branch)
13854{
13855 struct i40e_pf *pf = branch->pf;
13856 u16 branch_seid = branch->seid;
13857 u16 veb_idx = branch->idx;
13858 int i;
13859
13860
13861 for (i = 0; i < I40E_MAX_VEB; i++) {
13862 if (!pf->veb[i])
13863 continue;
13864 if (pf->veb[i]->uplink_seid == branch->seid)
13865 i40e_switch_branch_release(pf->veb[i]);
13866 }
13867
13868
13869
13870
13871
13872
13873 for (i = 0; i < pf->num_alloc_vsi; i++) {
13874 if (!pf->vsi[i])
13875 continue;
13876 if (pf->vsi[i]->uplink_seid == branch_seid &&
13877 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) {
13878 i40e_vsi_release(pf->vsi[i]);
13879 }
13880 }
13881
13882
13883
13884
13885
13886
13887 if (pf->veb[veb_idx])
13888 i40e_veb_release(pf->veb[veb_idx]);
13889}
13890
13891
13892
13893
13894
13895static void i40e_veb_clear(struct i40e_veb *veb)
13896{
13897 if (!veb)
13898 return;
13899
13900 if (veb->pf) {
13901 struct i40e_pf *pf = veb->pf;
13902
13903 mutex_lock(&pf->switch_mutex);
13904 if (pf->veb[veb->idx] == veb)
13905 pf->veb[veb->idx] = NULL;
13906 mutex_unlock(&pf->switch_mutex);
13907 }
13908
13909 kfree(veb);
13910}
13911
13912
13913
13914
13915
13916void i40e_veb_release(struct i40e_veb *veb)
13917{
13918 struct i40e_vsi *vsi = NULL;
13919 struct i40e_pf *pf;
13920 int i, n = 0;
13921
13922 pf = veb->pf;
13923
13924
13925 for (i = 0; i < pf->num_alloc_vsi; i++) {
13926 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) {
13927 n++;
13928 vsi = pf->vsi[i];
13929 }
13930 }
13931 if (n != 1) {
13932 dev_info(&pf->pdev->dev,
13933 "can't remove VEB %d with %d VSIs left\n",
13934 veb->seid, n);
13935 return;
13936 }
13937
13938
13939 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER;
13940 if (veb->uplink_seid) {
13941 vsi->uplink_seid = veb->uplink_seid;
13942 if (veb->uplink_seid == pf->mac_seid)
13943 vsi->veb_idx = I40E_NO_VEB;
13944 else
13945 vsi->veb_idx = veb->veb_idx;
13946 } else {
13947
13948 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid;
13949 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx;
13950 }
13951
13952 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
13953 i40e_veb_clear(veb);
13954}
13955
13956
13957
13958
13959
13960
13961static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi)
13962{
13963 struct i40e_pf *pf = veb->pf;
13964 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED);
13965 int ret;
13966
13967 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid,
13968 veb->enabled_tc, false,
13969 &veb->seid, enable_stats, NULL);
13970
13971
13972 if (ret) {
13973 dev_info(&pf->pdev->dev,
13974 "couldn't add VEB, err %s aq_err %s\n",
13975 i40e_stat_str(&pf->hw, ret),
13976 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13977 return -EPERM;
13978 }
13979
13980
13981 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL,
13982 &veb->stats_idx, NULL, NULL, NULL);
13983 if (ret) {
13984 dev_info(&pf->pdev->dev,
13985 "couldn't get VEB statistics idx, err %s aq_err %s\n",
13986 i40e_stat_str(&pf->hw, ret),
13987 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13988 return -EPERM;
13989 }
13990 ret = i40e_veb_get_bw_info(veb);
13991 if (ret) {
13992 dev_info(&pf->pdev->dev,
13993 "couldn't get VEB bw info, err %s aq_err %s\n",
13994 i40e_stat_str(&pf->hw, ret),
13995 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
13996 i40e_aq_delete_element(&pf->hw, veb->seid, NULL);
13997 return -ENOENT;
13998 }
13999
14000 vsi->uplink_seid = veb->seid;
14001 vsi->veb_idx = veb->idx;
14002 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER;
14003
14004 return 0;
14005}
14006
14007
14008
14009
14010
14011
14012
14013
14014
14015
14016
14017
14018
14019
14020
14021
14022
14023struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags,
14024 u16 uplink_seid, u16 vsi_seid,
14025 u8 enabled_tc)
14026{
14027 struct i40e_veb *veb, *uplink_veb = NULL;
14028 int vsi_idx, veb_idx;
14029 int ret;
14030
14031
14032 if ((uplink_seid == 0 || vsi_seid == 0) &&
14033 (uplink_seid + vsi_seid != 0)) {
14034 dev_info(&pf->pdev->dev,
14035 "one, not both seid's are 0: uplink=%d vsi=%d\n",
14036 uplink_seid, vsi_seid);
14037 return NULL;
14038 }
14039
14040
14041 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++)
14042 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid)
14043 break;
14044 if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) {
14045 dev_info(&pf->pdev->dev, "vsi seid %d not found\n",
14046 vsi_seid);
14047 return NULL;
14048 }
14049
14050 if (uplink_seid && uplink_seid != pf->mac_seid) {
14051 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) {
14052 if (pf->veb[veb_idx] &&
14053 pf->veb[veb_idx]->seid == uplink_seid) {
14054 uplink_veb = pf->veb[veb_idx];
14055 break;
14056 }
14057 }
14058 if (!uplink_veb) {
14059 dev_info(&pf->pdev->dev,
14060 "uplink seid %d not found\n", uplink_seid);
14061 return NULL;
14062 }
14063 }
14064
14065
14066 veb_idx = i40e_veb_mem_alloc(pf);
14067 if (veb_idx < 0)
14068 goto err_alloc;
14069 veb = pf->veb[veb_idx];
14070 veb->flags = flags;
14071 veb->uplink_seid = uplink_seid;
14072 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB);
14073 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1);
14074
14075
14076 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]);
14077 if (ret)
14078 goto err_veb;
14079 if (vsi_idx == pf->lan_vsi)
14080 pf->lan_veb = veb->idx;
14081
14082 return veb;
14083
14084err_veb:
14085 i40e_veb_clear(veb);
14086err_alloc:
14087 return NULL;
14088}
14089
14090
14091
14092
14093
14094
14095
14096
14097
14098
14099static void i40e_setup_pf_switch_element(struct i40e_pf *pf,
14100 struct i40e_aqc_switch_config_element_resp *ele,
14101 u16 num_reported, bool printconfig)
14102{
14103 u16 downlink_seid = le16_to_cpu(ele->downlink_seid);
14104 u16 uplink_seid = le16_to_cpu(ele->uplink_seid);
14105 u8 element_type = ele->element_type;
14106 u16 seid = le16_to_cpu(ele->seid);
14107
14108 if (printconfig)
14109 dev_info(&pf->pdev->dev,
14110 "type=%d seid=%d uplink=%d downlink=%d\n",
14111 element_type, seid, uplink_seid, downlink_seid);
14112
14113 switch (element_type) {
14114 case I40E_SWITCH_ELEMENT_TYPE_MAC:
14115 pf->mac_seid = seid;
14116 break;
14117 case I40E_SWITCH_ELEMENT_TYPE_VEB:
14118
14119 if (uplink_seid != pf->mac_seid)
14120 break;
14121 if (pf->lan_veb >= I40E_MAX_VEB) {
14122 int v;
14123
14124
14125 for (v = 0; v < I40E_MAX_VEB; v++) {
14126 if (pf->veb[v] && (pf->veb[v]->seid == seid)) {
14127 pf->lan_veb = v;
14128 break;
14129 }
14130 }
14131 if (pf->lan_veb >= I40E_MAX_VEB) {
14132 v = i40e_veb_mem_alloc(pf);
14133 if (v < 0)
14134 break;
14135 pf->lan_veb = v;
14136 }
14137 }
14138 if (pf->lan_veb >= I40E_MAX_VEB)
14139 break;
14140
14141 pf->veb[pf->lan_veb]->seid = seid;
14142 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid;
14143 pf->veb[pf->lan_veb]->pf = pf;
14144 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB;
14145 break;
14146 case I40E_SWITCH_ELEMENT_TYPE_VSI:
14147 if (num_reported != 1)
14148 break;
14149
14150
14151
14152 pf->mac_seid = uplink_seid;
14153 pf->pf_seid = downlink_seid;
14154 pf->main_vsi_seid = seid;
14155 if (printconfig)
14156 dev_info(&pf->pdev->dev,
14157 "pf_seid=%d main_vsi_seid=%d\n",
14158 pf->pf_seid, pf->main_vsi_seid);
14159 break;
14160 case I40E_SWITCH_ELEMENT_TYPE_PF:
14161 case I40E_SWITCH_ELEMENT_TYPE_VF:
14162 case I40E_SWITCH_ELEMENT_TYPE_EMP:
14163 case I40E_SWITCH_ELEMENT_TYPE_BMC:
14164 case I40E_SWITCH_ELEMENT_TYPE_PE:
14165 case I40E_SWITCH_ELEMENT_TYPE_PA:
14166
14167 break;
14168 default:
14169 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n",
14170 element_type, seid);
14171 break;
14172 }
14173}
14174
14175
14176
14177
14178
14179
14180
14181
14182
14183int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig)
14184{
14185 struct i40e_aqc_get_switch_config_resp *sw_config;
14186 u16 next_seid = 0;
14187 int ret = 0;
14188 u8 *aq_buf;
14189 int i;
14190
14191 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL);
14192 if (!aq_buf)
14193 return -ENOMEM;
14194
14195 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf;
14196 do {
14197 u16 num_reported, num_total;
14198
14199 ret = i40e_aq_get_switch_config(&pf->hw, sw_config,
14200 I40E_AQ_LARGE_BUF,
14201 &next_seid, NULL);
14202 if (ret) {
14203 dev_info(&pf->pdev->dev,
14204 "get switch config failed err %s aq_err %s\n",
14205 i40e_stat_str(&pf->hw, ret),
14206 i40e_aq_str(&pf->hw,
14207 pf->hw.aq.asq_last_status));
14208 kfree(aq_buf);
14209 return -ENOENT;
14210 }
14211
14212 num_reported = le16_to_cpu(sw_config->header.num_reported);
14213 num_total = le16_to_cpu(sw_config->header.num_total);
14214
14215 if (printconfig)
14216 dev_info(&pf->pdev->dev,
14217 "header: %d reported %d total\n",
14218 num_reported, num_total);
14219
14220 for (i = 0; i < num_reported; i++) {
14221 struct i40e_aqc_switch_config_element_resp *ele =
14222 &sw_config->element[i];
14223
14224 i40e_setup_pf_switch_element(pf, ele, num_reported,
14225 printconfig);
14226 }
14227 } while (next_seid != 0);
14228
14229 kfree(aq_buf);
14230 return ret;
14231}
14232
14233
14234
14235
14236
14237
14238
14239
14240static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit)
14241{
14242 u16 flags = 0;
14243 int ret;
14244
14245
14246 ret = i40e_fetch_switch_configuration(pf, false);
14247 if (ret) {
14248 dev_info(&pf->pdev->dev,
14249 "couldn't fetch switch config, err %s aq_err %s\n",
14250 i40e_stat_str(&pf->hw, ret),
14251 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
14252 return ret;
14253 }
14254 i40e_pf_reset_stats(pf);
14255
14256
14257
14258
14259
14260
14261
14262 if ((pf->hw.pf_id == 0) &&
14263 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) {
14264 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
14265 pf->last_sw_conf_flags = flags;
14266 }
14267
14268 if (pf->hw.pf_id == 0) {
14269 u16 valid_flags;
14270
14271 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC;
14272 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0,
14273 NULL);
14274 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) {
14275 dev_info(&pf->pdev->dev,
14276 "couldn't set switch config bits, err %s aq_err %s\n",
14277 i40e_stat_str(&pf->hw, ret),
14278 i40e_aq_str(&pf->hw,
14279 pf->hw.aq.asq_last_status));
14280
14281 }
14282 pf->last_sw_conf_valid_flags = valid_flags;
14283 }
14284
14285
14286 if (pf->lan_vsi == I40E_NO_VSI || reinit) {
14287 struct i40e_vsi *vsi = NULL;
14288 u16 uplink_seid;
14289
14290
14291
14292
14293 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb])
14294 uplink_seid = pf->veb[pf->lan_veb]->seid;
14295 else
14296 uplink_seid = pf->mac_seid;
14297 if (pf->lan_vsi == I40E_NO_VSI)
14298 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0);
14299 else if (reinit)
14300 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]);
14301 if (!vsi) {
14302 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n");
14303 i40e_cloud_filter_exit(pf);
14304 i40e_fdir_teardown(pf);
14305 return -EAGAIN;
14306 }
14307 } else {
14308
14309 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc;
14310
14311 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0;
14312 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid;
14313 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc);
14314 }
14315 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]);
14316
14317 i40e_fdir_sb_setup(pf);
14318
14319
14320 ret = i40e_setup_pf_filter_control(pf);
14321 if (ret) {
14322 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n",
14323 ret);
14324
14325 }
14326
14327
14328
14329
14330 if ((pf->flags & I40E_FLAG_RSS_ENABLED))
14331 i40e_pf_config_rss(pf);
14332
14333
14334 i40e_link_event(pf);
14335
14336
14337 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info &
14338 I40E_AQ_AN_COMPLETED) ? true : false);
14339
14340 i40e_ptp_init(pf);
14341
14342
14343 i40e_sync_udp_filters(pf);
14344
14345 return ret;
14346}
14347
14348
14349
14350
14351
14352static void i40e_determine_queue_usage(struct i40e_pf *pf)
14353{
14354 int queues_left;
14355 int q_max;
14356
14357 pf->num_lan_qps = 0;
14358
14359
14360
14361
14362
14363 queues_left = pf->hw.func_caps.num_tx_qp;
14364
14365 if ((queues_left == 1) ||
14366 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) {
14367
14368 queues_left = 0;
14369 pf->alloc_rss_size = pf->num_lan_qps = 1;
14370
14371
14372 pf->flags &= ~(I40E_FLAG_RSS_ENABLED |
14373 I40E_FLAG_IWARP_ENABLED |
14374 I40E_FLAG_FD_SB_ENABLED |
14375 I40E_FLAG_FD_ATR_ENABLED |
14376 I40E_FLAG_DCB_CAPABLE |
14377 I40E_FLAG_DCB_ENABLED |
14378 I40E_FLAG_SRIOV_ENABLED |
14379 I40E_FLAG_VMDQ_ENABLED);
14380 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14381 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED |
14382 I40E_FLAG_FD_SB_ENABLED |
14383 I40E_FLAG_FD_ATR_ENABLED |
14384 I40E_FLAG_DCB_CAPABLE))) {
14385
14386 pf->alloc_rss_size = pf->num_lan_qps = 1;
14387 queues_left -= pf->num_lan_qps;
14388
14389 pf->flags &= ~(I40E_FLAG_RSS_ENABLED |
14390 I40E_FLAG_IWARP_ENABLED |
14391 I40E_FLAG_FD_SB_ENABLED |
14392 I40E_FLAG_FD_ATR_ENABLED |
14393 I40E_FLAG_DCB_ENABLED |
14394 I40E_FLAG_VMDQ_ENABLED);
14395 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14396 } else {
14397
14398 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) &&
14399 (queues_left < I40E_MAX_TRAFFIC_CLASS)) {
14400 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE |
14401 I40E_FLAG_DCB_ENABLED);
14402 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n");
14403 }
14404
14405
14406 q_max = max_t(int, pf->rss_size_max, num_online_cpus());
14407 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp);
14408 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors);
14409 pf->num_lan_qps = q_max;
14410
14411 queues_left -= pf->num_lan_qps;
14412 }
14413
14414 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
14415 if (queues_left > 1) {
14416 queues_left -= 1;
14417 } else {
14418 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED;
14419 pf->flags |= I40E_FLAG_FD_SB_INACTIVE;
14420 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n");
14421 }
14422 }
14423
14424 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
14425 pf->num_vf_qps && pf->num_req_vfs && queues_left) {
14426 pf->num_req_vfs = min_t(int, pf->num_req_vfs,
14427 (queues_left / pf->num_vf_qps));
14428 queues_left -= (pf->num_req_vfs * pf->num_vf_qps);
14429 }
14430
14431 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) &&
14432 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) {
14433 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis,
14434 (queues_left / pf->num_vmdq_qps));
14435 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps);
14436 }
14437
14438 pf->queues_left = queues_left;
14439 dev_dbg(&pf->pdev->dev,
14440 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n",
14441 pf->hw.func_caps.num_tx_qp,
14442 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED),
14443 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs,
14444 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps,
14445 queues_left);
14446}
14447
14448
14449
14450
14451
14452
14453
14454
14455
14456
14457
14458
14459static int i40e_setup_pf_filter_control(struct i40e_pf *pf)
14460{
14461 struct i40e_filter_control_settings *settings = &pf->filter_settings;
14462
14463 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128;
14464
14465
14466 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED))
14467 settings->enable_fdir = true;
14468
14469
14470 settings->enable_ethtype = true;
14471 settings->enable_macvlan = true;
14472
14473 if (i40e_set_filter_control(&pf->hw, settings))
14474 return -ENOENT;
14475
14476 return 0;
14477}
14478
14479#define INFO_STRING_LEN 255
14480#define REMAIN(__x) (INFO_STRING_LEN - (__x))
14481static void i40e_print_features(struct i40e_pf *pf)
14482{
14483 struct i40e_hw *hw = &pf->hw;
14484 char *buf;
14485 int i;
14486
14487 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL);
14488 if (!buf)
14489 return;
14490
14491 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id);
14492#ifdef CONFIG_PCI_IOV
14493 i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs);
14494#endif
14495 i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d",
14496 pf->hw.func_caps.num_vsis,
14497 pf->vsi[pf->lan_vsi]->num_queue_pairs);
14498 if (pf->flags & I40E_FLAG_RSS_ENABLED)
14499 i += snprintf(&buf[i], REMAIN(i), " RSS");
14500 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED)
14501 i += snprintf(&buf[i], REMAIN(i), " FD_ATR");
14502 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) {
14503 i += snprintf(&buf[i], REMAIN(i), " FD_SB");
14504 i += snprintf(&buf[i], REMAIN(i), " NTUPLE");
14505 }
14506 if (pf->flags & I40E_FLAG_DCB_CAPABLE)
14507 i += snprintf(&buf[i], REMAIN(i), " DCB");
14508 i += snprintf(&buf[i], REMAIN(i), " VxLAN");
14509 i += snprintf(&buf[i], REMAIN(i), " Geneve");
14510 if (pf->flags & I40E_FLAG_PTP)
14511 i += snprintf(&buf[i], REMAIN(i), " PTP");
14512 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED)
14513 i += snprintf(&buf[i], REMAIN(i), " VEB");
14514 else
14515 i += snprintf(&buf[i], REMAIN(i), " VEPA");
14516
14517 dev_info(&pf->pdev->dev, "%s\n", buf);
14518 kfree(buf);
14519 WARN_ON(i > INFO_STRING_LEN);
14520}
14521
14522
14523
14524
14525
14526
14527
14528
14529
14530
14531
14532static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf)
14533{
14534 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr))
14535 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr);
14536}
14537
14538
14539
14540
14541
14542
14543void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags)
14544{
14545 if (fec_cfg & I40E_AQ_SET_FEC_AUTO)
14546 *flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC;
14547 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) ||
14548 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) {
14549 *flags |= I40E_FLAG_RS_FEC;
14550 *flags &= ~I40E_FLAG_BASE_R_FEC;
14551 }
14552 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) ||
14553 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) {
14554 *flags |= I40E_FLAG_BASE_R_FEC;
14555 *flags &= ~I40E_FLAG_RS_FEC;
14556 }
14557 if (fec_cfg == 0)
14558 *flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC);
14559}
14560
14561
14562
14563
14564
14565
14566
14567
14568
14569
14570static bool i40e_check_recovery_mode(struct i40e_pf *pf)
14571{
14572 u32 val = rd32(&pf->hw, I40E_GL_FWSTS);
14573
14574 if (val & I40E_GL_FWSTS_FWS1B_MASK) {
14575 dev_notice(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n");
14576 dev_notice(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n");
14577 set_bit(__I40E_RECOVERY_MODE, pf->state);
14578
14579 return true;
14580 }
14581 if (test_and_clear_bit(__I40E_RECOVERY_MODE, pf->state))
14582 dev_info(&pf->pdev->dev, "Reinitializing in normal mode with full functionality.\n");
14583
14584 return false;
14585}
14586
14587
14588
14589
14590
14591
14592
14593
14594
14595
14596
14597static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw)
14598{
14599 struct i40e_vsi *vsi;
14600 int err;
14601 int v_idx;
14602
14603 pci_save_state(pf->pdev);
14604
14605
14606 timer_setup(&pf->service_timer, i40e_service_timer, 0);
14607 pf->service_timer_period = HZ;
14608
14609 INIT_WORK(&pf->service_task, i40e_service_task);
14610 clear_bit(__I40E_SERVICE_SCHED, pf->state);
14611
14612 err = i40e_init_interrupt_scheme(pf);
14613 if (err)
14614 goto err_switch_setup;
14615
14616
14617
14618
14619
14620
14621 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
14622 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
14623 else
14624 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
14625
14626
14627 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
14628 GFP_KERNEL);
14629 if (!pf->vsi) {
14630 err = -ENOMEM;
14631 goto err_switch_setup;
14632 }
14633
14634
14635
14636
14637 v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN);
14638 if (v_idx < 0)
14639 goto err_switch_setup;
14640 pf->lan_vsi = v_idx;
14641 vsi = pf->vsi[v_idx];
14642 if (!vsi)
14643 goto err_switch_setup;
14644 vsi->alloc_queue_pairs = 1;
14645 err = i40e_config_netdev(vsi);
14646 if (err)
14647 goto err_switch_setup;
14648 err = register_netdev(vsi->netdev);
14649 if (err)
14650 goto err_switch_setup;
14651 vsi->netdev_registered = true;
14652 i40e_dbg_pf_init(pf);
14653
14654 err = i40e_setup_misc_vector_for_recovery_mode(pf);
14655 if (err)
14656 goto err_switch_setup;
14657
14658
14659 i40e_send_version(pf);
14660
14661
14662 mod_timer(&pf->service_timer,
14663 round_jiffies(jiffies + pf->service_timer_period));
14664
14665 return 0;
14666
14667err_switch_setup:
14668 i40e_reset_interrupt_capability(pf);
14669 del_timer_sync(&pf->service_timer);
14670 i40e_shutdown_adminq(hw);
14671 iounmap(hw->hw_addr);
14672 pci_disable_pcie_error_reporting(pf->pdev);
14673 pci_release_mem_regions(pf->pdev);
14674 pci_disable_device(pf->pdev);
14675 kfree(pf);
14676
14677 return err;
14678}
14679
14680
14681
14682
14683
14684
14685
14686
14687
14688
14689
14690
14691static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
14692{
14693 struct i40e_aq_get_phy_abilities_resp abilities;
14694 struct i40e_pf *pf;
14695 struct i40e_hw *hw;
14696 static u16 pfs_found;
14697 u16 wol_nvm_bits;
14698 u16 link_status;
14699 int err;
14700 u32 val;
14701 u32 i;
14702 u8 set_fc_aq_fail;
14703
14704 err = pci_enable_device_mem(pdev);
14705 if (err)
14706 return err;
14707
14708
14709 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
14710 if (err) {
14711 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
14712 if (err) {
14713 dev_err(&pdev->dev,
14714 "DMA configuration failed: 0x%x\n", err);
14715 goto err_dma;
14716 }
14717 }
14718
14719
14720 err = pci_request_mem_regions(pdev, i40e_driver_name);
14721 if (err) {
14722 dev_info(&pdev->dev,
14723 "pci_request_selected_regions failed %d\n", err);
14724 goto err_pci_reg;
14725 }
14726
14727 pci_enable_pcie_error_reporting(pdev);
14728 pci_set_master(pdev);
14729
14730
14731
14732
14733
14734
14735 pf = kzalloc(sizeof(*pf), GFP_KERNEL);
14736 if (!pf) {
14737 err = -ENOMEM;
14738 goto err_pf_alloc;
14739 }
14740 pf->next_vsi = 0;
14741 pf->pdev = pdev;
14742 set_bit(__I40E_DOWN, pf->state);
14743
14744 hw = &pf->hw;
14745 hw->back = pf;
14746
14747 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0),
14748 I40E_MAX_CSR_SPACE);
14749
14750
14751
14752
14753
14754 if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) {
14755 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n",
14756 pf->ioremap_len);
14757 err = -ENOMEM;
14758 goto err_ioremap;
14759 }
14760 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len);
14761 if (!hw->hw_addr) {
14762 err = -EIO;
14763 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n",
14764 (unsigned int)pci_resource_start(pdev, 0),
14765 pf->ioremap_len, err);
14766 goto err_ioremap;
14767 }
14768 hw->vendor_id = pdev->vendor;
14769 hw->device_id = pdev->device;
14770 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
14771 hw->subsystem_vendor_id = pdev->subsystem_vendor;
14772 hw->subsystem_device_id = pdev->subsystem_device;
14773 hw->bus.device = PCI_SLOT(pdev->devfn);
14774 hw->bus.func = PCI_FUNC(pdev->devfn);
14775 hw->bus.bus_id = pdev->bus->number;
14776 pf->instance = pfs_found;
14777
14778
14779
14780
14781 hw->switch_tag = 0xffff;
14782 hw->first_tag = ETH_P_8021AD;
14783 hw->second_tag = ETH_P_8021Q;
14784
14785 INIT_LIST_HEAD(&pf->l3_flex_pit_list);
14786 INIT_LIST_HEAD(&pf->l4_flex_pit_list);
14787 INIT_LIST_HEAD(&pf->ddp_old_prof);
14788
14789
14790
14791
14792 mutex_init(&hw->aq.asq_mutex);
14793 mutex_init(&hw->aq.arq_mutex);
14794
14795 pf->msg_enable = netif_msg_init(debug,
14796 NETIF_MSG_DRV |
14797 NETIF_MSG_PROBE |
14798 NETIF_MSG_LINK);
14799 if (debug < -1)
14800 pf->hw.debug_mask = debug;
14801
14802
14803 if (hw->revision_id == 0 &&
14804 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) {
14805 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK);
14806 i40e_flush(hw);
14807 msleep(200);
14808 pf->corer_count++;
14809
14810 i40e_clear_pxe_mode(hw);
14811 }
14812
14813
14814 i40e_clear_hw(hw);
14815 if (!i40e_check_recovery_mode(pf)) {
14816 err = i40e_pf_reset(hw);
14817 if (err) {
14818 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err);
14819 goto err_pf_reset;
14820 }
14821 pf->pfr_count++;
14822 }
14823 hw->aq.num_arq_entries = I40E_AQ_LEN;
14824 hw->aq.num_asq_entries = I40E_AQ_LEN;
14825 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE;
14826 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE;
14827 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT;
14828
14829 snprintf(pf->int_name, sizeof(pf->int_name) - 1,
14830 "%s-%s:misc",
14831 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev));
14832
14833 err = i40e_init_shared_code(hw);
14834 if (err) {
14835 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n",
14836 err);
14837 goto err_pf_reset;
14838 }
14839
14840
14841 pf->hw.fc.requested_mode = I40E_FC_NONE;
14842
14843 err = i40e_init_adminq(hw);
14844 if (err) {
14845 if (err == I40E_ERR_FIRMWARE_API_VERSION)
14846 dev_info(&pdev->dev,
14847 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n",
14848 hw->aq.api_maj_ver,
14849 hw->aq.api_min_ver,
14850 I40E_FW_API_VERSION_MAJOR,
14851 I40E_FW_MINOR_VERSION(hw));
14852 else
14853 dev_info(&pdev->dev,
14854 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n");
14855
14856 goto err_pf_reset;
14857 }
14858 i40e_get_oem_version(hw);
14859
14860
14861 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n",
14862 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build,
14863 hw->aq.api_maj_ver, hw->aq.api_min_ver,
14864 i40e_nvm_version_str(hw), hw->vendor_id, hw->device_id,
14865 hw->subsystem_vendor_id, hw->subsystem_device_id);
14866
14867 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR &&
14868 hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw))
14869 dev_info(&pdev->dev,
14870 "The driver for the device detected a newer version of the NVM image v%u.%u than expected v%u.%u. Please install the most recent version of the network driver.\n",
14871 hw->aq.api_maj_ver,
14872 hw->aq.api_min_ver,
14873 I40E_FW_API_VERSION_MAJOR,
14874 I40E_FW_MINOR_VERSION(hw));
14875 else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4)
14876 dev_info(&pdev->dev,
14877 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n",
14878 hw->aq.api_maj_ver,
14879 hw->aq.api_min_ver,
14880 I40E_FW_API_VERSION_MAJOR,
14881 I40E_FW_MINOR_VERSION(hw));
14882
14883 i40e_verify_eeprom(pf);
14884
14885
14886 if (hw->revision_id < 1)
14887 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n");
14888
14889 i40e_clear_pxe_mode(hw);
14890
14891 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities);
14892 if (err)
14893 goto err_adminq_setup;
14894
14895 err = i40e_sw_init(pf);
14896 if (err) {
14897 dev_info(&pdev->dev, "sw_init failed: %d\n", err);
14898 goto err_sw_init;
14899 }
14900
14901 if (test_bit(__I40E_RECOVERY_MODE, pf->state))
14902 return i40e_init_recovery_mode(pf, hw);
14903
14904 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp,
14905 hw->func_caps.num_rx_qp, 0, 0);
14906 if (err) {
14907 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err);
14908 goto err_init_lan_hmc;
14909 }
14910
14911 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY);
14912 if (err) {
14913 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err);
14914 err = -ENOENT;
14915 goto err_configure_lan_hmc;
14916 }
14917
14918
14919
14920
14921
14922 if (pf->hw_features & I40E_HW_STOP_FW_LLDP) {
14923 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n");
14924 i40e_aq_stop_lldp(hw, true, false, NULL);
14925 }
14926
14927
14928 i40e_get_platform_mac_addr(pdev, pf);
14929
14930 if (!is_valid_ether_addr(hw->mac.addr)) {
14931 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr);
14932 err = -EIO;
14933 goto err_mac_addr;
14934 }
14935 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr);
14936 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr);
14937 i40e_get_port_mac_addr(hw, hw->mac.port_addr);
14938 if (is_valid_ether_addr(hw->mac.port_addr))
14939 pf->hw_features |= I40E_HW_PORT_ID_VALID;
14940
14941 pci_set_drvdata(pdev, pf);
14942 pci_save_state(pdev);
14943
14944 dev_info(&pdev->dev,
14945 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ?
14946 "FW LLDP is disabled\n" :
14947 "FW LLDP is enabled\n");
14948
14949
14950 i40e_aq_set_dcb_parameters(hw, true, NULL);
14951
14952#ifdef CONFIG_I40E_DCB
14953 err = i40e_init_pf_dcb(pf);
14954 if (err) {
14955 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err);
14956 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED);
14957
14958 }
14959#endif
14960
14961
14962 timer_setup(&pf->service_timer, i40e_service_timer, 0);
14963 pf->service_timer_period = HZ;
14964
14965 INIT_WORK(&pf->service_task, i40e_service_task);
14966 clear_bit(__I40E_SERVICE_SCHED, pf->state);
14967
14968
14969 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits);
14970 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1)
14971 pf->wol_en = false;
14972 else
14973 pf->wol_en = true;
14974 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en);
14975
14976
14977 i40e_determine_queue_usage(pf);
14978 err = i40e_init_interrupt_scheme(pf);
14979 if (err)
14980 goto err_switch_setup;
14981
14982
14983
14984
14985
14986
14987 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC)
14988 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC;
14989 else
14990 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis;
14991
14992
14993 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *),
14994 GFP_KERNEL);
14995 if (!pf->vsi) {
14996 err = -ENOMEM;
14997 goto err_switch_setup;
14998 }
14999
15000#ifdef CONFIG_PCI_IOV
15001
15002 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15003 (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15004 !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15005 if (pci_num_vf(pdev))
15006 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED;
15007 }
15008#endif
15009 err = i40e_setup_pf_switch(pf, false);
15010 if (err) {
15011 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err);
15012 goto err_vsis;
15013 }
15014 INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list);
15015
15016
15017 err = i40e_set_fc(hw, &set_fc_aq_fail, true);
15018 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET)
15019 dev_dbg(&pf->pdev->dev,
15020 "Set fc with err %s aq_err %s on get_phy_cap\n",
15021 i40e_stat_str(hw, err),
15022 i40e_aq_str(hw, hw->aq.asq_last_status));
15023 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET)
15024 dev_dbg(&pf->pdev->dev,
15025 "Set fc with err %s aq_err %s on set_phy_config\n",
15026 i40e_stat_str(hw, err),
15027 i40e_aq_str(hw, hw->aq.asq_last_status));
15028 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE)
15029 dev_dbg(&pf->pdev->dev,
15030 "Set fc with err %s aq_err %s on get_link_info\n",
15031 i40e_stat_str(hw, err),
15032 i40e_aq_str(hw, hw->aq.asq_last_status));
15033
15034
15035 for (i = 0; i < pf->num_alloc_vsi; i++) {
15036 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) {
15037 i40e_vsi_open(pf->vsi[i]);
15038 break;
15039 }
15040 }
15041
15042
15043
15044
15045 err = i40e_aq_set_phy_int_mask(&pf->hw,
15046 ~(I40E_AQ_EVENT_LINK_UPDOWN |
15047 I40E_AQ_EVENT_MEDIA_NA |
15048 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL);
15049 if (err)
15050 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n",
15051 i40e_stat_str(&pf->hw, err),
15052 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15053
15054
15055
15056
15057
15058 val = rd32(hw, I40E_REG_MSS);
15059 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) {
15060 val &= ~I40E_REG_MSS_MIN_MASK;
15061 val |= I40E_64BYTE_MSS;
15062 wr32(hw, I40E_REG_MSS, val);
15063 }
15064
15065 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) {
15066 msleep(75);
15067 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL);
15068 if (err)
15069 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n",
15070 i40e_stat_str(&pf->hw, err),
15071 i40e_aq_str(&pf->hw,
15072 pf->hw.aq.asq_last_status));
15073 }
15074
15075
15076
15077
15078 clear_bit(__I40E_DOWN, pf->state);
15079
15080
15081
15082
15083
15084
15085 if (pf->flags & I40E_FLAG_MSIX_ENABLED) {
15086 err = i40e_setup_misc_vector(pf);
15087 if (err) {
15088 dev_info(&pdev->dev,
15089 "setup of misc vector failed: %d\n", err);
15090 goto err_vsis;
15091 }
15092 }
15093
15094#ifdef CONFIG_PCI_IOV
15095
15096 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) &&
15097 (pf->flags & I40E_FLAG_MSIX_ENABLED) &&
15098 !test_bit(__I40E_BAD_EEPROM, pf->state)) {
15099
15100 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM);
15101 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK;
15102 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val);
15103 i40e_flush(hw);
15104
15105 if (pci_num_vf(pdev)) {
15106 dev_info(&pdev->dev,
15107 "Active VFs found, allocating resources.\n");
15108 err = i40e_alloc_vfs(pf, pci_num_vf(pdev));
15109 if (err)
15110 dev_info(&pdev->dev,
15111 "Error %d allocating resources for existing VFs\n",
15112 err);
15113 }
15114 }
15115#endif
15116
15117 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15118 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile,
15119 pf->num_iwarp_msix,
15120 I40E_IWARP_IRQ_PILE_ID);
15121 if (pf->iwarp_base_vector < 0) {
15122 dev_info(&pdev->dev,
15123 "failed to get tracking for %d vectors for IWARP err=%d\n",
15124 pf->num_iwarp_msix, pf->iwarp_base_vector);
15125 pf->flags &= ~I40E_FLAG_IWARP_ENABLED;
15126 }
15127 }
15128
15129 i40e_dbg_pf_init(pf);
15130
15131
15132 i40e_send_version(pf);
15133
15134
15135 mod_timer(&pf->service_timer,
15136 round_jiffies(jiffies + pf->service_timer_period));
15137
15138
15139 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15140 err = i40e_lan_add_device(pf);
15141 if (err)
15142 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n",
15143 err);
15144 }
15145
15146#define PCI_SPEED_SIZE 8
15147#define PCI_WIDTH_SIZE 8
15148
15149
15150
15151
15152 if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) {
15153 char speed[PCI_SPEED_SIZE] = "Unknown";
15154 char width[PCI_WIDTH_SIZE] = "Unknown";
15155
15156
15157
15158
15159 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA,
15160 &link_status);
15161
15162 i40e_set_pci_config_data(hw, link_status);
15163
15164 switch (hw->bus.speed) {
15165 case i40e_bus_speed_8000:
15166 strlcpy(speed, "8.0", PCI_SPEED_SIZE); break;
15167 case i40e_bus_speed_5000:
15168 strlcpy(speed, "5.0", PCI_SPEED_SIZE); break;
15169 case i40e_bus_speed_2500:
15170 strlcpy(speed, "2.5", PCI_SPEED_SIZE); break;
15171 default:
15172 break;
15173 }
15174 switch (hw->bus.width) {
15175 case i40e_bus_width_pcie_x8:
15176 strlcpy(width, "8", PCI_WIDTH_SIZE); break;
15177 case i40e_bus_width_pcie_x4:
15178 strlcpy(width, "4", PCI_WIDTH_SIZE); break;
15179 case i40e_bus_width_pcie_x2:
15180 strlcpy(width, "2", PCI_WIDTH_SIZE); break;
15181 case i40e_bus_width_pcie_x1:
15182 strlcpy(width, "1", PCI_WIDTH_SIZE); break;
15183 default:
15184 break;
15185 }
15186
15187 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n",
15188 speed, width);
15189
15190 if (hw->bus.width < i40e_bus_width_pcie_x8 ||
15191 hw->bus.speed < i40e_bus_speed_8000) {
15192 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
15193 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
15194 }
15195 }
15196
15197
15198 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL);
15199 if (err)
15200 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n",
15201 i40e_stat_str(&pf->hw, err),
15202 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15203 pf->hw.phy.link_info.requested_speeds = abilities.link_speed;
15204
15205
15206 i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags);
15207
15208
15209 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL);
15210 if (err)
15211 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n",
15212 i40e_stat_str(&pf->hw, err),
15213 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status));
15214
15215
15216
15217
15218
15219
15220
15221 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw,
15222 pf->main_vsi_seid);
15223
15224 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) ||
15225 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4))
15226 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS;
15227 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722)
15228 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER;
15229
15230 i40e_print_features(pf);
15231
15232 return 0;
15233
15234
15235err_vsis:
15236 set_bit(__I40E_DOWN, pf->state);
15237 i40e_clear_interrupt_scheme(pf);
15238 kfree(pf->vsi);
15239err_switch_setup:
15240 i40e_reset_interrupt_capability(pf);
15241 del_timer_sync(&pf->service_timer);
15242err_mac_addr:
15243err_configure_lan_hmc:
15244 (void)i40e_shutdown_lan_hmc(hw);
15245err_init_lan_hmc:
15246 kfree(pf->qp_pile);
15247err_sw_init:
15248err_adminq_setup:
15249err_pf_reset:
15250 iounmap(hw->hw_addr);
15251err_ioremap:
15252 kfree(pf);
15253err_pf_alloc:
15254 pci_disable_pcie_error_reporting(pdev);
15255 pci_release_mem_regions(pdev);
15256err_pci_reg:
15257err_dma:
15258 pci_disable_device(pdev);
15259 return err;
15260}
15261
15262
15263
15264
15265
15266
15267
15268
15269
15270
15271static void i40e_remove(struct pci_dev *pdev)
15272{
15273 struct i40e_pf *pf = pci_get_drvdata(pdev);
15274 struct i40e_hw *hw = &pf->hw;
15275 i40e_status ret_code;
15276 int i;
15277
15278 i40e_dbg_pf_exit(pf);
15279
15280 i40e_ptp_stop(pf);
15281
15282
15283 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0);
15284 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0);
15285
15286
15287 set_bit(__I40E_SUSPENDED, pf->state);
15288 set_bit(__I40E_DOWN, pf->state);
15289 if (pf->service_timer.function)
15290 del_timer_sync(&pf->service_timer);
15291 if (pf->service_task.func)
15292 cancel_work_sync(&pf->service_task);
15293
15294 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) {
15295 struct i40e_vsi *vsi = pf->vsi[0];
15296
15297
15298
15299
15300
15301 unregister_netdev(vsi->netdev);
15302 free_netdev(vsi->netdev);
15303
15304 goto unmap;
15305 }
15306
15307
15308
15309
15310 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
15311
15312 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) {
15313 i40e_free_vfs(pf);
15314 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED;
15315 }
15316
15317 i40e_fdir_teardown(pf);
15318
15319
15320
15321
15322 for (i = 0; i < I40E_MAX_VEB; i++) {
15323 if (!pf->veb[i])
15324 continue;
15325
15326 if (pf->veb[i]->uplink_seid == pf->mac_seid ||
15327 pf->veb[i]->uplink_seid == 0)
15328 i40e_switch_branch_release(pf->veb[i]);
15329 }
15330
15331
15332
15333
15334 if (pf->vsi[pf->lan_vsi])
15335 i40e_vsi_release(pf->vsi[pf->lan_vsi]);
15336
15337 i40e_cloud_filter_exit(pf);
15338
15339
15340 if (pf->flags & I40E_FLAG_IWARP_ENABLED) {
15341 ret_code = i40e_lan_del_device(pf);
15342 if (ret_code)
15343 dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
15344 ret_code);
15345 }
15346
15347
15348 if (hw->hmc.hmc_obj) {
15349 ret_code = i40e_shutdown_lan_hmc(hw);
15350 if (ret_code)
15351 dev_warn(&pdev->dev,
15352 "Failed to destroy the HMC resources: %d\n",
15353 ret_code);
15354 }
15355
15356unmap:
15357
15358 if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
15359 !(pf->flags & I40E_FLAG_MSIX_ENABLED))
15360 free_irq(pf->pdev->irq, pf);
15361
15362
15363 i40e_shutdown_adminq(hw);
15364
15365
15366 mutex_destroy(&hw->aq.arq_mutex);
15367 mutex_destroy(&hw->aq.asq_mutex);
15368
15369
15370 rtnl_lock();
15371 i40e_clear_interrupt_scheme(pf);
15372 for (i = 0; i < pf->num_alloc_vsi; i++) {
15373 if (pf->vsi[i]) {
15374 if (!test_bit(__I40E_RECOVERY_MODE, pf->state))
15375 i40e_vsi_clear_rings(pf->vsi[i]);
15376 i40e_vsi_clear(pf->vsi[i]);
15377 pf->vsi[i] = NULL;
15378 }
15379 }
15380 rtnl_unlock();
15381
15382 for (i = 0; i < I40E_MAX_VEB; i++) {
15383 kfree(pf->veb[i]);
15384 pf->veb[i] = NULL;
15385 }
15386
15387 kfree(pf->qp_pile);
15388 kfree(pf->vsi);
15389
15390 iounmap(hw->hw_addr);
15391 kfree(pf);
15392 pci_release_mem_regions(pdev);
15393
15394 pci_disable_pcie_error_reporting(pdev);
15395 pci_disable_device(pdev);
15396}
15397
15398
15399
15400
15401
15402
15403
15404
15405
15406
15407static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev,
15408 enum pci_channel_state error)
15409{
15410 struct i40e_pf *pf = pci_get_drvdata(pdev);
15411
15412 dev_info(&pdev->dev, "%s: error %d\n", __func__, error);
15413
15414 if (!pf) {
15415 dev_info(&pdev->dev,
15416 "Cannot recover - error happened during device probe\n");
15417 return PCI_ERS_RESULT_DISCONNECT;
15418 }
15419
15420
15421 if (!test_bit(__I40E_SUSPENDED, pf->state))
15422 i40e_prep_for_reset(pf, false);
15423
15424
15425 return PCI_ERS_RESULT_NEED_RESET;
15426}
15427
15428
15429
15430
15431
15432
15433
15434
15435
15436
15437static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev)
15438{
15439 struct i40e_pf *pf = pci_get_drvdata(pdev);
15440 pci_ers_result_t result;
15441 u32 reg;
15442
15443 dev_dbg(&pdev->dev, "%s\n", __func__);
15444 if (pci_enable_device_mem(pdev)) {
15445 dev_info(&pdev->dev,
15446 "Cannot re-enable PCI device after reset.\n");
15447 result = PCI_ERS_RESULT_DISCONNECT;
15448 } else {
15449 pci_set_master(pdev);
15450 pci_restore_state(pdev);
15451 pci_save_state(pdev);
15452 pci_wake_from_d3(pdev, false);
15453
15454 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG);
15455 if (reg == 0)
15456 result = PCI_ERS_RESULT_RECOVERED;
15457 else
15458 result = PCI_ERS_RESULT_DISCONNECT;
15459 }
15460
15461 return result;
15462}
15463
15464
15465
15466
15467
15468static void i40e_pci_error_reset_prepare(struct pci_dev *pdev)
15469{
15470 struct i40e_pf *pf = pci_get_drvdata(pdev);
15471
15472 i40e_prep_for_reset(pf, false);
15473}
15474
15475
15476
15477
15478
15479static void i40e_pci_error_reset_done(struct pci_dev *pdev)
15480{
15481 struct i40e_pf *pf = pci_get_drvdata(pdev);
15482
15483 i40e_reset_and_rebuild(pf, false, false);
15484}
15485
15486
15487
15488
15489
15490
15491
15492
15493static void i40e_pci_error_resume(struct pci_dev *pdev)
15494{
15495 struct i40e_pf *pf = pci_get_drvdata(pdev);
15496
15497 dev_dbg(&pdev->dev, "%s\n", __func__);
15498 if (test_bit(__I40E_SUSPENDED, pf->state))
15499 return;
15500
15501 i40e_handle_reset_warning(pf, false);
15502}
15503
15504
15505
15506
15507
15508
15509static void i40e_enable_mc_magic_wake(struct i40e_pf *pf)
15510{
15511 struct i40e_hw *hw = &pf->hw;
15512 i40e_status ret;
15513 u8 mac_addr[6];
15514 u16 flags = 0;
15515
15516
15517 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) {
15518 ether_addr_copy(mac_addr,
15519 pf->vsi[pf->lan_vsi]->netdev->dev_addr);
15520 } else {
15521 dev_err(&pf->pdev->dev,
15522 "Failed to retrieve MAC address; using default\n");
15523 ether_addr_copy(mac_addr, hw->mac.addr);
15524 }
15525
15526
15527
15528
15529
15530 flags = I40E_AQC_WRITE_TYPE_LAA_WOL;
15531
15532 if (hw->func_caps.flex10_enable && hw->partition_id != 1)
15533 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY;
15534
15535 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
15536 if (ret) {
15537 dev_err(&pf->pdev->dev,
15538 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up");
15539 return;
15540 }
15541
15542 flags = I40E_AQC_MC_MAG_EN
15543 | I40E_AQC_WOL_PRESERVE_ON_PFR
15544 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG;
15545 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL);
15546 if (ret)
15547 dev_err(&pf->pdev->dev,
15548 "Failed to enable Multicast Magic Packet wake up\n");
15549}
15550
15551
15552
15553
15554
15555static void i40e_shutdown(struct pci_dev *pdev)
15556{
15557 struct i40e_pf *pf = pci_get_drvdata(pdev);
15558 struct i40e_hw *hw = &pf->hw;
15559
15560 set_bit(__I40E_SUSPENDED, pf->state);
15561 set_bit(__I40E_DOWN, pf->state);
15562
15563 del_timer_sync(&pf->service_timer);
15564 cancel_work_sync(&pf->service_task);
15565 i40e_cloud_filter_exit(pf);
15566 i40e_fdir_teardown(pf);
15567
15568
15569
15570
15571 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
15572
15573 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
15574 i40e_enable_mc_magic_wake(pf);
15575
15576 i40e_prep_for_reset(pf, false);
15577
15578 wr32(hw, I40E_PFPM_APM,
15579 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
15580 wr32(hw, I40E_PFPM_WUFC,
15581 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
15582
15583
15584 if (test_bit(__I40E_RECOVERY_MODE, pf->state) &&
15585 !(pf->flags & I40E_FLAG_MSIX_ENABLED))
15586 free_irq(pf->pdev->irq, pf);
15587
15588
15589
15590
15591
15592 rtnl_lock();
15593 i40e_clear_interrupt_scheme(pf);
15594 rtnl_unlock();
15595
15596 if (system_state == SYSTEM_POWER_OFF) {
15597 pci_wake_from_d3(pdev, pf->wol_en);
15598 pci_set_power_state(pdev, PCI_D3hot);
15599 }
15600}
15601
15602
15603
15604
15605
15606static int __maybe_unused i40e_suspend(struct device *dev)
15607{
15608 struct pci_dev *pdev = to_pci_dev(dev);
15609 struct i40e_pf *pf = pci_get_drvdata(pdev);
15610 struct i40e_hw *hw = &pf->hw;
15611
15612
15613 if (test_and_set_bit(__I40E_SUSPENDED, pf->state))
15614 return 0;
15615
15616 set_bit(__I40E_DOWN, pf->state);
15617
15618
15619 del_timer_sync(&pf->service_timer);
15620 cancel_work_sync(&pf->service_task);
15621
15622
15623
15624
15625 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false);
15626
15627 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE))
15628 i40e_enable_mc_magic_wake(pf);
15629
15630
15631
15632
15633
15634 rtnl_lock();
15635
15636 i40e_prep_for_reset(pf, true);
15637
15638 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0));
15639 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0));
15640
15641
15642
15643
15644
15645
15646 i40e_clear_interrupt_scheme(pf);
15647
15648 rtnl_unlock();
15649
15650 return 0;
15651}
15652
15653
15654
15655
15656
15657static int __maybe_unused i40e_resume(struct device *dev)
15658{
15659 struct pci_dev *pdev = to_pci_dev(dev);
15660 struct i40e_pf *pf = pci_get_drvdata(pdev);
15661 int err;
15662
15663
15664 if (!test_bit(__I40E_SUSPENDED, pf->state))
15665 return 0;
15666
15667
15668
15669
15670 rtnl_lock();
15671
15672
15673
15674
15675 err = i40e_restore_interrupt_scheme(pf);
15676 if (err) {
15677 dev_err(&pdev->dev, "Cannot restore interrupt scheme: %d\n",
15678 err);
15679 }
15680
15681 clear_bit(__I40E_DOWN, pf->state);
15682 i40e_reset_and_rebuild(pf, false, true);
15683
15684 rtnl_unlock();
15685
15686
15687 clear_bit(__I40E_SUSPENDED, pf->state);
15688
15689
15690 mod_timer(&pf->service_timer,
15691 round_jiffies(jiffies + pf->service_timer_period));
15692
15693 return 0;
15694}
15695
15696static const struct pci_error_handlers i40e_err_handler = {
15697 .error_detected = i40e_pci_error_detected,
15698 .slot_reset = i40e_pci_error_slot_reset,
15699 .reset_prepare = i40e_pci_error_reset_prepare,
15700 .reset_done = i40e_pci_error_reset_done,
15701 .resume = i40e_pci_error_resume,
15702};
15703
15704static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume);
15705
15706static struct pci_driver i40e_driver = {
15707 .name = i40e_driver_name,
15708 .id_table = i40e_pci_tbl,
15709 .probe = i40e_probe,
15710 .remove = i40e_remove,
15711 .driver = {
15712 .pm = &i40e_pm_ops,
15713 },
15714 .shutdown = i40e_shutdown,
15715 .err_handler = &i40e_err_handler,
15716 .sriov_configure = i40e_pci_sriov_configure,
15717};
15718
15719
15720
15721
15722
15723
15724
15725static int __init i40e_init_module(void)
15726{
15727 pr_info("%s: %s - version %s\n", i40e_driver_name,
15728 i40e_driver_string, i40e_driver_version_str);
15729 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright);
15730
15731
15732
15733
15734
15735
15736
15737
15738 i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name);
15739 if (!i40e_wq) {
15740 pr_err("%s: Failed to create workqueue\n", i40e_driver_name);
15741 return -ENOMEM;
15742 }
15743
15744 i40e_dbg_init();
15745 return pci_register_driver(&i40e_driver);
15746}
15747module_init(i40e_init_module);
15748
15749
15750
15751
15752
15753
15754
15755static void __exit i40e_exit_module(void)
15756{
15757 pci_unregister_driver(&i40e_driver);
15758 destroy_workqueue(i40e_wq);
15759 i40e_dbg_exit();
15760}
15761module_exit(i40e_exit_module);
15762