1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32#ifndef _QEDE_H_
33#define _QEDE_H_
34#include <linux/compiler.h>
35#include <linux/version.h>
36#include <linux/workqueue.h>
37#include <linux/netdevice.h>
38#include <linux/interrupt.h>
39#include <linux/bitmap.h>
40#include <linux/kernel.h>
41#include <linux/mutex.h>
42#include <linux/bpf.h>
43#include <net/xdp.h>
44#include <linux/qed/qede_rdma.h>
45#include <linux/io.h>
46#ifdef CONFIG_RFS_ACCEL
47#include <linux/cpu_rmap.h>
48#endif
49#include <linux/qed/common_hsi.h>
50#include <linux/qed/eth_common.h>
51#include <linux/qed/qed_if.h>
52#include <linux/qed/qed_chain.h>
53#include <linux/qed/qed_eth_if.h>
54
55#include <net/pkt_cls.h>
56#include <net/tc_act/tc_gact.h>
57
58#define QEDE_MAJOR_VERSION 8
59#define QEDE_MINOR_VERSION 37
60#define QEDE_REVISION_VERSION 0
61#define QEDE_ENGINEERING_VERSION 20
62#define DRV_MODULE_VERSION __stringify(QEDE_MAJOR_VERSION) "." \
63 __stringify(QEDE_MINOR_VERSION) "." \
64 __stringify(QEDE_REVISION_VERSION) "." \
65 __stringify(QEDE_ENGINEERING_VERSION)
66
67#define DRV_MODULE_SYM qede
68
69struct qede_stats_common {
70 u64 no_buff_discards;
71 u64 packet_too_big_discard;
72 u64 ttl0_discard;
73 u64 rx_ucast_bytes;
74 u64 rx_mcast_bytes;
75 u64 rx_bcast_bytes;
76 u64 rx_ucast_pkts;
77 u64 rx_mcast_pkts;
78 u64 rx_bcast_pkts;
79 u64 mftag_filter_discards;
80 u64 mac_filter_discards;
81 u64 gft_filter_drop;
82 u64 tx_ucast_bytes;
83 u64 tx_mcast_bytes;
84 u64 tx_bcast_bytes;
85 u64 tx_ucast_pkts;
86 u64 tx_mcast_pkts;
87 u64 tx_bcast_pkts;
88 u64 tx_err_drop_pkts;
89 u64 coalesced_pkts;
90 u64 coalesced_events;
91 u64 coalesced_aborts_num;
92 u64 non_coalesced_pkts;
93 u64 coalesced_bytes;
94 u64 link_change_count;
95 u64 ptp_skip_txts;
96
97
98 u64 rx_64_byte_packets;
99 u64 rx_65_to_127_byte_packets;
100 u64 rx_128_to_255_byte_packets;
101 u64 rx_256_to_511_byte_packets;
102 u64 rx_512_to_1023_byte_packets;
103 u64 rx_1024_to_1518_byte_packets;
104 u64 rx_crc_errors;
105 u64 rx_mac_crtl_frames;
106 u64 rx_pause_frames;
107 u64 rx_pfc_frames;
108 u64 rx_align_errors;
109 u64 rx_carrier_errors;
110 u64 rx_oversize_packets;
111 u64 rx_jabbers;
112 u64 rx_undersize_packets;
113 u64 rx_fragments;
114 u64 tx_64_byte_packets;
115 u64 tx_65_to_127_byte_packets;
116 u64 tx_128_to_255_byte_packets;
117 u64 tx_256_to_511_byte_packets;
118 u64 tx_512_to_1023_byte_packets;
119 u64 tx_1024_to_1518_byte_packets;
120 u64 tx_pause_frames;
121 u64 tx_pfc_frames;
122 u64 brb_truncates;
123 u64 brb_discards;
124 u64 tx_mac_ctrl_frames;
125};
126
127struct qede_stats_bb {
128 u64 rx_1519_to_1522_byte_packets;
129 u64 rx_1519_to_2047_byte_packets;
130 u64 rx_2048_to_4095_byte_packets;
131 u64 rx_4096_to_9216_byte_packets;
132 u64 rx_9217_to_16383_byte_packets;
133 u64 tx_1519_to_2047_byte_packets;
134 u64 tx_2048_to_4095_byte_packets;
135 u64 tx_4096_to_9216_byte_packets;
136 u64 tx_9217_to_16383_byte_packets;
137 u64 tx_lpi_entry_count;
138 u64 tx_total_collisions;
139};
140
141struct qede_stats_ah {
142 u64 rx_1519_to_max_byte_packets;
143 u64 tx_1519_to_max_byte_packets;
144};
145
146struct qede_stats {
147 struct qede_stats_common common;
148
149 union {
150 struct qede_stats_bb bb;
151 struct qede_stats_ah ah;
152 };
153};
154
155struct qede_vlan {
156 struct list_head list;
157 u16 vid;
158 bool configured;
159};
160
161struct qede_rdma_dev {
162 struct qedr_dev *qedr_dev;
163 struct list_head entry;
164 struct list_head rdma_event_list;
165 struct workqueue_struct *rdma_wq;
166 bool exp_recovery;
167};
168
169struct qede_ptp;
170
171#define QEDE_RFS_MAX_FLTR 256
172
173enum qede_flags_bit {
174 QEDE_FLAGS_IS_VF = 0,
175 QEDE_FLAGS_LINK_REQUESTED,
176 QEDE_FLAGS_PTP_TX_IN_PRORGESS,
177 QEDE_FLAGS_TX_TIMESTAMPING_EN
178};
179
180struct qede_dev {
181 struct qed_dev *cdev;
182 struct net_device *ndev;
183 struct pci_dev *pdev;
184
185 u32 dp_module;
186 u8 dp_level;
187
188 unsigned long flags;
189#define IS_VF(edev) (test_bit(QEDE_FLAGS_IS_VF, &(edev)->flags))
190
191 const struct qed_eth_ops *ops;
192 struct qede_ptp *ptp;
193 u64 ptp_skip_txts;
194
195 struct qed_dev_eth_info dev_info;
196#define QEDE_MAX_RSS_CNT(edev) ((edev)->dev_info.num_queues)
197#define QEDE_MAX_TSS_CNT(edev) ((edev)->dev_info.num_queues)
198#define QEDE_IS_BB(edev) \
199 ((edev)->dev_info.common.dev_type == QED_DEV_TYPE_BB)
200#define QEDE_IS_AH(edev) \
201 ((edev)->dev_info.common.dev_type == QED_DEV_TYPE_AH)
202
203 struct qede_fastpath *fp_array;
204 u8 req_num_tx;
205 u8 fp_num_tx;
206 u8 req_num_rx;
207 u8 fp_num_rx;
208 u16 req_queues;
209 u16 num_queues;
210#define QEDE_QUEUE_CNT(edev) ((edev)->num_queues)
211#define QEDE_RSS_COUNT(edev) ((edev)->num_queues - (edev)->fp_num_tx)
212#define QEDE_RX_QUEUE_IDX(edev, i) (i)
213#define QEDE_TSS_COUNT(edev) ((edev)->num_queues - (edev)->fp_num_rx)
214
215 struct qed_int_info int_info;
216
217
218 struct mutex qede_lock;
219 u32 state;
220 u16 rx_buf_size;
221 u32 rx_copybreak;
222
223
224#define ETH_OVERHEAD (ETH_HLEN + 8 + 8)
225
226
227
228#define QEDE_RX_ALIGN_SHIFT max(6, min(8, L1_CACHE_SHIFT))
229
230
231
232
233#define QEDE_FW_RX_ALIGN_END \
234 max_t(u64, 1UL << QEDE_RX_ALIGN_SHIFT, \
235 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
236
237 struct qede_stats stats;
238#define QEDE_RSS_INDIR_INITED BIT(0)
239#define QEDE_RSS_KEY_INITED BIT(1)
240#define QEDE_RSS_CAPS_INITED BIT(2)
241 u32 rss_params_inited;
242 u16 rss_ind_table[128];
243 u32 rss_key[10];
244 u8 rss_caps;
245
246 u16 q_num_rx_buffers;
247 u16 q_num_tx_buffers;
248
249 bool gro_disable;
250 struct list_head vlan_list;
251 u16 configured_vlans;
252 u16 non_configured_vlans;
253 bool accept_any_vlan;
254 struct delayed_work sp_task;
255 unsigned long sp_flags;
256 u16 vxlan_dst_port;
257 u16 geneve_dst_port;
258
259 struct qede_arfs *arfs;
260 bool wol_enabled;
261
262 struct qede_rdma_dev rdma_info;
263
264 struct bpf_prog *xdp_prog;
265};
266
267enum QEDE_STATE {
268 QEDE_STATE_CLOSED,
269 QEDE_STATE_OPEN,
270 QEDE_STATE_RECOVERY,
271};
272
273#define HILO_U64(hi, lo) ((((u64)(hi)) << 32) + (lo))
274
275#define MAX_NUM_TC 8
276#define MAX_NUM_PRI 8
277
278
279
280
281
282struct sw_rx_data {
283 struct page *data;
284 dma_addr_t mapping;
285 unsigned int page_offset;
286};
287
288enum qede_agg_state {
289 QEDE_AGG_STATE_NONE = 0,
290 QEDE_AGG_STATE_START = 1,
291 QEDE_AGG_STATE_ERROR = 2
292};
293
294struct qede_agg_info {
295
296
297
298
299
300
301
302
303
304
305 struct sw_rx_data buffer;
306 struct sk_buff *skb;
307
308
309 u16 vlan_tag;
310
311 bool tpa_start_fail;
312 u8 state;
313 u8 frag_id;
314
315 u8 tunnel_type;
316};
317
318struct qede_rx_queue {
319 __le16 *hw_cons_ptr;
320 void __iomem *hw_rxq_prod_addr;
321
322
323 struct device *dev;
324
325 struct bpf_prog *xdp_prog;
326
327 u16 sw_rx_cons;
328 u16 sw_rx_prod;
329
330 u16 filled_buffers;
331 u8 data_direction;
332 u8 rxq_id;
333
334
335 u16 num_rx_buffers;
336
337 u16 rx_headroom;
338
339 u32 rx_buf_size;
340 u32 rx_buf_seg_size;
341
342 struct sw_rx_data *sw_rx_ring;
343 struct qed_chain rx_bd_ring;
344 struct qed_chain rx_comp_ring ____cacheline_aligned;
345
346
347 struct qede_agg_info tpa_info[ETH_TPA_MAX_AGGS_NUM];
348
349
350 u64 rcv_pkts;
351
352 u64 rx_hw_errors;
353 u64 rx_alloc_errors;
354 u64 rx_ip_frags;
355
356 u64 xdp_no_pass;
357
358 void *handle;
359 struct xdp_rxq_info xdp_rxq;
360};
361
362union db_prod {
363 struct eth_db_data data;
364 u32 raw;
365};
366
367struct sw_tx_bd {
368 struct sk_buff *skb;
369 u8 flags;
370
371#define QEDE_TSO_SPLIT_BD BIT(0)
372};
373
374struct sw_tx_xdp {
375 struct page *page;
376 dma_addr_t mapping;
377};
378
379struct qede_tx_queue {
380 u8 is_xdp;
381 bool is_legacy;
382 u16 sw_tx_cons;
383 u16 sw_tx_prod;
384 u16 num_tx_buffers;
385
386 u64 xmit_pkts;
387 u64 stopped_cnt;
388 u64 tx_mem_alloc_err;
389
390 __le16 *hw_cons_ptr;
391
392
393 struct device *dev;
394
395 void __iomem *doorbell_addr;
396 union db_prod tx_db;
397 int index;
398#define QEDE_TXQ_XDP_TO_IDX(edev, txq) ((txq)->index - \
399 QEDE_MAX_TSS_CNT(edev))
400#define QEDE_TXQ_IDX_TO_XDP(edev, idx) ((idx) + QEDE_MAX_TSS_CNT(edev))
401#define QEDE_NDEV_TXQ_ID_TO_FP_ID(edev, idx) ((edev)->fp_num_rx + \
402 ((idx) % QEDE_TSS_COUNT(edev)))
403#define QEDE_NDEV_TXQ_ID_TO_TXQ_COS(edev, idx) ((idx) / QEDE_TSS_COUNT(edev))
404#define QEDE_TXQ_TO_NDEV_TXQ_ID(edev, txq) ((QEDE_TSS_COUNT(edev) * \
405 (txq)->cos) + (txq)->index)
406#define QEDE_NDEV_TXQ_ID_TO_TXQ(edev, idx) \
407 (&((edev)->fp_array[QEDE_NDEV_TXQ_ID_TO_FP_ID(edev, idx)].txq \
408 [QEDE_NDEV_TXQ_ID_TO_TXQ_COS(edev, idx)]))
409#define QEDE_FP_TC0_TXQ(fp) (&((fp)->txq[0]))
410
411
412
413
414 union {
415 struct sw_tx_bd *skbs;
416 struct sw_tx_xdp *xdp;
417 } sw_tx_ring;
418
419 struct qed_chain tx_pbl;
420
421
422 void *handle;
423 u16 cos;
424 u16 ndev_txq_id;
425};
426
427#define BD_UNMAP_ADDR(bd) HILO_U64(le32_to_cpu((bd)->addr.hi), \
428 le32_to_cpu((bd)->addr.lo))
429#define BD_SET_UNMAP_ADDR_LEN(bd, maddr, len) \
430 do { \
431 (bd)->addr.hi = cpu_to_le32(upper_32_bits(maddr)); \
432 (bd)->addr.lo = cpu_to_le32(lower_32_bits(maddr)); \
433 (bd)->nbytes = cpu_to_le16(len); \
434 } while (0)
435#define BD_UNMAP_LEN(bd) (le16_to_cpu((bd)->nbytes))
436
437struct qede_fastpath {
438 struct qede_dev *edev;
439#define QEDE_FASTPATH_TX BIT(0)
440#define QEDE_FASTPATH_RX BIT(1)
441#define QEDE_FASTPATH_XDP BIT(2)
442#define QEDE_FASTPATH_COMBINED (QEDE_FASTPATH_TX | QEDE_FASTPATH_RX)
443 u8 type;
444 u8 id;
445 u8 xdp_xmit;
446 struct napi_struct napi;
447 struct qed_sb_info *sb_info;
448 struct qede_rx_queue *rxq;
449 struct qede_tx_queue *txq;
450 struct qede_tx_queue *xdp_tx;
451
452#define VEC_NAME_SIZE (FIELD_SIZEOF(struct net_device, name) + 8)
453 char name[VEC_NAME_SIZE];
454};
455
456
457#define DP_NAME(edev) ((edev)->ndev->name)
458
459#define XMIT_PLAIN 0
460#define XMIT_L4_CSUM BIT(0)
461#define XMIT_LSO BIT(1)
462#define XMIT_ENC BIT(2)
463#define XMIT_ENC_GSO_L4_CSUM BIT(3)
464
465#define QEDE_CSUM_ERROR BIT(0)
466#define QEDE_CSUM_UNNECESSARY BIT(1)
467#define QEDE_TUNN_CSUM_UNNECESSARY BIT(2)
468
469#define QEDE_SP_RECOVERY 0
470#define QEDE_SP_RX_MODE 1
471
472#ifdef CONFIG_RFS_ACCEL
473int qede_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
474 u16 rxq_index, u32 flow_id);
475#define QEDE_SP_ARFS_CONFIG 4
476#define QEDE_SP_TASK_POLL_DELAY (5 * HZ)
477#endif
478
479void qede_process_arfs_filters(struct qede_dev *edev, bool free_fltr);
480void qede_poll_for_freeing_arfs_filters(struct qede_dev *edev);
481void qede_arfs_filter_op(void *dev, void *filter, u8 fw_rc);
482void qede_free_arfs(struct qede_dev *edev);
483int qede_alloc_arfs(struct qede_dev *edev);
484int qede_add_cls_rule(struct qede_dev *edev, struct ethtool_rxnfc *info);
485int qede_delete_flow_filter(struct qede_dev *edev, u64 cookie);
486int qede_get_cls_rule_entry(struct qede_dev *edev, struct ethtool_rxnfc *cmd);
487int qede_get_cls_rule_all(struct qede_dev *edev, struct ethtool_rxnfc *info,
488 u32 *rule_locs);
489int qede_get_arfs_filter_count(struct qede_dev *edev);
490
491struct qede_reload_args {
492 void (*func)(struct qede_dev *edev, struct qede_reload_args *args);
493 union {
494 netdev_features_t features;
495 struct bpf_prog *new_prog;
496 u16 mtu;
497 } u;
498};
499
500
501netdev_tx_t qede_start_xmit(struct sk_buff *skb, struct net_device *ndev);
502u16 qede_select_queue(struct net_device *dev, struct sk_buff *skb,
503 struct net_device *sb_dev);
504netdev_features_t qede_features_check(struct sk_buff *skb,
505 struct net_device *dev,
506 netdev_features_t features);
507void qede_tx_log_print(struct qede_dev *edev, struct qede_fastpath *fp);
508int qede_alloc_rx_buffer(struct qede_rx_queue *rxq, bool allow_lazy);
509int qede_free_tx_pkt(struct qede_dev *edev,
510 struct qede_tx_queue *txq, int *len);
511int qede_poll(struct napi_struct *napi, int budget);
512irqreturn_t qede_msix_fp_int(int irq, void *fp_cookie);
513
514
515void qede_force_mac(void *dev, u8 *mac, bool forced);
516void qede_udp_ports_update(void *dev, u16 vxlan_port, u16 geneve_port);
517int qede_set_mac_addr(struct net_device *ndev, void *p);
518
519int qede_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid);
520int qede_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid);
521void qede_vlan_mark_nonconfigured(struct qede_dev *edev);
522int qede_configure_vlan_filters(struct qede_dev *edev);
523
524netdev_features_t qede_fix_features(struct net_device *dev,
525 netdev_features_t features);
526int qede_set_features(struct net_device *dev, netdev_features_t features);
527void qede_set_rx_mode(struct net_device *ndev);
528void qede_config_rx_mode(struct net_device *ndev);
529void qede_fill_rss_params(struct qede_dev *edev,
530 struct qed_update_vport_rss_params *rss, u8 *update);
531
532void qede_udp_tunnel_add(struct net_device *dev, struct udp_tunnel_info *ti);
533void qede_udp_tunnel_del(struct net_device *dev, struct udp_tunnel_info *ti);
534
535int qede_xdp(struct net_device *dev, struct netdev_bpf *xdp);
536
537#ifdef CONFIG_DCB
538void qede_set_dcbnl_ops(struct net_device *ndev);
539#endif
540
541void qede_config_debug(uint debug, u32 *p_dp_module, u8 *p_dp_level);
542void qede_set_ethtool_ops(struct net_device *netdev);
543void qede_reload(struct qede_dev *edev,
544 struct qede_reload_args *args, bool is_locked);
545int qede_change_mtu(struct net_device *dev, int new_mtu);
546void qede_fill_by_demand_stats(struct qede_dev *edev);
547void __qede_lock(struct qede_dev *edev);
548void __qede_unlock(struct qede_dev *edev);
549bool qede_has_rx_work(struct qede_rx_queue *rxq);
550int qede_txq_has_work(struct qede_tx_queue *txq);
551void qede_recycle_rx_bd_ring(struct qede_rx_queue *rxq, u8 count);
552void qede_update_rx_prod(struct qede_dev *edev, struct qede_rx_queue *rxq);
553int qede_add_tc_flower_fltr(struct qede_dev *edev, __be16 proto,
554 struct flow_cls_offload *f);
555
556#define RX_RING_SIZE_POW 13
557#define RX_RING_SIZE ((u16)BIT(RX_RING_SIZE_POW))
558#define NUM_RX_BDS_MAX (RX_RING_SIZE - 1)
559#define NUM_RX_BDS_MIN 128
560#define NUM_RX_BDS_DEF ((u16)BIT(10) - 1)
561
562#define TX_RING_SIZE_POW 13
563#define TX_RING_SIZE ((u16)BIT(TX_RING_SIZE_POW))
564#define NUM_TX_BDS_MAX (TX_RING_SIZE - 1)
565#define NUM_TX_BDS_MIN 128
566#define NUM_TX_BDS_DEF NUM_TX_BDS_MAX
567
568#define QEDE_MIN_PKT_LEN 64
569#define QEDE_RX_HDR_SIZE 256
570#define QEDE_MAX_JUMBO_PACKET_SIZE 9600
571#define for_each_queue(i) for (i = 0; i < edev->num_queues; i++)
572#define for_each_cos_in_txq(edev, var) \
573 for ((var) = 0; (var) < (edev)->dev_info.num_tc; (var)++)
574
575#endif
576