1
2
3
4#ifndef __NFP_FLOWER_H__
5#define __NFP_FLOWER_H__ 1
6
7#include "cmsg.h"
8#include "../nfp_net.h"
9
10#include <linux/circ_buf.h>
11#include <linux/hashtable.h>
12#include <linux/rhashtable.h>
13#include <linux/time64.h>
14#include <linux/types.h>
15#include <net/pkt_cls.h>
16#include <net/tcp.h>
17#include <linux/workqueue.h>
18#include <linux/idr.h>
19
20struct nfp_fl_pre_lag;
21struct net_device;
22struct nfp_app;
23
24#define NFP_FL_STAT_ID_MU_NUM GENMASK(31, 22)
25#define NFP_FL_STAT_ID_STAT GENMASK(21, 0)
26
27#define NFP_FL_STATS_ELEM_RS sizeof_field(struct nfp_fl_stats_id, \
28 init_unalloc)
29#define NFP_FLOWER_MASK_ENTRY_RS 256
30#define NFP_FLOWER_MASK_ELEMENT_RS 1
31#define NFP_FLOWER_MASK_HASH_BITS 10
32
33#define NFP_FLOWER_KEY_MAX_LW 32
34
35#define NFP_FL_META_FLAG_MANAGE_MASK BIT(7)
36
37#define NFP_FL_MASK_REUSE_TIME_NS 40000
38#define NFP_FL_MASK_ID_LOCATION 1
39
40
41#define NFP_FL_FEATS_GENEVE BIT(0)
42#define NFP_FL_NBI_MTU_SETTING BIT(1)
43#define NFP_FL_FEATS_GENEVE_OPT BIT(2)
44#define NFP_FL_FEATS_VLAN_PCP BIT(3)
45#define NFP_FL_FEATS_VF_RLIM BIT(4)
46#define NFP_FL_FEATS_FLOW_MOD BIT(5)
47#define NFP_FL_FEATS_PRE_TUN_RULES BIT(6)
48#define NFP_FL_FEATS_IPV6_TUN BIT(7)
49#define NFP_FL_FEATS_VLAN_QINQ BIT(8)
50#define NFP_FL_FEATS_QOS_PPS BIT(9)
51#define NFP_FL_FEATS_HOST_ACK BIT(31)
52
53#define NFP_FL_ENABLE_FLOW_MERGE BIT(0)
54#define NFP_FL_ENABLE_LAG BIT(1)
55
56#define NFP_FL_FEATS_HOST \
57 (NFP_FL_FEATS_GENEVE | \
58 NFP_FL_NBI_MTU_SETTING | \
59 NFP_FL_FEATS_GENEVE_OPT | \
60 NFP_FL_FEATS_VLAN_PCP | \
61 NFP_FL_FEATS_VF_RLIM | \
62 NFP_FL_FEATS_FLOW_MOD | \
63 NFP_FL_FEATS_PRE_TUN_RULES | \
64 NFP_FL_FEATS_IPV6_TUN | \
65 NFP_FL_FEATS_VLAN_QINQ | \
66 NFP_FL_FEATS_QOS_PPS)
67
68struct nfp_fl_mask_id {
69 struct circ_buf mask_id_free_list;
70 ktime_t *last_used;
71 u8 init_unallocated;
72};
73
74struct nfp_fl_stats_id {
75 struct circ_buf free_list;
76 u32 init_unalloc;
77 u8 repeated_em_count;
78};
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94struct nfp_fl_tunnel_offloads {
95 struct rhashtable offloaded_macs;
96 struct list_head ipv4_off_list;
97 struct list_head ipv6_off_list;
98 struct list_head neigh_off_list_v4;
99 struct list_head neigh_off_list_v6;
100 struct mutex ipv4_off_lock;
101 struct mutex ipv6_off_lock;
102 spinlock_t neigh_off_lock_v4;
103 spinlock_t neigh_off_lock_v6;
104 struct ida mac_off_ids;
105 struct notifier_block neigh_nb;
106};
107
108
109
110
111
112
113
114
115
116struct nfp_mtu_conf {
117 u32 portnum;
118 unsigned int requested_val;
119 bool ack;
120 wait_queue_head_t wait_q;
121 spinlock_t lock;
122};
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137struct nfp_fl_lag {
138 struct delayed_work work;
139 struct mutex lock;
140 struct list_head group_list;
141 struct ida ida_handle;
142 unsigned int pkt_num;
143 unsigned int batch_ver;
144 u8 global_inst;
145 bool rst_cfg;
146 struct sk_buff_head retrans_skbs;
147};
148
149
150
151
152
153
154struct nfp_fl_internal_ports {
155 struct idr port_ids;
156 spinlock_t lock;
157};
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200struct nfp_flower_priv {
201 struct nfp_app *app;
202 struct nfp_net *nn;
203 u32 mask_id_seed;
204 u64 flower_version;
205 u64 flower_ext_feats;
206 u8 flower_en_feats;
207 struct nfp_fl_stats_id stats_ids;
208 struct nfp_fl_mask_id mask_ids;
209 DECLARE_HASHTABLE(mask_table, NFP_FLOWER_MASK_HASH_BITS);
210 u32 stats_ring_size;
211 struct rhashtable flow_table;
212 struct nfp_fl_stats *stats;
213 spinlock_t stats_lock;
214 struct rhashtable stats_ctx_table;
215 struct work_struct cmsg_work;
216 struct sk_buff_head cmsg_skbs_high;
217 struct sk_buff_head cmsg_skbs_low;
218 struct nfp_fl_tunnel_offloads tun;
219 atomic_t reify_replies;
220 wait_queue_head_t reify_wait_queue;
221 struct nfp_mtu_conf mtu_conf;
222 struct nfp_fl_lag nfp_lag;
223 struct list_head indr_block_cb_priv;
224 struct list_head non_repr_priv;
225 unsigned int active_mem_unit;
226 unsigned int total_mem_units;
227 struct nfp_fl_internal_ports internal_ports;
228 struct delayed_work qos_stats_work;
229 unsigned int qos_rate_limiters;
230 spinlock_t qos_stats_lock;
231 int pre_tun_rule_cnt;
232 struct rhashtable merge_table;
233 struct rhashtable ct_zone_table;
234 struct nfp_fl_ct_zone_entry *ct_zone_wc;
235 struct rhashtable ct_map_table;
236};
237
238
239
240
241
242
243
244
245struct nfp_fl_qos {
246 u32 netdev_port_id;
247 struct nfp_stat_pair curr_stats;
248 struct nfp_stat_pair prev_stats;
249 u64 last_update;
250};
251
252
253
254
255
256
257
258
259
260
261
262
263struct nfp_flower_repr_priv {
264 struct nfp_repr *nfp_repr;
265 unsigned long lag_port_flags;
266 bool mac_offloaded;
267 u8 offloaded_mac_addr[ETH_ALEN];
268 bool block_shared;
269 struct list_head mac_list;
270 struct nfp_fl_qos qos_table;
271 bool on_bridge;
272};
273
274
275
276
277
278
279
280
281
282struct nfp_flower_non_repr_priv {
283 struct list_head list;
284 struct net_device *netdev;
285 int ref_count;
286 bool mac_offloaded;
287 u8 offloaded_mac_addr[ETH_ALEN];
288};
289
290struct nfp_fl_key_ls {
291 u32 key_layer_two;
292 u8 key_layer;
293 int key_size;
294};
295
296struct nfp_fl_rule_metadata {
297 u8 key_len;
298 u8 mask_len;
299 u8 act_len;
300 u8 flags;
301 __be32 host_ctx_id;
302 __be64 host_cookie __packed;
303 __be64 flow_version __packed;
304 __be32 shortcut;
305};
306
307struct nfp_fl_stats {
308 u64 pkts;
309 u64 bytes;
310 u64 used;
311};
312
313
314
315
316
317
318
319struct nfp_ipv6_addr_entry {
320 struct in6_addr ipv6_addr;
321 int ref_count;
322 struct list_head list;
323};
324
325struct nfp_fl_payload {
326 struct nfp_fl_rule_metadata meta;
327 unsigned long tc_flower_cookie;
328 struct rhash_head fl_node;
329 struct rcu_head rcu;
330 __be32 nfp_tun_ipv4_addr;
331 struct nfp_ipv6_addr_entry *nfp_tun_ipv6;
332 struct net_device *ingress_dev;
333 char *unmasked_data;
334 char *mask_data;
335 char *action_data;
336 struct list_head linked_flows;
337 bool in_hw;
338 struct {
339 struct net_device *dev;
340 __be16 vlan_tci;
341 __be16 port_idx;
342 } pre_tun_rule;
343};
344
345struct nfp_fl_payload_link {
346
347
348
349
350
351
352
353
354
355
356 struct {
357 struct list_head list;
358 struct nfp_fl_payload *flow;
359 } merge_flow, sub_flow;
360};
361
362extern const struct rhashtable_params nfp_flower_table_params;
363extern const struct rhashtable_params merge_table_params;
364
365struct nfp_merge_info {
366 u64 parent_ctx;
367 struct rhash_head ht_node;
368};
369
370struct nfp_fl_stats_frame {
371 __be32 stats_con_id;
372 __be32 pkt_count;
373 __be64 byte_count;
374 __be64 stats_cookie;
375};
376
377static inline bool
378nfp_flower_internal_port_can_offload(struct nfp_app *app,
379 struct net_device *netdev)
380{
381 struct nfp_flower_priv *app_priv = app->priv;
382
383 if (!(app_priv->flower_en_feats & NFP_FL_ENABLE_FLOW_MERGE))
384 return false;
385 if (!netdev->rtnl_link_ops)
386 return false;
387 if (!strcmp(netdev->rtnl_link_ops->kind, "openvswitch"))
388 return true;
389
390 return false;
391}
392
393
394
395
396
397static inline bool nfp_flower_is_merge_flow(struct nfp_fl_payload *flow_pay)
398{
399 return flow_pay->tc_flower_cookie == (unsigned long)flow_pay;
400}
401
402static inline bool nfp_flower_is_supported_bridge(struct net_device *netdev)
403{
404 return netif_is_ovs_master(netdev);
405}
406
407int nfp_flower_metadata_init(struct nfp_app *app, u64 host_ctx_count,
408 unsigned int host_ctx_split);
409void nfp_flower_metadata_cleanup(struct nfp_app *app);
410
411int nfp_flower_setup_tc(struct nfp_app *app, struct net_device *netdev,
412 enum tc_setup_type type, void *type_data);
413int nfp_flower_merge_offloaded_flows(struct nfp_app *app,
414 struct nfp_fl_payload *sub_flow1,
415 struct nfp_fl_payload *sub_flow2);
416void
417nfp_flower_compile_meta(struct nfp_flower_meta_tci *ext,
418 struct nfp_flower_meta_tci *msk, u8 key_type);
419void
420nfp_flower_compile_tci(struct nfp_flower_meta_tci *ext,
421 struct nfp_flower_meta_tci *msk,
422 struct flow_rule *rule);
423void
424nfp_flower_compile_ext_meta(struct nfp_flower_ext_meta *frame, u32 key_ext);
425int
426nfp_flower_compile_port(struct nfp_flower_in_port *frame, u32 cmsg_port,
427 bool mask_version, enum nfp_flower_tun_type tun_type,
428 struct netlink_ext_ack *extack);
429void
430nfp_flower_compile_mac(struct nfp_flower_mac_mpls *ext,
431 struct nfp_flower_mac_mpls *msk,
432 struct flow_rule *rule);
433int
434nfp_flower_compile_mpls(struct nfp_flower_mac_mpls *ext,
435 struct nfp_flower_mac_mpls *msk,
436 struct flow_rule *rule,
437 struct netlink_ext_ack *extack);
438void
439nfp_flower_compile_tport(struct nfp_flower_tp_ports *ext,
440 struct nfp_flower_tp_ports *msk,
441 struct flow_rule *rule);
442void
443nfp_flower_compile_vlan(struct nfp_flower_vlan *ext,
444 struct nfp_flower_vlan *msk,
445 struct flow_rule *rule);
446void
447nfp_flower_compile_ipv4(struct nfp_flower_ipv4 *ext,
448 struct nfp_flower_ipv4 *msk, struct flow_rule *rule);
449void
450nfp_flower_compile_ipv6(struct nfp_flower_ipv6 *ext,
451 struct nfp_flower_ipv6 *msk, struct flow_rule *rule);
452void
453nfp_flower_compile_geneve_opt(u8 *ext, u8 *msk, struct flow_rule *rule);
454void
455nfp_flower_compile_ipv4_gre_tun(struct nfp_flower_ipv4_gre_tun *ext,
456 struct nfp_flower_ipv4_gre_tun *msk,
457 struct flow_rule *rule);
458void
459nfp_flower_compile_ipv4_udp_tun(struct nfp_flower_ipv4_udp_tun *ext,
460 struct nfp_flower_ipv4_udp_tun *msk,
461 struct flow_rule *rule);
462void
463nfp_flower_compile_ipv6_udp_tun(struct nfp_flower_ipv6_udp_tun *ext,
464 struct nfp_flower_ipv6_udp_tun *msk,
465 struct flow_rule *rule);
466void
467nfp_flower_compile_ipv6_gre_tun(struct nfp_flower_ipv6_gre_tun *ext,
468 struct nfp_flower_ipv6_gre_tun *msk,
469 struct flow_rule *rule);
470int nfp_flower_compile_flow_match(struct nfp_app *app,
471 struct flow_rule *rule,
472 struct nfp_fl_key_ls *key_ls,
473 struct net_device *netdev,
474 struct nfp_fl_payload *nfp_flow,
475 enum nfp_flower_tun_type tun_type,
476 struct netlink_ext_ack *extack);
477int nfp_flower_compile_action(struct nfp_app *app,
478 struct flow_rule *rule,
479 struct net_device *netdev,
480 struct nfp_fl_payload *nfp_flow,
481 struct netlink_ext_ack *extack);
482int nfp_compile_flow_metadata(struct nfp_app *app, u32 cookie,
483 struct nfp_fl_payload *nfp_flow,
484 struct net_device *netdev,
485 struct netlink_ext_ack *extack);
486void __nfp_modify_flow_metadata(struct nfp_flower_priv *priv,
487 struct nfp_fl_payload *nfp_flow);
488int nfp_modify_flow_metadata(struct nfp_app *app,
489 struct nfp_fl_payload *nfp_flow);
490
491struct nfp_fl_payload *
492nfp_flower_search_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie,
493 struct net_device *netdev);
494struct nfp_fl_payload *
495nfp_flower_get_fl_payload_from_ctx(struct nfp_app *app, u32 ctx_id);
496struct nfp_fl_payload *
497nfp_flower_remove_fl_table(struct nfp_app *app, unsigned long tc_flower_cookie);
498
499void nfp_flower_rx_flow_stats(struct nfp_app *app, struct sk_buff *skb);
500
501int nfp_tunnel_config_start(struct nfp_app *app);
502void nfp_tunnel_config_stop(struct nfp_app *app);
503int nfp_tunnel_mac_event_handler(struct nfp_app *app,
504 struct net_device *netdev,
505 unsigned long event, void *ptr);
506void nfp_tunnel_del_ipv4_off(struct nfp_app *app, __be32 ipv4);
507void nfp_tunnel_add_ipv4_off(struct nfp_app *app, __be32 ipv4);
508void
509nfp_tunnel_put_ipv6_off(struct nfp_app *app, struct nfp_ipv6_addr_entry *entry);
510struct nfp_ipv6_addr_entry *
511nfp_tunnel_add_ipv6_off(struct nfp_app *app, struct in6_addr *ipv6);
512void nfp_tunnel_request_route_v4(struct nfp_app *app, struct sk_buff *skb);
513void nfp_tunnel_request_route_v6(struct nfp_app *app, struct sk_buff *skb);
514void nfp_tunnel_keep_alive(struct nfp_app *app, struct sk_buff *skb);
515void nfp_tunnel_keep_alive_v6(struct nfp_app *app, struct sk_buff *skb);
516void nfp_flower_lag_init(struct nfp_fl_lag *lag);
517void nfp_flower_lag_cleanup(struct nfp_fl_lag *lag);
518int nfp_flower_lag_reset(struct nfp_fl_lag *lag);
519int nfp_flower_lag_netdev_event(struct nfp_flower_priv *priv,
520 struct net_device *netdev,
521 unsigned long event, void *ptr);
522bool nfp_flower_lag_unprocessed_msg(struct nfp_app *app, struct sk_buff *skb);
523int nfp_flower_lag_populate_pre_action(struct nfp_app *app,
524 struct net_device *master,
525 struct nfp_fl_pre_lag *pre_act,
526 struct netlink_ext_ack *extack);
527int nfp_flower_lag_get_output_id(struct nfp_app *app,
528 struct net_device *master);
529void nfp_flower_qos_init(struct nfp_app *app);
530void nfp_flower_qos_cleanup(struct nfp_app *app);
531int nfp_flower_setup_qos_offload(struct nfp_app *app, struct net_device *netdev,
532 struct tc_cls_matchall_offload *flow);
533void nfp_flower_stats_rlim_reply(struct nfp_app *app, struct sk_buff *skb);
534int nfp_flower_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch, void *cb_priv,
535 enum tc_setup_type type, void *type_data,
536 void *data,
537 void (*cleanup)(struct flow_block_cb *block_cb));
538void nfp_flower_setup_indr_tc_release(void *cb_priv);
539
540void
541__nfp_flower_non_repr_priv_get(struct nfp_flower_non_repr_priv *non_repr_priv);
542struct nfp_flower_non_repr_priv *
543nfp_flower_non_repr_priv_get(struct nfp_app *app, struct net_device *netdev);
544void
545__nfp_flower_non_repr_priv_put(struct nfp_flower_non_repr_priv *non_repr_priv);
546void
547nfp_flower_non_repr_priv_put(struct nfp_app *app, struct net_device *netdev);
548u32 nfp_flower_get_port_id_from_netdev(struct nfp_app *app,
549 struct net_device *netdev);
550int nfp_flower_xmit_pre_tun_flow(struct nfp_app *app,
551 struct nfp_fl_payload *flow);
552int nfp_flower_xmit_pre_tun_del_flow(struct nfp_app *app,
553 struct nfp_fl_payload *flow);
554
555struct nfp_fl_payload *
556nfp_flower_allocate_new(struct nfp_fl_key_ls *key_layer);
557int nfp_flower_calculate_key_layers(struct nfp_app *app,
558 struct net_device *netdev,
559 struct nfp_fl_key_ls *ret_key_ls,
560 struct flow_rule *flow,
561 enum nfp_flower_tun_type *tun_type,
562 struct netlink_ext_ack *extack);
563void
564nfp_flower_del_linked_merge_flows(struct nfp_app *app,
565 struct nfp_fl_payload *sub_flow);
566int
567nfp_flower_xmit_flow(struct nfp_app *app, struct nfp_fl_payload *nfp_flow,
568 u8 mtype);
569void
570nfp_flower_update_merge_stats(struct nfp_app *app,
571 struct nfp_fl_payload *sub_flow);
572#endif
573