1
2
3
4#include <net/pkt_cls.h>
5
6#include "../nfpcore/nfp_cpp.h"
7#include "../nfpcore/nfp_nffw.h"
8#include "../nfpcore/nfp_nsp.h"
9#include "../nfp_app.h"
10#include "../nfp_main.h"
11#include "../nfp_net.h"
12#include "../nfp_port.h"
13#include "fw.h"
14#include "main.h"
15
16const struct rhashtable_params nfp_bpf_maps_neutral_params = {
17 .nelem_hint = 4,
18 .key_len = FIELD_SIZEOF(struct bpf_map, id),
19 .key_offset = offsetof(struct nfp_bpf_neutral_map, map_id),
20 .head_offset = offsetof(struct nfp_bpf_neutral_map, l),
21 .automatic_shrinking = true,
22};
23
24static bool nfp_net_ebpf_capable(struct nfp_net *nn)
25{
26#ifdef __LITTLE_ENDIAN
27 struct nfp_app_bpf *bpf = nn->app->priv;
28
29 return nn->cap & NFP_NET_CFG_CTRL_BPF &&
30 bpf->abi_version &&
31 nn_readb(nn, NFP_NET_CFG_BPF_ABI) == bpf->abi_version;
32#else
33 return false;
34#endif
35}
36
37static int
38nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn,
39 struct bpf_prog *prog, struct netlink_ext_ack *extack)
40{
41 bool running, xdp_running;
42
43 if (!nfp_net_ebpf_capable(nn))
44 return -EINVAL;
45
46 running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF;
47 xdp_running = running && nn->xdp_hw.prog;
48
49 if (!prog && !xdp_running)
50 return 0;
51 if (prog && running && !xdp_running)
52 return -EBUSY;
53
54 return nfp_net_bpf_offload(nn, prog, running, extack);
55}
56
57static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn)
58{
59 return nfp_net_ebpf_capable(nn) ? "BPF" : "";
60}
61
62static int
63nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id)
64{
65 struct nfp_pf *pf = app->pf;
66 struct nfp_bpf_vnic *bv;
67 int err;
68
69 if (!pf->eth_tbl) {
70 nfp_err(pf->cpp, "No ETH table\n");
71 return -EINVAL;
72 }
73 if (pf->max_data_vnics != pf->eth_tbl->count) {
74 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n",
75 pf->max_data_vnics, pf->eth_tbl->count);
76 return -EINVAL;
77 }
78
79 bv = kzalloc(sizeof(*bv), GFP_KERNEL);
80 if (!bv)
81 return -ENOMEM;
82 nn->app_priv = bv;
83
84 err = nfp_app_nic_vnic_alloc(app, nn, id);
85 if (err)
86 goto err_free_priv;
87
88 bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
89 bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
90
91 return 0;
92err_free_priv:
93 kfree(nn->app_priv);
94 return err;
95}
96
97static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn)
98{
99 struct nfp_bpf_vnic *bv = nn->app_priv;
100
101 WARN_ON(bv->tc_prog);
102 kfree(bv);
103}
104
105static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type,
106 void *type_data, void *cb_priv)
107{
108 struct tc_cls_bpf_offload *cls_bpf = type_data;
109 struct nfp_net *nn = cb_priv;
110 struct bpf_prog *oldprog;
111 struct nfp_bpf_vnic *bv;
112 int err;
113
114 if (type != TC_SETUP_CLSBPF) {
115 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
116 "only offload of BPF classifiers supported");
117 return -EOPNOTSUPP;
118 }
119 if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common))
120 return -EOPNOTSUPP;
121 if (!nfp_net_ebpf_capable(nn)) {
122 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
123 "NFP firmware does not support eBPF offload");
124 return -EOPNOTSUPP;
125 }
126 if (cls_bpf->common.protocol != htons(ETH_P_ALL)) {
127 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
128 "only ETH_P_ALL supported as filter protocol");
129 return -EOPNOTSUPP;
130 }
131
132
133 if (!cls_bpf->exts_integrated ||
134 tcf_exts_has_actions(cls_bpf->exts)) {
135 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack,
136 "only direct action with no legacy actions supported");
137 return -EOPNOTSUPP;
138 }
139
140 if (cls_bpf->command != TC_CLSBPF_OFFLOAD)
141 return -EOPNOTSUPP;
142
143 bv = nn->app_priv;
144 oldprog = cls_bpf->oldprog;
145
146
147 if (bv->tc_prog != oldprog) {
148 oldprog = NULL;
149 if (!cls_bpf->prog)
150 return 0;
151 }
152
153 err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog,
154 cls_bpf->common.extack);
155 if (err)
156 return err;
157
158 bv->tc_prog = cls_bpf->prog;
159 nn->port->tc_offload_cnt = !!bv->tc_prog;
160 return 0;
161}
162
163static int nfp_bpf_setup_tc_block(struct net_device *netdev,
164 struct tc_block_offload *f)
165{
166 struct nfp_net *nn = netdev_priv(netdev);
167
168 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
169 return -EOPNOTSUPP;
170
171 switch (f->command) {
172 case TC_BLOCK_BIND:
173 return tcf_block_cb_register(f->block,
174 nfp_bpf_setup_tc_block_cb,
175 nn, nn, f->extack);
176 case TC_BLOCK_UNBIND:
177 tcf_block_cb_unregister(f->block,
178 nfp_bpf_setup_tc_block_cb,
179 nn);
180 return 0;
181 default:
182 return -EOPNOTSUPP;
183 }
184}
185
186static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev,
187 enum tc_setup_type type, void *type_data)
188{
189 switch (type) {
190 case TC_SETUP_BLOCK:
191 return nfp_bpf_setup_tc_block(netdev, type_data);
192 default:
193 return -EOPNOTSUPP;
194 }
195}
196
197static int
198nfp_bpf_check_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu)
199{
200 struct nfp_net *nn = netdev_priv(netdev);
201 unsigned int max_mtu;
202
203 if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
204 return 0;
205
206 max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
207 if (new_mtu > max_mtu) {
208 nn_info(nn, "BPF offload active, MTU over %u not supported\n",
209 max_mtu);
210 return -EBUSY;
211 }
212 return 0;
213}
214
215static int
216nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value,
217 u32 length)
218{
219 struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value;
220 struct nfp_cpp *cpp = bpf->app->pf->cpp;
221
222 if (length < sizeof(*cap)) {
223 nfp_err(cpp, "truncated adjust_head TLV: %d\n", length);
224 return -EINVAL;
225 }
226
227 bpf->adjust_head.flags = readl(&cap->flags);
228 bpf->adjust_head.off_min = readl(&cap->off_min);
229 bpf->adjust_head.off_max = readl(&cap->off_max);
230 bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub);
231 bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add);
232
233 if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) {
234 nfp_err(cpp, "invalid adjust_head TLV: min > max\n");
235 return -EINVAL;
236 }
237 if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) ||
238 !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) {
239 nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n");
240 memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head));
241 return 0;
242 }
243
244 return 0;
245}
246
247static int
248nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
249{
250 struct nfp_bpf_cap_tlv_func __iomem *cap = value;
251
252 if (length < sizeof(*cap)) {
253 nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length);
254 return -EINVAL;
255 }
256
257 switch (readl(&cap->func_id)) {
258 case BPF_FUNC_map_lookup_elem:
259 bpf->helpers.map_lookup = readl(&cap->func_addr);
260 break;
261 case BPF_FUNC_map_update_elem:
262 bpf->helpers.map_update = readl(&cap->func_addr);
263 break;
264 case BPF_FUNC_map_delete_elem:
265 bpf->helpers.map_delete = readl(&cap->func_addr);
266 break;
267 case BPF_FUNC_perf_event_output:
268 bpf->helpers.perf_event_output = readl(&cap->func_addr);
269 break;
270 }
271
272 return 0;
273}
274
275static int
276nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
277{
278 struct nfp_bpf_cap_tlv_maps __iomem *cap = value;
279
280 if (length < sizeof(*cap)) {
281 nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length);
282 return -EINVAL;
283 }
284
285 bpf->maps.types = readl(&cap->types);
286 bpf->maps.max_maps = readl(&cap->max_maps);
287 bpf->maps.max_elems = readl(&cap->max_elems);
288 bpf->maps.max_key_sz = readl(&cap->max_key_sz);
289 bpf->maps.max_val_sz = readl(&cap->max_val_sz);
290 bpf->maps.max_elem_sz = readl(&cap->max_elem_sz);
291
292 return 0;
293}
294
295static int
296nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value,
297 u32 length)
298{
299 bpf->pseudo_random = true;
300 return 0;
301}
302
303static int
304nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length)
305{
306 bpf->queue_select = true;
307 return 0;
308}
309
310static int
311nfp_bpf_parse_cap_adjust_tail(struct nfp_app_bpf *bpf, void __iomem *value,
312 u32 length)
313{
314 bpf->adjust_tail = true;
315 return 0;
316}
317
318static int
319nfp_bpf_parse_cap_abi_version(struct nfp_app_bpf *bpf, void __iomem *value,
320 u32 length)
321{
322 if (length < 4) {
323 nfp_err(bpf->app->cpp, "truncated ABI version TLV: %d\n",
324 length);
325 return -EINVAL;
326 }
327
328 bpf->abi_version = readl(value);
329 if (bpf->abi_version < 2 || bpf->abi_version > 3) {
330 nfp_warn(bpf->app->cpp, "unsupported BPF ABI version: %d\n",
331 bpf->abi_version);
332 bpf->abi_version = 0;
333 }
334
335 return 0;
336}
337
338static int nfp_bpf_parse_capabilities(struct nfp_app *app)
339{
340 struct nfp_cpp *cpp = app->pf->cpp;
341 struct nfp_cpp_area *area;
342 u8 __iomem *mem, *start;
343
344 mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap",
345 8, &area);
346 if (IS_ERR(mem))
347 return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem);
348
349 start = mem;
350 while (mem - start + 8 <= nfp_cpp_area_size(area)) {
351 u8 __iomem *value;
352 u32 type, length;
353
354 type = readl(mem);
355 length = readl(mem + 4);
356 value = mem + 8;
357
358 mem += 8 + length;
359 if (mem - start > nfp_cpp_area_size(area))
360 goto err_release_free;
361
362 switch (type) {
363 case NFP_BPF_CAP_TYPE_FUNC:
364 if (nfp_bpf_parse_cap_func(app->priv, value, length))
365 goto err_release_free;
366 break;
367 case NFP_BPF_CAP_TYPE_ADJUST_HEAD:
368 if (nfp_bpf_parse_cap_adjust_head(app->priv, value,
369 length))
370 goto err_release_free;
371 break;
372 case NFP_BPF_CAP_TYPE_MAPS:
373 if (nfp_bpf_parse_cap_maps(app->priv, value, length))
374 goto err_release_free;
375 break;
376 case NFP_BPF_CAP_TYPE_RANDOM:
377 if (nfp_bpf_parse_cap_random(app->priv, value, length))
378 goto err_release_free;
379 break;
380 case NFP_BPF_CAP_TYPE_QUEUE_SELECT:
381 if (nfp_bpf_parse_cap_qsel(app->priv, value, length))
382 goto err_release_free;
383 break;
384 case NFP_BPF_CAP_TYPE_ADJUST_TAIL:
385 if (nfp_bpf_parse_cap_adjust_tail(app->priv, value,
386 length))
387 goto err_release_free;
388 break;
389 case NFP_BPF_CAP_TYPE_ABI_VERSION:
390 if (nfp_bpf_parse_cap_abi_version(app->priv, value,
391 length))
392 goto err_release_free;
393 break;
394 default:
395 nfp_dbg(cpp, "unknown BPF capability: %d\n", type);
396 break;
397 }
398 }
399 if (mem - start != nfp_cpp_area_size(area)) {
400 nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n",
401 mem - start, nfp_cpp_area_size(area));
402 goto err_release_free;
403 }
404
405 nfp_cpp_area_release_free(area);
406
407 return 0;
408
409err_release_free:
410 nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start);
411 nfp_cpp_area_release_free(area);
412 return -EINVAL;
413}
414
415static void nfp_bpf_init_capabilities(struct nfp_app_bpf *bpf)
416{
417 bpf->abi_version = 2;
418}
419
420static int nfp_bpf_ndo_init(struct nfp_app *app, struct net_device *netdev)
421{
422 struct nfp_app_bpf *bpf = app->priv;
423
424 return bpf_offload_dev_netdev_register(bpf->bpf_dev, netdev);
425}
426
427static void nfp_bpf_ndo_uninit(struct nfp_app *app, struct net_device *netdev)
428{
429 struct nfp_app_bpf *bpf = app->priv;
430
431 bpf_offload_dev_netdev_unregister(bpf->bpf_dev, netdev);
432}
433
434static int nfp_bpf_init(struct nfp_app *app)
435{
436 struct nfp_app_bpf *bpf;
437 int err;
438
439 bpf = kzalloc(sizeof(*bpf), GFP_KERNEL);
440 if (!bpf)
441 return -ENOMEM;
442 bpf->app = app;
443 app->priv = bpf;
444
445 skb_queue_head_init(&bpf->cmsg_replies);
446 init_waitqueue_head(&bpf->cmsg_wq);
447 INIT_LIST_HEAD(&bpf->map_list);
448
449 err = rhashtable_init(&bpf->maps_neutral, &nfp_bpf_maps_neutral_params);
450 if (err)
451 goto err_free_bpf;
452
453 nfp_bpf_init_capabilities(bpf);
454
455 err = nfp_bpf_parse_capabilities(app);
456 if (err)
457 goto err_free_neutral_maps;
458
459 if (bpf->abi_version < 3) {
460 bpf->cmsg_key_sz = CMSG_MAP_KEY_LW * 4;
461 bpf->cmsg_val_sz = CMSG_MAP_VALUE_LW * 4;
462 } else {
463 bpf->cmsg_key_sz = bpf->maps.max_key_sz;
464 bpf->cmsg_val_sz = bpf->maps.max_val_sz;
465 app->ctrl_mtu = nfp_bpf_ctrl_cmsg_mtu(bpf);
466 }
467
468 bpf->bpf_dev = bpf_offload_dev_create();
469 err = PTR_ERR_OR_ZERO(bpf->bpf_dev);
470 if (err)
471 goto err_free_neutral_maps;
472
473 return 0;
474
475err_free_neutral_maps:
476 rhashtable_destroy(&bpf->maps_neutral);
477err_free_bpf:
478 kfree(bpf);
479 return err;
480}
481
482static void nfp_bpf_clean(struct nfp_app *app)
483{
484 struct nfp_app_bpf *bpf = app->priv;
485
486 bpf_offload_dev_destroy(bpf->bpf_dev);
487 WARN_ON(!skb_queue_empty(&bpf->cmsg_replies));
488 WARN_ON(!list_empty(&bpf->map_list));
489 WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use);
490 rhashtable_free_and_destroy(&bpf->maps_neutral,
491 nfp_check_rhashtable_empty, NULL);
492 kfree(bpf);
493}
494
495const struct nfp_app_type app_bpf = {
496 .id = NFP_APP_BPF_NIC,
497 .name = "ebpf",
498
499 .ctrl_cap_mask = 0,
500
501 .init = nfp_bpf_init,
502 .clean = nfp_bpf_clean,
503
504 .check_mtu = nfp_bpf_check_mtu,
505
506 .extra_cap = nfp_bpf_extra_cap,
507
508 .ndo_init = nfp_bpf_ndo_init,
509 .ndo_uninit = nfp_bpf_ndo_uninit,
510
511 .vnic_alloc = nfp_bpf_vnic_alloc,
512 .vnic_free = nfp_bpf_vnic_free,
513
514 .ctrl_msg_rx = nfp_bpf_ctrl_msg_rx,
515 .ctrl_msg_rx_raw = nfp_bpf_ctrl_msg_rx_raw,
516
517 .setup_tc = nfp_bpf_setup_tc,
518 .bpf = nfp_ndo_bpf,
519 .xdp_offload = nfp_bpf_xdp_offload,
520};
521
