1
2
3
4
5
6
7
8
9#define pr_fmt(fmt) "NFP net bpf: " fmt
10
11#include <linux/bpf.h>
12#include <linux/kernel.h>
13#include <linux/netdevice.h>
14#include <linux/pci.h>
15#include <linux/jiffies.h>
16#include <linux/timer.h>
17#include <linux/list.h>
18#include <linux/mm.h>
19
20#include <net/pkt_cls.h>
21#include <net/tc_act/tc_gact.h>
22#include <net/tc_act/tc_mirred.h>
23
24#include "main.h"
25#include "../nfp_app.h"
26#include "../nfp_net_ctrl.h"
27#include "../nfp_net.h"
28
29static int
30nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
31 struct bpf_map *map)
32{
33 struct nfp_bpf_neutral_map *record;
34 int err;
35
36
37 record = rhashtable_lookup_fast(&bpf->maps_neutral, &map->id,
38 nfp_bpf_maps_neutral_params);
39 if (record) {
40 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
41 record->count++;
42 return 0;
43 }
44
45
46
47
48 map = bpf_map_inc(map, false);
49 if (IS_ERR(map))
50 return PTR_ERR(map);
51
52 record = kmalloc(sizeof(*record), GFP_KERNEL);
53 if (!record) {
54 err = -ENOMEM;
55 goto err_map_put;
56 }
57
58 record->ptr = map;
59 record->map_id = map->id;
60 record->count = 1;
61
62 err = rhashtable_insert_fast(&bpf->maps_neutral, &record->l,
63 nfp_bpf_maps_neutral_params);
64 if (err)
65 goto err_free_rec;
66
67 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
68
69 return 0;
70
71err_free_rec:
72 kfree(record);
73err_map_put:
74 bpf_map_put(map);
75 return err;
76}
77
78static void
79nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog)
80{
81 bool freed = false;
82 int i;
83
84 for (i = 0; i < nfp_prog->map_records_cnt; i++) {
85 if (--nfp_prog->map_records[i]->count) {
86 nfp_prog->map_records[i] = NULL;
87 continue;
88 }
89
90 WARN_ON(rhashtable_remove_fast(&bpf->maps_neutral,
91 &nfp_prog->map_records[i]->l,
92 nfp_bpf_maps_neutral_params));
93 freed = true;
94 }
95
96 if (freed) {
97 synchronize_rcu();
98
99 for (i = 0; i < nfp_prog->map_records_cnt; i++)
100 if (nfp_prog->map_records[i]) {
101 bpf_map_put(nfp_prog->map_records[i]->ptr);
102 kfree(nfp_prog->map_records[i]);
103 }
104 }
105
106 kfree(nfp_prog->map_records);
107 nfp_prog->map_records = NULL;
108 nfp_prog->map_records_cnt = 0;
109}
110
111static int
112nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
113 struct bpf_prog *prog)
114{
115 int i, cnt, err;
116
117
118 cnt = 0;
119 for (i = 0; i < prog->aux->used_map_cnt; i++)
120 if (bpf_map_offload_neutral(prog->aux->used_maps[i]))
121 cnt++;
122 if (!cnt)
123 return 0;
124
125 nfp_prog->map_records = kmalloc_array(cnt,
126 sizeof(nfp_prog->map_records[0]),
127 GFP_KERNEL);
128 if (!nfp_prog->map_records)
129 return -ENOMEM;
130
131 for (i = 0; i < prog->aux->used_map_cnt; i++)
132 if (bpf_map_offload_neutral(prog->aux->used_maps[i])) {
133 err = nfp_map_ptr_record(bpf, nfp_prog,
134 prog->aux->used_maps[i]);
135 if (err) {
136 nfp_map_ptrs_forget(bpf, nfp_prog);
137 return err;
138 }
139 }
140 WARN_ON(cnt != nfp_prog->map_records_cnt);
141
142 return 0;
143}
144
145static int
146nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
147 unsigned int cnt)
148{
149 struct nfp_insn_meta *meta;
150 unsigned int i;
151
152 for (i = 0; i < cnt; i++) {
153 meta = kzalloc(sizeof(*meta), GFP_KERNEL);
154 if (!meta)
155 return -ENOMEM;
156
157 meta->insn = prog[i];
158 meta->n = i;
159 if (is_mbpf_alu(meta)) {
160 meta->umin_src = U64_MAX;
161 meta->umin_dst = U64_MAX;
162 }
163
164 list_add_tail(&meta->l, &nfp_prog->insns);
165 }
166
167 nfp_bpf_jit_prepare(nfp_prog, cnt);
168
169 return 0;
170}
171
172static void nfp_prog_free(struct nfp_prog *nfp_prog)
173{
174 struct nfp_insn_meta *meta, *tmp;
175
176 kfree(nfp_prog->subprog);
177
178 list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
179 list_del(&meta->l);
180 kfree(meta);
181 }
182 kfree(nfp_prog);
183}
184
185static int nfp_bpf_verifier_prep(struct bpf_prog *prog)
186{
187 struct nfp_net *nn = netdev_priv(prog->aux->offload->netdev);
188 struct nfp_app *app = nn->app;
189 struct nfp_prog *nfp_prog;
190 int ret;
191
192 nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
193 if (!nfp_prog)
194 return -ENOMEM;
195 prog->aux->offload->dev_priv = nfp_prog;
196
197 INIT_LIST_HEAD(&nfp_prog->insns);
198 nfp_prog->type = prog->type;
199 nfp_prog->bpf = app->priv;
200
201 ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len);
202 if (ret)
203 goto err_free;
204
205 nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog);
206
207 return 0;
208
209err_free:
210 nfp_prog_free(nfp_prog);
211
212 return ret;
213}
214
215static int nfp_bpf_translate(struct bpf_prog *prog)
216{
217 struct nfp_net *nn = netdev_priv(prog->aux->offload->netdev);
218 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
219 unsigned int max_instr;
220 int err;
221
222 max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
223 nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);
224
225 nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);
226 if (!nfp_prog->prog)
227 return -ENOMEM;
228
229 err = nfp_bpf_jit(nfp_prog);
230 if (err)
231 return err;
232
233 prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64);
234 prog->aux->offload->jited_image = nfp_prog->prog;
235
236 return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog);
237}
238
239static void nfp_bpf_destroy(struct bpf_prog *prog)
240{
241 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
242
243 kvfree(nfp_prog->prog);
244 nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog);
245 nfp_prog_free(nfp_prog);
246}
247
248
249
250
251static void nfp_map_bpf_byte_swap(struct nfp_bpf_map *nfp_map, void *value)
252{
253 u32 *word = value;
254 unsigned int i;
255
256 for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++)
257 if (nfp_map->use_map[i].type == NFP_MAP_USE_ATOMIC_CNT)
258 word[i] = (__force u32)cpu_to_be32(word[i]);
259}
260
261
262
263
264static void
265nfp_map_bpf_byte_swap_record(struct nfp_bpf_map *nfp_map, void *value)
266{
267 u32 *word = value;
268 unsigned int i;
269
270 for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++)
271 if (nfp_map->use_map[i].type == NFP_MAP_UNUSED &&
272 word[i] != (__force u32)cpu_to_be32(word[i]))
273 nfp_map->use_map[i].non_zero_update = 1;
274}
275
276static int
277nfp_bpf_map_lookup_entry(struct bpf_offloaded_map *offmap,
278 void *key, void *value)
279{
280 int err;
281
282 err = nfp_bpf_ctrl_lookup_entry(offmap, key, value);
283 if (err)
284 return err;
285
286 nfp_map_bpf_byte_swap(offmap->dev_priv, value);
287 return 0;
288}
289
290static int
291nfp_bpf_map_update_entry(struct bpf_offloaded_map *offmap,
292 void *key, void *value, u64 flags)
293{
294 nfp_map_bpf_byte_swap(offmap->dev_priv, value);
295 nfp_map_bpf_byte_swap_record(offmap->dev_priv, value);
296 return nfp_bpf_ctrl_update_entry(offmap, key, value, flags);
297}
298
299static int
300nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap,
301 void *key, void *next_key)
302{
303 if (!key)
304 return nfp_bpf_ctrl_getfirst_entry(offmap, next_key);
305 return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key);
306}
307
308static int
309nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
310{
311 if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
312 return -EINVAL;
313 return nfp_bpf_ctrl_del_entry(offmap, key);
314}
315
316static const struct bpf_map_dev_ops nfp_bpf_map_ops = {
317 .map_get_next_key = nfp_bpf_map_get_next_key,
318 .map_lookup_elem = nfp_bpf_map_lookup_entry,
319 .map_update_elem = nfp_bpf_map_update_entry,
320 .map_delete_elem = nfp_bpf_map_delete_elem,
321};
322
323static int
324nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
325{
326 struct nfp_bpf_map *nfp_map;
327 unsigned int use_map_size;
328 long long int res;
329
330 if (!bpf->maps.types)
331 return -EOPNOTSUPP;
332
333 if (offmap->map.map_flags ||
334 offmap->map.numa_node != NUMA_NO_NODE) {
335 pr_info("map flags are not supported\n");
336 return -EINVAL;
337 }
338
339 if (!(bpf->maps.types & 1 << offmap->map.map_type)) {
340 pr_info("map type not supported\n");
341 return -EOPNOTSUPP;
342 }
343 if (bpf->maps.max_maps == bpf->maps_in_use) {
344 pr_info("too many maps for a device\n");
345 return -ENOMEM;
346 }
347 if (bpf->maps.max_elems - bpf->map_elems_in_use <
348 offmap->map.max_entries) {
349 pr_info("map with too many elements: %u, left: %u\n",
350 offmap->map.max_entries,
351 bpf->maps.max_elems - bpf->map_elems_in_use);
352 return -ENOMEM;
353 }
354
355 if (round_up(offmap->map.key_size, 8) +
356 round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) {
357 pr_info("map elements too large: %u, FW max element size (key+value): %u\n",
358 round_up(offmap->map.key_size, 8) +
359 round_up(offmap->map.value_size, 8),
360 bpf->maps.max_elem_sz);
361 return -ENOMEM;
362 }
363 if (offmap->map.key_size > bpf->maps.max_key_sz) {
364 pr_info("map key size %u, FW max is %u\n",
365 offmap->map.key_size, bpf->maps.max_key_sz);
366 return -ENOMEM;
367 }
368 if (offmap->map.value_size > bpf->maps.max_val_sz) {
369 pr_info("map value size %u, FW max is %u\n",
370 offmap->map.value_size, bpf->maps.max_val_sz);
371 return -ENOMEM;
372 }
373
374 use_map_size = DIV_ROUND_UP(offmap->map.value_size, 4) *
375 FIELD_SIZEOF(struct nfp_bpf_map, use_map[0]);
376
377 nfp_map = kzalloc(sizeof(*nfp_map) + use_map_size, GFP_USER);
378 if (!nfp_map)
379 return -ENOMEM;
380
381 offmap->dev_priv = nfp_map;
382 nfp_map->offmap = offmap;
383 nfp_map->bpf = bpf;
384
385 res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map);
386 if (res < 0) {
387 kfree(nfp_map);
388 return res;
389 }
390
391 nfp_map->tid = res;
392 offmap->dev_ops = &nfp_bpf_map_ops;
393 bpf->maps_in_use++;
394 bpf->map_elems_in_use += offmap->map.max_entries;
395 list_add_tail(&nfp_map->l, &bpf->map_list);
396
397 return 0;
398}
399
400static int
401nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
402{
403 struct nfp_bpf_map *nfp_map = offmap->dev_priv;
404
405 nfp_bpf_ctrl_free_map(bpf, nfp_map);
406 list_del_init(&nfp_map->l);
407 bpf->map_elems_in_use -= offmap->map.max_entries;
408 bpf->maps_in_use--;
409 kfree(nfp_map);
410
411 return 0;
412}
413
414int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf)
415{
416 switch (bpf->command) {
417 case BPF_OFFLOAD_MAP_ALLOC:
418 return nfp_bpf_map_alloc(app->priv, bpf->offmap);
419 case BPF_OFFLOAD_MAP_FREE:
420 return nfp_bpf_map_free(app->priv, bpf->offmap);
421 default:
422 return -EINVAL;
423 }
424}
425
426static unsigned long
427nfp_bpf_perf_event_copy(void *dst, const void *src,
428 unsigned long off, unsigned long len)
429{
430 memcpy(dst, src + off, len);
431 return 0;
432}
433
434int nfp_bpf_event_output(struct nfp_app_bpf *bpf, const void *data,
435 unsigned int len)
436{
437 struct cmsg_bpf_event *cbe = (void *)data;
438 struct nfp_bpf_neutral_map *record;
439 u32 pkt_size, data_size, map_id;
440 u64 map_id_full;
441
442 if (len < sizeof(struct cmsg_bpf_event))
443 return -EINVAL;
444
445 pkt_size = be32_to_cpu(cbe->pkt_size);
446 data_size = be32_to_cpu(cbe->data_size);
447 map_id_full = be64_to_cpu(cbe->map_ptr);
448 map_id = map_id_full;
449
450 if (len < sizeof(struct cmsg_bpf_event) + pkt_size + data_size)
451 return -EINVAL;
452 if (cbe->hdr.ver != CMSG_MAP_ABI_VERSION)
453 return -EINVAL;
454
455 rcu_read_lock();
456 record = rhashtable_lookup_fast(&bpf->maps_neutral, &map_id,
457 nfp_bpf_maps_neutral_params);
458 if (!record || map_id_full > U32_MAX) {
459 rcu_read_unlock();
460 cmsg_warn(bpf, "perf event: map id %lld (0x%llx) not recognized, dropping event\n",
461 map_id_full, map_id_full);
462 return -EINVAL;
463 }
464
465 bpf_event_output(record->ptr, be32_to_cpu(cbe->cpu_id),
466 &cbe->data[round_up(pkt_size, 4)], data_size,
467 cbe->data, pkt_size, nfp_bpf_perf_event_copy);
468 rcu_read_unlock();
469
470 return 0;
471}
472
473static int
474nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog,
475 struct netlink_ext_ack *extack)
476{
477 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
478 unsigned int fw_mtu, pkt_off, max_stack, max_prog_len;
479 dma_addr_t dma_addr;
480 void *img;
481 int err;
482
483 fw_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
484 pkt_off = min(prog->aux->max_pkt_offset, nn->dp.netdev->mtu);
485 if (fw_mtu < pkt_off) {
486 NL_SET_ERR_MSG_MOD(extack, "BPF offload not supported with potential packet access beyond HW packet split boundary");
487 return -EOPNOTSUPP;
488 }
489
490 max_stack = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;
491 if (nfp_prog->stack_size > max_stack) {
492 NL_SET_ERR_MSG_MOD(extack, "stack too large");
493 return -EOPNOTSUPP;
494 }
495
496 max_prog_len = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
497 if (nfp_prog->prog_len > max_prog_len) {
498 NL_SET_ERR_MSG_MOD(extack, "program too long");
499 return -EOPNOTSUPP;
500 }
501
502 img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);
503 if (IS_ERR(img))
504 return PTR_ERR(img);
505
506 dma_addr = dma_map_single(nn->dp.dev, img,
507 nfp_prog->prog_len * sizeof(u64),
508 DMA_TO_DEVICE);
509 if (dma_mapping_error(nn->dp.dev, dma_addr)) {
510 kfree(img);
511 return -ENOMEM;
512 }
513
514 nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
515 nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
516
517
518 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
519 if (err)
520 NL_SET_ERR_MSG_MOD(extack,
521 "FW command error while loading BPF");
522
523 dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
524 DMA_TO_DEVICE);
525 kfree(img);
526
527 return err;
528}
529
530static void
531nfp_net_bpf_start(struct nfp_net *nn, struct netlink_ext_ack *extack)
532{
533 int err;
534
535
536 nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF;
537 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
538 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
539 if (err)
540 NL_SET_ERR_MSG_MOD(extack,
541 "FW command error while enabling BPF");
542}
543
544static int nfp_net_bpf_stop(struct nfp_net *nn)
545{
546 if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF))
547 return 0;
548
549 nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF;
550 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
551
552 return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
553}
554
555int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
556 bool old_prog, struct netlink_ext_ack *extack)
557{
558 int err;
559
560 if (prog && !bpf_offload_dev_match(prog, nn->dp.netdev))
561 return -EINVAL;
562
563 if (prog && old_prog) {
564 u8 cap;
565
566 cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP);
567 if (!(cap & NFP_NET_BPF_CAP_RELO)) {
568 NL_SET_ERR_MSG_MOD(extack,
569 "FW does not support live reload");
570 return -EBUSY;
571 }
572 }
573
574
575 if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
576 return -EBUSY;
577
578 if (old_prog && !prog)
579 return nfp_net_bpf_stop(nn);
580
581 err = nfp_net_bpf_load(nn, prog, extack);
582 if (err)
583 return err;
584
585 if (!old_prog)
586 nfp_net_bpf_start(nn, extack);
587
588 return 0;
589}
590
591const struct bpf_prog_offload_ops nfp_bpf_dev_ops = {
592 .insn_hook = nfp_verify_insn,
593 .finalize = nfp_bpf_finalize,
594 .prepare = nfp_bpf_verifier_prep,
595 .translate = nfp_bpf_translate,
596 .destroy = nfp_bpf_destroy,
597};
598
