1
2
3
4
5
6
7
8
9#define pr_fmt(fmt) "NFP net bpf: " fmt
10
11#include <linux/bpf.h>
12#include <linux/kernel.h>
13#include <linux/netdevice.h>
14#include <linux/pci.h>
15#include <linux/jiffies.h>
16#include <linux/timer.h>
17#include <linux/list.h>
18#include <linux/mm.h>
19
20#include <net/pkt_cls.h>
21#include <net/tc_act/tc_gact.h>
22#include <net/tc_act/tc_mirred.h>
23
24#include "main.h"
25#include "../ccm.h"
26#include "../nfp_app.h"
27#include "../nfp_net_ctrl.h"
28#include "../nfp_net.h"
29
30static int
31nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
32 struct bpf_map *map)
33{
34 struct nfp_bpf_neutral_map *record;
35 int err;
36
37
38 record = rhashtable_lookup_fast(&bpf->maps_neutral, &map->id,
39 nfp_bpf_maps_neutral_params);
40 if (record) {
41 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
42 record->count++;
43 return 0;
44 }
45
46
47
48
49 bpf_map_inc(map);
50
51 record = kmalloc(sizeof(*record), GFP_KERNEL);
52 if (!record) {
53 err = -ENOMEM;
54 goto err_map_put;
55 }
56
57 record->ptr = map;
58 record->map_id = map->id;
59 record->count = 1;
60
61 err = rhashtable_insert_fast(&bpf->maps_neutral, &record->l,
62 nfp_bpf_maps_neutral_params);
63 if (err)
64 goto err_free_rec;
65
66 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record;
67
68 return 0;
69
70err_free_rec:
71 kfree(record);
72err_map_put:
73 bpf_map_put(map);
74 return err;
75}
76
77static void
78nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog)
79{
80 bool freed = false;
81 int i;
82
83 for (i = 0; i < nfp_prog->map_records_cnt; i++) {
84 if (--nfp_prog->map_records[i]->count) {
85 nfp_prog->map_records[i] = NULL;
86 continue;
87 }
88
89 WARN_ON(rhashtable_remove_fast(&bpf->maps_neutral,
90 &nfp_prog->map_records[i]->l,
91 nfp_bpf_maps_neutral_params));
92 freed = true;
93 }
94
95 if (freed) {
96 synchronize_rcu();
97
98 for (i = 0; i < nfp_prog->map_records_cnt; i++)
99 if (nfp_prog->map_records[i]) {
100 bpf_map_put(nfp_prog->map_records[i]->ptr);
101 kfree(nfp_prog->map_records[i]);
102 }
103 }
104
105 kfree(nfp_prog->map_records);
106 nfp_prog->map_records = NULL;
107 nfp_prog->map_records_cnt = 0;
108}
109
110static int
111nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog,
112 struct bpf_prog *prog)
113{
114 int i, cnt, err = 0;
115
116 mutex_lock(&prog->aux->used_maps_mutex);
117
118
119 cnt = 0;
120 for (i = 0; i < prog->aux->used_map_cnt; i++)
121 if (bpf_map_offload_neutral(prog->aux->used_maps[i]))
122 cnt++;
123 if (!cnt)
124 goto out;
125
126 nfp_prog->map_records = kmalloc_array(cnt,
127 sizeof(nfp_prog->map_records[0]),
128 GFP_KERNEL);
129 if (!nfp_prog->map_records) {
130 err = -ENOMEM;
131 goto out;
132 }
133
134 for (i = 0; i < prog->aux->used_map_cnt; i++)
135 if (bpf_map_offload_neutral(prog->aux->used_maps[i])) {
136 err = nfp_map_ptr_record(bpf, nfp_prog,
137 prog->aux->used_maps[i]);
138 if (err) {
139 nfp_map_ptrs_forget(bpf, nfp_prog);
140 goto out;
141 }
142 }
143 WARN_ON(cnt != nfp_prog->map_records_cnt);
144
145out:
146 mutex_unlock(&prog->aux->used_maps_mutex);
147 return err;
148}
149
150static int
151nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog,
152 unsigned int cnt)
153{
154 struct nfp_insn_meta *meta;
155 unsigned int i;
156
157 for (i = 0; i < cnt; i++) {
158 meta = kzalloc(sizeof(*meta), GFP_KERNEL);
159 if (!meta)
160 return -ENOMEM;
161
162 meta->insn = prog[i];
163 meta->n = i;
164 if (is_mbpf_alu(meta)) {
165 meta->umin_src = U64_MAX;
166 meta->umin_dst = U64_MAX;
167 }
168
169 list_add_tail(&meta->l, &nfp_prog->insns);
170 }
171 nfp_prog->n_insns = cnt;
172
173 nfp_bpf_jit_prepare(nfp_prog);
174
175 return 0;
176}
177
178static void nfp_prog_free(struct nfp_prog *nfp_prog)
179{
180 struct nfp_insn_meta *meta, *tmp;
181
182 kfree(nfp_prog->subprog);
183
184 list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) {
185 list_del(&meta->l);
186 kfree(meta);
187 }
188 kfree(nfp_prog);
189}
190
191static int nfp_bpf_verifier_prep(struct bpf_prog *prog)
192{
193 struct nfp_prog *nfp_prog;
194 int ret;
195
196 nfp_prog = kzalloc(sizeof(*nfp_prog), GFP_KERNEL);
197 if (!nfp_prog)
198 return -ENOMEM;
199 prog->aux->offload->dev_priv = nfp_prog;
200
201 INIT_LIST_HEAD(&nfp_prog->insns);
202 nfp_prog->type = prog->type;
203 nfp_prog->bpf = bpf_offload_dev_priv(prog->aux->offload->offdev);
204
205 ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len);
206 if (ret)
207 goto err_free;
208
209 nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog);
210
211 return 0;
212
213err_free:
214 nfp_prog_free(nfp_prog);
215
216 return ret;
217}
218
219static int nfp_bpf_translate(struct bpf_prog *prog)
220{
221 struct nfp_net *nn = netdev_priv(prog->aux->offload->netdev);
222 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
223 unsigned int max_instr;
224 int err;
225
226
227 if (prog->aux->offload->opt_failed)
228 return -EINVAL;
229
230 max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
231 nfp_prog->__prog_alloc_len = max_instr * sizeof(u64);
232
233 nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL);
234 if (!nfp_prog->prog)
235 return -ENOMEM;
236
237 err = nfp_bpf_jit(nfp_prog);
238 if (err)
239 return err;
240
241 prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64);
242 prog->aux->offload->jited_image = nfp_prog->prog;
243
244 return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog);
245}
246
247static void nfp_bpf_destroy(struct bpf_prog *prog)
248{
249 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
250
251 kvfree(nfp_prog->prog);
252 nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog);
253 nfp_prog_free(nfp_prog);
254}
255
256
257
258
259static void nfp_map_bpf_byte_swap(struct nfp_bpf_map *nfp_map, void *value)
260{
261 u32 *word = value;
262 unsigned int i;
263
264 for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++)
265 if (nfp_map->use_map[i].type == NFP_MAP_USE_ATOMIC_CNT)
266 word[i] = (__force u32)cpu_to_be32(word[i]);
267}
268
269
270
271
272static void
273nfp_map_bpf_byte_swap_record(struct nfp_bpf_map *nfp_map, void *value)
274{
275 u32 *word = value;
276 unsigned int i;
277
278 for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++)
279 if (nfp_map->use_map[i].type == NFP_MAP_UNUSED &&
280 word[i] != (__force u32)cpu_to_be32(word[i]))
281 nfp_map->use_map[i].non_zero_update = 1;
282}
283
284static int
285nfp_bpf_map_lookup_entry(struct bpf_offloaded_map *offmap,
286 void *key, void *value)
287{
288 int err;
289
290 err = nfp_bpf_ctrl_lookup_entry(offmap, key, value);
291 if (err)
292 return err;
293
294 nfp_map_bpf_byte_swap(offmap->dev_priv, value);
295 return 0;
296}
297
298static int
299nfp_bpf_map_update_entry(struct bpf_offloaded_map *offmap,
300 void *key, void *value, u64 flags)
301{
302 nfp_map_bpf_byte_swap(offmap->dev_priv, value);
303 nfp_map_bpf_byte_swap_record(offmap->dev_priv, value);
304 return nfp_bpf_ctrl_update_entry(offmap, key, value, flags);
305}
306
307static int
308nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap,
309 void *key, void *next_key)
310{
311 if (!key)
312 return nfp_bpf_ctrl_getfirst_entry(offmap, next_key);
313 return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key);
314}
315
316static int
317nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key)
318{
319 if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY)
320 return -EINVAL;
321 return nfp_bpf_ctrl_del_entry(offmap, key);
322}
323
324static const struct bpf_map_dev_ops nfp_bpf_map_ops = {
325 .map_get_next_key = nfp_bpf_map_get_next_key,
326 .map_lookup_elem = nfp_bpf_map_lookup_entry,
327 .map_update_elem = nfp_bpf_map_update_entry,
328 .map_delete_elem = nfp_bpf_map_delete_elem,
329};
330
331static int
332nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
333{
334 struct nfp_bpf_map *nfp_map;
335 unsigned int use_map_size;
336 long long int res;
337
338 if (!bpf->maps.types)
339 return -EOPNOTSUPP;
340
341 if (offmap->map.map_flags ||
342 offmap->map.numa_node != NUMA_NO_NODE) {
343 pr_info("map flags are not supported\n");
344 return -EINVAL;
345 }
346
347 if (!(bpf->maps.types & 1 << offmap->map.map_type)) {
348 pr_info("map type not supported\n");
349 return -EOPNOTSUPP;
350 }
351 if (bpf->maps.max_maps == bpf->maps_in_use) {
352 pr_info("too many maps for a device\n");
353 return -ENOMEM;
354 }
355 if (bpf->maps.max_elems - bpf->map_elems_in_use <
356 offmap->map.max_entries) {
357 pr_info("map with too many elements: %u, left: %u\n",
358 offmap->map.max_entries,
359 bpf->maps.max_elems - bpf->map_elems_in_use);
360 return -ENOMEM;
361 }
362
363 if (round_up(offmap->map.key_size, 8) +
364 round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) {
365 pr_info("map elements too large: %u, FW max element size (key+value): %u\n",
366 round_up(offmap->map.key_size, 8) +
367 round_up(offmap->map.value_size, 8),
368 bpf->maps.max_elem_sz);
369 return -ENOMEM;
370 }
371 if (offmap->map.key_size > bpf->maps.max_key_sz) {
372 pr_info("map key size %u, FW max is %u\n",
373 offmap->map.key_size, bpf->maps.max_key_sz);
374 return -ENOMEM;
375 }
376 if (offmap->map.value_size > bpf->maps.max_val_sz) {
377 pr_info("map value size %u, FW max is %u\n",
378 offmap->map.value_size, bpf->maps.max_val_sz);
379 return -ENOMEM;
380 }
381
382 use_map_size = DIV_ROUND_UP(offmap->map.value_size, 4) *
383 sizeof_field(struct nfp_bpf_map, use_map[0]);
384
385 nfp_map = kzalloc(sizeof(*nfp_map) + use_map_size, GFP_USER);
386 if (!nfp_map)
387 return -ENOMEM;
388
389 offmap->dev_priv = nfp_map;
390 nfp_map->offmap = offmap;
391 nfp_map->bpf = bpf;
392 spin_lock_init(&nfp_map->cache_lock);
393
394 res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map);
395 if (res < 0) {
396 kfree(nfp_map);
397 return res;
398 }
399
400 nfp_map->tid = res;
401 offmap->dev_ops = &nfp_bpf_map_ops;
402 bpf->maps_in_use++;
403 bpf->map_elems_in_use += offmap->map.max_entries;
404 list_add_tail(&nfp_map->l, &bpf->map_list);
405
406 return 0;
407}
408
409static int
410nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap)
411{
412 struct nfp_bpf_map *nfp_map = offmap->dev_priv;
413
414 nfp_bpf_ctrl_free_map(bpf, nfp_map);
415 dev_consume_skb_any(nfp_map->cache);
416 WARN_ON_ONCE(nfp_map->cache_blockers);
417 list_del_init(&nfp_map->l);
418 bpf->map_elems_in_use -= offmap->map.max_entries;
419 bpf->maps_in_use--;
420 kfree(nfp_map);
421
422 return 0;
423}
424
425int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf)
426{
427 switch (bpf->command) {
428 case BPF_OFFLOAD_MAP_ALLOC:
429 return nfp_bpf_map_alloc(app->priv, bpf->offmap);
430 case BPF_OFFLOAD_MAP_FREE:
431 return nfp_bpf_map_free(app->priv, bpf->offmap);
432 default:
433 return -EINVAL;
434 }
435}
436
437static unsigned long
438nfp_bpf_perf_event_copy(void *dst, const void *src,
439 unsigned long off, unsigned long len)
440{
441 memcpy(dst, src + off, len);
442 return 0;
443}
444
445int nfp_bpf_event_output(struct nfp_app_bpf *bpf, const void *data,
446 unsigned int len)
447{
448 struct cmsg_bpf_event *cbe = (void *)data;
449 struct nfp_bpf_neutral_map *record;
450 u32 pkt_size, data_size, map_id;
451 u64 map_id_full;
452
453 if (len < sizeof(struct cmsg_bpf_event))
454 return -EINVAL;
455
456 pkt_size = be32_to_cpu(cbe->pkt_size);
457 data_size = be32_to_cpu(cbe->data_size);
458 map_id_full = be64_to_cpu(cbe->map_ptr);
459 map_id = map_id_full;
460
461 if (len < sizeof(struct cmsg_bpf_event) + pkt_size + data_size)
462 return -EINVAL;
463 if (cbe->hdr.ver != NFP_CCM_ABI_VERSION)
464 return -EINVAL;
465
466 rcu_read_lock();
467 record = rhashtable_lookup(&bpf->maps_neutral, &map_id,
468 nfp_bpf_maps_neutral_params);
469 if (!record || map_id_full > U32_MAX) {
470 rcu_read_unlock();
471 cmsg_warn(bpf, "perf event: map id %lld (0x%llx) not recognized, dropping event\n",
472 map_id_full, map_id_full);
473 return -EINVAL;
474 }
475
476 bpf_event_output(record->ptr, be32_to_cpu(cbe->cpu_id),
477 &cbe->data[round_up(pkt_size, 4)], data_size,
478 cbe->data, pkt_size, nfp_bpf_perf_event_copy);
479 rcu_read_unlock();
480
481 return 0;
482}
483
484bool nfp_bpf_offload_check_mtu(struct nfp_net *nn, struct bpf_prog *prog,
485 unsigned int mtu)
486{
487 unsigned int fw_mtu, pkt_off;
488
489 fw_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
490 pkt_off = min(prog->aux->max_pkt_offset, mtu);
491
492 return fw_mtu < pkt_off;
493}
494
495static int
496nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog,
497 struct netlink_ext_ack *extack)
498{
499 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv;
500 unsigned int max_stack, max_prog_len;
501 dma_addr_t dma_addr;
502 void *img;
503 int err;
504
505 if (nfp_bpf_offload_check_mtu(nn, prog, nn->dp.netdev->mtu)) {
506 NL_SET_ERR_MSG_MOD(extack, "BPF offload not supported with potential packet access beyond HW packet split boundary");
507 return -EOPNOTSUPP;
508 }
509
510 max_stack = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64;
511 if (nfp_prog->stack_size > max_stack) {
512 NL_SET_ERR_MSG_MOD(extack, "stack too large");
513 return -EOPNOTSUPP;
514 }
515
516 max_prog_len = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
517 if (nfp_prog->prog_len > max_prog_len) {
518 NL_SET_ERR_MSG_MOD(extack, "program too long");
519 return -EOPNOTSUPP;
520 }
521
522 img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv);
523 if (IS_ERR(img))
524 return PTR_ERR(img);
525
526 dma_addr = dma_map_single(nn->dp.dev, img,
527 nfp_prog->prog_len * sizeof(u64),
528 DMA_TO_DEVICE);
529 if (dma_mapping_error(nn->dp.dev, dma_addr)) {
530 kfree(img);
531 return -ENOMEM;
532 }
533
534 nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len);
535 nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr);
536
537
538 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
539 if (err)
540 NL_SET_ERR_MSG_MOD(extack,
541 "FW command error while loading BPF");
542
543 dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64),
544 DMA_TO_DEVICE);
545 kfree(img);
546
547 return err;
548}
549
550static void
551nfp_net_bpf_start(struct nfp_net *nn, struct netlink_ext_ack *extack)
552{
553 int err;
554
555
556 nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF;
557 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
558 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
559 if (err)
560 NL_SET_ERR_MSG_MOD(extack,
561 "FW command error while enabling BPF");
562}
563
564static int nfp_net_bpf_stop(struct nfp_net *nn)
565{
566 if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF))
567 return 0;
568
569 nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF;
570 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
571
572 return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
573}
574
575int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog,
576 bool old_prog, struct netlink_ext_ack *extack)
577{
578 int err;
579
580 if (prog && !bpf_offload_dev_match(prog, nn->dp.netdev))
581 return -EINVAL;
582
583 if (prog && old_prog) {
584 u8 cap;
585
586 cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP);
587 if (!(cap & NFP_NET_BPF_CAP_RELO)) {
588 NL_SET_ERR_MSG_MOD(extack,
589 "FW does not support live reload");
590 return -EBUSY;
591 }
592 }
593
594
595 if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
596 return -EBUSY;
597
598 if (old_prog && !prog)
599 return nfp_net_bpf_stop(nn);
600
601 err = nfp_net_bpf_load(nn, prog, extack);
602 if (err)
603 return err;
604
605 if (!old_prog)
606 nfp_net_bpf_start(nn, extack);
607
608 return 0;
609}
610
611const struct bpf_prog_offload_ops nfp_bpf_dev_ops = {
612 .insn_hook = nfp_verify_insn,
613 .finalize = nfp_bpf_finalize,
614 .replace_insn = nfp_bpf_opt_replace_insn,
615 .remove_insns = nfp_bpf_opt_remove_insns,
616 .prepare = nfp_bpf_verifier_prep,
617 .translate = nfp_bpf_translate,
618 .destroy = nfp_bpf_destroy,
619};
620
