1
2
3
4#include <linux/bpf.h>
5#include <linux/bitops.h>
6#include <linux/bug.h>
7#include <linux/jiffies.h>
8#include <linux/skbuff.h>
9#include <linux/wait.h>
10
11#include "../nfp_app.h"
12#include "../nfp_net.h"
13#include "fw.h"
14#include "main.h"
15
16#define NFP_BPF_TAG_ALLOC_SPAN (U16_MAX / 4)
17
18static bool nfp_bpf_all_tags_busy(struct nfp_app_bpf *bpf)
19{
20 u16 used_tags;
21
22 used_tags = bpf->tag_alloc_next - bpf->tag_alloc_last;
23
24 return used_tags > NFP_BPF_TAG_ALLOC_SPAN;
25}
26
27static int nfp_bpf_alloc_tag(struct nfp_app_bpf *bpf)
28{
29
30
31
32
33 if (nfp_bpf_all_tags_busy(bpf)) {
34 cmsg_warn(bpf, "all FW request contexts busy!\n");
35 return -EAGAIN;
36 }
37
38 WARN_ON(__test_and_set_bit(bpf->tag_alloc_next, bpf->tag_allocator));
39 return bpf->tag_alloc_next++;
40}
41
42static void nfp_bpf_free_tag(struct nfp_app_bpf *bpf, u16 tag)
43{
44 WARN_ON(!__test_and_clear_bit(tag, bpf->tag_allocator));
45
46 while (!test_bit(bpf->tag_alloc_last, bpf->tag_allocator) &&
47 bpf->tag_alloc_last != bpf->tag_alloc_next)
48 bpf->tag_alloc_last++;
49}
50
51static struct sk_buff *
52nfp_bpf_cmsg_alloc(struct nfp_app_bpf *bpf, unsigned int size)
53{
54 struct sk_buff *skb;
55
56 skb = nfp_app_ctrl_msg_alloc(bpf->app, size, GFP_KERNEL);
57 skb_put(skb, size);
58
59 return skb;
60}
61
62static unsigned int
63nfp_bpf_cmsg_map_req_size(struct nfp_app_bpf *bpf, unsigned int n)
64{
65 unsigned int size;
66
67 size = sizeof(struct cmsg_req_map_op);
68 size += (bpf->cmsg_key_sz + bpf->cmsg_val_sz) * n;
69
70 return size;
71}
72
73static struct sk_buff *
74nfp_bpf_cmsg_map_req_alloc(struct nfp_app_bpf *bpf, unsigned int n)
75{
76 return nfp_bpf_cmsg_alloc(bpf, nfp_bpf_cmsg_map_req_size(bpf, n));
77}
78
79static unsigned int
80nfp_bpf_cmsg_map_reply_size(struct nfp_app_bpf *bpf, unsigned int n)
81{
82 unsigned int size;
83
84 size = sizeof(struct cmsg_reply_map_op);
85 size += (bpf->cmsg_key_sz + bpf->cmsg_val_sz) * n;
86
87 return size;
88}
89
90static u8 nfp_bpf_cmsg_get_type(struct sk_buff *skb)
91{
92 struct cmsg_hdr *hdr;
93
94 hdr = (struct cmsg_hdr *)skb->data;
95
96 return hdr->type;
97}
98
99static unsigned int nfp_bpf_cmsg_get_tag(struct sk_buff *skb)
100{
101 struct cmsg_hdr *hdr;
102
103 hdr = (struct cmsg_hdr *)skb->data;
104
105 return be16_to_cpu(hdr->tag);
106}
107
108static struct sk_buff *__nfp_bpf_reply(struct nfp_app_bpf *bpf, u16 tag)
109{
110 unsigned int msg_tag;
111 struct sk_buff *skb;
112
113 skb_queue_walk(&bpf->cmsg_replies, skb) {
114 msg_tag = nfp_bpf_cmsg_get_tag(skb);
115 if (msg_tag == tag) {
116 nfp_bpf_free_tag(bpf, tag);
117 __skb_unlink(skb, &bpf->cmsg_replies);
118 return skb;
119 }
120 }
121
122 return NULL;
123}
124
125static struct sk_buff *nfp_bpf_reply(struct nfp_app_bpf *bpf, u16 tag)
126{
127 struct sk_buff *skb;
128
129 nfp_ctrl_lock(bpf->app->ctrl);
130 skb = __nfp_bpf_reply(bpf, tag);
131 nfp_ctrl_unlock(bpf->app->ctrl);
132
133 return skb;
134}
135
136static struct sk_buff *nfp_bpf_reply_drop_tag(struct nfp_app_bpf *bpf, u16 tag)
137{
138 struct sk_buff *skb;
139
140 nfp_ctrl_lock(bpf->app->ctrl);
141 skb = __nfp_bpf_reply(bpf, tag);
142 if (!skb)
143 nfp_bpf_free_tag(bpf, tag);
144 nfp_ctrl_unlock(bpf->app->ctrl);
145
146 return skb;
147}
148
149static struct sk_buff *
150nfp_bpf_cmsg_wait_reply(struct nfp_app_bpf *bpf, enum nfp_bpf_cmsg_type type,
151 int tag)
152{
153 struct sk_buff *skb;
154 int i, err;
155
156 for (i = 0; i < 50; i++) {
157 udelay(4);
158 skb = nfp_bpf_reply(bpf, tag);
159 if (skb)
160 return skb;
161 }
162
163 err = wait_event_interruptible_timeout(bpf->cmsg_wq,
164 skb = nfp_bpf_reply(bpf, tag),
165 msecs_to_jiffies(5000));
166
167
168
169 if (!skb)
170 skb = nfp_bpf_reply_drop_tag(bpf, tag);
171 if (err < 0) {
172 cmsg_warn(bpf, "%s waiting for response to 0x%02x: %d\n",
173 err == ERESTARTSYS ? "interrupted" : "error",
174 type, err);
175 return ERR_PTR(err);
176 }
177 if (!skb) {
178 cmsg_warn(bpf, "timeout waiting for response to 0x%02x\n",
179 type);
180 return ERR_PTR(-ETIMEDOUT);
181 }
182
183 return skb;
184}
185
186static struct sk_buff *
187nfp_bpf_cmsg_communicate(struct nfp_app_bpf *bpf, struct sk_buff *skb,
188 enum nfp_bpf_cmsg_type type, unsigned int reply_size)
189{
190 struct cmsg_hdr *hdr;
191 int tag;
192
193 nfp_ctrl_lock(bpf->app->ctrl);
194 tag = nfp_bpf_alloc_tag(bpf);
195 if (tag < 0) {
196 nfp_ctrl_unlock(bpf->app->ctrl);
197 dev_kfree_skb_any(skb);
198 return ERR_PTR(tag);
199 }
200
201 hdr = (void *)skb->data;
202 hdr->ver = CMSG_MAP_ABI_VERSION;
203 hdr->type = type;
204 hdr->tag = cpu_to_be16(tag);
205
206 __nfp_app_ctrl_tx(bpf->app, skb);
207
208 nfp_ctrl_unlock(bpf->app->ctrl);
209
210 skb = nfp_bpf_cmsg_wait_reply(bpf, type, tag);
211 if (IS_ERR(skb))
212 return skb;
213
214 hdr = (struct cmsg_hdr *)skb->data;
215 if (hdr->type != __CMSG_REPLY(type)) {
216 cmsg_warn(bpf, "cmsg drop - wrong type 0x%02x != 0x%02lx!\n",
217 hdr->type, __CMSG_REPLY(type));
218 goto err_free;
219 }
220
221 if (reply_size && skb->len != reply_size) {
222 cmsg_warn(bpf, "cmsg drop - type 0x%02x wrong size %d != %d!\n",
223 type, skb->len, reply_size);
224 goto err_free;
225 }
226
227 return skb;
228err_free:
229 dev_kfree_skb_any(skb);
230 return ERR_PTR(-EIO);
231}
232
233static int
234nfp_bpf_ctrl_rc_to_errno(struct nfp_app_bpf *bpf,
235 struct cmsg_reply_map_simple *reply)
236{
237 static const int res_table[] = {
238 [CMSG_RC_SUCCESS] = 0,
239 [CMSG_RC_ERR_MAP_FD] = -EBADFD,
240 [CMSG_RC_ERR_MAP_NOENT] = -ENOENT,
241 [CMSG_RC_ERR_MAP_ERR] = -EINVAL,
242 [CMSG_RC_ERR_MAP_PARSE] = -EIO,
243 [CMSG_RC_ERR_MAP_EXIST] = -EEXIST,
244 [CMSG_RC_ERR_MAP_NOMEM] = -ENOMEM,
245 [CMSG_RC_ERR_MAP_E2BIG] = -E2BIG,
246 };
247 u32 rc;
248
249 rc = be32_to_cpu(reply->rc);
250 if (rc >= ARRAY_SIZE(res_table)) {
251 cmsg_warn(bpf, "FW responded with invalid status: %u\n", rc);
252 return -EIO;
253 }
254
255 return res_table[rc];
256}
257
258long long int
259nfp_bpf_ctrl_alloc_map(struct nfp_app_bpf *bpf, struct bpf_map *map)
260{
261 struct cmsg_reply_map_alloc_tbl *reply;
262 struct cmsg_req_map_alloc_tbl *req;
263 struct sk_buff *skb;
264 u32 tid;
265 int err;
266
267 skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req));
268 if (!skb)
269 return -ENOMEM;
270
271 req = (void *)skb->data;
272 req->key_size = cpu_to_be32(map->key_size);
273 req->value_size = cpu_to_be32(map->value_size);
274 req->max_entries = cpu_to_be32(map->max_entries);
275 req->map_type = cpu_to_be32(map->map_type);
276 req->map_flags = 0;
277
278 skb = nfp_bpf_cmsg_communicate(bpf, skb, CMSG_TYPE_MAP_ALLOC,
279 sizeof(*reply));
280 if (IS_ERR(skb))
281 return PTR_ERR(skb);
282
283 reply = (void *)skb->data;
284 err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
285 if (err)
286 goto err_free;
287
288 tid = be32_to_cpu(reply->tid);
289 dev_consume_skb_any(skb);
290
291 return tid;
292err_free:
293 dev_kfree_skb_any(skb);
294 return err;
295}
296
297void nfp_bpf_ctrl_free_map(struct nfp_app_bpf *bpf, struct nfp_bpf_map *nfp_map)
298{
299 struct cmsg_reply_map_free_tbl *reply;
300 struct cmsg_req_map_free_tbl *req;
301 struct sk_buff *skb;
302 int err;
303
304 skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req));
305 if (!skb) {
306 cmsg_warn(bpf, "leaking map - failed to allocate msg\n");
307 return;
308 }
309
310 req = (void *)skb->data;
311 req->tid = cpu_to_be32(nfp_map->tid);
312
313 skb = nfp_bpf_cmsg_communicate(bpf, skb, CMSG_TYPE_MAP_FREE,
314 sizeof(*reply));
315 if (IS_ERR(skb)) {
316 cmsg_warn(bpf, "leaking map - I/O error\n");
317 return;
318 }
319
320 reply = (void *)skb->data;
321 err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
322 if (err)
323 cmsg_warn(bpf, "leaking map - FW responded with: %d\n", err);
324
325 dev_consume_skb_any(skb);
326}
327
328static void *
329nfp_bpf_ctrl_req_key(struct nfp_app_bpf *bpf, struct cmsg_req_map_op *req,
330 unsigned int n)
331{
332 return &req->data[bpf->cmsg_key_sz * n + bpf->cmsg_val_sz * n];
333}
334
335static void *
336nfp_bpf_ctrl_req_val(struct nfp_app_bpf *bpf, struct cmsg_req_map_op *req,
337 unsigned int n)
338{
339 return &req->data[bpf->cmsg_key_sz * (n + 1) + bpf->cmsg_val_sz * n];
340}
341
342static void *
343nfp_bpf_ctrl_reply_key(struct nfp_app_bpf *bpf, struct cmsg_reply_map_op *reply,
344 unsigned int n)
345{
346 return &reply->data[bpf->cmsg_key_sz * n + bpf->cmsg_val_sz * n];
347}
348
349static void *
350nfp_bpf_ctrl_reply_val(struct nfp_app_bpf *bpf, struct cmsg_reply_map_op *reply,
351 unsigned int n)
352{
353 return &reply->data[bpf->cmsg_key_sz * (n + 1) + bpf->cmsg_val_sz * n];
354}
355
356static int
357nfp_bpf_ctrl_entry_op(struct bpf_offloaded_map *offmap,
358 enum nfp_bpf_cmsg_type op,
359 u8 *key, u8 *value, u64 flags, u8 *out_key, u8 *out_value)
360{
361 struct nfp_bpf_map *nfp_map = offmap->dev_priv;
362 struct nfp_app_bpf *bpf = nfp_map->bpf;
363 struct bpf_map *map = &offmap->map;
364 struct cmsg_reply_map_op *reply;
365 struct cmsg_req_map_op *req;
366 struct sk_buff *skb;
367 int err;
368
369
370 if (flags >> 32)
371 return -EOPNOTSUPP;
372
373 skb = nfp_bpf_cmsg_map_req_alloc(bpf, 1);
374 if (!skb)
375 return -ENOMEM;
376
377 req = (void *)skb->data;
378 req->tid = cpu_to_be32(nfp_map->tid);
379 req->count = cpu_to_be32(1);
380 req->flags = cpu_to_be32(flags);
381
382
383 if (key)
384 memcpy(nfp_bpf_ctrl_req_key(bpf, req, 0), key, map->key_size);
385 if (value)
386 memcpy(nfp_bpf_ctrl_req_val(bpf, req, 0), value,
387 map->value_size);
388
389 skb = nfp_bpf_cmsg_communicate(bpf, skb, op,
390 nfp_bpf_cmsg_map_reply_size(bpf, 1));
391 if (IS_ERR(skb))
392 return PTR_ERR(skb);
393
394 reply = (void *)skb->data;
395 err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr);
396 if (err)
397 goto err_free;
398
399
400 if (out_key)
401 memcpy(out_key, nfp_bpf_ctrl_reply_key(bpf, reply, 0),
402 map->key_size);
403 if (out_value)
404 memcpy(out_value, nfp_bpf_ctrl_reply_val(bpf, reply, 0),
405 map->value_size);
406
407 dev_consume_skb_any(skb);
408
409 return 0;
410err_free:
411 dev_kfree_skb_any(skb);
412 return err;
413}
414
415int nfp_bpf_ctrl_update_entry(struct bpf_offloaded_map *offmap,
416 void *key, void *value, u64 flags)
417{
418 return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_UPDATE,
419 key, value, flags, NULL, NULL);
420}
421
422int nfp_bpf_ctrl_del_entry(struct bpf_offloaded_map *offmap, void *key)
423{
424 return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_DELETE,
425 key, NULL, 0, NULL, NULL);
426}
427
428int nfp_bpf_ctrl_lookup_entry(struct bpf_offloaded_map *offmap,
429 void *key, void *value)
430{
431 return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_LOOKUP,
432 key, NULL, 0, NULL, value);
433}
434
435int nfp_bpf_ctrl_getfirst_entry(struct bpf_offloaded_map *offmap,
436 void *next_key)
437{
438 return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_GETFIRST,
439 NULL, NULL, 0, next_key, NULL);
440}
441
442int nfp_bpf_ctrl_getnext_entry(struct bpf_offloaded_map *offmap,
443 void *key, void *next_key)
444{
445 return nfp_bpf_ctrl_entry_op(offmap, CMSG_TYPE_MAP_GETNEXT,
446 key, NULL, 0, next_key, NULL);
447}
448
449unsigned int nfp_bpf_ctrl_cmsg_mtu(struct nfp_app_bpf *bpf)
450{
451 return max3((unsigned int)NFP_NET_DEFAULT_MTU,
452 nfp_bpf_cmsg_map_req_size(bpf, 1),
453 nfp_bpf_cmsg_map_reply_size(bpf, 1));
454}
455
456void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb)
457{
458 struct nfp_app_bpf *bpf = app->priv;
459 unsigned int tag;
460
461 if (unlikely(skb->len < sizeof(struct cmsg_reply_map_simple))) {
462 cmsg_warn(bpf, "cmsg drop - too short %d!\n", skb->len);
463 goto err_free;
464 }
465
466 if (nfp_bpf_cmsg_get_type(skb) == CMSG_TYPE_BPF_EVENT) {
467 if (!nfp_bpf_event_output(bpf, skb->data, skb->len))
468 dev_consume_skb_any(skb);
469 else
470 dev_kfree_skb_any(skb);
471 return;
472 }
473
474 nfp_ctrl_lock(bpf->app->ctrl);
475
476 tag = nfp_bpf_cmsg_get_tag(skb);
477 if (unlikely(!test_bit(tag, bpf->tag_allocator))) {
478 cmsg_warn(bpf, "cmsg drop - no one is waiting for tag %u!\n",
479 tag);
480 goto err_unlock;
481 }
482
483 __skb_queue_tail(&bpf->cmsg_replies, skb);
484 wake_up_interruptible_all(&bpf->cmsg_wq);
485
486 nfp_ctrl_unlock(bpf->app->ctrl);
487
488 return;
489err_unlock:
490 nfp_ctrl_unlock(bpf->app->ctrl);
491err_free:
492 dev_kfree_skb_any(skb);
493}
494
495void
496nfp_bpf_ctrl_msg_rx_raw(struct nfp_app *app, const void *data, unsigned int len)
497{
498 struct nfp_app_bpf *bpf = app->priv;
499 const struct cmsg_hdr *hdr = data;
500
501 if (unlikely(len < sizeof(struct cmsg_reply_map_simple))) {
502 cmsg_warn(bpf, "cmsg drop - too short %d!\n", len);
503 return;
504 }
505
506 if (hdr->type == CMSG_TYPE_BPF_EVENT)
507 nfp_bpf_event_output(bpf, data, len);
508 else
509 cmsg_warn(bpf, "cmsg drop - msg type %d with raw buffer!\n",
510 hdr->type);
511}
512
