1
2
3#include <linux/compiler.h>
4#include <linux/err.h>
5
6#include <sys/resource.h>
7#include <sys/socket.h>
8#include <sys/types.h>
9#include <linux/btf.h>
10#include <unistd.h>
11#include <signal.h>
12#include <errno.h>
13#include <string.h>
14#include <pthread.h>
15
16#include <bpf/bpf.h>
17#include <bpf/libbpf.h>
18
19#include <test_btf.h>
20#include <test_maps.h>
21
22static struct bpf_create_map_attr xattr = {
23 .name = "sk_storage_map",
24 .map_type = BPF_MAP_TYPE_SK_STORAGE,
25 .map_flags = BPF_F_NO_PREALLOC,
26 .max_entries = 0,
27 .key_size = 4,
28 .value_size = 8,
29 .btf_key_type_id = 1,
30 .btf_value_type_id = 3,
31 .btf_fd = -1,
32};
33
34static unsigned int nr_sk_threads_done;
35static unsigned int nr_sk_threads_err;
36static unsigned int nr_sk_per_thread = 4096;
37static unsigned int nr_sk_threads = 4;
38static int sk_storage_map = -1;
39static unsigned int stop;
40static int runtime_s = 5;
41
42static bool is_stopped(void)
43{
44 return READ_ONCE(stop);
45}
46
47static unsigned int threads_err(void)
48{
49 return READ_ONCE(nr_sk_threads_err);
50}
51
52static void notify_thread_err(void)
53{
54 __sync_add_and_fetch(&nr_sk_threads_err, 1);
55}
56
57static bool wait_for_threads_err(void)
58{
59 while (!is_stopped() && !threads_err())
60 usleep(500);
61
62 return !is_stopped();
63}
64
65static unsigned int threads_done(void)
66{
67 return READ_ONCE(nr_sk_threads_done);
68}
69
70static void notify_thread_done(void)
71{
72 __sync_add_and_fetch(&nr_sk_threads_done, 1);
73}
74
75static void notify_thread_redo(void)
76{
77 __sync_sub_and_fetch(&nr_sk_threads_done, 1);
78}
79
80static bool wait_for_threads_done(void)
81{
82 while (threads_done() != nr_sk_threads && !is_stopped() &&
83 !threads_err())
84 usleep(50);
85
86 return !is_stopped() && !threads_err();
87}
88
89static bool wait_for_threads_redo(void)
90{
91 while (threads_done() && !is_stopped() && !threads_err())
92 usleep(50);
93
94 return !is_stopped() && !threads_err();
95}
96
97static bool wait_for_map(void)
98{
99 while (READ_ONCE(sk_storage_map) == -1 && !is_stopped())
100 usleep(50);
101
102 return !is_stopped();
103}
104
105static bool wait_for_map_close(void)
106{
107 while (READ_ONCE(sk_storage_map) != -1 && !is_stopped())
108 ;
109
110 return !is_stopped();
111}
112
113static int load_btf(void)
114{
115 const char btf_str_sec[] = "\0bpf_spin_lock\0val\0cnt\0l";
116 __u32 btf_raw_types[] = {
117
118 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4),
119
120 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 1), 4),
121 BTF_MEMBER_ENC(15, 1, 0),
122
123 BTF_TYPE_ENC(15, BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 2), 8),
124 BTF_MEMBER_ENC(19, 1, 0),
125 BTF_MEMBER_ENC(23, 2, 32),
126 };
127 struct btf_header btf_hdr = {
128 .magic = BTF_MAGIC,
129 .version = BTF_VERSION,
130 .hdr_len = sizeof(struct btf_header),
131 .type_len = sizeof(btf_raw_types),
132 .str_off = sizeof(btf_raw_types),
133 .str_len = sizeof(btf_str_sec),
134 };
135 __u8 raw_btf[sizeof(struct btf_header) + sizeof(btf_raw_types) +
136 sizeof(btf_str_sec)];
137
138 memcpy(raw_btf, &btf_hdr, sizeof(btf_hdr));
139 memcpy(raw_btf + sizeof(btf_hdr), btf_raw_types, sizeof(btf_raw_types));
140 memcpy(raw_btf + sizeof(btf_hdr) + sizeof(btf_raw_types),
141 btf_str_sec, sizeof(btf_str_sec));
142
143 return bpf_load_btf(raw_btf, sizeof(raw_btf), 0, 0, 0);
144}
145
146static int create_sk_storage_map(void)
147{
148 int btf_fd, map_fd;
149
150 btf_fd = load_btf();
151 CHECK(btf_fd == -1, "bpf_load_btf", "btf_fd:%d errno:%d\n",
152 btf_fd, errno);
153 xattr.btf_fd = btf_fd;
154
155 map_fd = bpf_create_map_xattr(&xattr);
156 xattr.btf_fd = -1;
157 close(btf_fd);
158 CHECK(map_fd == -1,
159 "bpf_create_map_xattr()", "errno:%d\n", errno);
160
161 return map_fd;
162}
163
164static void *insert_close_thread(void *arg)
165{
166 struct {
167 int cnt;
168 int lock;
169 } value = { .cnt = 0xeB9F, .lock = 0, };
170 int i, map_fd, err, *sk_fds;
171
172 sk_fds = malloc(sizeof(*sk_fds) * nr_sk_per_thread);
173 if (!sk_fds) {
174 notify_thread_err();
175 return ERR_PTR(-ENOMEM);
176 }
177
178 for (i = 0; i < nr_sk_per_thread; i++)
179 sk_fds[i] = -1;
180
181 while (!is_stopped()) {
182 if (!wait_for_map())
183 goto close_all;
184
185 map_fd = READ_ONCE(sk_storage_map);
186 for (i = 0; i < nr_sk_per_thread && !is_stopped(); i++) {
187 sk_fds[i] = socket(AF_INET6, SOCK_STREAM, 0);
188 if (sk_fds[i] == -1) {
189 err = -errno;
190 fprintf(stderr, "socket(): errno:%d\n", errno);
191 goto errout;
192 }
193 err = bpf_map_update_elem(map_fd, &sk_fds[i], &value,
194 BPF_NOEXIST);
195 if (err) {
196 err = -errno;
197 fprintf(stderr,
198 "bpf_map_update_elem(): errno:%d\n",
199 errno);
200 goto errout;
201 }
202 }
203
204 notify_thread_done();
205 wait_for_map_close();
206
207close_all:
208 for (i = 0; i < nr_sk_per_thread; i++) {
209 close(sk_fds[i]);
210 sk_fds[i] = -1;
211 }
212
213 notify_thread_redo();
214 }
215
216 free(sk_fds);
217 return NULL;
218
219errout:
220 for (i = 0; i < nr_sk_per_thread && sk_fds[i] != -1; i++)
221 close(sk_fds[i]);
222 free(sk_fds);
223 notify_thread_err();
224 return ERR_PTR(err);
225}
226
227static int do_sk_storage_map_stress_free(void)
228{
229 int i, map_fd = -1, err = 0, nr_threads_created = 0;
230 pthread_t *sk_thread_ids;
231 void *thread_ret;
232
233 sk_thread_ids = malloc(sizeof(pthread_t) * nr_sk_threads);
234 if (!sk_thread_ids) {
235 fprintf(stderr, "malloc(sk_threads): NULL\n");
236 return -ENOMEM;
237 }
238
239 for (i = 0; i < nr_sk_threads; i++) {
240 err = pthread_create(&sk_thread_ids[i], NULL,
241 insert_close_thread, NULL);
242 if (err) {
243 err = -errno;
244 goto done;
245 }
246 nr_threads_created++;
247 }
248
249 while (!is_stopped()) {
250 map_fd = create_sk_storage_map();
251 WRITE_ONCE(sk_storage_map, map_fd);
252
253 if (!wait_for_threads_done())
254 break;
255
256 WRITE_ONCE(sk_storage_map, -1);
257 close(map_fd);
258 map_fd = -1;
259
260 if (!wait_for_threads_redo())
261 break;
262 }
263
264done:
265 WRITE_ONCE(stop, 1);
266 for (i = 0; i < nr_threads_created; i++) {
267 pthread_join(sk_thread_ids[i], &thread_ret);
268 if (IS_ERR(thread_ret) && !err) {
269 err = PTR_ERR(thread_ret);
270 fprintf(stderr, "threads#%u: err:%d\n", i, err);
271 }
272 }
273 free(sk_thread_ids);
274
275 if (map_fd != -1)
276 close(map_fd);
277
278 return err;
279}
280
281static void *update_thread(void *arg)
282{
283 struct {
284 int cnt;
285 int lock;
286 } value = { .cnt = 0xeB9F, .lock = 0, };
287 int map_fd = READ_ONCE(sk_storage_map);
288 int sk_fd = *(int *)arg;
289 int err = 0;
290
291 while (!is_stopped()) {
292 err = bpf_map_update_elem(map_fd, &sk_fd, &value, 0);
293 if (err && errno != EAGAIN) {
294 err = -errno;
295 fprintf(stderr, "bpf_map_update_elem: %d %d\n",
296 err, errno);
297 break;
298 }
299 }
300
301 if (!is_stopped()) {
302 notify_thread_err();
303 return ERR_PTR(err);
304 }
305
306 return NULL;
307}
308
309static void *delete_thread(void *arg)
310{
311 int map_fd = READ_ONCE(sk_storage_map);
312 int sk_fd = *(int *)arg;
313 int err = 0;
314
315 while (!is_stopped()) {
316 err = bpf_map_delete_elem(map_fd, &sk_fd);
317 if (err && errno != ENOENT) {
318 err = -errno;
319 fprintf(stderr, "bpf_map_delete_elem: %d %d\n",
320 err, errno);
321 break;
322 }
323 }
324
325 if (!is_stopped()) {
326 notify_thread_err();
327 return ERR_PTR(err);
328 }
329
330 return NULL;
331}
332
333static int do_sk_storage_map_stress_change(void)
334{
335 int i, sk_fd, map_fd = -1, err = 0, nr_threads_created = 0;
336 pthread_t *sk_thread_ids;
337 void *thread_ret;
338
339 sk_thread_ids = malloc(sizeof(pthread_t) * nr_sk_threads);
340 if (!sk_thread_ids) {
341 fprintf(stderr, "malloc(sk_threads): NULL\n");
342 return -ENOMEM;
343 }
344
345 sk_fd = socket(AF_INET6, SOCK_STREAM, 0);
346 if (sk_fd == -1) {
347 err = -errno;
348 goto done;
349 }
350
351 map_fd = create_sk_storage_map();
352 WRITE_ONCE(sk_storage_map, map_fd);
353
354 for (i = 0; i < nr_sk_threads; i++) {
355 if (i & 0x1)
356 err = pthread_create(&sk_thread_ids[i], NULL,
357 update_thread, &sk_fd);
358 else
359 err = pthread_create(&sk_thread_ids[i], NULL,
360 delete_thread, &sk_fd);
361 if (err) {
362 err = -errno;
363 goto done;
364 }
365 nr_threads_created++;
366 }
367
368 wait_for_threads_err();
369
370done:
371 WRITE_ONCE(stop, 1);
372 for (i = 0; i < nr_threads_created; i++) {
373 pthread_join(sk_thread_ids[i], &thread_ret);
374 if (IS_ERR(thread_ret) && !err) {
375 err = PTR_ERR(thread_ret);
376 fprintf(stderr, "threads#%u: err:%d\n", i, err);
377 }
378 }
379 free(sk_thread_ids);
380
381 if (sk_fd != -1)
382 close(sk_fd);
383 close(map_fd);
384
385 return err;
386}
387
388static void stop_handler(int signum)
389{
390 if (signum != SIGALRM)
391 printf("stopping...\n");
392 WRITE_ONCE(stop, 1);
393}
394
395#define BPF_SK_STORAGE_MAP_TEST_NR_THREADS "BPF_SK_STORAGE_MAP_TEST_NR_THREADS"
396#define BPF_SK_STORAGE_MAP_TEST_SK_PER_THREAD "BPF_SK_STORAGE_MAP_TEST_SK_PER_THREAD"
397#define BPF_SK_STORAGE_MAP_TEST_RUNTIME_S "BPF_SK_STORAGE_MAP_TEST_RUNTIME_S"
398#define BPF_SK_STORAGE_MAP_TEST_NAME "BPF_SK_STORAGE_MAP_TEST_NAME"
399
400static void test_sk_storage_map_stress_free(void)
401{
402 struct rlimit rlim_old, rlim_new = {};
403 int err;
404
405 getrlimit(RLIMIT_NOFILE, &rlim_old);
406
407 signal(SIGTERM, stop_handler);
408 signal(SIGINT, stop_handler);
409 if (runtime_s > 0) {
410 signal(SIGALRM, stop_handler);
411 alarm(runtime_s);
412 }
413
414 if (rlim_old.rlim_cur < nr_sk_threads * nr_sk_per_thread) {
415 rlim_new.rlim_cur = nr_sk_threads * nr_sk_per_thread + 128;
416 rlim_new.rlim_max = rlim_new.rlim_cur + 128;
417 err = setrlimit(RLIMIT_NOFILE, &rlim_new);
418 CHECK(err, "setrlimit(RLIMIT_NOFILE)", "rlim_new:%lu errno:%d",
419 rlim_new.rlim_cur, errno);
420 }
421
422 err = do_sk_storage_map_stress_free();
423
424 signal(SIGTERM, SIG_DFL);
425 signal(SIGINT, SIG_DFL);
426 if (runtime_s > 0) {
427 signal(SIGALRM, SIG_DFL);
428 alarm(0);
429 }
430
431 if (rlim_new.rlim_cur)
432 setrlimit(RLIMIT_NOFILE, &rlim_old);
433
434 CHECK(err, "test_sk_storage_map_stress_free", "err:%d\n", err);
435}
436
437static void test_sk_storage_map_stress_change(void)
438{
439 int err;
440
441 signal(SIGTERM, stop_handler);
442 signal(SIGINT, stop_handler);
443 if (runtime_s > 0) {
444 signal(SIGALRM, stop_handler);
445 alarm(runtime_s);
446 }
447
448 err = do_sk_storage_map_stress_change();
449
450 signal(SIGTERM, SIG_DFL);
451 signal(SIGINT, SIG_DFL);
452 if (runtime_s > 0) {
453 signal(SIGALRM, SIG_DFL);
454 alarm(0);
455 }
456
457 CHECK(err, "test_sk_storage_map_stress_change", "err:%d\n", err);
458}
459
460static void test_sk_storage_map_basic(void)
461{
462 struct {
463 int cnt;
464 int lock;
465 } value = { .cnt = 0xeB9f, .lock = 0, }, lookup_value;
466 struct bpf_create_map_attr bad_xattr;
467 int btf_fd, map_fd, sk_fd, err;
468
469 btf_fd = load_btf();
470 CHECK(btf_fd == -1, "bpf_load_btf", "btf_fd:%d errno:%d\n",
471 btf_fd, errno);
472 xattr.btf_fd = btf_fd;
473
474 sk_fd = socket(AF_INET6, SOCK_STREAM, 0);
475 CHECK(sk_fd == -1, "socket()", "sk_fd:%d errno:%d\n",
476 sk_fd, errno);
477
478 map_fd = bpf_create_map_xattr(&xattr);
479 CHECK(map_fd == -1, "bpf_create_map_xattr(good_xattr)",
480 "map_fd:%d errno:%d\n", map_fd, errno);
481
482
483 memcpy(&lookup_value, &value, sizeof(value));
484 err = bpf_map_update_elem(map_fd, &sk_fd, &value,
485 BPF_NOEXIST | BPF_F_LOCK);
486 CHECK(err, "bpf_map_update_elem(BPF_NOEXIST|BPF_F_LOCK)",
487 "err:%d errno:%d\n", err, errno);
488 err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
489 BPF_F_LOCK);
490 CHECK(err || lookup_value.cnt != value.cnt,
491 "bpf_map_lookup_elem_flags(BPF_F_LOCK)",
492 "err:%d errno:%d cnt:%x(%x)\n",
493 err, errno, lookup_value.cnt, value.cnt);
494
495
496 value.cnt += 1;
497 err = bpf_map_update_elem(map_fd, &sk_fd, &value,
498 BPF_EXIST | BPF_F_LOCK);
499 CHECK(err, "bpf_map_update_elem(BPF_EXIST|BPF_F_LOCK)",
500 "err:%d errno:%d\n", err, errno);
501 err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
502 BPF_F_LOCK);
503 CHECK(err || lookup_value.cnt != value.cnt,
504 "bpf_map_lookup_elem_flags(BPF_F_LOCK)",
505 "err:%d errno:%d cnt:%x(%x)\n",
506 err, errno, lookup_value.cnt, value.cnt);
507
508
509 value.cnt += 1;
510 err = bpf_map_update_elem(map_fd, &sk_fd, &value, BPF_EXIST);
511 CHECK(err, "bpf_map_update_elem(BPF_EXIST)",
512 "err:%d errno:%d\n", err, errno);
513 err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
514 BPF_F_LOCK);
515 CHECK(err || lookup_value.cnt != value.cnt,
516 "bpf_map_lookup_elem_flags(BPF_F_LOCK)",
517 "err:%d errno:%d cnt:%x(%x)\n",
518 err, errno, lookup_value.cnt, value.cnt);
519
520
521 value.cnt += 1;
522 err = bpf_map_update_elem(map_fd, &sk_fd, &value,
523 BPF_NOEXIST | BPF_F_LOCK);
524 CHECK(!err || errno != EEXIST,
525 "bpf_map_update_elem(BPF_NOEXIST|BPF_F_LOCK)",
526 "err:%d errno:%d\n", err, errno);
527 err = bpf_map_update_elem(map_fd, &sk_fd, &value, BPF_NOEXIST);
528 CHECK(!err || errno != EEXIST, "bpf_map_update_elem(BPF_NOEXIST)",
529 "err:%d errno:%d\n", err, errno);
530 value.cnt -= 1;
531 err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
532 BPF_F_LOCK);
533 CHECK(err || lookup_value.cnt != value.cnt,
534 "bpf_map_lookup_elem_flags(BPF_F_LOCK)",
535 "err:%d errno:%d cnt:%x(%x)\n",
536 err, errno, lookup_value.cnt, value.cnt);
537
538
539 value.cnt += 1;
540 err = bpf_map_update_elem(map_fd, &sk_fd, &value, 0);
541 CHECK(err, "bpf_map_update_elem()", "err:%d errno:%d\n",
542 err, errno);
543 err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
544 BPF_F_LOCK);
545 CHECK(err || lookup_value.cnt != value.cnt,
546 "bpf_map_lookup_elem_flags(BPF_F_LOCK)",
547 "err:%d errno:%d cnt:%x(%x)\n",
548 err, errno, lookup_value.cnt, value.cnt);
549
550
551 err = bpf_map_delete_elem(map_fd, &sk_fd);
552 CHECK(err, "bpf_map_delete_elem()", "err:%d errno:%d\n",
553 err, errno);
554 err = bpf_map_lookup_elem_flags(map_fd, &sk_fd, &lookup_value,
555 BPF_F_LOCK);
556 CHECK(!err || errno != ENOENT,
557 "bpf_map_lookup_elem_flags(BPF_F_LOCK)",
558 "err:%d errno:%d\n", err, errno);
559 err = bpf_map_delete_elem(map_fd, &sk_fd);
560 CHECK(!err || errno != ENOENT, "bpf_map_delete_elem()",
561 "err:%d errno:%d\n", err, errno);
562
563 memcpy(&bad_xattr, &xattr, sizeof(xattr));
564 bad_xattr.btf_key_type_id = 0;
565 err = bpf_create_map_xattr(&bad_xattr);
566 CHECK(!err || errno != EINVAL, "bap_create_map_xattr(bad_xattr)",
567 "err:%d errno:%d\n", err, errno);
568
569 memcpy(&bad_xattr, &xattr, sizeof(xattr));
570 bad_xattr.btf_key_type_id = 3;
571 err = bpf_create_map_xattr(&bad_xattr);
572 CHECK(!err || errno != EINVAL, "bap_create_map_xattr(bad_xattr)",
573 "err:%d errno:%d\n", err, errno);
574
575 memcpy(&bad_xattr, &xattr, sizeof(xattr));
576 bad_xattr.max_entries = 1;
577 err = bpf_create_map_xattr(&bad_xattr);
578 CHECK(!err || errno != EINVAL, "bap_create_map_xattr(bad_xattr)",
579 "err:%d errno:%d\n", err, errno);
580
581 memcpy(&bad_xattr, &xattr, sizeof(xattr));
582 bad_xattr.map_flags = 0;
583 err = bpf_create_map_xattr(&bad_xattr);
584 CHECK(!err || errno != EINVAL, "bap_create_map_xattr(bad_xattr)",
585 "err:%d errno:%d\n", err, errno);
586
587 xattr.btf_fd = -1;
588 close(btf_fd);
589 close(map_fd);
590 close(sk_fd);
591}
592
593void test_sk_storage_map(void)
594{
595 const char *test_name, *env_opt;
596 bool test_ran = false;
597
598 test_name = getenv(BPF_SK_STORAGE_MAP_TEST_NAME);
599
600 env_opt = getenv(BPF_SK_STORAGE_MAP_TEST_NR_THREADS);
601 if (env_opt)
602 nr_sk_threads = atoi(env_opt);
603
604 env_opt = getenv(BPF_SK_STORAGE_MAP_TEST_SK_PER_THREAD);
605 if (env_opt)
606 nr_sk_per_thread = atoi(env_opt);
607
608 env_opt = getenv(BPF_SK_STORAGE_MAP_TEST_RUNTIME_S);
609 if (env_opt)
610 runtime_s = atoi(env_opt);
611
612 if (!test_name || !strcmp(test_name, "basic")) {
613 test_sk_storage_map_basic();
614 test_ran = true;
615 }
616 if (!test_name || !strcmp(test_name, "stress_free")) {
617 test_sk_storage_map_stress_free();
618 test_ran = true;
619 }
620 if (!test_name || !strcmp(test_name, "stress_change")) {
621 test_sk_storage_map_stress_change();
622 test_ran = true;
623 }
624
625 if (test_ran)
626 printf("%s:PASS\n", __func__);
627 else
628 CHECK(1, "Invalid test_name", "%s\n", test_name);
629}
630
