1
2
3#include <stdio.h>
4#include <stdlib.h>
5#include <sys/socket.h>
6#include <sys/ioctl.h>
7#include <sys/select.h>
8#include <netinet/in.h>
9#include <arpa/inet.h>
10#include <unistd.h>
11#include <string.h>
12#include <errno.h>
13#include <stdbool.h>
14#include <signal.h>
15#include <fcntl.h>
16#include <sys/wait.h>
17#include <time.h>
18#include <sched.h>
19
20#include <sys/time.h>
21#include <sys/resource.h>
22#include <sys/types.h>
23#include <sys/sendfile.h>
24
25#include <linux/netlink.h>
26#include <linux/socket.h>
27#include <linux/sock_diag.h>
28#include <linux/bpf.h>
29#include <linux/if_link.h>
30#include <linux/tls.h>
31#include <assert.h>
32#include <libgen.h>
33
34#include <getopt.h>
35
36#include <bpf/bpf.h>
37#include <bpf/libbpf.h>
38
39#include "bpf_util.h"
40#include "bpf_rlimit.h"
41#include "cgroup_helpers.h"
42
43int running;
44static void running_handler(int a);
45
46#ifndef TCP_ULP
47# define TCP_ULP 31
48#endif
49#ifndef SOL_TLS
50# define SOL_TLS 282
51#endif
52
53
54#define S1_PORT 10000
55#define S2_PORT 10001
56
57#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
58#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
59#define CG_PATH "/sockmap"
60
61
62int s1, s2, c1, c2, p1, p2;
63int test_cnt;
64int passed;
65int failed;
66int map_fd[9];
67struct bpf_map *maps[9];
68int prog_fd[11];
69
70int txmsg_pass;
71int txmsg_redir;
72int txmsg_drop;
73int txmsg_apply;
74int txmsg_cork;
75int txmsg_start;
76int txmsg_end;
77int txmsg_start_push;
78int txmsg_end_push;
79int txmsg_start_pop;
80int txmsg_pop;
81int txmsg_ingress;
82int txmsg_redir_skb;
83int txmsg_ktls_skb;
84int txmsg_ktls_skb_drop;
85int txmsg_ktls_skb_redir;
86int ktls;
87int peek_flag;
88int skb_use_parser;
89int txmsg_omit_skb_parser;
90
91static const struct option long_options[] = {
92 {"help", no_argument, NULL, 'h' },
93 {"cgroup", required_argument, NULL, 'c' },
94 {"rate", required_argument, NULL, 'r' },
95 {"verbose", optional_argument, NULL, 'v' },
96 {"iov_count", required_argument, NULL, 'i' },
97 {"length", required_argument, NULL, 'l' },
98 {"test", required_argument, NULL, 't' },
99 {"data_test", no_argument, NULL, 'd' },
100 {"txmsg", no_argument, &txmsg_pass, 1 },
101 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
102 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
103 {"txmsg_apply", required_argument, NULL, 'a'},
104 {"txmsg_cork", required_argument, NULL, 'k'},
105 {"txmsg_start", required_argument, NULL, 's'},
106 {"txmsg_end", required_argument, NULL, 'e'},
107 {"txmsg_start_push", required_argument, NULL, 'p'},
108 {"txmsg_end_push", required_argument, NULL, 'q'},
109 {"txmsg_start_pop", required_argument, NULL, 'w'},
110 {"txmsg_pop", required_argument, NULL, 'x'},
111 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
112 {"txmsg_redir_skb", no_argument, &txmsg_redir_skb, 1 },
113 {"ktls", no_argument, &ktls, 1 },
114 {"peek", no_argument, &peek_flag, 1 },
115 {"txmsg_omit_skb_parser", no_argument, &txmsg_omit_skb_parser, 1},
116 {"whitelist", required_argument, NULL, 'n' },
117 {"blacklist", required_argument, NULL, 'b' },
118 {0, 0, NULL, 0 }
119};
120
121struct test_env {
122 const char *type;
123 const char *subtest;
124 const char *prepend;
125
126 int test_num;
127 int subtest_num;
128
129 int succ_cnt;
130 int fail_cnt;
131 int fail_last;
132};
133
134struct test_env env;
135
136struct sockmap_options {
137 int verbose;
138 bool base;
139 bool sendpage;
140 bool data_test;
141 bool drop_expected;
142 bool check_recved_len;
143 int iov_count;
144 int iov_length;
145 int rate;
146 char *map;
147 char *whitelist;
148 char *blacklist;
149 char *prepend;
150};
151
152struct _test {
153 char *title;
154 void (*tester)(int cg_fd, struct sockmap_options *opt);
155};
156
157static void test_start(void)
158{
159 env.subtest_num++;
160}
161
162static void test_fail(void)
163{
164 env.fail_cnt++;
165}
166
167static void test_pass(void)
168{
169 env.succ_cnt++;
170}
171
172static void test_reset(void)
173{
174 txmsg_start = txmsg_end = 0;
175 txmsg_start_pop = txmsg_pop = 0;
176 txmsg_start_push = txmsg_end_push = 0;
177 txmsg_pass = txmsg_drop = txmsg_redir = 0;
178 txmsg_apply = txmsg_cork = 0;
179 txmsg_ingress = txmsg_redir_skb = 0;
180 txmsg_ktls_skb = txmsg_ktls_skb_drop = txmsg_ktls_skb_redir = 0;
181 txmsg_omit_skb_parser = 0;
182 skb_use_parser = 0;
183}
184
185static int test_start_subtest(const struct _test *t, struct sockmap_options *o)
186{
187 env.type = o->map;
188 env.subtest = t->title;
189 env.prepend = o->prepend;
190 env.test_num++;
191 env.subtest_num = 0;
192 env.fail_last = env.fail_cnt;
193 test_reset();
194 return 0;
195}
196
197static void test_end_subtest(void)
198{
199 int error = env.fail_cnt - env.fail_last;
200 int type = strcmp(env.type, BPF_SOCKMAP_FILENAME);
201
202 if (!error)
203 test_pass();
204
205 fprintf(stdout, "#%2d/%2d %8s:%s:%s:%s\n",
206 env.test_num, env.subtest_num,
207 !type ? "sockmap" : "sockhash",
208 env.prepend ? : "",
209 env.subtest, error ? "FAIL" : "OK");
210}
211
212static void test_print_results(void)
213{
214 fprintf(stdout, "Pass: %d Fail: %d\n",
215 env.succ_cnt, env.fail_cnt);
216}
217
218static void usage(char *argv[])
219{
220 int i;
221
222 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
223 printf(" options:\n");
224 for (i = 0; long_options[i].name != 0; i++) {
225 printf(" --%-12s", long_options[i].name);
226 if (long_options[i].flag != NULL)
227 printf(" flag (internal value:%d)\n",
228 *long_options[i].flag);
229 else
230 printf(" -%c\n", long_options[i].val);
231 }
232 printf("\n");
233}
234
235char *sock_to_string(int s)
236{
237 if (s == c1)
238 return "client1";
239 else if (s == c2)
240 return "client2";
241 else if (s == s1)
242 return "server1";
243 else if (s == s2)
244 return "server2";
245 else if (s == p1)
246 return "peer1";
247 else if (s == p2)
248 return "peer2";
249 else
250 return "unknown";
251}
252
253static int sockmap_init_ktls(int verbose, int s)
254{
255 struct tls12_crypto_info_aes_gcm_128 tls_tx = {
256 .info = {
257 .version = TLS_1_2_VERSION,
258 .cipher_type = TLS_CIPHER_AES_GCM_128,
259 },
260 };
261 struct tls12_crypto_info_aes_gcm_128 tls_rx = {
262 .info = {
263 .version = TLS_1_2_VERSION,
264 .cipher_type = TLS_CIPHER_AES_GCM_128,
265 },
266 };
267 int so_buf = 6553500;
268 int err;
269
270 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
271 if (err) {
272 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
273 return -EINVAL;
274 }
275 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
276 if (err) {
277 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
278 return -EINVAL;
279 }
280 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
281 if (err) {
282 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
283 return -EINVAL;
284 }
285 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
286 if (err) {
287 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
288 return -EINVAL;
289 }
290 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
291 if (err) {
292 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
293 return -EINVAL;
294 }
295
296 if (verbose)
297 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
298 return 0;
299}
300static int sockmap_init_sockets(int verbose)
301{
302 int i, err, one = 1;
303 struct sockaddr_in addr;
304 int *fds[4] = {&s1, &s2, &c1, &c2};
305
306 s1 = s2 = p1 = p2 = c1 = c2 = 0;
307
308
309 for (i = 0; i < 4; i++) {
310 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
311 if (*fds[i] < 0) {
312 perror("socket s1 failed()");
313 return errno;
314 }
315 }
316
317
318 for (i = 0; i < 2; i++) {
319 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
320 (char *)&one, sizeof(one));
321 if (err) {
322 perror("setsockopt failed()");
323 return errno;
324 }
325 }
326
327
328 for (i = 0; i < 2; i++) {
329 err = ioctl(*fds[i], FIONBIO, (char *)&one);
330 if (err < 0) {
331 perror("ioctl s1 failed()");
332 return errno;
333 }
334 }
335
336
337 memset(&addr, 0, sizeof(struct sockaddr_in));
338 addr.sin_family = AF_INET;
339 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
340
341 addr.sin_port = htons(S1_PORT);
342 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
343 if (err < 0) {
344 perror("bind s1 failed()");
345 return errno;
346 }
347
348 addr.sin_port = htons(S2_PORT);
349 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
350 if (err < 0) {
351 perror("bind s2 failed()");
352 return errno;
353 }
354
355
356 addr.sin_port = htons(S1_PORT);
357 err = listen(s1, 32);
358 if (err < 0) {
359 perror("listen s1 failed()");
360 return errno;
361 }
362
363 addr.sin_port = htons(S2_PORT);
364 err = listen(s2, 32);
365 if (err < 0) {
366 perror("listen s1 failed()");
367 return errno;
368 }
369
370
371 addr.sin_port = htons(S1_PORT);
372 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
373 if (err < 0 && errno != EINPROGRESS) {
374 perror("connect c1 failed()");
375 return errno;
376 }
377
378 addr.sin_port = htons(S2_PORT);
379 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
380 if (err < 0 && errno != EINPROGRESS) {
381 perror("connect c2 failed()");
382 return errno;
383 } else if (err < 0) {
384 err = 0;
385 }
386
387
388 p1 = accept(s1, NULL, NULL);
389 if (p1 < 0) {
390 perror("accept s1 failed()");
391 return errno;
392 }
393
394 p2 = accept(s2, NULL, NULL);
395 if (p2 < 0) {
396 perror("accept s1 failed()");
397 return errno;
398 }
399
400 if (verbose > 1) {
401 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
402 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
403 c1, s1, c2, s2);
404 }
405 return 0;
406}
407
408struct msg_stats {
409 size_t bytes_sent;
410 size_t bytes_recvd;
411 struct timespec start;
412 struct timespec end;
413};
414
415static int msg_loop_sendpage(int fd, int iov_length, int cnt,
416 struct msg_stats *s,
417 struct sockmap_options *opt)
418{
419 bool drop = opt->drop_expected;
420 unsigned char k = 0;
421 FILE *file;
422 int i, fp;
423
424 file = tmpfile();
425 if (!file) {
426 perror("create file for sendpage");
427 return 1;
428 }
429 for (i = 0; i < iov_length * cnt; i++, k++)
430 fwrite(&k, sizeof(char), 1, file);
431 fflush(file);
432 fseek(file, 0, SEEK_SET);
433
434 fp = fileno(file);
435
436 clock_gettime(CLOCK_MONOTONIC, &s->start);
437 for (i = 0; i < cnt; i++) {
438 int sent;
439
440 errno = 0;
441 sent = sendfile(fd, fp, NULL, iov_length);
442
443 if (!drop && sent < 0) {
444 perror("sendpage loop error");
445 fclose(file);
446 return sent;
447 } else if (drop && sent >= 0) {
448 printf("sendpage loop error expected: %i errno %i\n",
449 sent, errno);
450 fclose(file);
451 return -EIO;
452 }
453
454 if (sent > 0)
455 s->bytes_sent += sent;
456 }
457 clock_gettime(CLOCK_MONOTONIC, &s->end);
458 fclose(file);
459 return 0;
460}
461
462static void msg_free_iov(struct msghdr *msg)
463{
464 int i;
465
466 for (i = 0; i < msg->msg_iovlen; i++)
467 free(msg->msg_iov[i].iov_base);
468 free(msg->msg_iov);
469 msg->msg_iov = NULL;
470 msg->msg_iovlen = 0;
471}
472
473static int msg_alloc_iov(struct msghdr *msg,
474 int iov_count, int iov_length,
475 bool data, bool xmit)
476{
477 unsigned char k = 0;
478 struct iovec *iov;
479 int i;
480
481 iov = calloc(iov_count, sizeof(struct iovec));
482 if (!iov)
483 return errno;
484
485 for (i = 0; i < iov_count; i++) {
486 unsigned char *d = calloc(iov_length, sizeof(char));
487
488 if (!d) {
489 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
490 goto unwind_iov;
491 }
492 iov[i].iov_base = d;
493 iov[i].iov_len = iov_length;
494
495 if (data && xmit) {
496 int j;
497
498 for (j = 0; j < iov_length; j++)
499 d[j] = k++;
500 }
501 }
502
503 msg->msg_iov = iov;
504 msg->msg_iovlen = iov_count;
505
506 return 0;
507unwind_iov:
508 for (i--; i >= 0 ; i--)
509 free(msg->msg_iov[i].iov_base);
510 return -ENOMEM;
511}
512
513static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
514{
515 int i, j = 0, bytes_cnt = 0;
516 unsigned char k = 0;
517
518 for (i = 0; i < msg->msg_iovlen; i++) {
519 unsigned char *d = msg->msg_iov[i].iov_base;
520
521
522 if (i == 0 && txmsg_ktls_skb) {
523 if (msg->msg_iov[i].iov_len < 4)
524 return -EIO;
525 if (memcmp(d, "PASS", 4) != 0) {
526 fprintf(stderr,
527 "detected skb data error with skb ingress update @iov[%i]:%i \"%02x %02x %02x %02x\" != \"PASS\"\n",
528 i, 0, d[0], d[1], d[2], d[3]);
529 return -EIO;
530 }
531 j = 4;
532 }
533
534 for (; j < msg->msg_iov[i].iov_len && size; j++) {
535 if (d[j] != k++) {
536 fprintf(stderr,
537 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
538 i, j, d[j], k - 1, d[j+1], k);
539 return -EIO;
540 }
541 bytes_cnt++;
542 if (bytes_cnt == chunk_sz) {
543 k = 0;
544 bytes_cnt = 0;
545 }
546 size--;
547 }
548 }
549 return 0;
550}
551
552static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
553 struct msg_stats *s, bool tx,
554 struct sockmap_options *opt)
555{
556 struct msghdr msg = {0}, msg_peek = {0};
557 int err, i, flags = MSG_NOSIGNAL;
558 bool drop = opt->drop_expected;
559 bool data = opt->data_test;
560 int iov_alloc_length = iov_length;
561
562 if (!tx && opt->check_recved_len)
563 iov_alloc_length *= 2;
564
565 err = msg_alloc_iov(&msg, iov_count, iov_alloc_length, data, tx);
566 if (err)
567 goto out_errno;
568 if (peek_flag) {
569 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
570 if (err)
571 goto out_errno;
572 }
573
574 if (tx) {
575 clock_gettime(CLOCK_MONOTONIC, &s->start);
576 for (i = 0; i < cnt; i++) {
577 int sent;
578
579 errno = 0;
580 sent = sendmsg(fd, &msg, flags);
581
582 if (!drop && sent < 0) {
583 perror("sendmsg loop error");
584 goto out_errno;
585 } else if (drop && sent >= 0) {
586 fprintf(stderr,
587 "sendmsg loop error expected: %i errno %i\n",
588 sent, errno);
589 errno = -EIO;
590 goto out_errno;
591 }
592 if (sent > 0)
593 s->bytes_sent += sent;
594 }
595 clock_gettime(CLOCK_MONOTONIC, &s->end);
596 } else {
597 int slct, recvp = 0, recv, max_fd = fd;
598 float total_bytes, txmsg_pop_total;
599 int fd_flags = O_NONBLOCK;
600 struct timeval timeout;
601 fd_set w;
602
603 fcntl(fd, fd_flags);
604
605
606
607
608
609
610
611
612
613
614 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
615 if (txmsg_apply)
616 txmsg_pop_total = txmsg_pop * (total_bytes / txmsg_apply);
617 else
618 txmsg_pop_total = txmsg_pop * cnt;
619 total_bytes -= txmsg_pop_total;
620 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
621 if (err < 0)
622 perror("recv start time");
623 while (s->bytes_recvd < total_bytes) {
624 if (txmsg_cork) {
625 timeout.tv_sec = 0;
626 timeout.tv_usec = 300000;
627 } else {
628 timeout.tv_sec = 3;
629 timeout.tv_usec = 0;
630 }
631
632
633 FD_ZERO(&w);
634 FD_SET(fd, &w);
635
636 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
637 if (slct == -1) {
638 perror("select()");
639 clock_gettime(CLOCK_MONOTONIC, &s->end);
640 goto out_errno;
641 } else if (!slct) {
642 if (opt->verbose)
643 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
644 errno = -EIO;
645 clock_gettime(CLOCK_MONOTONIC, &s->end);
646 goto out_errno;
647 }
648
649 errno = 0;
650 if (peek_flag) {
651 flags |= MSG_PEEK;
652 recvp = recvmsg(fd, &msg_peek, flags);
653 if (recvp < 0) {
654 if (errno != EWOULDBLOCK) {
655 clock_gettime(CLOCK_MONOTONIC, &s->end);
656 goto out_errno;
657 }
658 }
659 flags = 0;
660 }
661
662 recv = recvmsg(fd, &msg, flags);
663 if (recv < 0) {
664 if (errno != EWOULDBLOCK) {
665 clock_gettime(CLOCK_MONOTONIC, &s->end);
666 perror("recv failed()");
667 goto out_errno;
668 }
669 }
670
671 s->bytes_recvd += recv;
672
673 if (opt->check_recved_len && s->bytes_recvd > total_bytes) {
674 errno = EMSGSIZE;
675 fprintf(stderr, "recv failed(), bytes_recvd:%zd, total_bytes:%f\n",
676 s->bytes_recvd, total_bytes);
677 goto out_errno;
678 }
679
680 if (data) {
681 int chunk_sz = opt->sendpage ?
682 iov_length * cnt :
683 iov_length * iov_count;
684
685 errno = msg_verify_data(&msg, recv, chunk_sz);
686 if (errno) {
687 perror("data verify msg failed");
688 goto out_errno;
689 }
690 if (recvp) {
691 errno = msg_verify_data(&msg_peek,
692 recvp,
693 chunk_sz);
694 if (errno) {
695 perror("data verify msg_peek failed");
696 goto out_errno;
697 }
698 }
699 }
700 }
701 clock_gettime(CLOCK_MONOTONIC, &s->end);
702 }
703
704 msg_free_iov(&msg);
705 msg_free_iov(&msg_peek);
706 return err;
707out_errno:
708 msg_free_iov(&msg);
709 msg_free_iov(&msg_peek);
710 return errno;
711}
712
713static float giga = 1000000000;
714
715static inline float sentBps(struct msg_stats s)
716{
717 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
718}
719
720static inline float recvdBps(struct msg_stats s)
721{
722 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
723}
724
725static int sendmsg_test(struct sockmap_options *opt)
726{
727 float sent_Bps = 0, recvd_Bps = 0;
728 int rx_fd, txpid, rxpid, err = 0;
729 struct msg_stats s = {0};
730 int iov_count = opt->iov_count;
731 int iov_buf = opt->iov_length;
732 int rx_status, tx_status;
733 int cnt = opt->rate;
734
735 errno = 0;
736
737 if (opt->base)
738 rx_fd = p1;
739 else
740 rx_fd = p2;
741
742 if (ktls) {
743
744
745
746
747 if (!txmsg_redir || txmsg_ingress) {
748 err = sockmap_init_ktls(opt->verbose, rx_fd);
749 if (err)
750 return err;
751 }
752 err = sockmap_init_ktls(opt->verbose, c1);
753 if (err)
754 return err;
755 }
756
757 rxpid = fork();
758 if (rxpid == 0) {
759 if (txmsg_pop || txmsg_start_pop)
760 iov_buf -= (txmsg_pop - txmsg_start_pop + 1);
761 if (opt->drop_expected || txmsg_ktls_skb_drop)
762 _exit(0);
763
764 if (!iov_buf)
765 _exit(0);
766
767 if (opt->sendpage)
768 iov_count = 1;
769 err = msg_loop(rx_fd, iov_count, iov_buf,
770 cnt, &s, false, opt);
771 if (opt->verbose > 1)
772 fprintf(stderr,
773 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
774 iov_count, iov_buf, cnt, err);
775 if (s.end.tv_sec - s.start.tv_sec) {
776 sent_Bps = sentBps(s);
777 recvd_Bps = recvdBps(s);
778 }
779 if (opt->verbose > 1)
780 fprintf(stdout,
781 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
782 s.bytes_sent, sent_Bps, sent_Bps/giga,
783 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
784 peek_flag ? "(peek_msg)" : "");
785 if (err && txmsg_cork)
786 err = 0;
787 exit(err ? 1 : 0);
788 } else if (rxpid == -1) {
789 perror("msg_loop_rx");
790 return errno;
791 }
792
793 txpid = fork();
794 if (txpid == 0) {
795 if (opt->sendpage)
796 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
797 else
798 err = msg_loop(c1, iov_count, iov_buf,
799 cnt, &s, true, opt);
800
801 if (err)
802 fprintf(stderr,
803 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
804 iov_count, iov_buf, cnt, err);
805 if (s.end.tv_sec - s.start.tv_sec) {
806 sent_Bps = sentBps(s);
807 recvd_Bps = recvdBps(s);
808 }
809 if (opt->verbose > 1)
810 fprintf(stdout,
811 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
812 s.bytes_sent, sent_Bps, sent_Bps/giga,
813 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
814 exit(err ? 1 : 0);
815 } else if (txpid == -1) {
816 perror("msg_loop_tx");
817 return errno;
818 }
819
820 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
821 assert(waitpid(txpid, &tx_status, 0) == txpid);
822 if (WIFEXITED(rx_status)) {
823 err = WEXITSTATUS(rx_status);
824 if (err) {
825 fprintf(stderr, "rx thread exited with err %d.\n", err);
826 goto out;
827 }
828 }
829 if (WIFEXITED(tx_status)) {
830 err = WEXITSTATUS(tx_status);
831 if (err)
832 fprintf(stderr, "tx thread exited with err %d.\n", err);
833 }
834out:
835 return err;
836}
837
838static int forever_ping_pong(int rate, struct sockmap_options *opt)
839{
840 struct timeval timeout;
841 char buf[1024] = {0};
842 int sc;
843
844 timeout.tv_sec = 10;
845 timeout.tv_usec = 0;
846
847
848 sc = send(c1, buf, sizeof(buf), 0);
849 if (sc < 0) {
850 perror("send failed()");
851 return sc;
852 }
853
854 do {
855 int s, rc, i, max_fd = p2;
856 fd_set w;
857
858
859 FD_ZERO(&w);
860 FD_SET(c1, &w);
861 FD_SET(c2, &w);
862 FD_SET(p1, &w);
863 FD_SET(p2, &w);
864
865 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
866 if (s == -1) {
867 perror("select()");
868 break;
869 } else if (!s) {
870 fprintf(stderr, "unexpected timeout\n");
871 break;
872 }
873
874 for (i = 0; i <= max_fd && s > 0; ++i) {
875 if (!FD_ISSET(i, &w))
876 continue;
877
878 s--;
879
880 rc = recv(i, buf, sizeof(buf), 0);
881 if (rc < 0) {
882 if (errno != EWOULDBLOCK) {
883 perror("recv failed()");
884 return rc;
885 }
886 }
887
888 if (rc == 0) {
889 close(i);
890 break;
891 }
892
893 sc = send(i, buf, rc, 0);
894 if (sc < 0) {
895 perror("send failed()");
896 return sc;
897 }
898 }
899
900 if (rate)
901 sleep(rate);
902
903 if (opt->verbose) {
904 printf(".");
905 fflush(stdout);
906
907 }
908 } while (running);
909
910 return 0;
911}
912
913enum {
914 SELFTESTS,
915 PING_PONG,
916 SENDMSG,
917 BASE,
918 BASE_SENDPAGE,
919 SENDPAGE,
920};
921
922static int run_options(struct sockmap_options *options, int cg_fd, int test)
923{
924 int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
925
926
927 if (test == BASE || test == BASE_SENDPAGE)
928 goto run;
929
930
931 if (!txmsg_omit_skb_parser) {
932 err = bpf_prog_attach(prog_fd[0], map_fd[0],
933 BPF_SK_SKB_STREAM_PARSER, 0);
934 if (err) {
935 fprintf(stderr,
936 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
937 prog_fd[0], map_fd[0], err, strerror(errno));
938 return err;
939 }
940 }
941
942 err = bpf_prog_attach(prog_fd[1], map_fd[0],
943 BPF_SK_SKB_STREAM_VERDICT, 0);
944 if (err) {
945 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
946 err, strerror(errno));
947 return err;
948 }
949
950
951 if (txmsg_ktls_skb) {
952 if (!txmsg_omit_skb_parser) {
953 err = bpf_prog_attach(prog_fd[0], map_fd[8],
954 BPF_SK_SKB_STREAM_PARSER, 0);
955 if (err) {
956 fprintf(stderr,
957 "ERROR: bpf_prog_attach (TLS sockmap %i->%i): %d (%s)\n",
958 prog_fd[0], map_fd[8], err, strerror(errno));
959 return err;
960 }
961 }
962
963 err = bpf_prog_attach(prog_fd[2], map_fd[8],
964 BPF_SK_SKB_STREAM_VERDICT, 0);
965 if (err) {
966 fprintf(stderr, "ERROR: bpf_prog_attach (TLS sockmap): %d (%s)\n",
967 err, strerror(errno));
968 return err;
969 }
970 }
971
972
973 err = bpf_prog_attach(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
974 if (err) {
975 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
976 err, strerror(errno));
977 return err;
978 }
979
980run:
981 err = sockmap_init_sockets(options->verbose);
982 if (err) {
983 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
984 goto out;
985 }
986
987
988 if (txmsg_pass)
989 tx_prog_fd = prog_fd[4];
990 else if (txmsg_redir)
991 tx_prog_fd = prog_fd[5];
992 else if (txmsg_apply)
993 tx_prog_fd = prog_fd[6];
994 else if (txmsg_cork)
995 tx_prog_fd = prog_fd[7];
996 else if (txmsg_drop)
997 tx_prog_fd = prog_fd[8];
998 else
999 tx_prog_fd = 0;
1000
1001 if (tx_prog_fd) {
1002 int redir_fd, i = 0;
1003
1004 err = bpf_prog_attach(tx_prog_fd,
1005 map_fd[1], BPF_SK_MSG_VERDICT, 0);
1006 if (err) {
1007 fprintf(stderr,
1008 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
1009 err, strerror(errno));
1010 goto out;
1011 }
1012
1013 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
1014 if (err) {
1015 fprintf(stderr,
1016 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1017 err, strerror(errno));
1018 goto out;
1019 }
1020
1021 if (txmsg_redir)
1022 redir_fd = c2;
1023 else
1024 redir_fd = c1;
1025
1026 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
1027 if (err) {
1028 fprintf(stderr,
1029 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
1030 err, strerror(errno));
1031 goto out;
1032 }
1033
1034 if (txmsg_apply) {
1035 err = bpf_map_update_elem(map_fd[3],
1036 &i, &txmsg_apply, BPF_ANY);
1037 if (err) {
1038 fprintf(stderr,
1039 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
1040 err, strerror(errno));
1041 goto out;
1042 }
1043 }
1044
1045 if (txmsg_cork) {
1046 err = bpf_map_update_elem(map_fd[4],
1047 &i, &txmsg_cork, BPF_ANY);
1048 if (err) {
1049 fprintf(stderr,
1050 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
1051 err, strerror(errno));
1052 goto out;
1053 }
1054 }
1055
1056 if (txmsg_start) {
1057 err = bpf_map_update_elem(map_fd[5],
1058 &i, &txmsg_start, BPF_ANY);
1059 if (err) {
1060 fprintf(stderr,
1061 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
1062 err, strerror(errno));
1063 goto out;
1064 }
1065 }
1066
1067 if (txmsg_end) {
1068 i = 1;
1069 err = bpf_map_update_elem(map_fd[5],
1070 &i, &txmsg_end, BPF_ANY);
1071 if (err) {
1072 fprintf(stderr,
1073 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
1074 err, strerror(errno));
1075 goto out;
1076 }
1077 }
1078
1079 if (txmsg_start_push) {
1080 i = 2;
1081 err = bpf_map_update_elem(map_fd[5],
1082 &i, &txmsg_start_push, BPF_ANY);
1083 if (err) {
1084 fprintf(stderr,
1085 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
1086 err, strerror(errno));
1087 goto out;
1088 }
1089 }
1090
1091 if (txmsg_end_push) {
1092 i = 3;
1093 err = bpf_map_update_elem(map_fd[5],
1094 &i, &txmsg_end_push, BPF_ANY);
1095 if (err) {
1096 fprintf(stderr,
1097 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
1098 txmsg_end_push, i, err, strerror(errno));
1099 goto out;
1100 }
1101 }
1102
1103 if (txmsg_start_pop) {
1104 i = 4;
1105 err = bpf_map_update_elem(map_fd[5],
1106 &i, &txmsg_start_pop, BPF_ANY);
1107 if (err) {
1108 fprintf(stderr,
1109 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
1110 txmsg_start_pop, i, err, strerror(errno));
1111 goto out;
1112 }
1113 } else {
1114 i = 4;
1115 bpf_map_update_elem(map_fd[5],
1116 &i, &txmsg_start_pop, BPF_ANY);
1117 }
1118
1119 if (txmsg_pop) {
1120 i = 5;
1121 err = bpf_map_update_elem(map_fd[5],
1122 &i, &txmsg_pop, BPF_ANY);
1123 if (err) {
1124 fprintf(stderr,
1125 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
1126 txmsg_pop, i, err, strerror(errno));
1127 goto out;
1128 }
1129 } else {
1130 i = 5;
1131 bpf_map_update_elem(map_fd[5],
1132 &i, &txmsg_pop, BPF_ANY);
1133
1134 }
1135
1136 if (txmsg_ingress) {
1137 int in = BPF_F_INGRESS;
1138
1139 i = 0;
1140 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
1141 if (err) {
1142 fprintf(stderr,
1143 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1144 err, strerror(errno));
1145 }
1146 i = 1;
1147 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1148 if (err) {
1149 fprintf(stderr,
1150 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1151 err, strerror(errno));
1152 }
1153 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1154 if (err) {
1155 fprintf(stderr,
1156 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1157 err, strerror(errno));
1158 }
1159
1160 i = 2;
1161 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1162 if (err) {
1163 fprintf(stderr,
1164 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1165 err, strerror(errno));
1166 }
1167 }
1168
1169 if (txmsg_ktls_skb) {
1170 int ingress = BPF_F_INGRESS;
1171
1172 i = 0;
1173 err = bpf_map_update_elem(map_fd[8], &i, &p2, BPF_ANY);
1174 if (err) {
1175 fprintf(stderr,
1176 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1177 err, strerror(errno));
1178 }
1179
1180 if (txmsg_ktls_skb_redir) {
1181 i = 1;
1182 err = bpf_map_update_elem(map_fd[7],
1183 &i, &ingress, BPF_ANY);
1184 if (err) {
1185 fprintf(stderr,
1186 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1187 err, strerror(errno));
1188 }
1189 }
1190
1191 if (txmsg_ktls_skb_drop) {
1192 i = 1;
1193 err = bpf_map_update_elem(map_fd[7], &i, &i, BPF_ANY);
1194 }
1195 }
1196
1197 if (txmsg_redir_skb) {
1198 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1199 p2 : p1;
1200 int ingress = BPF_F_INGRESS;
1201
1202 i = 0;
1203 err = bpf_map_update_elem(map_fd[7],
1204 &i, &ingress, BPF_ANY);
1205 if (err) {
1206 fprintf(stderr,
1207 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1208 err, strerror(errno));
1209 }
1210
1211 i = 3;
1212 err = bpf_map_update_elem(map_fd[0], &i, &skb_fd, BPF_ANY);
1213 if (err) {
1214 fprintf(stderr,
1215 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1216 err, strerror(errno));
1217 }
1218 }
1219 }
1220
1221 if (skb_use_parser) {
1222 i = 2;
1223 err = bpf_map_update_elem(map_fd[7], &i, &skb_use_parser, BPF_ANY);
1224 }
1225
1226 if (txmsg_drop)
1227 options->drop_expected = true;
1228
1229 if (test == PING_PONG)
1230 err = forever_ping_pong(options->rate, options);
1231 else if (test == SENDMSG) {
1232 options->base = false;
1233 options->sendpage = false;
1234 err = sendmsg_test(options);
1235 } else if (test == SENDPAGE) {
1236 options->base = false;
1237 options->sendpage = true;
1238 err = sendmsg_test(options);
1239 } else if (test == BASE) {
1240 options->base = true;
1241 options->sendpage = false;
1242 err = sendmsg_test(options);
1243 } else if (test == BASE_SENDPAGE) {
1244 options->base = true;
1245 options->sendpage = true;
1246 err = sendmsg_test(options);
1247 } else
1248 fprintf(stderr, "unknown test\n");
1249out:
1250
1251 bpf_prog_detach2(prog_fd[3], cg_fd, BPF_CGROUP_SOCK_OPS);
1252 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1253 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1254 bpf_prog_detach2(prog_fd[0], map_fd[8], BPF_SK_SKB_STREAM_PARSER);
1255 bpf_prog_detach2(prog_fd[2], map_fd[8], BPF_SK_SKB_STREAM_VERDICT);
1256
1257 if (tx_prog_fd >= 0)
1258 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1259
1260 for (i = 0; i < 8; i++) {
1261 key = next_key = 0;
1262 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1263 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1264 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1265 key = next_key;
1266 }
1267 }
1268
1269 close(s1);
1270 close(s2);
1271 close(p1);
1272 close(p2);
1273 close(c1);
1274 close(c2);
1275 return err;
1276}
1277
1278static char *test_to_str(int test)
1279{
1280 switch (test) {
1281 case SENDMSG:
1282 return "sendmsg";
1283 case SENDPAGE:
1284 return "sendpage";
1285 }
1286 return "unknown";
1287}
1288
1289static void append_str(char *dst, const char *src, size_t dst_cap)
1290{
1291 size_t avail = dst_cap - strlen(dst);
1292
1293 if (avail <= 1)
1294 return;
1295
1296 strncat(dst, src, avail - 1);
1297}
1298
1299#define OPTSTRING 60
1300static void test_options(char *options)
1301{
1302 char tstr[OPTSTRING];
1303
1304 memset(options, 0, OPTSTRING);
1305
1306 if (txmsg_pass)
1307 append_str(options, "pass,", OPTSTRING);
1308 if (txmsg_redir)
1309 append_str(options, "redir,", OPTSTRING);
1310 if (txmsg_drop)
1311 append_str(options, "drop,", OPTSTRING);
1312 if (txmsg_apply) {
1313 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1314 append_str(options, tstr, OPTSTRING);
1315 }
1316 if (txmsg_cork) {
1317 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1318 append_str(options, tstr, OPTSTRING);
1319 }
1320 if (txmsg_start) {
1321 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1322 append_str(options, tstr, OPTSTRING);
1323 }
1324 if (txmsg_end) {
1325 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1326 append_str(options, tstr, OPTSTRING);
1327 }
1328 if (txmsg_start_pop) {
1329 snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1330 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1331 append_str(options, tstr, OPTSTRING);
1332 }
1333 if (txmsg_ingress)
1334 append_str(options, "ingress,", OPTSTRING);
1335 if (txmsg_redir_skb)
1336 append_str(options, "redir_skb,", OPTSTRING);
1337 if (txmsg_ktls_skb)
1338 append_str(options, "ktls_skb,", OPTSTRING);
1339 if (ktls)
1340 append_str(options, "ktls,", OPTSTRING);
1341 if (peek_flag)
1342 append_str(options, "peek,", OPTSTRING);
1343}
1344
1345static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1346{
1347 char *options = calloc(OPTSTRING, sizeof(char));
1348 int err;
1349
1350 if (test == SENDPAGE)
1351 opt->sendpage = true;
1352 else
1353 opt->sendpage = false;
1354
1355 if (txmsg_drop)
1356 opt->drop_expected = true;
1357 else
1358 opt->drop_expected = false;
1359
1360 test_options(options);
1361
1362 if (opt->verbose) {
1363 fprintf(stdout,
1364 " [TEST %i]: (%i, %i, %i, %s, %s): ",
1365 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1366 test_to_str(test), options);
1367 fflush(stdout);
1368 }
1369 err = run_options(opt, cgrp, test);
1370 if (opt->verbose)
1371 fprintf(stdout, " %s\n", !err ? "PASS" : "FAILED");
1372 test_cnt++;
1373 !err ? passed++ : failed++;
1374 free(options);
1375 return err;
1376}
1377
1378static void test_exec(int cgrp, struct sockmap_options *opt)
1379{
1380 int type = strcmp(opt->map, BPF_SOCKMAP_FILENAME);
1381 int err;
1382
1383 if (type == 0) {
1384 test_start();
1385 err = __test_exec(cgrp, SENDMSG, opt);
1386 if (err)
1387 test_fail();
1388 } else {
1389 test_start();
1390 err = __test_exec(cgrp, SENDPAGE, opt);
1391 if (err)
1392 test_fail();
1393 }
1394}
1395
1396static void test_send_one(struct sockmap_options *opt, int cgrp)
1397{
1398 opt->iov_length = 1;
1399 opt->iov_count = 1;
1400 opt->rate = 1;
1401 test_exec(cgrp, opt);
1402
1403 opt->iov_length = 1;
1404 opt->iov_count = 1024;
1405 opt->rate = 1;
1406 test_exec(cgrp, opt);
1407
1408 opt->iov_length = 1024;
1409 opt->iov_count = 1;
1410 opt->rate = 1;
1411 test_exec(cgrp, opt);
1412
1413}
1414
1415static void test_send_many(struct sockmap_options *opt, int cgrp)
1416{
1417 opt->iov_length = 3;
1418 opt->iov_count = 1;
1419 opt->rate = 512;
1420 test_exec(cgrp, opt);
1421
1422 opt->rate = 100;
1423 opt->iov_count = 1;
1424 opt->iov_length = 5;
1425 test_exec(cgrp, opt);
1426}
1427
1428static void test_send_large(struct sockmap_options *opt, int cgrp)
1429{
1430 opt->iov_length = 256;
1431 opt->iov_count = 1024;
1432 opt->rate = 2;
1433 test_exec(cgrp, opt);
1434}
1435
1436static void test_send(struct sockmap_options *opt, int cgrp)
1437{
1438 test_send_one(opt, cgrp);
1439 test_send_many(opt, cgrp);
1440 test_send_large(opt, cgrp);
1441 sched_yield();
1442}
1443
1444static void test_txmsg_pass(int cgrp, struct sockmap_options *opt)
1445{
1446
1447 txmsg_pass = 1;
1448 test_send(opt, cgrp);
1449}
1450
1451static void test_txmsg_redir(int cgrp, struct sockmap_options *opt)
1452{
1453 txmsg_redir = 1;
1454 test_send(opt, cgrp);
1455}
1456
1457static void test_txmsg_drop(int cgrp, struct sockmap_options *opt)
1458{
1459 txmsg_drop = 1;
1460 test_send(opt, cgrp);
1461}
1462
1463static void test_txmsg_ingress_redir(int cgrp, struct sockmap_options *opt)
1464{
1465 txmsg_pass = txmsg_drop = 0;
1466 txmsg_ingress = txmsg_redir = 1;
1467 test_send(opt, cgrp);
1468}
1469
1470static void test_txmsg_skb(int cgrp, struct sockmap_options *opt)
1471{
1472 bool data = opt->data_test;
1473 int k = ktls;
1474
1475 opt->data_test = true;
1476 ktls = 1;
1477
1478 txmsg_pass = txmsg_drop = 0;
1479 txmsg_ingress = txmsg_redir = 0;
1480 txmsg_ktls_skb = 1;
1481 txmsg_pass = 1;
1482
1483
1484
1485
1486
1487 opt->iov_length = 100;
1488 opt->iov_count = 1;
1489 opt->rate = 1;
1490 test_exec(cgrp, opt);
1491
1492 txmsg_ktls_skb_drop = 1;
1493 test_exec(cgrp, opt);
1494
1495 txmsg_ktls_skb_drop = 0;
1496 txmsg_ktls_skb_redir = 1;
1497 test_exec(cgrp, opt);
1498 txmsg_ktls_skb_redir = 0;
1499
1500
1501 txmsg_omit_skb_parser = 1;
1502 ktls = 0;
1503 txmsg_ktls_skb = 0;
1504 test_exec(cgrp, opt);
1505
1506 txmsg_ktls_skb_drop = 1;
1507 test_exec(cgrp, opt);
1508 txmsg_ktls_skb_drop = 0;
1509
1510 txmsg_ktls_skb_redir = 1;
1511 test_exec(cgrp, opt);
1512
1513 ktls = 1;
1514 test_exec(cgrp, opt);
1515 txmsg_omit_skb_parser = 0;
1516
1517 opt->data_test = data;
1518 ktls = k;
1519}
1520
1521
1522
1523
1524
1525
1526
1527
1528static void test_txmsg_cork_hangs(int cgrp, struct sockmap_options *opt)
1529{
1530 txmsg_pass = 1;
1531 txmsg_redir = 0;
1532 txmsg_cork = 4097;
1533 txmsg_apply = 4097;
1534 test_send_large(opt, cgrp);
1535
1536 txmsg_pass = 0;
1537 txmsg_redir = 1;
1538 txmsg_apply = 0;
1539 txmsg_cork = 4097;
1540 test_send_large(opt, cgrp);
1541
1542 txmsg_pass = 0;
1543 txmsg_redir = 1;
1544 txmsg_apply = 4097;
1545 txmsg_cork = 4097;
1546 test_send_large(opt, cgrp);
1547}
1548
1549static void test_txmsg_pull(int cgrp, struct sockmap_options *opt)
1550{
1551
1552 txmsg_start = 1;
1553 txmsg_end = 2;
1554 test_send(opt, cgrp);
1555
1556
1557 txmsg_start = 4096;
1558 txmsg_end = 9182;
1559 test_send_large(opt, cgrp);
1560
1561
1562 txmsg_redir = 0;
1563 txmsg_start = 1;
1564 txmsg_end = 2;
1565 test_send(opt, cgrp);
1566
1567
1568 txmsg_redir = 0;
1569 txmsg_cork = 512;
1570 txmsg_start = 1;
1571 txmsg_end = 2;
1572 test_send_many(opt, cgrp);
1573
1574
1575 txmsg_redir = 1;
1576 txmsg_cork = 512;
1577 txmsg_start = 1;
1578 txmsg_end = 2;
1579 test_send_many(opt, cgrp);
1580}
1581
1582static void test_txmsg_pop(int cgrp, struct sockmap_options *opt)
1583{
1584
1585 txmsg_start_pop = 1;
1586 txmsg_pop = 2;
1587 test_send_many(opt, cgrp);
1588
1589
1590 txmsg_start_pop = 4096;
1591 txmsg_pop = 4096;
1592 test_send_large(opt, cgrp);
1593
1594
1595 txmsg_redir = 1;
1596 txmsg_start_pop = 1;
1597 txmsg_pop = 2;
1598 test_send_many(opt, cgrp);
1599
1600
1601 txmsg_redir = 0;
1602 txmsg_cork = 512;
1603 txmsg_start_pop = 1;
1604 txmsg_pop = 2;
1605 test_send_many(opt, cgrp);
1606
1607
1608 txmsg_redir = 1;
1609 txmsg_cork = 4;
1610 txmsg_start_pop = 1;
1611 txmsg_pop = 2;
1612 test_send_many(opt, cgrp);
1613}
1614
1615static void test_txmsg_push(int cgrp, struct sockmap_options *opt)
1616{
1617
1618 txmsg_start_push = 1;
1619 txmsg_end_push = 1;
1620 test_send(opt, cgrp);
1621
1622
1623 txmsg_start_push = 4096;
1624 txmsg_end_push = 4096;
1625 test_send_large(opt, cgrp);
1626
1627
1628 txmsg_redir = 1;
1629 txmsg_start_push = 1;
1630 txmsg_end_push = 2;
1631 test_send_many(opt, cgrp);
1632
1633
1634 txmsg_redir = 0;
1635 txmsg_cork = 512;
1636 txmsg_start_push = 1;
1637 txmsg_end_push = 2;
1638 test_send_many(opt, cgrp);
1639}
1640
1641static void test_txmsg_push_pop(int cgrp, struct sockmap_options *opt)
1642{
1643 txmsg_start_push = 1;
1644 txmsg_end_push = 10;
1645 txmsg_start_pop = 5;
1646 txmsg_pop = 4;
1647 test_send_large(opt, cgrp);
1648}
1649
1650static void test_txmsg_apply(int cgrp, struct sockmap_options *opt)
1651{
1652 txmsg_pass = 1;
1653 txmsg_redir = 0;
1654 txmsg_apply = 1;
1655 txmsg_cork = 0;
1656 test_send_one(opt, cgrp);
1657
1658 txmsg_pass = 0;
1659 txmsg_redir = 1;
1660 txmsg_apply = 1;
1661 txmsg_cork = 0;
1662 test_send_one(opt, cgrp);
1663
1664 txmsg_pass = 1;
1665 txmsg_redir = 0;
1666 txmsg_apply = 1024;
1667 txmsg_cork = 0;
1668 test_send_large(opt, cgrp);
1669
1670 txmsg_pass = 0;
1671 txmsg_redir = 1;
1672 txmsg_apply = 1024;
1673 txmsg_cork = 0;
1674 test_send_large(opt, cgrp);
1675}
1676
1677static void test_txmsg_cork(int cgrp, struct sockmap_options *opt)
1678{
1679 txmsg_pass = 1;
1680 txmsg_redir = 0;
1681 txmsg_apply = 0;
1682 txmsg_cork = 1;
1683 test_send(opt, cgrp);
1684
1685 txmsg_pass = 1;
1686 txmsg_redir = 0;
1687 txmsg_apply = 1;
1688 txmsg_cork = 1;
1689 test_send(opt, cgrp);
1690}
1691
1692static void test_txmsg_ingress_parser(int cgrp, struct sockmap_options *opt)
1693{
1694 txmsg_pass = 1;
1695 skb_use_parser = 512;
1696 if (ktls == 1)
1697 skb_use_parser = 570;
1698 opt->iov_length = 256;
1699 opt->iov_count = 1;
1700 opt->rate = 2;
1701 test_exec(cgrp, opt);
1702}
1703
1704static void test_txmsg_ingress_parser2(int cgrp, struct sockmap_options *opt)
1705{
1706 if (ktls == 1)
1707 return;
1708 skb_use_parser = 10;
1709 opt->iov_length = 20;
1710 opt->iov_count = 1;
1711 opt->rate = 1;
1712 opt->check_recved_len = true;
1713 test_exec(cgrp, opt);
1714 opt->check_recved_len = false;
1715}
1716
1717char *map_names[] = {
1718 "sock_map",
1719 "sock_map_txmsg",
1720 "sock_map_redir",
1721 "sock_apply_bytes",
1722 "sock_cork_bytes",
1723 "sock_bytes",
1724 "sock_redir_flags",
1725 "sock_skb_opts",
1726 "tls_sock_map",
1727};
1728
1729int prog_attach_type[] = {
1730 BPF_SK_SKB_STREAM_PARSER,
1731 BPF_SK_SKB_STREAM_VERDICT,
1732 BPF_SK_SKB_STREAM_VERDICT,
1733 BPF_CGROUP_SOCK_OPS,
1734 BPF_SK_MSG_VERDICT,
1735 BPF_SK_MSG_VERDICT,
1736 BPF_SK_MSG_VERDICT,
1737 BPF_SK_MSG_VERDICT,
1738 BPF_SK_MSG_VERDICT,
1739 BPF_SK_MSG_VERDICT,
1740 BPF_SK_MSG_VERDICT,
1741};
1742
1743int prog_type[] = {
1744 BPF_PROG_TYPE_SK_SKB,
1745 BPF_PROG_TYPE_SK_SKB,
1746 BPF_PROG_TYPE_SK_SKB,
1747 BPF_PROG_TYPE_SOCK_OPS,
1748 BPF_PROG_TYPE_SK_MSG,
1749 BPF_PROG_TYPE_SK_MSG,
1750 BPF_PROG_TYPE_SK_MSG,
1751 BPF_PROG_TYPE_SK_MSG,
1752 BPF_PROG_TYPE_SK_MSG,
1753 BPF_PROG_TYPE_SK_MSG,
1754 BPF_PROG_TYPE_SK_MSG,
1755};
1756
1757static int populate_progs(char *bpf_file)
1758{
1759 struct bpf_program *prog;
1760 struct bpf_object *obj;
1761 int i = 0;
1762 long err;
1763
1764 obj = bpf_object__open(bpf_file);
1765 err = libbpf_get_error(obj);
1766 if (err) {
1767 char err_buf[256];
1768
1769 libbpf_strerror(err, err_buf, sizeof(err_buf));
1770 printf("Unable to load eBPF objects in file '%s' : %s\n",
1771 bpf_file, err_buf);
1772 return -1;
1773 }
1774
1775 bpf_object__for_each_program(prog, obj) {
1776 bpf_program__set_type(prog, prog_type[i]);
1777 bpf_program__set_expected_attach_type(prog,
1778 prog_attach_type[i]);
1779 i++;
1780 }
1781
1782 i = bpf_object__load(obj);
1783 i = 0;
1784 bpf_object__for_each_program(prog, obj) {
1785 prog_fd[i] = bpf_program__fd(prog);
1786 i++;
1787 }
1788
1789 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) {
1790 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1791 map_fd[i] = bpf_map__fd(maps[i]);
1792 if (map_fd[i] < 0) {
1793 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1794 map_fd[i], strerror(errno));
1795 return -1;
1796 }
1797 }
1798
1799 return 0;
1800}
1801
1802struct _test test[] = {
1803 {"txmsg test passthrough", test_txmsg_pass},
1804 {"txmsg test redirect", test_txmsg_redir},
1805 {"txmsg test drop", test_txmsg_drop},
1806 {"txmsg test ingress redirect", test_txmsg_ingress_redir},
1807 {"txmsg test skb", test_txmsg_skb},
1808 {"txmsg test apply", test_txmsg_apply},
1809 {"txmsg test cork", test_txmsg_cork},
1810 {"txmsg test hanging corks", test_txmsg_cork_hangs},
1811 {"txmsg test push_data", test_txmsg_push},
1812 {"txmsg test pull-data", test_txmsg_pull},
1813 {"txmsg test pop-data", test_txmsg_pop},
1814 {"txmsg test push/pop data", test_txmsg_push_pop},
1815 {"txmsg test ingress parser", test_txmsg_ingress_parser},
1816 {"txmsg test ingress parser2", test_txmsg_ingress_parser2},
1817};
1818
1819static int check_whitelist(struct _test *t, struct sockmap_options *opt)
1820{
1821 char *entry, *ptr;
1822
1823 if (!opt->whitelist)
1824 return 0;
1825 ptr = strdup(opt->whitelist);
1826 if (!ptr)
1827 return -ENOMEM;
1828 entry = strtok(ptr, ",");
1829 while (entry) {
1830 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1831 strstr(opt->map, entry) != 0 ||
1832 strstr(t->title, entry) != 0)
1833 return 0;
1834 entry = strtok(NULL, ",");
1835 }
1836 return -EINVAL;
1837}
1838
1839static int check_blacklist(struct _test *t, struct sockmap_options *opt)
1840{
1841 char *entry, *ptr;
1842
1843 if (!opt->blacklist)
1844 return -EINVAL;
1845 ptr = strdup(opt->blacklist);
1846 if (!ptr)
1847 return -ENOMEM;
1848 entry = strtok(ptr, ",");
1849 while (entry) {
1850 if ((opt->prepend && strstr(opt->prepend, entry) != 0) ||
1851 strstr(opt->map, entry) != 0 ||
1852 strstr(t->title, entry) != 0)
1853 return 0;
1854 entry = strtok(NULL, ",");
1855 }
1856 return -EINVAL;
1857}
1858
1859static int __test_selftests(int cg_fd, struct sockmap_options *opt)
1860{
1861 int i, err;
1862
1863 err = populate_progs(opt->map);
1864 if (err < 0) {
1865 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1866 return err;
1867 }
1868
1869
1870 for (i = 0; i < sizeof(test)/sizeof(struct _test); i++) {
1871 struct _test t = test[i];
1872
1873 if (check_whitelist(&t, opt) != 0)
1874 continue;
1875 if (check_blacklist(&t, opt) == 0)
1876 continue;
1877
1878 test_start_subtest(&t, opt);
1879 t.tester(cg_fd, opt);
1880 test_end_subtest();
1881 }
1882
1883 return err;
1884}
1885
1886static void test_selftests_sockmap(int cg_fd, struct sockmap_options *opt)
1887{
1888 opt->map = BPF_SOCKMAP_FILENAME;
1889 __test_selftests(cg_fd, opt);
1890}
1891
1892static void test_selftests_sockhash(int cg_fd, struct sockmap_options *opt)
1893{
1894 opt->map = BPF_SOCKHASH_FILENAME;
1895 __test_selftests(cg_fd, opt);
1896}
1897
1898static void test_selftests_ktls(int cg_fd, struct sockmap_options *opt)
1899{
1900 opt->map = BPF_SOCKHASH_FILENAME;
1901 opt->prepend = "ktls";
1902 ktls = 1;
1903 __test_selftests(cg_fd, opt);
1904 ktls = 0;
1905}
1906
1907static int test_selftest(int cg_fd, struct sockmap_options *opt)
1908{
1909
1910 test_selftests_sockmap(cg_fd, opt);
1911 test_selftests_sockhash(cg_fd, opt);
1912 test_selftests_ktls(cg_fd, opt);
1913 test_print_results();
1914 return 0;
1915}
1916
1917int main(int argc, char **argv)
1918{
1919 int iov_count = 1, length = 1024, rate = 1;
1920 struct sockmap_options options = {0};
1921 int opt, longindex, err, cg_fd = 0;
1922 char *bpf_file = BPF_SOCKMAP_FILENAME;
1923 int test = SELFTESTS;
1924 bool cg_created = 0;
1925
1926 while ((opt = getopt_long(argc, argv, ":dhv:c:r:i:l:t:p:q:n:b:",
1927 long_options, &longindex)) != -1) {
1928 switch (opt) {
1929 case 's':
1930 txmsg_start = atoi(optarg);
1931 break;
1932 case 'e':
1933 txmsg_end = atoi(optarg);
1934 break;
1935 case 'p':
1936 txmsg_start_push = atoi(optarg);
1937 break;
1938 case 'q':
1939 txmsg_end_push = atoi(optarg);
1940 break;
1941 case 'w':
1942 txmsg_start_pop = atoi(optarg);
1943 break;
1944 case 'x':
1945 txmsg_pop = atoi(optarg);
1946 break;
1947 case 'a':
1948 txmsg_apply = atoi(optarg);
1949 break;
1950 case 'k':
1951 txmsg_cork = atoi(optarg);
1952 break;
1953 case 'c':
1954 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1955 if (cg_fd < 0) {
1956 fprintf(stderr,
1957 "ERROR: (%i) open cg path failed: %s\n",
1958 cg_fd, optarg);
1959 return cg_fd;
1960 }
1961 break;
1962 case 'r':
1963 rate = atoi(optarg);
1964 break;
1965 case 'v':
1966 options.verbose = 1;
1967 if (optarg)
1968 options.verbose = atoi(optarg);
1969 break;
1970 case 'i':
1971 iov_count = atoi(optarg);
1972 break;
1973 case 'l':
1974 length = atoi(optarg);
1975 break;
1976 case 'd':
1977 options.data_test = true;
1978 break;
1979 case 't':
1980 if (strcmp(optarg, "ping") == 0) {
1981 test = PING_PONG;
1982 } else if (strcmp(optarg, "sendmsg") == 0) {
1983 test = SENDMSG;
1984 } else if (strcmp(optarg, "base") == 0) {
1985 test = BASE;
1986 } else if (strcmp(optarg, "base_sendpage") == 0) {
1987 test = BASE_SENDPAGE;
1988 } else if (strcmp(optarg, "sendpage") == 0) {
1989 test = SENDPAGE;
1990 } else {
1991 usage(argv);
1992 return -1;
1993 }
1994 break;
1995 case 'n':
1996 options.whitelist = strdup(optarg);
1997 if (!options.whitelist)
1998 return -ENOMEM;
1999 break;
2000 case 'b':
2001 options.blacklist = strdup(optarg);
2002 if (!options.blacklist)
2003 return -ENOMEM;
2004 case 0:
2005 break;
2006 case 'h':
2007 default:
2008 usage(argv);
2009 return -1;
2010 }
2011 }
2012
2013 if (!cg_fd) {
2014 cg_fd = cgroup_setup_and_join(CG_PATH);
2015 if (cg_fd < 0)
2016 return cg_fd;
2017 cg_created = 1;
2018 }
2019
2020 if (test == SELFTESTS) {
2021 err = test_selftest(cg_fd, &options);
2022 goto out;
2023 }
2024
2025 err = populate_progs(bpf_file);
2026 if (err) {
2027 fprintf(stderr, "populate program: (%s) %s\n",
2028 bpf_file, strerror(errno));
2029 return 1;
2030 }
2031 running = 1;
2032
2033
2034 signal(SIGINT, running_handler);
2035
2036 options.iov_count = iov_count;
2037 options.iov_length = length;
2038 options.rate = rate;
2039
2040 err = run_options(&options, cg_fd, test);
2041out:
2042 if (options.whitelist)
2043 free(options.whitelist);
2044 if (options.blacklist)
2045 free(options.blacklist);
2046 if (cg_created)
2047 cleanup_cgroup_environment();
2048 close(cg_fd);
2049 return err;
2050}
2051
2052void running_handler(int a)
2053{
2054 running = 0;
2055}
2056
