1
2
3#include <stdio.h>
4#include <stdlib.h>
5#include <sys/socket.h>
6#include <sys/ioctl.h>
7#include <sys/select.h>
8#include <netinet/in.h>
9#include <arpa/inet.h>
10#include <unistd.h>
11#include <string.h>
12#include <errno.h>
13#include <stdbool.h>
14#include <signal.h>
15#include <fcntl.h>
16#include <sys/wait.h>
17#include <time.h>
18#include <sched.h>
19
20#include <sys/time.h>
21#include <sys/resource.h>
22#include <sys/types.h>
23#include <sys/sendfile.h>
24
25#include <linux/netlink.h>
26#include <linux/socket.h>
27#include <linux/sock_diag.h>
28#include <linux/bpf.h>
29#include <linux/if_link.h>
30#include <linux/tls.h>
31#include <assert.h>
32#include <libgen.h>
33
34#include <getopt.h>
35
36#include <bpf/bpf.h>
37#include <bpf/libbpf.h>
38
39#include "bpf_util.h"
40#include "bpf_rlimit.h"
41#include "cgroup_helpers.h"
42
43int running;
44static void running_handler(int a);
45
46#ifndef TCP_ULP
47# define TCP_ULP 31
48#endif
49#ifndef SOL_TLS
50# define SOL_TLS 282
51#endif
52
53
54#define S1_PORT 10000
55#define S2_PORT 10001
56
57#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
58#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
59#define CG_PATH "/sockmap"
60
61
62int s1, s2, c1, c2, p1, p2;
63int test_cnt;
64int passed;
65int failed;
66int map_fd[8];
67struct bpf_map *maps[8];
68int prog_fd[11];
69
70int txmsg_pass;
71int txmsg_noisy;
72int txmsg_redir;
73int txmsg_redir_noisy;
74int txmsg_drop;
75int txmsg_apply;
76int txmsg_cork;
77int txmsg_start;
78int txmsg_end;
79int txmsg_start_push;
80int txmsg_end_push;
81int txmsg_start_pop;
82int txmsg_pop;
83int txmsg_ingress;
84int txmsg_skb;
85int ktls;
86int peek_flag;
87
88static const struct option long_options[] = {
89 {"help", no_argument, NULL, 'h' },
90 {"cgroup", required_argument, NULL, 'c' },
91 {"rate", required_argument, NULL, 'r' },
92 {"verbose", no_argument, NULL, 'v' },
93 {"iov_count", required_argument, NULL, 'i' },
94 {"length", required_argument, NULL, 'l' },
95 {"test", required_argument, NULL, 't' },
96 {"data_test", no_argument, NULL, 'd' },
97 {"txmsg", no_argument, &txmsg_pass, 1 },
98 {"txmsg_noisy", no_argument, &txmsg_noisy, 1 },
99 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
100 {"txmsg_redir_noisy", no_argument, &txmsg_redir_noisy, 1},
101 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
102 {"txmsg_apply", required_argument, NULL, 'a'},
103 {"txmsg_cork", required_argument, NULL, 'k'},
104 {"txmsg_start", required_argument, NULL, 's'},
105 {"txmsg_end", required_argument, NULL, 'e'},
106 {"txmsg_start_push", required_argument, NULL, 'p'},
107 {"txmsg_end_push", required_argument, NULL, 'q'},
108 {"txmsg_start_pop", required_argument, NULL, 'w'},
109 {"txmsg_pop", required_argument, NULL, 'x'},
110 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
111 {"txmsg_skb", no_argument, &txmsg_skb, 1 },
112 {"ktls", no_argument, &ktls, 1 },
113 {"peek", no_argument, &peek_flag, 1 },
114 {0, 0, NULL, 0 }
115};
116
117static void usage(char *argv[])
118{
119 int i;
120
121 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
122 printf(" options:\n");
123 for (i = 0; long_options[i].name != 0; i++) {
124 printf(" --%-12s", long_options[i].name);
125 if (long_options[i].flag != NULL)
126 printf(" flag (internal value:%d)\n",
127 *long_options[i].flag);
128 else
129 printf(" -%c\n", long_options[i].val);
130 }
131 printf("\n");
132}
133
134char *sock_to_string(int s)
135{
136 if (s == c1)
137 return "client1";
138 else if (s == c2)
139 return "client2";
140 else if (s == s1)
141 return "server1";
142 else if (s == s2)
143 return "server2";
144 else if (s == p1)
145 return "peer1";
146 else if (s == p2)
147 return "peer2";
148 else
149 return "unknown";
150}
151
152static int sockmap_init_ktls(int verbose, int s)
153{
154 struct tls12_crypto_info_aes_gcm_128 tls_tx = {
155 .info = {
156 .version = TLS_1_2_VERSION,
157 .cipher_type = TLS_CIPHER_AES_GCM_128,
158 },
159 };
160 struct tls12_crypto_info_aes_gcm_128 tls_rx = {
161 .info = {
162 .version = TLS_1_2_VERSION,
163 .cipher_type = TLS_CIPHER_AES_GCM_128,
164 },
165 };
166 int so_buf = 6553500;
167 int err;
168
169 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
170 if (err) {
171 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
172 return -EINVAL;
173 }
174 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
175 if (err) {
176 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
177 return -EINVAL;
178 }
179 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
180 if (err) {
181 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
182 return -EINVAL;
183 }
184 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
185 if (err) {
186 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
187 return -EINVAL;
188 }
189 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
190 if (err) {
191 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
192 return -EINVAL;
193 }
194
195 if (verbose)
196 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
197 return 0;
198}
199static int sockmap_init_sockets(int verbose)
200{
201 int i, err, one = 1;
202 struct sockaddr_in addr;
203 int *fds[4] = {&s1, &s2, &c1, &c2};
204
205 s1 = s2 = p1 = p2 = c1 = c2 = 0;
206
207
208 for (i = 0; i < 4; i++) {
209 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
210 if (*fds[i] < 0) {
211 perror("socket s1 failed()");
212 return errno;
213 }
214 }
215
216
217 for (i = 0; i < 2; i++) {
218 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
219 (char *)&one, sizeof(one));
220 if (err) {
221 perror("setsockopt failed()");
222 return errno;
223 }
224 }
225
226
227 for (i = 0; i < 2; i++) {
228 err = ioctl(*fds[i], FIONBIO, (char *)&one);
229 if (err < 0) {
230 perror("ioctl s1 failed()");
231 return errno;
232 }
233 }
234
235
236 memset(&addr, 0, sizeof(struct sockaddr_in));
237 addr.sin_family = AF_INET;
238 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
239
240 addr.sin_port = htons(S1_PORT);
241 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
242 if (err < 0) {
243 perror("bind s1 failed()\n");
244 return errno;
245 }
246
247 addr.sin_port = htons(S2_PORT);
248 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
249 if (err < 0) {
250 perror("bind s2 failed()\n");
251 return errno;
252 }
253
254
255 addr.sin_port = htons(S1_PORT);
256 err = listen(s1, 32);
257 if (err < 0) {
258 perror("listen s1 failed()\n");
259 return errno;
260 }
261
262 addr.sin_port = htons(S2_PORT);
263 err = listen(s2, 32);
264 if (err < 0) {
265 perror("listen s1 failed()\n");
266 return errno;
267 }
268
269
270 addr.sin_port = htons(S1_PORT);
271 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
272 if (err < 0 && errno != EINPROGRESS) {
273 perror("connect c1 failed()\n");
274 return errno;
275 }
276
277 addr.sin_port = htons(S2_PORT);
278 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
279 if (err < 0 && errno != EINPROGRESS) {
280 perror("connect c2 failed()\n");
281 return errno;
282 } else if (err < 0) {
283 err = 0;
284 }
285
286
287 p1 = accept(s1, NULL, NULL);
288 if (p1 < 0) {
289 perror("accept s1 failed()\n");
290 return errno;
291 }
292
293 p2 = accept(s2, NULL, NULL);
294 if (p2 < 0) {
295 perror("accept s1 failed()\n");
296 return errno;
297 }
298
299 if (verbose) {
300 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
301 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
302 c1, s1, c2, s2);
303 }
304 return 0;
305}
306
307struct msg_stats {
308 size_t bytes_sent;
309 size_t bytes_recvd;
310 struct timespec start;
311 struct timespec end;
312};
313
314struct sockmap_options {
315 int verbose;
316 bool base;
317 bool sendpage;
318 bool data_test;
319 bool drop_expected;
320 int iov_count;
321 int iov_length;
322 int rate;
323};
324
325static int msg_loop_sendpage(int fd, int iov_length, int cnt,
326 struct msg_stats *s,
327 struct sockmap_options *opt)
328{
329 bool drop = opt->drop_expected;
330 unsigned char k = 0;
331 FILE *file;
332 int i, fp;
333
334 file = fopen(".sendpage_tst.tmp", "w+");
335 for (i = 0; i < iov_length * cnt; i++, k++)
336 fwrite(&k, sizeof(char), 1, file);
337 fflush(file);
338 fseek(file, 0, SEEK_SET);
339 fclose(file);
340
341 fp = open(".sendpage_tst.tmp", O_RDONLY);
342 clock_gettime(CLOCK_MONOTONIC, &s->start);
343 for (i = 0; i < cnt; i++) {
344 int sent = sendfile(fd, fp, NULL, iov_length);
345
346 if (!drop && sent < 0) {
347 perror("send loop error:");
348 close(fp);
349 return sent;
350 } else if (drop && sent >= 0) {
351 printf("sendpage loop error expected: %i\n", sent);
352 close(fp);
353 return -EIO;
354 }
355
356 if (sent > 0)
357 s->bytes_sent += sent;
358 }
359 clock_gettime(CLOCK_MONOTONIC, &s->end);
360 close(fp);
361 return 0;
362}
363
364static void msg_free_iov(struct msghdr *msg)
365{
366 int i;
367
368 for (i = 0; i < msg->msg_iovlen; i++)
369 free(msg->msg_iov[i].iov_base);
370 free(msg->msg_iov);
371 msg->msg_iov = NULL;
372 msg->msg_iovlen = 0;
373}
374
375static int msg_alloc_iov(struct msghdr *msg,
376 int iov_count, int iov_length,
377 bool data, bool xmit)
378{
379 unsigned char k = 0;
380 struct iovec *iov;
381 int i;
382
383 iov = calloc(iov_count, sizeof(struct iovec));
384 if (!iov)
385 return errno;
386
387 for (i = 0; i < iov_count; i++) {
388 unsigned char *d = calloc(iov_length, sizeof(char));
389
390 if (!d) {
391 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
392 goto unwind_iov;
393 }
394 iov[i].iov_base = d;
395 iov[i].iov_len = iov_length;
396
397 if (data && xmit) {
398 int j;
399
400 for (j = 0; j < iov_length; j++)
401 d[j] = k++;
402 }
403 }
404
405 msg->msg_iov = iov;
406 msg->msg_iovlen = iov_count;
407
408 return 0;
409unwind_iov:
410 for (i--; i >= 0 ; i--)
411 free(msg->msg_iov[i].iov_base);
412 return -ENOMEM;
413}
414
415static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
416{
417 int i, j, bytes_cnt = 0;
418 unsigned char k = 0;
419
420 for (i = 0; i < msg->msg_iovlen; i++) {
421 unsigned char *d = msg->msg_iov[i].iov_base;
422
423 for (j = 0;
424 j < msg->msg_iov[i].iov_len && size; j++) {
425 if (d[j] != k++) {
426 fprintf(stderr,
427 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
428 i, j, d[j], k - 1, d[j+1], k);
429 return -EIO;
430 }
431 bytes_cnt++;
432 if (bytes_cnt == chunk_sz) {
433 k = 0;
434 bytes_cnt = 0;
435 }
436 size--;
437 }
438 }
439 return 0;
440}
441
442static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
443 struct msg_stats *s, bool tx,
444 struct sockmap_options *opt)
445{
446 struct msghdr msg = {0}, msg_peek = {0};
447 int err, i, flags = MSG_NOSIGNAL;
448 bool drop = opt->drop_expected;
449 bool data = opt->data_test;
450
451 err = msg_alloc_iov(&msg, iov_count, iov_length, data, tx);
452 if (err)
453 goto out_errno;
454 if (peek_flag) {
455 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
456 if (err)
457 goto out_errno;
458 }
459
460 if (tx) {
461 clock_gettime(CLOCK_MONOTONIC, &s->start);
462 for (i = 0; i < cnt; i++) {
463 int sent = sendmsg(fd, &msg, flags);
464
465 if (!drop && sent < 0) {
466 perror("send loop error:");
467 goto out_errno;
468 } else if (drop && sent >= 0) {
469 printf("send loop error expected: %i\n", sent);
470 errno = -EIO;
471 goto out_errno;
472 }
473 if (sent > 0)
474 s->bytes_sent += sent;
475 }
476 clock_gettime(CLOCK_MONOTONIC, &s->end);
477 } else {
478 int slct, recvp = 0, recv, max_fd = fd;
479 float total_bytes, txmsg_pop_total;
480 int fd_flags = O_NONBLOCK;
481 struct timeval timeout;
482 fd_set w;
483
484 fcntl(fd, fd_flags);
485
486
487
488
489
490
491
492
493
494
495 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
496 txmsg_pop_total = txmsg_pop;
497 if (txmsg_apply)
498 txmsg_pop_total *= (total_bytes / txmsg_apply);
499 total_bytes -= txmsg_pop_total;
500 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
501 if (err < 0)
502 perror("recv start time: ");
503 while (s->bytes_recvd < total_bytes) {
504 if (txmsg_cork) {
505 timeout.tv_sec = 0;
506 timeout.tv_usec = 300000;
507 } else {
508 timeout.tv_sec = 3;
509 timeout.tv_usec = 0;
510 }
511
512
513 FD_ZERO(&w);
514 FD_SET(fd, &w);
515
516 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
517 if (slct == -1) {
518 perror("select()");
519 clock_gettime(CLOCK_MONOTONIC, &s->end);
520 goto out_errno;
521 } else if (!slct) {
522 if (opt->verbose)
523 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
524 errno = -EIO;
525 clock_gettime(CLOCK_MONOTONIC, &s->end);
526 goto out_errno;
527 }
528
529 errno = 0;
530 if (peek_flag) {
531 flags |= MSG_PEEK;
532 recvp = recvmsg(fd, &msg_peek, flags);
533 if (recvp < 0) {
534 if (errno != EWOULDBLOCK) {
535 clock_gettime(CLOCK_MONOTONIC, &s->end);
536 goto out_errno;
537 }
538 }
539 flags = 0;
540 }
541
542 recv = recvmsg(fd, &msg, flags);
543 if (recv < 0) {
544 if (errno != EWOULDBLOCK) {
545 clock_gettime(CLOCK_MONOTONIC, &s->end);
546 perror("recv failed()\n");
547 goto out_errno;
548 }
549 }
550
551 s->bytes_recvd += recv;
552
553 if (data) {
554 int chunk_sz = opt->sendpage ?
555 iov_length * cnt :
556 iov_length * iov_count;
557
558 errno = msg_verify_data(&msg, recv, chunk_sz);
559 if (errno) {
560 perror("data verify msg failed\n");
561 goto out_errno;
562 }
563 if (recvp) {
564 errno = msg_verify_data(&msg_peek,
565 recvp,
566 chunk_sz);
567 if (errno) {
568 perror("data verify msg_peek failed\n");
569 goto out_errno;
570 }
571 }
572 }
573 }
574 clock_gettime(CLOCK_MONOTONIC, &s->end);
575 }
576
577 msg_free_iov(&msg);
578 msg_free_iov(&msg_peek);
579 return err;
580out_errno:
581 msg_free_iov(&msg);
582 msg_free_iov(&msg_peek);
583 return errno;
584}
585
586static float giga = 1000000000;
587
588static inline float sentBps(struct msg_stats s)
589{
590 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
591}
592
593static inline float recvdBps(struct msg_stats s)
594{
595 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
596}
597
598static int sendmsg_test(struct sockmap_options *opt)
599{
600 float sent_Bps = 0, recvd_Bps = 0;
601 int rx_fd, txpid, rxpid, err = 0;
602 struct msg_stats s = {0};
603 int iov_count = opt->iov_count;
604 int iov_buf = opt->iov_length;
605 int rx_status, tx_status;
606 int cnt = opt->rate;
607
608 errno = 0;
609
610 if (opt->base)
611 rx_fd = p1;
612 else
613 rx_fd = p2;
614
615 if (ktls) {
616
617
618
619
620 if (!txmsg_redir || (txmsg_redir && txmsg_ingress)) {
621 err = sockmap_init_ktls(opt->verbose, rx_fd);
622 if (err)
623 return err;
624 }
625 err = sockmap_init_ktls(opt->verbose, c1);
626 if (err)
627 return err;
628 }
629
630 rxpid = fork();
631 if (rxpid == 0) {
632 if (opt->drop_expected)
633 exit(0);
634
635 if (opt->sendpage)
636 iov_count = 1;
637 err = msg_loop(rx_fd, iov_count, iov_buf,
638 cnt, &s, false, opt);
639 if (opt->verbose)
640 fprintf(stderr,
641 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
642 iov_count, iov_buf, cnt, err);
643 if (s.end.tv_sec - s.start.tv_sec) {
644 sent_Bps = sentBps(s);
645 recvd_Bps = recvdBps(s);
646 }
647 if (opt->verbose)
648 fprintf(stdout,
649 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
650 s.bytes_sent, sent_Bps, sent_Bps/giga,
651 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
652 peek_flag ? "(peek_msg)" : "");
653 if (err && txmsg_cork)
654 err = 0;
655 exit(err ? 1 : 0);
656 } else if (rxpid == -1) {
657 perror("msg_loop_rx: ");
658 return errno;
659 }
660
661 txpid = fork();
662 if (txpid == 0) {
663 if (opt->sendpage)
664 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
665 else
666 err = msg_loop(c1, iov_count, iov_buf,
667 cnt, &s, true, opt);
668
669 if (err)
670 fprintf(stderr,
671 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
672 iov_count, iov_buf, cnt, err);
673 if (s.end.tv_sec - s.start.tv_sec) {
674 sent_Bps = sentBps(s);
675 recvd_Bps = recvdBps(s);
676 }
677 if (opt->verbose)
678 fprintf(stdout,
679 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
680 s.bytes_sent, sent_Bps, sent_Bps/giga,
681 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
682 exit(err ? 1 : 0);
683 } else if (txpid == -1) {
684 perror("msg_loop_tx: ");
685 return errno;
686 }
687
688 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
689 assert(waitpid(txpid, &tx_status, 0) == txpid);
690 if (WIFEXITED(rx_status)) {
691 err = WEXITSTATUS(rx_status);
692 if (err) {
693 fprintf(stderr, "rx thread exited with err %d. ", err);
694 goto out;
695 }
696 }
697 if (WIFEXITED(tx_status)) {
698 err = WEXITSTATUS(tx_status);
699 if (err)
700 fprintf(stderr, "tx thread exited with err %d. ", err);
701 }
702out:
703 return err;
704}
705
706static int forever_ping_pong(int rate, struct sockmap_options *opt)
707{
708 struct timeval timeout;
709 char buf[1024] = {0};
710 int sc;
711
712 timeout.tv_sec = 10;
713 timeout.tv_usec = 0;
714
715
716 sc = send(c1, buf, sizeof(buf), 0);
717 if (sc < 0) {
718 perror("send failed()\n");
719 return sc;
720 }
721
722 do {
723 int s, rc, i, max_fd = p2;
724 fd_set w;
725
726
727 FD_ZERO(&w);
728 FD_SET(c1, &w);
729 FD_SET(c2, &w);
730 FD_SET(p1, &w);
731 FD_SET(p2, &w);
732
733 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
734 if (s == -1) {
735 perror("select()");
736 break;
737 } else if (!s) {
738 fprintf(stderr, "unexpected timeout\n");
739 break;
740 }
741
742 for (i = 0; i <= max_fd && s > 0; ++i) {
743 if (!FD_ISSET(i, &w))
744 continue;
745
746 s--;
747
748 rc = recv(i, buf, sizeof(buf), 0);
749 if (rc < 0) {
750 if (errno != EWOULDBLOCK) {
751 perror("recv failed()\n");
752 return rc;
753 }
754 }
755
756 if (rc == 0) {
757 close(i);
758 break;
759 }
760
761 sc = send(i, buf, rc, 0);
762 if (sc < 0) {
763 perror("send failed()\n");
764 return sc;
765 }
766 }
767
768 if (rate)
769 sleep(rate);
770
771 if (opt->verbose) {
772 printf(".");
773 fflush(stdout);
774
775 }
776 } while (running);
777
778 return 0;
779}
780
781enum {
782 PING_PONG,
783 SENDMSG,
784 BASE,
785 BASE_SENDPAGE,
786 SENDPAGE,
787};
788
789static int run_options(struct sockmap_options *options, int cg_fd, int test)
790{
791 int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
792
793
794 if (test == BASE || test == BASE_SENDPAGE)
795 goto run;
796
797
798 err = bpf_prog_attach(prog_fd[0], map_fd[0],
799 BPF_SK_SKB_STREAM_PARSER, 0);
800 if (err) {
801 fprintf(stderr,
802 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
803 prog_fd[0], map_fd[0], err, strerror(errno));
804 return err;
805 }
806
807 err = bpf_prog_attach(prog_fd[1], map_fd[0],
808 BPF_SK_SKB_STREAM_VERDICT, 0);
809 if (err) {
810 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
811 err, strerror(errno));
812 return err;
813 }
814
815
816 err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
817 if (err) {
818 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
819 err, strerror(errno));
820 return err;
821 }
822
823run:
824 err = sockmap_init_sockets(options->verbose);
825 if (err) {
826 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
827 goto out;
828 }
829
830
831 if (txmsg_pass)
832 tx_prog_fd = prog_fd[3];
833 else if (txmsg_noisy)
834 tx_prog_fd = prog_fd[4];
835 else if (txmsg_redir)
836 tx_prog_fd = prog_fd[5];
837 else if (txmsg_redir_noisy)
838 tx_prog_fd = prog_fd[6];
839 else if (txmsg_drop)
840 tx_prog_fd = prog_fd[9];
841
842 else if (txmsg_apply)
843 tx_prog_fd = prog_fd[7];
844 else if (txmsg_cork)
845 tx_prog_fd = prog_fd[8];
846 else
847 tx_prog_fd = 0;
848
849 if (tx_prog_fd) {
850 int redir_fd, i = 0;
851
852 err = bpf_prog_attach(tx_prog_fd,
853 map_fd[1], BPF_SK_MSG_VERDICT, 0);
854 if (err) {
855 fprintf(stderr,
856 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
857 err, strerror(errno));
858 goto out;
859 }
860
861 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
862 if (err) {
863 fprintf(stderr,
864 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
865 err, strerror(errno));
866 goto out;
867 }
868
869 if (txmsg_redir || txmsg_redir_noisy)
870 redir_fd = c2;
871 else
872 redir_fd = c1;
873
874 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
875 if (err) {
876 fprintf(stderr,
877 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
878 err, strerror(errno));
879 goto out;
880 }
881
882 if (txmsg_apply) {
883 err = bpf_map_update_elem(map_fd[3],
884 &i, &txmsg_apply, BPF_ANY);
885 if (err) {
886 fprintf(stderr,
887 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
888 err, strerror(errno));
889 goto out;
890 }
891 }
892
893 if (txmsg_cork) {
894 err = bpf_map_update_elem(map_fd[4],
895 &i, &txmsg_cork, BPF_ANY);
896 if (err) {
897 fprintf(stderr,
898 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
899 err, strerror(errno));
900 goto out;
901 }
902 }
903
904 if (txmsg_start) {
905 err = bpf_map_update_elem(map_fd[5],
906 &i, &txmsg_start, BPF_ANY);
907 if (err) {
908 fprintf(stderr,
909 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
910 err, strerror(errno));
911 goto out;
912 }
913 }
914
915 if (txmsg_end) {
916 i = 1;
917 err = bpf_map_update_elem(map_fd[5],
918 &i, &txmsg_end, BPF_ANY);
919 if (err) {
920 fprintf(stderr,
921 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
922 err, strerror(errno));
923 goto out;
924 }
925 }
926
927 if (txmsg_start_push) {
928 i = 2;
929 err = bpf_map_update_elem(map_fd[5],
930 &i, &txmsg_start_push, BPF_ANY);
931 if (err) {
932 fprintf(stderr,
933 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
934 err, strerror(errno));
935 goto out;
936 }
937 }
938
939 if (txmsg_end_push) {
940 i = 3;
941 err = bpf_map_update_elem(map_fd[5],
942 &i, &txmsg_end_push, BPF_ANY);
943 if (err) {
944 fprintf(stderr,
945 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
946 txmsg_end_push, i, err, strerror(errno));
947 goto out;
948 }
949 }
950
951 if (txmsg_start_pop) {
952 i = 4;
953 err = bpf_map_update_elem(map_fd[5],
954 &i, &txmsg_start_pop, BPF_ANY);
955 if (err) {
956 fprintf(stderr,
957 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
958 txmsg_start_pop, i, err, strerror(errno));
959 goto out;
960 }
961 } else {
962 i = 4;
963 bpf_map_update_elem(map_fd[5],
964 &i, &txmsg_start_pop, BPF_ANY);
965 }
966
967 if (txmsg_pop) {
968 i = 5;
969 err = bpf_map_update_elem(map_fd[5],
970 &i, &txmsg_pop, BPF_ANY);
971 if (err) {
972 fprintf(stderr,
973 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
974 txmsg_pop, i, err, strerror(errno));
975 goto out;
976 }
977 } else {
978 i = 5;
979 bpf_map_update_elem(map_fd[5],
980 &i, &txmsg_pop, BPF_ANY);
981
982 }
983
984 if (txmsg_ingress) {
985 int in = BPF_F_INGRESS;
986
987 i = 0;
988 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
989 if (err) {
990 fprintf(stderr,
991 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
992 err, strerror(errno));
993 }
994 i = 1;
995 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
996 if (err) {
997 fprintf(stderr,
998 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
999 err, strerror(errno));
1000 }
1001 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1002 if (err) {
1003 fprintf(stderr,
1004 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1005 err, strerror(errno));
1006 }
1007
1008 i = 2;
1009 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1010 if (err) {
1011 fprintf(stderr,
1012 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1013 err, strerror(errno));
1014 }
1015 }
1016
1017 if (txmsg_skb) {
1018 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1019 p2 : p1;
1020 int ingress = BPF_F_INGRESS;
1021
1022 i = 0;
1023 err = bpf_map_update_elem(map_fd[7],
1024 &i, &ingress, BPF_ANY);
1025 if (err) {
1026 fprintf(stderr,
1027 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1028 err, strerror(errno));
1029 }
1030
1031 i = 3;
1032 err = bpf_map_update_elem(map_fd[0],
1033 &i, &skb_fd, BPF_ANY);
1034 if (err) {
1035 fprintf(stderr,
1036 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1037 err, strerror(errno));
1038 }
1039 }
1040 }
1041
1042 if (txmsg_drop)
1043 options->drop_expected = true;
1044
1045 if (test == PING_PONG)
1046 err = forever_ping_pong(options->rate, options);
1047 else if (test == SENDMSG) {
1048 options->base = false;
1049 options->sendpage = false;
1050 err = sendmsg_test(options);
1051 } else if (test == SENDPAGE) {
1052 options->base = false;
1053 options->sendpage = true;
1054 err = sendmsg_test(options);
1055 } else if (test == BASE) {
1056 options->base = true;
1057 options->sendpage = false;
1058 err = sendmsg_test(options);
1059 } else if (test == BASE_SENDPAGE) {
1060 options->base = true;
1061 options->sendpage = true;
1062 err = sendmsg_test(options);
1063 } else
1064 fprintf(stderr, "unknown test\n");
1065out:
1066
1067 bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS);
1068 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1069 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1070 if (tx_prog_fd >= 0)
1071 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1072
1073 for (i = 0; i < 8; i++) {
1074 key = next_key = 0;
1075 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1076 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1077 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1078 key = next_key;
1079 }
1080 }
1081
1082 close(s1);
1083 close(s2);
1084 close(p1);
1085 close(p2);
1086 close(c1);
1087 close(c2);
1088 return err;
1089}
1090
1091static char *test_to_str(int test)
1092{
1093 switch (test) {
1094 case SENDMSG:
1095 return "sendmsg";
1096 case SENDPAGE:
1097 return "sendpage";
1098 }
1099 return "unknown";
1100}
1101
1102#define OPTSTRING 60
1103static void test_options(char *options)
1104{
1105 char tstr[OPTSTRING];
1106
1107 memset(options, 0, OPTSTRING);
1108
1109 if (txmsg_pass)
1110 strncat(options, "pass,", OPTSTRING);
1111 if (txmsg_noisy)
1112 strncat(options, "pass_noisy,", OPTSTRING);
1113 if (txmsg_redir)
1114 strncat(options, "redir,", OPTSTRING);
1115 if (txmsg_redir_noisy)
1116 strncat(options, "redir_noisy,", OPTSTRING);
1117 if (txmsg_drop)
1118 strncat(options, "drop,", OPTSTRING);
1119 if (txmsg_apply) {
1120 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1121 strncat(options, tstr, OPTSTRING);
1122 }
1123 if (txmsg_cork) {
1124 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1125 strncat(options, tstr, OPTSTRING);
1126 }
1127 if (txmsg_start) {
1128 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1129 strncat(options, tstr, OPTSTRING);
1130 }
1131 if (txmsg_end) {
1132 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1133 strncat(options, tstr, OPTSTRING);
1134 }
1135 if (txmsg_start_pop) {
1136 snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1137 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1138 strncat(options, tstr, OPTSTRING);
1139 }
1140 if (txmsg_ingress)
1141 strncat(options, "ingress,", OPTSTRING);
1142 if (txmsg_skb)
1143 strncat(options, "skb,", OPTSTRING);
1144 if (ktls)
1145 strncat(options, "ktls,", OPTSTRING);
1146 if (peek_flag)
1147 strncat(options, "peek,", OPTSTRING);
1148}
1149
1150static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1151{
1152 char *options = calloc(OPTSTRING, sizeof(char));
1153 int err;
1154
1155 if (test == SENDPAGE)
1156 opt->sendpage = true;
1157 else
1158 opt->sendpage = false;
1159
1160 if (txmsg_drop)
1161 opt->drop_expected = true;
1162 else
1163 opt->drop_expected = false;
1164
1165 test_options(options);
1166
1167 fprintf(stdout,
1168 "[TEST %i]: (%i, %i, %i, %s, %s): ",
1169 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1170 test_to_str(test), options);
1171 fflush(stdout);
1172 err = run_options(opt, cgrp, test);
1173 fprintf(stdout, "%s\n", !err ? "PASS" : "FAILED");
1174 test_cnt++;
1175 !err ? passed++ : failed++;
1176 free(options);
1177 return err;
1178}
1179
1180static int test_exec(int cgrp, struct sockmap_options *opt)
1181{
1182 int err = __test_exec(cgrp, SENDMSG, opt);
1183
1184 if (err)
1185 goto out;
1186
1187 err = __test_exec(cgrp, SENDPAGE, opt);
1188out:
1189 return err;
1190}
1191
1192static int test_loop(int cgrp)
1193{
1194 struct sockmap_options opt;
1195
1196 int err, i, l, r;
1197
1198 opt.verbose = 0;
1199 opt.base = false;
1200 opt.sendpage = false;
1201 opt.data_test = false;
1202 opt.drop_expected = false;
1203 opt.iov_count = 0;
1204 opt.iov_length = 0;
1205 opt.rate = 0;
1206
1207 r = 1;
1208 for (i = 1; i < 100; i += 33) {
1209 for (l = 1; l < 100; l += 33) {
1210 opt.rate = r;
1211 opt.iov_count = i;
1212 opt.iov_length = l;
1213 err = test_exec(cgrp, &opt);
1214 if (err)
1215 goto out;
1216 }
1217 }
1218 sched_yield();
1219out:
1220 return err;
1221}
1222
1223static int test_txmsg(int cgrp)
1224{
1225 int err;
1226
1227 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
1228 txmsg_apply = txmsg_cork = 0;
1229 txmsg_ingress = txmsg_skb = 0;
1230
1231 txmsg_pass = 1;
1232 err = test_loop(cgrp);
1233 txmsg_pass = 0;
1234 if (err)
1235 goto out;
1236
1237 txmsg_redir = 1;
1238 err = test_loop(cgrp);
1239 txmsg_redir = 0;
1240 if (err)
1241 goto out;
1242
1243 txmsg_drop = 1;
1244 err = test_loop(cgrp);
1245 txmsg_drop = 0;
1246 if (err)
1247 goto out;
1248
1249 txmsg_redir = 1;
1250 txmsg_ingress = 1;
1251 err = test_loop(cgrp);
1252 txmsg_redir = 0;
1253 txmsg_ingress = 0;
1254 if (err)
1255 goto out;
1256out:
1257 txmsg_pass = 0;
1258 txmsg_redir = 0;
1259 txmsg_drop = 0;
1260 return err;
1261}
1262
1263static int test_send(struct sockmap_options *opt, int cgrp)
1264{
1265 int err;
1266
1267 opt->iov_length = 1;
1268 opt->iov_count = 1;
1269 opt->rate = 1;
1270 err = test_exec(cgrp, opt);
1271 if (err)
1272 goto out;
1273
1274 opt->iov_length = 1;
1275 opt->iov_count = 1024;
1276 opt->rate = 1;
1277 err = test_exec(cgrp, opt);
1278 if (err)
1279 goto out;
1280
1281 opt->iov_length = 1024;
1282 opt->iov_count = 1;
1283 opt->rate = 1;
1284 err = test_exec(cgrp, opt);
1285 if (err)
1286 goto out;
1287
1288 opt->iov_length = 1;
1289 opt->iov_count = 1;
1290 opt->rate = 512;
1291 err = test_exec(cgrp, opt);
1292 if (err)
1293 goto out;
1294
1295 opt->iov_length = 256;
1296 opt->iov_count = 1024;
1297 opt->rate = 2;
1298 err = test_exec(cgrp, opt);
1299 if (err)
1300 goto out;
1301
1302 opt->rate = 100;
1303 opt->iov_count = 1;
1304 opt->iov_length = 5;
1305 err = test_exec(cgrp, opt);
1306 if (err)
1307 goto out;
1308out:
1309 sched_yield();
1310 return err;
1311}
1312
1313static int test_mixed(int cgrp)
1314{
1315 struct sockmap_options opt = {0};
1316 int err;
1317
1318 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
1319 txmsg_apply = txmsg_cork = 0;
1320 txmsg_start = txmsg_end = 0;
1321 txmsg_start_push = txmsg_end_push = 0;
1322 txmsg_start_pop = txmsg_pop = 0;
1323
1324
1325 txmsg_pass = 1;
1326 txmsg_redir = 0;
1327 txmsg_apply = 1;
1328 txmsg_cork = 0;
1329 err = test_send(&opt, cgrp);
1330 if (err)
1331 goto out;
1332
1333 txmsg_pass = 1;
1334 txmsg_redir = 0;
1335 txmsg_apply = 0;
1336 txmsg_cork = 1;
1337 err = test_send(&opt, cgrp);
1338 if (err)
1339 goto out;
1340
1341 txmsg_pass = 1;
1342 txmsg_redir = 0;
1343 txmsg_apply = 1;
1344 txmsg_cork = 1;
1345 err = test_send(&opt, cgrp);
1346 if (err)
1347 goto out;
1348
1349 txmsg_pass = 1;
1350 txmsg_redir = 0;
1351 txmsg_apply = 1024;
1352 txmsg_cork = 0;
1353 err = test_send(&opt, cgrp);
1354 if (err)
1355 goto out;
1356
1357 txmsg_pass = 1;
1358 txmsg_redir = 0;
1359 txmsg_apply = 0;
1360 txmsg_cork = 1024;
1361 err = test_send(&opt, cgrp);
1362 if (err)
1363 goto out;
1364
1365 txmsg_pass = 1;
1366 txmsg_redir = 0;
1367 txmsg_apply = 1024;
1368 txmsg_cork = 1024;
1369 err = test_send(&opt, cgrp);
1370 if (err)
1371 goto out;
1372
1373 txmsg_pass = 1;
1374 txmsg_redir = 0;
1375 txmsg_cork = 4096;
1376 txmsg_apply = 4096;
1377 err = test_send(&opt, cgrp);
1378 if (err)
1379 goto out;
1380
1381 txmsg_pass = 0;
1382 txmsg_redir = 1;
1383 txmsg_apply = 1;
1384 txmsg_cork = 0;
1385 err = test_send(&opt, cgrp);
1386 if (err)
1387 goto out;
1388
1389 txmsg_pass = 0;
1390 txmsg_redir = 1;
1391 txmsg_apply = 0;
1392 txmsg_cork = 1;
1393 err = test_send(&opt, cgrp);
1394 if (err)
1395 goto out;
1396
1397 txmsg_pass = 0;
1398 txmsg_redir = 1;
1399 txmsg_apply = 1024;
1400 txmsg_cork = 0;
1401 err = test_send(&opt, cgrp);
1402 if (err)
1403 goto out;
1404
1405 txmsg_pass = 0;
1406 txmsg_redir = 1;
1407 txmsg_apply = 0;
1408 txmsg_cork = 1024;
1409 err = test_send(&opt, cgrp);
1410 if (err)
1411 goto out;
1412
1413 txmsg_pass = 0;
1414 txmsg_redir = 1;
1415 txmsg_apply = 1024;
1416 txmsg_cork = 1024;
1417 err = test_send(&opt, cgrp);
1418 if (err)
1419 goto out;
1420
1421 txmsg_pass = 0;
1422 txmsg_redir = 1;
1423 txmsg_cork = 4096;
1424 txmsg_apply = 4096;
1425 err = test_send(&opt, cgrp);
1426 if (err)
1427 goto out;
1428out:
1429 return err;
1430}
1431
1432static int test_start_end(int cgrp)
1433{
1434 struct sockmap_options opt = {0};
1435 int err, i;
1436
1437
1438 txmsg_start = 1;
1439 txmsg_end = 2;
1440 txmsg_start_push = 1;
1441 txmsg_end_push = 2;
1442 txmsg_start_pop = 1;
1443 txmsg_pop = 1;
1444 err = test_txmsg(cgrp);
1445 if (err)
1446 goto out;
1447
1448
1449 txmsg_start = 1;
1450 txmsg_end = 2;
1451 txmsg_start_push = 1;
1452 txmsg_end_push = 3;
1453 txmsg_start_pop = 1;
1454 txmsg_pop = 1;
1455 err = test_txmsg(cgrp);
1456 if (err)
1457 goto out;
1458
1459
1460 opt.rate = 16;
1461 opt.iov_count = 1;
1462 opt.iov_length = 100;
1463 txmsg_cork = 1600;
1464
1465 txmsg_start_pop = 0;
1466 txmsg_pop = 0;
1467
1468 for (i = 99; i <= 1600; i += 500) {
1469 txmsg_start = 0;
1470 txmsg_end = i;
1471 txmsg_start_push = 0;
1472 txmsg_end_push = i;
1473 err = test_exec(cgrp, &opt);
1474 if (err)
1475 goto out;
1476 }
1477
1478
1479 for (i = 99; i <= 1600; i += 500) {
1480 txmsg_start_pop = 10;
1481 txmsg_pop = i;
1482 err = test_exec(cgrp, &opt);
1483 if (err)
1484 goto out;
1485 }
1486 txmsg_start_pop = 0;
1487 txmsg_pop = 0;
1488
1489
1490 for (i = 199; i <= 1600; i += 500) {
1491 txmsg_start = 100;
1492 txmsg_end = i;
1493 txmsg_start_push = 100;
1494 txmsg_end_push = i;
1495 err = test_exec(cgrp, &opt);
1496 if (err)
1497 goto out;
1498 }
1499
1500
1501 txmsg_start = 1500;
1502 txmsg_end = 1600;
1503 txmsg_start_push = 1500;
1504 txmsg_end_push = 1600;
1505 err = test_exec(cgrp, &opt);
1506 if (err)
1507 goto out;
1508
1509
1510 txmsg_start_pop = 1500;
1511 txmsg_pop = 1600;
1512 err = test_exec(cgrp, &opt);
1513 if (err)
1514 goto out;
1515 txmsg_start_pop = 0;
1516 txmsg_pop = 0;
1517
1518
1519 txmsg_start = 1111;
1520 txmsg_end = 1112;
1521 txmsg_start_push = 1111;
1522 txmsg_end_push = 1112;
1523 err = test_exec(cgrp, &opt);
1524 if (err)
1525 goto out;
1526
1527
1528 txmsg_start_pop = 1111;
1529 txmsg_pop = 1112;
1530 err = test_exec(cgrp, &opt);
1531 if (err)
1532 goto out;
1533
1534
1535 txmsg_start = 1111;
1536 txmsg_end = 0;
1537 txmsg_start_push = 1111;
1538 txmsg_end_push = 0;
1539 err = test_exec(cgrp, &opt);
1540 if (err)
1541 goto out;
1542
1543
1544 txmsg_start = 0;
1545 txmsg_end = 1601;
1546 txmsg_start_push = 0;
1547 txmsg_end_push = 1601;
1548 err = test_exec(cgrp, &opt);
1549 if (err)
1550 goto out;
1551
1552
1553 txmsg_start = 1601;
1554 txmsg_end = 1600;
1555 txmsg_start_push = 1601;
1556 txmsg_end_push = 1600;
1557 err = test_exec(cgrp, &opt);
1558 if (err)
1559 goto out;
1560
1561
1562 txmsg_start_pop = 1601;
1563 txmsg_pop = 1;
1564 err = test_exec(cgrp, &opt);
1565 if (err)
1566 goto out;
1567
1568
1569 txmsg_start_pop = 1599;
1570 txmsg_pop = 10;
1571 err = test_exec(cgrp, &opt);
1572out:
1573 txmsg_start = 0;
1574 txmsg_end = 0;
1575 sched_yield();
1576 return err;
1577}
1578
1579char *map_names[] = {
1580 "sock_map",
1581 "sock_map_txmsg",
1582 "sock_map_redir",
1583 "sock_apply_bytes",
1584 "sock_cork_bytes",
1585 "sock_bytes",
1586 "sock_redir_flags",
1587 "sock_skb_opts",
1588};
1589
1590int prog_attach_type[] = {
1591 BPF_SK_SKB_STREAM_PARSER,
1592 BPF_SK_SKB_STREAM_VERDICT,
1593 BPF_CGROUP_SOCK_OPS,
1594 BPF_SK_MSG_VERDICT,
1595 BPF_SK_MSG_VERDICT,
1596 BPF_SK_MSG_VERDICT,
1597 BPF_SK_MSG_VERDICT,
1598 BPF_SK_MSG_VERDICT,
1599 BPF_SK_MSG_VERDICT,
1600 BPF_SK_MSG_VERDICT,
1601};
1602
1603int prog_type[] = {
1604 BPF_PROG_TYPE_SK_SKB,
1605 BPF_PROG_TYPE_SK_SKB,
1606 BPF_PROG_TYPE_SOCK_OPS,
1607 BPF_PROG_TYPE_SK_MSG,
1608 BPF_PROG_TYPE_SK_MSG,
1609 BPF_PROG_TYPE_SK_MSG,
1610 BPF_PROG_TYPE_SK_MSG,
1611 BPF_PROG_TYPE_SK_MSG,
1612 BPF_PROG_TYPE_SK_MSG,
1613 BPF_PROG_TYPE_SK_MSG,
1614};
1615
1616static int populate_progs(char *bpf_file)
1617{
1618 struct bpf_program *prog;
1619 struct bpf_object *obj;
1620 int i = 0;
1621 long err;
1622
1623 obj = bpf_object__open(bpf_file);
1624 err = libbpf_get_error(obj);
1625 if (err) {
1626 char err_buf[256];
1627
1628 libbpf_strerror(err, err_buf, sizeof(err_buf));
1629 printf("Unable to load eBPF objects in file '%s' : %s\n",
1630 bpf_file, err_buf);
1631 return -1;
1632 }
1633
1634 bpf_object__for_each_program(prog, obj) {
1635 bpf_program__set_type(prog, prog_type[i]);
1636 bpf_program__set_expected_attach_type(prog,
1637 prog_attach_type[i]);
1638 i++;
1639 }
1640
1641 i = bpf_object__load(obj);
1642 i = 0;
1643 bpf_object__for_each_program(prog, obj) {
1644 prog_fd[i] = bpf_program__fd(prog);
1645 i++;
1646 }
1647
1648 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) {
1649 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1650 map_fd[i] = bpf_map__fd(maps[i]);
1651 if (map_fd[i] < 0) {
1652 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1653 map_fd[i], strerror(errno));
1654 return -1;
1655 }
1656 }
1657
1658 return 0;
1659}
1660
1661static int __test_suite(int cg_fd, char *bpf_file)
1662{
1663 int err, cleanup = cg_fd;
1664
1665 err = populate_progs(bpf_file);
1666 if (err < 0) {
1667 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1668 return err;
1669 }
1670
1671 if (cg_fd < 0) {
1672 if (setup_cgroup_environment()) {
1673 fprintf(stderr, "ERROR: cgroup env failed\n");
1674 return -EINVAL;
1675 }
1676
1677 cg_fd = create_and_get_cgroup(CG_PATH);
1678 if (cg_fd < 0) {
1679 fprintf(stderr,
1680 "ERROR: (%i) open cg path failed: %s\n",
1681 cg_fd, optarg);
1682 return cg_fd;
1683 }
1684
1685 if (join_cgroup(CG_PATH)) {
1686 fprintf(stderr, "ERROR: failed to join cgroup\n");
1687 return -EINVAL;
1688 }
1689 }
1690
1691
1692 txmsg_start = txmsg_end = txmsg_start_push = txmsg_end_push = 0;
1693 err = test_txmsg(cg_fd);
1694 if (err)
1695 goto out;
1696
1697
1698 err = test_mixed(cg_fd);
1699 if (err)
1700 goto out;
1701
1702
1703 err = test_start_end(cg_fd);
1704 if (err)
1705 goto out;
1706
1707out:
1708 printf("Summary: %i PASSED %i FAILED\n", passed, failed);
1709 if (cleanup < 0) {
1710 cleanup_cgroup_environment();
1711 close(cg_fd);
1712 }
1713 return err;
1714}
1715
1716static int test_suite(int cg_fd)
1717{
1718 int err;
1719
1720 err = __test_suite(cg_fd, BPF_SOCKMAP_FILENAME);
1721 if (err)
1722 goto out;
1723 err = __test_suite(cg_fd, BPF_SOCKHASH_FILENAME);
1724out:
1725 if (cg_fd > -1)
1726 close(cg_fd);
1727 return err;
1728}
1729
1730int main(int argc, char **argv)
1731{
1732 int iov_count = 1, length = 1024, rate = 1;
1733 struct sockmap_options options = {0};
1734 int opt, longindex, err, cg_fd = 0;
1735 char *bpf_file = BPF_SOCKMAP_FILENAME;
1736 int test = PING_PONG;
1737
1738 if (argc < 2)
1739 return test_suite(-1);
1740
1741 while ((opt = getopt_long(argc, argv, ":dhvc:r:i:l:t:p:q:",
1742 long_options, &longindex)) != -1) {
1743 switch (opt) {
1744 case 's':
1745 txmsg_start = atoi(optarg);
1746 break;
1747 case 'e':
1748 txmsg_end = atoi(optarg);
1749 break;
1750 case 'p':
1751 txmsg_start_push = atoi(optarg);
1752 break;
1753 case 'q':
1754 txmsg_end_push = atoi(optarg);
1755 break;
1756 case 'w':
1757 txmsg_start_pop = atoi(optarg);
1758 break;
1759 case 'x':
1760 txmsg_pop = atoi(optarg);
1761 break;
1762 case 'a':
1763 txmsg_apply = atoi(optarg);
1764 break;
1765 case 'k':
1766 txmsg_cork = atoi(optarg);
1767 break;
1768 case 'c':
1769 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1770 if (cg_fd < 0) {
1771 fprintf(stderr,
1772 "ERROR: (%i) open cg path failed: %s\n",
1773 cg_fd, optarg);
1774 return cg_fd;
1775 }
1776 break;
1777 case 'r':
1778 rate = atoi(optarg);
1779 break;
1780 case 'v':
1781 options.verbose = 1;
1782 break;
1783 case 'i':
1784 iov_count = atoi(optarg);
1785 break;
1786 case 'l':
1787 length = atoi(optarg);
1788 break;
1789 case 'd':
1790 options.data_test = true;
1791 break;
1792 case 't':
1793 if (strcmp(optarg, "ping") == 0) {
1794 test = PING_PONG;
1795 } else if (strcmp(optarg, "sendmsg") == 0) {
1796 test = SENDMSG;
1797 } else if (strcmp(optarg, "base") == 0) {
1798 test = BASE;
1799 } else if (strcmp(optarg, "base_sendpage") == 0) {
1800 test = BASE_SENDPAGE;
1801 } else if (strcmp(optarg, "sendpage") == 0) {
1802 test = SENDPAGE;
1803 } else {
1804 usage(argv);
1805 return -1;
1806 }
1807 break;
1808 case 0:
1809 break;
1810 case 'h':
1811 default:
1812 usage(argv);
1813 return -1;
1814 }
1815 }
1816
1817 if (argc <= 3 && cg_fd)
1818 return test_suite(cg_fd);
1819
1820 if (!cg_fd) {
1821 fprintf(stderr, "%s requires cgroup option: --cgroup <path>\n",
1822 argv[0]);
1823 return -1;
1824 }
1825
1826 err = populate_progs(bpf_file);
1827 if (err) {
1828 fprintf(stderr, "populate program: (%s) %s\n",
1829 bpf_file, strerror(errno));
1830 return 1;
1831 }
1832 running = 1;
1833
1834
1835 signal(SIGINT, running_handler);
1836
1837 options.iov_count = iov_count;
1838 options.iov_length = length;
1839 options.rate = rate;
1840
1841 err = run_options(&options, cg_fd, test);
1842 close(cg_fd);
1843 return err;
1844}
1845
1846void running_handler(int a)
1847{
1848 running = 0;
1849}
1850
