1
2
3#include <stdio.h>
4#include <stdlib.h>
5#include <sys/socket.h>
6#include <sys/ioctl.h>
7#include <sys/select.h>
8#include <netinet/in.h>
9#include <arpa/inet.h>
10#include <unistd.h>
11#include <string.h>
12#include <errno.h>
13#include <stdbool.h>
14#include <signal.h>
15#include <fcntl.h>
16#include <sys/wait.h>
17#include <time.h>
18#include <sched.h>
19
20#include <sys/time.h>
21#include <sys/resource.h>
22#include <sys/types.h>
23#include <sys/sendfile.h>
24
25#include <linux/netlink.h>
26#include <linux/socket.h>
27#include <linux/sock_diag.h>
28#include <linux/bpf.h>
29#include <linux/if_link.h>
30#include <linux/tls.h>
31#include <assert.h>
32#include <libgen.h>
33
34#include <getopt.h>
35
36#include <bpf/bpf.h>
37#include <bpf/libbpf.h>
38
39#include "bpf_util.h"
40#include "bpf_rlimit.h"
41#include "cgroup_helpers.h"
42
43int running;
44static void running_handler(int a);
45
46#ifndef TCP_ULP
47# define TCP_ULP 31
48#endif
49#ifndef SOL_TLS
50# define SOL_TLS 282
51#endif
52
53
54#define S1_PORT 10000
55#define S2_PORT 10001
56
57#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
58#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
59#define CG_PATH "/sockmap"
60
61
62int s1, s2, c1, c2, p1, p2;
63int test_cnt;
64int passed;
65int failed;
66int map_fd[8];
67struct bpf_map *maps[8];
68int prog_fd[11];
69
70int txmsg_pass;
71int txmsg_noisy;
72int txmsg_redir;
73int txmsg_redir_noisy;
74int txmsg_drop;
75int txmsg_apply;
76int txmsg_cork;
77int txmsg_start;
78int txmsg_end;
79int txmsg_start_push;
80int txmsg_end_push;
81int txmsg_start_pop;
82int txmsg_pop;
83int txmsg_ingress;
84int txmsg_skb;
85int ktls;
86int peek_flag;
87
88static const struct option long_options[] = {
89 {"help", no_argument, NULL, 'h' },
90 {"cgroup", required_argument, NULL, 'c' },
91 {"rate", required_argument, NULL, 'r' },
92 {"verbose", no_argument, NULL, 'v' },
93 {"iov_count", required_argument, NULL, 'i' },
94 {"length", required_argument, NULL, 'l' },
95 {"test", required_argument, NULL, 't' },
96 {"data_test", no_argument, NULL, 'd' },
97 {"txmsg", no_argument, &txmsg_pass, 1 },
98 {"txmsg_noisy", no_argument, &txmsg_noisy, 1 },
99 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
100 {"txmsg_redir_noisy", no_argument, &txmsg_redir_noisy, 1},
101 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
102 {"txmsg_apply", required_argument, NULL, 'a'},
103 {"txmsg_cork", required_argument, NULL, 'k'},
104 {"txmsg_start", required_argument, NULL, 's'},
105 {"txmsg_end", required_argument, NULL, 'e'},
106 {"txmsg_start_push", required_argument, NULL, 'p'},
107 {"txmsg_end_push", required_argument, NULL, 'q'},
108 {"txmsg_start_pop", required_argument, NULL, 'w'},
109 {"txmsg_pop", required_argument, NULL, 'x'},
110 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
111 {"txmsg_skb", no_argument, &txmsg_skb, 1 },
112 {"ktls", no_argument, &ktls, 1 },
113 {"peek", no_argument, &peek_flag, 1 },
114 {0, 0, NULL, 0 }
115};
116
117static void usage(char *argv[])
118{
119 int i;
120
121 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
122 printf(" options:\n");
123 for (i = 0; long_options[i].name != 0; i++) {
124 printf(" --%-12s", long_options[i].name);
125 if (long_options[i].flag != NULL)
126 printf(" flag (internal value:%d)\n",
127 *long_options[i].flag);
128 else
129 printf(" -%c\n", long_options[i].val);
130 }
131 printf("\n");
132}
133
134char *sock_to_string(int s)
135{
136 if (s == c1)
137 return "client1";
138 else if (s == c2)
139 return "client2";
140 else if (s == s1)
141 return "server1";
142 else if (s == s2)
143 return "server2";
144 else if (s == p1)
145 return "peer1";
146 else if (s == p2)
147 return "peer2";
148 else
149 return "unknown";
150}
151
152static int sockmap_init_ktls(int verbose, int s)
153{
154 struct tls12_crypto_info_aes_gcm_128 tls_tx = {
155 .info = {
156 .version = TLS_1_2_VERSION,
157 .cipher_type = TLS_CIPHER_AES_GCM_128,
158 },
159 };
160 struct tls12_crypto_info_aes_gcm_128 tls_rx = {
161 .info = {
162 .version = TLS_1_2_VERSION,
163 .cipher_type = TLS_CIPHER_AES_GCM_128,
164 },
165 };
166 int so_buf = 6553500;
167 int err;
168
169 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
170 if (err) {
171 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
172 return -EINVAL;
173 }
174 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
175 if (err) {
176 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
177 return -EINVAL;
178 }
179 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
180 if (err) {
181 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
182 return -EINVAL;
183 }
184 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
185 if (err) {
186 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
187 return -EINVAL;
188 }
189 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
190 if (err) {
191 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
192 return -EINVAL;
193 }
194
195 if (verbose)
196 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
197 return 0;
198}
199static int sockmap_init_sockets(int verbose)
200{
201 int i, err, one = 1;
202 struct sockaddr_in addr;
203 int *fds[4] = {&s1, &s2, &c1, &c2};
204
205 s1 = s2 = p1 = p2 = c1 = c2 = 0;
206
207
208 for (i = 0; i < 4; i++) {
209 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
210 if (*fds[i] < 0) {
211 perror("socket s1 failed()");
212 return errno;
213 }
214 }
215
216
217 for (i = 0; i < 2; i++) {
218 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
219 (char *)&one, sizeof(one));
220 if (err) {
221 perror("setsockopt failed()");
222 return errno;
223 }
224 }
225
226
227 for (i = 0; i < 2; i++) {
228 err = ioctl(*fds[i], FIONBIO, (char *)&one);
229 if (err < 0) {
230 perror("ioctl s1 failed()");
231 return errno;
232 }
233 }
234
235
236 memset(&addr, 0, sizeof(struct sockaddr_in));
237 addr.sin_family = AF_INET;
238 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
239
240 addr.sin_port = htons(S1_PORT);
241 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
242 if (err < 0) {
243 perror("bind s1 failed()");
244 return errno;
245 }
246
247 addr.sin_port = htons(S2_PORT);
248 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
249 if (err < 0) {
250 perror("bind s2 failed()");
251 return errno;
252 }
253
254
255 addr.sin_port = htons(S1_PORT);
256 err = listen(s1, 32);
257 if (err < 0) {
258 perror("listen s1 failed()");
259 return errno;
260 }
261
262 addr.sin_port = htons(S2_PORT);
263 err = listen(s2, 32);
264 if (err < 0) {
265 perror("listen s1 failed()");
266 return errno;
267 }
268
269
270 addr.sin_port = htons(S1_PORT);
271 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
272 if (err < 0 && errno != EINPROGRESS) {
273 perror("connect c1 failed()");
274 return errno;
275 }
276
277 addr.sin_port = htons(S2_PORT);
278 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
279 if (err < 0 && errno != EINPROGRESS) {
280 perror("connect c2 failed()");
281 return errno;
282 } else if (err < 0) {
283 err = 0;
284 }
285
286
287 p1 = accept(s1, NULL, NULL);
288 if (p1 < 0) {
289 perror("accept s1 failed()");
290 return errno;
291 }
292
293 p2 = accept(s2, NULL, NULL);
294 if (p2 < 0) {
295 perror("accept s1 failed()");
296 return errno;
297 }
298
299 if (verbose) {
300 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
301 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
302 c1, s1, c2, s2);
303 }
304 return 0;
305}
306
307struct msg_stats {
308 size_t bytes_sent;
309 size_t bytes_recvd;
310 struct timespec start;
311 struct timespec end;
312};
313
314struct sockmap_options {
315 int verbose;
316 bool base;
317 bool sendpage;
318 bool data_test;
319 bool drop_expected;
320 int iov_count;
321 int iov_length;
322 int rate;
323};
324
325static int msg_loop_sendpage(int fd, int iov_length, int cnt,
326 struct msg_stats *s,
327 struct sockmap_options *opt)
328{
329 bool drop = opt->drop_expected;
330 unsigned char k = 0;
331 FILE *file;
332 int i, fp;
333
334 file = tmpfile();
335 if (!file) {
336 perror("create file for sendpage");
337 return 1;
338 }
339 for (i = 0; i < iov_length * cnt; i++, k++)
340 fwrite(&k, sizeof(char), 1, file);
341 fflush(file);
342 fseek(file, 0, SEEK_SET);
343
344 fp = fileno(file);
345
346 clock_gettime(CLOCK_MONOTONIC, &s->start);
347 for (i = 0; i < cnt; i++) {
348 int sent = sendfile(fd, fp, NULL, iov_length);
349
350 if (!drop && sent < 0) {
351 perror("send loop error");
352 fclose(file);
353 return sent;
354 } else if (drop && sent >= 0) {
355 printf("sendpage loop error expected: %i\n", sent);
356 fclose(file);
357 return -EIO;
358 }
359
360 if (sent > 0)
361 s->bytes_sent += sent;
362 }
363 clock_gettime(CLOCK_MONOTONIC, &s->end);
364 fclose(file);
365 return 0;
366}
367
368static void msg_free_iov(struct msghdr *msg)
369{
370 int i;
371
372 for (i = 0; i < msg->msg_iovlen; i++)
373 free(msg->msg_iov[i].iov_base);
374 free(msg->msg_iov);
375 msg->msg_iov = NULL;
376 msg->msg_iovlen = 0;
377}
378
379static int msg_alloc_iov(struct msghdr *msg,
380 int iov_count, int iov_length,
381 bool data, bool xmit)
382{
383 unsigned char k = 0;
384 struct iovec *iov;
385 int i;
386
387 iov = calloc(iov_count, sizeof(struct iovec));
388 if (!iov)
389 return errno;
390
391 for (i = 0; i < iov_count; i++) {
392 unsigned char *d = calloc(iov_length, sizeof(char));
393
394 if (!d) {
395 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
396 goto unwind_iov;
397 }
398 iov[i].iov_base = d;
399 iov[i].iov_len = iov_length;
400
401 if (data && xmit) {
402 int j;
403
404 for (j = 0; j < iov_length; j++)
405 d[j] = k++;
406 }
407 }
408
409 msg->msg_iov = iov;
410 msg->msg_iovlen = iov_count;
411
412 return 0;
413unwind_iov:
414 for (i--; i >= 0 ; i--)
415 free(msg->msg_iov[i].iov_base);
416 return -ENOMEM;
417}
418
419static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
420{
421 int i, j, bytes_cnt = 0;
422 unsigned char k = 0;
423
424 for (i = 0; i < msg->msg_iovlen; i++) {
425 unsigned char *d = msg->msg_iov[i].iov_base;
426
427 for (j = 0;
428 j < msg->msg_iov[i].iov_len && size; j++) {
429 if (d[j] != k++) {
430 fprintf(stderr,
431 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
432 i, j, d[j], k - 1, d[j+1], k);
433 return -EIO;
434 }
435 bytes_cnt++;
436 if (bytes_cnt == chunk_sz) {
437 k = 0;
438 bytes_cnt = 0;
439 }
440 size--;
441 }
442 }
443 return 0;
444}
445
446static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
447 struct msg_stats *s, bool tx,
448 struct sockmap_options *opt)
449{
450 struct msghdr msg = {0}, msg_peek = {0};
451 int err, i, flags = MSG_NOSIGNAL;
452 bool drop = opt->drop_expected;
453 bool data = opt->data_test;
454
455 err = msg_alloc_iov(&msg, iov_count, iov_length, data, tx);
456 if (err)
457 goto out_errno;
458 if (peek_flag) {
459 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
460 if (err)
461 goto out_errno;
462 }
463
464 if (tx) {
465 clock_gettime(CLOCK_MONOTONIC, &s->start);
466 for (i = 0; i < cnt; i++) {
467 int sent = sendmsg(fd, &msg, flags);
468
469 if (!drop && sent < 0) {
470 perror("send loop error");
471 goto out_errno;
472 } else if (drop && sent >= 0) {
473 printf("send loop error expected: %i\n", sent);
474 errno = -EIO;
475 goto out_errno;
476 }
477 if (sent > 0)
478 s->bytes_sent += sent;
479 }
480 clock_gettime(CLOCK_MONOTONIC, &s->end);
481 } else {
482 int slct, recvp = 0, recv, max_fd = fd;
483 float total_bytes, txmsg_pop_total;
484 int fd_flags = O_NONBLOCK;
485 struct timeval timeout;
486 fd_set w;
487
488 fcntl(fd, fd_flags);
489
490
491
492
493
494
495
496
497
498
499 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
500 txmsg_pop_total = txmsg_pop;
501 if (txmsg_apply)
502 txmsg_pop_total *= (total_bytes / txmsg_apply);
503 total_bytes -= txmsg_pop_total;
504 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
505 if (err < 0)
506 perror("recv start time");
507 while (s->bytes_recvd < total_bytes) {
508 if (txmsg_cork) {
509 timeout.tv_sec = 0;
510 timeout.tv_usec = 300000;
511 } else {
512 timeout.tv_sec = 3;
513 timeout.tv_usec = 0;
514 }
515
516
517 FD_ZERO(&w);
518 FD_SET(fd, &w);
519
520 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
521 if (slct == -1) {
522 perror("select()");
523 clock_gettime(CLOCK_MONOTONIC, &s->end);
524 goto out_errno;
525 } else if (!slct) {
526 if (opt->verbose)
527 fprintf(stderr, "unexpected timeout: recved %zu/%f pop_total %f\n", s->bytes_recvd, total_bytes, txmsg_pop_total);
528 errno = -EIO;
529 clock_gettime(CLOCK_MONOTONIC, &s->end);
530 goto out_errno;
531 }
532
533 errno = 0;
534 if (peek_flag) {
535 flags |= MSG_PEEK;
536 recvp = recvmsg(fd, &msg_peek, flags);
537 if (recvp < 0) {
538 if (errno != EWOULDBLOCK) {
539 clock_gettime(CLOCK_MONOTONIC, &s->end);
540 goto out_errno;
541 }
542 }
543 flags = 0;
544 }
545
546 recv = recvmsg(fd, &msg, flags);
547 if (recv < 0) {
548 if (errno != EWOULDBLOCK) {
549 clock_gettime(CLOCK_MONOTONIC, &s->end);
550 perror("recv failed()");
551 goto out_errno;
552 }
553 }
554
555 s->bytes_recvd += recv;
556
557 if (data) {
558 int chunk_sz = opt->sendpage ?
559 iov_length * cnt :
560 iov_length * iov_count;
561
562 errno = msg_verify_data(&msg, recv, chunk_sz);
563 if (errno) {
564 perror("data verify msg failed");
565 goto out_errno;
566 }
567 if (recvp) {
568 errno = msg_verify_data(&msg_peek,
569 recvp,
570 chunk_sz);
571 if (errno) {
572 perror("data verify msg_peek failed");
573 goto out_errno;
574 }
575 }
576 }
577 }
578 clock_gettime(CLOCK_MONOTONIC, &s->end);
579 }
580
581 msg_free_iov(&msg);
582 msg_free_iov(&msg_peek);
583 return err;
584out_errno:
585 msg_free_iov(&msg);
586 msg_free_iov(&msg_peek);
587 return errno;
588}
589
590static float giga = 1000000000;
591
592static inline float sentBps(struct msg_stats s)
593{
594 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
595}
596
597static inline float recvdBps(struct msg_stats s)
598{
599 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
600}
601
602static int sendmsg_test(struct sockmap_options *opt)
603{
604 float sent_Bps = 0, recvd_Bps = 0;
605 int rx_fd, txpid, rxpid, err = 0;
606 struct msg_stats s = {0};
607 int iov_count = opt->iov_count;
608 int iov_buf = opt->iov_length;
609 int rx_status, tx_status;
610 int cnt = opt->rate;
611
612 errno = 0;
613
614 if (opt->base)
615 rx_fd = p1;
616 else
617 rx_fd = p2;
618
619 if (ktls) {
620
621
622
623
624 if (!txmsg_redir || (txmsg_redir && txmsg_ingress)) {
625 err = sockmap_init_ktls(opt->verbose, rx_fd);
626 if (err)
627 return err;
628 }
629 err = sockmap_init_ktls(opt->verbose, c1);
630 if (err)
631 return err;
632 }
633
634 rxpid = fork();
635 if (rxpid == 0) {
636 if (opt->drop_expected)
637 exit(0);
638
639 if (opt->sendpage)
640 iov_count = 1;
641 err = msg_loop(rx_fd, iov_count, iov_buf,
642 cnt, &s, false, opt);
643 if (opt->verbose)
644 fprintf(stderr,
645 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
646 iov_count, iov_buf, cnt, err);
647 if (s.end.tv_sec - s.start.tv_sec) {
648 sent_Bps = sentBps(s);
649 recvd_Bps = recvdBps(s);
650 }
651 if (opt->verbose)
652 fprintf(stdout,
653 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
654 s.bytes_sent, sent_Bps, sent_Bps/giga,
655 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
656 peek_flag ? "(peek_msg)" : "");
657 if (err && txmsg_cork)
658 err = 0;
659 exit(err ? 1 : 0);
660 } else if (rxpid == -1) {
661 perror("msg_loop_rx");
662 return errno;
663 }
664
665 txpid = fork();
666 if (txpid == 0) {
667 if (opt->sendpage)
668 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
669 else
670 err = msg_loop(c1, iov_count, iov_buf,
671 cnt, &s, true, opt);
672
673 if (err)
674 fprintf(stderr,
675 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
676 iov_count, iov_buf, cnt, err);
677 if (s.end.tv_sec - s.start.tv_sec) {
678 sent_Bps = sentBps(s);
679 recvd_Bps = recvdBps(s);
680 }
681 if (opt->verbose)
682 fprintf(stdout,
683 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
684 s.bytes_sent, sent_Bps, sent_Bps/giga,
685 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
686 exit(err ? 1 : 0);
687 } else if (txpid == -1) {
688 perror("msg_loop_tx");
689 return errno;
690 }
691
692 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
693 assert(waitpid(txpid, &tx_status, 0) == txpid);
694 if (WIFEXITED(rx_status)) {
695 err = WEXITSTATUS(rx_status);
696 if (err) {
697 fprintf(stderr, "rx thread exited with err %d. ", err);
698 goto out;
699 }
700 }
701 if (WIFEXITED(tx_status)) {
702 err = WEXITSTATUS(tx_status);
703 if (err)
704 fprintf(stderr, "tx thread exited with err %d. ", err);
705 }
706out:
707 return err;
708}
709
710static int forever_ping_pong(int rate, struct sockmap_options *opt)
711{
712 struct timeval timeout;
713 char buf[1024] = {0};
714 int sc;
715
716 timeout.tv_sec = 10;
717 timeout.tv_usec = 0;
718
719
720 sc = send(c1, buf, sizeof(buf), 0);
721 if (sc < 0) {
722 perror("send failed()");
723 return sc;
724 }
725
726 do {
727 int s, rc, i, max_fd = p2;
728 fd_set w;
729
730
731 FD_ZERO(&w);
732 FD_SET(c1, &w);
733 FD_SET(c2, &w);
734 FD_SET(p1, &w);
735 FD_SET(p2, &w);
736
737 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
738 if (s == -1) {
739 perror("select()");
740 break;
741 } else if (!s) {
742 fprintf(stderr, "unexpected timeout\n");
743 break;
744 }
745
746 for (i = 0; i <= max_fd && s > 0; ++i) {
747 if (!FD_ISSET(i, &w))
748 continue;
749
750 s--;
751
752 rc = recv(i, buf, sizeof(buf), 0);
753 if (rc < 0) {
754 if (errno != EWOULDBLOCK) {
755 perror("recv failed()");
756 return rc;
757 }
758 }
759
760 if (rc == 0) {
761 close(i);
762 break;
763 }
764
765 sc = send(i, buf, rc, 0);
766 if (sc < 0) {
767 perror("send failed()");
768 return sc;
769 }
770 }
771
772 if (rate)
773 sleep(rate);
774
775 if (opt->verbose) {
776 printf(".");
777 fflush(stdout);
778
779 }
780 } while (running);
781
782 return 0;
783}
784
785enum {
786 PING_PONG,
787 SENDMSG,
788 BASE,
789 BASE_SENDPAGE,
790 SENDPAGE,
791};
792
793static int run_options(struct sockmap_options *options, int cg_fd, int test)
794{
795 int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
796
797
798 if (test == BASE || test == BASE_SENDPAGE)
799 goto run;
800
801
802 err = bpf_prog_attach(prog_fd[0], map_fd[0],
803 BPF_SK_SKB_STREAM_PARSER, 0);
804 if (err) {
805 fprintf(stderr,
806 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
807 prog_fd[0], map_fd[0], err, strerror(errno));
808 return err;
809 }
810
811 err = bpf_prog_attach(prog_fd[1], map_fd[0],
812 BPF_SK_SKB_STREAM_VERDICT, 0);
813 if (err) {
814 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
815 err, strerror(errno));
816 return err;
817 }
818
819
820 err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
821 if (err) {
822 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
823 err, strerror(errno));
824 return err;
825 }
826
827run:
828 err = sockmap_init_sockets(options->verbose);
829 if (err) {
830 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
831 goto out;
832 }
833
834
835 if (txmsg_pass)
836 tx_prog_fd = prog_fd[3];
837 else if (txmsg_noisy)
838 tx_prog_fd = prog_fd[4];
839 else if (txmsg_redir)
840 tx_prog_fd = prog_fd[5];
841 else if (txmsg_redir_noisy)
842 tx_prog_fd = prog_fd[6];
843 else if (txmsg_drop)
844 tx_prog_fd = prog_fd[9];
845
846 else if (txmsg_apply)
847 tx_prog_fd = prog_fd[7];
848 else if (txmsg_cork)
849 tx_prog_fd = prog_fd[8];
850 else
851 tx_prog_fd = 0;
852
853 if (tx_prog_fd) {
854 int redir_fd, i = 0;
855
856 err = bpf_prog_attach(tx_prog_fd,
857 map_fd[1], BPF_SK_MSG_VERDICT, 0);
858 if (err) {
859 fprintf(stderr,
860 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
861 err, strerror(errno));
862 goto out;
863 }
864
865 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
866 if (err) {
867 fprintf(stderr,
868 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
869 err, strerror(errno));
870 goto out;
871 }
872
873 if (txmsg_redir || txmsg_redir_noisy)
874 redir_fd = c2;
875 else
876 redir_fd = c1;
877
878 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
879 if (err) {
880 fprintf(stderr,
881 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
882 err, strerror(errno));
883 goto out;
884 }
885
886 if (txmsg_apply) {
887 err = bpf_map_update_elem(map_fd[3],
888 &i, &txmsg_apply, BPF_ANY);
889 if (err) {
890 fprintf(stderr,
891 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
892 err, strerror(errno));
893 goto out;
894 }
895 }
896
897 if (txmsg_cork) {
898 err = bpf_map_update_elem(map_fd[4],
899 &i, &txmsg_cork, BPF_ANY);
900 if (err) {
901 fprintf(stderr,
902 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
903 err, strerror(errno));
904 goto out;
905 }
906 }
907
908 if (txmsg_start) {
909 err = bpf_map_update_elem(map_fd[5],
910 &i, &txmsg_start, BPF_ANY);
911 if (err) {
912 fprintf(stderr,
913 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
914 err, strerror(errno));
915 goto out;
916 }
917 }
918
919 if (txmsg_end) {
920 i = 1;
921 err = bpf_map_update_elem(map_fd[5],
922 &i, &txmsg_end, BPF_ANY);
923 if (err) {
924 fprintf(stderr,
925 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
926 err, strerror(errno));
927 goto out;
928 }
929 }
930
931 if (txmsg_start_push) {
932 i = 2;
933 err = bpf_map_update_elem(map_fd[5],
934 &i, &txmsg_start_push, BPF_ANY);
935 if (err) {
936 fprintf(stderr,
937 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
938 err, strerror(errno));
939 goto out;
940 }
941 }
942
943 if (txmsg_end_push) {
944 i = 3;
945 err = bpf_map_update_elem(map_fd[5],
946 &i, &txmsg_end_push, BPF_ANY);
947 if (err) {
948 fprintf(stderr,
949 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
950 txmsg_end_push, i, err, strerror(errno));
951 goto out;
952 }
953 }
954
955 if (txmsg_start_pop) {
956 i = 4;
957 err = bpf_map_update_elem(map_fd[5],
958 &i, &txmsg_start_pop, BPF_ANY);
959 if (err) {
960 fprintf(stderr,
961 "ERROR: bpf_map_update_elem %i@%i (txmsg_start_pop): %d (%s)\n",
962 txmsg_start_pop, i, err, strerror(errno));
963 goto out;
964 }
965 } else {
966 i = 4;
967 bpf_map_update_elem(map_fd[5],
968 &i, &txmsg_start_pop, BPF_ANY);
969 }
970
971 if (txmsg_pop) {
972 i = 5;
973 err = bpf_map_update_elem(map_fd[5],
974 &i, &txmsg_pop, BPF_ANY);
975 if (err) {
976 fprintf(stderr,
977 "ERROR: bpf_map_update_elem %i@%i (txmsg_pop): %d (%s)\n",
978 txmsg_pop, i, err, strerror(errno));
979 goto out;
980 }
981 } else {
982 i = 5;
983 bpf_map_update_elem(map_fd[5],
984 &i, &txmsg_pop, BPF_ANY);
985
986 }
987
988 if (txmsg_ingress) {
989 int in = BPF_F_INGRESS;
990
991 i = 0;
992 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
993 if (err) {
994 fprintf(stderr,
995 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
996 err, strerror(errno));
997 }
998 i = 1;
999 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
1000 if (err) {
1001 fprintf(stderr,
1002 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
1003 err, strerror(errno));
1004 }
1005 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
1006 if (err) {
1007 fprintf(stderr,
1008 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
1009 err, strerror(errno));
1010 }
1011
1012 i = 2;
1013 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
1014 if (err) {
1015 fprintf(stderr,
1016 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
1017 err, strerror(errno));
1018 }
1019 }
1020
1021 if (txmsg_skb) {
1022 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
1023 p2 : p1;
1024 int ingress = BPF_F_INGRESS;
1025
1026 i = 0;
1027 err = bpf_map_update_elem(map_fd[7],
1028 &i, &ingress, BPF_ANY);
1029 if (err) {
1030 fprintf(stderr,
1031 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
1032 err, strerror(errno));
1033 }
1034
1035 i = 3;
1036 err = bpf_map_update_elem(map_fd[0],
1037 &i, &skb_fd, BPF_ANY);
1038 if (err) {
1039 fprintf(stderr,
1040 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
1041 err, strerror(errno));
1042 }
1043 }
1044 }
1045
1046 if (txmsg_drop)
1047 options->drop_expected = true;
1048
1049 if (test == PING_PONG)
1050 err = forever_ping_pong(options->rate, options);
1051 else if (test == SENDMSG) {
1052 options->base = false;
1053 options->sendpage = false;
1054 err = sendmsg_test(options);
1055 } else if (test == SENDPAGE) {
1056 options->base = false;
1057 options->sendpage = true;
1058 err = sendmsg_test(options);
1059 } else if (test == BASE) {
1060 options->base = true;
1061 options->sendpage = false;
1062 err = sendmsg_test(options);
1063 } else if (test == BASE_SENDPAGE) {
1064 options->base = true;
1065 options->sendpage = true;
1066 err = sendmsg_test(options);
1067 } else
1068 fprintf(stderr, "unknown test\n");
1069out:
1070
1071 bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS);
1072 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1073 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1074 if (tx_prog_fd >= 0)
1075 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1076
1077 for (i = 0; i < 8; i++) {
1078 key = next_key = 0;
1079 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1080 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1081 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1082 key = next_key;
1083 }
1084 }
1085
1086 close(s1);
1087 close(s2);
1088 close(p1);
1089 close(p2);
1090 close(c1);
1091 close(c2);
1092 return err;
1093}
1094
1095static char *test_to_str(int test)
1096{
1097 switch (test) {
1098 case SENDMSG:
1099 return "sendmsg";
1100 case SENDPAGE:
1101 return "sendpage";
1102 }
1103 return "unknown";
1104}
1105
1106#define OPTSTRING 60
1107static void test_options(char *options)
1108{
1109 char tstr[OPTSTRING];
1110
1111 memset(options, 0, OPTSTRING);
1112
1113 if (txmsg_pass)
1114 strncat(options, "pass,", OPTSTRING);
1115 if (txmsg_noisy)
1116 strncat(options, "pass_noisy,", OPTSTRING);
1117 if (txmsg_redir)
1118 strncat(options, "redir,", OPTSTRING);
1119 if (txmsg_redir_noisy)
1120 strncat(options, "redir_noisy,", OPTSTRING);
1121 if (txmsg_drop)
1122 strncat(options, "drop,", OPTSTRING);
1123 if (txmsg_apply) {
1124 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1125 strncat(options, tstr, OPTSTRING);
1126 }
1127 if (txmsg_cork) {
1128 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1129 strncat(options, tstr, OPTSTRING);
1130 }
1131 if (txmsg_start) {
1132 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1133 strncat(options, tstr, OPTSTRING);
1134 }
1135 if (txmsg_end) {
1136 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1137 strncat(options, tstr, OPTSTRING);
1138 }
1139 if (txmsg_start_pop) {
1140 snprintf(tstr, OPTSTRING, "pop (%d,%d),",
1141 txmsg_start_pop, txmsg_start_pop + txmsg_pop);
1142 strncat(options, tstr, OPTSTRING);
1143 }
1144 if (txmsg_ingress)
1145 strncat(options, "ingress,", OPTSTRING);
1146 if (txmsg_skb)
1147 strncat(options, "skb,", OPTSTRING);
1148 if (ktls)
1149 strncat(options, "ktls,", OPTSTRING);
1150 if (peek_flag)
1151 strncat(options, "peek,", OPTSTRING);
1152}
1153
1154static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1155{
1156 char *options = calloc(OPTSTRING, sizeof(char));
1157 int err;
1158
1159 if (test == SENDPAGE)
1160 opt->sendpage = true;
1161 else
1162 opt->sendpage = false;
1163
1164 if (txmsg_drop)
1165 opt->drop_expected = true;
1166 else
1167 opt->drop_expected = false;
1168
1169 test_options(options);
1170
1171 fprintf(stdout,
1172 "[TEST %i]: (%i, %i, %i, %s, %s): ",
1173 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1174 test_to_str(test), options);
1175 fflush(stdout);
1176 err = run_options(opt, cgrp, test);
1177 fprintf(stdout, "%s\n", !err ? "PASS" : "FAILED");
1178 test_cnt++;
1179 !err ? passed++ : failed++;
1180 free(options);
1181 return err;
1182}
1183
1184static int test_exec(int cgrp, struct sockmap_options *opt)
1185{
1186 int err = __test_exec(cgrp, SENDMSG, opt);
1187
1188 if (err)
1189 goto out;
1190
1191 err = __test_exec(cgrp, SENDPAGE, opt);
1192out:
1193 return err;
1194}
1195
1196static int test_loop(int cgrp)
1197{
1198 struct sockmap_options opt;
1199
1200 int err, i, l, r;
1201
1202 opt.verbose = 0;
1203 opt.base = false;
1204 opt.sendpage = false;
1205 opt.data_test = false;
1206 opt.drop_expected = false;
1207 opt.iov_count = 0;
1208 opt.iov_length = 0;
1209 opt.rate = 0;
1210
1211 r = 1;
1212 for (i = 1; i < 100; i += 33) {
1213 for (l = 1; l < 100; l += 33) {
1214 opt.rate = r;
1215 opt.iov_count = i;
1216 opt.iov_length = l;
1217 err = test_exec(cgrp, &opt);
1218 if (err)
1219 goto out;
1220 }
1221 }
1222 sched_yield();
1223out:
1224 return err;
1225}
1226
1227static int test_txmsg(int cgrp)
1228{
1229 int err;
1230
1231 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
1232 txmsg_apply = txmsg_cork = 0;
1233 txmsg_ingress = txmsg_skb = 0;
1234
1235 txmsg_pass = 1;
1236 err = test_loop(cgrp);
1237 txmsg_pass = 0;
1238 if (err)
1239 goto out;
1240
1241 txmsg_redir = 1;
1242 err = test_loop(cgrp);
1243 txmsg_redir = 0;
1244 if (err)
1245 goto out;
1246
1247 txmsg_drop = 1;
1248 err = test_loop(cgrp);
1249 txmsg_drop = 0;
1250 if (err)
1251 goto out;
1252
1253 txmsg_redir = 1;
1254 txmsg_ingress = 1;
1255 err = test_loop(cgrp);
1256 txmsg_redir = 0;
1257 txmsg_ingress = 0;
1258 if (err)
1259 goto out;
1260out:
1261 txmsg_pass = 0;
1262 txmsg_redir = 0;
1263 txmsg_drop = 0;
1264 return err;
1265}
1266
1267static int test_send(struct sockmap_options *opt, int cgrp)
1268{
1269 int err;
1270
1271 opt->iov_length = 1;
1272 opt->iov_count = 1;
1273 opt->rate = 1;
1274 err = test_exec(cgrp, opt);
1275 if (err)
1276 goto out;
1277
1278 opt->iov_length = 1;
1279 opt->iov_count = 1024;
1280 opt->rate = 1;
1281 err = test_exec(cgrp, opt);
1282 if (err)
1283 goto out;
1284
1285 opt->iov_length = 1024;
1286 opt->iov_count = 1;
1287 opt->rate = 1;
1288 err = test_exec(cgrp, opt);
1289 if (err)
1290 goto out;
1291
1292 opt->iov_length = 1;
1293 opt->iov_count = 1;
1294 opt->rate = 512;
1295 err = test_exec(cgrp, opt);
1296 if (err)
1297 goto out;
1298
1299 opt->iov_length = 256;
1300 opt->iov_count = 1024;
1301 opt->rate = 2;
1302 err = test_exec(cgrp, opt);
1303 if (err)
1304 goto out;
1305
1306 opt->rate = 100;
1307 opt->iov_count = 1;
1308 opt->iov_length = 5;
1309 err = test_exec(cgrp, opt);
1310 if (err)
1311 goto out;
1312out:
1313 sched_yield();
1314 return err;
1315}
1316
1317static int test_mixed(int cgrp)
1318{
1319 struct sockmap_options opt = {0};
1320 int err;
1321
1322 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
1323 txmsg_apply = txmsg_cork = 0;
1324 txmsg_start = txmsg_end = 0;
1325 txmsg_start_push = txmsg_end_push = 0;
1326 txmsg_start_pop = txmsg_pop = 0;
1327
1328
1329 txmsg_pass = 1;
1330 txmsg_redir = 0;
1331 txmsg_apply = 1;
1332 txmsg_cork = 0;
1333 err = test_send(&opt, cgrp);
1334 if (err)
1335 goto out;
1336
1337 txmsg_pass = 1;
1338 txmsg_redir = 0;
1339 txmsg_apply = 0;
1340 txmsg_cork = 1;
1341 err = test_send(&opt, cgrp);
1342 if (err)
1343 goto out;
1344
1345 txmsg_pass = 1;
1346 txmsg_redir = 0;
1347 txmsg_apply = 1;
1348 txmsg_cork = 1;
1349 err = test_send(&opt, cgrp);
1350 if (err)
1351 goto out;
1352
1353 txmsg_pass = 1;
1354 txmsg_redir = 0;
1355 txmsg_apply = 1024;
1356 txmsg_cork = 0;
1357 err = test_send(&opt, cgrp);
1358 if (err)
1359 goto out;
1360
1361 txmsg_pass = 1;
1362 txmsg_redir = 0;
1363 txmsg_apply = 0;
1364 txmsg_cork = 1024;
1365 err = test_send(&opt, cgrp);
1366 if (err)
1367 goto out;
1368
1369 txmsg_pass = 1;
1370 txmsg_redir = 0;
1371 txmsg_apply = 1024;
1372 txmsg_cork = 1024;
1373 err = test_send(&opt, cgrp);
1374 if (err)
1375 goto out;
1376
1377 txmsg_pass = 1;
1378 txmsg_redir = 0;
1379 txmsg_cork = 4096;
1380 txmsg_apply = 4096;
1381 err = test_send(&opt, cgrp);
1382 if (err)
1383 goto out;
1384
1385 txmsg_pass = 0;
1386 txmsg_redir = 1;
1387 txmsg_apply = 1;
1388 txmsg_cork = 0;
1389 err = test_send(&opt, cgrp);
1390 if (err)
1391 goto out;
1392
1393 txmsg_pass = 0;
1394 txmsg_redir = 1;
1395 txmsg_apply = 0;
1396 txmsg_cork = 1;
1397 err = test_send(&opt, cgrp);
1398 if (err)
1399 goto out;
1400
1401 txmsg_pass = 0;
1402 txmsg_redir = 1;
1403 txmsg_apply = 1024;
1404 txmsg_cork = 0;
1405 err = test_send(&opt, cgrp);
1406 if (err)
1407 goto out;
1408
1409 txmsg_pass = 0;
1410 txmsg_redir = 1;
1411 txmsg_apply = 0;
1412 txmsg_cork = 1024;
1413 err = test_send(&opt, cgrp);
1414 if (err)
1415 goto out;
1416
1417 txmsg_pass = 0;
1418 txmsg_redir = 1;
1419 txmsg_apply = 1024;
1420 txmsg_cork = 1024;
1421 err = test_send(&opt, cgrp);
1422 if (err)
1423 goto out;
1424
1425 txmsg_pass = 0;
1426 txmsg_redir = 1;
1427 txmsg_cork = 4096;
1428 txmsg_apply = 4096;
1429 err = test_send(&opt, cgrp);
1430 if (err)
1431 goto out;
1432out:
1433 return err;
1434}
1435
1436static int test_start_end(int cgrp)
1437{
1438 struct sockmap_options opt = {0};
1439 int err, i;
1440
1441
1442 txmsg_start = 1;
1443 txmsg_end = 2;
1444 txmsg_start_push = 1;
1445 txmsg_end_push = 2;
1446 txmsg_start_pop = 1;
1447 txmsg_pop = 1;
1448 err = test_txmsg(cgrp);
1449 if (err)
1450 goto out;
1451
1452
1453 txmsg_start = 1;
1454 txmsg_end = 2;
1455 txmsg_start_push = 1;
1456 txmsg_end_push = 3;
1457 txmsg_start_pop = 1;
1458 txmsg_pop = 1;
1459 err = test_txmsg(cgrp);
1460 if (err)
1461 goto out;
1462
1463
1464 opt.rate = 16;
1465 opt.iov_count = 1;
1466 opt.iov_length = 100;
1467 txmsg_cork = 1600;
1468
1469 txmsg_start_pop = 0;
1470 txmsg_pop = 0;
1471
1472 for (i = 99; i <= 1600; i += 500) {
1473 txmsg_start = 0;
1474 txmsg_end = i;
1475 txmsg_start_push = 0;
1476 txmsg_end_push = i;
1477 err = test_exec(cgrp, &opt);
1478 if (err)
1479 goto out;
1480 }
1481
1482
1483 for (i = 99; i <= 1600; i += 500) {
1484 txmsg_start_pop = 10;
1485 txmsg_pop = i;
1486 err = test_exec(cgrp, &opt);
1487 if (err)
1488 goto out;
1489 }
1490 txmsg_start_pop = 0;
1491 txmsg_pop = 0;
1492
1493
1494 for (i = 199; i <= 1600; i += 500) {
1495 txmsg_start = 100;
1496 txmsg_end = i;
1497 txmsg_start_push = 100;
1498 txmsg_end_push = i;
1499 err = test_exec(cgrp, &opt);
1500 if (err)
1501 goto out;
1502 }
1503
1504
1505 txmsg_start = 1500;
1506 txmsg_end = 1600;
1507 txmsg_start_push = 1500;
1508 txmsg_end_push = 1600;
1509 err = test_exec(cgrp, &opt);
1510 if (err)
1511 goto out;
1512
1513
1514 txmsg_start_pop = 1500;
1515 txmsg_pop = 1600;
1516 err = test_exec(cgrp, &opt);
1517 if (err)
1518 goto out;
1519 txmsg_start_pop = 0;
1520 txmsg_pop = 0;
1521
1522
1523 txmsg_start = 1111;
1524 txmsg_end = 1112;
1525 txmsg_start_push = 1111;
1526 txmsg_end_push = 1112;
1527 err = test_exec(cgrp, &opt);
1528 if (err)
1529 goto out;
1530
1531
1532 txmsg_start_pop = 1111;
1533 txmsg_pop = 1112;
1534 err = test_exec(cgrp, &opt);
1535 if (err)
1536 goto out;
1537
1538
1539 txmsg_start = 1111;
1540 txmsg_end = 0;
1541 txmsg_start_push = 1111;
1542 txmsg_end_push = 0;
1543 err = test_exec(cgrp, &opt);
1544 if (err)
1545 goto out;
1546
1547
1548 txmsg_start = 0;
1549 txmsg_end = 1601;
1550 txmsg_start_push = 0;
1551 txmsg_end_push = 1601;
1552 err = test_exec(cgrp, &opt);
1553 if (err)
1554 goto out;
1555
1556
1557 txmsg_start = 1601;
1558 txmsg_end = 1600;
1559 txmsg_start_push = 1601;
1560 txmsg_end_push = 1600;
1561 err = test_exec(cgrp, &opt);
1562 if (err)
1563 goto out;
1564
1565
1566 txmsg_start_pop = 1601;
1567 txmsg_pop = 1;
1568 err = test_exec(cgrp, &opt);
1569 if (err)
1570 goto out;
1571
1572
1573 txmsg_start_pop = 1599;
1574 txmsg_pop = 10;
1575 err = test_exec(cgrp, &opt);
1576out:
1577 txmsg_start = 0;
1578 txmsg_end = 0;
1579 sched_yield();
1580 return err;
1581}
1582
1583char *map_names[] = {
1584 "sock_map",
1585 "sock_map_txmsg",
1586 "sock_map_redir",
1587 "sock_apply_bytes",
1588 "sock_cork_bytes",
1589 "sock_bytes",
1590 "sock_redir_flags",
1591 "sock_skb_opts",
1592};
1593
1594int prog_attach_type[] = {
1595 BPF_SK_SKB_STREAM_PARSER,
1596 BPF_SK_SKB_STREAM_VERDICT,
1597 BPF_CGROUP_SOCK_OPS,
1598 BPF_SK_MSG_VERDICT,
1599 BPF_SK_MSG_VERDICT,
1600 BPF_SK_MSG_VERDICT,
1601 BPF_SK_MSG_VERDICT,
1602 BPF_SK_MSG_VERDICT,
1603 BPF_SK_MSG_VERDICT,
1604 BPF_SK_MSG_VERDICT,
1605};
1606
1607int prog_type[] = {
1608 BPF_PROG_TYPE_SK_SKB,
1609 BPF_PROG_TYPE_SK_SKB,
1610 BPF_PROG_TYPE_SOCK_OPS,
1611 BPF_PROG_TYPE_SK_MSG,
1612 BPF_PROG_TYPE_SK_MSG,
1613 BPF_PROG_TYPE_SK_MSG,
1614 BPF_PROG_TYPE_SK_MSG,
1615 BPF_PROG_TYPE_SK_MSG,
1616 BPF_PROG_TYPE_SK_MSG,
1617 BPF_PROG_TYPE_SK_MSG,
1618};
1619
1620static int populate_progs(char *bpf_file)
1621{
1622 struct bpf_program *prog;
1623 struct bpf_object *obj;
1624 int i = 0;
1625 long err;
1626
1627 obj = bpf_object__open(bpf_file);
1628 err = libbpf_get_error(obj);
1629 if (err) {
1630 char err_buf[256];
1631
1632 libbpf_strerror(err, err_buf, sizeof(err_buf));
1633 printf("Unable to load eBPF objects in file '%s' : %s\n",
1634 bpf_file, err_buf);
1635 return -1;
1636 }
1637
1638 bpf_object__for_each_program(prog, obj) {
1639 bpf_program__set_type(prog, prog_type[i]);
1640 bpf_program__set_expected_attach_type(prog,
1641 prog_attach_type[i]);
1642 i++;
1643 }
1644
1645 i = bpf_object__load(obj);
1646 i = 0;
1647 bpf_object__for_each_program(prog, obj) {
1648 prog_fd[i] = bpf_program__fd(prog);
1649 i++;
1650 }
1651
1652 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) {
1653 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1654 map_fd[i] = bpf_map__fd(maps[i]);
1655 if (map_fd[i] < 0) {
1656 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1657 map_fd[i], strerror(errno));
1658 return -1;
1659 }
1660 }
1661
1662 return 0;
1663}
1664
1665static int __test_suite(int cg_fd, char *bpf_file)
1666{
1667 int err, cleanup = cg_fd;
1668
1669 err = populate_progs(bpf_file);
1670 if (err < 0) {
1671 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1672 return err;
1673 }
1674
1675 if (cg_fd < 0) {
1676 if (setup_cgroup_environment()) {
1677 fprintf(stderr, "ERROR: cgroup env failed\n");
1678 return -EINVAL;
1679 }
1680
1681 cg_fd = create_and_get_cgroup(CG_PATH);
1682 if (cg_fd < 0) {
1683 fprintf(stderr,
1684 "ERROR: (%i) open cg path failed: %s\n",
1685 cg_fd, optarg);
1686 return cg_fd;
1687 }
1688
1689 if (join_cgroup(CG_PATH)) {
1690 fprintf(stderr, "ERROR: failed to join cgroup\n");
1691 return -EINVAL;
1692 }
1693 }
1694
1695
1696 txmsg_start = txmsg_end = txmsg_start_push = txmsg_end_push = 0;
1697 err = test_txmsg(cg_fd);
1698 if (err)
1699 goto out;
1700
1701
1702 err = test_mixed(cg_fd);
1703 if (err)
1704 goto out;
1705
1706
1707 err = test_start_end(cg_fd);
1708 if (err)
1709 goto out;
1710
1711out:
1712 printf("Summary: %i PASSED %i FAILED\n", passed, failed);
1713 if (cleanup < 0) {
1714 cleanup_cgroup_environment();
1715 close(cg_fd);
1716 }
1717 return err;
1718}
1719
1720static int test_suite(int cg_fd)
1721{
1722 int err;
1723
1724 err = __test_suite(cg_fd, BPF_SOCKMAP_FILENAME);
1725 if (err)
1726 goto out;
1727 err = __test_suite(cg_fd, BPF_SOCKHASH_FILENAME);
1728out:
1729 if (cg_fd > -1)
1730 close(cg_fd);
1731 return err;
1732}
1733
1734int main(int argc, char **argv)
1735{
1736 int iov_count = 1, length = 1024, rate = 1;
1737 struct sockmap_options options = {0};
1738 int opt, longindex, err, cg_fd = 0;
1739 char *bpf_file = BPF_SOCKMAP_FILENAME;
1740 int test = PING_PONG;
1741
1742 if (argc < 2)
1743 return test_suite(-1);
1744
1745 while ((opt = getopt_long(argc, argv, ":dhvc:r:i:l:t:p:q:",
1746 long_options, &longindex)) != -1) {
1747 switch (opt) {
1748 case 's':
1749 txmsg_start = atoi(optarg);
1750 break;
1751 case 'e':
1752 txmsg_end = atoi(optarg);
1753 break;
1754 case 'p':
1755 txmsg_start_push = atoi(optarg);
1756 break;
1757 case 'q':
1758 txmsg_end_push = atoi(optarg);
1759 break;
1760 case 'w':
1761 txmsg_start_pop = atoi(optarg);
1762 break;
1763 case 'x':
1764 txmsg_pop = atoi(optarg);
1765 break;
1766 case 'a':
1767 txmsg_apply = atoi(optarg);
1768 break;
1769 case 'k':
1770 txmsg_cork = atoi(optarg);
1771 break;
1772 case 'c':
1773 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1774 if (cg_fd < 0) {
1775 fprintf(stderr,
1776 "ERROR: (%i) open cg path failed: %s\n",
1777 cg_fd, optarg);
1778 return cg_fd;
1779 }
1780 break;
1781 case 'r':
1782 rate = atoi(optarg);
1783 break;
1784 case 'v':
1785 options.verbose = 1;
1786 break;
1787 case 'i':
1788 iov_count = atoi(optarg);
1789 break;
1790 case 'l':
1791 length = atoi(optarg);
1792 break;
1793 case 'd':
1794 options.data_test = true;
1795 break;
1796 case 't':
1797 if (strcmp(optarg, "ping") == 0) {
1798 test = PING_PONG;
1799 } else if (strcmp(optarg, "sendmsg") == 0) {
1800 test = SENDMSG;
1801 } else if (strcmp(optarg, "base") == 0) {
1802 test = BASE;
1803 } else if (strcmp(optarg, "base_sendpage") == 0) {
1804 test = BASE_SENDPAGE;
1805 } else if (strcmp(optarg, "sendpage") == 0) {
1806 test = SENDPAGE;
1807 } else {
1808 usage(argv);
1809 return -1;
1810 }
1811 break;
1812 case 0:
1813 break;
1814 case 'h':
1815 default:
1816 usage(argv);
1817 return -1;
1818 }
1819 }
1820
1821 if (argc <= 3 && cg_fd)
1822 return test_suite(cg_fd);
1823
1824 if (!cg_fd) {
1825 fprintf(stderr, "%s requires cgroup option: --cgroup <path>\n",
1826 argv[0]);
1827 return -1;
1828 }
1829
1830 err = populate_progs(bpf_file);
1831 if (err) {
1832 fprintf(stderr, "populate program: (%s) %s\n",
1833 bpf_file, strerror(errno));
1834 return 1;
1835 }
1836 running = 1;
1837
1838
1839 signal(SIGINT, running_handler);
1840
1841 options.iov_count = iov_count;
1842 options.iov_length = length;
1843 options.rate = rate;
1844
1845 err = run_options(&options, cg_fd, test);
1846 close(cg_fd);
1847 return err;
1848}
1849
1850void running_handler(int a)
1851{
1852 running = 0;
1853}
1854
