1
2#define _GNU_SOURCE
3
4#include <arpa/inet.h>
5#include <bpf/bpf.h>
6#include <bpf/libbpf.h>
7#include <errno.h>
8#include <fcntl.h>
9#include <getopt.h>
10#include <linux/ethtool.h>
11#include <linux/hashtable.h>
12#include <linux/if_link.h>
13#include <linux/jhash.h>
14#include <linux/limits.h>
15#include <linux/list.h>
16#include <linux/sockios.h>
17#include <locale.h>
18#include <math.h>
19#include <net/if.h>
20#include <poll.h>
21#include <signal.h>
22#include <stdbool.h>
23#include <stdio.h>
24#include <stdlib.h>
25#include <string.h>
26#include <sys/ioctl.h>
27#include <sys/mman.h>
28#include <sys/resource.h>
29#include <sys/signalfd.h>
30#include <sys/sysinfo.h>
31#include <sys/timerfd.h>
32#include <sys/utsname.h>
33#include <time.h>
34#include <unistd.h>
35
36#include "bpf_util.h"
37#include "xdp_sample_user.h"
38
39#define __sample_print(fmt, cond, ...) \
40 ({ \
41 if (cond) \
42 printf(fmt, ##__VA_ARGS__); \
43 })
44
45#define print_always(fmt, ...) __sample_print(fmt, 1, ##__VA_ARGS__)
46#define print_default(fmt, ...) \
47 __sample_print(fmt, sample_log_level & LL_DEFAULT, ##__VA_ARGS__)
48#define __print_err(err, fmt, ...) \
49 ({ \
50 __sample_print(fmt, err > 0 || sample_log_level & LL_DEFAULT, \
51 ##__VA_ARGS__); \
52 sample_err_exp = sample_err_exp ? true : err > 0; \
53 })
54#define print_err(err, fmt, ...) __print_err(err, fmt, ##__VA_ARGS__)
55
56#define __COLUMN(x) "%'10" x " %-13s"
57#define FMT_COLUMNf __COLUMN(".0f")
58#define FMT_COLUMNd __COLUMN("d")
59#define FMT_COLUMNl __COLUMN("llu")
60#define RX(rx) rx, "rx/s"
61#define PPS(pps) pps, "pkt/s"
62#define DROP(drop) drop, "drop/s"
63#define ERR(err) err, "error/s"
64#define HITS(hits) hits, "hit/s"
65#define XMIT(xmit) xmit, "xmit/s"
66#define PASS(pass) pass, "pass/s"
67#define REDIR(redir) redir, "redir/s"
68#define NANOSEC_PER_SEC 1000000000
69
70#define XDP_UNKNOWN (XDP_REDIRECT + 1)
71#define XDP_ACTION_MAX (XDP_UNKNOWN + 1)
72#define XDP_REDIRECT_ERR_MAX 7
73
74enum map_type {
75 MAP_RX,
76 MAP_REDIRECT_ERR,
77 MAP_CPUMAP_ENQUEUE,
78 MAP_CPUMAP_KTHREAD,
79 MAP_EXCEPTION,
80 MAP_DEVMAP_XMIT,
81 MAP_DEVMAP_XMIT_MULTI,
82 NUM_MAP,
83};
84
85enum log_level {
86 LL_DEFAULT = 1U << 0,
87 LL_SIMPLE = 1U << 1,
88 LL_DEBUG = 1U << 2,
89};
90
91struct record {
92 __u64 timestamp;
93 struct datarec total;
94 struct datarec *cpu;
95};
96
97struct map_entry {
98 struct hlist_node node;
99 __u64 pair;
100 struct record val;
101};
102
103struct stats_record {
104 struct record rx_cnt;
105 struct record redir_err[XDP_REDIRECT_ERR_MAX];
106 struct record kthread;
107 struct record exception[XDP_ACTION_MAX];
108 struct record devmap_xmit;
109 DECLARE_HASHTABLE(xmit_map, 5);
110 struct record enq[];
111};
112
113struct sample_output {
114 struct {
115 __u64 rx;
116 __u64 redir;
117 __u64 drop;
118 __u64 drop_xmit;
119 __u64 err;
120 __u64 xmit;
121 } totals;
122 struct {
123 __u64 pps;
124 __u64 drop;
125 __u64 err;
126 } rx_cnt;
127 struct {
128 __u64 suc;
129 __u64 err;
130 } redir_cnt;
131 struct {
132 __u64 hits;
133 } except_cnt;
134 struct {
135 __u64 pps;
136 __u64 drop;
137 __u64 err;
138 double bavg;
139 } xmit_cnt;
140};
141
142struct xdp_desc {
143 int ifindex;
144 __u32 prog_id;
145 int flags;
146} sample_xdp_progs[32];
147
148struct datarec *sample_mmap[NUM_MAP];
149struct bpf_map *sample_map[NUM_MAP];
150size_t sample_map_count[NUM_MAP];
151enum log_level sample_log_level;
152struct sample_output sample_out;
153unsigned long sample_interval;
154bool sample_err_exp;
155int sample_xdp_cnt;
156int sample_n_cpus;
157int sample_sig_fd;
158int sample_mask;
159
160static const char *xdp_redirect_err_names[XDP_REDIRECT_ERR_MAX] = {
161
162 "Success",
163 "Unknown",
164 "EINVAL",
165 "ENETDOWN",
166 "EMSGSIZE",
167 "EOPNOTSUPP",
168 "ENOSPC",
169};
170
171
172static const char *xdp_redirect_err_help[XDP_REDIRECT_ERR_MAX - 1] = {
173 "Unknown error",
174 "Invalid redirection",
175 "Device being redirected to is down",
176 "Packet length too large for device",
177 "Operation not supported",
178 "No space in ptr_ring of cpumap kthread",
179};
180
181static const char *xdp_action_names[XDP_ACTION_MAX] = {
182 [XDP_ABORTED] = "XDP_ABORTED",
183 [XDP_DROP] = "XDP_DROP",
184 [XDP_PASS] = "XDP_PASS",
185 [XDP_TX] = "XDP_TX",
186 [XDP_REDIRECT] = "XDP_REDIRECT",
187 [XDP_UNKNOWN] = "XDP_UNKNOWN",
188};
189
190static __u64 gettime(void)
191{
192 struct timespec t;
193 int res;
194
195 res = clock_gettime(CLOCK_MONOTONIC, &t);
196 if (res < 0) {
197 fprintf(stderr, "Error with gettimeofday! (%i)\n", res);
198 return UINT64_MAX;
199 }
200 return (__u64)t.tv_sec * NANOSEC_PER_SEC + t.tv_nsec;
201}
202
203static const char *action2str(int action)
204{
205 if (action < XDP_ACTION_MAX)
206 return xdp_action_names[action];
207 return NULL;
208}
209
210static void sample_print_help(int mask)
211{
212 printf("Output format description\n\n"
213 "By default, redirect success statistics are disabled, use -s to enable.\n"
214 "The terse output mode is default, verbose mode can be activated using -v\n"
215 "Use SIGQUIT (Ctrl + \\) to switch the mode dynamically at runtime\n\n"
216 "Terse mode displays at most the following fields:\n"
217 " rx/s Number of packets received per second\n"
218 " redir/s Number of packets successfully redirected per second\n"
219 " err,drop/s Aggregated count of errors per second (including dropped packets)\n"
220 " xmit/s Number of packets transmitted on the output device per second\n\n"
221 "Output description for verbose mode:\n"
222 " FIELD DESCRIPTION\n");
223
224 if (mask & SAMPLE_RX_CNT) {
225 printf(" receive\t\tDisplays the number of packets received & errors encountered\n"
226 " \t\t\tWhenever an error or packet drop occurs, details of per CPU error\n"
227 " \t\t\tand drop statistics will be expanded inline in terse mode.\n"
228 " \t\t\t\tpkt/s - Packets received per second\n"
229 " \t\t\t\tdrop/s - Packets dropped per second\n"
230 " \t\t\t\terror/s - Errors encountered per second\n\n");
231 }
232 if (mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) {
233 printf(" redirect\t\tDisplays the number of packets successfully redirected\n"
234 " \t\t\tErrors encountered are expanded under redirect_err field\n"
235 " \t\t\tNote that passing -s to enable it has a per packet overhead\n"
236 " \t\t\t\tredir/s - Packets redirected successfully per second\n\n"
237 " redirect_err\t\tDisplays the number of packets that failed redirection\n"
238 " \t\t\tThe errno is expanded under this field with per CPU count\n"
239 " \t\t\tThe recognized errors are:\n");
240
241 for (int i = 2; i < XDP_REDIRECT_ERR_MAX; i++)
242 printf("\t\t\t %s: %s\n", xdp_redirect_err_names[i],
243 xdp_redirect_err_help[i - 1]);
244
245 printf(" \n\t\t\t\terror/s - Packets that failed redirection per second\n\n");
246 }
247
248 if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT) {
249 printf(" enqueue to cpu N\tDisplays the number of packets enqueued to bulk queue of CPU N\n"
250 " \t\t\tExpands to cpu:FROM->N to display enqueue stats for each CPU enqueuing to CPU N\n"
251 " \t\t\tReceived packets can be associated with the CPU redirect program is enqueuing \n"
252 " \t\t\tpackets to.\n"
253 " \t\t\t\tpkt/s - Packets enqueued per second from other CPU to CPU N\n"
254 " \t\t\t\tdrop/s - Packets dropped when trying to enqueue to CPU N\n"
255 " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n");
256 }
257
258 if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
259 printf(" kthread\t\tDisplays the number of packets processed in CPUMAP kthread for each CPU\n"
260 " \t\t\tPackets consumed from ptr_ring in kthread, and its xdp_stats (after calling \n"
261 " \t\t\tCPUMAP bpf prog) are expanded below this. xdp_stats are expanded as a total and\n"
262 " \t\t\tthen per-CPU to associate it to each CPU's pinned CPUMAP kthread.\n"
263 " \t\t\t\tpkt/s - Packets consumed per second from ptr_ring\n"
264 " \t\t\t\tdrop/s - Packets dropped per second in kthread\n"
265 " \t\t\t\tsched - Number of times kthread called schedule()\n\n"
266 " \t\t\txdp_stats (also expands to per-CPU counts)\n"
267 " \t\t\t\tpass/s - XDP_PASS count for CPUMAP program execution\n"
268 " \t\t\t\tdrop/s - XDP_DROP count for CPUMAP program execution\n"
269 " \t\t\t\tredir/s - XDP_REDIRECT count for CPUMAP program execution\n\n");
270 }
271
272 if (mask & SAMPLE_EXCEPTION_CNT) {
273 printf(" xdp_exception\t\tDisplays xdp_exception tracepoint events\n"
274 " \t\t\tThis can occur due to internal driver errors, unrecognized\n"
275 " \t\t\tXDP actions and due to explicit user trigger by use of XDP_ABORTED\n"
276 " \t\t\tEach action is expanded below this field with its count\n"
277 " \t\t\t\thit/s - Number of times the tracepoint was hit per second\n\n");
278 }
279
280 if (mask & SAMPLE_DEVMAP_XMIT_CNT) {
281 printf(" devmap_xmit\t\tDisplays devmap_xmit tracepoint events\n"
282 " \t\t\tThis tracepoint is invoked for successful transmissions on output\n"
283 " \t\t\tdevice but these statistics are not available for generic XDP mode,\n"
284 " \t\t\thence they will be omitted from the output when using SKB mode\n"
285 " \t\t\t\txmit/s - Number of packets that were transmitted per second\n"
286 " \t\t\t\tdrop/s - Number of packets that failed transmissions per second\n"
287 " \t\t\t\tdrv_err/s - Number of internal driver errors per second\n"
288 " \t\t\t\tbulk-avg - Average number of packets processed for each event\n\n");
289 }
290}
291
292void sample_usage(char *argv[], const struct option *long_options,
293 const char *doc, int mask, bool error)
294{
295 int i;
296
297 if (!error)
298 sample_print_help(mask);
299
300 printf("\n%s\nOption for %s:\n", doc, argv[0]);
301 for (i = 0; long_options[i].name != 0; i++) {
302 printf(" --%-15s", long_options[i].name);
303 if (long_options[i].flag != NULL)
304 printf(" flag (internal value: %d)",
305 *long_options[i].flag);
306 else
307 printf("\t short-option: -%c", long_options[i].val);
308 printf("\n");
309 }
310 printf("\n");
311}
312
313static struct datarec *alloc_record_per_cpu(void)
314{
315 unsigned int nr_cpus = libbpf_num_possible_cpus();
316 struct datarec *array;
317
318 array = calloc(nr_cpus, sizeof(*array));
319 if (!array) {
320 fprintf(stderr, "Failed to allocate memory (nr_cpus: %u)\n",
321 nr_cpus);
322 return NULL;
323 }
324 return array;
325}
326
327static int map_entry_init(struct map_entry *e, __u64 pair)
328{
329 e->pair = pair;
330 INIT_HLIST_NODE(&e->node);
331 e->val.timestamp = gettime();
332 e->val.cpu = alloc_record_per_cpu();
333 if (!e->val.cpu)
334 return -ENOMEM;
335 return 0;
336}
337
338static void map_collect_percpu(struct datarec *values, struct record *rec)
339{
340
341 unsigned int nr_cpus = libbpf_num_possible_cpus();
342 __u64 sum_xdp_redirect = 0;
343 __u64 sum_processed = 0;
344 __u64 sum_xdp_pass = 0;
345 __u64 sum_xdp_drop = 0;
346 __u64 sum_dropped = 0;
347 __u64 sum_issue = 0;
348 int i;
349
350
351 rec->timestamp = gettime();
352
353
354 for (i = 0; i < nr_cpus; i++) {
355 rec->cpu[i].processed = READ_ONCE(values[i].processed);
356 rec->cpu[i].dropped = READ_ONCE(values[i].dropped);
357 rec->cpu[i].issue = READ_ONCE(values[i].issue);
358 rec->cpu[i].xdp_pass = READ_ONCE(values[i].xdp_pass);
359 rec->cpu[i].xdp_drop = READ_ONCE(values[i].xdp_drop);
360 rec->cpu[i].xdp_redirect = READ_ONCE(values[i].xdp_redirect);
361
362 sum_processed += rec->cpu[i].processed;
363 sum_dropped += rec->cpu[i].dropped;
364 sum_issue += rec->cpu[i].issue;
365 sum_xdp_pass += rec->cpu[i].xdp_pass;
366 sum_xdp_drop += rec->cpu[i].xdp_drop;
367 sum_xdp_redirect += rec->cpu[i].xdp_redirect;
368 }
369
370 rec->total.processed = sum_processed;
371 rec->total.dropped = sum_dropped;
372 rec->total.issue = sum_issue;
373 rec->total.xdp_pass = sum_xdp_pass;
374 rec->total.xdp_drop = sum_xdp_drop;
375 rec->total.xdp_redirect = sum_xdp_redirect;
376}
377
378static int map_collect_percpu_devmap(int map_fd, struct stats_record *rec)
379{
380 unsigned int nr_cpus = bpf_num_possible_cpus();
381 __u32 batch, count = 32;
382 struct datarec *values;
383 bool init = false;
384 __u64 *keys;
385 int i, ret;
386
387 keys = calloc(count, sizeof(__u64));
388 if (!keys)
389 return -ENOMEM;
390 values = calloc(count * nr_cpus, sizeof(struct datarec));
391 if (!values) {
392 free(keys);
393 return -ENOMEM;
394 }
395
396 for (;;) {
397 bool exit = false;
398
399 ret = bpf_map_lookup_batch(map_fd, init ? &batch : NULL, &batch,
400 keys, values, &count, NULL);
401 if (ret < 0 && errno != ENOENT)
402 break;
403 if (errno == ENOENT)
404 exit = true;
405
406 init = true;
407 for (i = 0; i < count; i++) {
408 struct map_entry *e, *x = NULL;
409 __u64 pair = keys[i];
410 struct datarec *arr;
411
412 arr = &values[i * nr_cpus];
413 hash_for_each_possible(rec->xmit_map, e, node, pair) {
414 if (e->pair == pair) {
415 x = e;
416 break;
417 }
418 }
419 if (!x) {
420 x = calloc(1, sizeof(*x));
421 if (!x)
422 goto cleanup;
423 if (map_entry_init(x, pair) < 0) {
424 free(x);
425 goto cleanup;
426 }
427 hash_add(rec->xmit_map, &x->node, pair);
428 }
429 map_collect_percpu(arr, &x->val);
430 }
431
432 if (exit)
433 break;
434 count = 32;
435 }
436
437 free(values);
438 free(keys);
439 return 0;
440cleanup:
441 free(values);
442 free(keys);
443 return -ENOMEM;
444}
445
446static struct stats_record *alloc_stats_record(void)
447{
448 struct stats_record *rec;
449 int i;
450
451 rec = calloc(1, sizeof(*rec) + sample_n_cpus * sizeof(struct record));
452 if (!rec) {
453 fprintf(stderr, "Failed to allocate memory\n");
454 return NULL;
455 }
456
457 if (sample_mask & SAMPLE_RX_CNT) {
458 rec->rx_cnt.cpu = alloc_record_per_cpu();
459 if (!rec->rx_cnt.cpu) {
460 fprintf(stderr,
461 "Failed to allocate rx_cnt per-CPU array\n");
462 goto end_rec;
463 }
464 }
465 if (sample_mask & (SAMPLE_REDIRECT_CNT | SAMPLE_REDIRECT_ERR_CNT)) {
466 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++) {
467 rec->redir_err[i].cpu = alloc_record_per_cpu();
468 if (!rec->redir_err[i].cpu) {
469 fprintf(stderr,
470 "Failed to allocate redir_err per-CPU array for "
471 "\"%s\" case\n",
472 xdp_redirect_err_names[i]);
473 while (i--)
474 free(rec->redir_err[i].cpu);
475 goto end_rx_cnt;
476 }
477 }
478 }
479 if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
480 rec->kthread.cpu = alloc_record_per_cpu();
481 if (!rec->kthread.cpu) {
482 fprintf(stderr,
483 "Failed to allocate kthread per-CPU array\n");
484 goto end_redir;
485 }
486 }
487 if (sample_mask & SAMPLE_EXCEPTION_CNT) {
488 for (i = 0; i < XDP_ACTION_MAX; i++) {
489 rec->exception[i].cpu = alloc_record_per_cpu();
490 if (!rec->exception[i].cpu) {
491 fprintf(stderr,
492 "Failed to allocate exception per-CPU array for "
493 "\"%s\" case\n",
494 action2str(i));
495 while (i--)
496 free(rec->exception[i].cpu);
497 goto end_kthread;
498 }
499 }
500 }
501 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT) {
502 rec->devmap_xmit.cpu = alloc_record_per_cpu();
503 if (!rec->devmap_xmit.cpu) {
504 fprintf(stderr,
505 "Failed to allocate devmap_xmit per-CPU array\n");
506 goto end_exception;
507 }
508 }
509 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
510 hash_init(rec->xmit_map);
511 if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT) {
512 for (i = 0; i < sample_n_cpus; i++) {
513 rec->enq[i].cpu = alloc_record_per_cpu();
514 if (!rec->enq[i].cpu) {
515 fprintf(stderr,
516 "Failed to allocate enqueue per-CPU array for "
517 "CPU %d\n",
518 i);
519 while (i--)
520 free(rec->enq[i].cpu);
521 goto end_devmap_xmit;
522 }
523 }
524 }
525
526 return rec;
527
528end_devmap_xmit:
529 free(rec->devmap_xmit.cpu);
530end_exception:
531 for (i = 0; i < XDP_ACTION_MAX; i++)
532 free(rec->exception[i].cpu);
533end_kthread:
534 free(rec->kthread.cpu);
535end_redir:
536 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++)
537 free(rec->redir_err[i].cpu);
538end_rx_cnt:
539 free(rec->rx_cnt.cpu);
540end_rec:
541 free(rec);
542 return NULL;
543}
544
545static void free_stats_record(struct stats_record *r)
546{
547 struct hlist_node *tmp;
548 struct map_entry *e;
549 int i;
550
551 for (i = 0; i < sample_n_cpus; i++)
552 free(r->enq[i].cpu);
553 hash_for_each_safe(r->xmit_map, i, tmp, e, node) {
554 hash_del(&e->node);
555 free(e->val.cpu);
556 free(e);
557 }
558 free(r->devmap_xmit.cpu);
559 for (i = 0; i < XDP_ACTION_MAX; i++)
560 free(r->exception[i].cpu);
561 free(r->kthread.cpu);
562 for (i = 0; i < XDP_REDIRECT_ERR_MAX; i++)
563 free(r->redir_err[i].cpu);
564 free(r->rx_cnt.cpu);
565 free(r);
566}
567
568static double calc_period(struct record *r, struct record *p)
569{
570 double period_ = 0;
571 __u64 period = 0;
572
573 period = r->timestamp - p->timestamp;
574 if (period > 0)
575 period_ = ((double)period / NANOSEC_PER_SEC);
576
577 return period_;
578}
579
580static double sample_round(double val)
581{
582 if (val - floor(val) < 0.5)
583 return floor(val);
584 return ceil(val);
585}
586
587static __u64 calc_pps(struct datarec *r, struct datarec *p, double period_)
588{
589 __u64 packets = 0;
590 __u64 pps = 0;
591
592 if (period_ > 0) {
593 packets = r->processed - p->processed;
594 pps = sample_round(packets / period_);
595 }
596 return pps;
597}
598
599static __u64 calc_drop_pps(struct datarec *r, struct datarec *p, double period_)
600{
601 __u64 packets = 0;
602 __u64 pps = 0;
603
604 if (period_ > 0) {
605 packets = r->dropped - p->dropped;
606 pps = sample_round(packets / period_);
607 }
608 return pps;
609}
610
611static __u64 calc_errs_pps(struct datarec *r, struct datarec *p, double period_)
612{
613 __u64 packets = 0;
614 __u64 pps = 0;
615
616 if (period_ > 0) {
617 packets = r->issue - p->issue;
618 pps = sample_round(packets / period_);
619 }
620 return pps;
621}
622
623static __u64 calc_info_pps(struct datarec *r, struct datarec *p, double period_)
624{
625 __u64 packets = 0;
626 __u64 pps = 0;
627
628 if (period_ > 0) {
629 packets = r->info - p->info;
630 pps = sample_round(packets / period_);
631 }
632 return pps;
633}
634
635static void calc_xdp_pps(struct datarec *r, struct datarec *p, double *xdp_pass,
636 double *xdp_drop, double *xdp_redirect, double period_)
637{
638 *xdp_pass = 0, *xdp_drop = 0, *xdp_redirect = 0;
639 if (period_ > 0) {
640 *xdp_redirect = (r->xdp_redirect - p->xdp_redirect) / period_;
641 *xdp_pass = (r->xdp_pass - p->xdp_pass) / period_;
642 *xdp_drop = (r->xdp_drop - p->xdp_drop) / period_;
643 }
644}
645
646static void stats_get_rx_cnt(struct stats_record *stats_rec,
647 struct stats_record *stats_prev,
648 unsigned int nr_cpus, struct sample_output *out)
649{
650 struct record *rec, *prev;
651 double t, pps, drop, err;
652 int i;
653
654 rec = &stats_rec->rx_cnt;
655 prev = &stats_prev->rx_cnt;
656 t = calc_period(rec, prev);
657
658 for (i = 0; i < nr_cpus; i++) {
659 struct datarec *r = &rec->cpu[i];
660 struct datarec *p = &prev->cpu[i];
661 char str[64];
662
663 pps = calc_pps(r, p, t);
664 drop = calc_drop_pps(r, p, t);
665 err = calc_errs_pps(r, p, t);
666 if (!pps && !drop && !err)
667 continue;
668
669 snprintf(str, sizeof(str), "cpu:%d", i);
670 print_default(" %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
671 "\n",
672 str, PPS(pps), DROP(drop), ERR(err));
673 }
674
675 if (out) {
676 pps = calc_pps(&rec->total, &prev->total, t);
677 drop = calc_drop_pps(&rec->total, &prev->total, t);
678 err = calc_errs_pps(&rec->total, &prev->total, t);
679
680 out->rx_cnt.pps = pps;
681 out->rx_cnt.drop = drop;
682 out->rx_cnt.err = err;
683 out->totals.rx += pps;
684 out->totals.drop += drop;
685 out->totals.err += err;
686 }
687}
688
689static void stats_get_cpumap_enqueue(struct stats_record *stats_rec,
690 struct stats_record *stats_prev,
691 unsigned int nr_cpus)
692{
693 struct record *rec, *prev;
694 double t, pps, drop, err;
695 int i, to_cpu;
696
697
698 for (to_cpu = 0; to_cpu < sample_n_cpus; to_cpu++) {
699 rec = &stats_rec->enq[to_cpu];
700 prev = &stats_prev->enq[to_cpu];
701 t = calc_period(rec, prev);
702
703 pps = calc_pps(&rec->total, &prev->total, t);
704 drop = calc_drop_pps(&rec->total, &prev->total, t);
705 err = calc_errs_pps(&rec->total, &prev->total, t);
706
707 if (pps > 0 || drop > 0) {
708 char str[64];
709
710 snprintf(str, sizeof(str), "enqueue to cpu %d", to_cpu);
711
712 if (err > 0)
713 err = pps / err;
714
715 print_err(drop,
716 " %-20s " FMT_COLUMNf FMT_COLUMNf __COLUMN(
717 ".2f") "\n",
718 str, PPS(pps), DROP(drop), err, "bulk-avg");
719 }
720
721 for (i = 0; i < nr_cpus; i++) {
722 struct datarec *r = &rec->cpu[i];
723 struct datarec *p = &prev->cpu[i];
724 char str[64];
725
726 pps = calc_pps(r, p, t);
727 drop = calc_drop_pps(r, p, t);
728 err = calc_errs_pps(r, p, t);
729 if (!pps && !drop && !err)
730 continue;
731
732 snprintf(str, sizeof(str), "cpu:%d->%d", i, to_cpu);
733 if (err > 0)
734 err = pps / err;
735 print_default(
736 " %-18s " FMT_COLUMNf FMT_COLUMNf __COLUMN(
737 ".2f") "\n",
738 str, PPS(pps), DROP(drop), err, "bulk-avg");
739 }
740 }
741}
742
743static void stats_get_cpumap_remote(struct stats_record *stats_rec,
744 struct stats_record *stats_prev,
745 unsigned int nr_cpus)
746{
747 double xdp_pass, xdp_drop, xdp_redirect;
748 struct record *rec, *prev;
749 double t;
750 int i;
751
752 rec = &stats_rec->kthread;
753 prev = &stats_prev->kthread;
754 t = calc_period(rec, prev);
755
756 calc_xdp_pps(&rec->total, &prev->total, &xdp_pass, &xdp_drop,
757 &xdp_redirect, t);
758 if (xdp_pass || xdp_drop || xdp_redirect) {
759 print_err(xdp_drop,
760 " %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n",
761 "xdp_stats", PASS(xdp_pass), DROP(xdp_drop),
762 REDIR(xdp_redirect));
763 }
764
765 for (i = 0; i < nr_cpus; i++) {
766 struct datarec *r = &rec->cpu[i];
767 struct datarec *p = &prev->cpu[i];
768 char str[64];
769
770 calc_xdp_pps(r, p, &xdp_pass, &xdp_drop, &xdp_redirect, t);
771 if (!xdp_pass && !xdp_drop && !xdp_redirect)
772 continue;
773
774 snprintf(str, sizeof(str), "cpu:%d", i);
775 print_default(" %-16s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
776 "\n",
777 str, PASS(xdp_pass), DROP(xdp_drop),
778 REDIR(xdp_redirect));
779 }
780}
781
782static void stats_get_cpumap_kthread(struct stats_record *stats_rec,
783 struct stats_record *stats_prev,
784 unsigned int nr_cpus)
785{
786 struct record *rec, *prev;
787 double t, pps, drop, err;
788 int i;
789
790 rec = &stats_rec->kthread;
791 prev = &stats_prev->kthread;
792 t = calc_period(rec, prev);
793
794 pps = calc_pps(&rec->total, &prev->total, t);
795 drop = calc_drop_pps(&rec->total, &prev->total, t);
796 err = calc_errs_pps(&rec->total, &prev->total, t);
797
798 print_err(drop, " %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf "\n",
799 pps ? "kthread total" : "kthread", PPS(pps), DROP(drop), err,
800 "sched");
801
802 for (i = 0; i < nr_cpus; i++) {
803 struct datarec *r = &rec->cpu[i];
804 struct datarec *p = &prev->cpu[i];
805 char str[64];
806
807 pps = calc_pps(r, p, t);
808 drop = calc_drop_pps(r, p, t);
809 err = calc_errs_pps(r, p, t);
810 if (!pps && !drop && !err)
811 continue;
812
813 snprintf(str, sizeof(str), "cpu:%d", i);
814 print_default(" %-18s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
815 "\n",
816 str, PPS(pps), DROP(drop), err, "sched");
817 }
818}
819
820static void stats_get_redirect_cnt(struct stats_record *stats_rec,
821 struct stats_record *stats_prev,
822 unsigned int nr_cpus,
823 struct sample_output *out)
824{
825 struct record *rec, *prev;
826 double t, pps;
827 int i;
828
829 rec = &stats_rec->redir_err[0];
830 prev = &stats_prev->redir_err[0];
831 t = calc_period(rec, prev);
832 for (i = 0; i < nr_cpus; i++) {
833 struct datarec *r = &rec->cpu[i];
834 struct datarec *p = &prev->cpu[i];
835 char str[64];
836
837 pps = calc_pps(r, p, t);
838 if (!pps)
839 continue;
840
841 snprintf(str, sizeof(str), "cpu:%d", i);
842 print_default(" %-18s " FMT_COLUMNf "\n", str, REDIR(pps));
843 }
844
845 if (out) {
846 pps = calc_pps(&rec->total, &prev->total, t);
847 out->redir_cnt.suc = pps;
848 out->totals.redir += pps;
849 }
850}
851
852static void stats_get_redirect_err_cnt(struct stats_record *stats_rec,
853 struct stats_record *stats_prev,
854 unsigned int nr_cpus,
855 struct sample_output *out)
856{
857 struct record *rec, *prev;
858 double t, drop, sum = 0;
859 int rec_i, i;
860
861 for (rec_i = 1; rec_i < XDP_REDIRECT_ERR_MAX; rec_i++) {
862 char str[64];
863
864 rec = &stats_rec->redir_err[rec_i];
865 prev = &stats_prev->redir_err[rec_i];
866 t = calc_period(rec, prev);
867
868 drop = calc_drop_pps(&rec->total, &prev->total, t);
869 if (drop > 0 && !out) {
870 snprintf(str, sizeof(str),
871 sample_log_level & LL_DEFAULT ? "%s total" :
872 "%s",
873 xdp_redirect_err_names[rec_i]);
874 print_err(drop, " %-18s " FMT_COLUMNf "\n", str,
875 ERR(drop));
876 }
877
878 for (i = 0; i < nr_cpus; i++) {
879 struct datarec *r = &rec->cpu[i];
880 struct datarec *p = &prev->cpu[i];
881 double drop;
882
883 drop = calc_drop_pps(r, p, t);
884 if (!drop)
885 continue;
886
887 snprintf(str, sizeof(str), "cpu:%d", i);
888 print_default(" %-16s" FMT_COLUMNf "\n", str,
889 ERR(drop));
890 }
891
892 sum += drop;
893 }
894
895 if (out) {
896 out->redir_cnt.err = sum;
897 out->totals.err += sum;
898 }
899}
900
901static void stats_get_exception_cnt(struct stats_record *stats_rec,
902 struct stats_record *stats_prev,
903 unsigned int nr_cpus,
904 struct sample_output *out)
905{
906 double t, drop, sum = 0;
907 struct record *rec, *prev;
908 int rec_i, i;
909
910 for (rec_i = 0; rec_i < XDP_ACTION_MAX; rec_i++) {
911 rec = &stats_rec->exception[rec_i];
912 prev = &stats_prev->exception[rec_i];
913 t = calc_period(rec, prev);
914
915 drop = calc_drop_pps(&rec->total, &prev->total, t);
916
917 sum += drop;
918
919 if (drop > 0 && !out) {
920 print_always(" %-18s " FMT_COLUMNf "\n",
921 action2str(rec_i), ERR(drop));
922
923 for (i = 0; i < nr_cpus; i++) {
924 struct datarec *r = &rec->cpu[i];
925 struct datarec *p = &prev->cpu[i];
926 char str[64];
927 double drop;
928
929 drop = calc_drop_pps(r, p, t);
930 if (!drop)
931 continue;
932
933 snprintf(str, sizeof(str), "cpu:%d", i);
934 print_default(" %-16s" FMT_COLUMNf "\n",
935 str, ERR(drop));
936 }
937 }
938 }
939
940 if (out) {
941 out->except_cnt.hits = sum;
942 out->totals.err += sum;
943 }
944}
945
946static void stats_get_devmap_xmit(struct stats_record *stats_rec,
947 struct stats_record *stats_prev,
948 unsigned int nr_cpus,
949 struct sample_output *out)
950{
951 double pps, drop, info, err;
952 struct record *rec, *prev;
953 double t;
954 int i;
955
956 rec = &stats_rec->devmap_xmit;
957 prev = &stats_prev->devmap_xmit;
958 t = calc_period(rec, prev);
959 for (i = 0; i < nr_cpus; i++) {
960 struct datarec *r = &rec->cpu[i];
961 struct datarec *p = &prev->cpu[i];
962 char str[64];
963
964 pps = calc_pps(r, p, t);
965 drop = calc_drop_pps(r, p, t);
966 err = calc_errs_pps(r, p, t);
967
968 if (!pps && !drop && !err)
969 continue;
970
971 snprintf(str, sizeof(str), "cpu:%d", i);
972 info = calc_info_pps(r, p, t);
973 if (info > 0)
974 info = (pps + drop) / info;
975 print_default(" %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
976 __COLUMN(".2f") "\n",
977 str, XMIT(pps), DROP(drop), err, "drv_err/s",
978 info, "bulk-avg");
979 }
980 if (out) {
981 pps = calc_pps(&rec->total, &prev->total, t);
982 drop = calc_drop_pps(&rec->total, &prev->total, t);
983 info = calc_info_pps(&rec->total, &prev->total, t);
984 if (info > 0)
985 info = (pps + drop) / info;
986 err = calc_errs_pps(&rec->total, &prev->total, t);
987
988 out->xmit_cnt.pps = pps;
989 out->xmit_cnt.drop = drop;
990 out->xmit_cnt.bavg = info;
991 out->xmit_cnt.err = err;
992 out->totals.xmit += pps;
993 out->totals.drop_xmit += drop;
994 out->totals.err += err;
995 }
996}
997
998static void stats_get_devmap_xmit_multi(struct stats_record *stats_rec,
999 struct stats_record *stats_prev,
1000 unsigned int nr_cpus,
1001 struct sample_output *out,
1002 bool xmit_total)
1003{
1004 double pps, drop, info, err;
1005 struct map_entry *entry;
1006 struct record *r, *p;
1007 double t;
1008 int bkt;
1009
1010 hash_for_each(stats_rec->xmit_map, bkt, entry, node) {
1011 struct map_entry *e, *x = NULL;
1012 char ifname_from[IFNAMSIZ];
1013 char ifname_to[IFNAMSIZ];
1014 const char *fstr, *tstr;
1015 unsigned long prev_time;
1016 struct record beg = {};
1017 __u32 from_idx, to_idx;
1018 char str[128];
1019 __u64 pair;
1020 int i;
1021
1022 prev_time = sample_interval * NANOSEC_PER_SEC;
1023
1024 pair = entry->pair;
1025 from_idx = pair >> 32;
1026 to_idx = pair & 0xFFFFFFFF;
1027
1028 r = &entry->val;
1029 beg.timestamp = r->timestamp - prev_time;
1030
1031
1032 hash_for_each_possible(stats_prev->xmit_map, e, node, pair) {
1033 if (e->pair == pair) {
1034 x = e;
1035 break;
1036 }
1037 }
1038 if (x)
1039 p = &x->val;
1040 else
1041 p = &beg;
1042 t = calc_period(r, p);
1043 pps = calc_pps(&r->total, &p->total, t);
1044 drop = calc_drop_pps(&r->total, &p->total, t);
1045 info = calc_info_pps(&r->total, &p->total, t);
1046 if (info > 0)
1047 info = (pps + drop) / info;
1048 err = calc_errs_pps(&r->total, &p->total, t);
1049
1050 if (out) {
1051
1052 out->totals.xmit += pps;
1053 out->totals.drop_xmit += drop;
1054 out->totals.err += err;
1055 continue;
1056 }
1057
1058 fstr = tstr = NULL;
1059 if (if_indextoname(from_idx, ifname_from))
1060 fstr = ifname_from;
1061 if (if_indextoname(to_idx, ifname_to))
1062 tstr = ifname_to;
1063
1064 snprintf(str, sizeof(str), "xmit %s->%s", fstr ?: "?",
1065 tstr ?: "?");
1066
1067 if (pps || drop || err) {
1068 print_err(drop,
1069 " %-20s " FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
1070 __COLUMN(".2f") "\n", str, XMIT(pps), DROP(drop),
1071 err, "drv_err/s", info, "bulk-avg");
1072 }
1073
1074 for (i = 0; i < nr_cpus; i++) {
1075 struct datarec *rc = &r->cpu[i];
1076 struct datarec *pc, p_beg = {};
1077 char str[64];
1078
1079 pc = p == &beg ? &p_beg : &p->cpu[i];
1080
1081 pps = calc_pps(rc, pc, t);
1082 drop = calc_drop_pps(rc, pc, t);
1083 err = calc_errs_pps(rc, pc, t);
1084
1085 if (!pps && !drop && !err)
1086 continue;
1087
1088 snprintf(str, sizeof(str), "cpu:%d", i);
1089 info = calc_info_pps(rc, pc, t);
1090 if (info > 0)
1091 info = (pps + drop) / info;
1092
1093 print_default(" %-18s" FMT_COLUMNf FMT_COLUMNf FMT_COLUMNf
1094 __COLUMN(".2f") "\n", str, XMIT(pps),
1095 DROP(drop), err, "drv_err/s", info, "bulk-avg");
1096 }
1097 }
1098}
1099
1100static void stats_print(const char *prefix, int mask, struct stats_record *r,
1101 struct stats_record *p, struct sample_output *out)
1102{
1103 int nr_cpus = libbpf_num_possible_cpus();
1104 const char *str;
1105
1106 print_always("%-23s", prefix ?: "Summary");
1107 if (mask & SAMPLE_RX_CNT)
1108 print_always(FMT_COLUMNl, RX(out->totals.rx));
1109 if (mask & SAMPLE_REDIRECT_CNT)
1110 print_always(FMT_COLUMNl, REDIR(out->totals.redir));
1111 printf(FMT_COLUMNl,
1112 out->totals.err + out->totals.drop + out->totals.drop_xmit,
1113 "err,drop/s");
1114 if (mask & SAMPLE_DEVMAP_XMIT_CNT ||
1115 mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1116 printf(FMT_COLUMNl, XMIT(out->totals.xmit));
1117 printf("\n");
1118
1119 if (mask & SAMPLE_RX_CNT) {
1120 str = (sample_log_level & LL_DEFAULT) && out->rx_cnt.pps ?
1121 "receive total" :
1122 "receive";
1123 print_err((out->rx_cnt.err || out->rx_cnt.drop),
1124 " %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl "\n",
1125 str, PPS(out->rx_cnt.pps), DROP(out->rx_cnt.drop),
1126 ERR(out->rx_cnt.err));
1127
1128 stats_get_rx_cnt(r, p, nr_cpus, NULL);
1129 }
1130
1131 if (mask & SAMPLE_CPUMAP_ENQUEUE_CNT)
1132 stats_get_cpumap_enqueue(r, p, nr_cpus);
1133
1134 if (mask & SAMPLE_CPUMAP_KTHREAD_CNT) {
1135 stats_get_cpumap_kthread(r, p, nr_cpus);
1136 stats_get_cpumap_remote(r, p, nr_cpus);
1137 }
1138
1139 if (mask & SAMPLE_REDIRECT_CNT) {
1140 str = out->redir_cnt.suc ? "redirect total" : "redirect";
1141 print_default(" %-20s " FMT_COLUMNl "\n", str,
1142 REDIR(out->redir_cnt.suc));
1143
1144 stats_get_redirect_cnt(r, p, nr_cpus, NULL);
1145 }
1146
1147 if (mask & SAMPLE_REDIRECT_ERR_CNT) {
1148 str = (sample_log_level & LL_DEFAULT) && out->redir_cnt.err ?
1149 "redirect_err total" :
1150 "redirect_err";
1151 print_err(out->redir_cnt.err, " %-20s " FMT_COLUMNl "\n", str,
1152 ERR(out->redir_cnt.err));
1153
1154 stats_get_redirect_err_cnt(r, p, nr_cpus, NULL);
1155 }
1156
1157 if (mask & SAMPLE_EXCEPTION_CNT) {
1158 str = out->except_cnt.hits ? "xdp_exception total" :
1159 "xdp_exception";
1160
1161 print_err(out->except_cnt.hits, " %-20s " FMT_COLUMNl "\n", str,
1162 HITS(out->except_cnt.hits));
1163
1164 stats_get_exception_cnt(r, p, nr_cpus, NULL);
1165 }
1166
1167 if (mask & SAMPLE_DEVMAP_XMIT_CNT) {
1168 str = (sample_log_level & LL_DEFAULT) && out->xmit_cnt.pps ?
1169 "devmap_xmit total" :
1170 "devmap_xmit";
1171
1172 print_err(out->xmit_cnt.err || out->xmit_cnt.drop,
1173 " %-20s " FMT_COLUMNl FMT_COLUMNl FMT_COLUMNl
1174 __COLUMN(".2f") "\n",
1175 str, XMIT(out->xmit_cnt.pps),
1176 DROP(out->xmit_cnt.drop), out->xmit_cnt.err,
1177 "drv_err/s", out->xmit_cnt.bavg, "bulk-avg");
1178
1179 stats_get_devmap_xmit(r, p, nr_cpus, NULL);
1180 }
1181
1182 if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1183 stats_get_devmap_xmit_multi(r, p, nr_cpus, NULL,
1184 mask & SAMPLE_DEVMAP_XMIT_CNT);
1185
1186 if (sample_log_level & LL_DEFAULT ||
1187 ((sample_log_level & LL_SIMPLE) && sample_err_exp)) {
1188 sample_err_exp = false;
1189 printf("\n");
1190 }
1191}
1192
1193int sample_setup_maps(struct bpf_map **maps)
1194{
1195 sample_n_cpus = libbpf_num_possible_cpus();
1196
1197 for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) {
1198 sample_map[i] = maps[i];
1199
1200 switch (i) {
1201 case MAP_RX:
1202 case MAP_CPUMAP_KTHREAD:
1203 case MAP_DEVMAP_XMIT:
1204 sample_map_count[i] = sample_n_cpus;
1205 break;
1206 case MAP_REDIRECT_ERR:
1207 sample_map_count[i] =
1208 XDP_REDIRECT_ERR_MAX * sample_n_cpus;
1209 break;
1210 case MAP_EXCEPTION:
1211 sample_map_count[i] = XDP_ACTION_MAX * sample_n_cpus;
1212 case MAP_CPUMAP_ENQUEUE:
1213 sample_map_count[i] = sample_n_cpus * sample_n_cpus;
1214 break;
1215 default:
1216 return -EINVAL;
1217 }
1218 if (bpf_map__resize(sample_map[i], sample_map_count[i]) < 0)
1219 return -errno;
1220 }
1221 sample_map[MAP_DEVMAP_XMIT_MULTI] = maps[MAP_DEVMAP_XMIT_MULTI];
1222 return 0;
1223}
1224
1225static int sample_setup_maps_mappings(void)
1226{
1227 for (int i = 0; i < MAP_DEVMAP_XMIT_MULTI; i++) {
1228 size_t size = sample_map_count[i] * sizeof(struct datarec);
1229
1230 sample_mmap[i] = mmap(NULL, size, PROT_READ | PROT_WRITE,
1231 MAP_SHARED, bpf_map__fd(sample_map[i]), 0);
1232 if (sample_mmap[i] == MAP_FAILED)
1233 return -errno;
1234 }
1235 return 0;
1236}
1237
1238int __sample_init(int mask)
1239{
1240 sigset_t st;
1241
1242 sigemptyset(&st);
1243 sigaddset(&st, SIGQUIT);
1244 sigaddset(&st, SIGINT);
1245 sigaddset(&st, SIGTERM);
1246
1247 if (sigprocmask(SIG_BLOCK, &st, NULL) < 0)
1248 return -errno;
1249
1250 sample_sig_fd = signalfd(-1, &st, SFD_CLOEXEC | SFD_NONBLOCK);
1251 if (sample_sig_fd < 0)
1252 return -errno;
1253
1254 sample_mask = mask;
1255
1256 return sample_setup_maps_mappings();
1257}
1258
1259static int __sample_remove_xdp(int ifindex, __u32 prog_id, int xdp_flags)
1260{
1261 __u32 cur_prog_id = 0;
1262 int ret;
1263
1264 if (prog_id) {
1265 ret = bpf_get_link_xdp_id(ifindex, &cur_prog_id, xdp_flags);
1266 if (ret < 0)
1267 return -errno;
1268
1269 if (prog_id != cur_prog_id) {
1270 print_always(
1271 "Program on ifindex %d does not match installed "
1272 "program, skipping unload\n",
1273 ifindex);
1274 return -ENOENT;
1275 }
1276 }
1277
1278 return bpf_set_link_xdp_fd(ifindex, -1, xdp_flags);
1279}
1280
1281int sample_install_xdp(struct bpf_program *xdp_prog, int ifindex, bool generic,
1282 bool force)
1283{
1284 int ret, xdp_flags = 0;
1285 __u32 prog_id = 0;
1286
1287 if (sample_xdp_cnt == 32) {
1288 fprintf(stderr,
1289 "Total limit for installed XDP programs in a sample reached\n");
1290 return -ENOTSUP;
1291 }
1292
1293 xdp_flags |= !force ? XDP_FLAGS_UPDATE_IF_NOEXIST : 0;
1294 xdp_flags |= generic ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE;
1295 ret = bpf_set_link_xdp_fd(ifindex, bpf_program__fd(xdp_prog),
1296 xdp_flags);
1297 if (ret < 0) {
1298 ret = -errno;
1299 fprintf(stderr,
1300 "Failed to install program \"%s\" on ifindex %d, mode = %s, "
1301 "force = %s: %s\n",
1302 bpf_program__name(xdp_prog), ifindex,
1303 generic ? "skb" : "native", force ? "true" : "false",
1304 strerror(-ret));
1305 return ret;
1306 }
1307
1308 ret = bpf_get_link_xdp_id(ifindex, &prog_id, xdp_flags);
1309 if (ret < 0) {
1310 ret = -errno;
1311 fprintf(stderr,
1312 "Failed to get XDP program id for ifindex %d, removing program: %s\n",
1313 ifindex, strerror(errno));
1314 __sample_remove_xdp(ifindex, 0, xdp_flags);
1315 return ret;
1316 }
1317 sample_xdp_progs[sample_xdp_cnt++] =
1318 (struct xdp_desc){ ifindex, prog_id, xdp_flags };
1319
1320 return 0;
1321}
1322
1323static void sample_summary_print(void)
1324{
1325 double period = sample_out.rx_cnt.pps;
1326
1327 if (sample_out.totals.rx) {
1328 double pkts = sample_out.totals.rx;
1329
1330 print_always(" Packets received : %'-10llu\n",
1331 sample_out.totals.rx);
1332 print_always(" Average packets/s : %'-10.0f\n",
1333 sample_round(pkts / period));
1334 }
1335 if (sample_out.totals.redir) {
1336 double pkts = sample_out.totals.redir;
1337
1338 print_always(" Packets redirected : %'-10llu\n",
1339 sample_out.totals.redir);
1340 print_always(" Average redir/s : %'-10.0f\n",
1341 sample_round(pkts / period));
1342 }
1343 if (sample_out.totals.drop)
1344 print_always(" Rx dropped : %'-10llu\n",
1345 sample_out.totals.drop);
1346 if (sample_out.totals.drop_xmit)
1347 print_always(" Tx dropped : %'-10llu\n",
1348 sample_out.totals.drop_xmit);
1349 if (sample_out.totals.err)
1350 print_always(" Errors recorded : %'-10llu\n",
1351 sample_out.totals.err);
1352 if (sample_out.totals.xmit) {
1353 double pkts = sample_out.totals.xmit;
1354
1355 print_always(" Packets transmitted : %'-10llu\n",
1356 sample_out.totals.xmit);
1357 print_always(" Average transmit/s : %'-10.0f\n",
1358 sample_round(pkts / period));
1359 }
1360}
1361
1362void sample_exit(int status)
1363{
1364 size_t size;
1365
1366 for (int i = 0; i < NUM_MAP; i++) {
1367 size = sample_map_count[i] * sizeof(**sample_mmap);
1368 munmap(sample_mmap[i], size);
1369 }
1370 while (sample_xdp_cnt--) {
1371 int i = sample_xdp_cnt, ifindex, xdp_flags;
1372 __u32 prog_id;
1373
1374 prog_id = sample_xdp_progs[i].prog_id;
1375 ifindex = sample_xdp_progs[i].ifindex;
1376 xdp_flags = sample_xdp_progs[i].flags;
1377
1378 __sample_remove_xdp(ifindex, prog_id, xdp_flags);
1379 }
1380 sample_summary_print();
1381 close(sample_sig_fd);
1382 exit(status);
1383}
1384
1385static int sample_stats_collect(struct stats_record *rec)
1386{
1387 int i;
1388
1389 if (sample_mask & SAMPLE_RX_CNT)
1390 map_collect_percpu(sample_mmap[MAP_RX], &rec->rx_cnt);
1391
1392 if (sample_mask & SAMPLE_REDIRECT_CNT)
1393 map_collect_percpu(sample_mmap[MAP_REDIRECT_ERR], &rec->redir_err[0]);
1394
1395 if (sample_mask & SAMPLE_REDIRECT_ERR_CNT) {
1396 for (i = 1; i < XDP_REDIRECT_ERR_MAX; i++)
1397 map_collect_percpu(&sample_mmap[MAP_REDIRECT_ERR][i * sample_n_cpus],
1398 &rec->redir_err[i]);
1399 }
1400
1401 if (sample_mask & SAMPLE_CPUMAP_ENQUEUE_CNT)
1402 for (i = 0; i < sample_n_cpus; i++)
1403 map_collect_percpu(&sample_mmap[MAP_CPUMAP_ENQUEUE][i * sample_n_cpus],
1404 &rec->enq[i]);
1405
1406 if (sample_mask & SAMPLE_CPUMAP_KTHREAD_CNT)
1407 map_collect_percpu(sample_mmap[MAP_CPUMAP_KTHREAD],
1408 &rec->kthread);
1409
1410 if (sample_mask & SAMPLE_EXCEPTION_CNT)
1411 for (i = 0; i < XDP_ACTION_MAX; i++)
1412 map_collect_percpu(&sample_mmap[MAP_EXCEPTION][i * sample_n_cpus],
1413 &rec->exception[i]);
1414
1415 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT)
1416 map_collect_percpu(sample_mmap[MAP_DEVMAP_XMIT], &rec->devmap_xmit);
1417
1418 if (sample_mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI) {
1419 if (map_collect_percpu_devmap(bpf_map__fd(sample_map[MAP_DEVMAP_XMIT_MULTI]), rec) < 0)
1420 return -EINVAL;
1421 }
1422 return 0;
1423}
1424
1425static void sample_summary_update(struct sample_output *out, int interval)
1426{
1427 sample_out.totals.rx += out->totals.rx;
1428 sample_out.totals.redir += out->totals.redir;
1429 sample_out.totals.drop += out->totals.drop;
1430 sample_out.totals.drop_xmit += out->totals.drop_xmit;
1431 sample_out.totals.err += out->totals.err;
1432 sample_out.totals.xmit += out->totals.xmit;
1433 sample_out.rx_cnt.pps += interval;
1434}
1435
1436static void sample_stats_print(int mask, struct stats_record *cur,
1437 struct stats_record *prev, char *prog_name,
1438 int interval)
1439{
1440 struct sample_output out = {};
1441
1442 if (mask & SAMPLE_RX_CNT)
1443 stats_get_rx_cnt(cur, prev, 0, &out);
1444 if (mask & SAMPLE_REDIRECT_CNT)
1445 stats_get_redirect_cnt(cur, prev, 0, &out);
1446 if (mask & SAMPLE_REDIRECT_ERR_CNT)
1447 stats_get_redirect_err_cnt(cur, prev, 0, &out);
1448 if (mask & SAMPLE_EXCEPTION_CNT)
1449 stats_get_exception_cnt(cur, prev, 0, &out);
1450 if (mask & SAMPLE_DEVMAP_XMIT_CNT)
1451 stats_get_devmap_xmit(cur, prev, 0, &out);
1452 else if (mask & SAMPLE_DEVMAP_XMIT_CNT_MULTI)
1453 stats_get_devmap_xmit_multi(cur, prev, 0, &out,
1454 mask & SAMPLE_DEVMAP_XMIT_CNT);
1455 sample_summary_update(&out, interval);
1456
1457 stats_print(prog_name, mask, cur, prev, &out);
1458}
1459
1460void sample_switch_mode(void)
1461{
1462 sample_log_level ^= LL_DEBUG - 1;
1463}
1464
1465static int sample_signal_cb(void)
1466{
1467 struct signalfd_siginfo si;
1468 int r;
1469
1470 r = read(sample_sig_fd, &si, sizeof(si));
1471 if (r < 0)
1472 return -errno;
1473
1474 switch (si.ssi_signo) {
1475 case SIGQUIT:
1476 sample_switch_mode();
1477 printf("\n");
1478 break;
1479 default:
1480 printf("\n");
1481 return 1;
1482 }
1483
1484 return 0;
1485}
1486
1487
1488static void swap(struct stats_record **a, struct stats_record **b)
1489{
1490 struct stats_record *tmp;
1491
1492 tmp = *a;
1493 *a = *b;
1494 *b = tmp;
1495}
1496
1497static int sample_timer_cb(int timerfd, struct stats_record **rec,
1498 struct stats_record **prev, int interval)
1499{
1500 char line[64] = "Summary";
1501 int ret;
1502 __u64 t;
1503
1504 ret = read(timerfd, &t, sizeof(t));
1505 if (ret < 0)
1506 return -errno;
1507
1508 swap(prev, rec);
1509 ret = sample_stats_collect(*rec);
1510 if (ret < 0)
1511 return ret;
1512
1513 if (sample_xdp_cnt == 2 && !(sample_mask & SAMPLE_SKIP_HEADING)) {
1514 char fi[IFNAMSIZ];
1515 char to[IFNAMSIZ];
1516 const char *f, *t;
1517
1518 f = t = NULL;
1519 if (if_indextoname(sample_xdp_progs[0].ifindex, fi))
1520 f = fi;
1521 if (if_indextoname(sample_xdp_progs[1].ifindex, to))
1522 t = to;
1523
1524 snprintf(line, sizeof(line), "%s->%s", f ?: "?", t ?: "?");
1525 }
1526
1527 sample_stats_print(sample_mask, *rec, *prev, line, interval);
1528 return 0;
1529}
1530
1531int sample_run(int interval, void (*post_cb)(void *), void *ctx)
1532{
1533 struct timespec ts = { interval, 0 };
1534 struct itimerspec its = { ts, ts };
1535 struct stats_record *rec, *prev;
1536 struct pollfd pfd[2] = {};
1537 int timerfd, ret;
1538
1539 if (!interval) {
1540 fprintf(stderr, "Incorrect interval 0\n");
1541 return -EINVAL;
1542 }
1543 sample_interval = interval;
1544
1545 setlocale(LC_NUMERIC, "en_US.UTF-8");
1546
1547 timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC | TFD_NONBLOCK);
1548 if (timerfd < 0)
1549 return -errno;
1550 timerfd_settime(timerfd, 0, &its, NULL);
1551
1552 pfd[0].fd = sample_sig_fd;
1553 pfd[0].events = POLLIN;
1554
1555 pfd[1].fd = timerfd;
1556 pfd[1].events = POLLIN;
1557
1558 ret = -ENOMEM;
1559 rec = alloc_stats_record();
1560 if (!rec)
1561 goto end;
1562 prev = alloc_stats_record();
1563 if (!prev)
1564 goto end_rec;
1565
1566 ret = sample_stats_collect(rec);
1567 if (ret < 0)
1568 goto end_rec_prev;
1569
1570 for (;;) {
1571 ret = poll(pfd, 2, -1);
1572 if (ret < 0) {
1573 if (errno == EINTR)
1574 continue;
1575 else
1576 break;
1577 }
1578
1579 if (pfd[0].revents & POLLIN)
1580 ret = sample_signal_cb();
1581 else if (pfd[1].revents & POLLIN)
1582 ret = sample_timer_cb(timerfd, &rec, &prev, interval);
1583
1584 if (ret)
1585 break;
1586
1587 if (post_cb)
1588 post_cb(ctx);
1589 }
1590
1591end_rec_prev:
1592 free_stats_record(prev);
1593end_rec:
1594 free_stats_record(rec);
1595end:
1596 close(timerfd);
1597
1598 return ret;
1599}
1600
1601const char *get_driver_name(int ifindex)
1602{
1603 struct ethtool_drvinfo drv = {};
1604 char ifname[IF_NAMESIZE];
1605 static char drvname[32];
1606 struct ifreq ifr = {};
1607 int fd, r = 0;
1608
1609 fd = socket(AF_INET, SOCK_DGRAM, 0);
1610 if (fd < 0)
1611 return "[error]";
1612
1613 if (!if_indextoname(ifindex, ifname))
1614 goto end;
1615
1616 drv.cmd = ETHTOOL_GDRVINFO;
1617 safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name));
1618 ifr.ifr_data = (void *)&drv;
1619
1620 r = ioctl(fd, SIOCETHTOOL, &ifr);
1621 if (r)
1622 goto end;
1623
1624 safe_strncpy(drvname, drv.driver, sizeof(drvname));
1625
1626 close(fd);
1627 return drvname;
1628
1629end:
1630 r = errno;
1631 close(fd);
1632 return r == EOPNOTSUPP ? "loopback" : "[error]";
1633}
1634
1635int get_mac_addr(int ifindex, void *mac_addr)
1636{
1637 char ifname[IF_NAMESIZE];
1638 struct ifreq ifr = {};
1639 int fd, r;
1640
1641 fd = socket(AF_INET, SOCK_DGRAM, 0);
1642 if (fd < 0)
1643 return -errno;
1644
1645 if (!if_indextoname(ifindex, ifname)) {
1646 r = -errno;
1647 goto end;
1648 }
1649
1650 safe_strncpy(ifr.ifr_name, ifname, sizeof(ifr.ifr_name));
1651
1652 r = ioctl(fd, SIOCGIFHWADDR, &ifr);
1653 if (r) {
1654 r = -errno;
1655 goto end;
1656 }
1657
1658 memcpy(mac_addr, ifr.ifr_hwaddr.sa_data, 6 * sizeof(char));
1659
1660end:
1661 close(fd);
1662 return r;
1663}
1664
1665__attribute__((constructor)) static void sample_ctor(void)
1666{
1667 if (libbpf_set_strict_mode(LIBBPF_STRICT_ALL) < 0) {
1668 fprintf(stderr, "Failed to set libbpf strict mode: %s\n",
1669 strerror(errno));
1670
1671 exit(EXIT_FAIL_BPF);
1672 }
1673}
1674
