1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24#define _GNU_SOURCE
25
26#include <arpa/inet.h>
27#include <errno.h>
28#include <error.h>
29#include <fcntl.h>
30#include <getopt.h>
31#include <linux/filter.h>
32#include <linux/if_ether.h>
33#include <linux/if_packet.h>
34#include <net/if.h>
35#include <netdb.h>
36#include <netinet/ip.h>
37#include <netinet/ip6.h>
38#include <netinet/tcp.h>
39#include <netinet/udp.h>
40#include <poll.h>
41#include <stdbool.h>
42#include <stddef.h>
43#include <stdint.h>
44#include <stdio.h>
45#include <stdlib.h>
46#include <string.h>
47#include <sys/mman.h>
48#include <sys/socket.h>
49#include <sys/stat.h>
50#include <sys/sysinfo.h>
51#include <sys/time.h>
52#include <sys/types.h>
53#include <unistd.h>
54
55#include "../kselftest.h"
56
57#define TOEPLITZ_KEY_MIN_LEN 40
58#define TOEPLITZ_KEY_MAX_LEN 60
59
60#define TOEPLITZ_STR_LEN(K) (((K) * 3) - 1)
61#define TOEPLITZ_STR_MIN_LEN TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MIN_LEN)
62#define TOEPLITZ_STR_MAX_LEN TOEPLITZ_STR_LEN(TOEPLITZ_KEY_MAX_LEN)
63
64#define FOUR_TUPLE_MAX_LEN ((sizeof(struct in6_addr) * 2) + (sizeof(uint16_t) * 2))
65
66#define RSS_MAX_CPUS (1 << 16)
67
68#define RPS_MAX_CPUS 16UL
69
70
71static uint16_t cfg_dport = 8000;
72static int cfg_family = AF_INET6;
73static char *cfg_ifname = "eth0";
74static int cfg_num_queues;
75static int cfg_num_rps_cpus;
76static bool cfg_sink;
77static int cfg_type = SOCK_STREAM;
78static int cfg_timeout_msec = 1000;
79static bool cfg_verbose;
80
81
82static int num_cpus;
83static int ring_block_nr;
84static int ring_block_sz;
85
86
87static int frames_received;
88static int frames_nohash;
89static int frames_error;
90
91#define log_verbose(args...) do { if (cfg_verbose) fprintf(stderr, args); } while (0)
92
93
94struct ring_state {
95 int fd;
96 char *mmap;
97 int idx;
98 int cpu;
99};
100
101static unsigned int rx_irq_cpus[RSS_MAX_CPUS];
102static int rps_silo_to_cpu[RPS_MAX_CPUS];
103static unsigned char toeplitz_key[TOEPLITZ_KEY_MAX_LEN];
104static struct ring_state rings[RSS_MAX_CPUS];
105
106static inline uint32_t toeplitz(const unsigned char *four_tuple,
107 const unsigned char *key)
108{
109 int i, bit, ret = 0;
110 uint32_t key32;
111
112 key32 = ntohl(*((uint32_t *)key));
113 key += 4;
114
115 for (i = 0; i < FOUR_TUPLE_MAX_LEN; i++) {
116 for (bit = 7; bit >= 0; bit--) {
117 if (four_tuple[i] & (1 << bit))
118 ret ^= key32;
119
120 key32 <<= 1;
121 key32 |= !!(key[0] & (1 << bit));
122 }
123 key++;
124 }
125
126 return ret;
127}
128
129
130static void verify_rss(uint32_t rx_hash, int cpu)
131{
132 int queue = rx_hash % cfg_num_queues;
133
134 log_verbose(" rxq %d (cpu %d)", queue, rx_irq_cpus[queue]);
135 if (rx_irq_cpus[queue] != cpu) {
136 log_verbose(". error: rss cpu mismatch (%d)", cpu);
137 frames_error++;
138 }
139}
140
141static void verify_rps(uint64_t rx_hash, int cpu)
142{
143 int silo = (rx_hash * cfg_num_rps_cpus) >> 32;
144
145 log_verbose(" silo %d (cpu %d)", silo, rps_silo_to_cpu[silo]);
146 if (rps_silo_to_cpu[silo] != cpu) {
147 log_verbose(". error: rps cpu mismatch (%d)", cpu);
148 frames_error++;
149 }
150}
151
152static void log_rxhash(int cpu, uint32_t rx_hash,
153 const char *addrs, int addr_len)
154{
155 char saddr[INET6_ADDRSTRLEN], daddr[INET6_ADDRSTRLEN];
156 uint16_t *ports;
157
158 if (!inet_ntop(cfg_family, addrs, saddr, sizeof(saddr)) ||
159 !inet_ntop(cfg_family, addrs + addr_len, daddr, sizeof(daddr)))
160 error(1, 0, "address parse error");
161
162 ports = (void *)addrs + (addr_len * 2);
163 log_verbose("cpu %d: rx_hash 0x%08x [saddr %s daddr %s sport %02hu dport %02hu]",
164 cpu, rx_hash, saddr, daddr,
165 ntohs(ports[0]), ntohs(ports[1]));
166}
167
168
169static void verify_rxhash(const char *pkt, uint32_t rx_hash, int cpu)
170{
171 unsigned char four_tuple[FOUR_TUPLE_MAX_LEN] = {0};
172 uint32_t rx_hash_sw;
173 const char *addrs;
174 int addr_len;
175
176 if (cfg_family == AF_INET) {
177 addr_len = sizeof(struct in_addr);
178 addrs = pkt + offsetof(struct iphdr, saddr);
179 } else {
180 addr_len = sizeof(struct in6_addr);
181 addrs = pkt + offsetof(struct ip6_hdr, ip6_src);
182 }
183
184 memcpy(four_tuple, addrs, (addr_len * 2) + (sizeof(uint16_t) * 2));
185 rx_hash_sw = toeplitz(four_tuple, toeplitz_key);
186
187 if (cfg_verbose)
188 log_rxhash(cpu, rx_hash, addrs, addr_len);
189
190 if (rx_hash != rx_hash_sw) {
191 log_verbose(" != expected 0x%x\n", rx_hash_sw);
192 frames_error++;
193 return;
194 }
195
196 log_verbose(" OK");
197 if (cfg_num_queues)
198 verify_rss(rx_hash, cpu);
199 else if (cfg_num_rps_cpus)
200 verify_rps(rx_hash, cpu);
201 log_verbose("\n");
202}
203
204static char *recv_frame(const struct ring_state *ring, char *frame)
205{
206 struct tpacket3_hdr *hdr = (void *)frame;
207
208 if (hdr->hv1.tp_rxhash)
209 verify_rxhash(frame + hdr->tp_net, hdr->hv1.tp_rxhash,
210 ring->cpu);
211 else
212 frames_nohash++;
213
214 return frame + hdr->tp_next_offset;
215}
216
217
218static void recv_block(struct ring_state *ring)
219{
220 struct tpacket_block_desc *block;
221 char *frame;
222 int i;
223
224 block = (void *)(ring->mmap + ring->idx * ring_block_sz);
225 if (!(block->hdr.bh1.block_status & TP_STATUS_USER))
226 return;
227
228 frame = (char *)block;
229 frame += block->hdr.bh1.offset_to_first_pkt;
230
231 for (i = 0; i < block->hdr.bh1.num_pkts; i++) {
232 frame = recv_frame(ring, frame);
233 frames_received++;
234 }
235
236 block->hdr.bh1.block_status = TP_STATUS_KERNEL;
237 ring->idx = (ring->idx + 1) % ring_block_nr;
238}
239
240
241static void process_rings(void)
242{
243 int i;
244
245 usleep(1000 * cfg_timeout_msec);
246
247 for (i = 0; i < num_cpus; i++)
248 recv_block(&rings[i]);
249
250 fprintf(stderr, "count: pass=%u nohash=%u fail=%u\n",
251 frames_received - frames_nohash - frames_error,
252 frames_nohash, frames_error);
253}
254
255static char *setup_ring(int fd)
256{
257 struct tpacket_req3 req3 = {0};
258 void *ring;
259
260 req3.tp_retire_blk_tov = cfg_timeout_msec;
261 req3.tp_feature_req_word = TP_FT_REQ_FILL_RXHASH;
262
263 req3.tp_frame_size = 2048;
264 req3.tp_frame_nr = 1 << 10;
265 req3.tp_block_nr = 2;
266
267 req3.tp_block_size = req3.tp_frame_size * req3.tp_frame_nr;
268 req3.tp_block_size /= req3.tp_block_nr;
269
270 if (setsockopt(fd, SOL_PACKET, PACKET_RX_RING, &req3, sizeof(req3)))
271 error(1, errno, "setsockopt PACKET_RX_RING");
272
273 ring_block_sz = req3.tp_block_size;
274 ring_block_nr = req3.tp_block_nr;
275
276 ring = mmap(0, req3.tp_block_size * req3.tp_block_nr,
277 PROT_READ | PROT_WRITE,
278 MAP_SHARED | MAP_LOCKED | MAP_POPULATE, fd, 0);
279 if (ring == MAP_FAILED)
280 error(1, 0, "mmap failed");
281
282 return ring;
283}
284
285static void __set_filter(int fd, int off_proto, uint8_t proto, int off_dport)
286{
287 struct sock_filter filter[] = {
288 BPF_STMT(BPF_LD + BPF_B + BPF_ABS, SKF_AD_OFF + SKF_AD_PKTTYPE),
289 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, PACKET_HOST, 0, 4),
290 BPF_STMT(BPF_LD + BPF_B + BPF_ABS, off_proto),
291 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, proto, 0, 2),
292 BPF_STMT(BPF_LD + BPF_H + BPF_ABS, off_dport),
293 BPF_JUMP(BPF_JMP + BPF_JEQ + BPF_K, cfg_dport, 1, 0),
294 BPF_STMT(BPF_RET + BPF_K, 0),
295 BPF_STMT(BPF_RET + BPF_K, 0xFFFF),
296 };
297 struct sock_fprog prog = {};
298
299 prog.filter = filter;
300 prog.len = ARRAY_SIZE(filter);
301 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog)))
302 error(1, errno, "setsockopt filter");
303}
304
305
306static void set_filter(int fd)
307{
308 const int off_dport = offsetof(struct tcphdr, dest);
309 uint8_t proto;
310
311 proto = cfg_type == SOCK_STREAM ? IPPROTO_TCP : IPPROTO_UDP;
312 if (cfg_family == AF_INET)
313 __set_filter(fd, offsetof(struct iphdr, protocol), proto,
314 sizeof(struct iphdr) + off_dport);
315 else
316 __set_filter(fd, offsetof(struct ip6_hdr, ip6_nxt), proto,
317 sizeof(struct ip6_hdr) + off_dport);
318}
319
320
321static void set_filter_null(int fd)
322{
323 struct sock_filter filter[] = {
324 BPF_STMT(BPF_RET + BPF_K, 0),
325 };
326 struct sock_fprog prog = {};
327
328 prog.filter = filter;
329 prog.len = ARRAY_SIZE(filter);
330 if (setsockopt(fd, SOL_SOCKET, SO_ATTACH_FILTER, &prog, sizeof(prog)))
331 error(1, errno, "setsockopt filter");
332}
333
334static int create_ring(char **ring)
335{
336 struct fanout_args args = {
337 .id = 1,
338 .type_flags = PACKET_FANOUT_CPU,
339 .max_num_members = RSS_MAX_CPUS
340 };
341 struct sockaddr_ll ll = { 0 };
342 int fd, val;
343
344 fd = socket(PF_PACKET, SOCK_DGRAM, 0);
345 if (fd == -1)
346 error(1, errno, "socket creation failed");
347
348 val = TPACKET_V3;
349 if (setsockopt(fd, SOL_PACKET, PACKET_VERSION, &val, sizeof(val)))
350 error(1, errno, "setsockopt PACKET_VERSION");
351 *ring = setup_ring(fd);
352
353
354
355
356 set_filter_null(fd);
357
358 ll.sll_family = AF_PACKET;
359 ll.sll_ifindex = if_nametoindex(cfg_ifname);
360 ll.sll_protocol = cfg_family == AF_INET ? htons(ETH_P_IP) :
361 htons(ETH_P_IPV6);
362 if (bind(fd, (void *)&ll, sizeof(ll)))
363 error(1, errno, "bind");
364
365
366 if (setsockopt(fd, SOL_PACKET, PACKET_FANOUT, &args, sizeof(args))) {
367
368
369
370
371
372 if (cfg_num_queues || num_cpus > 256 ||
373 setsockopt(fd, SOL_PACKET, PACKET_FANOUT,
374 &args, sizeof(uint32_t)))
375 error(1, errno, "setsockopt PACKET_FANOUT cpu");
376 }
377
378 return fd;
379}
380
381
382static int setup_sink(void)
383{
384 int fd, val;
385
386 fd = socket(cfg_family, cfg_type, 0);
387 if (fd == -1)
388 error(1, errno, "socket %d.%d", cfg_family, cfg_type);
389
390 val = 1 << 20;
391 if (setsockopt(fd, SOL_SOCKET, SO_RCVBUFFORCE, &val, sizeof(val)))
392 error(1, errno, "setsockopt rcvbuf");
393
394 return fd;
395}
396
397static void setup_rings(void)
398{
399 int i;
400
401 for (i = 0; i < num_cpus; i++) {
402 rings[i].cpu = i;
403 rings[i].fd = create_ring(&rings[i].mmap);
404 }
405
406
407 for (i = 0; i < num_cpus; i++)
408 set_filter(rings[i].fd);
409}
410
411static void cleanup_rings(void)
412{
413 int i;
414
415 for (i = 0; i < num_cpus; i++) {
416 if (munmap(rings[i].mmap, ring_block_nr * ring_block_sz))
417 error(1, errno, "munmap");
418 if (close(rings[i].fd))
419 error(1, errno, "close");
420 }
421}
422
423static void parse_cpulist(const char *arg)
424{
425 do {
426 rx_irq_cpus[cfg_num_queues++] = strtol(arg, NULL, 10);
427
428 arg = strchr(arg, ',');
429 if (!arg)
430 break;
431 arg++;
432 } while (1);
433}
434
435static void show_cpulist(void)
436{
437 int i;
438
439 for (i = 0; i < cfg_num_queues; i++)
440 fprintf(stderr, "rxq %d: cpu %d\n", i, rx_irq_cpus[i]);
441}
442
443static void show_silos(void)
444{
445 int i;
446
447 for (i = 0; i < cfg_num_rps_cpus; i++)
448 fprintf(stderr, "silo %d: cpu %d\n", i, rps_silo_to_cpu[i]);
449}
450
451static void parse_toeplitz_key(const char *str, int slen, unsigned char *key)
452{
453 int i, ret, off;
454
455 if (slen < TOEPLITZ_STR_MIN_LEN ||
456 slen > TOEPLITZ_STR_MAX_LEN + 1)
457 error(1, 0, "invalid toeplitz key");
458
459 for (i = 0, off = 0; off < slen; i++, off += 3) {
460 ret = sscanf(str + off, "%hhx", &key[i]);
461 if (ret != 1)
462 error(1, 0, "key parse error at %d off %d len %d",
463 i, off, slen);
464 }
465}
466
467static void parse_rps_bitmap(const char *arg)
468{
469 unsigned long bitmap;
470 int i;
471
472 bitmap = strtoul(arg, NULL, 0);
473
474 if (bitmap & ~(RPS_MAX_CPUS - 1))
475 error(1, 0, "rps bitmap 0x%lx out of bounds 0..%lu",
476 bitmap, RPS_MAX_CPUS - 1);
477
478 for (i = 0; i < RPS_MAX_CPUS; i++)
479 if (bitmap & 1UL << i)
480 rps_silo_to_cpu[cfg_num_rps_cpus++] = i;
481}
482
483static void parse_opts(int argc, char **argv)
484{
485 static struct option long_options[] = {
486 {"dport", required_argument, 0, 'd'},
487 {"cpus", required_argument, 0, 'C'},
488 {"key", required_argument, 0, 'k'},
489 {"iface", required_argument, 0, 'i'},
490 {"ipv4", no_argument, 0, '4'},
491 {"ipv6", no_argument, 0, '6'},
492 {"sink", no_argument, 0, 's'},
493 {"tcp", no_argument, 0, 't'},
494 {"timeout", required_argument, 0, 'T'},
495 {"udp", no_argument, 0, 'u'},
496 {"verbose", no_argument, 0, 'v'},
497 {"rps", required_argument, 0, 'r'},
498 {0, 0, 0, 0}
499 };
500 bool have_toeplitz = false;
501 int index, c;
502
503 while ((c = getopt_long(argc, argv, "46C:d:i:k:r:stT:uv", long_options, &index)) != -1) {
504 switch (c) {
505 case '4':
506 cfg_family = AF_INET;
507 break;
508 case '6':
509 cfg_family = AF_INET6;
510 break;
511 case 'C':
512 parse_cpulist(optarg);
513 break;
514 case 'd':
515 cfg_dport = strtol(optarg, NULL, 0);
516 break;
517 case 'i':
518 cfg_ifname = optarg;
519 break;
520 case 'k':
521 parse_toeplitz_key(optarg, strlen(optarg),
522 toeplitz_key);
523 have_toeplitz = true;
524 break;
525 case 'r':
526 parse_rps_bitmap(optarg);
527 break;
528 case 's':
529 cfg_sink = true;
530 break;
531 case 't':
532 cfg_type = SOCK_STREAM;
533 break;
534 case 'T':
535 cfg_timeout_msec = strtol(optarg, NULL, 0);
536 break;
537 case 'u':
538 cfg_type = SOCK_DGRAM;
539 break;
540 case 'v':
541 cfg_verbose = true;
542 break;
543
544 default:
545 error(1, 0, "unknown option %c", optopt);
546 break;
547 }
548 }
549
550 if (!have_toeplitz)
551 error(1, 0, "Must supply rss key ('-k')");
552
553 num_cpus = get_nprocs();
554 if (num_cpus > RSS_MAX_CPUS)
555 error(1, 0, "increase RSS_MAX_CPUS");
556
557 if (cfg_num_queues && cfg_num_rps_cpus)
558 error(1, 0,
559 "Can't supply both RSS cpus ('-C') and RPS map ('-r')");
560 if (cfg_verbose) {
561 show_cpulist();
562 show_silos();
563 }
564}
565
566int main(int argc, char **argv)
567{
568 const int min_tests = 10;
569 int fd_sink = -1;
570
571 parse_opts(argc, argv);
572
573 if (cfg_sink)
574 fd_sink = setup_sink();
575
576 setup_rings();
577 process_rings();
578 cleanup_rings();
579
580 if (cfg_sink && close(fd_sink))
581 error(1, errno, "close sink");
582
583 if (frames_received - frames_nohash < min_tests)
584 error(1, 0, "too few frames for verification");
585
586 return frames_error;
587}
588
