1
2
3
4
5
6
7
8#ifdef HAVE_EVENTFD
9
10#include <string.h>
11#include <pthread.h>
12
13#include <errno.h>
14#include <inttypes.h>
15#include <signal.h>
16#include <stdlib.h>
17#include <linux/compiler.h>
18#include <linux/kernel.h>
19#include <sys/time.h>
20#include <sys/resource.h>
21#include <sys/epoll.h>
22#include <sys/eventfd.h>
23
24#include "../util/stat.h"
25#include <subcmd/parse-options.h>
26#include "bench.h"
27#include "cpumap.h"
28
29#include <err.h>
30
31#define printinfo(fmt, arg...) \
32 do { if (__verbose) printf(fmt, ## arg); } while (0)
33
34static unsigned int nthreads = 0;
35static unsigned int nsecs = 8;
36struct timeval start, end, runtime;
37static bool done, __verbose, randomize;
38
39
40
41
42
43
44#define EPOLL_MAXNESTS 4
45
46enum {
47 OP_EPOLL_ADD,
48 OP_EPOLL_MOD,
49 OP_EPOLL_DEL,
50 EPOLL_NR_OPS,
51};
52
53static int epollfd;
54static int *epollfdp;
55static bool noaffinity;
56static unsigned int nested = 0;
57
58
59static unsigned int nfds = 64;
60
61static pthread_mutex_t thread_lock;
62static unsigned int threads_starting;
63static struct stats all_stats[EPOLL_NR_OPS];
64static pthread_cond_t thread_parent, thread_worker;
65
66struct worker {
67 int tid;
68 pthread_t thread;
69 unsigned long ops[EPOLL_NR_OPS];
70 int *fdmap;
71};
72
73static const struct option options[] = {
74 OPT_UINTEGER('t', "threads", &nthreads, "Specify amount of threads"),
75 OPT_UINTEGER('r', "runtime", &nsecs, "Specify runtime (in seconds)"),
76 OPT_UINTEGER('f', "nfds", &nfds, "Specify amount of file descriptors to monitor for each thread"),
77 OPT_BOOLEAN( 'n', "noaffinity", &noaffinity, "Disables CPU affinity"),
78 OPT_UINTEGER( 'N', "nested", &nested, "Nesting level epoll hierarchy (default is 0, no nesting)"),
79 OPT_BOOLEAN( 'R', "randomize", &randomize, "Perform random operations on random fds"),
80 OPT_BOOLEAN( 'v', "verbose", &__verbose, "Verbose mode"),
81 OPT_END()
82};
83
84static const char * const bench_epoll_ctl_usage[] = {
85 "perf bench epoll ctl <options>",
86 NULL
87};
88
89static void toggle_done(int sig __maybe_unused,
90 siginfo_t *info __maybe_unused,
91 void *uc __maybe_unused)
92{
93
94 done = true;
95 gettimeofday(&end, NULL);
96 timersub(&end, &start, &runtime);
97}
98
99static void nest_epollfd(void)
100{
101 unsigned int i;
102 struct epoll_event ev;
103
104 if (nested > EPOLL_MAXNESTS)
105 nested = EPOLL_MAXNESTS;
106 printinfo("Nesting level(s): %d\n", nested);
107
108 epollfdp = calloc(nested, sizeof(int));
109 if (!epollfd)
110 err(EXIT_FAILURE, "calloc");
111
112 for (i = 0; i < nested; i++) {
113 epollfdp[i] = epoll_create(1);
114 if (epollfd < 0)
115 err(EXIT_FAILURE, "epoll_create");
116 }
117
118 ev.events = EPOLLHUP;
119 ev.data.u64 = i;
120
121 for (i = nested - 1; i; i--) {
122 if (epoll_ctl(epollfdp[i - 1], EPOLL_CTL_ADD,
123 epollfdp[i], &ev) < 0)
124 err(EXIT_FAILURE, "epoll_ctl");
125 }
126
127 if (epoll_ctl(epollfd, EPOLL_CTL_ADD, *epollfdp, &ev) < 0)
128 err(EXIT_FAILURE, "epoll_ctl");
129}
130
131static inline void do_epoll_op(struct worker *w, int op, int fd)
132{
133 int error;
134 struct epoll_event ev;
135
136 ev.events = EPOLLIN;
137 ev.data.u64 = fd;
138
139 switch (op) {
140 case OP_EPOLL_ADD:
141 error = epoll_ctl(epollfd, EPOLL_CTL_ADD, fd, &ev);
142 break;
143 case OP_EPOLL_MOD:
144 ev.events = EPOLLOUT;
145 error = epoll_ctl(epollfd, EPOLL_CTL_MOD, fd, &ev);
146 break;
147 case OP_EPOLL_DEL:
148 error = epoll_ctl(epollfd, EPOLL_CTL_DEL, fd, NULL);
149 break;
150 default:
151 error = 1;
152 break;
153 }
154
155 if (!error)
156 w->ops[op]++;
157}
158
159static inline void do_random_epoll_op(struct worker *w)
160{
161 unsigned long rnd1 = random(), rnd2 = random();
162 int op, fd;
163
164 fd = w->fdmap[rnd1 % nfds];
165 op = rnd2 % EPOLL_NR_OPS;
166
167 do_epoll_op(w, op, fd);
168}
169
170static void *workerfn(void *arg)
171{
172 unsigned int i;
173 struct worker *w = (struct worker *) arg;
174 struct timespec ts = { .tv_sec = 0,
175 .tv_nsec = 250 };
176
177 pthread_mutex_lock(&thread_lock);
178 threads_starting--;
179 if (!threads_starting)
180 pthread_cond_signal(&thread_parent);
181 pthread_cond_wait(&thread_worker, &thread_lock);
182 pthread_mutex_unlock(&thread_lock);
183
184
185 do {
186
187 if (randomize) {
188 do_random_epoll_op(w);
189 } else {
190 for (i = 0; i < nfds; i++) {
191 do_epoll_op(w, OP_EPOLL_ADD, w->fdmap[i]);
192 do_epoll_op(w, OP_EPOLL_MOD, w->fdmap[i]);
193 do_epoll_op(w, OP_EPOLL_DEL, w->fdmap[i]);
194 }
195 }
196
197 nanosleep(&ts, NULL);
198 } while (!done);
199
200 return NULL;
201}
202
203static void init_fdmaps(struct worker *w, int pct)
204{
205 unsigned int i;
206 int inc;
207 struct epoll_event ev;
208
209 if (!pct)
210 return;
211
212 inc = 100/pct;
213 for (i = 0; i < nfds; i+=inc) {
214 ev.data.fd = w->fdmap[i];
215 ev.events = EPOLLIN;
216
217 if (epoll_ctl(epollfd, EPOLL_CTL_ADD, w->fdmap[i], &ev) < 0)
218 err(EXIT_FAILURE, "epoll_ct");
219 }
220}
221
222static int do_threads(struct worker *worker, struct cpu_map *cpu)
223{
224 pthread_attr_t thread_attr, *attrp = NULL;
225 cpu_set_t cpuset;
226 unsigned int i, j;
227 int ret = 0;
228
229 if (!noaffinity)
230 pthread_attr_init(&thread_attr);
231
232 for (i = 0; i < nthreads; i++) {
233 struct worker *w = &worker[i];
234
235 w->tid = i;
236 w->fdmap = calloc(nfds, sizeof(int));
237 if (!w->fdmap)
238 return 1;
239
240 for (j = 0; j < nfds; j++) {
241 w->fdmap[j] = eventfd(0, EFD_NONBLOCK);
242 if (w->fdmap[j] < 0)
243 err(EXIT_FAILURE, "eventfd");
244 }
245
246
247
248
249
250
251 if (randomize)
252 init_fdmaps(w, 50);
253
254 if (!noaffinity) {
255 CPU_ZERO(&cpuset);
256 CPU_SET(cpu->map[i % cpu->nr], &cpuset);
257
258 ret = pthread_attr_setaffinity_np(&thread_attr, sizeof(cpu_set_t), &cpuset);
259 if (ret)
260 err(EXIT_FAILURE, "pthread_attr_setaffinity_np");
261
262 attrp = &thread_attr;
263 }
264
265 ret = pthread_create(&w->thread, attrp, workerfn,
266 (void *)(struct worker *) w);
267 if (ret)
268 err(EXIT_FAILURE, "pthread_create");
269 }
270
271 if (!noaffinity)
272 pthread_attr_destroy(&thread_attr);
273
274 return ret;
275}
276
277static void print_summary(void)
278{
279 int i;
280 unsigned long avg[EPOLL_NR_OPS];
281 double stddev[EPOLL_NR_OPS];
282
283 for (i = 0; i < EPOLL_NR_OPS; i++) {
284 avg[i] = avg_stats(&all_stats[i]);
285 stddev[i] = stddev_stats(&all_stats[i]);
286 }
287
288 printf("\nAveraged %ld ADD operations (+- %.2f%%)\n",
289 avg[OP_EPOLL_ADD], rel_stddev_stats(stddev[OP_EPOLL_ADD],
290 avg[OP_EPOLL_ADD]));
291 printf("Averaged %ld MOD operations (+- %.2f%%)\n",
292 avg[OP_EPOLL_MOD], rel_stddev_stats(stddev[OP_EPOLL_MOD],
293 avg[OP_EPOLL_MOD]));
294 printf("Averaged %ld DEL operations (+- %.2f%%)\n",
295 avg[OP_EPOLL_DEL], rel_stddev_stats(stddev[OP_EPOLL_DEL],
296 avg[OP_EPOLL_DEL]));
297}
298
299int bench_epoll_ctl(int argc, const char **argv)
300{
301 int j, ret = 0;
302 struct sigaction act;
303 struct worker *worker = NULL;
304 struct cpu_map *cpu;
305 struct rlimit rl, prevrl;
306 unsigned int i;
307
308 argc = parse_options(argc, argv, options, bench_epoll_ctl_usage, 0);
309 if (argc) {
310 usage_with_options(bench_epoll_ctl_usage, options);
311 exit(EXIT_FAILURE);
312 }
313
314 sigfillset(&act.sa_mask);
315 act.sa_sigaction = toggle_done;
316 sigaction(SIGINT, &act, NULL);
317
318 cpu = cpu_map__new(NULL);
319 if (!cpu)
320 goto errmem;
321
322
323 epollfd = epoll_create(1);
324 if (epollfd < 0)
325 err(EXIT_FAILURE, "epoll_create");
326
327
328
329
330 if (nested)
331 nest_epollfd();
332
333
334 if (!nthreads)
335 nthreads = cpu->nr;
336
337 worker = calloc(nthreads, sizeof(*worker));
338 if (!worker)
339 goto errmem;
340
341 if (getrlimit(RLIMIT_NOFILE, &prevrl))
342 err(EXIT_FAILURE, "getrlimit");
343 rl.rlim_cur = rl.rlim_max = nfds * nthreads * 2 + 50;
344 printinfo("Setting RLIMIT_NOFILE rlimit from %" PRIu64 " to: %" PRIu64 "\n",
345 (uint64_t)prevrl.rlim_max, (uint64_t)rl.rlim_max);
346 if (setrlimit(RLIMIT_NOFILE, &rl) < 0)
347 err(EXIT_FAILURE, "setrlimit");
348
349 printf("Run summary [PID %d]: %d threads doing epoll_ctl ops "
350 "%d file-descriptors for %d secs.\n\n",
351 getpid(), nthreads, nfds, nsecs);
352
353 for (i = 0; i < EPOLL_NR_OPS; i++)
354 init_stats(&all_stats[i]);
355
356 pthread_mutex_init(&thread_lock, NULL);
357 pthread_cond_init(&thread_parent, NULL);
358 pthread_cond_init(&thread_worker, NULL);
359
360 threads_starting = nthreads;
361
362 gettimeofday(&start, NULL);
363
364 do_threads(worker, cpu);
365
366 pthread_mutex_lock(&thread_lock);
367 while (threads_starting)
368 pthread_cond_wait(&thread_parent, &thread_lock);
369 pthread_cond_broadcast(&thread_worker);
370 pthread_mutex_unlock(&thread_lock);
371
372 sleep(nsecs);
373 toggle_done(0, NULL, NULL);
374 printinfo("main thread: toggling done\n");
375
376 for (i = 0; i < nthreads; i++) {
377 ret = pthread_join(worker[i].thread, NULL);
378 if (ret)
379 err(EXIT_FAILURE, "pthread_join");
380 }
381
382
383 pthread_cond_destroy(&thread_parent);
384 pthread_cond_destroy(&thread_worker);
385 pthread_mutex_destroy(&thread_lock);
386
387 for (i = 0; i < nthreads; i++) {
388 unsigned long t[EPOLL_NR_OPS];
389
390 for (j = 0; j < EPOLL_NR_OPS; j++) {
391 t[j] = worker[i].ops[j];
392 update_stats(&all_stats[j], t[j]);
393 }
394
395 if (nfds == 1)
396 printf("[thread %2d] fdmap: %p [ add: %04ld; mod: %04ld; del: %04lds ops ]\n",
397 worker[i].tid, &worker[i].fdmap[0],
398 t[OP_EPOLL_ADD], t[OP_EPOLL_MOD], t[OP_EPOLL_DEL]);
399 else
400 printf("[thread %2d] fdmap: %p ... %p [ add: %04ld ops; mod: %04ld ops; del: %04ld ops ]\n",
401 worker[i].tid, &worker[i].fdmap[0],
402 &worker[i].fdmap[nfds-1],
403 t[OP_EPOLL_ADD], t[OP_EPOLL_MOD], t[OP_EPOLL_DEL]);
404 }
405
406 print_summary();
407
408 close(epollfd);
409 return ret;
410errmem:
411 err(EXIT_FAILURE, "calloc");
412}
413#endif
414