1
2
3
4
5
6
7
8#include <linux/file.h>
9#include <linux/poll.h>
10#include <linux/init.h>
11#include <linux/fs.h>
12#include <linux/sched.h>
13#include <linux/kernel.h>
14#include <linux/slab.h>
15#include <linux/list.h>
16#include <linux/spinlock.h>
17#include <linux/anon_inodes.h>
18#include <linux/syscalls.h>
19#include <linux/export.h>
20#include <linux/kref.h>
21#include <linux/eventfd.h>
22#include <linux/proc_fs.h>
23#include <linux/seq_file.h>
24
25struct eventfd_ctx {
26 struct kref kref;
27 wait_queue_head_t wqh;
28
29
30
31
32
33
34
35
36 __u64 count;
37 unsigned int flags;
38};
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54__u64 eventfd_signal(struct eventfd_ctx *ctx, __u64 n)
55{
56 unsigned long flags;
57
58 spin_lock_irqsave(&ctx->wqh.lock, flags);
59 if (ULLONG_MAX - ctx->count < n)
60 n = ULLONG_MAX - ctx->count;
61 ctx->count += n;
62 if (waitqueue_active(&ctx->wqh))
63 wake_up_locked_poll(&ctx->wqh, POLLIN);
64 spin_unlock_irqrestore(&ctx->wqh.lock, flags);
65
66 return n;
67}
68EXPORT_SYMBOL_GPL(eventfd_signal);
69
70static void eventfd_free_ctx(struct eventfd_ctx *ctx)
71{
72 kfree(ctx);
73}
74
75static void eventfd_free(struct kref *kref)
76{
77 struct eventfd_ctx *ctx = container_of(kref, struct eventfd_ctx, kref);
78
79 eventfd_free_ctx(ctx);
80}
81
82
83
84
85
86
87
88struct eventfd_ctx *eventfd_ctx_get(struct eventfd_ctx *ctx)
89{
90 kref_get(&ctx->kref);
91 return ctx;
92}
93EXPORT_SYMBOL_GPL(eventfd_ctx_get);
94
95
96
97
98
99
100
101
102void eventfd_ctx_put(struct eventfd_ctx *ctx)
103{
104 kref_put(&ctx->kref, eventfd_free);
105}
106EXPORT_SYMBOL_GPL(eventfd_ctx_put);
107
108static int eventfd_release(struct inode *inode, struct file *file)
109{
110 struct eventfd_ctx *ctx = file->private_data;
111
112 wake_up_poll(&ctx->wqh, POLLHUP);
113 eventfd_ctx_put(ctx);
114 return 0;
115}
116
117static unsigned int eventfd_poll(struct file *file, poll_table *wait)
118{
119 struct eventfd_ctx *ctx = file->private_data;
120 unsigned int events = 0;
121 u64 count;
122
123 poll_wait(file, &ctx->wqh, wait);
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163 count = READ_ONCE(ctx->count);
164
165 if (count > 0)
166 events |= POLLIN;
167 if (count == ULLONG_MAX)
168 events |= POLLERR;
169 if (ULLONG_MAX - 1 > count)
170 events |= POLLOUT;
171
172 return events;
173}
174
175static void eventfd_ctx_do_read(struct eventfd_ctx *ctx, __u64 *cnt)
176{
177 *cnt = (ctx->flags & EFD_SEMAPHORE) ? 1 : ctx->count;
178 ctx->count -= *cnt;
179}
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194int eventfd_ctx_remove_wait_queue(struct eventfd_ctx *ctx, wait_queue_t *wait,
195 __u64 *cnt)
196{
197 unsigned long flags;
198
199 spin_lock_irqsave(&ctx->wqh.lock, flags);
200 eventfd_ctx_do_read(ctx, cnt);
201 __remove_wait_queue(&ctx->wqh, wait);
202 if (*cnt != 0 && waitqueue_active(&ctx->wqh))
203 wake_up_locked_poll(&ctx->wqh, POLLOUT);
204 spin_unlock_irqrestore(&ctx->wqh.lock, flags);
205
206 return *cnt != 0 ? 0 : -EAGAIN;
207}
208EXPORT_SYMBOL_GPL(eventfd_ctx_remove_wait_queue);
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224ssize_t eventfd_ctx_read(struct eventfd_ctx *ctx, int no_wait, __u64 *cnt)
225{
226 ssize_t res;
227 DECLARE_WAITQUEUE(wait, current);
228
229 spin_lock_irq(&ctx->wqh.lock);
230 *cnt = 0;
231 res = -EAGAIN;
232 if (ctx->count > 0)
233 res = 0;
234 else if (!no_wait) {
235 __add_wait_queue(&ctx->wqh, &wait);
236 for (;;) {
237 set_current_state(TASK_INTERRUPTIBLE);
238 if (ctx->count > 0) {
239 res = 0;
240 break;
241 }
242 if (signal_pending(current)) {
243 res = -ERESTARTSYS;
244 break;
245 }
246 spin_unlock_irq(&ctx->wqh.lock);
247 schedule();
248 spin_lock_irq(&ctx->wqh.lock);
249 }
250 __remove_wait_queue(&ctx->wqh, &wait);
251 __set_current_state(TASK_RUNNING);
252 }
253 if (likely(res == 0)) {
254 eventfd_ctx_do_read(ctx, cnt);
255 if (waitqueue_active(&ctx->wqh))
256 wake_up_locked_poll(&ctx->wqh, POLLOUT);
257 }
258 spin_unlock_irq(&ctx->wqh.lock);
259
260 return res;
261}
262EXPORT_SYMBOL_GPL(eventfd_ctx_read);
263
264static ssize_t eventfd_read(struct file *file, char __user *buf, size_t count,
265 loff_t *ppos)
266{
267 struct eventfd_ctx *ctx = file->private_data;
268 ssize_t res;
269 __u64 cnt;
270
271 if (count < sizeof(cnt))
272 return -EINVAL;
273 res = eventfd_ctx_read(ctx, file->f_flags & O_NONBLOCK, &cnt);
274 if (res < 0)
275 return res;
276
277 return put_user(cnt, (__u64 __user *) buf) ? -EFAULT : sizeof(cnt);
278}
279
280static ssize_t eventfd_write(struct file *file, const char __user *buf, size_t count,
281 loff_t *ppos)
282{
283 struct eventfd_ctx *ctx = file->private_data;
284 ssize_t res;
285 __u64 ucnt;
286 DECLARE_WAITQUEUE(wait, current);
287
288 if (count < sizeof(ucnt))
289 return -EINVAL;
290 if (copy_from_user(&ucnt, buf, sizeof(ucnt)))
291 return -EFAULT;
292 if (ucnt == ULLONG_MAX)
293 return -EINVAL;
294 spin_lock_irq(&ctx->wqh.lock);
295 res = -EAGAIN;
296 if (ULLONG_MAX - ctx->count > ucnt)
297 res = sizeof(ucnt);
298 else if (!(file->f_flags & O_NONBLOCK)) {
299 __add_wait_queue(&ctx->wqh, &wait);
300 for (res = 0;;) {
301 set_current_state(TASK_INTERRUPTIBLE);
302 if (ULLONG_MAX - ctx->count > ucnt) {
303 res = sizeof(ucnt);
304 break;
305 }
306 if (signal_pending(current)) {
307 res = -ERESTARTSYS;
308 break;
309 }
310 spin_unlock_irq(&ctx->wqh.lock);
311 schedule();
312 spin_lock_irq(&ctx->wqh.lock);
313 }
314 __remove_wait_queue(&ctx->wqh, &wait);
315 __set_current_state(TASK_RUNNING);
316 }
317 if (likely(res > 0)) {
318 ctx->count += ucnt;
319 if (waitqueue_active(&ctx->wqh))
320 wake_up_locked_poll(&ctx->wqh, POLLIN);
321 }
322 spin_unlock_irq(&ctx->wqh.lock);
323
324 return res;
325}
326
327#ifdef CONFIG_PROC_FS
328static void eventfd_show_fdinfo(struct seq_file *m, struct file *f)
329{
330 struct eventfd_ctx *ctx = f->private_data;
331
332 spin_lock_irq(&ctx->wqh.lock);
333 seq_printf(m, "eventfd-count: %16llx\n",
334 (unsigned long long)ctx->count);
335 spin_unlock_irq(&ctx->wqh.lock);
336}
337#endif
338
339static const struct file_operations eventfd_fops = {
340#ifdef CONFIG_PROC_FS
341 .show_fdinfo = eventfd_show_fdinfo,
342#endif
343 .release = eventfd_release,
344 .poll = eventfd_poll,
345 .read = eventfd_read,
346 .write = eventfd_write,
347 .llseek = noop_llseek,
348};
349
350
351
352
353
354
355
356
357
358
359
360struct file *eventfd_fget(int fd)
361{
362 struct file *file;
363
364 file = fget(fd);
365 if (!file)
366 return ERR_PTR(-EBADF);
367 if (file->f_op != &eventfd_fops) {
368 fput(file);
369 return ERR_PTR(-EINVAL);
370 }
371
372 return file;
373}
374EXPORT_SYMBOL_GPL(eventfd_fget);
375
376
377
378
379
380
381
382
383
384
385struct eventfd_ctx *eventfd_ctx_fdget(int fd)
386{
387 struct eventfd_ctx *ctx;
388 struct fd f = fdget(fd);
389 if (!f.file)
390 return ERR_PTR(-EBADF);
391 ctx = eventfd_ctx_fileget(f.file);
392 fdput(f);
393 return ctx;
394}
395EXPORT_SYMBOL_GPL(eventfd_ctx_fdget);
396
397
398
399
400
401
402
403
404
405
406struct eventfd_ctx *eventfd_ctx_fileget(struct file *file)
407{
408 if (file->f_op != &eventfd_fops)
409 return ERR_PTR(-EINVAL);
410
411 return eventfd_ctx_get(file->private_data);
412}
413EXPORT_SYMBOL_GPL(eventfd_ctx_fileget);
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429struct file *eventfd_file_create(unsigned int count, int flags)
430{
431 struct file *file;
432 struct eventfd_ctx *ctx;
433
434
435 BUILD_BUG_ON(EFD_CLOEXEC != O_CLOEXEC);
436 BUILD_BUG_ON(EFD_NONBLOCK != O_NONBLOCK);
437
438 if (flags & ~EFD_FLAGS_SET)
439 return ERR_PTR(-EINVAL);
440
441 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
442 if (!ctx)
443 return ERR_PTR(-ENOMEM);
444
445 kref_init(&ctx->kref);
446 init_waitqueue_head(&ctx->wqh);
447 ctx->count = count;
448 ctx->flags = flags;
449
450 file = anon_inode_getfile("[eventfd]", &eventfd_fops, ctx,
451 O_RDWR | (flags & EFD_SHARED_FCNTL_FLAGS));
452 if (IS_ERR(file))
453 eventfd_free_ctx(ctx);
454
455 return file;
456}
457
458SYSCALL_DEFINE2(eventfd2, unsigned int, count, int, flags)
459{
460 int fd, error;
461 struct file *file;
462
463 error = get_unused_fd_flags(flags & EFD_SHARED_FCNTL_FLAGS);
464 if (error < 0)
465 return error;
466 fd = error;
467
468 file = eventfd_file_create(count, flags);
469 if (IS_ERR(file)) {
470 error = PTR_ERR(file);
471 goto err_put_unused_fd;
472 }
473 fd_install(fd, file);
474
475 return fd;
476
477err_put_unused_fd:
478 put_unused_fd(fd);
479
480 return error;
481}
482
483SYSCALL_DEFINE1(eventfd, unsigned int, count)
484{
485 return sys_eventfd2(count, 0);
486}
487
488