1
2
3
4
5
6
7
8
9
10
11#include <linux/kernel.h>
12#include <linux/init.h>
13#include <linux/errno.h>
14#include <linux/time.h>
15#include <linux/aio_abi.h>
16#include <linux/module.h>
17#include <linux/syscalls.h>
18#include <linux/uio.h>
19
20#define DEBUG 0
21
22#include <linux/sched.h>
23#include <linux/fs.h>
24#include <linux/file.h>
25#include <linux/mm.h>
26#include <linux/mman.h>
27#include <linux/mmu_context.h>
28#include <linux/slab.h>
29#include <linux/timer.h>
30#include <linux/aio.h>
31#include <linux/highmem.h>
32#include <linux/workqueue.h>
33#include <linux/security.h>
34#include <linux/eventfd.h>
35
36#include <asm/kmap_types.h>
37#include <asm/uaccess.h>
38
39#if DEBUG > 1
40#define dprintk printk
41#else
42#define dprintk(x...) do { ; } while (0)
43#endif
44
45
46static DEFINE_SPINLOCK(aio_nr_lock);
47unsigned long aio_nr;
48unsigned long aio_max_nr = 0x10000;
49
50
51static struct kmem_cache *kiocb_cachep;
52static struct kmem_cache *kioctx_cachep;
53
54static struct workqueue_struct *aio_wq;
55
56
57static void aio_fput_routine(struct work_struct *);
58static DECLARE_WORK(fput_work, aio_fput_routine);
59
60static DEFINE_SPINLOCK(fput_lock);
61static LIST_HEAD(fput_head);
62
63static void aio_kick_handler(struct work_struct *);
64static void aio_queue_work(struct kioctx *);
65
66
67
68
69
70static int __init aio_setup(void)
71{
72 kiocb_cachep = KMEM_CACHE(kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC);
73 kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC);
74
75 aio_wq = create_workqueue("aio");
76
77 pr_debug("aio_setup: sizeof(struct page) = %d\n", (int)sizeof(struct page));
78
79 return 0;
80}
81__initcall(aio_setup);
82
83static void aio_free_ring(struct kioctx *ctx)
84{
85 struct aio_ring_info *info = &ctx->ring_info;
86 long i;
87
88 for (i=0; i<info->nr_pages; i++)
89 put_page(info->ring_pages[i]);
90
91 if (info->mmap_size) {
92 down_write(&ctx->mm->mmap_sem);
93 do_munmap(ctx->mm, info->mmap_base, info->mmap_size);
94 up_write(&ctx->mm->mmap_sem);
95 }
96
97 if (info->ring_pages && info->ring_pages != info->internal_pages)
98 kfree(info->ring_pages);
99 info->ring_pages = NULL;
100 info->nr = 0;
101}
102
103static int aio_setup_ring(struct kioctx *ctx)
104{
105 struct aio_ring *ring;
106 struct aio_ring_info *info = &ctx->ring_info;
107 unsigned nr_events = ctx->max_reqs;
108 unsigned long size;
109 int nr_pages;
110
111
112 nr_events += 2;
113
114 size = sizeof(struct aio_ring);
115 size += sizeof(struct io_event) * nr_events;
116 nr_pages = (size + PAGE_SIZE-1) >> PAGE_SHIFT;
117
118 if (nr_pages < 0)
119 return -EINVAL;
120
121 nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring)) / sizeof(struct io_event);
122
123 info->nr = 0;
124 info->ring_pages = info->internal_pages;
125 if (nr_pages > AIO_RING_PAGES) {
126 info->ring_pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
127 if (!info->ring_pages)
128 return -ENOMEM;
129 }
130
131 info->mmap_size = nr_pages * PAGE_SIZE;
132 dprintk("attempting mmap of %lu bytes\n", info->mmap_size);
133 down_write(&ctx->mm->mmap_sem);
134 info->mmap_base = do_mmap(NULL, 0, info->mmap_size,
135 PROT_READ|PROT_WRITE, MAP_ANONYMOUS|MAP_PRIVATE,
136 0);
137 if (IS_ERR((void *)info->mmap_base)) {
138 up_write(&ctx->mm->mmap_sem);
139 info->mmap_size = 0;
140 aio_free_ring(ctx);
141 return -EAGAIN;
142 }
143
144 dprintk("mmap address: 0x%08lx\n", info->mmap_base);
145 info->nr_pages = get_user_pages(current, ctx->mm,
146 info->mmap_base, nr_pages,
147 1, 0, info->ring_pages, NULL);
148 up_write(&ctx->mm->mmap_sem);
149
150 if (unlikely(info->nr_pages != nr_pages)) {
151 aio_free_ring(ctx);
152 return -EAGAIN;
153 }
154
155 ctx->user_id = info->mmap_base;
156
157 info->nr = nr_events;
158
159 ring = kmap_atomic(info->ring_pages[0], KM_USER0);
160 ring->nr = nr_events;
161 ring->id = ctx->user_id;
162 ring->head = ring->tail = 0;
163 ring->magic = AIO_RING_MAGIC;
164 ring->compat_features = AIO_RING_COMPAT_FEATURES;
165 ring->incompat_features = AIO_RING_INCOMPAT_FEATURES;
166 ring->header_length = sizeof(struct aio_ring);
167 kunmap_atomic(ring, KM_USER0);
168
169 return 0;
170}
171
172
173
174
175
176#define AIO_EVENTS_PER_PAGE (PAGE_SIZE / sizeof(struct io_event))
177#define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event))
178#define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE)
179
180#define aio_ring_event(info, nr, km) ({ \
181 unsigned pos = (nr) + AIO_EVENTS_OFFSET; \
182 struct io_event *__event; \
183 __event = kmap_atomic( \
184 (info)->ring_pages[pos / AIO_EVENTS_PER_PAGE], km); \
185 __event += pos % AIO_EVENTS_PER_PAGE; \
186 __event; \
187})
188
189#define put_aio_ring_event(event, km) do { \
190 struct io_event *__event = (event); \
191 (void)__event; \
192 kunmap_atomic((void *)((unsigned long)__event & PAGE_MASK), km); \
193} while(0)
194
195static void ctx_rcu_free(struct rcu_head *head)
196{
197 struct kioctx *ctx = container_of(head, struct kioctx, rcu_head);
198 unsigned nr_events = ctx->max_reqs;
199
200 kmem_cache_free(kioctx_cachep, ctx);
201
202 if (nr_events) {
203 spin_lock(&aio_nr_lock);
204 BUG_ON(aio_nr - nr_events > aio_nr);
205 aio_nr -= nr_events;
206 spin_unlock(&aio_nr_lock);
207 }
208}
209
210
211
212
213
214static void __put_ioctx(struct kioctx *ctx)
215{
216 BUG_ON(ctx->reqs_active);
217
218 cancel_delayed_work(&ctx->wq);
219 cancel_work_sync(&ctx->wq.work);
220 aio_free_ring(ctx);
221 mmdrop(ctx->mm);
222 ctx->mm = NULL;
223 pr_debug("__put_ioctx: freeing %p\n", ctx);
224 call_rcu(&ctx->rcu_head, ctx_rcu_free);
225}
226
227#define get_ioctx(kioctx) do { \
228 BUG_ON(atomic_read(&(kioctx)->users) <= 0); \
229 atomic_inc(&(kioctx)->users); \
230} while (0)
231#define put_ioctx(kioctx) do { \
232 BUG_ON(atomic_read(&(kioctx)->users) <= 0); \
233 if (unlikely(atomic_dec_and_test(&(kioctx)->users))) \
234 __put_ioctx(kioctx); \
235} while (0)
236
237
238
239
240static struct kioctx *ioctx_alloc(unsigned nr_events)
241{
242 struct mm_struct *mm;
243 struct kioctx *ctx;
244 int did_sync = 0;
245
246
247 if ((nr_events > (0x10000000U / sizeof(struct io_event))) ||
248 (nr_events > (0x10000000U / sizeof(struct kiocb)))) {
249 pr_debug("ENOMEM: nr_events too high\n");
250 return ERR_PTR(-EINVAL);
251 }
252
253 if ((unsigned long)nr_events > aio_max_nr)
254 return ERR_PTR(-EAGAIN);
255
256 ctx = kmem_cache_zalloc(kioctx_cachep, GFP_KERNEL);
257 if (!ctx)
258 return ERR_PTR(-ENOMEM);
259
260 ctx->max_reqs = nr_events;
261 mm = ctx->mm = current->mm;
262 atomic_inc(&mm->mm_count);
263
264 atomic_set(&ctx->users, 1);
265 spin_lock_init(&ctx->ctx_lock);
266 spin_lock_init(&ctx->ring_info.ring_lock);
267 init_waitqueue_head(&ctx->wait);
268
269 INIT_LIST_HEAD(&ctx->active_reqs);
270 INIT_LIST_HEAD(&ctx->run_list);
271 INIT_DELAYED_WORK(&ctx->wq, aio_kick_handler);
272
273 if (aio_setup_ring(ctx) < 0)
274 goto out_freectx;
275
276
277 do {
278 spin_lock_bh(&aio_nr_lock);
279 if (aio_nr + nr_events > aio_max_nr ||
280 aio_nr + nr_events < aio_nr)
281 ctx->max_reqs = 0;
282 else
283 aio_nr += ctx->max_reqs;
284 spin_unlock_bh(&aio_nr_lock);
285 if (ctx->max_reqs || did_sync)
286 break;
287
288
289 synchronize_rcu();
290 did_sync = 1;
291 ctx->max_reqs = nr_events;
292 } while (1);
293
294 if (ctx->max_reqs == 0)
295 goto out_cleanup;
296
297
298 spin_lock(&mm->ioctx_lock);
299 hlist_add_head_rcu(&ctx->list, &mm->ioctx_list);
300 spin_unlock(&mm->ioctx_lock);
301
302 dprintk("aio: allocated ioctx %p[%ld]: mm=%p mask=0x%x\n",
303 ctx, ctx->user_id, current->mm, ctx->ring_info.nr);
304 return ctx;
305
306out_cleanup:
307 __put_ioctx(ctx);
308 return ERR_PTR(-EAGAIN);
309
310out_freectx:
311 mmdrop(mm);
312 kmem_cache_free(kioctx_cachep, ctx);
313 ctx = ERR_PTR(-ENOMEM);
314
315 dprintk("aio: error allocating ioctx %p\n", ctx);
316 return ctx;
317}
318
319
320
321
322
323
324static void aio_cancel_all(struct kioctx *ctx)
325{
326 int (*cancel)(struct kiocb *, struct io_event *);
327 struct io_event res;
328 spin_lock_irq(&ctx->ctx_lock);
329 ctx->dead = 1;
330 while (!list_empty(&ctx->active_reqs)) {
331 struct list_head *pos = ctx->active_reqs.next;
332 struct kiocb *iocb = list_kiocb(pos);
333 list_del_init(&iocb->ki_list);
334 cancel = iocb->ki_cancel;
335 kiocbSetCancelled(iocb);
336 if (cancel) {
337 iocb->ki_users++;
338 spin_unlock_irq(&ctx->ctx_lock);
339 cancel(iocb, &res);
340 spin_lock_irq(&ctx->ctx_lock);
341 }
342 }
343 spin_unlock_irq(&ctx->ctx_lock);
344}
345
346static void wait_for_all_aios(struct kioctx *ctx)
347{
348 struct task_struct *tsk = current;
349 DECLARE_WAITQUEUE(wait, tsk);
350
351 spin_lock_irq(&ctx->ctx_lock);
352 if (!ctx->reqs_active)
353 goto out;
354
355 add_wait_queue(&ctx->wait, &wait);
356 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
357 while (ctx->reqs_active) {
358 spin_unlock_irq(&ctx->ctx_lock);
359 io_schedule();
360 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
361 spin_lock_irq(&ctx->ctx_lock);
362 }
363 __set_task_state(tsk, TASK_RUNNING);
364 remove_wait_queue(&ctx->wait, &wait);
365
366out:
367 spin_unlock_irq(&ctx->ctx_lock);
368}
369
370
371
372
373ssize_t wait_on_sync_kiocb(struct kiocb *iocb)
374{
375 while (iocb->ki_users) {
376 set_current_state(TASK_UNINTERRUPTIBLE);
377 if (!iocb->ki_users)
378 break;
379 io_schedule();
380 }
381 __set_current_state(TASK_RUNNING);
382 return iocb->ki_user_data;
383}
384EXPORT_SYMBOL(wait_on_sync_kiocb);
385
386
387
388
389
390
391
392
393void exit_aio(struct mm_struct *mm)
394{
395 struct kioctx *ctx;
396
397 while (!hlist_empty(&mm->ioctx_list)) {
398 ctx = hlist_entry(mm->ioctx_list.first, struct kioctx, list);
399 hlist_del_rcu(&ctx->list);
400
401 aio_cancel_all(ctx);
402
403 wait_for_all_aios(ctx);
404
405
406
407 cancel_work_sync(&ctx->wq.work);
408
409 if (1 != atomic_read(&ctx->users))
410 printk(KERN_DEBUG
411 "exit_aio:ioctx still alive: %d %d %d\n",
412 atomic_read(&ctx->users), ctx->dead,
413 ctx->reqs_active);
414 put_ioctx(ctx);
415 }
416}
417
418
419
420
421
422
423
424
425
426
427
428static struct kiocb *__aio_get_req(struct kioctx *ctx)
429{
430 struct kiocb *req = NULL;
431 struct aio_ring *ring;
432 int okay = 0;
433
434 req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL);
435 if (unlikely(!req))
436 return NULL;
437
438 req->ki_flags = 0;
439 req->ki_users = 2;
440 req->ki_key = 0;
441 req->ki_ctx = ctx;
442 req->ki_cancel = NULL;
443 req->ki_retry = NULL;
444 req->ki_dtor = NULL;
445 req->private = NULL;
446 req->ki_iovec = NULL;
447 INIT_LIST_HEAD(&req->ki_run_list);
448 req->ki_eventfd = NULL;
449
450
451
452
453 spin_lock_irq(&ctx->ctx_lock);
454 ring = kmap_atomic(ctx->ring_info.ring_pages[0], KM_USER0);
455 if (ctx->reqs_active < aio_ring_avail(&ctx->ring_info, ring)) {
456 list_add(&req->ki_list, &ctx->active_reqs);
457 ctx->reqs_active++;
458 okay = 1;
459 }
460 kunmap_atomic(ring, KM_USER0);
461 spin_unlock_irq(&ctx->ctx_lock);
462
463 if (!okay) {
464 kmem_cache_free(kiocb_cachep, req);
465 req = NULL;
466 }
467
468 return req;
469}
470
471static inline struct kiocb *aio_get_req(struct kioctx *ctx)
472{
473 struct kiocb *req;
474
475
476
477
478 req = __aio_get_req(ctx);
479 if (unlikely(NULL == req)) {
480 aio_fput_routine(NULL);
481 req = __aio_get_req(ctx);
482 }
483 return req;
484}
485
486static inline void really_put_req(struct kioctx *ctx, struct kiocb *req)
487{
488 assert_spin_locked(&ctx->ctx_lock);
489
490 if (req->ki_eventfd != NULL)
491 eventfd_ctx_put(req->ki_eventfd);
492 if (req->ki_dtor)
493 req->ki_dtor(req);
494 if (req->ki_iovec != &req->ki_inline_vec)
495 kfree(req->ki_iovec);
496 kmem_cache_free(kiocb_cachep, req);
497 ctx->reqs_active--;
498
499 if (unlikely(!ctx->reqs_active && ctx->dead))
500 wake_up(&ctx->wait);
501}
502
503static void aio_fput_routine(struct work_struct *data)
504{
505 spin_lock_irq(&fput_lock);
506 while (likely(!list_empty(&fput_head))) {
507 struct kiocb *req = list_kiocb(fput_head.next);
508 struct kioctx *ctx = req->ki_ctx;
509
510 list_del(&req->ki_list);
511 spin_unlock_irq(&fput_lock);
512
513
514 if (req->ki_filp != NULL)
515 __fput(req->ki_filp);
516
517
518 spin_lock_irq(&ctx->ctx_lock);
519 really_put_req(ctx, req);
520 spin_unlock_irq(&ctx->ctx_lock);
521
522 put_ioctx(ctx);
523 spin_lock_irq(&fput_lock);
524 }
525 spin_unlock_irq(&fput_lock);
526}
527
528
529
530
531static int __aio_put_req(struct kioctx *ctx, struct kiocb *req)
532{
533 dprintk(KERN_DEBUG "aio_put(%p): f_count=%ld\n",
534 req, atomic_long_read(&req->ki_filp->f_count));
535
536 assert_spin_locked(&ctx->ctx_lock);
537
538 req->ki_users--;
539 BUG_ON(req->ki_users < 0);
540 if (likely(req->ki_users))
541 return 0;
542 list_del(&req->ki_list);
543 req->ki_cancel = NULL;
544 req->ki_retry = NULL;
545
546
547
548
549
550
551
552 if (unlikely(atomic_long_dec_and_test(&req->ki_filp->f_count))) {
553 get_ioctx(ctx);
554 spin_lock(&fput_lock);
555 list_add(&req->ki_list, &fput_head);
556 spin_unlock(&fput_lock);
557 queue_work(aio_wq, &fput_work);
558 } else {
559 req->ki_filp = NULL;
560 really_put_req(ctx, req);
561 }
562 return 1;
563}
564
565
566
567
568
569int aio_put_req(struct kiocb *req)
570{
571 struct kioctx *ctx = req->ki_ctx;
572 int ret;
573 spin_lock_irq(&ctx->ctx_lock);
574 ret = __aio_put_req(ctx, req);
575 spin_unlock_irq(&ctx->ctx_lock);
576 return ret;
577}
578EXPORT_SYMBOL(aio_put_req);
579
580static struct kioctx *lookup_ioctx(unsigned long ctx_id)
581{
582 struct mm_struct *mm = current->mm;
583 struct kioctx *ctx, *ret = NULL;
584 struct hlist_node *n;
585
586 rcu_read_lock();
587
588 hlist_for_each_entry_rcu(ctx, n, &mm->ioctx_list, list) {
589 if (ctx->user_id == ctx_id && !ctx->dead) {
590 get_ioctx(ctx);
591 ret = ctx;
592 break;
593 }
594 }
595
596 rcu_read_unlock();
597 return ret;
598}
599
600
601
602
603
604
605
606
607
608
609static inline int __queue_kicked_iocb(struct kiocb *iocb)
610{
611 struct kioctx *ctx = iocb->ki_ctx;
612
613 assert_spin_locked(&ctx->ctx_lock);
614
615 if (list_empty(&iocb->ki_run_list)) {
616 list_add_tail(&iocb->ki_run_list,
617 &ctx->run_list);
618 return 1;
619 }
620 return 0;
621}
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645static ssize_t aio_run_iocb(struct kiocb *iocb)
646{
647 struct kioctx *ctx = iocb->ki_ctx;
648 ssize_t (*retry)(struct kiocb *);
649 ssize_t ret;
650
651 if (!(retry = iocb->ki_retry)) {
652 printk("aio_run_iocb: iocb->ki_retry = NULL\n");
653 return 0;
654 }
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675 kiocbClearKicked(iocb);
676
677
678
679
680
681
682
683 iocb->ki_run_list.next = iocb->ki_run_list.prev = NULL;
684 spin_unlock_irq(&ctx->ctx_lock);
685
686
687 if (kiocbIsCancelled(iocb)) {
688 ret = -EINTR;
689 aio_complete(iocb, ret, 0);
690
691 goto out;
692 }
693
694
695
696
697
698 ret = retry(iocb);
699
700 if (ret != -EIOCBRETRY && ret != -EIOCBQUEUED) {
701 BUG_ON(!list_empty(&iocb->ki_wait.task_list));
702 aio_complete(iocb, ret, 0);
703 }
704out:
705 spin_lock_irq(&ctx->ctx_lock);
706
707 if (-EIOCBRETRY == ret) {
708
709
710
711
712
713
714
715
716 INIT_LIST_HEAD(&iocb->ki_run_list);
717
718
719 if (kiocbIsKicked(iocb)) {
720 __queue_kicked_iocb(iocb);
721
722
723
724
725
726
727
728 aio_queue_work(ctx);
729 }
730 }
731 return ret;
732}
733
734
735
736
737
738
739
740
741static int __aio_run_iocbs(struct kioctx *ctx)
742{
743 struct kiocb *iocb;
744 struct list_head run_list;
745
746 assert_spin_locked(&ctx->ctx_lock);
747
748 list_replace_init(&ctx->run_list, &run_list);
749 while (!list_empty(&run_list)) {
750 iocb = list_entry(run_list.next, struct kiocb,
751 ki_run_list);
752 list_del(&iocb->ki_run_list);
753
754
755
756 iocb->ki_users++;
757 aio_run_iocb(iocb);
758 __aio_put_req(ctx, iocb);
759 }
760 if (!list_empty(&ctx->run_list))
761 return 1;
762 return 0;
763}
764
765static void aio_queue_work(struct kioctx * ctx)
766{
767 unsigned long timeout;
768
769
770
771
772 smp_mb();
773 if (waitqueue_active(&ctx->wait))
774 timeout = 1;
775 else
776 timeout = HZ/10;
777 queue_delayed_work(aio_wq, &ctx->wq, timeout);
778}
779
780
781
782
783
784
785
786
787
788static inline void aio_run_iocbs(struct kioctx *ctx)
789{
790 int requeue;
791
792 spin_lock_irq(&ctx->ctx_lock);
793
794 requeue = __aio_run_iocbs(ctx);
795 spin_unlock_irq(&ctx->ctx_lock);
796 if (requeue)
797 aio_queue_work(ctx);
798}
799
800
801
802
803
804static inline void aio_run_all_iocbs(struct kioctx *ctx)
805{
806 spin_lock_irq(&ctx->ctx_lock);
807 while (__aio_run_iocbs(ctx))
808 ;
809 spin_unlock_irq(&ctx->ctx_lock);
810}
811
812
813
814
815
816
817
818
819
820
821static void aio_kick_handler(struct work_struct *work)
822{
823 struct kioctx *ctx = container_of(work, struct kioctx, wq.work);
824 mm_segment_t oldfs = get_fs();
825 struct mm_struct *mm;
826 int requeue;
827
828 set_fs(USER_DS);
829 use_mm(ctx->mm);
830 spin_lock_irq(&ctx->ctx_lock);
831 requeue =__aio_run_iocbs(ctx);
832 mm = ctx->mm;
833 spin_unlock_irq(&ctx->ctx_lock);
834 unuse_mm(mm);
835 set_fs(oldfs);
836
837
838
839 if (requeue)
840 queue_delayed_work(aio_wq, &ctx->wq, 0);
841}
842
843
844
845
846
847
848
849static void try_queue_kicked_iocb(struct kiocb *iocb)
850{
851 struct kioctx *ctx = iocb->ki_ctx;
852 unsigned long flags;
853 int run = 0;
854
855
856
857
858
859
860 BUG_ON((!list_empty(&iocb->ki_wait.task_list)));
861
862 spin_lock_irqsave(&ctx->ctx_lock, flags);
863
864
865 if (!kiocbTryKick(iocb))
866 run = __queue_kicked_iocb(iocb);
867 spin_unlock_irqrestore(&ctx->ctx_lock, flags);
868 if (run)
869 aio_queue_work(ctx);
870}
871
872
873
874
875
876
877
878
879void kick_iocb(struct kiocb *iocb)
880{
881
882
883 if (is_sync_kiocb(iocb)) {
884 kiocbSetKicked(iocb);
885 wake_up_process(iocb->ki_obj.tsk);
886 return;
887 }
888
889 try_queue_kicked_iocb(iocb);
890}
891EXPORT_SYMBOL(kick_iocb);
892
893
894
895
896
897
898int aio_complete(struct kiocb *iocb, long res, long res2)
899{
900 struct kioctx *ctx = iocb->ki_ctx;
901 struct aio_ring_info *info;
902 struct aio_ring *ring;
903 struct io_event *event;
904 unsigned long flags;
905 unsigned long tail;
906 int ret;
907
908
909
910
911
912
913
914
915 if (is_sync_kiocb(iocb)) {
916 BUG_ON(iocb->ki_users != 1);
917 iocb->ki_user_data = res;
918 iocb->ki_users = 0;
919 wake_up_process(iocb->ki_obj.tsk);
920 return 1;
921 }
922
923 info = &ctx->ring_info;
924
925
926
927
928
929
930
931 spin_lock_irqsave(&ctx->ctx_lock, flags);
932
933 if (iocb->ki_run_list.prev && !list_empty(&iocb->ki_run_list))
934 list_del_init(&iocb->ki_run_list);
935
936
937
938
939
940 if (kiocbIsCancelled(iocb))
941 goto put_rq;
942
943 ring = kmap_atomic(info->ring_pages[0], KM_IRQ1);
944
945 tail = info->tail;
946 event = aio_ring_event(info, tail, KM_IRQ0);
947 if (++tail >= info->nr)
948 tail = 0;
949
950 event->obj = (u64)(unsigned long)iocb->ki_obj.user;
951 event->data = iocb->ki_user_data;
952 event->res = res;
953 event->res2 = res2;
954
955 dprintk("aio_complete: %p[%lu]: %p: %p %Lx %lx %lx\n",
956 ctx, tail, iocb, iocb->ki_obj.user, iocb->ki_user_data,
957 res, res2);
958
959
960
961
962 smp_wmb();
963
964 info->tail = tail;
965 ring->tail = tail;
966
967 put_aio_ring_event(event, KM_IRQ0);
968 kunmap_atomic(ring, KM_IRQ1);
969
970 pr_debug("added to ring %p at [%lu]\n", iocb, tail);
971
972
973
974
975
976
977 if (iocb->ki_eventfd != NULL)
978 eventfd_signal(iocb->ki_eventfd, 1);
979
980put_rq:
981
982 ret = __aio_put_req(ctx, iocb);
983
984
985
986
987
988
989
990 smp_mb();
991
992 if (waitqueue_active(&ctx->wait))
993 wake_up(&ctx->wait);
994
995 spin_unlock_irqrestore(&ctx->ctx_lock, flags);
996 return ret;
997}
998EXPORT_SYMBOL(aio_complete);
999
1000
1001
1002
1003
1004
1005
1006static int aio_read_evt(struct kioctx *ioctx, struct io_event *ent)
1007{
1008 struct aio_ring_info *info = &ioctx->ring_info;
1009 struct aio_ring *ring;
1010 unsigned long head;
1011 int ret = 0;
1012
1013 ring = kmap_atomic(info->ring_pages[0], KM_USER0);
1014 dprintk("in aio_read_evt h%lu t%lu m%lu\n",
1015 (unsigned long)ring->head, (unsigned long)ring->tail,
1016 (unsigned long)ring->nr);
1017
1018 if (ring->head == ring->tail)
1019 goto out;
1020
1021 spin_lock(&info->ring_lock);
1022
1023 head = ring->head % info->nr;
1024 if (head != ring->tail) {
1025 struct io_event *evp = aio_ring_event(info, head, KM_USER1);
1026 *ent = *evp;
1027 head = (head + 1) % info->nr;
1028 smp_mb();
1029 ring->head = head;
1030 ret = 1;
1031 put_aio_ring_event(evp, KM_USER1);
1032 }
1033 spin_unlock(&info->ring_lock);
1034
1035out:
1036 kunmap_atomic(ring, KM_USER0);
1037 dprintk("leaving aio_read_evt: %d h%lu t%lu\n", ret,
1038 (unsigned long)ring->head, (unsigned long)ring->tail);
1039 return ret;
1040}
1041
1042struct aio_timeout {
1043 struct timer_list timer;
1044 int timed_out;
1045 struct task_struct *p;
1046};
1047
1048static void timeout_func(unsigned long data)
1049{
1050 struct aio_timeout *to = (struct aio_timeout *)data;
1051
1052 to->timed_out = 1;
1053 wake_up_process(to->p);
1054}
1055
1056static inline void init_timeout(struct aio_timeout *to)
1057{
1058 setup_timer_on_stack(&to->timer, timeout_func, (unsigned long) to);
1059 to->timed_out = 0;
1060 to->p = current;
1061}
1062
1063static inline void set_timeout(long start_jiffies, struct aio_timeout *to,
1064 const struct timespec *ts)
1065{
1066 to->timer.expires = start_jiffies + timespec_to_jiffies(ts);
1067 if (time_after(to->timer.expires, jiffies))
1068 add_timer(&to->timer);
1069 else
1070 to->timed_out = 1;
1071}
1072
1073static inline void clear_timeout(struct aio_timeout *to)
1074{
1075 del_singleshot_timer_sync(&to->timer);
1076}
1077
1078static int read_events(struct kioctx *ctx,
1079 long min_nr, long nr,
1080 struct io_event __user *event,
1081 struct timespec __user *timeout)
1082{
1083 long start_jiffies = jiffies;
1084 struct task_struct *tsk = current;
1085 DECLARE_WAITQUEUE(wait, tsk);
1086 int ret;
1087 int i = 0;
1088 struct io_event ent;
1089 struct aio_timeout to;
1090 int retry = 0;
1091
1092
1093
1094
1095 memset(&ent, 0, sizeof(ent));
1096retry:
1097 ret = 0;
1098 while (likely(i < nr)) {
1099 ret = aio_read_evt(ctx, &ent);
1100 if (unlikely(ret <= 0))
1101 break;
1102
1103 dprintk("read event: %Lx %Lx %Lx %Lx\n",
1104 ent.data, ent.obj, ent.res, ent.res2);
1105
1106
1107 ret = -EFAULT;
1108 if (unlikely(copy_to_user(event, &ent, sizeof(ent)))) {
1109 dprintk("aio: lost an event due to EFAULT.\n");
1110 break;
1111 }
1112 ret = 0;
1113
1114
1115 event ++;
1116 i ++;
1117 }
1118
1119 if (min_nr <= i)
1120 return i;
1121 if (ret)
1122 return ret;
1123
1124
1125
1126
1127 if (!retry && unlikely(!list_empty(&ctx->run_list))) {
1128 retry = 1;
1129 aio_run_all_iocbs(ctx);
1130 goto retry;
1131 }
1132
1133 init_timeout(&to);
1134 if (timeout) {
1135 struct timespec ts;
1136 ret = -EFAULT;
1137 if (unlikely(copy_from_user(&ts, timeout, sizeof(ts))))
1138 goto out;
1139
1140 set_timeout(start_jiffies, &to, &ts);
1141 }
1142
1143 while (likely(i < nr)) {
1144 add_wait_queue_exclusive(&ctx->wait, &wait);
1145 do {
1146 set_task_state(tsk, TASK_INTERRUPTIBLE);
1147 ret = aio_read_evt(ctx, &ent);
1148 if (ret)
1149 break;
1150 if (min_nr <= i)
1151 break;
1152 if (unlikely(ctx->dead)) {
1153 ret = -EINVAL;
1154 break;
1155 }
1156 if (to.timed_out)
1157 break;
1158
1159
1160 if (ctx->reqs_active)
1161 io_schedule();
1162 else
1163 schedule();
1164 if (signal_pending(tsk)) {
1165 ret = -EINTR;
1166 break;
1167 }
1168
1169 } while (1) ;
1170
1171 set_task_state(tsk, TASK_RUNNING);
1172 remove_wait_queue(&ctx->wait, &wait);
1173
1174 if (unlikely(ret <= 0))
1175 break;
1176
1177 ret = -EFAULT;
1178 if (unlikely(copy_to_user(event, &ent, sizeof(ent)))) {
1179 dprintk("aio: lost an event due to EFAULT.\n");
1180 break;
1181 }
1182
1183
1184 event ++;
1185 i ++;
1186 }
1187
1188 if (timeout)
1189 clear_timeout(&to);
1190out:
1191 destroy_timer_on_stack(&to.timer);
1192 return i ? i : ret;
1193}
1194
1195
1196
1197
1198static void io_destroy(struct kioctx *ioctx)
1199{
1200 struct mm_struct *mm = current->mm;
1201 int was_dead;
1202
1203
1204 spin_lock(&mm->ioctx_lock);
1205 was_dead = ioctx->dead;
1206 ioctx->dead = 1;
1207 hlist_del_rcu(&ioctx->list);
1208 spin_unlock(&mm->ioctx_lock);
1209
1210 dprintk("aio_release(%p)\n", ioctx);
1211 if (likely(!was_dead))
1212 put_ioctx(ioctx);
1213
1214 aio_cancel_all(ioctx);
1215 wait_for_all_aios(ioctx);
1216
1217
1218
1219
1220
1221
1222 wake_up(&ioctx->wait);
1223 put_ioctx(ioctx);
1224}
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239SYSCALL_DEFINE2(io_setup, unsigned, nr_events, aio_context_t __user *, ctxp)
1240{
1241 struct kioctx *ioctx = NULL;
1242 unsigned long ctx;
1243 long ret;
1244
1245 ret = get_user(ctx, ctxp);
1246 if (unlikely(ret))
1247 goto out;
1248
1249 ret = -EINVAL;
1250 if (unlikely(ctx || nr_events == 0)) {
1251 pr_debug("EINVAL: io_setup: ctx %lu nr_events %u\n",
1252 ctx, nr_events);
1253 goto out;
1254 }
1255
1256 ioctx = ioctx_alloc(nr_events);
1257 ret = PTR_ERR(ioctx);
1258 if (!IS_ERR(ioctx)) {
1259 ret = put_user(ioctx->user_id, ctxp);
1260 if (!ret)
1261 return 0;
1262
1263 get_ioctx(ioctx);
1264 io_destroy(ioctx);
1265 }
1266
1267out:
1268 return ret;
1269}
1270
1271
1272
1273
1274
1275
1276
1277SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx)
1278{
1279 struct kioctx *ioctx = lookup_ioctx(ctx);
1280 if (likely(NULL != ioctx)) {
1281 io_destroy(ioctx);
1282 return 0;
1283 }
1284 pr_debug("EINVAL: io_destroy: invalid context id\n");
1285 return -EINVAL;
1286}
1287
1288static void aio_advance_iovec(struct kiocb *iocb, ssize_t ret)
1289{
1290 struct iovec *iov = &iocb->ki_iovec[iocb->ki_cur_seg];
1291
1292 BUG_ON(ret <= 0);
1293
1294 while (iocb->ki_cur_seg < iocb->ki_nr_segs && ret > 0) {
1295 ssize_t this = min((ssize_t)iov->iov_len, ret);
1296 iov->iov_base += this;
1297 iov->iov_len -= this;
1298 iocb->ki_left -= this;
1299 ret -= this;
1300 if (iov->iov_len == 0) {
1301 iocb->ki_cur_seg++;
1302 iov++;
1303 }
1304 }
1305
1306
1307
1308 BUG_ON(ret > 0 && iocb->ki_left == 0);
1309}
1310
1311static ssize_t aio_rw_vect_retry(struct kiocb *iocb)
1312{
1313 struct file *file = iocb->ki_filp;
1314 struct address_space *mapping = file->f_mapping;
1315 struct inode *inode = mapping->host;
1316 ssize_t (*rw_op)(struct kiocb *, const struct iovec *,
1317 unsigned long, loff_t);
1318 ssize_t ret = 0;
1319 unsigned short opcode;
1320
1321 if ((iocb->ki_opcode == IOCB_CMD_PREADV) ||
1322 (iocb->ki_opcode == IOCB_CMD_PREAD)) {
1323 rw_op = file->f_op->aio_read;
1324 opcode = IOCB_CMD_PREADV;
1325 } else {
1326 rw_op = file->f_op->aio_write;
1327 opcode = IOCB_CMD_PWRITEV;
1328 }
1329
1330
1331 if (iocb->ki_pos < 0)
1332 return -EINVAL;
1333
1334 do {
1335 ret = rw_op(iocb, &iocb->ki_iovec[iocb->ki_cur_seg],
1336 iocb->ki_nr_segs - iocb->ki_cur_seg,
1337 iocb->ki_pos);
1338 if (ret > 0)
1339 aio_advance_iovec(iocb, ret);
1340
1341
1342
1343 } while (ret > 0 && iocb->ki_left > 0 &&
1344 (opcode == IOCB_CMD_PWRITEV ||
1345 (!S_ISFIFO(inode->i_mode) && !S_ISSOCK(inode->i_mode))));
1346
1347
1348
1349 if ((ret == 0) || (iocb->ki_left == 0))
1350 ret = iocb->ki_nbytes - iocb->ki_left;
1351
1352
1353
1354 if (opcode == IOCB_CMD_PWRITEV
1355 && ret < 0 && ret != -EIOCBQUEUED && ret != -EIOCBRETRY
1356 && iocb->ki_nbytes - iocb->ki_left)
1357 ret = iocb->ki_nbytes - iocb->ki_left;
1358
1359 return ret;
1360}
1361
1362static ssize_t aio_fdsync(struct kiocb *iocb)
1363{
1364 struct file *file = iocb->ki_filp;
1365 ssize_t ret = -EINVAL;
1366
1367 if (file->f_op->aio_fsync)
1368 ret = file->f_op->aio_fsync(iocb, 1);
1369 return ret;
1370}
1371
1372static ssize_t aio_fsync(struct kiocb *iocb)
1373{
1374 struct file *file = iocb->ki_filp;
1375 ssize_t ret = -EINVAL;
1376
1377 if (file->f_op->aio_fsync)
1378 ret = file->f_op->aio_fsync(iocb, 0);
1379 return ret;
1380}
1381
1382static ssize_t aio_setup_vectored_rw(int type, struct kiocb *kiocb)
1383{
1384 ssize_t ret;
1385
1386 ret = rw_copy_check_uvector(type, (struct iovec __user *)kiocb->ki_buf,
1387 kiocb->ki_nbytes, 1,
1388 &kiocb->ki_inline_vec, &kiocb->ki_iovec);
1389 if (ret < 0)
1390 goto out;
1391
1392 kiocb->ki_nr_segs = kiocb->ki_nbytes;
1393 kiocb->ki_cur_seg = 0;
1394
1395 kiocb->ki_nbytes = ret;
1396 kiocb->ki_left = ret;
1397
1398 ret = 0;
1399out:
1400 return ret;
1401}
1402
1403static ssize_t aio_setup_single_vector(struct kiocb *kiocb)
1404{
1405 kiocb->ki_iovec = &kiocb->ki_inline_vec;
1406 kiocb->ki_iovec->iov_base = kiocb->ki_buf;
1407 kiocb->ki_iovec->iov_len = kiocb->ki_left;
1408 kiocb->ki_nr_segs = 1;
1409 kiocb->ki_cur_seg = 0;
1410 return 0;
1411}
1412
1413
1414
1415
1416
1417
1418static ssize_t aio_setup_iocb(struct kiocb *kiocb)
1419{
1420 struct file *file = kiocb->ki_filp;
1421 ssize_t ret = 0;
1422
1423 switch (kiocb->ki_opcode) {
1424 case IOCB_CMD_PREAD:
1425 ret = -EBADF;
1426 if (unlikely(!(file->f_mode & FMODE_READ)))
1427 break;
1428 ret = -EFAULT;
1429 if (unlikely(!access_ok(VERIFY_WRITE, kiocb->ki_buf,
1430 kiocb->ki_left)))
1431 break;
1432 ret = security_file_permission(file, MAY_READ);
1433 if (unlikely(ret))
1434 break;
1435 ret = aio_setup_single_vector(kiocb);
1436 if (ret)
1437 break;
1438 ret = -EINVAL;
1439 if (file->f_op->aio_read)
1440 kiocb->ki_retry = aio_rw_vect_retry;
1441 break;
1442 case IOCB_CMD_PWRITE:
1443 ret = -EBADF;
1444 if (unlikely(!(file->f_mode & FMODE_WRITE)))
1445 break;
1446 ret = -EFAULT;
1447 if (unlikely(!access_ok(VERIFY_READ, kiocb->ki_buf,
1448 kiocb->ki_left)))
1449 break;
1450 ret = security_file_permission(file, MAY_WRITE);
1451 if (unlikely(ret))
1452 break;
1453 ret = aio_setup_single_vector(kiocb);
1454 if (ret)
1455 break;
1456 ret = -EINVAL;
1457 if (file->f_op->aio_write)
1458 kiocb->ki_retry = aio_rw_vect_retry;
1459 break;
1460 case IOCB_CMD_PREADV:
1461 ret = -EBADF;
1462 if (unlikely(!(file->f_mode & FMODE_READ)))
1463 break;
1464 ret = security_file_permission(file, MAY_READ);
1465 if (unlikely(ret))
1466 break;
1467 ret = aio_setup_vectored_rw(READ, kiocb);
1468 if (ret)
1469 break;
1470 ret = -EINVAL;
1471 if (file->f_op->aio_read)
1472 kiocb->ki_retry = aio_rw_vect_retry;
1473 break;
1474 case IOCB_CMD_PWRITEV:
1475 ret = -EBADF;
1476 if (unlikely(!(file->f_mode & FMODE_WRITE)))
1477 break;
1478 ret = security_file_permission(file, MAY_WRITE);
1479 if (unlikely(ret))
1480 break;
1481 ret = aio_setup_vectored_rw(WRITE, kiocb);
1482 if (ret)
1483 break;
1484 ret = -EINVAL;
1485 if (file->f_op->aio_write)
1486 kiocb->ki_retry = aio_rw_vect_retry;
1487 break;
1488 case IOCB_CMD_FDSYNC:
1489 ret = -EINVAL;
1490 if (file->f_op->aio_fsync)
1491 kiocb->ki_retry = aio_fdsync;
1492 break;
1493 case IOCB_CMD_FSYNC:
1494 ret = -EINVAL;
1495 if (file->f_op->aio_fsync)
1496 kiocb->ki_retry = aio_fsync;
1497 break;
1498 default:
1499 dprintk("EINVAL: io_submit: no operation provided\n");
1500 ret = -EINVAL;
1501 }
1502
1503 if (!kiocb->ki_retry)
1504 return ret;
1505
1506 return 0;
1507}
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524static int aio_wake_function(wait_queue_t *wait, unsigned mode,
1525 int sync, void *key)
1526{
1527 struct kiocb *iocb = container_of(wait, struct kiocb, ki_wait);
1528
1529 list_del_init(&wait->task_list);
1530 kick_iocb(iocb);
1531 return 1;
1532}
1533
1534static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb,
1535 struct iocb *iocb)
1536{
1537 struct kiocb *req;
1538 struct file *file;
1539 ssize_t ret;
1540
1541
1542 if (unlikely(iocb->aio_reserved1 || iocb->aio_reserved2)) {
1543 pr_debug("EINVAL: io_submit: reserve field set\n");
1544 return -EINVAL;
1545 }
1546
1547
1548 if (unlikely(
1549 (iocb->aio_buf != (unsigned long)iocb->aio_buf) ||
1550 (iocb->aio_nbytes != (size_t)iocb->aio_nbytes) ||
1551 ((ssize_t)iocb->aio_nbytes < 0)
1552 )) {
1553 pr_debug("EINVAL: io_submit: overflow check\n");
1554 return -EINVAL;
1555 }
1556
1557 file = fget(iocb->aio_fildes);
1558 if (unlikely(!file))
1559 return -EBADF;
1560
1561 req = aio_get_req(ctx);
1562 if (unlikely(!req)) {
1563 fput(file);
1564 return -EAGAIN;
1565 }
1566 req->ki_filp = file;
1567 if (iocb->aio_flags & IOCB_FLAG_RESFD) {
1568
1569
1570
1571
1572
1573
1574 req->ki_eventfd = eventfd_ctx_fdget((int) iocb->aio_resfd);
1575 if (IS_ERR(req->ki_eventfd)) {
1576 ret = PTR_ERR(req->ki_eventfd);
1577 req->ki_eventfd = NULL;
1578 goto out_put_req;
1579 }
1580 }
1581
1582 ret = put_user(req->ki_key, &user_iocb->aio_key);
1583 if (unlikely(ret)) {
1584 dprintk("EFAULT: aio_key\n");
1585 goto out_put_req;
1586 }
1587
1588 req->ki_obj.user = user_iocb;
1589 req->ki_user_data = iocb->aio_data;
1590 req->ki_pos = iocb->aio_offset;
1591
1592 req->ki_buf = (char __user *)(unsigned long)iocb->aio_buf;
1593 req->ki_left = req->ki_nbytes = iocb->aio_nbytes;
1594 req->ki_opcode = iocb->aio_lio_opcode;
1595 init_waitqueue_func_entry(&req->ki_wait, aio_wake_function);
1596 INIT_LIST_HEAD(&req->ki_wait.task_list);
1597
1598 ret = aio_setup_iocb(req);
1599
1600 if (ret)
1601 goto out_put_req;
1602
1603 spin_lock_irq(&ctx->ctx_lock);
1604 aio_run_iocb(req);
1605 if (!list_empty(&ctx->run_list)) {
1606
1607 while (__aio_run_iocbs(ctx))
1608 ;
1609 }
1610 spin_unlock_irq(&ctx->ctx_lock);
1611 aio_put_req(req);
1612 return 0;
1613
1614out_put_req:
1615 aio_put_req(req);
1616 aio_put_req(req);
1617 return ret;
1618}
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr,
1633 struct iocb __user * __user *, iocbpp)
1634{
1635 struct kioctx *ctx;
1636 long ret = 0;
1637 int i;
1638
1639 if (unlikely(nr < 0))
1640 return -EINVAL;
1641
1642 if (unlikely(!access_ok(VERIFY_READ, iocbpp, (nr*sizeof(*iocbpp)))))
1643 return -EFAULT;
1644
1645 ctx = lookup_ioctx(ctx_id);
1646 if (unlikely(!ctx)) {
1647 pr_debug("EINVAL: io_submit: invalid context id\n");
1648 return -EINVAL;
1649 }
1650
1651
1652
1653
1654
1655 for (i=0; i<nr; i++) {
1656 struct iocb __user *user_iocb;
1657 struct iocb tmp;
1658
1659 if (unlikely(__get_user(user_iocb, iocbpp + i))) {
1660 ret = -EFAULT;
1661 break;
1662 }
1663
1664 if (unlikely(copy_from_user(&tmp, user_iocb, sizeof(tmp)))) {
1665 ret = -EFAULT;
1666 break;
1667 }
1668
1669 ret = io_submit_one(ctx, user_iocb, &tmp);
1670 if (ret)
1671 break;
1672 }
1673
1674 put_ioctx(ctx);
1675 return i ? i : ret;
1676}
1677
1678
1679
1680
1681static struct kiocb *lookup_kiocb(struct kioctx *ctx, struct iocb __user *iocb,
1682 u32 key)
1683{
1684 struct list_head *pos;
1685
1686 assert_spin_locked(&ctx->ctx_lock);
1687
1688
1689 list_for_each(pos, &ctx->active_reqs) {
1690 struct kiocb *kiocb = list_kiocb(pos);
1691 if (kiocb->ki_obj.user == iocb && kiocb->ki_key == key)
1692 return kiocb;
1693 }
1694 return NULL;
1695}
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb,
1708 struct io_event __user *, result)
1709{
1710 int (*cancel)(struct kiocb *iocb, struct io_event *res);
1711 struct kioctx *ctx;
1712 struct kiocb *kiocb;
1713 u32 key;
1714 int ret;
1715
1716 ret = get_user(key, &iocb->aio_key);
1717 if (unlikely(ret))
1718 return -EFAULT;
1719
1720 ctx = lookup_ioctx(ctx_id);
1721 if (unlikely(!ctx))
1722 return -EINVAL;
1723
1724 spin_lock_irq(&ctx->ctx_lock);
1725 ret = -EAGAIN;
1726 kiocb = lookup_kiocb(ctx, iocb, key);
1727 if (kiocb && kiocb->ki_cancel) {
1728 cancel = kiocb->ki_cancel;
1729 kiocb->ki_users ++;
1730 kiocbSetCancelled(kiocb);
1731 } else
1732 cancel = NULL;
1733 spin_unlock_irq(&ctx->ctx_lock);
1734
1735 if (NULL != cancel) {
1736 struct io_event tmp;
1737 pr_debug("calling cancel\n");
1738 memset(&tmp, 0, sizeof(tmp));
1739 tmp.obj = (u64)(unsigned long)kiocb->ki_obj.user;
1740 tmp.data = kiocb->ki_user_data;
1741 ret = cancel(kiocb, &tmp);
1742 if (!ret) {
1743
1744
1745
1746 if (copy_to_user(result, &tmp, sizeof(tmp)))
1747 ret = -EFAULT;
1748 }
1749 } else
1750 ret = -EINVAL;
1751
1752 put_ioctx(ctx);
1753
1754 return ret;
1755}
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id,
1770 long, min_nr,
1771 long, nr,
1772 struct io_event __user *, events,
1773 struct timespec __user *, timeout)
1774{
1775 struct kioctx *ioctx = lookup_ioctx(ctx_id);
1776 long ret = -EINVAL;
1777
1778 if (likely(ioctx)) {
1779 if (likely(min_nr <= nr && min_nr >= 0 && nr >= 0))
1780 ret = read_events(ioctx, min_nr, nr, events, timeout);
1781 put_ioctx(ioctx);
1782 }
1783
1784 asmlinkage_protect(5, ret, ctx_id, min_nr, nr, events, timeout);
1785 return ret;
1786}
1787