1
2
3
4
5
6
7
8
9#include "fuse_i.h"
10
11#include <linux/init.h>
12#include <linux/module.h>
13#include <linux/poll.h>
14#include <linux/uio.h>
15#include <linux/miscdevice.h>
16#include <linux/pagemap.h>
17#include <linux/file.h>
18#include <linux/slab.h>
19#include <linux/pipe_fs_i.h>
20#include <linux/swap.h>
21#include <linux/splice.h>
22
23MODULE_ALIAS_MISCDEV(FUSE_MINOR);
24MODULE_ALIAS("devname:fuse");
25
26static struct kmem_cache *fuse_req_cachep;
27
28static struct fuse_dev *fuse_get_dev(struct file *file)
29{
30
31
32
33
34 return ACCESS_ONCE(file->private_data);
35}
36
37static void fuse_request_init(struct fuse_req *req, struct page **pages,
38 struct fuse_page_desc *page_descs,
39 unsigned npages)
40{
41 memset(req, 0, sizeof(*req));
42 memset(pages, 0, sizeof(*pages) * npages);
43 memset(page_descs, 0, sizeof(*page_descs) * npages);
44 INIT_LIST_HEAD(&req->list);
45 INIT_LIST_HEAD(&req->intr_entry);
46 init_waitqueue_head(&req->waitq);
47 atomic_set(&req->count, 1);
48 req->pages = pages;
49 req->page_descs = page_descs;
50 req->max_pages = npages;
51 __set_bit(FR_PENDING, &req->flags);
52}
53
54static struct fuse_req *__fuse_request_alloc(unsigned npages, gfp_t flags)
55{
56 struct fuse_req *req = kmem_cache_alloc(fuse_req_cachep, flags);
57 if (req) {
58 struct page **pages;
59 struct fuse_page_desc *page_descs;
60
61 if (npages <= FUSE_REQ_INLINE_PAGES) {
62 pages = req->inline_pages;
63 page_descs = req->inline_page_descs;
64 } else {
65 pages = kmalloc(sizeof(struct page *) * npages, flags);
66 page_descs = kmalloc(sizeof(struct fuse_page_desc) *
67 npages, flags);
68 }
69
70 if (!pages || !page_descs) {
71 kfree(pages);
72 kfree(page_descs);
73 kmem_cache_free(fuse_req_cachep, req);
74 return NULL;
75 }
76
77 fuse_request_init(req, pages, page_descs, npages);
78 }
79 return req;
80}
81
82struct fuse_req *fuse_request_alloc(unsigned npages)
83{
84 return __fuse_request_alloc(npages, GFP_KERNEL);
85}
86EXPORT_SYMBOL_GPL(fuse_request_alloc);
87
88struct fuse_req *fuse_request_alloc_nofs(unsigned npages)
89{
90 return __fuse_request_alloc(npages, GFP_NOFS);
91}
92
93void fuse_request_free(struct fuse_req *req)
94{
95 if (req->pages != req->inline_pages) {
96 kfree(req->pages);
97 kfree(req->page_descs);
98 }
99 kmem_cache_free(fuse_req_cachep, req);
100}
101
102void __fuse_get_request(struct fuse_req *req)
103{
104 atomic_inc(&req->count);
105}
106
107
108static void __fuse_put_request(struct fuse_req *req)
109{
110 BUG_ON(atomic_read(&req->count) < 2);
111 atomic_dec(&req->count);
112}
113
114static void fuse_req_init_context(struct fuse_req *req)
115{
116 req->in.h.uid = from_kuid_munged(&init_user_ns, current_fsuid());
117 req->in.h.gid = from_kgid_munged(&init_user_ns, current_fsgid());
118 req->in.h.pid = current->pid;
119}
120
121void fuse_set_initialized(struct fuse_conn *fc)
122{
123
124 smp_wmb();
125 fc->initialized = 1;
126}
127
128static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
129{
130 return !fc->initialized || (for_background && fc->blocked);
131}
132
133static struct fuse_req *__fuse_get_req(struct fuse_conn *fc, unsigned npages,
134 bool for_background)
135{
136 struct fuse_req *req;
137 int err;
138 atomic_inc(&fc->num_waiting);
139
140 if (fuse_block_alloc(fc, for_background)) {
141 err = -EINTR;
142 if (wait_event_killable_exclusive(fc->blocked_waitq,
143 !fuse_block_alloc(fc, for_background)))
144 goto out;
145 }
146
147 smp_rmb();
148
149 err = -ENOTCONN;
150 if (!fc->connected)
151 goto out;
152
153 err = -ECONNREFUSED;
154 if (fc->conn_error)
155 goto out;
156
157 req = fuse_request_alloc(npages);
158 err = -ENOMEM;
159 if (!req) {
160 if (for_background)
161 wake_up(&fc->blocked_waitq);
162 goto out;
163 }
164
165 fuse_req_init_context(req);
166 __set_bit(FR_WAITING, &req->flags);
167 if (for_background)
168 __set_bit(FR_BACKGROUND, &req->flags);
169
170 return req;
171
172 out:
173 atomic_dec(&fc->num_waiting);
174 return ERR_PTR(err);
175}
176
177struct fuse_req *fuse_get_req(struct fuse_conn *fc, unsigned npages)
178{
179 return __fuse_get_req(fc, npages, false);
180}
181EXPORT_SYMBOL_GPL(fuse_get_req);
182
183struct fuse_req *fuse_get_req_for_background(struct fuse_conn *fc,
184 unsigned npages)
185{
186 return __fuse_get_req(fc, npages, true);
187}
188EXPORT_SYMBOL_GPL(fuse_get_req_for_background);
189
190
191
192
193
194
195static struct fuse_req *get_reserved_req(struct fuse_conn *fc,
196 struct file *file)
197{
198 struct fuse_req *req = NULL;
199 struct fuse_file *ff = file->private_data;
200
201 do {
202 wait_event(fc->reserved_req_waitq, ff->reserved_req);
203 spin_lock(&fc->lock);
204 if (ff->reserved_req) {
205 req = ff->reserved_req;
206 ff->reserved_req = NULL;
207 req->stolen_file = get_file(file);
208 }
209 spin_unlock(&fc->lock);
210 } while (!req);
211
212 return req;
213}
214
215
216
217
218static void put_reserved_req(struct fuse_conn *fc, struct fuse_req *req)
219{
220 struct file *file = req->stolen_file;
221 struct fuse_file *ff = file->private_data;
222
223 spin_lock(&fc->lock);
224 fuse_request_init(req, req->pages, req->page_descs, req->max_pages);
225 BUG_ON(ff->reserved_req);
226 ff->reserved_req = req;
227 wake_up_all(&fc->reserved_req_waitq);
228 spin_unlock(&fc->lock);
229 fput(file);
230}
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245struct fuse_req *fuse_get_req_nofail_nopages(struct fuse_conn *fc,
246 struct file *file)
247{
248 struct fuse_req *req;
249
250 atomic_inc(&fc->num_waiting);
251 wait_event(fc->blocked_waitq, fc->initialized);
252
253 smp_rmb();
254 req = fuse_request_alloc(0);
255 if (!req)
256 req = get_reserved_req(fc, file);
257
258 fuse_req_init_context(req);
259 __set_bit(FR_WAITING, &req->flags);
260 __clear_bit(FR_BACKGROUND, &req->flags);
261 return req;
262}
263
264void fuse_put_request(struct fuse_conn *fc, struct fuse_req *req)
265{
266 if (atomic_dec_and_test(&req->count)) {
267 if (test_bit(FR_BACKGROUND, &req->flags)) {
268
269
270
271
272 spin_lock(&fc->lock);
273 if (!fc->blocked)
274 wake_up(&fc->blocked_waitq);
275 spin_unlock(&fc->lock);
276 }
277
278 if (test_bit(FR_WAITING, &req->flags)) {
279 __clear_bit(FR_WAITING, &req->flags);
280 atomic_dec(&fc->num_waiting);
281 }
282
283 if (req->stolen_file)
284 put_reserved_req(fc, req);
285 else
286 fuse_request_free(req);
287 }
288}
289EXPORT_SYMBOL_GPL(fuse_put_request);
290
291static unsigned len_args(unsigned numargs, struct fuse_arg *args)
292{
293 unsigned nbytes = 0;
294 unsigned i;
295
296 for (i = 0; i < numargs; i++)
297 nbytes += args[i].size;
298
299 return nbytes;
300}
301
302static u64 fuse_get_unique(struct fuse_iqueue *fiq)
303{
304 return ++fiq->reqctr;
305}
306
307static void queue_request(struct fuse_iqueue *fiq, struct fuse_req *req)
308{
309 req->in.h.len = sizeof(struct fuse_in_header) +
310 len_args(req->in.numargs, (struct fuse_arg *) req->in.args);
311 list_add_tail(&req->list, &fiq->pending);
312 wake_up_locked(&fiq->waitq);
313 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
314}
315
316void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
317 u64 nodeid, u64 nlookup)
318{
319 struct fuse_iqueue *fiq = &fc->iq;
320
321 forget->forget_one.nodeid = nodeid;
322 forget->forget_one.nlookup = nlookup;
323
324 spin_lock(&fiq->waitq.lock);
325 if (fiq->connected) {
326 fiq->forget_list_tail->next = forget;
327 fiq->forget_list_tail = forget;
328 wake_up_locked(&fiq->waitq);
329 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
330 } else {
331 kfree(forget);
332 }
333 spin_unlock(&fiq->waitq.lock);
334}
335
336static void flush_bg_queue(struct fuse_conn *fc)
337{
338 while (fc->active_background < fc->max_background &&
339 !list_empty(&fc->bg_queue)) {
340 struct fuse_req *req;
341 struct fuse_iqueue *fiq = &fc->iq;
342
343 req = list_entry(fc->bg_queue.next, struct fuse_req, list);
344 list_del(&req->list);
345 fc->active_background++;
346 spin_lock(&fiq->waitq.lock);
347 req->in.h.unique = fuse_get_unique(fiq);
348 queue_request(fiq, req);
349 spin_unlock(&fiq->waitq.lock);
350 }
351}
352
353
354
355
356
357
358
359
360
361static void request_end(struct fuse_conn *fc, struct fuse_req *req)
362{
363 struct fuse_iqueue *fiq = &fc->iq;
364
365 if (test_and_set_bit(FR_FINISHED, &req->flags))
366 return;
367
368 spin_lock(&fiq->waitq.lock);
369 list_del_init(&req->intr_entry);
370 spin_unlock(&fiq->waitq.lock);
371 WARN_ON(test_bit(FR_PENDING, &req->flags));
372 WARN_ON(test_bit(FR_SENT, &req->flags));
373 if (test_bit(FR_BACKGROUND, &req->flags)) {
374 spin_lock(&fc->lock);
375 clear_bit(FR_BACKGROUND, &req->flags);
376 if (fc->num_background == fc->max_background)
377 fc->blocked = 0;
378
379
380 if (!fc->blocked && waitqueue_active(&fc->blocked_waitq))
381 wake_up(&fc->blocked_waitq);
382
383 if (fc->num_background == fc->congestion_threshold &&
384 fc->connected && fc->bdi_initialized) {
385 clear_bdi_congested(&fc->bdi, BLK_RW_SYNC);
386 clear_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
387 }
388 fc->num_background--;
389 fc->active_background--;
390 flush_bg_queue(fc);
391 spin_unlock(&fc->lock);
392 }
393 wake_up(&req->waitq);
394 if (req->end)
395 req->end(fc, req);
396 fuse_put_request(fc, req);
397}
398
399static void queue_interrupt(struct fuse_iqueue *fiq, struct fuse_req *req)
400{
401 spin_lock(&fiq->waitq.lock);
402 if (list_empty(&req->intr_entry)) {
403 list_add_tail(&req->intr_entry, &fiq->interrupts);
404 wake_up_locked(&fiq->waitq);
405 }
406 spin_unlock(&fiq->waitq.lock);
407 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
408}
409
410static void request_wait_answer(struct fuse_conn *fc, struct fuse_req *req)
411{
412 struct fuse_iqueue *fiq = &fc->iq;
413 int err;
414
415 if (!fc->no_interrupt) {
416
417 err = wait_event_interruptible(req->waitq,
418 test_bit(FR_FINISHED, &req->flags));
419 if (!err)
420 return;
421
422 set_bit(FR_INTERRUPTED, &req->flags);
423
424 smp_mb__after_atomic();
425 if (test_bit(FR_SENT, &req->flags))
426 queue_interrupt(fiq, req);
427 }
428
429 if (!test_bit(FR_FORCE, &req->flags)) {
430
431 err = wait_event_killable(req->waitq,
432 test_bit(FR_FINISHED, &req->flags));
433 if (!err)
434 return;
435
436 spin_lock(&fiq->waitq.lock);
437
438 if (test_bit(FR_PENDING, &req->flags)) {
439 list_del(&req->list);
440 spin_unlock(&fiq->waitq.lock);
441 __fuse_put_request(req);
442 req->out.h.error = -EINTR;
443 return;
444 }
445 spin_unlock(&fiq->waitq.lock);
446 }
447
448
449
450
451
452 wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags));
453}
454
455static void __fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
456{
457 struct fuse_iqueue *fiq = &fc->iq;
458
459 BUG_ON(test_bit(FR_BACKGROUND, &req->flags));
460 spin_lock(&fiq->waitq.lock);
461 if (!fiq->connected) {
462 spin_unlock(&fiq->waitq.lock);
463 req->out.h.error = -ENOTCONN;
464 } else {
465 req->in.h.unique = fuse_get_unique(fiq);
466 queue_request(fiq, req);
467
468
469 __fuse_get_request(req);
470 spin_unlock(&fiq->waitq.lock);
471
472 request_wait_answer(fc, req);
473
474 smp_rmb();
475 }
476}
477
478void fuse_request_send(struct fuse_conn *fc, struct fuse_req *req)
479{
480 __set_bit(FR_ISREPLY, &req->flags);
481 if (!test_bit(FR_WAITING, &req->flags)) {
482 __set_bit(FR_WAITING, &req->flags);
483 atomic_inc(&fc->num_waiting);
484 }
485 __fuse_request_send(fc, req);
486}
487EXPORT_SYMBOL_GPL(fuse_request_send);
488
489static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
490{
491 if (fc->minor < 4 && args->in.h.opcode == FUSE_STATFS)
492 args->out.args[0].size = FUSE_COMPAT_STATFS_SIZE;
493
494 if (fc->minor < 9) {
495 switch (args->in.h.opcode) {
496 case FUSE_LOOKUP:
497 case FUSE_CREATE:
498 case FUSE_MKNOD:
499 case FUSE_MKDIR:
500 case FUSE_SYMLINK:
501 case FUSE_LINK:
502 args->out.args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE;
503 break;
504 case FUSE_GETATTR:
505 case FUSE_SETATTR:
506 args->out.args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE;
507 break;
508 }
509 }
510 if (fc->minor < 12) {
511 switch (args->in.h.opcode) {
512 case FUSE_CREATE:
513 args->in.args[0].size = sizeof(struct fuse_open_in);
514 break;
515 case FUSE_MKNOD:
516 args->in.args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE;
517 break;
518 }
519 }
520}
521
522ssize_t fuse_simple_request(struct fuse_conn *fc, struct fuse_args *args)
523{
524 struct fuse_req *req;
525 ssize_t ret;
526
527 req = fuse_get_req(fc, 0);
528 if (IS_ERR(req))
529 return PTR_ERR(req);
530
531
532 fuse_adjust_compat(fc, args);
533
534 req->in.h.opcode = args->in.h.opcode;
535 req->in.h.nodeid = args->in.h.nodeid;
536 req->in.numargs = args->in.numargs;
537 memcpy(req->in.args, args->in.args,
538 args->in.numargs * sizeof(struct fuse_in_arg));
539 req->out.argvar = args->out.argvar;
540 req->out.numargs = args->out.numargs;
541 memcpy(req->out.args, args->out.args,
542 args->out.numargs * sizeof(struct fuse_arg));
543 fuse_request_send(fc, req);
544 ret = req->out.h.error;
545 if (!ret && args->out.argvar) {
546 BUG_ON(args->out.numargs != 1);
547 ret = req->out.args[0].size;
548 }
549 fuse_put_request(fc, req);
550
551 return ret;
552}
553
554
555
556
557
558
559void fuse_request_send_background_locked(struct fuse_conn *fc,
560 struct fuse_req *req)
561{
562 BUG_ON(!test_bit(FR_BACKGROUND, &req->flags));
563 if (!test_bit(FR_WAITING, &req->flags)) {
564 __set_bit(FR_WAITING, &req->flags);
565 atomic_inc(&fc->num_waiting);
566 }
567 __set_bit(FR_ISREPLY, &req->flags);
568 fc->num_background++;
569 if (fc->num_background == fc->max_background)
570 fc->blocked = 1;
571 if (fc->num_background == fc->congestion_threshold &&
572 fc->bdi_initialized) {
573 set_bdi_congested(&fc->bdi, BLK_RW_SYNC);
574 set_bdi_congested(&fc->bdi, BLK_RW_ASYNC);
575 }
576 list_add_tail(&req->list, &fc->bg_queue);
577 flush_bg_queue(fc);
578}
579
580void fuse_request_send_background(struct fuse_conn *fc, struct fuse_req *req)
581{
582 BUG_ON(!req->end);
583 spin_lock(&fc->lock);
584 if (fc->connected) {
585 fuse_request_send_background_locked(fc, req);
586 spin_unlock(&fc->lock);
587 } else {
588 spin_unlock(&fc->lock);
589 req->out.h.error = -ENOTCONN;
590 req->end(fc, req);
591 fuse_put_request(fc, req);
592 }
593}
594EXPORT_SYMBOL_GPL(fuse_request_send_background);
595
596static int fuse_request_send_notify_reply(struct fuse_conn *fc,
597 struct fuse_req *req, u64 unique)
598{
599 int err = -ENODEV;
600 struct fuse_iqueue *fiq = &fc->iq;
601
602 __clear_bit(FR_ISREPLY, &req->flags);
603 req->in.h.unique = unique;
604 spin_lock(&fiq->waitq.lock);
605 if (fiq->connected) {
606 queue_request(fiq, req);
607 err = 0;
608 }
609 spin_unlock(&fiq->waitq.lock);
610
611 return err;
612}
613
614void fuse_force_forget(struct file *file, u64 nodeid)
615{
616 struct inode *inode = file_inode(file);
617 struct fuse_conn *fc = get_fuse_conn(inode);
618 struct fuse_req *req;
619 struct fuse_forget_in inarg;
620
621 memset(&inarg, 0, sizeof(inarg));
622 inarg.nlookup = 1;
623 req = fuse_get_req_nofail_nopages(fc, file);
624 req->in.h.opcode = FUSE_FORGET;
625 req->in.h.nodeid = nodeid;
626 req->in.numargs = 1;
627 req->in.args[0].size = sizeof(inarg);
628 req->in.args[0].value = &inarg;
629 __clear_bit(FR_ISREPLY, &req->flags);
630 __fuse_request_send(fc, req);
631
632 fuse_put_request(fc, req);
633}
634
635
636
637
638
639
640static int lock_request(struct fuse_req *req)
641{
642 int err = 0;
643 if (req) {
644 spin_lock(&req->waitq.lock);
645 if (test_bit(FR_ABORTED, &req->flags))
646 err = -ENOENT;
647 else
648 set_bit(FR_LOCKED, &req->flags);
649 spin_unlock(&req->waitq.lock);
650 }
651 return err;
652}
653
654
655
656
657
658static int unlock_request(struct fuse_req *req)
659{
660 int err = 0;
661 if (req) {
662 spin_lock(&req->waitq.lock);
663 if (test_bit(FR_ABORTED, &req->flags))
664 err = -ENOENT;
665 else
666 clear_bit(FR_LOCKED, &req->flags);
667 spin_unlock(&req->waitq.lock);
668 }
669 return err;
670}
671
672struct fuse_copy_state {
673 int write;
674 struct fuse_req *req;
675 struct iov_iter *iter;
676 struct pipe_buffer *pipebufs;
677 struct pipe_buffer *currbuf;
678 struct pipe_inode_info *pipe;
679 unsigned long nr_segs;
680 struct page *pg;
681 unsigned len;
682 unsigned offset;
683 unsigned move_pages:1;
684};
685
686static void fuse_copy_init(struct fuse_copy_state *cs, int write,
687 struct iov_iter *iter)
688{
689 memset(cs, 0, sizeof(*cs));
690 cs->write = write;
691 cs->iter = iter;
692}
693
694
695static void fuse_copy_finish(struct fuse_copy_state *cs)
696{
697 if (cs->currbuf) {
698 struct pipe_buffer *buf = cs->currbuf;
699
700 if (cs->write)
701 buf->len = PAGE_SIZE - cs->len;
702 cs->currbuf = NULL;
703 } else if (cs->pg) {
704 if (cs->write) {
705 flush_dcache_page(cs->pg);
706 set_page_dirty_lock(cs->pg);
707 }
708 put_page(cs->pg);
709 }
710 cs->pg = NULL;
711}
712
713
714
715
716
717static int fuse_copy_fill(struct fuse_copy_state *cs)
718{
719 struct page *page;
720 int err;
721
722 err = unlock_request(cs->req);
723 if (err)
724 return err;
725
726 fuse_copy_finish(cs);
727 if (cs->pipebufs) {
728 struct pipe_buffer *buf = cs->pipebufs;
729
730 if (!cs->write) {
731 err = buf->ops->confirm(cs->pipe, buf);
732 if (err)
733 return err;
734
735 BUG_ON(!cs->nr_segs);
736 cs->currbuf = buf;
737 cs->pg = buf->page;
738 cs->offset = buf->offset;
739 cs->len = buf->len;
740 cs->pipebufs++;
741 cs->nr_segs--;
742 } else {
743 if (cs->nr_segs == cs->pipe->buffers)
744 return -EIO;
745
746 page = alloc_page(GFP_HIGHUSER);
747 if (!page)
748 return -ENOMEM;
749
750 buf->page = page;
751 buf->offset = 0;
752 buf->len = 0;
753
754 cs->currbuf = buf;
755 cs->pg = page;
756 cs->offset = 0;
757 cs->len = PAGE_SIZE;
758 cs->pipebufs++;
759 cs->nr_segs++;
760 }
761 } else {
762 size_t off;
763 err = iov_iter_get_pages(cs->iter, &page, PAGE_SIZE, 1, &off);
764 if (err < 0)
765 return err;
766 BUG_ON(!err);
767 cs->len = err;
768 cs->offset = off;
769 cs->pg = page;
770 cs->offset = off;
771 iov_iter_advance(cs->iter, err);
772 }
773
774 return lock_request(cs->req);
775}
776
777
778static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
779{
780 unsigned ncpy = min(*size, cs->len);
781 if (val) {
782 void *pgaddr = kmap_atomic(cs->pg);
783 void *buf = pgaddr + cs->offset;
784
785 if (cs->write)
786 memcpy(buf, *val, ncpy);
787 else
788 memcpy(*val, buf, ncpy);
789
790 kunmap_atomic(pgaddr);
791 *val += ncpy;
792 }
793 *size -= ncpy;
794 cs->len -= ncpy;
795 cs->offset += ncpy;
796 return ncpy;
797}
798
799static int fuse_check_page(struct page *page)
800{
801 if (page_mapcount(page) ||
802 page->mapping != NULL ||
803 page_count(page) != 1 ||
804 (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
805 ~(1 << PG_locked |
806 1 << PG_referenced |
807 1 << PG_uptodate |
808 1 << PG_lru |
809 1 << PG_active |
810 1 << PG_reclaim))) {
811 printk(KERN_WARNING "fuse: trying to steal weird page\n");
812 printk(KERN_WARNING " page=%p index=%li flags=%08lx, count=%i, mapcount=%i, mapping=%p\n", page, page->index, page->flags, page_count(page), page_mapcount(page), page->mapping);
813 return 1;
814 }
815 return 0;
816}
817
818static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
819{
820 int err;
821 struct page *oldpage = *pagep;
822 struct page *newpage;
823 struct pipe_buffer *buf = cs->pipebufs;
824
825 err = unlock_request(cs->req);
826 if (err)
827 return err;
828
829 fuse_copy_finish(cs);
830
831 err = buf->ops->confirm(cs->pipe, buf);
832 if (err)
833 return err;
834
835 BUG_ON(!cs->nr_segs);
836 cs->currbuf = buf;
837 cs->len = buf->len;
838 cs->pipebufs++;
839 cs->nr_segs--;
840
841 if (cs->len != PAGE_SIZE)
842 goto out_fallback;
843
844 if (buf->ops->steal(cs->pipe, buf) != 0)
845 goto out_fallback;
846
847 newpage = buf->page;
848
849 if (!PageUptodate(newpage))
850 SetPageUptodate(newpage);
851
852 ClearPageMappedToDisk(newpage);
853
854 if (fuse_check_page(newpage) != 0)
855 goto out_fallback_unlock;
856
857
858
859
860
861 if (WARN_ON(page_mapped(oldpage)))
862 goto out_fallback_unlock;
863 if (WARN_ON(page_has_private(oldpage)))
864 goto out_fallback_unlock;
865 if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
866 goto out_fallback_unlock;
867 if (WARN_ON(PageMlocked(oldpage)))
868 goto out_fallback_unlock;
869
870 err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL);
871 if (err) {
872 unlock_page(newpage);
873 return err;
874 }
875
876 get_page(newpage);
877
878 if (!(buf->flags & PIPE_BUF_FLAG_LRU))
879 lru_cache_add_file(newpage);
880
881 err = 0;
882 spin_lock(&cs->req->waitq.lock);
883 if (test_bit(FR_ABORTED, &cs->req->flags))
884 err = -ENOENT;
885 else
886 *pagep = newpage;
887 spin_unlock(&cs->req->waitq.lock);
888
889 if (err) {
890 unlock_page(newpage);
891 put_page(newpage);
892 return err;
893 }
894
895 unlock_page(oldpage);
896 put_page(oldpage);
897 cs->len = 0;
898
899 return 0;
900
901out_fallback_unlock:
902 unlock_page(newpage);
903out_fallback:
904 cs->pg = buf->page;
905 cs->offset = buf->offset;
906
907 err = lock_request(cs->req);
908 if (err)
909 return err;
910
911 return 1;
912}
913
914static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
915 unsigned offset, unsigned count)
916{
917 struct pipe_buffer *buf;
918 int err;
919
920 if (cs->nr_segs == cs->pipe->buffers)
921 return -EIO;
922
923 err = unlock_request(cs->req);
924 if (err)
925 return err;
926
927 fuse_copy_finish(cs);
928
929 buf = cs->pipebufs;
930 get_page(page);
931 buf->page = page;
932 buf->offset = offset;
933 buf->len = count;
934
935 cs->pipebufs++;
936 cs->nr_segs++;
937 cs->len = 0;
938
939 return 0;
940}
941
942
943
944
945
946static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
947 unsigned offset, unsigned count, int zeroing)
948{
949 int err;
950 struct page *page = *pagep;
951
952 if (page && zeroing && count < PAGE_SIZE)
953 clear_highpage(page);
954
955 while (count) {
956 if (cs->write && cs->pipebufs && page) {
957 return fuse_ref_page(cs, page, offset, count);
958 } else if (!cs->len) {
959 if (cs->move_pages && page &&
960 offset == 0 && count == PAGE_SIZE) {
961 err = fuse_try_move_page(cs, pagep);
962 if (err <= 0)
963 return err;
964 } else {
965 err = fuse_copy_fill(cs);
966 if (err)
967 return err;
968 }
969 }
970 if (page) {
971 void *mapaddr = kmap_atomic(page);
972 void *buf = mapaddr + offset;
973 offset += fuse_copy_do(cs, &buf, &count);
974 kunmap_atomic(mapaddr);
975 } else
976 offset += fuse_copy_do(cs, NULL, &count);
977 }
978 if (page && !cs->write)
979 flush_dcache_page(page);
980 return 0;
981}
982
983
984static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
985 int zeroing)
986{
987 unsigned i;
988 struct fuse_req *req = cs->req;
989
990 for (i = 0; i < req->num_pages && (nbytes || zeroing); i++) {
991 int err;
992 unsigned offset = req->page_descs[i].offset;
993 unsigned count = min(nbytes, req->page_descs[i].length);
994
995 err = fuse_copy_page(cs, &req->pages[i], offset, count,
996 zeroing);
997 if (err)
998 return err;
999
1000 nbytes -= count;
1001 }
1002 return 0;
1003}
1004
1005
1006static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
1007{
1008 while (size) {
1009 if (!cs->len) {
1010 int err = fuse_copy_fill(cs);
1011 if (err)
1012 return err;
1013 }
1014 fuse_copy_do(cs, &val, &size);
1015 }
1016 return 0;
1017}
1018
1019
1020static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
1021 unsigned argpages, struct fuse_arg *args,
1022 int zeroing)
1023{
1024 int err = 0;
1025 unsigned i;
1026
1027 for (i = 0; !err && i < numargs; i++) {
1028 struct fuse_arg *arg = &args[i];
1029 if (i == numargs - 1 && argpages)
1030 err = fuse_copy_pages(cs, arg->size, zeroing);
1031 else
1032 err = fuse_copy_one(cs, arg->value, arg->size);
1033 }
1034 return err;
1035}
1036
1037static int forget_pending(struct fuse_iqueue *fiq)
1038{
1039 return fiq->forget_list_head.next != NULL;
1040}
1041
1042static int request_pending(struct fuse_iqueue *fiq)
1043{
1044 return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||
1045 forget_pending(fiq);
1046}
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056static int fuse_read_interrupt(struct fuse_iqueue *fiq,
1057 struct fuse_copy_state *cs,
1058 size_t nbytes, struct fuse_req *req)
1059__releases(fiq->waitq.lock)
1060{
1061 struct fuse_in_header ih;
1062 struct fuse_interrupt_in arg;
1063 unsigned reqsize = sizeof(ih) + sizeof(arg);
1064 int err;
1065
1066 list_del_init(&req->intr_entry);
1067 req->intr_unique = fuse_get_unique(fiq);
1068 memset(&ih, 0, sizeof(ih));
1069 memset(&arg, 0, sizeof(arg));
1070 ih.len = reqsize;
1071 ih.opcode = FUSE_INTERRUPT;
1072 ih.unique = req->intr_unique;
1073 arg.unique = req->in.h.unique;
1074
1075 spin_unlock(&fiq->waitq.lock);
1076 if (nbytes < reqsize)
1077 return -EINVAL;
1078
1079 err = fuse_copy_one(cs, &ih, sizeof(ih));
1080 if (!err)
1081 err = fuse_copy_one(cs, &arg, sizeof(arg));
1082 fuse_copy_finish(cs);
1083
1084 return err ? err : reqsize;
1085}
1086
1087static struct fuse_forget_link *dequeue_forget(struct fuse_iqueue *fiq,
1088 unsigned max,
1089 unsigned *countp)
1090{
1091 struct fuse_forget_link *head = fiq->forget_list_head.next;
1092 struct fuse_forget_link **newhead = &head;
1093 unsigned count;
1094
1095 for (count = 0; *newhead != NULL && count < max; count++)
1096 newhead = &(*newhead)->next;
1097
1098 fiq->forget_list_head.next = *newhead;
1099 *newhead = NULL;
1100 if (fiq->forget_list_head.next == NULL)
1101 fiq->forget_list_tail = &fiq->forget_list_head;
1102
1103 if (countp != NULL)
1104 *countp = count;
1105
1106 return head;
1107}
1108
1109static int fuse_read_single_forget(struct fuse_iqueue *fiq,
1110 struct fuse_copy_state *cs,
1111 size_t nbytes)
1112__releases(fiq->waitq.lock)
1113{
1114 int err;
1115 struct fuse_forget_link *forget = dequeue_forget(fiq, 1, NULL);
1116 struct fuse_forget_in arg = {
1117 .nlookup = forget->forget_one.nlookup,
1118 };
1119 struct fuse_in_header ih = {
1120 .opcode = FUSE_FORGET,
1121 .nodeid = forget->forget_one.nodeid,
1122 .unique = fuse_get_unique(fiq),
1123 .len = sizeof(ih) + sizeof(arg),
1124 };
1125
1126 spin_unlock(&fiq->waitq.lock);
1127 kfree(forget);
1128 if (nbytes < ih.len)
1129 return -EINVAL;
1130
1131 err = fuse_copy_one(cs, &ih, sizeof(ih));
1132 if (!err)
1133 err = fuse_copy_one(cs, &arg, sizeof(arg));
1134 fuse_copy_finish(cs);
1135
1136 if (err)
1137 return err;
1138
1139 return ih.len;
1140}
1141
1142static int fuse_read_batch_forget(struct fuse_iqueue *fiq,
1143 struct fuse_copy_state *cs, size_t nbytes)
1144__releases(fiq->waitq.lock)
1145{
1146 int err;
1147 unsigned max_forgets;
1148 unsigned count;
1149 struct fuse_forget_link *head;
1150 struct fuse_batch_forget_in arg = { .count = 0 };
1151 struct fuse_in_header ih = {
1152 .opcode = FUSE_BATCH_FORGET,
1153 .unique = fuse_get_unique(fiq),
1154 .len = sizeof(ih) + sizeof(arg),
1155 };
1156
1157 if (nbytes < ih.len) {
1158 spin_unlock(&fiq->waitq.lock);
1159 return -EINVAL;
1160 }
1161
1162 max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
1163 head = dequeue_forget(fiq, max_forgets, &count);
1164 spin_unlock(&fiq->waitq.lock);
1165
1166 arg.count = count;
1167 ih.len += count * sizeof(struct fuse_forget_one);
1168 err = fuse_copy_one(cs, &ih, sizeof(ih));
1169 if (!err)
1170 err = fuse_copy_one(cs, &arg, sizeof(arg));
1171
1172 while (head) {
1173 struct fuse_forget_link *forget = head;
1174
1175 if (!err) {
1176 err = fuse_copy_one(cs, &forget->forget_one,
1177 sizeof(forget->forget_one));
1178 }
1179 head = forget->next;
1180 kfree(forget);
1181 }
1182
1183 fuse_copy_finish(cs);
1184
1185 if (err)
1186 return err;
1187
1188 return ih.len;
1189}
1190
1191static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,
1192 struct fuse_copy_state *cs,
1193 size_t nbytes)
1194__releases(fiq->waitq.lock)
1195{
1196 if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)
1197 return fuse_read_single_forget(fiq, cs, nbytes);
1198 else
1199 return fuse_read_batch_forget(fiq, cs, nbytes);
1200}
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
1212 struct fuse_copy_state *cs, size_t nbytes)
1213{
1214 ssize_t err;
1215 struct fuse_conn *fc = fud->fc;
1216 struct fuse_iqueue *fiq = &fc->iq;
1217 struct fuse_pqueue *fpq = &fud->pq;
1218 struct fuse_req *req;
1219 struct fuse_in *in;
1220 unsigned reqsize;
1221
1222 restart:
1223 spin_lock(&fiq->waitq.lock);
1224 err = -EAGAIN;
1225 if ((file->f_flags & O_NONBLOCK) && fiq->connected &&
1226 !request_pending(fiq))
1227 goto err_unlock;
1228
1229 err = wait_event_interruptible_exclusive_locked(fiq->waitq,
1230 !fiq->connected || request_pending(fiq));
1231 if (err)
1232 goto err_unlock;
1233
1234 err = -ENODEV;
1235 if (!fiq->connected)
1236 goto err_unlock;
1237
1238 if (!list_empty(&fiq->interrupts)) {
1239 req = list_entry(fiq->interrupts.next, struct fuse_req,
1240 intr_entry);
1241 return fuse_read_interrupt(fiq, cs, nbytes, req);
1242 }
1243
1244 if (forget_pending(fiq)) {
1245 if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)
1246 return fuse_read_forget(fc, fiq, cs, nbytes);
1247
1248 if (fiq->forget_batch <= -8)
1249 fiq->forget_batch = 16;
1250 }
1251
1252 req = list_entry(fiq->pending.next, struct fuse_req, list);
1253 clear_bit(FR_PENDING, &req->flags);
1254 list_del_init(&req->list);
1255 spin_unlock(&fiq->waitq.lock);
1256
1257 in = &req->in;
1258 reqsize = in->h.len;
1259
1260 if (nbytes < reqsize) {
1261 req->out.h.error = -EIO;
1262
1263 if (in->h.opcode == FUSE_SETXATTR)
1264 req->out.h.error = -E2BIG;
1265 request_end(fc, req);
1266 goto restart;
1267 }
1268 spin_lock(&fpq->lock);
1269 list_add(&req->list, &fpq->io);
1270 spin_unlock(&fpq->lock);
1271 cs->req = req;
1272 err = fuse_copy_one(cs, &in->h, sizeof(in->h));
1273 if (!err)
1274 err = fuse_copy_args(cs, in->numargs, in->argpages,
1275 (struct fuse_arg *) in->args, 0);
1276 fuse_copy_finish(cs);
1277 spin_lock(&fpq->lock);
1278 clear_bit(FR_LOCKED, &req->flags);
1279 if (!fpq->connected) {
1280 err = -ENODEV;
1281 goto out_end;
1282 }
1283 if (err) {
1284 req->out.h.error = -EIO;
1285 goto out_end;
1286 }
1287 if (!test_bit(FR_ISREPLY, &req->flags)) {
1288 err = reqsize;
1289 goto out_end;
1290 }
1291 list_move_tail(&req->list, &fpq->processing);
1292 spin_unlock(&fpq->lock);
1293 set_bit(FR_SENT, &req->flags);
1294
1295 smp_mb__after_atomic();
1296 if (test_bit(FR_INTERRUPTED, &req->flags))
1297 queue_interrupt(fiq, req);
1298
1299 return reqsize;
1300
1301out_end:
1302 if (!test_bit(FR_PRIVATE, &req->flags))
1303 list_del_init(&req->list);
1304 spin_unlock(&fpq->lock);
1305 request_end(fc, req);
1306 return err;
1307
1308 err_unlock:
1309 spin_unlock(&fiq->waitq.lock);
1310 return err;
1311}
1312
1313static int fuse_dev_open(struct inode *inode, struct file *file)
1314{
1315
1316
1317
1318
1319
1320 file->private_data = NULL;
1321 return 0;
1322}
1323
1324static ssize_t fuse_dev_read(struct kiocb *iocb, struct iov_iter *to)
1325{
1326 struct fuse_copy_state cs;
1327 struct file *file = iocb->ki_filp;
1328 struct fuse_dev *fud = fuse_get_dev(file);
1329
1330 if (!fud)
1331 return -EPERM;
1332
1333 if (!iter_is_iovec(to))
1334 return -EINVAL;
1335
1336 fuse_copy_init(&cs, 1, to);
1337
1338 return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to));
1339}
1340
1341static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
1342 struct pipe_inode_info *pipe,
1343 size_t len, unsigned int flags)
1344{
1345 int ret;
1346 int page_nr = 0;
1347 int do_wakeup = 0;
1348 struct pipe_buffer *bufs;
1349 struct fuse_copy_state cs;
1350 struct fuse_dev *fud = fuse_get_dev(in);
1351
1352 if (!fud)
1353 return -EPERM;
1354
1355 bufs = kmalloc(pipe->buffers * sizeof(struct pipe_buffer), GFP_KERNEL);
1356 if (!bufs)
1357 return -ENOMEM;
1358
1359 fuse_copy_init(&cs, 1, NULL);
1360 cs.pipebufs = bufs;
1361 cs.pipe = pipe;
1362 ret = fuse_dev_do_read(fud, in, &cs, len);
1363 if (ret < 0)
1364 goto out;
1365
1366 ret = 0;
1367 pipe_lock(pipe);
1368
1369 if (!pipe->readers) {
1370 send_sig(SIGPIPE, current, 0);
1371 if (!ret)
1372 ret = -EPIPE;
1373 goto out_unlock;
1374 }
1375
1376 if (pipe->nrbufs + cs.nr_segs > pipe->buffers) {
1377 ret = -EIO;
1378 goto out_unlock;
1379 }
1380
1381 while (page_nr < cs.nr_segs) {
1382 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
1383 struct pipe_buffer *buf = pipe->bufs + newbuf;
1384
1385 buf->page = bufs[page_nr].page;
1386 buf->offset = bufs[page_nr].offset;
1387 buf->len = bufs[page_nr].len;
1388
1389
1390
1391
1392 buf->ops = &nosteal_pipe_buf_ops;
1393
1394 pipe->nrbufs++;
1395 page_nr++;
1396 ret += buf->len;
1397
1398 if (pipe->files)
1399 do_wakeup = 1;
1400 }
1401
1402out_unlock:
1403 pipe_unlock(pipe);
1404
1405 if (do_wakeup) {
1406 smp_mb();
1407 if (waitqueue_active(&pipe->wait))
1408 wake_up_interruptible(&pipe->wait);
1409 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
1410 }
1411
1412out:
1413 for (; page_nr < cs.nr_segs; page_nr++)
1414 put_page(bufs[page_nr].page);
1415
1416 kfree(bufs);
1417 return ret;
1418}
1419
1420static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
1421 struct fuse_copy_state *cs)
1422{
1423 struct fuse_notify_poll_wakeup_out outarg;
1424 int err = -EINVAL;
1425
1426 if (size != sizeof(outarg))
1427 goto err;
1428
1429 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1430 if (err)
1431 goto err;
1432
1433 fuse_copy_finish(cs);
1434 return fuse_notify_poll_wakeup(fc, &outarg);
1435
1436err:
1437 fuse_copy_finish(cs);
1438 return err;
1439}
1440
1441static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
1442 struct fuse_copy_state *cs)
1443{
1444 struct fuse_notify_inval_inode_out outarg;
1445 int err = -EINVAL;
1446
1447 if (size != sizeof(outarg))
1448 goto err;
1449
1450 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1451 if (err)
1452 goto err;
1453 fuse_copy_finish(cs);
1454
1455 down_read(&fc->killsb);
1456 err = -ENOENT;
1457 if (fc->sb) {
1458 err = fuse_reverse_inval_inode(fc->sb, outarg.ino,
1459 outarg.off, outarg.len);
1460 }
1461 up_read(&fc->killsb);
1462 return err;
1463
1464err:
1465 fuse_copy_finish(cs);
1466 return err;
1467}
1468
1469static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
1470 struct fuse_copy_state *cs)
1471{
1472 struct fuse_notify_inval_entry_out outarg;
1473 int err = -ENOMEM;
1474 char *buf;
1475 struct qstr name;
1476
1477 buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1478 if (!buf)
1479 goto err;
1480
1481 err = -EINVAL;
1482 if (size < sizeof(outarg))
1483 goto err;
1484
1485 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1486 if (err)
1487 goto err;
1488
1489 err = -ENAMETOOLONG;
1490 if (outarg.namelen > FUSE_NAME_MAX)
1491 goto err;
1492
1493 err = -EINVAL;
1494 if (size != sizeof(outarg) + outarg.namelen + 1)
1495 goto err;
1496
1497 name.name = buf;
1498 name.len = outarg.namelen;
1499 err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1500 if (err)
1501 goto err;
1502 fuse_copy_finish(cs);
1503 buf[outarg.namelen] = 0;
1504
1505 down_read(&fc->killsb);
1506 err = -ENOENT;
1507 if (fc->sb)
1508 err = fuse_reverse_inval_entry(fc->sb, outarg.parent, 0, &name);
1509 up_read(&fc->killsb);
1510 kfree(buf);
1511 return err;
1512
1513err:
1514 kfree(buf);
1515 fuse_copy_finish(cs);
1516 return err;
1517}
1518
1519static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
1520 struct fuse_copy_state *cs)
1521{
1522 struct fuse_notify_delete_out outarg;
1523 int err = -ENOMEM;
1524 char *buf;
1525 struct qstr name;
1526
1527 buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
1528 if (!buf)
1529 goto err;
1530
1531 err = -EINVAL;
1532 if (size < sizeof(outarg))
1533 goto err;
1534
1535 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1536 if (err)
1537 goto err;
1538
1539 err = -ENAMETOOLONG;
1540 if (outarg.namelen > FUSE_NAME_MAX)
1541 goto err;
1542
1543 err = -EINVAL;
1544 if (size != sizeof(outarg) + outarg.namelen + 1)
1545 goto err;
1546
1547 name.name = buf;
1548 name.len = outarg.namelen;
1549 err = fuse_copy_one(cs, buf, outarg.namelen + 1);
1550 if (err)
1551 goto err;
1552 fuse_copy_finish(cs);
1553 buf[outarg.namelen] = 0;
1554
1555 down_read(&fc->killsb);
1556 err = -ENOENT;
1557 if (fc->sb)
1558 err = fuse_reverse_inval_entry(fc->sb, outarg.parent,
1559 outarg.child, &name);
1560 up_read(&fc->killsb);
1561 kfree(buf);
1562 return err;
1563
1564err:
1565 kfree(buf);
1566 fuse_copy_finish(cs);
1567 return err;
1568}
1569
1570static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
1571 struct fuse_copy_state *cs)
1572{
1573 struct fuse_notify_store_out outarg;
1574 struct inode *inode;
1575 struct address_space *mapping;
1576 u64 nodeid;
1577 int err;
1578 pgoff_t index;
1579 unsigned int offset;
1580 unsigned int num;
1581 loff_t file_size;
1582 loff_t end;
1583
1584 err = -EINVAL;
1585 if (size < sizeof(outarg))
1586 goto out_finish;
1587
1588 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1589 if (err)
1590 goto out_finish;
1591
1592 err = -EINVAL;
1593 if (size - sizeof(outarg) != outarg.size)
1594 goto out_finish;
1595
1596 nodeid = outarg.nodeid;
1597
1598 down_read(&fc->killsb);
1599
1600 err = -ENOENT;
1601 if (!fc->sb)
1602 goto out_up_killsb;
1603
1604 inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
1605 if (!inode)
1606 goto out_up_killsb;
1607
1608 mapping = inode->i_mapping;
1609 index = outarg.offset >> PAGE_SHIFT;
1610 offset = outarg.offset & ~PAGE_MASK;
1611 file_size = i_size_read(inode);
1612 end = outarg.offset + outarg.size;
1613 if (end > file_size) {
1614 file_size = end;
1615 fuse_write_update_size(inode, file_size);
1616 }
1617
1618 num = outarg.size;
1619 while (num) {
1620 struct page *page;
1621 unsigned int this_num;
1622
1623 err = -ENOMEM;
1624 page = find_or_create_page(mapping, index,
1625 mapping_gfp_mask(mapping));
1626 if (!page)
1627 goto out_iput;
1628
1629 this_num = min_t(unsigned, num, PAGE_SIZE - offset);
1630 err = fuse_copy_page(cs, &page, offset, this_num, 0);
1631 if (!err && offset == 0 &&
1632 (this_num == PAGE_SIZE || file_size == end))
1633 SetPageUptodate(page);
1634 unlock_page(page);
1635 put_page(page);
1636
1637 if (err)
1638 goto out_iput;
1639
1640 num -= this_num;
1641 offset = 0;
1642 index++;
1643 }
1644
1645 err = 0;
1646
1647out_iput:
1648 iput(inode);
1649out_up_killsb:
1650 up_read(&fc->killsb);
1651out_finish:
1652 fuse_copy_finish(cs);
1653 return err;
1654}
1655
1656static void fuse_retrieve_end(struct fuse_conn *fc, struct fuse_req *req)
1657{
1658 release_pages(req->pages, req->num_pages, false);
1659}
1660
1661static int fuse_retrieve(struct fuse_conn *fc, struct inode *inode,
1662 struct fuse_notify_retrieve_out *outarg)
1663{
1664 int err;
1665 struct address_space *mapping = inode->i_mapping;
1666 struct fuse_req *req;
1667 pgoff_t index;
1668 loff_t file_size;
1669 unsigned int num;
1670 unsigned int offset;
1671 size_t total_len = 0;
1672 int num_pages;
1673
1674 offset = outarg->offset & ~PAGE_MASK;
1675 file_size = i_size_read(inode);
1676
1677 num = outarg->size;
1678 if (outarg->offset > file_size)
1679 num = 0;
1680 else if (outarg->offset + num > file_size)
1681 num = file_size - outarg->offset;
1682
1683 num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
1684 num_pages = min(num_pages, FUSE_MAX_PAGES_PER_REQ);
1685
1686 req = fuse_get_req(fc, num_pages);
1687 if (IS_ERR(req))
1688 return PTR_ERR(req);
1689
1690 req->in.h.opcode = FUSE_NOTIFY_REPLY;
1691 req->in.h.nodeid = outarg->nodeid;
1692 req->in.numargs = 2;
1693 req->in.argpages = 1;
1694 req->page_descs[0].offset = offset;
1695 req->end = fuse_retrieve_end;
1696
1697 index = outarg->offset >> PAGE_SHIFT;
1698
1699 while (num && req->num_pages < num_pages) {
1700 struct page *page;
1701 unsigned int this_num;
1702
1703 page = find_get_page(mapping, index);
1704 if (!page)
1705 break;
1706
1707 this_num = min_t(unsigned, num, PAGE_SIZE - offset);
1708 req->pages[req->num_pages] = page;
1709 req->page_descs[req->num_pages].length = this_num;
1710 req->num_pages++;
1711
1712 offset = 0;
1713 num -= this_num;
1714 total_len += this_num;
1715 index++;
1716 }
1717 req->misc.retrieve_in.offset = outarg->offset;
1718 req->misc.retrieve_in.size = total_len;
1719 req->in.args[0].size = sizeof(req->misc.retrieve_in);
1720 req->in.args[0].value = &req->misc.retrieve_in;
1721 req->in.args[1].size = total_len;
1722
1723 err = fuse_request_send_notify_reply(fc, req, outarg->notify_unique);
1724 if (err)
1725 fuse_retrieve_end(fc, req);
1726
1727 return err;
1728}
1729
1730static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
1731 struct fuse_copy_state *cs)
1732{
1733 struct fuse_notify_retrieve_out outarg;
1734 struct inode *inode;
1735 int err;
1736
1737 err = -EINVAL;
1738 if (size != sizeof(outarg))
1739 goto copy_finish;
1740
1741 err = fuse_copy_one(cs, &outarg, sizeof(outarg));
1742 if (err)
1743 goto copy_finish;
1744
1745 fuse_copy_finish(cs);
1746
1747 down_read(&fc->killsb);
1748 err = -ENOENT;
1749 if (fc->sb) {
1750 u64 nodeid = outarg.nodeid;
1751
1752 inode = ilookup5(fc->sb, nodeid, fuse_inode_eq, &nodeid);
1753 if (inode) {
1754 err = fuse_retrieve(fc, inode, &outarg);
1755 iput(inode);
1756 }
1757 }
1758 up_read(&fc->killsb);
1759
1760 return err;
1761
1762copy_finish:
1763 fuse_copy_finish(cs);
1764 return err;
1765}
1766
1767static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
1768 unsigned int size, struct fuse_copy_state *cs)
1769{
1770
1771 cs->move_pages = 0;
1772
1773 switch (code) {
1774 case FUSE_NOTIFY_POLL:
1775 return fuse_notify_poll(fc, size, cs);
1776
1777 case FUSE_NOTIFY_INVAL_INODE:
1778 return fuse_notify_inval_inode(fc, size, cs);
1779
1780 case FUSE_NOTIFY_INVAL_ENTRY:
1781 return fuse_notify_inval_entry(fc, size, cs);
1782
1783 case FUSE_NOTIFY_STORE:
1784 return fuse_notify_store(fc, size, cs);
1785
1786 case FUSE_NOTIFY_RETRIEVE:
1787 return fuse_notify_retrieve(fc, size, cs);
1788
1789 case FUSE_NOTIFY_DELETE:
1790 return fuse_notify_delete(fc, size, cs);
1791
1792 default:
1793 fuse_copy_finish(cs);
1794 return -EINVAL;
1795 }
1796}
1797
1798
1799static struct fuse_req *request_find(struct fuse_pqueue *fpq, u64 unique)
1800{
1801 struct fuse_req *req;
1802
1803 list_for_each_entry(req, &fpq->processing, list) {
1804 if (req->in.h.unique == unique || req->intr_unique == unique)
1805 return req;
1806 }
1807 return NULL;
1808}
1809
1810static int copy_out_args(struct fuse_copy_state *cs, struct fuse_out *out,
1811 unsigned nbytes)
1812{
1813 unsigned reqsize = sizeof(struct fuse_out_header);
1814
1815 if (out->h.error)
1816 return nbytes != reqsize ? -EINVAL : 0;
1817
1818 reqsize += len_args(out->numargs, out->args);
1819
1820 if (reqsize < nbytes || (reqsize > nbytes && !out->argvar))
1821 return -EINVAL;
1822 else if (reqsize > nbytes) {
1823 struct fuse_arg *lastarg = &out->args[out->numargs-1];
1824 unsigned diffsize = reqsize - nbytes;
1825 if (diffsize > lastarg->size)
1826 return -EINVAL;
1827 lastarg->size -= diffsize;
1828 }
1829 return fuse_copy_args(cs, out->numargs, out->argpages, out->args,
1830 out->page_zeroing);
1831}
1832
1833
1834
1835
1836
1837
1838
1839
1840static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
1841 struct fuse_copy_state *cs, size_t nbytes)
1842{
1843 int err;
1844 struct fuse_conn *fc = fud->fc;
1845 struct fuse_pqueue *fpq = &fud->pq;
1846 struct fuse_req *req;
1847 struct fuse_out_header oh;
1848
1849 if (nbytes < sizeof(struct fuse_out_header))
1850 return -EINVAL;
1851
1852 err = fuse_copy_one(cs, &oh, sizeof(oh));
1853 if (err)
1854 goto err_finish;
1855
1856 err = -EINVAL;
1857 if (oh.len != nbytes)
1858 goto err_finish;
1859
1860
1861
1862
1863
1864 if (!oh.unique) {
1865 err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
1866 return err ? err : nbytes;
1867 }
1868
1869 err = -EINVAL;
1870 if (oh.error <= -1000 || oh.error > 0)
1871 goto err_finish;
1872
1873 spin_lock(&fpq->lock);
1874 err = -ENOENT;
1875 if (!fpq->connected)
1876 goto err_unlock_pq;
1877
1878 req = request_find(fpq, oh.unique);
1879 if (!req)
1880 goto err_unlock_pq;
1881
1882
1883 if (req->intr_unique == oh.unique) {
1884 spin_unlock(&fpq->lock);
1885
1886 err = -EINVAL;
1887 if (nbytes != sizeof(struct fuse_out_header))
1888 goto err_finish;
1889
1890 if (oh.error == -ENOSYS)
1891 fc->no_interrupt = 1;
1892 else if (oh.error == -EAGAIN)
1893 queue_interrupt(&fc->iq, req);
1894
1895 fuse_copy_finish(cs);
1896 return nbytes;
1897 }
1898
1899 clear_bit(FR_SENT, &req->flags);
1900 list_move(&req->list, &fpq->io);
1901 req->out.h = oh;
1902 set_bit(FR_LOCKED, &req->flags);
1903 spin_unlock(&fpq->lock);
1904 cs->req = req;
1905 if (!req->out.page_replace)
1906 cs->move_pages = 0;
1907
1908 err = copy_out_args(cs, &req->out, nbytes);
1909 fuse_copy_finish(cs);
1910
1911 spin_lock(&fpq->lock);
1912 clear_bit(FR_LOCKED, &req->flags);
1913 if (!fpq->connected)
1914 err = -ENOENT;
1915 else if (err)
1916 req->out.h.error = -EIO;
1917 if (!test_bit(FR_PRIVATE, &req->flags))
1918 list_del_init(&req->list);
1919 spin_unlock(&fpq->lock);
1920
1921 request_end(fc, req);
1922
1923 return err ? err : nbytes;
1924
1925 err_unlock_pq:
1926 spin_unlock(&fpq->lock);
1927 err_finish:
1928 fuse_copy_finish(cs);
1929 return err;
1930}
1931
1932static ssize_t fuse_dev_write(struct kiocb *iocb, struct iov_iter *from)
1933{
1934 struct fuse_copy_state cs;
1935 struct fuse_dev *fud = fuse_get_dev(iocb->ki_filp);
1936
1937 if (!fud)
1938 return -EPERM;
1939
1940 if (!iter_is_iovec(from))
1941 return -EINVAL;
1942
1943 fuse_copy_init(&cs, 0, from);
1944
1945 return fuse_dev_do_write(fud, &cs, iov_iter_count(from));
1946}
1947
1948static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
1949 struct file *out, loff_t *ppos,
1950 size_t len, unsigned int flags)
1951{
1952 unsigned nbuf;
1953 unsigned idx;
1954 struct pipe_buffer *bufs;
1955 struct fuse_copy_state cs;
1956 struct fuse_dev *fud;
1957 size_t rem;
1958 ssize_t ret;
1959
1960 fud = fuse_get_dev(out);
1961 if (!fud)
1962 return -EPERM;
1963
1964 bufs = kmalloc(pipe->buffers * sizeof(struct pipe_buffer), GFP_KERNEL);
1965 if (!bufs)
1966 return -ENOMEM;
1967
1968 pipe_lock(pipe);
1969 nbuf = 0;
1970 rem = 0;
1971 for (idx = 0; idx < pipe->nrbufs && rem < len; idx++)
1972 rem += pipe->bufs[(pipe->curbuf + idx) & (pipe->buffers - 1)].len;
1973
1974 ret = -EINVAL;
1975 if (rem < len) {
1976 pipe_unlock(pipe);
1977 goto out;
1978 }
1979
1980 rem = len;
1981 while (rem) {
1982 struct pipe_buffer *ibuf;
1983 struct pipe_buffer *obuf;
1984
1985 BUG_ON(nbuf >= pipe->buffers);
1986 BUG_ON(!pipe->nrbufs);
1987 ibuf = &pipe->bufs[pipe->curbuf];
1988 obuf = &bufs[nbuf];
1989
1990 if (rem >= ibuf->len) {
1991 *obuf = *ibuf;
1992 ibuf->ops = NULL;
1993 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
1994 pipe->nrbufs--;
1995 } else {
1996 ibuf->ops->get(pipe, ibuf);
1997 *obuf = *ibuf;
1998 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1999 obuf->len = rem;
2000 ibuf->offset += obuf->len;
2001 ibuf->len -= obuf->len;
2002 }
2003 nbuf++;
2004 rem -= obuf->len;
2005 }
2006 pipe_unlock(pipe);
2007
2008 fuse_copy_init(&cs, 0, NULL);
2009 cs.pipebufs = bufs;
2010 cs.nr_segs = nbuf;
2011 cs.pipe = pipe;
2012
2013 if (flags & SPLICE_F_MOVE)
2014 cs.move_pages = 1;
2015
2016 ret = fuse_dev_do_write(fud, &cs, len);
2017
2018 for (idx = 0; idx < nbuf; idx++) {
2019 struct pipe_buffer *buf = &bufs[idx];
2020 buf->ops->release(pipe, buf);
2021 }
2022out:
2023 kfree(bufs);
2024 return ret;
2025}
2026
2027static unsigned fuse_dev_poll(struct file *file, poll_table *wait)
2028{
2029 unsigned mask = POLLOUT | POLLWRNORM;
2030 struct fuse_iqueue *fiq;
2031 struct fuse_dev *fud = fuse_get_dev(file);
2032
2033 if (!fud)
2034 return POLLERR;
2035
2036 fiq = &fud->fc->iq;
2037 poll_wait(file, &fiq->waitq, wait);
2038
2039 spin_lock(&fiq->waitq.lock);
2040 if (!fiq->connected)
2041 mask = POLLERR;
2042 else if (request_pending(fiq))
2043 mask |= POLLIN | POLLRDNORM;
2044 spin_unlock(&fiq->waitq.lock);
2045
2046 return mask;
2047}
2048
2049
2050
2051
2052
2053
2054static void end_requests(struct fuse_conn *fc, struct list_head *head)
2055{
2056 while (!list_empty(head)) {
2057 struct fuse_req *req;
2058 req = list_entry(head->next, struct fuse_req, list);
2059 req->out.h.error = -ECONNABORTED;
2060 clear_bit(FR_PENDING, &req->flags);
2061 clear_bit(FR_SENT, &req->flags);
2062 list_del_init(&req->list);
2063 request_end(fc, req);
2064 }
2065}
2066
2067static void end_polls(struct fuse_conn *fc)
2068{
2069 struct rb_node *p;
2070
2071 p = rb_first(&fc->polled_files);
2072
2073 while (p) {
2074 struct fuse_file *ff;
2075 ff = rb_entry(p, struct fuse_file, polled_node);
2076 wake_up_interruptible_all(&ff->poll_wait);
2077
2078 p = rb_next(p);
2079 }
2080}
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100void fuse_abort_conn(struct fuse_conn *fc)
2101{
2102 struct fuse_iqueue *fiq = &fc->iq;
2103
2104 spin_lock(&fc->lock);
2105 if (fc->connected) {
2106 struct fuse_dev *fud;
2107 struct fuse_req *req, *next;
2108 LIST_HEAD(to_end1);
2109 LIST_HEAD(to_end2);
2110
2111 fc->connected = 0;
2112 fc->blocked = 0;
2113 fuse_set_initialized(fc);
2114 list_for_each_entry(fud, &fc->devices, entry) {
2115 struct fuse_pqueue *fpq = &fud->pq;
2116
2117 spin_lock(&fpq->lock);
2118 fpq->connected = 0;
2119 list_for_each_entry_safe(req, next, &fpq->io, list) {
2120 req->out.h.error = -ECONNABORTED;
2121 spin_lock(&req->waitq.lock);
2122 set_bit(FR_ABORTED, &req->flags);
2123 if (!test_bit(FR_LOCKED, &req->flags)) {
2124 set_bit(FR_PRIVATE, &req->flags);
2125 list_move(&req->list, &to_end1);
2126 }
2127 spin_unlock(&req->waitq.lock);
2128 }
2129 list_splice_init(&fpq->processing, &to_end2);
2130 spin_unlock(&fpq->lock);
2131 }
2132 fc->max_background = UINT_MAX;
2133 flush_bg_queue(fc);
2134
2135 spin_lock(&fiq->waitq.lock);
2136 fiq->connected = 0;
2137 list_splice_init(&fiq->pending, &to_end2);
2138 while (forget_pending(fiq))
2139 kfree(dequeue_forget(fiq, 1, NULL));
2140 wake_up_all_locked(&fiq->waitq);
2141 spin_unlock(&fiq->waitq.lock);
2142 kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
2143 end_polls(fc);
2144 wake_up_all(&fc->blocked_waitq);
2145 spin_unlock(&fc->lock);
2146
2147 while (!list_empty(&to_end1)) {
2148 req = list_first_entry(&to_end1, struct fuse_req, list);
2149 __fuse_get_request(req);
2150 list_del_init(&req->list);
2151 request_end(fc, req);
2152 }
2153 end_requests(fc, &to_end2);
2154 } else {
2155 spin_unlock(&fc->lock);
2156 }
2157}
2158EXPORT_SYMBOL_GPL(fuse_abort_conn);
2159
2160int fuse_dev_release(struct inode *inode, struct file *file)
2161{
2162 struct fuse_dev *fud = fuse_get_dev(file);
2163
2164 if (fud) {
2165 struct fuse_conn *fc = fud->fc;
2166 struct fuse_pqueue *fpq = &fud->pq;
2167
2168 WARN_ON(!list_empty(&fpq->io));
2169 end_requests(fc, &fpq->processing);
2170
2171 if (atomic_dec_and_test(&fc->dev_count)) {
2172 WARN_ON(fc->iq.fasync != NULL);
2173 fuse_abort_conn(fc);
2174 }
2175 fuse_dev_free(fud);
2176 }
2177 return 0;
2178}
2179EXPORT_SYMBOL_GPL(fuse_dev_release);
2180
2181static int fuse_dev_fasync(int fd, struct file *file, int on)
2182{
2183 struct fuse_dev *fud = fuse_get_dev(file);
2184
2185 if (!fud)
2186 return -EPERM;
2187
2188
2189 return fasync_helper(fd, file, on, &fud->fc->iq.fasync);
2190}
2191
2192static int fuse_device_clone(struct fuse_conn *fc, struct file *new)
2193{
2194 struct fuse_dev *fud;
2195
2196 if (new->private_data)
2197 return -EINVAL;
2198
2199 fud = fuse_dev_alloc(fc);
2200 if (!fud)
2201 return -ENOMEM;
2202
2203 new->private_data = fud;
2204 atomic_inc(&fc->dev_count);
2205
2206 return 0;
2207}
2208
2209static long fuse_dev_ioctl(struct file *file, unsigned int cmd,
2210 unsigned long arg)
2211{
2212 int err = -ENOTTY;
2213
2214 if (cmd == FUSE_DEV_IOC_CLONE) {
2215 int oldfd;
2216
2217 err = -EFAULT;
2218 if (!get_user(oldfd, (__u32 __user *) arg)) {
2219 struct file *old = fget(oldfd);
2220
2221 err = -EINVAL;
2222 if (old) {
2223 struct fuse_dev *fud = NULL;
2224
2225
2226
2227
2228
2229 if (old->f_op == file->f_op &&
2230 old->f_cred->user_ns == file->f_cred->user_ns)
2231 fud = fuse_get_dev(old);
2232
2233 if (fud) {
2234 mutex_lock(&fuse_mutex);
2235 err = fuse_device_clone(fud->fc, file);
2236 mutex_unlock(&fuse_mutex);
2237 }
2238 fput(old);
2239 }
2240 }
2241 }
2242 return err;
2243}
2244
2245const struct file_operations fuse_dev_operations = {
2246 .owner = THIS_MODULE,
2247 .open = fuse_dev_open,
2248 .llseek = no_llseek,
2249 .read_iter = fuse_dev_read,
2250 .splice_read = fuse_dev_splice_read,
2251 .write_iter = fuse_dev_write,
2252 .splice_write = fuse_dev_splice_write,
2253 .poll = fuse_dev_poll,
2254 .release = fuse_dev_release,
2255 .fasync = fuse_dev_fasync,
2256 .unlocked_ioctl = fuse_dev_ioctl,
2257 .compat_ioctl = fuse_dev_ioctl,
2258};
2259EXPORT_SYMBOL_GPL(fuse_dev_operations);
2260
2261static struct miscdevice fuse_miscdevice = {
2262 .minor = FUSE_MINOR,
2263 .name = "fuse",
2264 .fops = &fuse_dev_operations,
2265};
2266
2267int __init fuse_dev_init(void)
2268{
2269 int err = -ENOMEM;
2270 fuse_req_cachep = kmem_cache_create("fuse_request",
2271 sizeof(struct fuse_req),
2272 0, 0, NULL);
2273 if (!fuse_req_cachep)
2274 goto out;
2275
2276 err = misc_register(&fuse_miscdevice);
2277 if (err)
2278 goto out_cache_clean;
2279
2280 return 0;
2281
2282 out_cache_clean:
2283 kmem_cache_destroy(fuse_req_cachep);
2284 out:
2285 return err;
2286}
2287
2288void fuse_dev_cleanup(void)
2289{
2290 misc_deregister(&fuse_miscdevice);
2291 kmem_cache_destroy(fuse_req_cachep);
2292}
2293
