1
2
3
4
5
6
7
8
9#include "fuse_i.h"
10
11#include <linux/pagemap.h>
12#include <linux/slab.h>
13#include <linux/kernel.h>
14#include <linux/sched.h>
15#include <linux/module.h>
16#include <linux/compat.h>
17#include <linux/swap.h>
18#include <linux/falloc.h>
19#include <linux/uio.h>
20
21static const struct file_operations fuse_direct_io_file_operations;
22
23static int fuse_send_open(struct fuse_conn *fc, u64 nodeid, struct file *file,
24 int opcode, struct fuse_open_out *outargp)
25{
26 struct fuse_open_in inarg;
27 FUSE_ARGS(args);
28
29 memset(&inarg, 0, sizeof(inarg));
30 inarg.flags = file->f_flags & ~(O_CREAT | O_EXCL | O_NOCTTY);
31 if (!fc->atomic_o_trunc)
32 inarg.flags &= ~O_TRUNC;
33 args.in.h.opcode = opcode;
34 args.in.h.nodeid = nodeid;
35 args.in.numargs = 1;
36 args.in.args[0].size = sizeof(inarg);
37 args.in.args[0].value = &inarg;
38 args.out.numargs = 1;
39 args.out.args[0].size = sizeof(*outargp);
40 args.out.args[0].value = outargp;
41
42 return fuse_simple_request(fc, &args);
43}
44
45struct fuse_file *fuse_file_alloc(struct fuse_conn *fc)
46{
47 struct fuse_file *ff;
48
49 ff = kzalloc(sizeof(struct fuse_file), GFP_KERNEL);
50 if (unlikely(!ff))
51 return NULL;
52
53 ff->fc = fc;
54 ff->reserved_req = fuse_request_alloc(0);
55 if (unlikely(!ff->reserved_req)) {
56 kfree(ff);
57 return NULL;
58 }
59
60 INIT_LIST_HEAD(&ff->write_entry);
61 refcount_set(&ff->count, 1);
62 RB_CLEAR_NODE(&ff->polled_node);
63 init_waitqueue_head(&ff->poll_wait);
64
65 spin_lock(&fc->lock);
66 ff->kh = ++fc->khctr;
67 spin_unlock(&fc->lock);
68
69 return ff;
70}
71
72void fuse_file_free(struct fuse_file *ff)
73{
74 fuse_request_free(ff->reserved_req);
75 kfree(ff);
76}
77
78static struct fuse_file *fuse_file_get(struct fuse_file *ff)
79{
80 refcount_inc(&ff->count);
81 return ff;
82}
83
84static void fuse_release_end(struct fuse_conn *fc, struct fuse_req *req)
85{
86 iput(req->misc.release.inode);
87}
88
89static void fuse_file_put(struct fuse_file *ff, bool sync)
90{
91 if (refcount_dec_and_test(&ff->count)) {
92 struct fuse_req *req = ff->reserved_req;
93
94 if (ff->fc->no_open) {
95
96
97
98
99 __clear_bit(FR_BACKGROUND, &req->flags);
100 iput(req->misc.release.inode);
101 fuse_put_request(ff->fc, req);
102 } else if (sync) {
103 __set_bit(FR_FORCE, &req->flags);
104 __clear_bit(FR_BACKGROUND, &req->flags);
105 fuse_request_send(ff->fc, req);
106 iput(req->misc.release.inode);
107 fuse_put_request(ff->fc, req);
108 } else {
109 req->end = fuse_release_end;
110 __set_bit(FR_BACKGROUND, &req->flags);
111 fuse_request_send_background(ff->fc, req);
112 }
113 kfree(ff);
114 }
115}
116
117int fuse_do_open(struct fuse_conn *fc, u64 nodeid, struct file *file,
118 bool isdir)
119{
120 struct fuse_file *ff;
121 int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN;
122
123 ff = fuse_file_alloc(fc);
124 if (!ff)
125 return -ENOMEM;
126
127 ff->fh = 0;
128 ff->open_flags = FOPEN_KEEP_CACHE;
129 if (!fc->no_open || isdir) {
130 struct fuse_open_out outarg;
131 int err;
132
133 err = fuse_send_open(fc, nodeid, file, opcode, &outarg);
134 if (!err) {
135 ff->fh = outarg.fh;
136 ff->open_flags = outarg.open_flags;
137
138 } else if (err != -ENOSYS || isdir) {
139 fuse_file_free(ff);
140 return err;
141 } else {
142 fc->no_open = 1;
143 }
144 }
145
146 if (isdir)
147 ff->open_flags &= ~FOPEN_DIRECT_IO;
148
149 ff->nodeid = nodeid;
150 file->private_data = ff;
151
152 return 0;
153}
154EXPORT_SYMBOL_GPL(fuse_do_open);
155
156static void fuse_link_write_file(struct file *file)
157{
158 struct inode *inode = file_inode(file);
159 struct fuse_conn *fc = get_fuse_conn(inode);
160 struct fuse_inode *fi = get_fuse_inode(inode);
161 struct fuse_file *ff = file->private_data;
162
163
164
165
166 spin_lock(&fc->lock);
167 if (list_empty(&ff->write_entry))
168 list_add(&ff->write_entry, &fi->write_files);
169 spin_unlock(&fc->lock);
170}
171
172void fuse_finish_open(struct inode *inode, struct file *file)
173{
174 struct fuse_file *ff = file->private_data;
175 struct fuse_conn *fc = get_fuse_conn(inode);
176
177 if (ff->open_flags & FOPEN_DIRECT_IO)
178 file->f_op = &fuse_direct_io_file_operations;
179 if (!(ff->open_flags & FOPEN_KEEP_CACHE))
180 invalidate_inode_pages2(inode->i_mapping);
181 if (ff->open_flags & FOPEN_NONSEEKABLE)
182 nonseekable_open(inode, file);
183 if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) {
184 struct fuse_inode *fi = get_fuse_inode(inode);
185
186 spin_lock(&fc->lock);
187 fi->attr_version = ++fc->attr_version;
188 i_size_write(inode, 0);
189 spin_unlock(&fc->lock);
190 fuse_invalidate_attr(inode);
191 if (fc->writeback_cache)
192 file_update_time(file);
193 }
194 if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache)
195 fuse_link_write_file(file);
196}
197
198int fuse_open_common(struct inode *inode, struct file *file, bool isdir)
199{
200 struct fuse_conn *fc = get_fuse_conn(inode);
201 int err;
202 bool lock_inode = (file->f_flags & O_TRUNC) &&
203 fc->atomic_o_trunc &&
204 fc->writeback_cache;
205
206 err = generic_file_open(inode, file);
207 if (err)
208 return err;
209
210 if (lock_inode)
211 inode_lock(inode);
212
213 err = fuse_do_open(fc, get_node_id(inode), file, isdir);
214
215 if (!err)
216 fuse_finish_open(inode, file);
217
218 if (lock_inode)
219 inode_unlock(inode);
220
221 return err;
222}
223
224static void fuse_prepare_release(struct fuse_file *ff, int flags, int opcode)
225{
226 struct fuse_conn *fc = ff->fc;
227 struct fuse_req *req = ff->reserved_req;
228 struct fuse_release_in *inarg = &req->misc.release.in;
229
230 spin_lock(&fc->lock);
231 list_del(&ff->write_entry);
232 if (!RB_EMPTY_NODE(&ff->polled_node))
233 rb_erase(&ff->polled_node, &fc->polled_files);
234 spin_unlock(&fc->lock);
235
236 wake_up_interruptible_all(&ff->poll_wait);
237
238 inarg->fh = ff->fh;
239 inarg->flags = flags;
240 req->in.h.opcode = opcode;
241 req->in.h.nodeid = ff->nodeid;
242 req->in.numargs = 1;
243 req->in.args[0].size = sizeof(struct fuse_release_in);
244 req->in.args[0].value = inarg;
245}
246
247void fuse_release_common(struct file *file, int opcode)
248{
249 struct fuse_file *ff = file->private_data;
250 struct fuse_req *req = ff->reserved_req;
251
252 fuse_prepare_release(ff, file->f_flags, opcode);
253
254 if (ff->flock) {
255 struct fuse_release_in *inarg = &req->misc.release.in;
256 inarg->release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
257 inarg->lock_owner = fuse_lock_owner_id(ff->fc,
258 (fl_owner_t) file);
259 }
260
261 req->misc.release.inode = igrab(file_inode(file));
262
263
264
265
266
267
268
269
270
271
272 fuse_file_put(ff, ff->fc->destroy_req != NULL);
273}
274
275static int fuse_open(struct inode *inode, struct file *file)
276{
277 return fuse_open_common(inode, file, false);
278}
279
280static int fuse_release(struct inode *inode, struct file *file)
281{
282 struct fuse_conn *fc = get_fuse_conn(inode);
283
284
285 if (fc->writeback_cache)
286 write_inode_now(inode, 1);
287
288 fuse_release_common(file, FUSE_RELEASE);
289
290
291 return 0;
292}
293
294void fuse_sync_release(struct fuse_file *ff, int flags)
295{
296 WARN_ON(refcount_read(&ff->count) > 1);
297 fuse_prepare_release(ff, flags, FUSE_RELEASE);
298
299
300
301
302 fuse_file_put(ff, true);
303}
304EXPORT_SYMBOL_GPL(fuse_sync_release);
305
306
307
308
309
310u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id)
311{
312 u32 *k = fc->scramble_key;
313 u64 v = (unsigned long) id;
314 u32 v0 = v;
315 u32 v1 = v >> 32;
316 u32 sum = 0;
317 int i;
318
319 for (i = 0; i < 32; i++) {
320 v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]);
321 sum += 0x9E3779B9;
322 v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]);
323 }
324
325 return (u64) v0 + ((u64) v1 << 32);
326}
327
328
329
330
331
332
333
334static bool fuse_range_is_writeback(struct inode *inode, pgoff_t idx_from,
335 pgoff_t idx_to)
336{
337 struct fuse_conn *fc = get_fuse_conn(inode);
338 struct fuse_inode *fi = get_fuse_inode(inode);
339 struct fuse_req *req;
340 bool found = false;
341
342 spin_lock(&fc->lock);
343 list_for_each_entry(req, &fi->writepages, writepages_entry) {
344 pgoff_t curr_index;
345
346 BUG_ON(req->inode != inode);
347 curr_index = req->misc.write.in.offset >> PAGE_SHIFT;
348 if (idx_from < curr_index + req->num_pages &&
349 curr_index <= idx_to) {
350 found = true;
351 break;
352 }
353 }
354 spin_unlock(&fc->lock);
355
356 return found;
357}
358
359static inline bool fuse_page_is_writeback(struct inode *inode, pgoff_t index)
360{
361 return fuse_range_is_writeback(inode, index, index);
362}
363
364
365
366
367
368
369
370static int fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index)
371{
372 struct fuse_inode *fi = get_fuse_inode(inode);
373
374 wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index));
375 return 0;
376}
377
378
379
380
381
382
383
384
385
386
387static void fuse_sync_writes(struct inode *inode)
388{
389 fuse_set_nowrite(inode);
390 fuse_release_nowrite(inode);
391}
392
393static int fuse_flush(struct file *file, fl_owner_t id)
394{
395 struct inode *inode = file_inode(file);
396 struct fuse_conn *fc = get_fuse_conn(inode);
397 struct fuse_file *ff = file->private_data;
398 struct fuse_req *req;
399 struct fuse_flush_in inarg;
400 int err;
401
402 if (is_bad_inode(inode))
403 return -EIO;
404
405 if (fc->no_flush)
406 return 0;
407
408 err = write_inode_now(inode, 1);
409 if (err)
410 return err;
411
412 inode_lock(inode);
413 fuse_sync_writes(inode);
414 inode_unlock(inode);
415
416 err = filemap_check_errors(file->f_mapping);
417 if (err)
418 return err;
419
420 req = fuse_get_req_nofail_nopages(fc, file);
421 memset(&inarg, 0, sizeof(inarg));
422 inarg.fh = ff->fh;
423 inarg.lock_owner = fuse_lock_owner_id(fc, id);
424 req->in.h.opcode = FUSE_FLUSH;
425 req->in.h.nodeid = get_node_id(inode);
426 req->in.numargs = 1;
427 req->in.args[0].size = sizeof(inarg);
428 req->in.args[0].value = &inarg;
429 __set_bit(FR_FORCE, &req->flags);
430 fuse_request_send(fc, req);
431 err = req->out.h.error;
432 fuse_put_request(fc, req);
433 if (err == -ENOSYS) {
434 fc->no_flush = 1;
435 err = 0;
436 }
437 return err;
438}
439
440int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
441 int datasync, int isdir)
442{
443 struct inode *inode = file->f_mapping->host;
444 struct fuse_conn *fc = get_fuse_conn(inode);
445 struct fuse_file *ff = file->private_data;
446 FUSE_ARGS(args);
447 struct fuse_fsync_in inarg;
448 int err;
449
450 if (is_bad_inode(inode))
451 return -EIO;
452
453 inode_lock(inode);
454
455
456
457
458
459
460 err = file_write_and_wait_range(file, start, end);
461 if (err)
462 goto out;
463
464 fuse_sync_writes(inode);
465
466
467
468
469
470
471 err = file_check_and_advance_wb_err(file);
472 if (err)
473 goto out;
474
475 err = sync_inode_metadata(inode, 1);
476 if (err)
477 goto out;
478
479 if ((!isdir && fc->no_fsync) || (isdir && fc->no_fsyncdir))
480 goto out;
481
482 memset(&inarg, 0, sizeof(inarg));
483 inarg.fh = ff->fh;
484 inarg.fsync_flags = datasync ? 1 : 0;
485 args.in.h.opcode = isdir ? FUSE_FSYNCDIR : FUSE_FSYNC;
486 args.in.h.nodeid = get_node_id(inode);
487 args.in.numargs = 1;
488 args.in.args[0].size = sizeof(inarg);
489 args.in.args[0].value = &inarg;
490 err = fuse_simple_request(fc, &args);
491 if (err == -ENOSYS) {
492 if (isdir)
493 fc->no_fsyncdir = 1;
494 else
495 fc->no_fsync = 1;
496 err = 0;
497 }
498out:
499 inode_unlock(inode);
500 return err;
501}
502
503static int fuse_fsync(struct file *file, loff_t start, loff_t end,
504 int datasync)
505{
506 return fuse_fsync_common(file, start, end, datasync, 0);
507}
508
509void fuse_read_fill(struct fuse_req *req, struct file *file, loff_t pos,
510 size_t count, int opcode)
511{
512 struct fuse_read_in *inarg = &req->misc.read.in;
513 struct fuse_file *ff = file->private_data;
514
515 inarg->fh = ff->fh;
516 inarg->offset = pos;
517 inarg->size = count;
518 inarg->flags = file->f_flags;
519 req->in.h.opcode = opcode;
520 req->in.h.nodeid = ff->nodeid;
521 req->in.numargs = 1;
522 req->in.args[0].size = sizeof(struct fuse_read_in);
523 req->in.args[0].value = inarg;
524 req->out.argvar = 1;
525 req->out.numargs = 1;
526 req->out.args[0].size = count;
527}
528
529static void fuse_release_user_pages(struct fuse_req *req, bool should_dirty)
530{
531 unsigned i;
532
533 for (i = 0; i < req->num_pages; i++) {
534 struct page *page = req->pages[i];
535 if (should_dirty)
536 set_page_dirty_lock(page);
537 put_page(page);
538 }
539}
540
541static void fuse_io_release(struct kref *kref)
542{
543 kfree(container_of(kref, struct fuse_io_priv, refcnt));
544}
545
546static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)
547{
548 if (io->err)
549 return io->err;
550
551 if (io->bytes >= 0 && io->write)
552 return -EIO;
553
554 return io->bytes < 0 ? io->size : io->bytes;
555}
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
574{
575 int left;
576
577 spin_lock(&io->lock);
578 if (err)
579 io->err = io->err ? : err;
580 else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))
581 io->bytes = pos;
582
583 left = --io->reqs;
584 if (!left && io->blocking)
585 complete(io->done);
586 spin_unlock(&io->lock);
587
588 if (!left && !io->blocking) {
589 ssize_t res = fuse_get_res_by_io(io);
590
591 if (res >= 0) {
592 struct inode *inode = file_inode(io->iocb->ki_filp);
593 struct fuse_conn *fc = get_fuse_conn(inode);
594 struct fuse_inode *fi = get_fuse_inode(inode);
595
596 spin_lock(&fc->lock);
597 fi->attr_version = ++fc->attr_version;
598 spin_unlock(&fc->lock);
599 }
600
601 io->iocb->ki_complete(io->iocb, res, 0);
602 }
603
604 kref_put(&io->refcnt, fuse_io_release);
605}
606
607static void fuse_aio_complete_req(struct fuse_conn *fc, struct fuse_req *req)
608{
609 struct fuse_io_priv *io = req->io;
610 ssize_t pos = -1;
611
612 fuse_release_user_pages(req, io->should_dirty);
613
614 if (io->write) {
615 if (req->misc.write.in.size != req->misc.write.out.size)
616 pos = req->misc.write.in.offset - io->offset +
617 req->misc.write.out.size;
618 } else {
619 if (req->misc.read.in.size != req->out.args[0].size)
620 pos = req->misc.read.in.offset - io->offset +
621 req->out.args[0].size;
622 }
623
624 fuse_aio_complete(io, req->out.h.error, pos);
625}
626
627static size_t fuse_async_req_send(struct fuse_conn *fc, struct fuse_req *req,
628 size_t num_bytes, struct fuse_io_priv *io)
629{
630 spin_lock(&io->lock);
631 kref_get(&io->refcnt);
632 io->size += num_bytes;
633 io->reqs++;
634 spin_unlock(&io->lock);
635
636 req->io = io;
637 req->end = fuse_aio_complete_req;
638
639 __fuse_get_request(req);
640 fuse_request_send_background(fc, req);
641
642 return num_bytes;
643}
644
645static size_t fuse_send_read(struct fuse_req *req, struct fuse_io_priv *io,
646 loff_t pos, size_t count, fl_owner_t owner)
647{
648 struct file *file = io->iocb->ki_filp;
649 struct fuse_file *ff = file->private_data;
650 struct fuse_conn *fc = ff->fc;
651
652 fuse_read_fill(req, file, pos, count, FUSE_READ);
653 if (owner != NULL) {
654 struct fuse_read_in *inarg = &req->misc.read.in;
655
656 inarg->read_flags |= FUSE_READ_LOCKOWNER;
657 inarg->lock_owner = fuse_lock_owner_id(fc, owner);
658 }
659
660 if (io->async)
661 return fuse_async_req_send(fc, req, count, io);
662
663 fuse_request_send(fc, req);
664 return req->out.args[0].size;
665}
666
667static void fuse_read_update_size(struct inode *inode, loff_t size,
668 u64 attr_ver)
669{
670 struct fuse_conn *fc = get_fuse_conn(inode);
671 struct fuse_inode *fi = get_fuse_inode(inode);
672
673 spin_lock(&fc->lock);
674 if (attr_ver == fi->attr_version && size < inode->i_size &&
675 !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) {
676 fi->attr_version = ++fc->attr_version;
677 i_size_write(inode, size);
678 }
679 spin_unlock(&fc->lock);
680}
681
682static void fuse_short_read(struct fuse_req *req, struct inode *inode,
683 u64 attr_ver)
684{
685 size_t num_read = req->out.args[0].size;
686 struct fuse_conn *fc = get_fuse_conn(inode);
687
688 if (fc->writeback_cache) {
689
690
691
692
693
694 int i;
695 int start_idx = num_read >> PAGE_SHIFT;
696 size_t off = num_read & (PAGE_SIZE - 1);
697
698 for (i = start_idx; i < req->num_pages; i++) {
699 zero_user_segment(req->pages[i], off, PAGE_SIZE);
700 off = 0;
701 }
702 } else {
703 loff_t pos = page_offset(req->pages[0]) + num_read;
704 fuse_read_update_size(inode, pos, attr_ver);
705 }
706}
707
708static int fuse_do_readpage(struct file *file, struct page *page)
709{
710 struct kiocb iocb;
711 struct fuse_io_priv io;
712 struct inode *inode = page->mapping->host;
713 struct fuse_conn *fc = get_fuse_conn(inode);
714 struct fuse_req *req;
715 size_t num_read;
716 loff_t pos = page_offset(page);
717 size_t count = PAGE_SIZE;
718 u64 attr_ver;
719 int err;
720
721
722
723
724
725
726 fuse_wait_on_page_writeback(inode, page->index);
727
728 req = fuse_get_req(fc, 1);
729 if (IS_ERR(req))
730 return PTR_ERR(req);
731
732 attr_ver = fuse_get_attr_version(fc);
733
734 req->out.page_zeroing = 1;
735 req->out.argpages = 1;
736 req->num_pages = 1;
737 req->pages[0] = page;
738 req->page_descs[0].length = count;
739 init_sync_kiocb(&iocb, file);
740 io = (struct fuse_io_priv) FUSE_IO_PRIV_SYNC(&iocb);
741 num_read = fuse_send_read(req, &io, pos, count, NULL);
742 err = req->out.h.error;
743
744 if (!err) {
745
746
747
748 if (num_read < count)
749 fuse_short_read(req, inode, attr_ver);
750
751 SetPageUptodate(page);
752 }
753
754 fuse_put_request(fc, req);
755
756 return err;
757}
758
759static int fuse_readpage(struct file *file, struct page *page)
760{
761 struct inode *inode = page->mapping->host;
762 int err;
763
764 err = -EIO;
765 if (is_bad_inode(inode))
766 goto out;
767
768 err = fuse_do_readpage(file, page);
769 fuse_invalidate_atime(inode);
770 out:
771 unlock_page(page);
772 return err;
773}
774
775static void fuse_readpages_end(struct fuse_conn *fc, struct fuse_req *req)
776{
777 int i;
778 size_t count = req->misc.read.in.size;
779 size_t num_read = req->out.args[0].size;
780 struct address_space *mapping = NULL;
781
782 for (i = 0; mapping == NULL && i < req->num_pages; i++)
783 mapping = req->pages[i]->mapping;
784
785 if (mapping) {
786 struct inode *inode = mapping->host;
787
788
789
790
791 if (!req->out.h.error && num_read < count)
792 fuse_short_read(req, inode, req->misc.read.attr_ver);
793
794 fuse_invalidate_atime(inode);
795 }
796
797 for (i = 0; i < req->num_pages; i++) {
798 struct page *page = req->pages[i];
799 if (!req->out.h.error)
800 SetPageUptodate(page);
801 else
802 SetPageError(page);
803 unlock_page(page);
804 put_page(page);
805 }
806 if (req->ff)
807 fuse_file_put(req->ff, false);
808}
809
810static void fuse_send_readpages(struct fuse_req *req, struct file *file)
811{
812 struct fuse_file *ff = file->private_data;
813 struct fuse_conn *fc = ff->fc;
814 loff_t pos = page_offset(req->pages[0]);
815 size_t count = req->num_pages << PAGE_SHIFT;
816
817 req->out.argpages = 1;
818 req->out.page_zeroing = 1;
819 req->out.page_replace = 1;
820 fuse_read_fill(req, file, pos, count, FUSE_READ);
821 req->misc.read.attr_ver = fuse_get_attr_version(fc);
822 if (fc->async_read) {
823 req->ff = fuse_file_get(ff);
824 req->end = fuse_readpages_end;
825 fuse_request_send_background(fc, req);
826 } else {
827 fuse_request_send(fc, req);
828 fuse_readpages_end(fc, req);
829 fuse_put_request(fc, req);
830 }
831}
832
833struct fuse_fill_data {
834 struct fuse_req *req;
835 struct file *file;
836 struct inode *inode;
837 unsigned nr_pages;
838};
839
840static int fuse_readpages_fill(void *_data, struct page *page)
841{
842 struct fuse_fill_data *data = _data;
843 struct fuse_req *req = data->req;
844 struct inode *inode = data->inode;
845 struct fuse_conn *fc = get_fuse_conn(inode);
846
847 fuse_wait_on_page_writeback(inode, page->index);
848
849 if (req->num_pages &&
850 (req->num_pages == FUSE_MAX_PAGES_PER_REQ ||
851 (req->num_pages + 1) * PAGE_SIZE > fc->max_read ||
852 req->pages[req->num_pages - 1]->index + 1 != page->index)) {
853 int nr_alloc = min_t(unsigned, data->nr_pages,
854 FUSE_MAX_PAGES_PER_REQ);
855 fuse_send_readpages(req, data->file);
856 if (fc->async_read)
857 req = fuse_get_req_for_background(fc, nr_alloc);
858 else
859 req = fuse_get_req(fc, nr_alloc);
860
861 data->req = req;
862 if (IS_ERR(req)) {
863 unlock_page(page);
864 return PTR_ERR(req);
865 }
866 }
867
868 if (WARN_ON(req->num_pages >= req->max_pages)) {
869 fuse_put_request(fc, req);
870 return -EIO;
871 }
872
873 get_page(page);
874 req->pages[req->num_pages] = page;
875 req->page_descs[req->num_pages].length = PAGE_SIZE;
876 req->num_pages++;
877 data->nr_pages--;
878 return 0;
879}
880
881static int fuse_readpages(struct file *file, struct address_space *mapping,
882 struct list_head *pages, unsigned nr_pages)
883{
884 struct inode *inode = mapping->host;
885 struct fuse_conn *fc = get_fuse_conn(inode);
886 struct fuse_fill_data data;
887 int err;
888 int nr_alloc = min_t(unsigned, nr_pages, FUSE_MAX_PAGES_PER_REQ);
889
890 err = -EIO;
891 if (is_bad_inode(inode))
892 goto out;
893
894 data.file = file;
895 data.inode = inode;
896 if (fc->async_read)
897 data.req = fuse_get_req_for_background(fc, nr_alloc);
898 else
899 data.req = fuse_get_req(fc, nr_alloc);
900 data.nr_pages = nr_pages;
901 err = PTR_ERR(data.req);
902 if (IS_ERR(data.req))
903 goto out;
904
905 err = read_cache_pages(mapping, pages, fuse_readpages_fill, &data);
906 if (!err) {
907 if (data.req->num_pages)
908 fuse_send_readpages(data.req, file);
909 else
910 fuse_put_request(fc, data.req);
911 }
912out:
913 return err;
914}
915
916static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
917{
918 struct inode *inode = iocb->ki_filp->f_mapping->host;
919 struct fuse_conn *fc = get_fuse_conn(inode);
920
921
922
923
924
925
926 if (fc->auto_inval_data ||
927 (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) {
928 int err;
929 err = fuse_update_attributes(inode, iocb->ki_filp);
930 if (err)
931 return err;
932 }
933
934 return generic_file_read_iter(iocb, to);
935}
936
937static void fuse_write_fill(struct fuse_req *req, struct fuse_file *ff,
938 loff_t pos, size_t count)
939{
940 struct fuse_write_in *inarg = &req->misc.write.in;
941 struct fuse_write_out *outarg = &req->misc.write.out;
942
943 inarg->fh = ff->fh;
944 inarg->offset = pos;
945 inarg->size = count;
946 req->in.h.opcode = FUSE_WRITE;
947 req->in.h.nodeid = ff->nodeid;
948 req->in.numargs = 2;
949 if (ff->fc->minor < 9)
950 req->in.args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
951 else
952 req->in.args[0].size = sizeof(struct fuse_write_in);
953 req->in.args[0].value = inarg;
954 req->in.args[1].size = count;
955 req->out.numargs = 1;
956 req->out.args[0].size = sizeof(struct fuse_write_out);
957 req->out.args[0].value = outarg;
958}
959
960static size_t fuse_send_write(struct fuse_req *req, struct fuse_io_priv *io,
961 loff_t pos, size_t count, fl_owner_t owner)
962{
963 struct kiocb *iocb = io->iocb;
964 struct file *file = iocb->ki_filp;
965 struct fuse_file *ff = file->private_data;
966 struct fuse_conn *fc = ff->fc;
967 struct fuse_write_in *inarg = &req->misc.write.in;
968
969 fuse_write_fill(req, ff, pos, count);
970 inarg->flags = file->f_flags;
971 if (iocb->ki_flags & IOCB_DSYNC)
972 inarg->flags |= O_DSYNC;
973 if (iocb->ki_flags & IOCB_SYNC)
974 inarg->flags |= O_SYNC;
975 if (owner != NULL) {
976 inarg->write_flags |= FUSE_WRITE_LOCKOWNER;
977 inarg->lock_owner = fuse_lock_owner_id(fc, owner);
978 }
979
980 if (io->async)
981 return fuse_async_req_send(fc, req, count, io);
982
983 fuse_request_send(fc, req);
984 return req->misc.write.out.size;
985}
986
987bool fuse_write_update_size(struct inode *inode, loff_t pos)
988{
989 struct fuse_conn *fc = get_fuse_conn(inode);
990 struct fuse_inode *fi = get_fuse_inode(inode);
991 bool ret = false;
992
993 spin_lock(&fc->lock);
994 fi->attr_version = ++fc->attr_version;
995 if (pos > inode->i_size) {
996 i_size_write(inode, pos);
997 ret = true;
998 }
999 spin_unlock(&fc->lock);
1000
1001 return ret;
1002}
1003
1004static size_t fuse_send_write_pages(struct fuse_req *req, struct kiocb *iocb,
1005 struct inode *inode, loff_t pos,
1006 size_t count)
1007{
1008 size_t res;
1009 unsigned offset;
1010 unsigned i;
1011 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1012
1013 for (i = 0; i < req->num_pages; i++)
1014 fuse_wait_on_page_writeback(inode, req->pages[i]->index);
1015
1016 res = fuse_send_write(req, &io, pos, count, NULL);
1017
1018 offset = req->page_descs[0].offset;
1019 count = res;
1020 for (i = 0; i < req->num_pages; i++) {
1021 struct page *page = req->pages[i];
1022
1023 if (!req->out.h.error && !offset && count >= PAGE_SIZE)
1024 SetPageUptodate(page);
1025
1026 if (count > PAGE_SIZE - offset)
1027 count -= PAGE_SIZE - offset;
1028 else
1029 count = 0;
1030 offset = 0;
1031
1032 unlock_page(page);
1033 put_page(page);
1034 }
1035
1036 return res;
1037}
1038
1039static ssize_t fuse_fill_write_pages(struct fuse_req *req,
1040 struct address_space *mapping,
1041 struct iov_iter *ii, loff_t pos)
1042{
1043 struct fuse_conn *fc = get_fuse_conn(mapping->host);
1044 unsigned offset = pos & (PAGE_SIZE - 1);
1045 size_t count = 0;
1046 int err;
1047
1048 req->in.argpages = 1;
1049 req->page_descs[0].offset = offset;
1050
1051 do {
1052 size_t tmp;
1053 struct page *page;
1054 pgoff_t index = pos >> PAGE_SHIFT;
1055 size_t bytes = min_t(size_t, PAGE_SIZE - offset,
1056 iov_iter_count(ii));
1057
1058 bytes = min_t(size_t, bytes, fc->max_write - count);
1059
1060 again:
1061 err = -EFAULT;
1062 if (iov_iter_fault_in_readable(ii, bytes))
1063 break;
1064
1065 err = -ENOMEM;
1066 page = grab_cache_page_write_begin(mapping, index, 0);
1067 if (!page)
1068 break;
1069
1070 if (mapping_writably_mapped(mapping))
1071 flush_dcache_page(page);
1072
1073 tmp = iov_iter_copy_from_user_atomic(page, ii, offset, bytes);
1074 flush_dcache_page(page);
1075
1076 iov_iter_advance(ii, tmp);
1077 if (!tmp) {
1078 unlock_page(page);
1079 put_page(page);
1080 bytes = min(bytes, iov_iter_single_seg_count(ii));
1081 goto again;
1082 }
1083
1084 err = 0;
1085 req->pages[req->num_pages] = page;
1086 req->page_descs[req->num_pages].length = tmp;
1087 req->num_pages++;
1088
1089 count += tmp;
1090 pos += tmp;
1091 offset += tmp;
1092 if (offset == PAGE_SIZE)
1093 offset = 0;
1094
1095 if (!fc->big_writes)
1096 break;
1097 } while (iov_iter_count(ii) && count < fc->max_write &&
1098 req->num_pages < req->max_pages && offset == 0);
1099
1100 return count > 0 ? count : err;
1101}
1102
1103static inline unsigned fuse_wr_pages(loff_t pos, size_t len)
1104{
1105 return min_t(unsigned,
1106 ((pos + len - 1) >> PAGE_SHIFT) -
1107 (pos >> PAGE_SHIFT) + 1,
1108 FUSE_MAX_PAGES_PER_REQ);
1109}
1110
1111static ssize_t fuse_perform_write(struct kiocb *iocb,
1112 struct address_space *mapping,
1113 struct iov_iter *ii, loff_t pos)
1114{
1115 struct inode *inode = mapping->host;
1116 struct fuse_conn *fc = get_fuse_conn(inode);
1117 struct fuse_inode *fi = get_fuse_inode(inode);
1118 int err = 0;
1119 ssize_t res = 0;
1120
1121 if (is_bad_inode(inode))
1122 return -EIO;
1123
1124 if (inode->i_size < pos + iov_iter_count(ii))
1125 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1126
1127 do {
1128 struct fuse_req *req;
1129 ssize_t count;
1130 unsigned nr_pages = fuse_wr_pages(pos, iov_iter_count(ii));
1131
1132 req = fuse_get_req(fc, nr_pages);
1133 if (IS_ERR(req)) {
1134 err = PTR_ERR(req);
1135 break;
1136 }
1137
1138 count = fuse_fill_write_pages(req, mapping, ii, pos);
1139 if (count <= 0) {
1140 err = count;
1141 } else {
1142 size_t num_written;
1143
1144 num_written = fuse_send_write_pages(req, iocb, inode,
1145 pos, count);
1146 err = req->out.h.error;
1147 if (!err) {
1148 res += num_written;
1149 pos += num_written;
1150
1151
1152 if (num_written != count)
1153 err = -EIO;
1154 }
1155 }
1156 fuse_put_request(fc, req);
1157 } while (!err && iov_iter_count(ii));
1158
1159 if (res > 0)
1160 fuse_write_update_size(inode, pos);
1161
1162 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1163 fuse_invalidate_attr(inode);
1164
1165 return res > 0 ? res : err;
1166}
1167
1168static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
1169{
1170 struct file *file = iocb->ki_filp;
1171 struct address_space *mapping = file->f_mapping;
1172 ssize_t written = 0;
1173 ssize_t written_buffered = 0;
1174 struct inode *inode = mapping->host;
1175 ssize_t err;
1176 loff_t endbyte = 0;
1177
1178 if (get_fuse_conn(inode)->writeback_cache) {
1179
1180 err = fuse_update_attributes(mapping->host, file);
1181 if (err)
1182 return err;
1183
1184 return generic_file_write_iter(iocb, from);
1185 }
1186
1187 inode_lock(inode);
1188
1189
1190 current->backing_dev_info = inode_to_bdi(inode);
1191
1192 err = generic_write_checks(iocb, from);
1193 if (err <= 0)
1194 goto out;
1195
1196 err = file_remove_privs(file);
1197 if (err)
1198 goto out;
1199
1200 err = file_update_time(file);
1201 if (err)
1202 goto out;
1203
1204 if (iocb->ki_flags & IOCB_DIRECT) {
1205 loff_t pos = iocb->ki_pos;
1206 written = generic_file_direct_write(iocb, from);
1207 if (written < 0 || !iov_iter_count(from))
1208 goto out;
1209
1210 pos += written;
1211
1212 written_buffered = fuse_perform_write(iocb, mapping, from, pos);
1213 if (written_buffered < 0) {
1214 err = written_buffered;
1215 goto out;
1216 }
1217 endbyte = pos + written_buffered - 1;
1218
1219 err = filemap_write_and_wait_range(file->f_mapping, pos,
1220 endbyte);
1221 if (err)
1222 goto out;
1223
1224 invalidate_mapping_pages(file->f_mapping,
1225 pos >> PAGE_SHIFT,
1226 endbyte >> PAGE_SHIFT);
1227
1228 written += written_buffered;
1229 iocb->ki_pos = pos + written_buffered;
1230 } else {
1231 written = fuse_perform_write(iocb, mapping, from, iocb->ki_pos);
1232 if (written >= 0)
1233 iocb->ki_pos += written;
1234 }
1235out:
1236 current->backing_dev_info = NULL;
1237 inode_unlock(inode);
1238 if (written > 0)
1239 written = generic_write_sync(iocb, written);
1240
1241 return written ? written : err;
1242}
1243
1244static inline void fuse_page_descs_length_init(struct fuse_req *req,
1245 unsigned index, unsigned nr_pages)
1246{
1247 int i;
1248
1249 for (i = index; i < index + nr_pages; i++)
1250 req->page_descs[i].length = PAGE_SIZE -
1251 req->page_descs[i].offset;
1252}
1253
1254static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
1255{
1256 return (unsigned long)ii->iov->iov_base + ii->iov_offset;
1257}
1258
1259static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
1260 size_t max_size)
1261{
1262 return min(iov_iter_single_seg_count(ii), max_size);
1263}
1264
1265static int fuse_get_user_pages(struct fuse_req *req, struct iov_iter *ii,
1266 size_t *nbytesp, int write)
1267{
1268 size_t nbytes = 0;
1269 ssize_t ret = 0;
1270
1271
1272 if (ii->type & ITER_KVEC) {
1273 unsigned long user_addr = fuse_get_user_addr(ii);
1274 size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
1275
1276 if (write)
1277 req->in.args[1].value = (void *) user_addr;
1278 else
1279 req->out.args[0].value = (void *) user_addr;
1280
1281 iov_iter_advance(ii, frag_size);
1282 *nbytesp = frag_size;
1283 return 0;
1284 }
1285
1286 while (nbytes < *nbytesp && req->num_pages < req->max_pages) {
1287 unsigned npages;
1288 size_t start;
1289 ret = iov_iter_get_pages(ii, &req->pages[req->num_pages],
1290 *nbytesp - nbytes,
1291 req->max_pages - req->num_pages,
1292 &start);
1293 if (ret < 0)
1294 break;
1295
1296 iov_iter_advance(ii, ret);
1297 nbytes += ret;
1298
1299 ret += start;
1300 npages = (ret + PAGE_SIZE - 1) / PAGE_SIZE;
1301
1302 req->page_descs[req->num_pages].offset = start;
1303 fuse_page_descs_length_init(req, req->num_pages, npages);
1304
1305 req->num_pages += npages;
1306 req->page_descs[req->num_pages - 1].length -=
1307 (PAGE_SIZE - ret) & (PAGE_SIZE - 1);
1308 }
1309
1310 if (write)
1311 req->in.argpages = 1;
1312 else
1313 req->out.argpages = 1;
1314
1315 *nbytesp = nbytes;
1316
1317 return ret < 0 ? ret : 0;
1318}
1319
1320static inline int fuse_iter_npages(const struct iov_iter *ii_p)
1321{
1322 return iov_iter_npages(ii_p, FUSE_MAX_PAGES_PER_REQ);
1323}
1324
1325ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
1326 loff_t *ppos, int flags)
1327{
1328 int write = flags & FUSE_DIO_WRITE;
1329 int cuse = flags & FUSE_DIO_CUSE;
1330 struct file *file = io->iocb->ki_filp;
1331 struct inode *inode = file->f_mapping->host;
1332 struct fuse_file *ff = file->private_data;
1333 struct fuse_conn *fc = ff->fc;
1334 size_t nmax = write ? fc->max_write : fc->max_read;
1335 loff_t pos = *ppos;
1336 size_t count = iov_iter_count(iter);
1337 pgoff_t idx_from = pos >> PAGE_SHIFT;
1338 pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT;
1339 ssize_t res = 0;
1340 struct fuse_req *req;
1341 int err = 0;
1342
1343 if (io->async)
1344 req = fuse_get_req_for_background(fc, fuse_iter_npages(iter));
1345 else
1346 req = fuse_get_req(fc, fuse_iter_npages(iter));
1347 if (IS_ERR(req))
1348 return PTR_ERR(req);
1349
1350 if (!cuse && fuse_range_is_writeback(inode, idx_from, idx_to)) {
1351 if (!write)
1352 inode_lock(inode);
1353 fuse_sync_writes(inode);
1354 if (!write)
1355 inode_unlock(inode);
1356 }
1357
1358 io->should_dirty = !write && iter_is_iovec(iter);
1359 while (count) {
1360 size_t nres;
1361 fl_owner_t owner = current->files;
1362 size_t nbytes = min(count, nmax);
1363 err = fuse_get_user_pages(req, iter, &nbytes, write);
1364 if (err && !nbytes)
1365 break;
1366
1367 if (write)
1368 nres = fuse_send_write(req, io, pos, nbytes, owner);
1369 else
1370 nres = fuse_send_read(req, io, pos, nbytes, owner);
1371
1372 if (!io->async)
1373 fuse_release_user_pages(req, io->should_dirty);
1374 if (req->out.h.error) {
1375 err = req->out.h.error;
1376 break;
1377 } else if (nres > nbytes) {
1378 res = 0;
1379 err = -EIO;
1380 break;
1381 }
1382 count -= nres;
1383 res += nres;
1384 pos += nres;
1385 if (nres != nbytes)
1386 break;
1387 if (count) {
1388 fuse_put_request(fc, req);
1389 if (io->async)
1390 req = fuse_get_req_for_background(fc,
1391 fuse_iter_npages(iter));
1392 else
1393 req = fuse_get_req(fc, fuse_iter_npages(iter));
1394 if (IS_ERR(req))
1395 break;
1396 }
1397 }
1398 if (!IS_ERR(req))
1399 fuse_put_request(fc, req);
1400 if (res > 0)
1401 *ppos = pos;
1402
1403 return res > 0 ? res : err;
1404}
1405EXPORT_SYMBOL_GPL(fuse_direct_io);
1406
1407static ssize_t __fuse_direct_read(struct fuse_io_priv *io,
1408 struct iov_iter *iter,
1409 loff_t *ppos)
1410{
1411 ssize_t res;
1412 struct inode *inode = file_inode(io->iocb->ki_filp);
1413
1414 if (is_bad_inode(inode))
1415 return -EIO;
1416
1417 res = fuse_direct_io(io, iter, ppos, 0);
1418
1419 fuse_invalidate_attr(inode);
1420
1421 return res;
1422}
1423
1424static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to)
1425{
1426 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1427 return __fuse_direct_read(&io, to, &iocb->ki_pos);
1428}
1429
1430static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
1431{
1432 struct inode *inode = file_inode(iocb->ki_filp);
1433 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1434 ssize_t res;
1435
1436 if (is_bad_inode(inode))
1437 return -EIO;
1438
1439
1440 inode_lock(inode);
1441 res = generic_write_checks(iocb, from);
1442 if (res > 0)
1443 res = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE);
1444 fuse_invalidate_attr(inode);
1445 if (res > 0)
1446 fuse_write_update_size(inode, iocb->ki_pos);
1447 inode_unlock(inode);
1448
1449 return res;
1450}
1451
1452static void fuse_writepage_free(struct fuse_conn *fc, struct fuse_req *req)
1453{
1454 int i;
1455
1456 for (i = 0; i < req->num_pages; i++)
1457 __free_page(req->pages[i]);
1458
1459 if (req->ff)
1460 fuse_file_put(req->ff, false);
1461}
1462
1463static void fuse_writepage_finish(struct fuse_conn *fc, struct fuse_req *req)
1464{
1465 struct inode *inode = req->inode;
1466 struct fuse_inode *fi = get_fuse_inode(inode);
1467 struct backing_dev_info *bdi = inode_to_bdi(inode);
1468 int i;
1469
1470 list_del(&req->writepages_entry);
1471 for (i = 0; i < req->num_pages; i++) {
1472 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
1473 dec_node_page_state(req->pages[i], NR_WRITEBACK_TEMP);
1474 wb_writeout_inc(&bdi->wb);
1475 }
1476 wake_up(&fi->page_waitq);
1477}
1478
1479
1480static void fuse_send_writepage(struct fuse_conn *fc, struct fuse_req *req,
1481 loff_t size)
1482__releases(fc->lock)
1483__acquires(fc->lock)
1484{
1485 struct fuse_inode *fi = get_fuse_inode(req->inode);
1486 struct fuse_write_in *inarg = &req->misc.write.in;
1487 __u64 data_size = req->num_pages * PAGE_SIZE;
1488
1489 if (!fc->connected)
1490 goto out_free;
1491
1492 if (inarg->offset + data_size <= size) {
1493 inarg->size = data_size;
1494 } else if (inarg->offset < size) {
1495 inarg->size = size - inarg->offset;
1496 } else {
1497
1498 goto out_free;
1499 }
1500
1501 req->in.args[1].size = inarg->size;
1502 fi->writectr++;
1503 fuse_request_send_background_locked(fc, req);
1504 return;
1505
1506 out_free:
1507 fuse_writepage_finish(fc, req);
1508 spin_unlock(&fc->lock);
1509 fuse_writepage_free(fc, req);
1510 fuse_put_request(fc, req);
1511 spin_lock(&fc->lock);
1512}
1513
1514
1515
1516
1517
1518
1519
1520void fuse_flush_writepages(struct inode *inode)
1521__releases(fc->lock)
1522__acquires(fc->lock)
1523{
1524 struct fuse_conn *fc = get_fuse_conn(inode);
1525 struct fuse_inode *fi = get_fuse_inode(inode);
1526 size_t crop = i_size_read(inode);
1527 struct fuse_req *req;
1528
1529 while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) {
1530 req = list_entry(fi->queued_writes.next, struct fuse_req, list);
1531 list_del_init(&req->list);
1532 fuse_send_writepage(fc, req, crop);
1533 }
1534}
1535
1536static void fuse_writepage_end(struct fuse_conn *fc, struct fuse_req *req)
1537{
1538 struct inode *inode = req->inode;
1539 struct fuse_inode *fi = get_fuse_inode(inode);
1540
1541 mapping_set_error(inode->i_mapping, req->out.h.error);
1542 spin_lock(&fc->lock);
1543 while (req->misc.write.next) {
1544 struct fuse_conn *fc = get_fuse_conn(inode);
1545 struct fuse_write_in *inarg = &req->misc.write.in;
1546 struct fuse_req *next = req->misc.write.next;
1547 req->misc.write.next = next->misc.write.next;
1548 next->misc.write.next = NULL;
1549 next->ff = fuse_file_get(req->ff);
1550 list_add(&next->writepages_entry, &fi->writepages);
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575 fuse_send_writepage(fc, next, inarg->offset + inarg->size);
1576 }
1577 fi->writectr--;
1578 fuse_writepage_finish(fc, req);
1579 spin_unlock(&fc->lock);
1580 fuse_writepage_free(fc, req);
1581}
1582
1583static struct fuse_file *__fuse_write_file_get(struct fuse_conn *fc,
1584 struct fuse_inode *fi)
1585{
1586 struct fuse_file *ff = NULL;
1587
1588 spin_lock(&fc->lock);
1589 if (!list_empty(&fi->write_files)) {
1590 ff = list_entry(fi->write_files.next, struct fuse_file,
1591 write_entry);
1592 fuse_file_get(ff);
1593 }
1594 spin_unlock(&fc->lock);
1595
1596 return ff;
1597}
1598
1599static struct fuse_file *fuse_write_file_get(struct fuse_conn *fc,
1600 struct fuse_inode *fi)
1601{
1602 struct fuse_file *ff = __fuse_write_file_get(fc, fi);
1603 WARN_ON(!ff);
1604 return ff;
1605}
1606
1607int fuse_write_inode(struct inode *inode, struct writeback_control *wbc)
1608{
1609 struct fuse_conn *fc = get_fuse_conn(inode);
1610 struct fuse_inode *fi = get_fuse_inode(inode);
1611 struct fuse_file *ff;
1612 int err;
1613
1614 ff = __fuse_write_file_get(fc, fi);
1615 err = fuse_flush_times(inode, ff);
1616 if (ff)
1617 fuse_file_put(ff, 0);
1618
1619 return err;
1620}
1621
1622static int fuse_writepage_locked(struct page *page)
1623{
1624 struct address_space *mapping = page->mapping;
1625 struct inode *inode = mapping->host;
1626 struct fuse_conn *fc = get_fuse_conn(inode);
1627 struct fuse_inode *fi = get_fuse_inode(inode);
1628 struct fuse_req *req;
1629 struct page *tmp_page;
1630 int error = -ENOMEM;
1631
1632 set_page_writeback(page);
1633
1634 req = fuse_request_alloc_nofs(1);
1635 if (!req)
1636 goto err;
1637
1638
1639 __set_bit(FR_BACKGROUND, &req->flags);
1640 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
1641 if (!tmp_page)
1642 goto err_free;
1643
1644 error = -EIO;
1645 req->ff = fuse_write_file_get(fc, fi);
1646 if (!req->ff)
1647 goto err_nofile;
1648
1649 fuse_write_fill(req, req->ff, page_offset(page), 0);
1650
1651 copy_highpage(tmp_page, page);
1652 req->misc.write.in.write_flags |= FUSE_WRITE_CACHE;
1653 req->misc.write.next = NULL;
1654 req->in.argpages = 1;
1655 req->num_pages = 1;
1656 req->pages[0] = tmp_page;
1657 req->page_descs[0].offset = 0;
1658 req->page_descs[0].length = PAGE_SIZE;
1659 req->end = fuse_writepage_end;
1660 req->inode = inode;
1661
1662 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
1663 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
1664
1665 spin_lock(&fc->lock);
1666 list_add(&req->writepages_entry, &fi->writepages);
1667 list_add_tail(&req->list, &fi->queued_writes);
1668 fuse_flush_writepages(inode);
1669 spin_unlock(&fc->lock);
1670
1671 end_page_writeback(page);
1672
1673 return 0;
1674
1675err_nofile:
1676 __free_page(tmp_page);
1677err_free:
1678 fuse_request_free(req);
1679err:
1680 mapping_set_error(page->mapping, error);
1681 end_page_writeback(page);
1682 return error;
1683}
1684
1685static int fuse_writepage(struct page *page, struct writeback_control *wbc)
1686{
1687 int err;
1688
1689 if (fuse_page_is_writeback(page->mapping->host, page->index)) {
1690
1691
1692
1693
1694
1695 WARN_ON(wbc->sync_mode == WB_SYNC_ALL);
1696
1697 redirty_page_for_writepage(wbc, page);
1698 return 0;
1699 }
1700
1701 err = fuse_writepage_locked(page);
1702 unlock_page(page);
1703
1704 return err;
1705}
1706
1707struct fuse_fill_wb_data {
1708 struct fuse_req *req;
1709 struct fuse_file *ff;
1710 struct inode *inode;
1711 struct page **orig_pages;
1712};
1713
1714static void fuse_writepages_send(struct fuse_fill_wb_data *data)
1715{
1716 struct fuse_req *req = data->req;
1717 struct inode *inode = data->inode;
1718 struct fuse_conn *fc = get_fuse_conn(inode);
1719 struct fuse_inode *fi = get_fuse_inode(inode);
1720 int num_pages = req->num_pages;
1721 int i;
1722
1723 req->ff = fuse_file_get(data->ff);
1724 spin_lock(&fc->lock);
1725 list_add_tail(&req->list, &fi->queued_writes);
1726 fuse_flush_writepages(inode);
1727 spin_unlock(&fc->lock);
1728
1729 for (i = 0; i < num_pages; i++)
1730 end_page_writeback(data->orig_pages[i]);
1731}
1732
1733static bool fuse_writepage_in_flight(struct fuse_req *new_req,
1734 struct page *page)
1735{
1736 struct fuse_conn *fc = get_fuse_conn(new_req->inode);
1737 struct fuse_inode *fi = get_fuse_inode(new_req->inode);
1738 struct fuse_req *tmp;
1739 struct fuse_req *old_req;
1740 bool found = false;
1741 pgoff_t curr_index;
1742
1743 BUG_ON(new_req->num_pages != 0);
1744
1745 spin_lock(&fc->lock);
1746 list_del(&new_req->writepages_entry);
1747 list_for_each_entry(old_req, &fi->writepages, writepages_entry) {
1748 BUG_ON(old_req->inode != new_req->inode);
1749 curr_index = old_req->misc.write.in.offset >> PAGE_SHIFT;
1750 if (curr_index <= page->index &&
1751 page->index < curr_index + old_req->num_pages) {
1752 found = true;
1753 break;
1754 }
1755 }
1756 if (!found) {
1757 list_add(&new_req->writepages_entry, &fi->writepages);
1758 goto out_unlock;
1759 }
1760
1761 new_req->num_pages = 1;
1762 for (tmp = old_req; tmp != NULL; tmp = tmp->misc.write.next) {
1763 BUG_ON(tmp->inode != new_req->inode);
1764 curr_index = tmp->misc.write.in.offset >> PAGE_SHIFT;
1765 if (tmp->num_pages == 1 &&
1766 curr_index == page->index) {
1767 old_req = tmp;
1768 }
1769 }
1770
1771 if (old_req->num_pages == 1 && test_bit(FR_PENDING, &old_req->flags)) {
1772 struct backing_dev_info *bdi = inode_to_bdi(page->mapping->host);
1773
1774 copy_highpage(old_req->pages[0], page);
1775 spin_unlock(&fc->lock);
1776
1777 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
1778 dec_node_page_state(page, NR_WRITEBACK_TEMP);
1779 wb_writeout_inc(&bdi->wb);
1780 fuse_writepage_free(fc, new_req);
1781 fuse_request_free(new_req);
1782 goto out;
1783 } else {
1784 new_req->misc.write.next = old_req->misc.write.next;
1785 old_req->misc.write.next = new_req;
1786 }
1787out_unlock:
1788 spin_unlock(&fc->lock);
1789out:
1790 return found;
1791}
1792
1793static int fuse_writepages_fill(struct page *page,
1794 struct writeback_control *wbc, void *_data)
1795{
1796 struct fuse_fill_wb_data *data = _data;
1797 struct fuse_req *req = data->req;
1798 struct inode *inode = data->inode;
1799 struct fuse_conn *fc = get_fuse_conn(inode);
1800 struct page *tmp_page;
1801 bool is_writeback;
1802 int err;
1803
1804 if (!data->ff) {
1805 err = -EIO;
1806 data->ff = fuse_write_file_get(fc, get_fuse_inode(inode));
1807 if (!data->ff)
1808 goto out_unlock;
1809 }
1810
1811
1812
1813
1814
1815
1816
1817 is_writeback = fuse_page_is_writeback(inode, page->index);
1818
1819 if (req && req->num_pages &&
1820 (is_writeback || req->num_pages == FUSE_MAX_PAGES_PER_REQ ||
1821 (req->num_pages + 1) * PAGE_SIZE > fc->max_write ||
1822 data->orig_pages[req->num_pages - 1]->index + 1 != page->index)) {
1823 fuse_writepages_send(data);
1824 data->req = NULL;
1825 }
1826 err = -ENOMEM;
1827 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
1828 if (!tmp_page)
1829 goto out_unlock;
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844 if (data->req == NULL) {
1845 struct fuse_inode *fi = get_fuse_inode(inode);
1846
1847 err = -ENOMEM;
1848 req = fuse_request_alloc_nofs(FUSE_MAX_PAGES_PER_REQ);
1849 if (!req) {
1850 __free_page(tmp_page);
1851 goto out_unlock;
1852 }
1853
1854 fuse_write_fill(req, data->ff, page_offset(page), 0);
1855 req->misc.write.in.write_flags |= FUSE_WRITE_CACHE;
1856 req->misc.write.next = NULL;
1857 req->in.argpages = 1;
1858 __set_bit(FR_BACKGROUND, &req->flags);
1859 req->num_pages = 0;
1860 req->end = fuse_writepage_end;
1861 req->inode = inode;
1862
1863 spin_lock(&fc->lock);
1864 list_add(&req->writepages_entry, &fi->writepages);
1865 spin_unlock(&fc->lock);
1866
1867 data->req = req;
1868 }
1869 set_page_writeback(page);
1870
1871 copy_highpage(tmp_page, page);
1872 req->pages[req->num_pages] = tmp_page;
1873 req->page_descs[req->num_pages].offset = 0;
1874 req->page_descs[req->num_pages].length = PAGE_SIZE;
1875
1876 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
1877 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
1878
1879 err = 0;
1880 if (is_writeback && fuse_writepage_in_flight(req, page)) {
1881 end_page_writeback(page);
1882 data->req = NULL;
1883 goto out_unlock;
1884 }
1885 data->orig_pages[req->num_pages] = page;
1886
1887
1888
1889
1890
1891 spin_lock(&fc->lock);
1892 req->num_pages++;
1893 spin_unlock(&fc->lock);
1894
1895out_unlock:
1896 unlock_page(page);
1897
1898 return err;
1899}
1900
1901static int fuse_writepages(struct address_space *mapping,
1902 struct writeback_control *wbc)
1903{
1904 struct inode *inode = mapping->host;
1905 struct fuse_fill_wb_data data;
1906 int err;
1907
1908 err = -EIO;
1909 if (is_bad_inode(inode))
1910 goto out;
1911
1912 data.inode = inode;
1913 data.req = NULL;
1914 data.ff = NULL;
1915
1916 err = -ENOMEM;
1917 data.orig_pages = kcalloc(FUSE_MAX_PAGES_PER_REQ,
1918 sizeof(struct page *),
1919 GFP_NOFS);
1920 if (!data.orig_pages)
1921 goto out;
1922
1923 err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data);
1924 if (data.req) {
1925
1926 BUG_ON(!data.req->num_pages);
1927 fuse_writepages_send(&data);
1928 err = 0;
1929 }
1930 if (data.ff)
1931 fuse_file_put(data.ff, false);
1932
1933 kfree(data.orig_pages);
1934out:
1935 return err;
1936}
1937
1938
1939
1940
1941
1942static int fuse_write_begin(struct file *file, struct address_space *mapping,
1943 loff_t pos, unsigned len, unsigned flags,
1944 struct page **pagep, void **fsdata)
1945{
1946 pgoff_t index = pos >> PAGE_SHIFT;
1947 struct fuse_conn *fc = get_fuse_conn(file_inode(file));
1948 struct page *page;
1949 loff_t fsize;
1950 int err = -ENOMEM;
1951
1952 WARN_ON(!fc->writeback_cache);
1953
1954 page = grab_cache_page_write_begin(mapping, index, flags);
1955 if (!page)
1956 goto error;
1957
1958 fuse_wait_on_page_writeback(mapping->host, page->index);
1959
1960 if (PageUptodate(page) || len == PAGE_SIZE)
1961 goto success;
1962
1963
1964
1965
1966 fsize = i_size_read(mapping->host);
1967 if (fsize <= (pos & PAGE_MASK)) {
1968 size_t off = pos & ~PAGE_MASK;
1969 if (off)
1970 zero_user_segment(page, 0, off);
1971 goto success;
1972 }
1973 err = fuse_do_readpage(file, page);
1974 if (err)
1975 goto cleanup;
1976success:
1977 *pagep = page;
1978 return 0;
1979
1980cleanup:
1981 unlock_page(page);
1982 put_page(page);
1983error:
1984 return err;
1985}
1986
1987static int fuse_write_end(struct file *file, struct address_space *mapping,
1988 loff_t pos, unsigned len, unsigned copied,
1989 struct page *page, void *fsdata)
1990{
1991 struct inode *inode = page->mapping->host;
1992
1993
1994 if (!copied)
1995 goto unlock;
1996
1997 if (!PageUptodate(page)) {
1998
1999 size_t endoff = (pos + copied) & ~PAGE_MASK;
2000 if (endoff)
2001 zero_user_segment(page, endoff, PAGE_SIZE);
2002 SetPageUptodate(page);
2003 }
2004
2005 fuse_write_update_size(inode, pos + copied);
2006 set_page_dirty(page);
2007
2008unlock:
2009 unlock_page(page);
2010 put_page(page);
2011
2012 return copied;
2013}
2014
2015static int fuse_launder_page(struct page *page)
2016{
2017 int err = 0;
2018 if (clear_page_dirty_for_io(page)) {
2019 struct inode *inode = page->mapping->host;
2020 err = fuse_writepage_locked(page);
2021 if (!err)
2022 fuse_wait_on_page_writeback(inode, page->index);
2023 }
2024 return err;
2025}
2026
2027
2028
2029
2030
2031static void fuse_vma_close(struct vm_area_struct *vma)
2032{
2033 filemap_write_and_wait(vma->vm_file->f_mapping);
2034}
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051static int fuse_page_mkwrite(struct vm_fault *vmf)
2052{
2053 struct page *page = vmf->page;
2054 struct inode *inode = file_inode(vmf->vma->vm_file);
2055
2056 file_update_time(vmf->vma->vm_file);
2057 lock_page(page);
2058 if (page->mapping != inode->i_mapping) {
2059 unlock_page(page);
2060 return VM_FAULT_NOPAGE;
2061 }
2062
2063 fuse_wait_on_page_writeback(inode, page->index);
2064 return VM_FAULT_LOCKED;
2065}
2066
2067static const struct vm_operations_struct fuse_file_vm_ops = {
2068 .close = fuse_vma_close,
2069 .fault = filemap_fault,
2070 .map_pages = filemap_map_pages,
2071 .page_mkwrite = fuse_page_mkwrite,
2072};
2073
2074static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
2075{
2076 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2077 fuse_link_write_file(file);
2078
2079 file_accessed(file);
2080 vma->vm_ops = &fuse_file_vm_ops;
2081 return 0;
2082}
2083
2084static int fuse_direct_mmap(struct file *file, struct vm_area_struct *vma)
2085{
2086
2087 if (vma->vm_flags & VM_MAYSHARE)
2088 return -ENODEV;
2089
2090 invalidate_inode_pages2(file->f_mapping);
2091
2092 return generic_file_mmap(file, vma);
2093}
2094
2095static int convert_fuse_file_lock(struct fuse_conn *fc,
2096 const struct fuse_file_lock *ffl,
2097 struct file_lock *fl)
2098{
2099 switch (ffl->type) {
2100 case F_UNLCK:
2101 break;
2102
2103 case F_RDLCK:
2104 case F_WRLCK:
2105 if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX ||
2106 ffl->end < ffl->start)
2107 return -EIO;
2108
2109 fl->fl_start = ffl->start;
2110 fl->fl_end = ffl->end;
2111
2112
2113
2114
2115
2116 rcu_read_lock();
2117 fl->fl_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns);
2118 rcu_read_unlock();
2119 break;
2120
2121 default:
2122 return -EIO;
2123 }
2124 fl->fl_type = ffl->type;
2125 return 0;
2126}
2127
2128static void fuse_lk_fill(struct fuse_args *args, struct file *file,
2129 const struct file_lock *fl, int opcode, pid_t pid,
2130 int flock, struct fuse_lk_in *inarg)
2131{
2132 struct inode *inode = file_inode(file);
2133 struct fuse_conn *fc = get_fuse_conn(inode);
2134 struct fuse_file *ff = file->private_data;
2135
2136 memset(inarg, 0, sizeof(*inarg));
2137 inarg->fh = ff->fh;
2138 inarg->owner = fuse_lock_owner_id(fc, fl->fl_owner);
2139 inarg->lk.start = fl->fl_start;
2140 inarg->lk.end = fl->fl_end;
2141 inarg->lk.type = fl->fl_type;
2142 inarg->lk.pid = pid;
2143 if (flock)
2144 inarg->lk_flags |= FUSE_LK_FLOCK;
2145 args->in.h.opcode = opcode;
2146 args->in.h.nodeid = get_node_id(inode);
2147 args->in.numargs = 1;
2148 args->in.args[0].size = sizeof(*inarg);
2149 args->in.args[0].value = inarg;
2150}
2151
2152static int fuse_getlk(struct file *file, struct file_lock *fl)
2153{
2154 struct inode *inode = file_inode(file);
2155 struct fuse_conn *fc = get_fuse_conn(inode);
2156 FUSE_ARGS(args);
2157 struct fuse_lk_in inarg;
2158 struct fuse_lk_out outarg;
2159 int err;
2160
2161 fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg);
2162 args.out.numargs = 1;
2163 args.out.args[0].size = sizeof(outarg);
2164 args.out.args[0].value = &outarg;
2165 err = fuse_simple_request(fc, &args);
2166 if (!err)
2167 err = convert_fuse_file_lock(fc, &outarg.lk, fl);
2168
2169 return err;
2170}
2171
2172static int fuse_setlk(struct file *file, struct file_lock *fl, int flock)
2173{
2174 struct inode *inode = file_inode(file);
2175 struct fuse_conn *fc = get_fuse_conn(inode);
2176 FUSE_ARGS(args);
2177 struct fuse_lk_in inarg;
2178 int opcode = (fl->fl_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK;
2179 struct pid *pid = fl->fl_type != F_UNLCK ? task_tgid(current) : NULL;
2180 pid_t pid_nr = pid_nr_ns(pid, fc->pid_ns);
2181 int err;
2182
2183 if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
2184
2185 return -ENOLCK;
2186 }
2187
2188
2189 if ((fl->fl_flags & FL_CLOSE_POSIX) == FL_CLOSE_POSIX)
2190 return 0;
2191
2192 fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg);
2193 err = fuse_simple_request(fc, &args);
2194
2195
2196 if (err == -EINTR)
2197 err = -ERESTARTSYS;
2198
2199 return err;
2200}
2201
2202static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl)
2203{
2204 struct inode *inode = file_inode(file);
2205 struct fuse_conn *fc = get_fuse_conn(inode);
2206 int err;
2207
2208 if (cmd == F_CANCELLK) {
2209 err = 0;
2210 } else if (cmd == F_GETLK) {
2211 if (fc->no_lock) {
2212 posix_test_lock(file, fl);
2213 err = 0;
2214 } else
2215 err = fuse_getlk(file, fl);
2216 } else {
2217 if (fc->no_lock)
2218 err = posix_lock_file(file, fl, NULL);
2219 else
2220 err = fuse_setlk(file, fl, 0);
2221 }
2222 return err;
2223}
2224
2225static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl)
2226{
2227 struct inode *inode = file_inode(file);
2228 struct fuse_conn *fc = get_fuse_conn(inode);
2229 int err;
2230
2231 if (fc->no_flock) {
2232 err = locks_lock_file_wait(file, fl);
2233 } else {
2234 struct fuse_file *ff = file->private_data;
2235
2236
2237 ff->flock = true;
2238 err = fuse_setlk(file, fl, 1);
2239 }
2240
2241 return err;
2242}
2243
2244static sector_t fuse_bmap(struct address_space *mapping, sector_t block)
2245{
2246 struct inode *inode = mapping->host;
2247 struct fuse_conn *fc = get_fuse_conn(inode);
2248 FUSE_ARGS(args);
2249 struct fuse_bmap_in inarg;
2250 struct fuse_bmap_out outarg;
2251 int err;
2252
2253 if (!inode->i_sb->s_bdev || fc->no_bmap)
2254 return 0;
2255
2256 memset(&inarg, 0, sizeof(inarg));
2257 inarg.block = block;
2258 inarg.blocksize = inode->i_sb->s_blocksize;
2259 args.in.h.opcode = FUSE_BMAP;
2260 args.in.h.nodeid = get_node_id(inode);
2261 args.in.numargs = 1;
2262 args.in.args[0].size = sizeof(inarg);
2263 args.in.args[0].value = &inarg;
2264 args.out.numargs = 1;
2265 args.out.args[0].size = sizeof(outarg);
2266 args.out.args[0].value = &outarg;
2267 err = fuse_simple_request(fc, &args);
2268 if (err == -ENOSYS)
2269 fc->no_bmap = 1;
2270
2271 return err ? 0 : outarg.block;
2272}
2273
2274static loff_t fuse_lseek(struct file *file, loff_t offset, int whence)
2275{
2276 struct inode *inode = file->f_mapping->host;
2277 struct fuse_conn *fc = get_fuse_conn(inode);
2278 struct fuse_file *ff = file->private_data;
2279 FUSE_ARGS(args);
2280 struct fuse_lseek_in inarg = {
2281 .fh = ff->fh,
2282 .offset = offset,
2283 .whence = whence
2284 };
2285 struct fuse_lseek_out outarg;
2286 int err;
2287
2288 if (fc->no_lseek)
2289 goto fallback;
2290
2291 args.in.h.opcode = FUSE_LSEEK;
2292 args.in.h.nodeid = ff->nodeid;
2293 args.in.numargs = 1;
2294 args.in.args[0].size = sizeof(inarg);
2295 args.in.args[0].value = &inarg;
2296 args.out.numargs = 1;
2297 args.out.args[0].size = sizeof(outarg);
2298 args.out.args[0].value = &outarg;
2299 err = fuse_simple_request(fc, &args);
2300 if (err) {
2301 if (err == -ENOSYS) {
2302 fc->no_lseek = 1;
2303 goto fallback;
2304 }
2305 return err;
2306 }
2307
2308 return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes);
2309
2310fallback:
2311 err = fuse_update_attributes(inode, file);
2312 if (!err)
2313 return generic_file_llseek(file, offset, whence);
2314 else
2315 return err;
2316}
2317
2318static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
2319{
2320 loff_t retval;
2321 struct inode *inode = file_inode(file);
2322
2323 switch (whence) {
2324 case SEEK_SET:
2325 case SEEK_CUR:
2326
2327 retval = generic_file_llseek(file, offset, whence);
2328 break;
2329 case SEEK_END:
2330 inode_lock(inode);
2331 retval = fuse_update_attributes(inode, file);
2332 if (!retval)
2333 retval = generic_file_llseek(file, offset, whence);
2334 inode_unlock(inode);
2335 break;
2336 case SEEK_HOLE:
2337 case SEEK_DATA:
2338 inode_lock(inode);
2339 retval = fuse_lseek(file, offset, whence);
2340 inode_unlock(inode);
2341 break;
2342 default:
2343 retval = -EINVAL;
2344 }
2345
2346 return retval;
2347}
2348
2349
2350
2351
2352
2353
2354
2355static int fuse_copy_ioctl_iovec_old(struct iovec *dst, void *src,
2356 size_t transferred, unsigned count,
2357 bool is_compat)
2358{
2359#ifdef CONFIG_COMPAT
2360 if (count * sizeof(struct compat_iovec) == transferred) {
2361 struct compat_iovec *ciov = src;
2362 unsigned i;
2363
2364
2365
2366
2367
2368
2369 if (!is_compat)
2370 return -EINVAL;
2371
2372 for (i = 0; i < count; i++) {
2373 dst[i].iov_base = compat_ptr(ciov[i].iov_base);
2374 dst[i].iov_len = ciov[i].iov_len;
2375 }
2376 return 0;
2377 }
2378#endif
2379
2380 if (count * sizeof(struct iovec) != transferred)
2381 return -EIO;
2382
2383 memcpy(dst, src, transferred);
2384 return 0;
2385}
2386
2387
2388static int fuse_verify_ioctl_iov(struct iovec *iov, size_t count)
2389{
2390 size_t n;
2391 u32 max = FUSE_MAX_PAGES_PER_REQ << PAGE_SHIFT;
2392
2393 for (n = 0; n < count; n++, iov++) {
2394 if (iov->iov_len > (size_t) max)
2395 return -ENOMEM;
2396 max -= iov->iov_len;
2397 }
2398 return 0;
2399}
2400
2401static int fuse_copy_ioctl_iovec(struct fuse_conn *fc, struct iovec *dst,
2402 void *src, size_t transferred, unsigned count,
2403 bool is_compat)
2404{
2405 unsigned i;
2406 struct fuse_ioctl_iovec *fiov = src;
2407
2408 if (fc->minor < 16) {
2409 return fuse_copy_ioctl_iovec_old(dst, src, transferred,
2410 count, is_compat);
2411 }
2412
2413 if (count * sizeof(struct fuse_ioctl_iovec) != transferred)
2414 return -EIO;
2415
2416 for (i = 0; i < count; i++) {
2417
2418 if (fiov[i].base != (unsigned long) fiov[i].base ||
2419 fiov[i].len != (unsigned long) fiov[i].len)
2420 return -EIO;
2421
2422 dst[i].iov_base = (void __user *) (unsigned long) fiov[i].base;
2423 dst[i].iov_len = (size_t) fiov[i].len;
2424
2425#ifdef CONFIG_COMPAT
2426 if (is_compat &&
2427 (ptr_to_compat(dst[i].iov_base) != fiov[i].base ||
2428 (compat_size_t) dst[i].iov_len != fiov[i].len))
2429 return -EIO;
2430#endif
2431 }
2432
2433 return 0;
2434}
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
2484 unsigned int flags)
2485{
2486 struct fuse_file *ff = file->private_data;
2487 struct fuse_conn *fc = ff->fc;
2488 struct fuse_ioctl_in inarg = {
2489 .fh = ff->fh,
2490 .cmd = cmd,
2491 .arg = arg,
2492 .flags = flags
2493 };
2494 struct fuse_ioctl_out outarg;
2495 struct fuse_req *req = NULL;
2496 struct page **pages = NULL;
2497 struct iovec *iov_page = NULL;
2498 struct iovec *in_iov = NULL, *out_iov = NULL;
2499 unsigned int in_iovs = 0, out_iovs = 0, num_pages = 0, max_pages;
2500 size_t in_size, out_size, transferred, c;
2501 int err, i;
2502 struct iov_iter ii;
2503
2504#if BITS_PER_LONG == 32
2505 inarg.flags |= FUSE_IOCTL_32BIT;
2506#else
2507 if (flags & FUSE_IOCTL_COMPAT)
2508 inarg.flags |= FUSE_IOCTL_32BIT;
2509#endif
2510
2511
2512 BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE);
2513
2514 err = -ENOMEM;
2515 pages = kcalloc(FUSE_MAX_PAGES_PER_REQ, sizeof(pages[0]), GFP_KERNEL);
2516 iov_page = (struct iovec *) __get_free_page(GFP_KERNEL);
2517 if (!pages || !iov_page)
2518 goto out;
2519
2520
2521
2522
2523
2524 if (!(flags & FUSE_IOCTL_UNRESTRICTED)) {
2525 struct iovec *iov = iov_page;
2526
2527 iov->iov_base = (void __user *)arg;
2528 iov->iov_len = _IOC_SIZE(cmd);
2529
2530 if (_IOC_DIR(cmd) & _IOC_WRITE) {
2531 in_iov = iov;
2532 in_iovs = 1;
2533 }
2534
2535 if (_IOC_DIR(cmd) & _IOC_READ) {
2536 out_iov = iov;
2537 out_iovs = 1;
2538 }
2539 }
2540
2541 retry:
2542 inarg.in_size = in_size = iov_length(in_iov, in_iovs);
2543 inarg.out_size = out_size = iov_length(out_iov, out_iovs);
2544
2545
2546
2547
2548
2549 out_size = max_t(size_t, out_size, PAGE_SIZE);
2550 max_pages = DIV_ROUND_UP(max(in_size, out_size), PAGE_SIZE);
2551
2552
2553 err = -ENOMEM;
2554 if (max_pages > FUSE_MAX_PAGES_PER_REQ)
2555 goto out;
2556 while (num_pages < max_pages) {
2557 pages[num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
2558 if (!pages[num_pages])
2559 goto out;
2560 num_pages++;
2561 }
2562
2563 req = fuse_get_req(fc, num_pages);
2564 if (IS_ERR(req)) {
2565 err = PTR_ERR(req);
2566 req = NULL;
2567 goto out;
2568 }
2569 memcpy(req->pages, pages, sizeof(req->pages[0]) * num_pages);
2570 req->num_pages = num_pages;
2571 fuse_page_descs_length_init(req, 0, req->num_pages);
2572
2573
2574 req->in.h.opcode = FUSE_IOCTL;
2575 req->in.h.nodeid = ff->nodeid;
2576 req->in.numargs = 1;
2577 req->in.args[0].size = sizeof(inarg);
2578 req->in.args[0].value = &inarg;
2579 if (in_size) {
2580 req->in.numargs++;
2581 req->in.args[1].size = in_size;
2582 req->in.argpages = 1;
2583
2584 err = -EFAULT;
2585 iov_iter_init(&ii, WRITE, in_iov, in_iovs, in_size);
2586 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= num_pages); i++) {
2587 c = copy_page_from_iter(pages[i], 0, PAGE_SIZE, &ii);
2588 if (c != PAGE_SIZE && iov_iter_count(&ii))
2589 goto out;
2590 }
2591 }
2592
2593 req->out.numargs = 2;
2594 req->out.args[0].size = sizeof(outarg);
2595 req->out.args[0].value = &outarg;
2596 req->out.args[1].size = out_size;
2597 req->out.argpages = 1;
2598 req->out.argvar = 1;
2599
2600 fuse_request_send(fc, req);
2601 err = req->out.h.error;
2602 transferred = req->out.args[1].size;
2603 fuse_put_request(fc, req);
2604 req = NULL;
2605 if (err)
2606 goto out;
2607
2608
2609 if (outarg.flags & FUSE_IOCTL_RETRY) {
2610 void *vaddr;
2611
2612
2613 err = -EIO;
2614 if (!(flags & FUSE_IOCTL_UNRESTRICTED))
2615 goto out;
2616
2617 in_iovs = outarg.in_iovs;
2618 out_iovs = outarg.out_iovs;
2619
2620
2621
2622
2623
2624 err = -ENOMEM;
2625 if (in_iovs > FUSE_IOCTL_MAX_IOV ||
2626 out_iovs > FUSE_IOCTL_MAX_IOV ||
2627 in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV)
2628 goto out;
2629
2630 vaddr = kmap_atomic(pages[0]);
2631 err = fuse_copy_ioctl_iovec(fc, iov_page, vaddr,
2632 transferred, in_iovs + out_iovs,
2633 (flags & FUSE_IOCTL_COMPAT) != 0);
2634 kunmap_atomic(vaddr);
2635 if (err)
2636 goto out;
2637
2638 in_iov = iov_page;
2639 out_iov = in_iov + in_iovs;
2640
2641 err = fuse_verify_ioctl_iov(in_iov, in_iovs);
2642 if (err)
2643 goto out;
2644
2645 err = fuse_verify_ioctl_iov(out_iov, out_iovs);
2646 if (err)
2647 goto out;
2648
2649 goto retry;
2650 }
2651
2652 err = -EIO;
2653 if (transferred > inarg.out_size)
2654 goto out;
2655
2656 err = -EFAULT;
2657 iov_iter_init(&ii, READ, out_iov, out_iovs, transferred);
2658 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= num_pages); i++) {
2659 c = copy_page_to_iter(pages[i], 0, PAGE_SIZE, &ii);
2660 if (c != PAGE_SIZE && iov_iter_count(&ii))
2661 goto out;
2662 }
2663 err = 0;
2664 out:
2665 if (req)
2666 fuse_put_request(fc, req);
2667 free_page((unsigned long) iov_page);
2668 while (num_pages)
2669 __free_page(pages[--num_pages]);
2670 kfree(pages);
2671
2672 return err ? err : outarg.result;
2673}
2674EXPORT_SYMBOL_GPL(fuse_do_ioctl);
2675
2676long fuse_ioctl_common(struct file *file, unsigned int cmd,
2677 unsigned long arg, unsigned int flags)
2678{
2679 struct inode *inode = file_inode(file);
2680 struct fuse_conn *fc = get_fuse_conn(inode);
2681
2682 if (!fuse_allow_current_process(fc))
2683 return -EACCES;
2684
2685 if (is_bad_inode(inode))
2686 return -EIO;
2687
2688 return fuse_do_ioctl(file, cmd, arg, flags);
2689}
2690
2691static long fuse_file_ioctl(struct file *file, unsigned int cmd,
2692 unsigned long arg)
2693{
2694 return fuse_ioctl_common(file, cmd, arg, 0);
2695}
2696
2697static long fuse_file_compat_ioctl(struct file *file, unsigned int cmd,
2698 unsigned long arg)
2699{
2700 return fuse_ioctl_common(file, cmd, arg, FUSE_IOCTL_COMPAT);
2701}
2702
2703
2704
2705
2706
2707
2708static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
2709 struct rb_node **parent_out)
2710{
2711 struct rb_node **link = &fc->polled_files.rb_node;
2712 struct rb_node *last = NULL;
2713
2714 while (*link) {
2715 struct fuse_file *ff;
2716
2717 last = *link;
2718 ff = rb_entry(last, struct fuse_file, polled_node);
2719
2720 if (kh < ff->kh)
2721 link = &last->rb_left;
2722 else if (kh > ff->kh)
2723 link = &last->rb_right;
2724 else
2725 return link;
2726 }
2727
2728 if (parent_out)
2729 *parent_out = last;
2730 return link;
2731}
2732
2733
2734
2735
2736
2737
2738
2739static void fuse_register_polled_file(struct fuse_conn *fc,
2740 struct fuse_file *ff)
2741{
2742 spin_lock(&fc->lock);
2743 if (RB_EMPTY_NODE(&ff->polled_node)) {
2744 struct rb_node **link, *uninitialized_var(parent);
2745
2746 link = fuse_find_polled_node(fc, ff->kh, &parent);
2747 BUG_ON(*link);
2748 rb_link_node(&ff->polled_node, parent, link);
2749 rb_insert_color(&ff->polled_node, &fc->polled_files);
2750 }
2751 spin_unlock(&fc->lock);
2752}
2753
2754__poll_t fuse_file_poll(struct file *file, poll_table *wait)
2755{
2756 struct fuse_file *ff = file->private_data;
2757 struct fuse_conn *fc = ff->fc;
2758 struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
2759 struct fuse_poll_out outarg;
2760 FUSE_ARGS(args);
2761 int err;
2762
2763 if (fc->no_poll)
2764 return DEFAULT_POLLMASK;
2765
2766 poll_wait(file, &ff->poll_wait, wait);
2767 inarg.events = mangle_poll(poll_requested_events(wait));
2768
2769
2770
2771
2772
2773 if (waitqueue_active(&ff->poll_wait)) {
2774 inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
2775 fuse_register_polled_file(fc, ff);
2776 }
2777
2778 args.in.h.opcode = FUSE_POLL;
2779 args.in.h.nodeid = ff->nodeid;
2780 args.in.numargs = 1;
2781 args.in.args[0].size = sizeof(inarg);
2782 args.in.args[0].value = &inarg;
2783 args.out.numargs = 1;
2784 args.out.args[0].size = sizeof(outarg);
2785 args.out.args[0].value = &outarg;
2786 err = fuse_simple_request(fc, &args);
2787
2788 if (!err)
2789 return demangle_poll(outarg.revents);
2790 if (err == -ENOSYS) {
2791 fc->no_poll = 1;
2792 return DEFAULT_POLLMASK;
2793 }
2794 return EPOLLERR;
2795}
2796EXPORT_SYMBOL_GPL(fuse_file_poll);
2797
2798
2799
2800
2801
2802int fuse_notify_poll_wakeup(struct fuse_conn *fc,
2803 struct fuse_notify_poll_wakeup_out *outarg)
2804{
2805 u64 kh = outarg->kh;
2806 struct rb_node **link;
2807
2808 spin_lock(&fc->lock);
2809
2810 link = fuse_find_polled_node(fc, kh, NULL);
2811 if (*link) {
2812 struct fuse_file *ff;
2813
2814 ff = rb_entry(*link, struct fuse_file, polled_node);
2815 wake_up_interruptible_sync(&ff->poll_wait);
2816 }
2817
2818 spin_unlock(&fc->lock);
2819 return 0;
2820}
2821
2822static void fuse_do_truncate(struct file *file)
2823{
2824 struct inode *inode = file->f_mapping->host;
2825 struct iattr attr;
2826
2827 attr.ia_valid = ATTR_SIZE;
2828 attr.ia_size = i_size_read(inode);
2829
2830 attr.ia_file = file;
2831 attr.ia_valid |= ATTR_FILE;
2832
2833 fuse_do_setattr(file_dentry(file), &attr, file);
2834}
2835
2836static inline loff_t fuse_round_up(loff_t off)
2837{
2838 return round_up(off, FUSE_MAX_PAGES_PER_REQ << PAGE_SHIFT);
2839}
2840
2841static ssize_t
2842fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
2843{
2844 DECLARE_COMPLETION_ONSTACK(wait);
2845 ssize_t ret = 0;
2846 struct file *file = iocb->ki_filp;
2847 struct fuse_file *ff = file->private_data;
2848 bool async_dio = ff->fc->async_dio;
2849 loff_t pos = 0;
2850 struct inode *inode;
2851 loff_t i_size;
2852 size_t count = iov_iter_count(iter);
2853 loff_t offset = iocb->ki_pos;
2854 struct fuse_io_priv *io;
2855
2856 pos = offset;
2857 inode = file->f_mapping->host;
2858 i_size = i_size_read(inode);
2859
2860 if ((iov_iter_rw(iter) == READ) && (offset > i_size))
2861 return 0;
2862
2863
2864 if (async_dio && iov_iter_rw(iter) != WRITE && offset + count > i_size) {
2865 if (offset >= i_size)
2866 return 0;
2867 iov_iter_truncate(iter, fuse_round_up(i_size - offset));
2868 count = iov_iter_count(iter);
2869 }
2870
2871 io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL);
2872 if (!io)
2873 return -ENOMEM;
2874 spin_lock_init(&io->lock);
2875 kref_init(&io->refcnt);
2876 io->reqs = 1;
2877 io->bytes = -1;
2878 io->size = 0;
2879 io->offset = offset;
2880 io->write = (iov_iter_rw(iter) == WRITE);
2881 io->err = 0;
2882
2883
2884
2885
2886 io->async = async_dio;
2887 io->iocb = iocb;
2888 io->blocking = is_sync_kiocb(iocb);
2889
2890
2891
2892
2893
2894 if ((offset + count > i_size) && iov_iter_rw(iter) == WRITE)
2895 io->blocking = true;
2896
2897 if (io->async && io->blocking) {
2898
2899
2900
2901
2902 kref_get(&io->refcnt);
2903 io->done = &wait;
2904 }
2905
2906 if (iov_iter_rw(iter) == WRITE) {
2907 ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE);
2908 fuse_invalidate_attr(inode);
2909 } else {
2910 ret = __fuse_direct_read(io, iter, &pos);
2911 }
2912
2913 if (io->async) {
2914 fuse_aio_complete(io, ret < 0 ? ret : 0, -1);
2915
2916
2917 if (!io->blocking)
2918 return -EIOCBQUEUED;
2919
2920 wait_for_completion(&wait);
2921 ret = fuse_get_res_by_io(io);
2922 }
2923
2924 kref_put(&io->refcnt, fuse_io_release);
2925
2926 if (iov_iter_rw(iter) == WRITE) {
2927 if (ret > 0)
2928 fuse_write_update_size(inode, pos);
2929 else if (ret < 0 && offset + count > i_size)
2930 fuse_do_truncate(file);
2931 }
2932
2933 return ret;
2934}
2935
2936static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
2937 loff_t length)
2938{
2939 struct fuse_file *ff = file->private_data;
2940 struct inode *inode = file_inode(file);
2941 struct fuse_inode *fi = get_fuse_inode(inode);
2942 struct fuse_conn *fc = ff->fc;
2943 FUSE_ARGS(args);
2944 struct fuse_fallocate_in inarg = {
2945 .fh = ff->fh,
2946 .offset = offset,
2947 .length = length,
2948 .mode = mode
2949 };
2950 int err;
2951 bool lock_inode = !(mode & FALLOC_FL_KEEP_SIZE) ||
2952 (mode & FALLOC_FL_PUNCH_HOLE);
2953
2954 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2955 return -EOPNOTSUPP;
2956
2957 if (fc->no_fallocate)
2958 return -EOPNOTSUPP;
2959
2960 if (lock_inode) {
2961 inode_lock(inode);
2962 if (mode & FALLOC_FL_PUNCH_HOLE) {
2963 loff_t endbyte = offset + length - 1;
2964 err = filemap_write_and_wait_range(inode->i_mapping,
2965 offset, endbyte);
2966 if (err)
2967 goto out;
2968
2969 fuse_sync_writes(inode);
2970 }
2971 }
2972
2973 if (!(mode & FALLOC_FL_KEEP_SIZE))
2974 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
2975
2976 args.in.h.opcode = FUSE_FALLOCATE;
2977 args.in.h.nodeid = ff->nodeid;
2978 args.in.numargs = 1;
2979 args.in.args[0].size = sizeof(inarg);
2980 args.in.args[0].value = &inarg;
2981 err = fuse_simple_request(fc, &args);
2982 if (err == -ENOSYS) {
2983 fc->no_fallocate = 1;
2984 err = -EOPNOTSUPP;
2985 }
2986 if (err)
2987 goto out;
2988
2989
2990 if (!(mode & FALLOC_FL_KEEP_SIZE)) {
2991 bool changed = fuse_write_update_size(inode, offset + length);
2992
2993 if (changed && fc->writeback_cache)
2994 file_update_time(file);
2995 }
2996
2997 if (mode & FALLOC_FL_PUNCH_HOLE)
2998 truncate_pagecache_range(inode, offset, offset + length - 1);
2999
3000 fuse_invalidate_attr(inode);
3001
3002out:
3003 if (!(mode & FALLOC_FL_KEEP_SIZE))
3004 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
3005
3006 if (lock_inode)
3007 inode_unlock(inode);
3008
3009 return err;
3010}
3011
3012static const struct file_operations fuse_file_operations = {
3013 .llseek = fuse_file_llseek,
3014 .read_iter = fuse_file_read_iter,
3015 .write_iter = fuse_file_write_iter,
3016 .mmap = fuse_file_mmap,
3017 .open = fuse_open,
3018 .flush = fuse_flush,
3019 .release = fuse_release,
3020 .fsync = fuse_fsync,
3021 .lock = fuse_file_lock,
3022 .flock = fuse_file_flock,
3023 .splice_read = generic_file_splice_read,
3024 .unlocked_ioctl = fuse_file_ioctl,
3025 .compat_ioctl = fuse_file_compat_ioctl,
3026 .poll = fuse_file_poll,
3027 .fallocate = fuse_file_fallocate,
3028};
3029
3030static const struct file_operations fuse_direct_io_file_operations = {
3031 .llseek = fuse_file_llseek,
3032 .read_iter = fuse_direct_read_iter,
3033 .write_iter = fuse_direct_write_iter,
3034 .mmap = fuse_direct_mmap,
3035 .open = fuse_open,
3036 .flush = fuse_flush,
3037 .release = fuse_release,
3038 .fsync = fuse_fsync,
3039 .lock = fuse_file_lock,
3040 .flock = fuse_file_flock,
3041 .unlocked_ioctl = fuse_file_ioctl,
3042 .compat_ioctl = fuse_file_compat_ioctl,
3043 .poll = fuse_file_poll,
3044 .fallocate = fuse_file_fallocate,
3045
3046};
3047
3048static const struct address_space_operations fuse_file_aops = {
3049 .readpage = fuse_readpage,
3050 .writepage = fuse_writepage,
3051 .writepages = fuse_writepages,
3052 .launder_page = fuse_launder_page,
3053 .readpages = fuse_readpages,
3054 .set_page_dirty = __set_page_dirty_nobuffers,
3055 .bmap = fuse_bmap,
3056 .direct_IO = fuse_direct_IO,
3057 .write_begin = fuse_write_begin,
3058 .write_end = fuse_write_end,
3059};
3060
3061void fuse_init_file_inode(struct inode *inode)
3062{
3063 inode->i_fop = &fuse_file_operations;
3064 inode->i_data.a_ops = &fuse_file_aops;
3065}
3066