1
2
3
4
5
6
7
8
9#include "fuse_i.h"
10
11#include <linux/pagemap.h>
12#include <linux/slab.h>
13#include <linux/kernel.h>
14#include <linux/sched.h>
15#include <linux/module.h>
16#include <linux/compat.h>
17#include <linux/swap.h>
18#include <linux/falloc.h>
19#include <linux/uio.h>
20
21static const struct file_operations fuse_direct_io_file_operations;
22
23static int fuse_send_open(struct fuse_conn *fc, u64 nodeid, struct file *file,
24 int opcode, struct fuse_open_out *outargp)
25{
26 struct fuse_open_in inarg;
27 FUSE_ARGS(args);
28
29 memset(&inarg, 0, sizeof(inarg));
30 inarg.flags = file->f_flags & ~(O_CREAT | O_EXCL | O_NOCTTY);
31 if (!fc->atomic_o_trunc)
32 inarg.flags &= ~O_TRUNC;
33 args.in.h.opcode = opcode;
34 args.in.h.nodeid = nodeid;
35 args.in.numargs = 1;
36 args.in.args[0].size = sizeof(inarg);
37 args.in.args[0].value = &inarg;
38 args.out.numargs = 1;
39 args.out.args[0].size = sizeof(*outargp);
40 args.out.args[0].value = outargp;
41
42 return fuse_simple_request(fc, &args);
43}
44
45struct fuse_file *fuse_file_alloc(struct fuse_conn *fc)
46{
47 struct fuse_file *ff;
48
49 ff = kmalloc(sizeof(struct fuse_file), GFP_KERNEL);
50 if (unlikely(!ff))
51 return NULL;
52
53 ff->fc = fc;
54 ff->reserved_req = fuse_request_alloc(0);
55 if (unlikely(!ff->reserved_req)) {
56 kfree(ff);
57 return NULL;
58 }
59
60 INIT_LIST_HEAD(&ff->write_entry);
61 atomic_set(&ff->count, 0);
62 RB_CLEAR_NODE(&ff->polled_node);
63 init_waitqueue_head(&ff->poll_wait);
64
65 spin_lock(&fc->lock);
66 ff->kh = ++fc->khctr;
67 spin_unlock(&fc->lock);
68
69 return ff;
70}
71
72void fuse_file_free(struct fuse_file *ff)
73{
74 fuse_request_free(ff->reserved_req);
75 kfree(ff);
76}
77
78struct fuse_file *fuse_file_get(struct fuse_file *ff)
79{
80 atomic_inc(&ff->count);
81 return ff;
82}
83
84static void fuse_release_end(struct fuse_conn *fc, struct fuse_req *req)
85{
86 iput(req->misc.release.inode);
87}
88
89static void fuse_file_put(struct fuse_file *ff, bool sync)
90{
91 if (atomic_dec_and_test(&ff->count)) {
92 struct fuse_req *req = ff->reserved_req;
93
94 if (ff->fc->no_open) {
95
96
97
98
99 __clear_bit(FR_BACKGROUND, &req->flags);
100 iput(req->misc.release.inode);
101 fuse_put_request(ff->fc, req);
102 } else if (sync) {
103 __clear_bit(FR_BACKGROUND, &req->flags);
104 fuse_request_send(ff->fc, req);
105 iput(req->misc.release.inode);
106 fuse_put_request(ff->fc, req);
107 } else {
108 req->end = fuse_release_end;
109 __set_bit(FR_BACKGROUND, &req->flags);
110 fuse_request_send_background(ff->fc, req);
111 }
112 kfree(ff);
113 }
114}
115
116int fuse_do_open(struct fuse_conn *fc, u64 nodeid, struct file *file,
117 bool isdir)
118{
119 struct fuse_file *ff;
120 int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN;
121
122 ff = fuse_file_alloc(fc);
123 if (!ff)
124 return -ENOMEM;
125
126 ff->fh = 0;
127 ff->open_flags = FOPEN_KEEP_CACHE;
128 if (!fc->no_open || isdir) {
129 struct fuse_open_out outarg;
130 int err;
131
132 err = fuse_send_open(fc, nodeid, file, opcode, &outarg);
133 if (!err) {
134 ff->fh = outarg.fh;
135 ff->open_flags = outarg.open_flags;
136
137 } else if (err != -ENOSYS || isdir) {
138 fuse_file_free(ff);
139 return err;
140 } else {
141 fc->no_open = 1;
142 }
143 }
144
145 if (isdir)
146 ff->open_flags &= ~FOPEN_DIRECT_IO;
147
148 ff->nodeid = nodeid;
149 file->private_data = fuse_file_get(ff);
150
151 return 0;
152}
153EXPORT_SYMBOL_GPL(fuse_do_open);
154
155static void fuse_link_write_file(struct file *file)
156{
157 struct inode *inode = file_inode(file);
158 struct fuse_conn *fc = get_fuse_conn(inode);
159 struct fuse_inode *fi = get_fuse_inode(inode);
160 struct fuse_file *ff = file->private_data;
161
162
163
164
165 spin_lock(&fc->lock);
166 if (list_empty(&ff->write_entry))
167 list_add(&ff->write_entry, &fi->write_files);
168 spin_unlock(&fc->lock);
169}
170
171void fuse_finish_open(struct inode *inode, struct file *file)
172{
173 struct fuse_file *ff = file->private_data;
174 struct fuse_conn *fc = get_fuse_conn(inode);
175
176 if (ff->open_flags & FOPEN_DIRECT_IO)
177 file->f_op = &fuse_direct_io_file_operations;
178 if (!(ff->open_flags & FOPEN_KEEP_CACHE))
179 invalidate_inode_pages2(inode->i_mapping);
180 if (ff->open_flags & FOPEN_NONSEEKABLE)
181 nonseekable_open(inode, file);
182 if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) {
183 struct fuse_inode *fi = get_fuse_inode(inode);
184
185 spin_lock(&fc->lock);
186 fi->attr_version = ++fc->attr_version;
187 i_size_write(inode, 0);
188 spin_unlock(&fc->lock);
189 fuse_invalidate_attr(inode);
190 if (fc->writeback_cache)
191 file_update_time(file);
192 }
193 if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache)
194 fuse_link_write_file(file);
195}
196
197int fuse_open_common(struct inode *inode, struct file *file, bool isdir)
198{
199 struct fuse_conn *fc = get_fuse_conn(inode);
200 int err;
201 bool lock_inode = (file->f_flags & O_TRUNC) &&
202 fc->atomic_o_trunc &&
203 fc->writeback_cache;
204
205 err = generic_file_open(inode, file);
206 if (err)
207 return err;
208
209 if (lock_inode)
210 inode_lock(inode);
211
212 err = fuse_do_open(fc, get_node_id(inode), file, isdir);
213
214 if (!err)
215 fuse_finish_open(inode, file);
216
217 if (lock_inode)
218 inode_unlock(inode);
219
220 return err;
221}
222
223static void fuse_prepare_release(struct fuse_file *ff, int flags, int opcode)
224{
225 struct fuse_conn *fc = ff->fc;
226 struct fuse_req *req = ff->reserved_req;
227 struct fuse_release_in *inarg = &req->misc.release.in;
228
229 spin_lock(&fc->lock);
230 list_del(&ff->write_entry);
231 if (!RB_EMPTY_NODE(&ff->polled_node))
232 rb_erase(&ff->polled_node, &fc->polled_files);
233 spin_unlock(&fc->lock);
234
235 wake_up_interruptible_all(&ff->poll_wait);
236
237 inarg->fh = ff->fh;
238 inarg->flags = flags;
239 req->in.h.opcode = opcode;
240 req->in.h.nodeid = ff->nodeid;
241 req->in.numargs = 1;
242 req->in.args[0].size = sizeof(struct fuse_release_in);
243 req->in.args[0].value = inarg;
244}
245
246void fuse_release_common(struct file *file, int opcode)
247{
248 struct fuse_file *ff;
249 struct fuse_req *req;
250
251 ff = file->private_data;
252 if (unlikely(!ff))
253 return;
254
255 req = ff->reserved_req;
256 fuse_prepare_release(ff, file->f_flags, opcode);
257
258 if (ff->flock) {
259 struct fuse_release_in *inarg = &req->misc.release.in;
260 inarg->release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
261 inarg->lock_owner = fuse_lock_owner_id(ff->fc,
262 (fl_owner_t) file);
263 }
264
265 req->misc.release.inode = igrab(file_inode(file));
266
267
268
269
270
271
272
273
274
275
276 fuse_file_put(ff, ff->fc->destroy_req != NULL);
277}
278
279static int fuse_open(struct inode *inode, struct file *file)
280{
281 return fuse_open_common(inode, file, false);
282}
283
284static int fuse_release(struct inode *inode, struct file *file)
285{
286 struct fuse_conn *fc = get_fuse_conn(inode);
287
288
289 if (fc->writeback_cache)
290 write_inode_now(inode, 1);
291
292 fuse_release_common(file, FUSE_RELEASE);
293
294
295 return 0;
296}
297
298void fuse_sync_release(struct fuse_file *ff, int flags)
299{
300 WARN_ON(atomic_read(&ff->count) > 1);
301 fuse_prepare_release(ff, flags, FUSE_RELEASE);
302 __set_bit(FR_FORCE, &ff->reserved_req->flags);
303 __clear_bit(FR_BACKGROUND, &ff->reserved_req->flags);
304 fuse_request_send(ff->fc, ff->reserved_req);
305 fuse_put_request(ff->fc, ff->reserved_req);
306 kfree(ff);
307}
308EXPORT_SYMBOL_GPL(fuse_sync_release);
309
310
311
312
313
314u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id)
315{
316 u32 *k = fc->scramble_key;
317 u64 v = (unsigned long) id;
318 u32 v0 = v;
319 u32 v1 = v >> 32;
320 u32 sum = 0;
321 int i;
322
323 for (i = 0; i < 32; i++) {
324 v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]);
325 sum += 0x9E3779B9;
326 v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]);
327 }
328
329 return (u64) v0 + ((u64) v1 << 32);
330}
331
332
333
334
335
336
337
338static bool fuse_range_is_writeback(struct inode *inode, pgoff_t idx_from,
339 pgoff_t idx_to)
340{
341 struct fuse_conn *fc = get_fuse_conn(inode);
342 struct fuse_inode *fi = get_fuse_inode(inode);
343 struct fuse_req *req;
344 bool found = false;
345
346 spin_lock(&fc->lock);
347 list_for_each_entry(req, &fi->writepages, writepages_entry) {
348 pgoff_t curr_index;
349
350 BUG_ON(req->inode != inode);
351 curr_index = req->misc.write.in.offset >> PAGE_SHIFT;
352 if (idx_from < curr_index + req->num_pages &&
353 curr_index <= idx_to) {
354 found = true;
355 break;
356 }
357 }
358 spin_unlock(&fc->lock);
359
360 return found;
361}
362
363static inline bool fuse_page_is_writeback(struct inode *inode, pgoff_t index)
364{
365 return fuse_range_is_writeback(inode, index, index);
366}
367
368
369
370
371
372
373
374static int fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index)
375{
376 struct fuse_inode *fi = get_fuse_inode(inode);
377
378 wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index));
379 return 0;
380}
381
382
383
384
385
386
387
388
389
390
391static void fuse_sync_writes(struct inode *inode)
392{
393 fuse_set_nowrite(inode);
394 fuse_release_nowrite(inode);
395}
396
397static int fuse_flush(struct file *file, fl_owner_t id)
398{
399 struct inode *inode = file_inode(file);
400 struct fuse_conn *fc = get_fuse_conn(inode);
401 struct fuse_file *ff = file->private_data;
402 struct fuse_req *req;
403 struct fuse_flush_in inarg;
404 int err;
405
406 if (is_bad_inode(inode))
407 return -EIO;
408
409 if (fc->no_flush)
410 return 0;
411
412 err = write_inode_now(inode, 1);
413 if (err)
414 return err;
415
416 inode_lock(inode);
417 fuse_sync_writes(inode);
418 inode_unlock(inode);
419
420 err = filemap_check_errors(file->f_mapping);
421 if (err)
422 return err;
423
424 req = fuse_get_req_nofail_nopages(fc, file);
425 memset(&inarg, 0, sizeof(inarg));
426 inarg.fh = ff->fh;
427 inarg.lock_owner = fuse_lock_owner_id(fc, id);
428 req->in.h.opcode = FUSE_FLUSH;
429 req->in.h.nodeid = get_node_id(inode);
430 req->in.numargs = 1;
431 req->in.args[0].size = sizeof(inarg);
432 req->in.args[0].value = &inarg;
433 __set_bit(FR_FORCE, &req->flags);
434 fuse_request_send(fc, req);
435 err = req->out.h.error;
436 fuse_put_request(fc, req);
437 if (err == -ENOSYS) {
438 fc->no_flush = 1;
439 err = 0;
440 }
441 return err;
442}
443
444int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
445 int datasync, int isdir)
446{
447 struct inode *inode = file->f_mapping->host;
448 struct fuse_conn *fc = get_fuse_conn(inode);
449 struct fuse_file *ff = file->private_data;
450 FUSE_ARGS(args);
451 struct fuse_fsync_in inarg;
452 int err;
453
454 if (is_bad_inode(inode))
455 return -EIO;
456
457 inode_lock(inode);
458
459
460
461
462
463
464 err = filemap_write_and_wait_range(inode->i_mapping, start, end);
465 if (err)
466 goto out;
467
468 fuse_sync_writes(inode);
469
470
471
472
473
474
475 err = filemap_check_errors(file->f_mapping);
476 if (err)
477 goto out;
478
479 err = sync_inode_metadata(inode, 1);
480 if (err)
481 goto out;
482
483 if ((!isdir && fc->no_fsync) || (isdir && fc->no_fsyncdir))
484 goto out;
485
486 memset(&inarg, 0, sizeof(inarg));
487 inarg.fh = ff->fh;
488 inarg.fsync_flags = datasync ? 1 : 0;
489 args.in.h.opcode = isdir ? FUSE_FSYNCDIR : FUSE_FSYNC;
490 args.in.h.nodeid = get_node_id(inode);
491 args.in.numargs = 1;
492 args.in.args[0].size = sizeof(inarg);
493 args.in.args[0].value = &inarg;
494 err = fuse_simple_request(fc, &args);
495 if (err == -ENOSYS) {
496 if (isdir)
497 fc->no_fsyncdir = 1;
498 else
499 fc->no_fsync = 1;
500 err = 0;
501 }
502out:
503 inode_unlock(inode);
504 return err;
505}
506
507static int fuse_fsync(struct file *file, loff_t start, loff_t end,
508 int datasync)
509{
510 return fuse_fsync_common(file, start, end, datasync, 0);
511}
512
513void fuse_read_fill(struct fuse_req *req, struct file *file, loff_t pos,
514 size_t count, int opcode)
515{
516 struct fuse_read_in *inarg = &req->misc.read.in;
517 struct fuse_file *ff = file->private_data;
518
519 inarg->fh = ff->fh;
520 inarg->offset = pos;
521 inarg->size = count;
522 inarg->flags = file->f_flags;
523 req->in.h.opcode = opcode;
524 req->in.h.nodeid = ff->nodeid;
525 req->in.numargs = 1;
526 req->in.args[0].size = sizeof(struct fuse_read_in);
527 req->in.args[0].value = inarg;
528 req->out.argvar = 1;
529 req->out.numargs = 1;
530 req->out.args[0].size = count;
531}
532
533static void fuse_release_user_pages(struct fuse_req *req, bool should_dirty)
534{
535 unsigned i;
536
537 for (i = 0; i < req->num_pages; i++) {
538 struct page *page = req->pages[i];
539 if (should_dirty)
540 set_page_dirty_lock(page);
541 put_page(page);
542 }
543}
544
545static void fuse_io_release(struct kref *kref)
546{
547 kfree(container_of(kref, struct fuse_io_priv, refcnt));
548}
549
550static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)
551{
552 if (io->err)
553 return io->err;
554
555 if (io->bytes >= 0 && io->write)
556 return -EIO;
557
558 return io->bytes < 0 ? io->size : io->bytes;
559}
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
578{
579 int left;
580
581 spin_lock(&io->lock);
582 if (err)
583 io->err = io->err ? : err;
584 else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))
585 io->bytes = pos;
586
587 left = --io->reqs;
588 if (!left && io->blocking)
589 complete(io->done);
590 spin_unlock(&io->lock);
591
592 if (!left && !io->blocking) {
593 ssize_t res = fuse_get_res_by_io(io);
594
595 if (res >= 0) {
596 struct inode *inode = file_inode(io->iocb->ki_filp);
597 struct fuse_conn *fc = get_fuse_conn(inode);
598 struct fuse_inode *fi = get_fuse_inode(inode);
599
600 spin_lock(&fc->lock);
601 fi->attr_version = ++fc->attr_version;
602 spin_unlock(&fc->lock);
603 }
604
605 io->iocb->ki_complete(io->iocb, res, 0);
606 }
607
608 kref_put(&io->refcnt, fuse_io_release);
609}
610
611static void fuse_aio_complete_req(struct fuse_conn *fc, struct fuse_req *req)
612{
613 struct fuse_io_priv *io = req->io;
614 ssize_t pos = -1;
615
616 fuse_release_user_pages(req, !io->write);
617
618 if (io->write) {
619 if (req->misc.write.in.size != req->misc.write.out.size)
620 pos = req->misc.write.in.offset - io->offset +
621 req->misc.write.out.size;
622 } else {
623 if (req->misc.read.in.size != req->out.args[0].size)
624 pos = req->misc.read.in.offset - io->offset +
625 req->out.args[0].size;
626 }
627
628 fuse_aio_complete(io, req->out.h.error, pos);
629}
630
631static size_t fuse_async_req_send(struct fuse_conn *fc, struct fuse_req *req,
632 size_t num_bytes, struct fuse_io_priv *io)
633{
634 spin_lock(&io->lock);
635 kref_get(&io->refcnt);
636 io->size += num_bytes;
637 io->reqs++;
638 spin_unlock(&io->lock);
639
640 req->io = io;
641 req->end = fuse_aio_complete_req;
642
643 __fuse_get_request(req);
644 fuse_request_send_background(fc, req);
645
646 return num_bytes;
647}
648
649static size_t fuse_send_read(struct fuse_req *req, struct fuse_io_priv *io,
650 loff_t pos, size_t count, fl_owner_t owner)
651{
652 struct file *file = io->file;
653 struct fuse_file *ff = file->private_data;
654 struct fuse_conn *fc = ff->fc;
655
656 fuse_read_fill(req, file, pos, count, FUSE_READ);
657 if (owner != NULL) {
658 struct fuse_read_in *inarg = &req->misc.read.in;
659
660 inarg->read_flags |= FUSE_READ_LOCKOWNER;
661 inarg->lock_owner = fuse_lock_owner_id(fc, owner);
662 }
663
664 if (io->async)
665 return fuse_async_req_send(fc, req, count, io);
666
667 fuse_request_send(fc, req);
668 return req->out.args[0].size;
669}
670
671static void fuse_read_update_size(struct inode *inode, loff_t size,
672 u64 attr_ver)
673{
674 struct fuse_conn *fc = get_fuse_conn(inode);
675 struct fuse_inode *fi = get_fuse_inode(inode);
676
677 spin_lock(&fc->lock);
678 if (attr_ver == fi->attr_version && size < inode->i_size &&
679 !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) {
680 fi->attr_version = ++fc->attr_version;
681 i_size_write(inode, size);
682 }
683 spin_unlock(&fc->lock);
684}
685
686static void fuse_short_read(struct fuse_req *req, struct inode *inode,
687 u64 attr_ver)
688{
689 size_t num_read = req->out.args[0].size;
690 struct fuse_conn *fc = get_fuse_conn(inode);
691
692 if (fc->writeback_cache) {
693
694
695
696
697
698 int i;
699 int start_idx = num_read >> PAGE_SHIFT;
700 size_t off = num_read & (PAGE_SIZE - 1);
701
702 for (i = start_idx; i < req->num_pages; i++) {
703 zero_user_segment(req->pages[i], off, PAGE_SIZE);
704 off = 0;
705 }
706 } else {
707 loff_t pos = page_offset(req->pages[0]) + num_read;
708 fuse_read_update_size(inode, pos, attr_ver);
709 }
710}
711
712static int fuse_do_readpage(struct file *file, struct page *page)
713{
714 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(file);
715 struct inode *inode = page->mapping->host;
716 struct fuse_conn *fc = get_fuse_conn(inode);
717 struct fuse_req *req;
718 size_t num_read;
719 loff_t pos = page_offset(page);
720 size_t count = PAGE_SIZE;
721 u64 attr_ver;
722 int err;
723
724
725
726
727
728
729 fuse_wait_on_page_writeback(inode, page->index);
730
731 req = fuse_get_req(fc, 1);
732 if (IS_ERR(req))
733 return PTR_ERR(req);
734
735 attr_ver = fuse_get_attr_version(fc);
736
737 req->out.page_zeroing = 1;
738 req->out.argpages = 1;
739 req->num_pages = 1;
740 req->pages[0] = page;
741 req->page_descs[0].length = count;
742 num_read = fuse_send_read(req, &io, pos, count, NULL);
743 err = req->out.h.error;
744
745 if (!err) {
746
747
748
749 if (num_read < count)
750 fuse_short_read(req, inode, attr_ver);
751
752 SetPageUptodate(page);
753 }
754
755 fuse_put_request(fc, req);
756
757 return err;
758}
759
760static int fuse_readpage(struct file *file, struct page *page)
761{
762 struct inode *inode = page->mapping->host;
763 int err;
764
765 err = -EIO;
766 if (is_bad_inode(inode))
767 goto out;
768
769 err = fuse_do_readpage(file, page);
770 fuse_invalidate_atime(inode);
771 out:
772 unlock_page(page);
773 return err;
774}
775
776static void fuse_readpages_end(struct fuse_conn *fc, struct fuse_req *req)
777{
778 int i;
779 size_t count = req->misc.read.in.size;
780 size_t num_read = req->out.args[0].size;
781 struct address_space *mapping = NULL;
782
783 for (i = 0; mapping == NULL && i < req->num_pages; i++)
784 mapping = req->pages[i]->mapping;
785
786 if (mapping) {
787 struct inode *inode = mapping->host;
788
789
790
791
792 if (!req->out.h.error && num_read < count)
793 fuse_short_read(req, inode, req->misc.read.attr_ver);
794
795 fuse_invalidate_atime(inode);
796 }
797
798 for (i = 0; i < req->num_pages; i++) {
799 struct page *page = req->pages[i];
800 if (!req->out.h.error)
801 SetPageUptodate(page);
802 else
803 SetPageError(page);
804 unlock_page(page);
805 put_page(page);
806 }
807 if (req->ff)
808 fuse_file_put(req->ff, false);
809}
810
811static void fuse_send_readpages(struct fuse_req *req, struct file *file)
812{
813 struct fuse_file *ff = file->private_data;
814 struct fuse_conn *fc = ff->fc;
815 loff_t pos = page_offset(req->pages[0]);
816 size_t count = req->num_pages << PAGE_SHIFT;
817
818 req->out.argpages = 1;
819 req->out.page_zeroing = 1;
820 req->out.page_replace = 1;
821 fuse_read_fill(req, file, pos, count, FUSE_READ);
822 req->misc.read.attr_ver = fuse_get_attr_version(fc);
823 if (fc->async_read) {
824 req->ff = fuse_file_get(ff);
825 req->end = fuse_readpages_end;
826 fuse_request_send_background(fc, req);
827 } else {
828 fuse_request_send(fc, req);
829 fuse_readpages_end(fc, req);
830 fuse_put_request(fc, req);
831 }
832}
833
834struct fuse_fill_data {
835 struct fuse_req *req;
836 struct file *file;
837 struct inode *inode;
838 unsigned nr_pages;
839};
840
841static int fuse_readpages_fill(void *_data, struct page *page)
842{
843 struct fuse_fill_data *data = _data;
844 struct fuse_req *req = data->req;
845 struct inode *inode = data->inode;
846 struct fuse_conn *fc = get_fuse_conn(inode);
847
848 fuse_wait_on_page_writeback(inode, page->index);
849
850 if (req->num_pages &&
851 (req->num_pages == FUSE_MAX_PAGES_PER_REQ ||
852 (req->num_pages + 1) * PAGE_SIZE > fc->max_read ||
853 req->pages[req->num_pages - 1]->index + 1 != page->index)) {
854 int nr_alloc = min_t(unsigned, data->nr_pages,
855 FUSE_MAX_PAGES_PER_REQ);
856 fuse_send_readpages(req, data->file);
857 if (fc->async_read)
858 req = fuse_get_req_for_background(fc, nr_alloc);
859 else
860 req = fuse_get_req(fc, nr_alloc);
861
862 data->req = req;
863 if (IS_ERR(req)) {
864 unlock_page(page);
865 return PTR_ERR(req);
866 }
867 }
868
869 if (WARN_ON(req->num_pages >= req->max_pages)) {
870 fuse_put_request(fc, req);
871 return -EIO;
872 }
873
874 get_page(page);
875 req->pages[req->num_pages] = page;
876 req->page_descs[req->num_pages].length = PAGE_SIZE;
877 req->num_pages++;
878 data->nr_pages--;
879 return 0;
880}
881
882static int fuse_readpages(struct file *file, struct address_space *mapping,
883 struct list_head *pages, unsigned nr_pages)
884{
885 struct inode *inode = mapping->host;
886 struct fuse_conn *fc = get_fuse_conn(inode);
887 struct fuse_fill_data data;
888 int err;
889 int nr_alloc = min_t(unsigned, nr_pages, FUSE_MAX_PAGES_PER_REQ);
890
891 err = -EIO;
892 if (is_bad_inode(inode))
893 goto out;
894
895 data.file = file;
896 data.inode = inode;
897 if (fc->async_read)
898 data.req = fuse_get_req_for_background(fc, nr_alloc);
899 else
900 data.req = fuse_get_req(fc, nr_alloc);
901 data.nr_pages = nr_pages;
902 err = PTR_ERR(data.req);
903 if (IS_ERR(data.req))
904 goto out;
905
906 err = read_cache_pages(mapping, pages, fuse_readpages_fill, &data);
907 if (!err) {
908 if (data.req->num_pages)
909 fuse_send_readpages(data.req, file);
910 else
911 fuse_put_request(fc, data.req);
912 }
913out:
914 return err;
915}
916
917static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
918{
919 struct inode *inode = iocb->ki_filp->f_mapping->host;
920 struct fuse_conn *fc = get_fuse_conn(inode);
921
922
923
924
925
926
927 if (fc->auto_inval_data ||
928 (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) {
929 int err;
930 err = fuse_update_attributes(inode, NULL, iocb->ki_filp, NULL);
931 if (err)
932 return err;
933 }
934
935 return generic_file_read_iter(iocb, to);
936}
937
938static void fuse_write_fill(struct fuse_req *req, struct fuse_file *ff,
939 loff_t pos, size_t count)
940{
941 struct fuse_write_in *inarg = &req->misc.write.in;
942 struct fuse_write_out *outarg = &req->misc.write.out;
943
944 inarg->fh = ff->fh;
945 inarg->offset = pos;
946 inarg->size = count;
947 req->in.h.opcode = FUSE_WRITE;
948 req->in.h.nodeid = ff->nodeid;
949 req->in.numargs = 2;
950 if (ff->fc->minor < 9)
951 req->in.args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
952 else
953 req->in.args[0].size = sizeof(struct fuse_write_in);
954 req->in.args[0].value = inarg;
955 req->in.args[1].size = count;
956 req->out.numargs = 1;
957 req->out.args[0].size = sizeof(struct fuse_write_out);
958 req->out.args[0].value = outarg;
959}
960
961static size_t fuse_send_write(struct fuse_req *req, struct fuse_io_priv *io,
962 loff_t pos, size_t count, fl_owner_t owner)
963{
964 struct file *file = io->file;
965 struct fuse_file *ff = file->private_data;
966 struct fuse_conn *fc = ff->fc;
967 struct fuse_write_in *inarg = &req->misc.write.in;
968
969 fuse_write_fill(req, ff, pos, count);
970 inarg->flags = file->f_flags;
971 if (owner != NULL) {
972 inarg->write_flags |= FUSE_WRITE_LOCKOWNER;
973 inarg->lock_owner = fuse_lock_owner_id(fc, owner);
974 }
975
976 if (io->async)
977 return fuse_async_req_send(fc, req, count, io);
978
979 fuse_request_send(fc, req);
980 return req->misc.write.out.size;
981}
982
983bool fuse_write_update_size(struct inode *inode, loff_t pos)
984{
985 struct fuse_conn *fc = get_fuse_conn(inode);
986 struct fuse_inode *fi = get_fuse_inode(inode);
987 bool ret = false;
988
989 spin_lock(&fc->lock);
990 fi->attr_version = ++fc->attr_version;
991 if (pos > inode->i_size) {
992 i_size_write(inode, pos);
993 ret = true;
994 }
995 spin_unlock(&fc->lock);
996
997 return ret;
998}
999
1000static size_t fuse_send_write_pages(struct fuse_req *req, struct file *file,
1001 struct inode *inode, loff_t pos,
1002 size_t count)
1003{
1004 size_t res;
1005 unsigned offset;
1006 unsigned i;
1007 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(file);
1008
1009 for (i = 0; i < req->num_pages; i++)
1010 fuse_wait_on_page_writeback(inode, req->pages[i]->index);
1011
1012 res = fuse_send_write(req, &io, pos, count, NULL);
1013
1014 offset = req->page_descs[0].offset;
1015 count = res;
1016 for (i = 0; i < req->num_pages; i++) {
1017 struct page *page = req->pages[i];
1018
1019 if (!req->out.h.error && !offset && count >= PAGE_SIZE)
1020 SetPageUptodate(page);
1021
1022 if (count > PAGE_SIZE - offset)
1023 count -= PAGE_SIZE - offset;
1024 else
1025 count = 0;
1026 offset = 0;
1027
1028 unlock_page(page);
1029 put_page(page);
1030 }
1031
1032 return res;
1033}
1034
1035static ssize_t fuse_fill_write_pages(struct fuse_req *req,
1036 struct address_space *mapping,
1037 struct iov_iter *ii, loff_t pos)
1038{
1039 struct fuse_conn *fc = get_fuse_conn(mapping->host);
1040 unsigned offset = pos & (PAGE_SIZE - 1);
1041 size_t count = 0;
1042 int err;
1043
1044 req->in.argpages = 1;
1045 req->page_descs[0].offset = offset;
1046
1047 do {
1048 size_t tmp;
1049 struct page *page;
1050 pgoff_t index = pos >> PAGE_SHIFT;
1051 size_t bytes = min_t(size_t, PAGE_SIZE - offset,
1052 iov_iter_count(ii));
1053
1054 bytes = min_t(size_t, bytes, fc->max_write - count);
1055
1056 again:
1057 err = -EFAULT;
1058 if (iov_iter_fault_in_readable(ii, bytes))
1059 break;
1060
1061 err = -ENOMEM;
1062 page = grab_cache_page_write_begin(mapping, index, 0);
1063 if (!page)
1064 break;
1065
1066 if (mapping_writably_mapped(mapping))
1067 flush_dcache_page(page);
1068
1069 tmp = iov_iter_copy_from_user_atomic(page, ii, offset, bytes);
1070 flush_dcache_page(page);
1071
1072 iov_iter_advance(ii, tmp);
1073 if (!tmp) {
1074 unlock_page(page);
1075 put_page(page);
1076 bytes = min(bytes, iov_iter_single_seg_count(ii));
1077 goto again;
1078 }
1079
1080 err = 0;
1081 req->pages[req->num_pages] = page;
1082 req->page_descs[req->num_pages].length = tmp;
1083 req->num_pages++;
1084
1085 count += tmp;
1086 pos += tmp;
1087 offset += tmp;
1088 if (offset == PAGE_SIZE)
1089 offset = 0;
1090
1091 if (!fc->big_writes)
1092 break;
1093 } while (iov_iter_count(ii) && count < fc->max_write &&
1094 req->num_pages < req->max_pages && offset == 0);
1095
1096 return count > 0 ? count : err;
1097}
1098
1099static inline unsigned fuse_wr_pages(loff_t pos, size_t len)
1100{
1101 return min_t(unsigned,
1102 ((pos + len - 1) >> PAGE_SHIFT) -
1103 (pos >> PAGE_SHIFT) + 1,
1104 FUSE_MAX_PAGES_PER_REQ);
1105}
1106
1107static ssize_t fuse_perform_write(struct file *file,
1108 struct address_space *mapping,
1109 struct iov_iter *ii, loff_t pos)
1110{
1111 struct inode *inode = mapping->host;
1112 struct fuse_conn *fc = get_fuse_conn(inode);
1113 struct fuse_inode *fi = get_fuse_inode(inode);
1114 int err = 0;
1115 ssize_t res = 0;
1116
1117 if (is_bad_inode(inode))
1118 return -EIO;
1119
1120 if (inode->i_size < pos + iov_iter_count(ii))
1121 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1122
1123 do {
1124 struct fuse_req *req;
1125 ssize_t count;
1126 unsigned nr_pages = fuse_wr_pages(pos, iov_iter_count(ii));
1127
1128 req = fuse_get_req(fc, nr_pages);
1129 if (IS_ERR(req)) {
1130 err = PTR_ERR(req);
1131 break;
1132 }
1133
1134 count = fuse_fill_write_pages(req, mapping, ii, pos);
1135 if (count <= 0) {
1136 err = count;
1137 } else {
1138 size_t num_written;
1139
1140 num_written = fuse_send_write_pages(req, file, inode,
1141 pos, count);
1142 err = req->out.h.error;
1143 if (!err) {
1144 res += num_written;
1145 pos += num_written;
1146
1147
1148 if (num_written != count)
1149 err = -EIO;
1150 }
1151 }
1152 fuse_put_request(fc, req);
1153 } while (!err && iov_iter_count(ii));
1154
1155 if (res > 0)
1156 fuse_write_update_size(inode, pos);
1157
1158 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1159 fuse_invalidate_attr(inode);
1160
1161 return res > 0 ? res : err;
1162}
1163
1164static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
1165{
1166 struct file *file = iocb->ki_filp;
1167 struct address_space *mapping = file->f_mapping;
1168 ssize_t written = 0;
1169 ssize_t written_buffered = 0;
1170 struct inode *inode = mapping->host;
1171 ssize_t err;
1172 loff_t endbyte = 0;
1173
1174 if (get_fuse_conn(inode)->writeback_cache) {
1175
1176 err = fuse_update_attributes(mapping->host, NULL, file, NULL);
1177 if (err)
1178 return err;
1179
1180 return generic_file_write_iter(iocb, from);
1181 }
1182
1183 inode_lock(inode);
1184
1185
1186 current->backing_dev_info = inode_to_bdi(inode);
1187
1188 err = generic_write_checks(iocb, from);
1189 if (err <= 0)
1190 goto out;
1191
1192 err = file_remove_privs(file);
1193 if (err)
1194 goto out;
1195
1196 err = file_update_time(file);
1197 if (err)
1198 goto out;
1199
1200 if (iocb->ki_flags & IOCB_DIRECT) {
1201 loff_t pos = iocb->ki_pos;
1202 written = generic_file_direct_write(iocb, from);
1203 if (written < 0 || !iov_iter_count(from))
1204 goto out;
1205
1206 pos += written;
1207
1208 written_buffered = fuse_perform_write(file, mapping, from, pos);
1209 if (written_buffered < 0) {
1210 err = written_buffered;
1211 goto out;
1212 }
1213 endbyte = pos + written_buffered - 1;
1214
1215 err = filemap_write_and_wait_range(file->f_mapping, pos,
1216 endbyte);
1217 if (err)
1218 goto out;
1219
1220 invalidate_mapping_pages(file->f_mapping,
1221 pos >> PAGE_SHIFT,
1222 endbyte >> PAGE_SHIFT);
1223
1224 written += written_buffered;
1225 iocb->ki_pos = pos + written_buffered;
1226 } else {
1227 written = fuse_perform_write(file, mapping, from, iocb->ki_pos);
1228 if (written >= 0)
1229 iocb->ki_pos += written;
1230 }
1231out:
1232 current->backing_dev_info = NULL;
1233 inode_unlock(inode);
1234
1235 return written ? written : err;
1236}
1237
1238static inline void fuse_page_descs_length_init(struct fuse_req *req,
1239 unsigned index, unsigned nr_pages)
1240{
1241 int i;
1242
1243 for (i = index; i < index + nr_pages; i++)
1244 req->page_descs[i].length = PAGE_SIZE -
1245 req->page_descs[i].offset;
1246}
1247
1248static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
1249{
1250 return (unsigned long)ii->iov->iov_base + ii->iov_offset;
1251}
1252
1253static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
1254 size_t max_size)
1255{
1256 return min(iov_iter_single_seg_count(ii), max_size);
1257}
1258
1259static int fuse_get_user_pages(struct fuse_req *req, struct iov_iter *ii,
1260 size_t *nbytesp, int write)
1261{
1262 size_t nbytes = 0;
1263 ssize_t ret = 0;
1264
1265
1266 if (ii->type & ITER_KVEC) {
1267 unsigned long user_addr = fuse_get_user_addr(ii);
1268 size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
1269
1270 if (write)
1271 req->in.args[1].value = (void *) user_addr;
1272 else
1273 req->out.args[0].value = (void *) user_addr;
1274
1275 iov_iter_advance(ii, frag_size);
1276 *nbytesp = frag_size;
1277 return 0;
1278 }
1279
1280 while (nbytes < *nbytesp && req->num_pages < req->max_pages) {
1281 unsigned npages;
1282 size_t start;
1283 ret = iov_iter_get_pages(ii, &req->pages[req->num_pages],
1284 *nbytesp - nbytes,
1285 req->max_pages - req->num_pages,
1286 &start);
1287 if (ret < 0)
1288 break;
1289
1290 iov_iter_advance(ii, ret);
1291 nbytes += ret;
1292
1293 ret += start;
1294 npages = (ret + PAGE_SIZE - 1) / PAGE_SIZE;
1295
1296 req->page_descs[req->num_pages].offset = start;
1297 fuse_page_descs_length_init(req, req->num_pages, npages);
1298
1299 req->num_pages += npages;
1300 req->page_descs[req->num_pages - 1].length -=
1301 (PAGE_SIZE - ret) & (PAGE_SIZE - 1);
1302 }
1303
1304 if (write)
1305 req->in.argpages = 1;
1306 else
1307 req->out.argpages = 1;
1308
1309 *nbytesp = nbytes;
1310
1311 return ret < 0 ? ret : 0;
1312}
1313
1314static inline int fuse_iter_npages(const struct iov_iter *ii_p)
1315{
1316 return iov_iter_npages(ii_p, FUSE_MAX_PAGES_PER_REQ);
1317}
1318
1319ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
1320 loff_t *ppos, int flags)
1321{
1322 int write = flags & FUSE_DIO_WRITE;
1323 bool should_dirty = !write && iter_is_iovec(iter);
1324 int cuse = flags & FUSE_DIO_CUSE;
1325 struct file *file = io->file;
1326 struct inode *inode = file->f_mapping->host;
1327 struct fuse_file *ff = file->private_data;
1328 struct fuse_conn *fc = ff->fc;
1329 size_t nmax = write ? fc->max_write : fc->max_read;
1330 loff_t pos = *ppos;
1331 size_t count = iov_iter_count(iter);
1332 pgoff_t idx_from = pos >> PAGE_SHIFT;
1333 pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT;
1334 ssize_t res = 0;
1335 struct fuse_req *req;
1336 int err = 0;
1337
1338 if (io->async)
1339 req = fuse_get_req_for_background(fc, fuse_iter_npages(iter));
1340 else
1341 req = fuse_get_req(fc, fuse_iter_npages(iter));
1342 if (IS_ERR(req))
1343 return PTR_ERR(req);
1344
1345 if (!cuse && fuse_range_is_writeback(inode, idx_from, idx_to)) {
1346 if (!write)
1347 inode_lock(inode);
1348 fuse_sync_writes(inode);
1349 if (!write)
1350 inode_unlock(inode);
1351 }
1352
1353 while (count) {
1354 size_t nres;
1355 fl_owner_t owner = current->files;
1356 size_t nbytes = min(count, nmax);
1357 err = fuse_get_user_pages(req, iter, &nbytes, write);
1358 if (err && !nbytes)
1359 break;
1360
1361 if (write)
1362 nres = fuse_send_write(req, io, pos, nbytes, owner);
1363 else
1364 nres = fuse_send_read(req, io, pos, nbytes, owner);
1365
1366 if (!io->async)
1367 fuse_release_user_pages(req, should_dirty);
1368 if (req->out.h.error) {
1369 err = req->out.h.error;
1370 break;
1371 } else if (nres > nbytes) {
1372 res = 0;
1373 err = -EIO;
1374 break;
1375 }
1376 count -= nres;
1377 res += nres;
1378 pos += nres;
1379 if (nres != nbytes)
1380 break;
1381 if (count) {
1382 fuse_put_request(fc, req);
1383 if (io->async)
1384 req = fuse_get_req_for_background(fc,
1385 fuse_iter_npages(iter));
1386 else
1387 req = fuse_get_req(fc, fuse_iter_npages(iter));
1388 if (IS_ERR(req))
1389 break;
1390 }
1391 }
1392 if (!IS_ERR(req))
1393 fuse_put_request(fc, req);
1394 if (res > 0)
1395 *ppos = pos;
1396
1397 return res > 0 ? res : err;
1398}
1399EXPORT_SYMBOL_GPL(fuse_direct_io);
1400
1401static ssize_t __fuse_direct_read(struct fuse_io_priv *io,
1402 struct iov_iter *iter,
1403 loff_t *ppos)
1404{
1405 ssize_t res;
1406 struct file *file = io->file;
1407 struct inode *inode = file_inode(file);
1408
1409 if (is_bad_inode(inode))
1410 return -EIO;
1411
1412 res = fuse_direct_io(io, iter, ppos, 0);
1413
1414 fuse_invalidate_attr(inode);
1415
1416 return res;
1417}
1418
1419static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to)
1420{
1421 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb->ki_filp);
1422 return __fuse_direct_read(&io, to, &iocb->ki_pos);
1423}
1424
1425static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
1426{
1427 struct file *file = iocb->ki_filp;
1428 struct inode *inode = file_inode(file);
1429 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(file);
1430 ssize_t res;
1431
1432 if (is_bad_inode(inode))
1433 return -EIO;
1434
1435
1436 inode_lock(inode);
1437 res = generic_write_checks(iocb, from);
1438 if (res > 0)
1439 res = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE);
1440 fuse_invalidate_attr(inode);
1441 if (res > 0)
1442 fuse_write_update_size(inode, iocb->ki_pos);
1443 inode_unlock(inode);
1444
1445 return res;
1446}
1447
1448static void fuse_writepage_free(struct fuse_conn *fc, struct fuse_req *req)
1449{
1450 int i;
1451
1452 for (i = 0; i < req->num_pages; i++)
1453 __free_page(req->pages[i]);
1454
1455 if (req->ff)
1456 fuse_file_put(req->ff, false);
1457}
1458
1459static void fuse_writepage_finish(struct fuse_conn *fc, struct fuse_req *req)
1460{
1461 struct inode *inode = req->inode;
1462 struct fuse_inode *fi = get_fuse_inode(inode);
1463 struct backing_dev_info *bdi = inode_to_bdi(inode);
1464 int i;
1465
1466 list_del(&req->writepages_entry);
1467 for (i = 0; i < req->num_pages; i++) {
1468 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
1469 dec_node_page_state(req->pages[i], NR_WRITEBACK_TEMP);
1470 wb_writeout_inc(&bdi->wb);
1471 }
1472 wake_up(&fi->page_waitq);
1473}
1474
1475
1476static void fuse_send_writepage(struct fuse_conn *fc, struct fuse_req *req,
1477 loff_t size)
1478__releases(fc->lock)
1479__acquires(fc->lock)
1480{
1481 struct fuse_inode *fi = get_fuse_inode(req->inode);
1482 struct fuse_write_in *inarg = &req->misc.write.in;
1483 __u64 data_size = req->num_pages * PAGE_SIZE;
1484
1485 if (!fc->connected)
1486 goto out_free;
1487
1488 if (inarg->offset + data_size <= size) {
1489 inarg->size = data_size;
1490 } else if (inarg->offset < size) {
1491 inarg->size = size - inarg->offset;
1492 } else {
1493
1494 goto out_free;
1495 }
1496
1497 req->in.args[1].size = inarg->size;
1498 fi->writectr++;
1499 fuse_request_send_background_locked(fc, req);
1500 return;
1501
1502 out_free:
1503 fuse_writepage_finish(fc, req);
1504 spin_unlock(&fc->lock);
1505 fuse_writepage_free(fc, req);
1506 fuse_put_request(fc, req);
1507 spin_lock(&fc->lock);
1508}
1509
1510
1511
1512
1513
1514
1515
1516void fuse_flush_writepages(struct inode *inode)
1517__releases(fc->lock)
1518__acquires(fc->lock)
1519{
1520 struct fuse_conn *fc = get_fuse_conn(inode);
1521 struct fuse_inode *fi = get_fuse_inode(inode);
1522 size_t crop = i_size_read(inode);
1523 struct fuse_req *req;
1524
1525 while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) {
1526 req = list_entry(fi->queued_writes.next, struct fuse_req, list);
1527 list_del_init(&req->list);
1528 fuse_send_writepage(fc, req, crop);
1529 }
1530}
1531
1532static void fuse_writepage_end(struct fuse_conn *fc, struct fuse_req *req)
1533{
1534 struct inode *inode = req->inode;
1535 struct fuse_inode *fi = get_fuse_inode(inode);
1536
1537 mapping_set_error(inode->i_mapping, req->out.h.error);
1538 spin_lock(&fc->lock);
1539 while (req->misc.write.next) {
1540 struct fuse_conn *fc = get_fuse_conn(inode);
1541 struct fuse_write_in *inarg = &req->misc.write.in;
1542 struct fuse_req *next = req->misc.write.next;
1543 req->misc.write.next = next->misc.write.next;
1544 next->misc.write.next = NULL;
1545 next->ff = fuse_file_get(req->ff);
1546 list_add(&next->writepages_entry, &fi->writepages);
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571 fuse_send_writepage(fc, next, inarg->offset + inarg->size);
1572 }
1573 fi->writectr--;
1574 fuse_writepage_finish(fc, req);
1575 spin_unlock(&fc->lock);
1576 fuse_writepage_free(fc, req);
1577}
1578
1579static struct fuse_file *__fuse_write_file_get(struct fuse_conn *fc,
1580 struct fuse_inode *fi)
1581{
1582 struct fuse_file *ff = NULL;
1583
1584 spin_lock(&fc->lock);
1585 if (!list_empty(&fi->write_files)) {
1586 ff = list_entry(fi->write_files.next, struct fuse_file,
1587 write_entry);
1588 fuse_file_get(ff);
1589 }
1590 spin_unlock(&fc->lock);
1591
1592 return ff;
1593}
1594
1595static struct fuse_file *fuse_write_file_get(struct fuse_conn *fc,
1596 struct fuse_inode *fi)
1597{
1598 struct fuse_file *ff = __fuse_write_file_get(fc, fi);
1599 WARN_ON(!ff);
1600 return ff;
1601}
1602
1603int fuse_write_inode(struct inode *inode, struct writeback_control *wbc)
1604{
1605 struct fuse_conn *fc = get_fuse_conn(inode);
1606 struct fuse_inode *fi = get_fuse_inode(inode);
1607 struct fuse_file *ff;
1608 int err;
1609
1610 ff = __fuse_write_file_get(fc, fi);
1611 err = fuse_flush_times(inode, ff);
1612 if (ff)
1613 fuse_file_put(ff, 0);
1614
1615 return err;
1616}
1617
1618static int fuse_writepage_locked(struct page *page)
1619{
1620 struct address_space *mapping = page->mapping;
1621 struct inode *inode = mapping->host;
1622 struct fuse_conn *fc = get_fuse_conn(inode);
1623 struct fuse_inode *fi = get_fuse_inode(inode);
1624 struct fuse_req *req;
1625 struct page *tmp_page;
1626 int error = -ENOMEM;
1627
1628 set_page_writeback(page);
1629
1630 req = fuse_request_alloc_nofs(1);
1631 if (!req)
1632 goto err;
1633
1634
1635 __set_bit(FR_BACKGROUND, &req->flags);
1636 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
1637 if (!tmp_page)
1638 goto err_free;
1639
1640 error = -EIO;
1641 req->ff = fuse_write_file_get(fc, fi);
1642 if (!req->ff)
1643 goto err_nofile;
1644
1645 fuse_write_fill(req, req->ff, page_offset(page), 0);
1646
1647 copy_highpage(tmp_page, page);
1648 req->misc.write.in.write_flags |= FUSE_WRITE_CACHE;
1649 req->misc.write.next = NULL;
1650 req->in.argpages = 1;
1651 req->num_pages = 1;
1652 req->pages[0] = tmp_page;
1653 req->page_descs[0].offset = 0;
1654 req->page_descs[0].length = PAGE_SIZE;
1655 req->end = fuse_writepage_end;
1656 req->inode = inode;
1657
1658 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
1659 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
1660
1661 spin_lock(&fc->lock);
1662 list_add(&req->writepages_entry, &fi->writepages);
1663 list_add_tail(&req->list, &fi->queued_writes);
1664 fuse_flush_writepages(inode);
1665 spin_unlock(&fc->lock);
1666
1667 end_page_writeback(page);
1668
1669 return 0;
1670
1671err_nofile:
1672 __free_page(tmp_page);
1673err_free:
1674 fuse_request_free(req);
1675err:
1676 end_page_writeback(page);
1677 return error;
1678}
1679
1680static int fuse_writepage(struct page *page, struct writeback_control *wbc)
1681{
1682 int err;
1683
1684 if (fuse_page_is_writeback(page->mapping->host, page->index)) {
1685
1686
1687
1688
1689
1690 WARN_ON(wbc->sync_mode == WB_SYNC_ALL);
1691
1692 redirty_page_for_writepage(wbc, page);
1693 return 0;
1694 }
1695
1696 err = fuse_writepage_locked(page);
1697 unlock_page(page);
1698
1699 return err;
1700}
1701
1702struct fuse_fill_wb_data {
1703 struct fuse_req *req;
1704 struct fuse_file *ff;
1705 struct inode *inode;
1706 struct page **orig_pages;
1707};
1708
1709static void fuse_writepages_send(struct fuse_fill_wb_data *data)
1710{
1711 struct fuse_req *req = data->req;
1712 struct inode *inode = data->inode;
1713 struct fuse_conn *fc = get_fuse_conn(inode);
1714 struct fuse_inode *fi = get_fuse_inode(inode);
1715 int num_pages = req->num_pages;
1716 int i;
1717
1718 req->ff = fuse_file_get(data->ff);
1719 spin_lock(&fc->lock);
1720 list_add_tail(&req->list, &fi->queued_writes);
1721 fuse_flush_writepages(inode);
1722 spin_unlock(&fc->lock);
1723
1724 for (i = 0; i < num_pages; i++)
1725 end_page_writeback(data->orig_pages[i]);
1726}
1727
1728static bool fuse_writepage_in_flight(struct fuse_req *new_req,
1729 struct page *page)
1730{
1731 struct fuse_conn *fc = get_fuse_conn(new_req->inode);
1732 struct fuse_inode *fi = get_fuse_inode(new_req->inode);
1733 struct fuse_req *tmp;
1734 struct fuse_req *old_req;
1735 bool found = false;
1736 pgoff_t curr_index;
1737
1738 BUG_ON(new_req->num_pages != 0);
1739
1740 spin_lock(&fc->lock);
1741 list_del(&new_req->writepages_entry);
1742 list_for_each_entry(old_req, &fi->writepages, writepages_entry) {
1743 BUG_ON(old_req->inode != new_req->inode);
1744 curr_index = old_req->misc.write.in.offset >> PAGE_SHIFT;
1745 if (curr_index <= page->index &&
1746 page->index < curr_index + old_req->num_pages) {
1747 found = true;
1748 break;
1749 }
1750 }
1751 if (!found) {
1752 list_add(&new_req->writepages_entry, &fi->writepages);
1753 goto out_unlock;
1754 }
1755
1756 new_req->num_pages = 1;
1757 for (tmp = old_req; tmp != NULL; tmp = tmp->misc.write.next) {
1758 BUG_ON(tmp->inode != new_req->inode);
1759 curr_index = tmp->misc.write.in.offset >> PAGE_SHIFT;
1760 if (tmp->num_pages == 1 &&
1761 curr_index == page->index) {
1762 old_req = tmp;
1763 }
1764 }
1765
1766 if (old_req->num_pages == 1 && test_bit(FR_PENDING, &old_req->flags)) {
1767 struct backing_dev_info *bdi = inode_to_bdi(page->mapping->host);
1768
1769 copy_highpage(old_req->pages[0], page);
1770 spin_unlock(&fc->lock);
1771
1772 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
1773 dec_node_page_state(page, NR_WRITEBACK_TEMP);
1774 wb_writeout_inc(&bdi->wb);
1775 fuse_writepage_free(fc, new_req);
1776 fuse_request_free(new_req);
1777 goto out;
1778 } else {
1779 new_req->misc.write.next = old_req->misc.write.next;
1780 old_req->misc.write.next = new_req;
1781 }
1782out_unlock:
1783 spin_unlock(&fc->lock);
1784out:
1785 return found;
1786}
1787
1788static int fuse_writepages_fill(struct page *page,
1789 struct writeback_control *wbc, void *_data)
1790{
1791 struct fuse_fill_wb_data *data = _data;
1792 struct fuse_req *req = data->req;
1793 struct inode *inode = data->inode;
1794 struct fuse_conn *fc = get_fuse_conn(inode);
1795 struct page *tmp_page;
1796 bool is_writeback;
1797 int err;
1798
1799 if (!data->ff) {
1800 err = -EIO;
1801 data->ff = fuse_write_file_get(fc, get_fuse_inode(inode));
1802 if (!data->ff)
1803 goto out_unlock;
1804 }
1805
1806
1807
1808
1809
1810
1811
1812 is_writeback = fuse_page_is_writeback(inode, page->index);
1813
1814 if (req && req->num_pages &&
1815 (is_writeback || req->num_pages == FUSE_MAX_PAGES_PER_REQ ||
1816 (req->num_pages + 1) * PAGE_SIZE > fc->max_write ||
1817 data->orig_pages[req->num_pages - 1]->index + 1 != page->index)) {
1818 fuse_writepages_send(data);
1819 data->req = NULL;
1820 }
1821 err = -ENOMEM;
1822 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
1823 if (!tmp_page)
1824 goto out_unlock;
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839 if (data->req == NULL) {
1840 struct fuse_inode *fi = get_fuse_inode(inode);
1841
1842 err = -ENOMEM;
1843 req = fuse_request_alloc_nofs(FUSE_MAX_PAGES_PER_REQ);
1844 if (!req) {
1845 __free_page(tmp_page);
1846 goto out_unlock;
1847 }
1848
1849 fuse_write_fill(req, data->ff, page_offset(page), 0);
1850 req->misc.write.in.write_flags |= FUSE_WRITE_CACHE;
1851 req->misc.write.next = NULL;
1852 req->in.argpages = 1;
1853 __set_bit(FR_BACKGROUND, &req->flags);
1854 req->num_pages = 0;
1855 req->end = fuse_writepage_end;
1856 req->inode = inode;
1857
1858 spin_lock(&fc->lock);
1859 list_add(&req->writepages_entry, &fi->writepages);
1860 spin_unlock(&fc->lock);
1861
1862 data->req = req;
1863 }
1864 set_page_writeback(page);
1865
1866 copy_highpage(tmp_page, page);
1867 req->pages[req->num_pages] = tmp_page;
1868 req->page_descs[req->num_pages].offset = 0;
1869 req->page_descs[req->num_pages].length = PAGE_SIZE;
1870
1871 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
1872 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
1873
1874 err = 0;
1875 if (is_writeback && fuse_writepage_in_flight(req, page)) {
1876 end_page_writeback(page);
1877 data->req = NULL;
1878 goto out_unlock;
1879 }
1880 data->orig_pages[req->num_pages] = page;
1881
1882
1883
1884
1885
1886 spin_lock(&fc->lock);
1887 req->num_pages++;
1888 spin_unlock(&fc->lock);
1889
1890out_unlock:
1891 unlock_page(page);
1892
1893 return err;
1894}
1895
1896static int fuse_writepages(struct address_space *mapping,
1897 struct writeback_control *wbc)
1898{
1899 struct inode *inode = mapping->host;
1900 struct fuse_fill_wb_data data;
1901 int err;
1902
1903 err = -EIO;
1904 if (is_bad_inode(inode))
1905 goto out;
1906
1907 data.inode = inode;
1908 data.req = NULL;
1909 data.ff = NULL;
1910
1911 err = -ENOMEM;
1912 data.orig_pages = kcalloc(FUSE_MAX_PAGES_PER_REQ,
1913 sizeof(struct page *),
1914 GFP_NOFS);
1915 if (!data.orig_pages)
1916 goto out;
1917
1918 err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data);
1919 if (data.req) {
1920
1921 BUG_ON(!data.req->num_pages);
1922 fuse_writepages_send(&data);
1923 err = 0;
1924 }
1925 if (data.ff)
1926 fuse_file_put(data.ff, false);
1927
1928 kfree(data.orig_pages);
1929out:
1930 return err;
1931}
1932
1933
1934
1935
1936
1937static int fuse_write_begin(struct file *file, struct address_space *mapping,
1938 loff_t pos, unsigned len, unsigned flags,
1939 struct page **pagep, void **fsdata)
1940{
1941 pgoff_t index = pos >> PAGE_SHIFT;
1942 struct fuse_conn *fc = get_fuse_conn(file_inode(file));
1943 struct page *page;
1944 loff_t fsize;
1945 int err = -ENOMEM;
1946
1947 WARN_ON(!fc->writeback_cache);
1948
1949 page = grab_cache_page_write_begin(mapping, index, flags);
1950 if (!page)
1951 goto error;
1952
1953 fuse_wait_on_page_writeback(mapping->host, page->index);
1954
1955 if (PageUptodate(page) || len == PAGE_SIZE)
1956 goto success;
1957
1958
1959
1960
1961 fsize = i_size_read(mapping->host);
1962 if (fsize <= (pos & PAGE_MASK)) {
1963 size_t off = pos & ~PAGE_MASK;
1964 if (off)
1965 zero_user_segment(page, 0, off);
1966 goto success;
1967 }
1968 err = fuse_do_readpage(file, page);
1969 if (err)
1970 goto cleanup;
1971success:
1972 *pagep = page;
1973 return 0;
1974
1975cleanup:
1976 unlock_page(page);
1977 put_page(page);
1978error:
1979 return err;
1980}
1981
1982static int fuse_write_end(struct file *file, struct address_space *mapping,
1983 loff_t pos, unsigned len, unsigned copied,
1984 struct page *page, void *fsdata)
1985{
1986 struct inode *inode = page->mapping->host;
1987
1988
1989 if (!copied)
1990 goto unlock;
1991
1992 if (!PageUptodate(page)) {
1993
1994 size_t endoff = (pos + copied) & ~PAGE_MASK;
1995 if (endoff)
1996 zero_user_segment(page, endoff, PAGE_SIZE);
1997 SetPageUptodate(page);
1998 }
1999
2000 fuse_write_update_size(inode, pos + copied);
2001 set_page_dirty(page);
2002
2003unlock:
2004 unlock_page(page);
2005 put_page(page);
2006
2007 return copied;
2008}
2009
2010static int fuse_launder_page(struct page *page)
2011{
2012 int err = 0;
2013 if (clear_page_dirty_for_io(page)) {
2014 struct inode *inode = page->mapping->host;
2015 err = fuse_writepage_locked(page);
2016 if (!err)
2017 fuse_wait_on_page_writeback(inode, page->index);
2018 }
2019 return err;
2020}
2021
2022
2023
2024
2025
2026static void fuse_vma_close(struct vm_area_struct *vma)
2027{
2028 filemap_write_and_wait(vma->vm_file->f_mapping);
2029}
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046static int fuse_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
2047{
2048 struct page *page = vmf->page;
2049 struct inode *inode = file_inode(vma->vm_file);
2050
2051 file_update_time(vma->vm_file);
2052 lock_page(page);
2053 if (page->mapping != inode->i_mapping) {
2054 unlock_page(page);
2055 return VM_FAULT_NOPAGE;
2056 }
2057
2058 fuse_wait_on_page_writeback(inode, page->index);
2059 return VM_FAULT_LOCKED;
2060}
2061
2062static const struct vm_operations_struct fuse_file_vm_ops = {
2063 .close = fuse_vma_close,
2064 .fault = filemap_fault,
2065 .map_pages = filemap_map_pages,
2066 .page_mkwrite = fuse_page_mkwrite,
2067};
2068
2069static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
2070{
2071 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2072 fuse_link_write_file(file);
2073
2074 file_accessed(file);
2075 vma->vm_ops = &fuse_file_vm_ops;
2076 return 0;
2077}
2078
2079static int fuse_direct_mmap(struct file *file, struct vm_area_struct *vma)
2080{
2081
2082 if (vma->vm_flags & VM_MAYSHARE)
2083 return -ENODEV;
2084
2085 invalidate_inode_pages2(file->f_mapping);
2086
2087 return generic_file_mmap(file, vma);
2088}
2089
2090static int convert_fuse_file_lock(const struct fuse_file_lock *ffl,
2091 struct file_lock *fl)
2092{
2093 switch (ffl->type) {
2094 case F_UNLCK:
2095 break;
2096
2097 case F_RDLCK:
2098 case F_WRLCK:
2099 if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX ||
2100 ffl->end < ffl->start)
2101 return -EIO;
2102
2103 fl->fl_start = ffl->start;
2104 fl->fl_end = ffl->end;
2105 fl->fl_pid = ffl->pid;
2106 break;
2107
2108 default:
2109 return -EIO;
2110 }
2111 fl->fl_type = ffl->type;
2112 return 0;
2113}
2114
2115static void fuse_lk_fill(struct fuse_args *args, struct file *file,
2116 const struct file_lock *fl, int opcode, pid_t pid,
2117 int flock, struct fuse_lk_in *inarg)
2118{
2119 struct inode *inode = file_inode(file);
2120 struct fuse_conn *fc = get_fuse_conn(inode);
2121 struct fuse_file *ff = file->private_data;
2122
2123 memset(inarg, 0, sizeof(*inarg));
2124 inarg->fh = ff->fh;
2125 inarg->owner = fuse_lock_owner_id(fc, fl->fl_owner);
2126 inarg->lk.start = fl->fl_start;
2127 inarg->lk.end = fl->fl_end;
2128 inarg->lk.type = fl->fl_type;
2129 inarg->lk.pid = pid;
2130 if (flock)
2131 inarg->lk_flags |= FUSE_LK_FLOCK;
2132 args->in.h.opcode = opcode;
2133 args->in.h.nodeid = get_node_id(inode);
2134 args->in.numargs = 1;
2135 args->in.args[0].size = sizeof(*inarg);
2136 args->in.args[0].value = inarg;
2137}
2138
2139static int fuse_getlk(struct file *file, struct file_lock *fl)
2140{
2141 struct inode *inode = file_inode(file);
2142 struct fuse_conn *fc = get_fuse_conn(inode);
2143 FUSE_ARGS(args);
2144 struct fuse_lk_in inarg;
2145 struct fuse_lk_out outarg;
2146 int err;
2147
2148 fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg);
2149 args.out.numargs = 1;
2150 args.out.args[0].size = sizeof(outarg);
2151 args.out.args[0].value = &outarg;
2152 err = fuse_simple_request(fc, &args);
2153 if (!err)
2154 err = convert_fuse_file_lock(&outarg.lk, fl);
2155
2156 return err;
2157}
2158
2159static int fuse_setlk(struct file *file, struct file_lock *fl, int flock)
2160{
2161 struct inode *inode = file_inode(file);
2162 struct fuse_conn *fc = get_fuse_conn(inode);
2163 FUSE_ARGS(args);
2164 struct fuse_lk_in inarg;
2165 int opcode = (fl->fl_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK;
2166 pid_t pid = fl->fl_type != F_UNLCK ? current->tgid : 0;
2167 int err;
2168
2169 if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
2170
2171 return -ENOLCK;
2172 }
2173
2174
2175 if (fl->fl_flags & FL_CLOSE)
2176 return 0;
2177
2178 fuse_lk_fill(&args, file, fl, opcode, pid, flock, &inarg);
2179 err = fuse_simple_request(fc, &args);
2180
2181
2182 if (err == -EINTR)
2183 err = -ERESTARTSYS;
2184
2185 return err;
2186}
2187
2188static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl)
2189{
2190 struct inode *inode = file_inode(file);
2191 struct fuse_conn *fc = get_fuse_conn(inode);
2192 int err;
2193
2194 if (cmd == F_CANCELLK) {
2195 err = 0;
2196 } else if (cmd == F_GETLK) {
2197 if (fc->no_lock) {
2198 posix_test_lock(file, fl);
2199 err = 0;
2200 } else
2201 err = fuse_getlk(file, fl);
2202 } else {
2203 if (fc->no_lock)
2204 err = posix_lock_file(file, fl, NULL);
2205 else
2206 err = fuse_setlk(file, fl, 0);
2207 }
2208 return err;
2209}
2210
2211static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl)
2212{
2213 struct inode *inode = file_inode(file);
2214 struct fuse_conn *fc = get_fuse_conn(inode);
2215 int err;
2216
2217 if (fc->no_flock) {
2218 err = locks_lock_file_wait(file, fl);
2219 } else {
2220 struct fuse_file *ff = file->private_data;
2221
2222
2223 ff->flock = true;
2224 err = fuse_setlk(file, fl, 1);
2225 }
2226
2227 return err;
2228}
2229
2230static sector_t fuse_bmap(struct address_space *mapping, sector_t block)
2231{
2232 struct inode *inode = mapping->host;
2233 struct fuse_conn *fc = get_fuse_conn(inode);
2234 FUSE_ARGS(args);
2235 struct fuse_bmap_in inarg;
2236 struct fuse_bmap_out outarg;
2237 int err;
2238
2239 if (!inode->i_sb->s_bdev || fc->no_bmap)
2240 return 0;
2241
2242 memset(&inarg, 0, sizeof(inarg));
2243 inarg.block = block;
2244 inarg.blocksize = inode->i_sb->s_blocksize;
2245 args.in.h.opcode = FUSE_BMAP;
2246 args.in.h.nodeid = get_node_id(inode);
2247 args.in.numargs = 1;
2248 args.in.args[0].size = sizeof(inarg);
2249 args.in.args[0].value = &inarg;
2250 args.out.numargs = 1;
2251 args.out.args[0].size = sizeof(outarg);
2252 args.out.args[0].value = &outarg;
2253 err = fuse_simple_request(fc, &args);
2254 if (err == -ENOSYS)
2255 fc->no_bmap = 1;
2256
2257 return err ? 0 : outarg.block;
2258}
2259
2260static loff_t fuse_lseek(struct file *file, loff_t offset, int whence)
2261{
2262 struct inode *inode = file->f_mapping->host;
2263 struct fuse_conn *fc = get_fuse_conn(inode);
2264 struct fuse_file *ff = file->private_data;
2265 FUSE_ARGS(args);
2266 struct fuse_lseek_in inarg = {
2267 .fh = ff->fh,
2268 .offset = offset,
2269 .whence = whence
2270 };
2271 struct fuse_lseek_out outarg;
2272 int err;
2273
2274 if (fc->no_lseek)
2275 goto fallback;
2276
2277 args.in.h.opcode = FUSE_LSEEK;
2278 args.in.h.nodeid = ff->nodeid;
2279 args.in.numargs = 1;
2280 args.in.args[0].size = sizeof(inarg);
2281 args.in.args[0].value = &inarg;
2282 args.out.numargs = 1;
2283 args.out.args[0].size = sizeof(outarg);
2284 args.out.args[0].value = &outarg;
2285 err = fuse_simple_request(fc, &args);
2286 if (err) {
2287 if (err == -ENOSYS) {
2288 fc->no_lseek = 1;
2289 goto fallback;
2290 }
2291 return err;
2292 }
2293
2294 return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes);
2295
2296fallback:
2297 err = fuse_update_attributes(inode, NULL, file, NULL);
2298 if (!err)
2299 return generic_file_llseek(file, offset, whence);
2300 else
2301 return err;
2302}
2303
2304static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
2305{
2306 loff_t retval;
2307 struct inode *inode = file_inode(file);
2308
2309 switch (whence) {
2310 case SEEK_SET:
2311 case SEEK_CUR:
2312
2313 retval = generic_file_llseek(file, offset, whence);
2314 break;
2315 case SEEK_END:
2316 inode_lock(inode);
2317 retval = fuse_update_attributes(inode, NULL, file, NULL);
2318 if (!retval)
2319 retval = generic_file_llseek(file, offset, whence);
2320 inode_unlock(inode);
2321 break;
2322 case SEEK_HOLE:
2323 case SEEK_DATA:
2324 inode_lock(inode);
2325 retval = fuse_lseek(file, offset, whence);
2326 inode_unlock(inode);
2327 break;
2328 default:
2329 retval = -EINVAL;
2330 }
2331
2332 return retval;
2333}
2334
2335
2336
2337
2338
2339
2340
2341static int fuse_copy_ioctl_iovec_old(struct iovec *dst, void *src,
2342 size_t transferred, unsigned count,
2343 bool is_compat)
2344{
2345#ifdef CONFIG_COMPAT
2346 if (count * sizeof(struct compat_iovec) == transferred) {
2347 struct compat_iovec *ciov = src;
2348 unsigned i;
2349
2350
2351
2352
2353
2354
2355 if (!is_compat)
2356 return -EINVAL;
2357
2358 for (i = 0; i < count; i++) {
2359 dst[i].iov_base = compat_ptr(ciov[i].iov_base);
2360 dst[i].iov_len = ciov[i].iov_len;
2361 }
2362 return 0;
2363 }
2364#endif
2365
2366 if (count * sizeof(struct iovec) != transferred)
2367 return -EIO;
2368
2369 memcpy(dst, src, transferred);
2370 return 0;
2371}
2372
2373
2374static int fuse_verify_ioctl_iov(struct iovec *iov, size_t count)
2375{
2376 size_t n;
2377 u32 max = FUSE_MAX_PAGES_PER_REQ << PAGE_SHIFT;
2378
2379 for (n = 0; n < count; n++, iov++) {
2380 if (iov->iov_len > (size_t) max)
2381 return -ENOMEM;
2382 max -= iov->iov_len;
2383 }
2384 return 0;
2385}
2386
2387static int fuse_copy_ioctl_iovec(struct fuse_conn *fc, struct iovec *dst,
2388 void *src, size_t transferred, unsigned count,
2389 bool is_compat)
2390{
2391 unsigned i;
2392 struct fuse_ioctl_iovec *fiov = src;
2393
2394 if (fc->minor < 16) {
2395 return fuse_copy_ioctl_iovec_old(dst, src, transferred,
2396 count, is_compat);
2397 }
2398
2399 if (count * sizeof(struct fuse_ioctl_iovec) != transferred)
2400 return -EIO;
2401
2402 for (i = 0; i < count; i++) {
2403
2404 if (fiov[i].base != (unsigned long) fiov[i].base ||
2405 fiov[i].len != (unsigned long) fiov[i].len)
2406 return -EIO;
2407
2408 dst[i].iov_base = (void __user *) (unsigned long) fiov[i].base;
2409 dst[i].iov_len = (size_t) fiov[i].len;
2410
2411#ifdef CONFIG_COMPAT
2412 if (is_compat &&
2413 (ptr_to_compat(dst[i].iov_base) != fiov[i].base ||
2414 (compat_size_t) dst[i].iov_len != fiov[i].len))
2415 return -EIO;
2416#endif
2417 }
2418
2419 return 0;
2420}
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
2470 unsigned int flags)
2471{
2472 struct fuse_file *ff = file->private_data;
2473 struct fuse_conn *fc = ff->fc;
2474 struct fuse_ioctl_in inarg = {
2475 .fh = ff->fh,
2476 .cmd = cmd,
2477 .arg = arg,
2478 .flags = flags
2479 };
2480 struct fuse_ioctl_out outarg;
2481 struct fuse_req *req = NULL;
2482 struct page **pages = NULL;
2483 struct iovec *iov_page = NULL;
2484 struct iovec *in_iov = NULL, *out_iov = NULL;
2485 unsigned int in_iovs = 0, out_iovs = 0, num_pages = 0, max_pages;
2486 size_t in_size, out_size, transferred, c;
2487 int err, i;
2488 struct iov_iter ii;
2489
2490#if BITS_PER_LONG == 32
2491 inarg.flags |= FUSE_IOCTL_32BIT;
2492#else
2493 if (flags & FUSE_IOCTL_COMPAT)
2494 inarg.flags |= FUSE_IOCTL_32BIT;
2495#endif
2496
2497
2498 BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE);
2499
2500 err = -ENOMEM;
2501 pages = kcalloc(FUSE_MAX_PAGES_PER_REQ, sizeof(pages[0]), GFP_KERNEL);
2502 iov_page = (struct iovec *) __get_free_page(GFP_KERNEL);
2503 if (!pages || !iov_page)
2504 goto out;
2505
2506
2507
2508
2509
2510 if (!(flags & FUSE_IOCTL_UNRESTRICTED)) {
2511 struct iovec *iov = iov_page;
2512
2513 iov->iov_base = (void __user *)arg;
2514 iov->iov_len = _IOC_SIZE(cmd);
2515
2516 if (_IOC_DIR(cmd) & _IOC_WRITE) {
2517 in_iov = iov;
2518 in_iovs = 1;
2519 }
2520
2521 if (_IOC_DIR(cmd) & _IOC_READ) {
2522 out_iov = iov;
2523 out_iovs = 1;
2524 }
2525 }
2526
2527 retry:
2528 inarg.in_size = in_size = iov_length(in_iov, in_iovs);
2529 inarg.out_size = out_size = iov_length(out_iov, out_iovs);
2530
2531
2532
2533
2534
2535 out_size = max_t(size_t, out_size, PAGE_SIZE);
2536 max_pages = DIV_ROUND_UP(max(in_size, out_size), PAGE_SIZE);
2537
2538
2539 err = -ENOMEM;
2540 if (max_pages > FUSE_MAX_PAGES_PER_REQ)
2541 goto out;
2542 while (num_pages < max_pages) {
2543 pages[num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
2544 if (!pages[num_pages])
2545 goto out;
2546 num_pages++;
2547 }
2548
2549 req = fuse_get_req(fc, num_pages);
2550 if (IS_ERR(req)) {
2551 err = PTR_ERR(req);
2552 req = NULL;
2553 goto out;
2554 }
2555 memcpy(req->pages, pages, sizeof(req->pages[0]) * num_pages);
2556 req->num_pages = num_pages;
2557 fuse_page_descs_length_init(req, 0, req->num_pages);
2558
2559
2560 req->in.h.opcode = FUSE_IOCTL;
2561 req->in.h.nodeid = ff->nodeid;
2562 req->in.numargs = 1;
2563 req->in.args[0].size = sizeof(inarg);
2564 req->in.args[0].value = &inarg;
2565 if (in_size) {
2566 req->in.numargs++;
2567 req->in.args[1].size = in_size;
2568 req->in.argpages = 1;
2569
2570 err = -EFAULT;
2571 iov_iter_init(&ii, WRITE, in_iov, in_iovs, in_size);
2572 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= num_pages); i++) {
2573 c = copy_page_from_iter(pages[i], 0, PAGE_SIZE, &ii);
2574 if (c != PAGE_SIZE && iov_iter_count(&ii))
2575 goto out;
2576 }
2577 }
2578
2579 req->out.numargs = 2;
2580 req->out.args[0].size = sizeof(outarg);
2581 req->out.args[0].value = &outarg;
2582 req->out.args[1].size = out_size;
2583 req->out.argpages = 1;
2584 req->out.argvar = 1;
2585
2586 fuse_request_send(fc, req);
2587 err = req->out.h.error;
2588 transferred = req->out.args[1].size;
2589 fuse_put_request(fc, req);
2590 req = NULL;
2591 if (err)
2592 goto out;
2593
2594
2595 if (outarg.flags & FUSE_IOCTL_RETRY) {
2596 void *vaddr;
2597
2598
2599 err = -EIO;
2600 if (!(flags & FUSE_IOCTL_UNRESTRICTED))
2601 goto out;
2602
2603 in_iovs = outarg.in_iovs;
2604 out_iovs = outarg.out_iovs;
2605
2606
2607
2608
2609
2610 err = -ENOMEM;
2611 if (in_iovs > FUSE_IOCTL_MAX_IOV ||
2612 out_iovs > FUSE_IOCTL_MAX_IOV ||
2613 in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV)
2614 goto out;
2615
2616 vaddr = kmap_atomic(pages[0]);
2617 err = fuse_copy_ioctl_iovec(fc, iov_page, vaddr,
2618 transferred, in_iovs + out_iovs,
2619 (flags & FUSE_IOCTL_COMPAT) != 0);
2620 kunmap_atomic(vaddr);
2621 if (err)
2622 goto out;
2623
2624 in_iov = iov_page;
2625 out_iov = in_iov + in_iovs;
2626
2627 err = fuse_verify_ioctl_iov(in_iov, in_iovs);
2628 if (err)
2629 goto out;
2630
2631 err = fuse_verify_ioctl_iov(out_iov, out_iovs);
2632 if (err)
2633 goto out;
2634
2635 goto retry;
2636 }
2637
2638 err = -EIO;
2639 if (transferred > inarg.out_size)
2640 goto out;
2641
2642 err = -EFAULT;
2643 iov_iter_init(&ii, READ, out_iov, out_iovs, transferred);
2644 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= num_pages); i++) {
2645 c = copy_page_to_iter(pages[i], 0, PAGE_SIZE, &ii);
2646 if (c != PAGE_SIZE && iov_iter_count(&ii))
2647 goto out;
2648 }
2649 err = 0;
2650 out:
2651 if (req)
2652 fuse_put_request(fc, req);
2653 free_page((unsigned long) iov_page);
2654 while (num_pages)
2655 __free_page(pages[--num_pages]);
2656 kfree(pages);
2657
2658 return err ? err : outarg.result;
2659}
2660EXPORT_SYMBOL_GPL(fuse_do_ioctl);
2661
2662long fuse_ioctl_common(struct file *file, unsigned int cmd,
2663 unsigned long arg, unsigned int flags)
2664{
2665 struct inode *inode = file_inode(file);
2666 struct fuse_conn *fc = get_fuse_conn(inode);
2667
2668 if (!fuse_allow_current_process(fc))
2669 return -EACCES;
2670
2671 if (is_bad_inode(inode))
2672 return -EIO;
2673
2674 return fuse_do_ioctl(file, cmd, arg, flags);
2675}
2676
2677static long fuse_file_ioctl(struct file *file, unsigned int cmd,
2678 unsigned long arg)
2679{
2680 return fuse_ioctl_common(file, cmd, arg, 0);
2681}
2682
2683static long fuse_file_compat_ioctl(struct file *file, unsigned int cmd,
2684 unsigned long arg)
2685{
2686 return fuse_ioctl_common(file, cmd, arg, FUSE_IOCTL_COMPAT);
2687}
2688
2689
2690
2691
2692
2693
2694static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
2695 struct rb_node **parent_out)
2696{
2697 struct rb_node **link = &fc->polled_files.rb_node;
2698 struct rb_node *last = NULL;
2699
2700 while (*link) {
2701 struct fuse_file *ff;
2702
2703 last = *link;
2704 ff = rb_entry(last, struct fuse_file, polled_node);
2705
2706 if (kh < ff->kh)
2707 link = &last->rb_left;
2708 else if (kh > ff->kh)
2709 link = &last->rb_right;
2710 else
2711 return link;
2712 }
2713
2714 if (parent_out)
2715 *parent_out = last;
2716 return link;
2717}
2718
2719
2720
2721
2722
2723
2724
2725static void fuse_register_polled_file(struct fuse_conn *fc,
2726 struct fuse_file *ff)
2727{
2728 spin_lock(&fc->lock);
2729 if (RB_EMPTY_NODE(&ff->polled_node)) {
2730 struct rb_node **link, *uninitialized_var(parent);
2731
2732 link = fuse_find_polled_node(fc, ff->kh, &parent);
2733 BUG_ON(*link);
2734 rb_link_node(&ff->polled_node, parent, link);
2735 rb_insert_color(&ff->polled_node, &fc->polled_files);
2736 }
2737 spin_unlock(&fc->lock);
2738}
2739
2740unsigned fuse_file_poll(struct file *file, poll_table *wait)
2741{
2742 struct fuse_file *ff = file->private_data;
2743 struct fuse_conn *fc = ff->fc;
2744 struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
2745 struct fuse_poll_out outarg;
2746 FUSE_ARGS(args);
2747 int err;
2748
2749 if (fc->no_poll)
2750 return DEFAULT_POLLMASK;
2751
2752 poll_wait(file, &ff->poll_wait, wait);
2753 inarg.events = (__u32)poll_requested_events(wait);
2754
2755
2756
2757
2758
2759 if (waitqueue_active(&ff->poll_wait)) {
2760 inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
2761 fuse_register_polled_file(fc, ff);
2762 }
2763
2764 args.in.h.opcode = FUSE_POLL;
2765 args.in.h.nodeid = ff->nodeid;
2766 args.in.numargs = 1;
2767 args.in.args[0].size = sizeof(inarg);
2768 args.in.args[0].value = &inarg;
2769 args.out.numargs = 1;
2770 args.out.args[0].size = sizeof(outarg);
2771 args.out.args[0].value = &outarg;
2772 err = fuse_simple_request(fc, &args);
2773
2774 if (!err)
2775 return outarg.revents;
2776 if (err == -ENOSYS) {
2777 fc->no_poll = 1;
2778 return DEFAULT_POLLMASK;
2779 }
2780 return POLLERR;
2781}
2782EXPORT_SYMBOL_GPL(fuse_file_poll);
2783
2784
2785
2786
2787
2788int fuse_notify_poll_wakeup(struct fuse_conn *fc,
2789 struct fuse_notify_poll_wakeup_out *outarg)
2790{
2791 u64 kh = outarg->kh;
2792 struct rb_node **link;
2793
2794 spin_lock(&fc->lock);
2795
2796 link = fuse_find_polled_node(fc, kh, NULL);
2797 if (*link) {
2798 struct fuse_file *ff;
2799
2800 ff = rb_entry(*link, struct fuse_file, polled_node);
2801 wake_up_interruptible_sync(&ff->poll_wait);
2802 }
2803
2804 spin_unlock(&fc->lock);
2805 return 0;
2806}
2807
2808static void fuse_do_truncate(struct file *file)
2809{
2810 struct inode *inode = file->f_mapping->host;
2811 struct iattr attr;
2812
2813 attr.ia_valid = ATTR_SIZE;
2814 attr.ia_size = i_size_read(inode);
2815
2816 attr.ia_file = file;
2817 attr.ia_valid |= ATTR_FILE;
2818
2819 fuse_do_setattr(file_dentry(file), &attr, file);
2820}
2821
2822static inline loff_t fuse_round_up(loff_t off)
2823{
2824 return round_up(off, FUSE_MAX_PAGES_PER_REQ << PAGE_SHIFT);
2825}
2826
2827static ssize_t
2828fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
2829{
2830 DECLARE_COMPLETION_ONSTACK(wait);
2831 ssize_t ret = 0;
2832 struct file *file = iocb->ki_filp;
2833 struct fuse_file *ff = file->private_data;
2834 bool async_dio = ff->fc->async_dio;
2835 loff_t pos = 0;
2836 struct inode *inode;
2837 loff_t i_size;
2838 size_t count = iov_iter_count(iter);
2839 loff_t offset = iocb->ki_pos;
2840 struct fuse_io_priv *io;
2841
2842 pos = offset;
2843 inode = file->f_mapping->host;
2844 i_size = i_size_read(inode);
2845
2846 if ((iov_iter_rw(iter) == READ) && (offset > i_size))
2847 return 0;
2848
2849
2850 if (async_dio && iov_iter_rw(iter) != WRITE && offset + count > i_size) {
2851 if (offset >= i_size)
2852 return 0;
2853 iov_iter_truncate(iter, fuse_round_up(i_size - offset));
2854 count = iov_iter_count(iter);
2855 }
2856
2857 io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL);
2858 if (!io)
2859 return -ENOMEM;
2860 spin_lock_init(&io->lock);
2861 kref_init(&io->refcnt);
2862 io->reqs = 1;
2863 io->bytes = -1;
2864 io->size = 0;
2865 io->offset = offset;
2866 io->write = (iov_iter_rw(iter) == WRITE);
2867 io->err = 0;
2868 io->file = file;
2869
2870
2871
2872
2873 io->async = async_dio;
2874 io->iocb = iocb;
2875 io->blocking = is_sync_kiocb(iocb);
2876
2877
2878
2879
2880
2881 if ((offset + count > i_size) && iov_iter_rw(iter) == WRITE)
2882 io->blocking = true;
2883
2884 if (io->async && io->blocking) {
2885
2886
2887
2888
2889 kref_get(&io->refcnt);
2890 io->done = &wait;
2891 }
2892
2893 if (iov_iter_rw(iter) == WRITE) {
2894 ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE);
2895 fuse_invalidate_attr(inode);
2896 } else {
2897 ret = __fuse_direct_read(io, iter, &pos);
2898 }
2899
2900 if (io->async) {
2901 fuse_aio_complete(io, ret < 0 ? ret : 0, -1);
2902
2903
2904 if (!io->blocking)
2905 return -EIOCBQUEUED;
2906
2907 wait_for_completion(&wait);
2908 ret = fuse_get_res_by_io(io);
2909 }
2910
2911 kref_put(&io->refcnt, fuse_io_release);
2912
2913 if (iov_iter_rw(iter) == WRITE) {
2914 if (ret > 0)
2915 fuse_write_update_size(inode, pos);
2916 else if (ret < 0 && offset + count > i_size)
2917 fuse_do_truncate(file);
2918 }
2919
2920 return ret;
2921}
2922
2923static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
2924 loff_t length)
2925{
2926 struct fuse_file *ff = file->private_data;
2927 struct inode *inode = file_inode(file);
2928 struct fuse_inode *fi = get_fuse_inode(inode);
2929 struct fuse_conn *fc = ff->fc;
2930 FUSE_ARGS(args);
2931 struct fuse_fallocate_in inarg = {
2932 .fh = ff->fh,
2933 .offset = offset,
2934 .length = length,
2935 .mode = mode
2936 };
2937 int err;
2938 bool lock_inode = !(mode & FALLOC_FL_KEEP_SIZE) ||
2939 (mode & FALLOC_FL_PUNCH_HOLE);
2940
2941 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2942 return -EOPNOTSUPP;
2943
2944 if (fc->no_fallocate)
2945 return -EOPNOTSUPP;
2946
2947 if (lock_inode) {
2948 inode_lock(inode);
2949 if (mode & FALLOC_FL_PUNCH_HOLE) {
2950 loff_t endbyte = offset + length - 1;
2951 err = filemap_write_and_wait_range(inode->i_mapping,
2952 offset, endbyte);
2953 if (err)
2954 goto out;
2955
2956 fuse_sync_writes(inode);
2957 }
2958 }
2959
2960 if (!(mode & FALLOC_FL_KEEP_SIZE))
2961 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
2962
2963 args.in.h.opcode = FUSE_FALLOCATE;
2964 args.in.h.nodeid = ff->nodeid;
2965 args.in.numargs = 1;
2966 args.in.args[0].size = sizeof(inarg);
2967 args.in.args[0].value = &inarg;
2968 err = fuse_simple_request(fc, &args);
2969 if (err == -ENOSYS) {
2970 fc->no_fallocate = 1;
2971 err = -EOPNOTSUPP;
2972 }
2973 if (err)
2974 goto out;
2975
2976
2977 if (!(mode & FALLOC_FL_KEEP_SIZE)) {
2978 bool changed = fuse_write_update_size(inode, offset + length);
2979
2980 if (changed && fc->writeback_cache)
2981 file_update_time(file);
2982 }
2983
2984 if (mode & FALLOC_FL_PUNCH_HOLE)
2985 truncate_pagecache_range(inode, offset, offset + length - 1);
2986
2987 fuse_invalidate_attr(inode);
2988
2989out:
2990 if (!(mode & FALLOC_FL_KEEP_SIZE))
2991 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
2992
2993 if (lock_inode)
2994 inode_unlock(inode);
2995
2996 return err;
2997}
2998
2999static const struct file_operations fuse_file_operations = {
3000 .llseek = fuse_file_llseek,
3001 .read_iter = fuse_file_read_iter,
3002 .write_iter = fuse_file_write_iter,
3003 .mmap = fuse_file_mmap,
3004 .open = fuse_open,
3005 .flush = fuse_flush,
3006 .release = fuse_release,
3007 .fsync = fuse_fsync,
3008 .lock = fuse_file_lock,
3009 .flock = fuse_file_flock,
3010 .splice_read = generic_file_splice_read,
3011 .unlocked_ioctl = fuse_file_ioctl,
3012 .compat_ioctl = fuse_file_compat_ioctl,
3013 .poll = fuse_file_poll,
3014 .fallocate = fuse_file_fallocate,
3015};
3016
3017static const struct file_operations fuse_direct_io_file_operations = {
3018 .llseek = fuse_file_llseek,
3019 .read_iter = fuse_direct_read_iter,
3020 .write_iter = fuse_direct_write_iter,
3021 .mmap = fuse_direct_mmap,
3022 .open = fuse_open,
3023 .flush = fuse_flush,
3024 .release = fuse_release,
3025 .fsync = fuse_fsync,
3026 .lock = fuse_file_lock,
3027 .flock = fuse_file_flock,
3028 .unlocked_ioctl = fuse_file_ioctl,
3029 .compat_ioctl = fuse_file_compat_ioctl,
3030 .poll = fuse_file_poll,
3031 .fallocate = fuse_file_fallocate,
3032
3033};
3034
3035static const struct address_space_operations fuse_file_aops = {
3036 .readpage = fuse_readpage,
3037 .writepage = fuse_writepage,
3038 .writepages = fuse_writepages,
3039 .launder_page = fuse_launder_page,
3040 .readpages = fuse_readpages,
3041 .set_page_dirty = __set_page_dirty_nobuffers,
3042 .bmap = fuse_bmap,
3043 .direct_IO = fuse_direct_IO,
3044 .write_begin = fuse_write_begin,
3045 .write_end = fuse_write_end,
3046};
3047
3048void fuse_init_file_inode(struct inode *inode)
3049{
3050 inode->i_fop = &fuse_file_operations;
3051 inode->i_data.a_ops = &fuse_file_aops;
3052}
3053