1
2
3
4
5
6
7
8
9#include "fuse_i.h"
10
11#include <linux/pagemap.h>
12#include <linux/slab.h>
13#include <linux/kernel.h>
14#include <linux/sched.h>
15#include <linux/sched/signal.h>
16#include <linux/module.h>
17#include <linux/compat.h>
18#include <linux/swap.h>
19#include <linux/falloc.h>
20#include <linux/uio.h>
21
22static int fuse_send_open(struct fuse_conn *fc, u64 nodeid, struct file *file,
23 int opcode, struct fuse_open_out *outargp)
24{
25 struct fuse_open_in inarg;
26 FUSE_ARGS(args);
27
28 memset(&inarg, 0, sizeof(inarg));
29 inarg.flags = file->f_flags & ~(O_CREAT | O_EXCL | O_NOCTTY);
30 if (!fc->atomic_o_trunc)
31 inarg.flags &= ~O_TRUNC;
32 args.in.h.opcode = opcode;
33 args.in.h.nodeid = nodeid;
34 args.in.numargs = 1;
35 args.in.args[0].size = sizeof(inarg);
36 args.in.args[0].value = &inarg;
37 args.out.numargs = 1;
38 args.out.args[0].size = sizeof(*outargp);
39 args.out.args[0].value = outargp;
40
41 return fuse_simple_request(fc, &args);
42}
43
44struct fuse_file *fuse_file_alloc(struct fuse_conn *fc)
45{
46 struct fuse_file *ff;
47
48 ff = kzalloc(sizeof(struct fuse_file), GFP_KERNEL);
49 if (unlikely(!ff))
50 return NULL;
51
52 ff->fc = fc;
53 ff->reserved_req = fuse_request_alloc(0);
54 if (unlikely(!ff->reserved_req)) {
55 kfree(ff);
56 return NULL;
57 }
58
59 INIT_LIST_HEAD(&ff->write_entry);
60 mutex_init(&ff->readdir.lock);
61 refcount_set(&ff->count, 1);
62 RB_CLEAR_NODE(&ff->polled_node);
63 init_waitqueue_head(&ff->poll_wait);
64
65 ff->kh = atomic64_inc_return(&fc->khctr);
66
67 return ff;
68}
69
70void fuse_file_free(struct fuse_file *ff)
71{
72 fuse_request_free(ff->reserved_req);
73 mutex_destroy(&ff->readdir.lock);
74 kfree(ff);
75}
76
77static struct fuse_file *fuse_file_get(struct fuse_file *ff)
78{
79 refcount_inc(&ff->count);
80 return ff;
81}
82
83static void fuse_release_end(struct fuse_conn *fc, struct fuse_req *req)
84{
85 iput(req->misc.release.inode);
86}
87
88static void fuse_file_put(struct fuse_file *ff, bool sync, bool isdir)
89{
90 if (refcount_dec_and_test(&ff->count)) {
91 struct fuse_req *req = ff->reserved_req;
92
93 if (isdir ? ff->fc->no_opendir : ff->fc->no_open) {
94
95
96
97
98 __clear_bit(FR_BACKGROUND, &req->flags);
99 iput(req->misc.release.inode);
100 fuse_put_request(ff->fc, req);
101 } else if (sync) {
102 __set_bit(FR_FORCE, &req->flags);
103 __clear_bit(FR_BACKGROUND, &req->flags);
104 fuse_request_send(ff->fc, req);
105 iput(req->misc.release.inode);
106 fuse_put_request(ff->fc, req);
107 } else {
108 req->end = fuse_release_end;
109 __set_bit(FR_BACKGROUND, &req->flags);
110 fuse_request_send_background(ff->fc, req);
111 }
112 kfree(ff);
113 }
114}
115
116int fuse_do_open(struct fuse_conn *fc, u64 nodeid, struct file *file,
117 bool isdir)
118{
119 struct fuse_file *ff;
120 int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN;
121
122 ff = fuse_file_alloc(fc);
123 if (!ff)
124 return -ENOMEM;
125
126 ff->fh = 0;
127
128 ff->open_flags = FOPEN_KEEP_CACHE | (isdir ? FOPEN_CACHE_DIR : 0);
129 if (isdir ? !fc->no_opendir : !fc->no_open) {
130 struct fuse_open_out outarg;
131 int err;
132
133 err = fuse_send_open(fc, nodeid, file, opcode, &outarg);
134 if (!err) {
135 ff->fh = outarg.fh;
136 ff->open_flags = outarg.open_flags;
137
138 } else if (err != -ENOSYS) {
139 fuse_file_free(ff);
140 return err;
141 } else {
142 if (isdir)
143 fc->no_opendir = 1;
144 else
145 fc->no_open = 1;
146 }
147 }
148
149 if (isdir)
150 ff->open_flags &= ~FOPEN_DIRECT_IO;
151
152 ff->nodeid = nodeid;
153 file->private_data = ff;
154
155 return 0;
156}
157EXPORT_SYMBOL_GPL(fuse_do_open);
158
159static void fuse_link_write_file(struct file *file)
160{
161 struct inode *inode = file_inode(file);
162 struct fuse_inode *fi = get_fuse_inode(inode);
163 struct fuse_file *ff = file->private_data;
164
165
166
167
168 spin_lock(&fi->lock);
169 if (list_empty(&ff->write_entry))
170 list_add(&ff->write_entry, &fi->write_files);
171 spin_unlock(&fi->lock);
172}
173
174void fuse_finish_open(struct inode *inode, struct file *file)
175{
176 struct fuse_file *ff = file->private_data;
177 struct fuse_conn *fc = get_fuse_conn(inode);
178
179 if (!(ff->open_flags & FOPEN_KEEP_CACHE))
180 invalidate_inode_pages2(inode->i_mapping);
181 if (ff->open_flags & FOPEN_STREAM)
182 stream_open(inode, file);
183 else if (ff->open_flags & FOPEN_NONSEEKABLE)
184 nonseekable_open(inode, file);
185 if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) {
186 struct fuse_inode *fi = get_fuse_inode(inode);
187
188 spin_lock(&fi->lock);
189 fi->attr_version = atomic64_inc_return(&fc->attr_version);
190 i_size_write(inode, 0);
191 spin_unlock(&fi->lock);
192 fuse_invalidate_attr(inode);
193 if (fc->writeback_cache)
194 file_update_time(file);
195 }
196 if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache)
197 fuse_link_write_file(file);
198}
199
200int fuse_open_common(struct inode *inode, struct file *file, bool isdir)
201{
202 struct fuse_conn *fc = get_fuse_conn(inode);
203 int err;
204 bool lock_inode = (file->f_flags & O_TRUNC) &&
205 fc->atomic_o_trunc &&
206 fc->writeback_cache;
207
208 err = generic_file_open(inode, file);
209 if (err)
210 return err;
211
212 if (lock_inode)
213 inode_lock(inode);
214
215 err = fuse_do_open(fc, get_node_id(inode), file, isdir);
216
217 if (!err)
218 fuse_finish_open(inode, file);
219
220 if (lock_inode)
221 inode_unlock(inode);
222
223 return err;
224}
225
226static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff,
227 int flags, int opcode)
228{
229 struct fuse_conn *fc = ff->fc;
230 struct fuse_req *req = ff->reserved_req;
231 struct fuse_release_in *inarg = &req->misc.release.in;
232
233
234 if (likely(fi)) {
235 spin_lock(&fi->lock);
236 list_del(&ff->write_entry);
237 spin_unlock(&fi->lock);
238 }
239 spin_lock(&fc->lock);
240 if (!RB_EMPTY_NODE(&ff->polled_node))
241 rb_erase(&ff->polled_node, &fc->polled_files);
242 spin_unlock(&fc->lock);
243
244 wake_up_interruptible_all(&ff->poll_wait);
245
246 inarg->fh = ff->fh;
247 inarg->flags = flags;
248 req->in.h.opcode = opcode;
249 req->in.h.nodeid = ff->nodeid;
250 req->in.numargs = 1;
251 req->in.args[0].size = sizeof(struct fuse_release_in);
252 req->in.args[0].value = inarg;
253}
254
255void fuse_release_common(struct file *file, bool isdir)
256{
257 struct fuse_inode *fi = get_fuse_inode(file_inode(file));
258 struct fuse_file *ff = file->private_data;
259 struct fuse_req *req = ff->reserved_req;
260 int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE;
261
262 fuse_prepare_release(fi, ff, file->f_flags, opcode);
263
264 if (ff->flock) {
265 struct fuse_release_in *inarg = &req->misc.release.in;
266 inarg->release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
267 inarg->lock_owner = fuse_lock_owner_id(ff->fc,
268 (fl_owner_t) file);
269 }
270
271 req->misc.release.inode = igrab(file_inode(file));
272
273
274
275
276
277
278
279
280
281
282 fuse_file_put(ff, ff->fc->destroy_req != NULL, isdir);
283}
284
285static int fuse_open(struct inode *inode, struct file *file)
286{
287 return fuse_open_common(inode, file, false);
288}
289
290static int fuse_release(struct inode *inode, struct file *file)
291{
292 struct fuse_conn *fc = get_fuse_conn(inode);
293
294
295 if (fc->writeback_cache)
296 write_inode_now(inode, 1);
297
298 fuse_release_common(file, false);
299
300
301 return 0;
302}
303
304void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, int flags)
305{
306 WARN_ON(refcount_read(&ff->count) > 1);
307 fuse_prepare_release(fi, ff, flags, FUSE_RELEASE);
308
309
310
311
312 fuse_file_put(ff, true, false);
313}
314EXPORT_SYMBOL_GPL(fuse_sync_release);
315
316
317
318
319
320u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id)
321{
322 u32 *k = fc->scramble_key;
323 u64 v = (unsigned long) id;
324 u32 v0 = v;
325 u32 v1 = v >> 32;
326 u32 sum = 0;
327 int i;
328
329 for (i = 0; i < 32; i++) {
330 v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]);
331 sum += 0x9E3779B9;
332 v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]);
333 }
334
335 return (u64) v0 + ((u64) v1 << 32);
336}
337
338static struct fuse_req *fuse_find_writeback(struct fuse_inode *fi,
339 pgoff_t idx_from, pgoff_t idx_to)
340{
341 struct fuse_req *req;
342
343 list_for_each_entry(req, &fi->writepages, writepages_entry) {
344 pgoff_t curr_index;
345
346 WARN_ON(get_fuse_inode(req->inode) != fi);
347 curr_index = req->misc.write.in.offset >> PAGE_SHIFT;
348 if (idx_from < curr_index + req->num_pages &&
349 curr_index <= idx_to) {
350 return req;
351 }
352 }
353 return NULL;
354}
355
356
357
358
359
360
361
362static bool fuse_range_is_writeback(struct inode *inode, pgoff_t idx_from,
363 pgoff_t idx_to)
364{
365 struct fuse_inode *fi = get_fuse_inode(inode);
366 bool found;
367
368 spin_lock(&fi->lock);
369 found = fuse_find_writeback(fi, idx_from, idx_to);
370 spin_unlock(&fi->lock);
371
372 return found;
373}
374
375static inline bool fuse_page_is_writeback(struct inode *inode, pgoff_t index)
376{
377 return fuse_range_is_writeback(inode, index, index);
378}
379
380
381
382
383
384
385
386static int fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index)
387{
388 struct fuse_inode *fi = get_fuse_inode(inode);
389
390 wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index));
391 return 0;
392}
393
394
395
396
397
398
399
400
401
402
403static void fuse_sync_writes(struct inode *inode)
404{
405 fuse_set_nowrite(inode);
406 fuse_release_nowrite(inode);
407}
408
409static int fuse_flush(struct file *file, fl_owner_t id)
410{
411 struct inode *inode = file_inode(file);
412 struct fuse_conn *fc = get_fuse_conn(inode);
413 struct fuse_file *ff = file->private_data;
414 struct fuse_req *req;
415 struct fuse_flush_in inarg;
416 int err;
417
418 if (is_bad_inode(inode))
419 return -EIO;
420
421 if (fc->no_flush)
422 return 0;
423
424 err = write_inode_now(inode, 1);
425 if (err)
426 return err;
427
428 inode_lock(inode);
429 fuse_sync_writes(inode);
430 inode_unlock(inode);
431
432 err = filemap_check_errors(file->f_mapping);
433 if (err)
434 return err;
435
436 req = fuse_get_req_nofail_nopages(fc, file);
437 memset(&inarg, 0, sizeof(inarg));
438 inarg.fh = ff->fh;
439 inarg.lock_owner = fuse_lock_owner_id(fc, id);
440 req->in.h.opcode = FUSE_FLUSH;
441 req->in.h.nodeid = get_node_id(inode);
442 req->in.numargs = 1;
443 req->in.args[0].size = sizeof(inarg);
444 req->in.args[0].value = &inarg;
445 __set_bit(FR_FORCE, &req->flags);
446 fuse_request_send(fc, req);
447 err = req->out.h.error;
448 fuse_put_request(fc, req);
449 if (err == -ENOSYS) {
450 fc->no_flush = 1;
451 err = 0;
452 }
453 return err;
454}
455
456int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
457 int datasync, int opcode)
458{
459 struct inode *inode = file->f_mapping->host;
460 struct fuse_conn *fc = get_fuse_conn(inode);
461 struct fuse_file *ff = file->private_data;
462 FUSE_ARGS(args);
463 struct fuse_fsync_in inarg;
464
465 memset(&inarg, 0, sizeof(inarg));
466 inarg.fh = ff->fh;
467 inarg.fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0;
468 args.in.h.opcode = opcode;
469 args.in.h.nodeid = get_node_id(inode);
470 args.in.numargs = 1;
471 args.in.args[0].size = sizeof(inarg);
472 args.in.args[0].value = &inarg;
473 return fuse_simple_request(fc, &args);
474}
475
476static int fuse_fsync(struct file *file, loff_t start, loff_t end,
477 int datasync)
478{
479 struct inode *inode = file->f_mapping->host;
480 struct fuse_conn *fc = get_fuse_conn(inode);
481 int err;
482
483 if (is_bad_inode(inode))
484 return -EIO;
485
486 inode_lock(inode);
487
488
489
490
491
492
493 err = file_write_and_wait_range(file, start, end);
494 if (err)
495 goto out;
496
497 fuse_sync_writes(inode);
498
499
500
501
502
503
504 err = file_check_and_advance_wb_err(file);
505 if (err)
506 goto out;
507
508 err = sync_inode_metadata(inode, 1);
509 if (err)
510 goto out;
511
512 if (fc->no_fsync)
513 goto out;
514
515 err = fuse_fsync_common(file, start, end, datasync, FUSE_FSYNC);
516 if (err == -ENOSYS) {
517 fc->no_fsync = 1;
518 err = 0;
519 }
520out:
521 inode_unlock(inode);
522
523 return err;
524}
525
526void fuse_read_fill(struct fuse_req *req, struct file *file, loff_t pos,
527 size_t count, int opcode)
528{
529 struct fuse_read_in *inarg = &req->misc.read.in;
530 struct fuse_file *ff = file->private_data;
531
532 inarg->fh = ff->fh;
533 inarg->offset = pos;
534 inarg->size = count;
535 inarg->flags = file->f_flags;
536 req->in.h.opcode = opcode;
537 req->in.h.nodeid = ff->nodeid;
538 req->in.numargs = 1;
539 req->in.args[0].size = sizeof(struct fuse_read_in);
540 req->in.args[0].value = inarg;
541 req->out.argvar = 1;
542 req->out.numargs = 1;
543 req->out.args[0].size = count;
544}
545
546static void fuse_release_user_pages(struct fuse_req *req, bool should_dirty)
547{
548 unsigned i;
549
550 for (i = 0; i < req->num_pages; i++) {
551 struct page *page = req->pages[i];
552 if (should_dirty)
553 set_page_dirty_lock(page);
554 put_page(page);
555 }
556}
557
558static void fuse_io_release(struct kref *kref)
559{
560 kfree(container_of(kref, struct fuse_io_priv, refcnt));
561}
562
563static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)
564{
565 if (io->err)
566 return io->err;
567
568 if (io->bytes >= 0 && io->write)
569 return -EIO;
570
571 return io->bytes < 0 ? io->size : io->bytes;
572}
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
591{
592 int left;
593
594 spin_lock(&io->lock);
595 if (err)
596 io->err = io->err ? : err;
597 else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))
598 io->bytes = pos;
599
600 left = --io->reqs;
601 if (!left && io->blocking)
602 complete(io->done);
603 spin_unlock(&io->lock);
604
605 if (!left && !io->blocking) {
606 ssize_t res = fuse_get_res_by_io(io);
607
608 if (res >= 0) {
609 struct inode *inode = file_inode(io->iocb->ki_filp);
610 struct fuse_conn *fc = get_fuse_conn(inode);
611 struct fuse_inode *fi = get_fuse_inode(inode);
612
613 spin_lock(&fi->lock);
614 fi->attr_version = atomic64_inc_return(&fc->attr_version);
615 spin_unlock(&fi->lock);
616 }
617
618 io->iocb->ki_complete(io->iocb, res, 0);
619 }
620
621 kref_put(&io->refcnt, fuse_io_release);
622}
623
624static void fuse_aio_complete_req(struct fuse_conn *fc, struct fuse_req *req)
625{
626 struct fuse_io_priv *io = req->io;
627 ssize_t pos = -1;
628
629 fuse_release_user_pages(req, io->should_dirty);
630
631 if (io->write) {
632 if (req->misc.write.in.size != req->misc.write.out.size)
633 pos = req->misc.write.in.offset - io->offset +
634 req->misc.write.out.size;
635 } else {
636 if (req->misc.read.in.size != req->out.args[0].size)
637 pos = req->misc.read.in.offset - io->offset +
638 req->out.args[0].size;
639 }
640
641 fuse_aio_complete(io, req->out.h.error, pos);
642}
643
644static size_t fuse_async_req_send(struct fuse_conn *fc, struct fuse_req *req,
645 size_t num_bytes, struct fuse_io_priv *io)
646{
647 spin_lock(&io->lock);
648 kref_get(&io->refcnt);
649 io->size += num_bytes;
650 io->reqs++;
651 spin_unlock(&io->lock);
652
653 req->io = io;
654 req->end = fuse_aio_complete_req;
655
656 __fuse_get_request(req);
657 fuse_request_send_background(fc, req);
658
659 return num_bytes;
660}
661
662static size_t fuse_send_read(struct fuse_req *req, struct fuse_io_priv *io,
663 loff_t pos, size_t count, fl_owner_t owner)
664{
665 struct file *file = io->iocb->ki_filp;
666 struct fuse_file *ff = file->private_data;
667 struct fuse_conn *fc = ff->fc;
668
669 fuse_read_fill(req, file, pos, count, FUSE_READ);
670 if (owner != NULL) {
671 struct fuse_read_in *inarg = &req->misc.read.in;
672
673 inarg->read_flags |= FUSE_READ_LOCKOWNER;
674 inarg->lock_owner = fuse_lock_owner_id(fc, owner);
675 }
676
677 if (io->async)
678 return fuse_async_req_send(fc, req, count, io);
679
680 fuse_request_send(fc, req);
681 return req->out.args[0].size;
682}
683
684static void fuse_read_update_size(struct inode *inode, loff_t size,
685 u64 attr_ver)
686{
687 struct fuse_conn *fc = get_fuse_conn(inode);
688 struct fuse_inode *fi = get_fuse_inode(inode);
689
690 spin_lock(&fi->lock);
691 if (attr_ver == fi->attr_version && size < inode->i_size &&
692 !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) {
693 fi->attr_version = atomic64_inc_return(&fc->attr_version);
694 i_size_write(inode, size);
695 }
696 spin_unlock(&fi->lock);
697}
698
699static void fuse_short_read(struct fuse_req *req, struct inode *inode,
700 u64 attr_ver)
701{
702 size_t num_read = req->out.args[0].size;
703 struct fuse_conn *fc = get_fuse_conn(inode);
704
705 if (fc->writeback_cache) {
706
707
708
709
710
711 int i;
712 int start_idx = num_read >> PAGE_SHIFT;
713 size_t off = num_read & (PAGE_SIZE - 1);
714
715 for (i = start_idx; i < req->num_pages; i++) {
716 zero_user_segment(req->pages[i], off, PAGE_SIZE);
717 off = 0;
718 }
719 } else {
720 loff_t pos = page_offset(req->pages[0]) + num_read;
721 fuse_read_update_size(inode, pos, attr_ver);
722 }
723}
724
725static int fuse_do_readpage(struct file *file, struct page *page)
726{
727 struct kiocb iocb;
728 struct fuse_io_priv io;
729 struct inode *inode = page->mapping->host;
730 struct fuse_conn *fc = get_fuse_conn(inode);
731 struct fuse_req *req;
732 size_t num_read;
733 loff_t pos = page_offset(page);
734 size_t count = PAGE_SIZE;
735 u64 attr_ver;
736 int err;
737
738
739
740
741
742
743 fuse_wait_on_page_writeback(inode, page->index);
744
745 req = fuse_get_req(fc, 1);
746 if (IS_ERR(req))
747 return PTR_ERR(req);
748
749 attr_ver = fuse_get_attr_version(fc);
750
751 req->out.page_zeroing = 1;
752 req->out.argpages = 1;
753 req->num_pages = 1;
754 req->pages[0] = page;
755 req->page_descs[0].length = count;
756 init_sync_kiocb(&iocb, file);
757 io = (struct fuse_io_priv) FUSE_IO_PRIV_SYNC(&iocb);
758 num_read = fuse_send_read(req, &io, pos, count, NULL);
759 err = req->out.h.error;
760
761 if (!err) {
762
763
764
765 if (num_read < count)
766 fuse_short_read(req, inode, attr_ver);
767
768 SetPageUptodate(page);
769 }
770
771 fuse_put_request(fc, req);
772
773 return err;
774}
775
776static int fuse_readpage(struct file *file, struct page *page)
777{
778 struct inode *inode = page->mapping->host;
779 int err;
780
781 err = -EIO;
782 if (is_bad_inode(inode))
783 goto out;
784
785 err = fuse_do_readpage(file, page);
786 fuse_invalidate_atime(inode);
787 out:
788 unlock_page(page);
789 return err;
790}
791
792static void fuse_readpages_end(struct fuse_conn *fc, struct fuse_req *req)
793{
794 int i;
795 size_t count = req->misc.read.in.size;
796 size_t num_read = req->out.args[0].size;
797 struct address_space *mapping = NULL;
798
799 for (i = 0; mapping == NULL && i < req->num_pages; i++)
800 mapping = req->pages[i]->mapping;
801
802 if (mapping) {
803 struct inode *inode = mapping->host;
804
805
806
807
808 if (!req->out.h.error && num_read < count)
809 fuse_short_read(req, inode, req->misc.read.attr_ver);
810
811 fuse_invalidate_atime(inode);
812 }
813
814 for (i = 0; i < req->num_pages; i++) {
815 struct page *page = req->pages[i];
816 if (!req->out.h.error)
817 SetPageUptodate(page);
818 else
819 SetPageError(page);
820 unlock_page(page);
821 put_page(page);
822 }
823 if (req->ff)
824 fuse_file_put(req->ff, false, false);
825}
826
827static void fuse_send_readpages(struct fuse_req *req, struct file *file)
828{
829 struct fuse_file *ff = file->private_data;
830 struct fuse_conn *fc = ff->fc;
831 loff_t pos = page_offset(req->pages[0]);
832 size_t count = req->num_pages << PAGE_SHIFT;
833
834 req->out.argpages = 1;
835 req->out.page_zeroing = 1;
836 req->out.page_replace = 1;
837 fuse_read_fill(req, file, pos, count, FUSE_READ);
838 req->misc.read.attr_ver = fuse_get_attr_version(fc);
839 if (fc->async_read) {
840 req->ff = fuse_file_get(ff);
841 req->end = fuse_readpages_end;
842 fuse_request_send_background(fc, req);
843 } else {
844 fuse_request_send(fc, req);
845 fuse_readpages_end(fc, req);
846 fuse_put_request(fc, req);
847 }
848}
849
850struct fuse_fill_data {
851 struct fuse_req *req;
852 struct file *file;
853 struct inode *inode;
854 unsigned nr_pages;
855};
856
857static int fuse_readpages_fill(void *_data, struct page *page)
858{
859 struct fuse_fill_data *data = _data;
860 struct fuse_req *req = data->req;
861 struct inode *inode = data->inode;
862 struct fuse_conn *fc = get_fuse_conn(inode);
863
864 fuse_wait_on_page_writeback(inode, page->index);
865
866 if (req->num_pages &&
867 (req->num_pages == fc->max_pages ||
868 (req->num_pages + 1) * PAGE_SIZE > fc->max_read ||
869 req->pages[req->num_pages - 1]->index + 1 != page->index)) {
870 unsigned int nr_alloc = min_t(unsigned int, data->nr_pages,
871 fc->max_pages);
872 fuse_send_readpages(req, data->file);
873 if (fc->async_read)
874 req = fuse_get_req_for_background(fc, nr_alloc);
875 else
876 req = fuse_get_req(fc, nr_alloc);
877
878 data->req = req;
879 if (IS_ERR(req)) {
880 unlock_page(page);
881 return PTR_ERR(req);
882 }
883 }
884
885 if (WARN_ON(req->num_pages >= req->max_pages)) {
886 unlock_page(page);
887 fuse_put_request(fc, req);
888 return -EIO;
889 }
890
891 get_page(page);
892 req->pages[req->num_pages] = page;
893 req->page_descs[req->num_pages].length = PAGE_SIZE;
894 req->num_pages++;
895 data->nr_pages--;
896 return 0;
897}
898
899static int fuse_readpages(struct file *file, struct address_space *mapping,
900 struct list_head *pages, unsigned nr_pages)
901{
902 struct inode *inode = mapping->host;
903 struct fuse_conn *fc = get_fuse_conn(inode);
904 struct fuse_fill_data data;
905 int err;
906 unsigned int nr_alloc = min_t(unsigned int, nr_pages, fc->max_pages);
907
908 err = -EIO;
909 if (is_bad_inode(inode))
910 goto out;
911
912 data.file = file;
913 data.inode = inode;
914 if (fc->async_read)
915 data.req = fuse_get_req_for_background(fc, nr_alloc);
916 else
917 data.req = fuse_get_req(fc, nr_alloc);
918 data.nr_pages = nr_pages;
919 err = PTR_ERR(data.req);
920 if (IS_ERR(data.req))
921 goto out;
922
923 err = read_cache_pages(mapping, pages, fuse_readpages_fill, &data);
924 if (!err) {
925 if (data.req->num_pages)
926 fuse_send_readpages(data.req, file);
927 else
928 fuse_put_request(fc, data.req);
929 }
930out:
931 return err;
932}
933
934static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
935{
936 struct inode *inode = iocb->ki_filp->f_mapping->host;
937 struct fuse_conn *fc = get_fuse_conn(inode);
938
939
940
941
942
943
944 if (fc->auto_inval_data ||
945 (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) {
946 int err;
947 err = fuse_update_attributes(inode, iocb->ki_filp);
948 if (err)
949 return err;
950 }
951
952 return generic_file_read_iter(iocb, to);
953}
954
955static void fuse_write_fill(struct fuse_req *req, struct fuse_file *ff,
956 loff_t pos, size_t count)
957{
958 struct fuse_write_in *inarg = &req->misc.write.in;
959 struct fuse_write_out *outarg = &req->misc.write.out;
960
961 inarg->fh = ff->fh;
962 inarg->offset = pos;
963 inarg->size = count;
964 req->in.h.opcode = FUSE_WRITE;
965 req->in.h.nodeid = ff->nodeid;
966 req->in.numargs = 2;
967 if (ff->fc->minor < 9)
968 req->in.args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
969 else
970 req->in.args[0].size = sizeof(struct fuse_write_in);
971 req->in.args[0].value = inarg;
972 req->in.args[1].size = count;
973 req->out.numargs = 1;
974 req->out.args[0].size = sizeof(struct fuse_write_out);
975 req->out.args[0].value = outarg;
976}
977
978static size_t fuse_send_write(struct fuse_req *req, struct fuse_io_priv *io,
979 loff_t pos, size_t count, fl_owner_t owner)
980{
981 struct kiocb *iocb = io->iocb;
982 struct file *file = iocb->ki_filp;
983 struct fuse_file *ff = file->private_data;
984 struct fuse_conn *fc = ff->fc;
985 struct fuse_write_in *inarg = &req->misc.write.in;
986
987 fuse_write_fill(req, ff, pos, count);
988 inarg->flags = file->f_flags;
989 if (iocb->ki_flags & IOCB_DSYNC)
990 inarg->flags |= O_DSYNC;
991 if (iocb->ki_flags & IOCB_SYNC)
992 inarg->flags |= O_SYNC;
993 if (owner != NULL) {
994 inarg->write_flags |= FUSE_WRITE_LOCKOWNER;
995 inarg->lock_owner = fuse_lock_owner_id(fc, owner);
996 }
997
998 if (io->async)
999 return fuse_async_req_send(fc, req, count, io);
1000
1001 fuse_request_send(fc, req);
1002 return req->misc.write.out.size;
1003}
1004
1005bool fuse_write_update_size(struct inode *inode, loff_t pos)
1006{
1007 struct fuse_conn *fc = get_fuse_conn(inode);
1008 struct fuse_inode *fi = get_fuse_inode(inode);
1009 bool ret = false;
1010
1011 spin_lock(&fi->lock);
1012 fi->attr_version = atomic64_inc_return(&fc->attr_version);
1013 if (pos > inode->i_size) {
1014 i_size_write(inode, pos);
1015 ret = true;
1016 }
1017 spin_unlock(&fi->lock);
1018
1019 return ret;
1020}
1021
1022static size_t fuse_send_write_pages(struct fuse_req *req, struct kiocb *iocb,
1023 struct inode *inode, loff_t pos,
1024 size_t count)
1025{
1026 size_t res;
1027 unsigned offset;
1028 unsigned i;
1029 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1030
1031 for (i = 0; i < req->num_pages; i++)
1032 fuse_wait_on_page_writeback(inode, req->pages[i]->index);
1033
1034 res = fuse_send_write(req, &io, pos, count, NULL);
1035
1036 offset = req->page_descs[0].offset;
1037 count = res;
1038 for (i = 0; i < req->num_pages; i++) {
1039 struct page *page = req->pages[i];
1040
1041 if (!req->out.h.error && !offset && count >= PAGE_SIZE)
1042 SetPageUptodate(page);
1043
1044 if (count > PAGE_SIZE - offset)
1045 count -= PAGE_SIZE - offset;
1046 else
1047 count = 0;
1048 offset = 0;
1049
1050 unlock_page(page);
1051 put_page(page);
1052 }
1053
1054 return res;
1055}
1056
1057static ssize_t fuse_fill_write_pages(struct fuse_req *req,
1058 struct address_space *mapping,
1059 struct iov_iter *ii, loff_t pos)
1060{
1061 struct fuse_conn *fc = get_fuse_conn(mapping->host);
1062 unsigned offset = pos & (PAGE_SIZE - 1);
1063 size_t count = 0;
1064 int err;
1065
1066 req->in.argpages = 1;
1067 req->page_descs[0].offset = offset;
1068
1069 do {
1070 size_t tmp;
1071 struct page *page;
1072 pgoff_t index = pos >> PAGE_SHIFT;
1073 size_t bytes = min_t(size_t, PAGE_SIZE - offset,
1074 iov_iter_count(ii));
1075
1076 bytes = min_t(size_t, bytes, fc->max_write - count);
1077
1078 again:
1079 err = -EFAULT;
1080 if (iov_iter_fault_in_readable(ii, bytes))
1081 break;
1082
1083 err = -ENOMEM;
1084 page = grab_cache_page_write_begin(mapping, index, 0);
1085 if (!page)
1086 break;
1087
1088 if (mapping_writably_mapped(mapping))
1089 flush_dcache_page(page);
1090
1091 tmp = iov_iter_copy_from_user_atomic(page, ii, offset, bytes);
1092 flush_dcache_page(page);
1093
1094 iov_iter_advance(ii, tmp);
1095 if (!tmp) {
1096 unlock_page(page);
1097 put_page(page);
1098 bytes = min(bytes, iov_iter_single_seg_count(ii));
1099 goto again;
1100 }
1101
1102 err = 0;
1103 req->pages[req->num_pages] = page;
1104 req->page_descs[req->num_pages].length = tmp;
1105 req->num_pages++;
1106
1107 count += tmp;
1108 pos += tmp;
1109 offset += tmp;
1110 if (offset == PAGE_SIZE)
1111 offset = 0;
1112
1113 if (!fc->big_writes)
1114 break;
1115 } while (iov_iter_count(ii) && count < fc->max_write &&
1116 req->num_pages < req->max_pages && offset == 0);
1117
1118 return count > 0 ? count : err;
1119}
1120
1121static inline unsigned int fuse_wr_pages(loff_t pos, size_t len,
1122 unsigned int max_pages)
1123{
1124 return min_t(unsigned int,
1125 ((pos + len - 1) >> PAGE_SHIFT) -
1126 (pos >> PAGE_SHIFT) + 1,
1127 max_pages);
1128}
1129
1130static ssize_t fuse_perform_write(struct kiocb *iocb,
1131 struct address_space *mapping,
1132 struct iov_iter *ii, loff_t pos)
1133{
1134 struct inode *inode = mapping->host;
1135 struct fuse_conn *fc = get_fuse_conn(inode);
1136 struct fuse_inode *fi = get_fuse_inode(inode);
1137 int err = 0;
1138 ssize_t res = 0;
1139
1140 if (inode->i_size < pos + iov_iter_count(ii))
1141 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1142
1143 do {
1144 struct fuse_req *req;
1145 ssize_t count;
1146 unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii),
1147 fc->max_pages);
1148
1149 req = fuse_get_req(fc, nr_pages);
1150 if (IS_ERR(req)) {
1151 err = PTR_ERR(req);
1152 break;
1153 }
1154
1155 count = fuse_fill_write_pages(req, mapping, ii, pos);
1156 if (count <= 0) {
1157 err = count;
1158 } else {
1159 size_t num_written;
1160
1161 num_written = fuse_send_write_pages(req, iocb, inode,
1162 pos, count);
1163 err = req->out.h.error;
1164 if (!err) {
1165 res += num_written;
1166 pos += num_written;
1167
1168
1169 if (num_written != count)
1170 err = -EIO;
1171 }
1172 }
1173 fuse_put_request(fc, req);
1174 } while (!err && iov_iter_count(ii));
1175
1176 if (res > 0)
1177 fuse_write_update_size(inode, pos);
1178
1179 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1180 fuse_invalidate_attr(inode);
1181
1182 return res > 0 ? res : err;
1183}
1184
1185static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
1186{
1187 struct file *file = iocb->ki_filp;
1188 struct address_space *mapping = file->f_mapping;
1189 ssize_t written = 0;
1190 ssize_t written_buffered = 0;
1191 struct inode *inode = mapping->host;
1192 ssize_t err;
1193 loff_t endbyte = 0;
1194
1195 if (get_fuse_conn(inode)->writeback_cache) {
1196
1197 err = fuse_update_attributes(mapping->host, file);
1198 if (err)
1199 return err;
1200
1201 return generic_file_write_iter(iocb, from);
1202 }
1203
1204 inode_lock(inode);
1205
1206
1207 current->backing_dev_info = inode_to_bdi(inode);
1208
1209 err = generic_write_checks(iocb, from);
1210 if (err <= 0)
1211 goto out;
1212
1213 err = file_remove_privs(file);
1214 if (err)
1215 goto out;
1216
1217 err = file_update_time(file);
1218 if (err)
1219 goto out;
1220
1221 if (iocb->ki_flags & IOCB_DIRECT) {
1222 loff_t pos = iocb->ki_pos;
1223 written = generic_file_direct_write(iocb, from);
1224 if (written < 0 || !iov_iter_count(from))
1225 goto out;
1226
1227 pos += written;
1228
1229 written_buffered = fuse_perform_write(iocb, mapping, from, pos);
1230 if (written_buffered < 0) {
1231 err = written_buffered;
1232 goto out;
1233 }
1234 endbyte = pos + written_buffered - 1;
1235
1236 err = filemap_write_and_wait_range(file->f_mapping, pos,
1237 endbyte);
1238 if (err)
1239 goto out;
1240
1241 invalidate_mapping_pages(file->f_mapping,
1242 pos >> PAGE_SHIFT,
1243 endbyte >> PAGE_SHIFT);
1244
1245 written += written_buffered;
1246 iocb->ki_pos = pos + written_buffered;
1247 } else {
1248 written = fuse_perform_write(iocb, mapping, from, iocb->ki_pos);
1249 if (written >= 0)
1250 iocb->ki_pos += written;
1251 }
1252out:
1253 current->backing_dev_info = NULL;
1254 inode_unlock(inode);
1255 if (written > 0)
1256 written = generic_write_sync(iocb, written);
1257
1258 return written ? written : err;
1259}
1260
1261static inline void fuse_page_descs_length_init(struct fuse_req *req,
1262 unsigned index, unsigned nr_pages)
1263{
1264 int i;
1265
1266 for (i = index; i < index + nr_pages; i++)
1267 req->page_descs[i].length = PAGE_SIZE -
1268 req->page_descs[i].offset;
1269}
1270
1271static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
1272{
1273 return (unsigned long)ii->iov->iov_base + ii->iov_offset;
1274}
1275
1276static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
1277 size_t max_size)
1278{
1279 return min(iov_iter_single_seg_count(ii), max_size);
1280}
1281
1282static int fuse_get_user_pages(struct fuse_req *req, struct iov_iter *ii,
1283 size_t *nbytesp, int write)
1284{
1285 size_t nbytes = 0;
1286 ssize_t ret = 0;
1287
1288
1289 if (iov_iter_is_kvec(ii)) {
1290 unsigned long user_addr = fuse_get_user_addr(ii);
1291 size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
1292
1293 if (write)
1294 req->in.args[1].value = (void *) user_addr;
1295 else
1296 req->out.args[0].value = (void *) user_addr;
1297
1298 iov_iter_advance(ii, frag_size);
1299 *nbytesp = frag_size;
1300 return 0;
1301 }
1302
1303 while (nbytes < *nbytesp && req->num_pages < req->max_pages) {
1304 unsigned npages;
1305 size_t start;
1306 ret = iov_iter_get_pages(ii, &req->pages[req->num_pages],
1307 *nbytesp - nbytes,
1308 req->max_pages - req->num_pages,
1309 &start);
1310 if (ret < 0)
1311 break;
1312
1313 iov_iter_advance(ii, ret);
1314 nbytes += ret;
1315
1316 ret += start;
1317 npages = (ret + PAGE_SIZE - 1) / PAGE_SIZE;
1318
1319 req->page_descs[req->num_pages].offset = start;
1320 fuse_page_descs_length_init(req, req->num_pages, npages);
1321
1322 req->num_pages += npages;
1323 req->page_descs[req->num_pages - 1].length -=
1324 (PAGE_SIZE - ret) & (PAGE_SIZE - 1);
1325 }
1326
1327 if (write)
1328 req->in.argpages = 1;
1329 else
1330 req->out.argpages = 1;
1331
1332 *nbytesp = nbytes;
1333
1334 return ret < 0 ? ret : 0;
1335}
1336
1337ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
1338 loff_t *ppos, int flags)
1339{
1340 int write = flags & FUSE_DIO_WRITE;
1341 int cuse = flags & FUSE_DIO_CUSE;
1342 struct file *file = io->iocb->ki_filp;
1343 struct inode *inode = file->f_mapping->host;
1344 struct fuse_file *ff = file->private_data;
1345 struct fuse_conn *fc = ff->fc;
1346 size_t nmax = write ? fc->max_write : fc->max_read;
1347 loff_t pos = *ppos;
1348 size_t count = iov_iter_count(iter);
1349 pgoff_t idx_from = pos >> PAGE_SHIFT;
1350 pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT;
1351 ssize_t res = 0;
1352 struct fuse_req *req;
1353 int err = 0;
1354
1355 if (io->async)
1356 req = fuse_get_req_for_background(fc, iov_iter_npages(iter,
1357 fc->max_pages));
1358 else
1359 req = fuse_get_req(fc, iov_iter_npages(iter, fc->max_pages));
1360 if (IS_ERR(req))
1361 return PTR_ERR(req);
1362
1363 if (!cuse && fuse_range_is_writeback(inode, idx_from, idx_to)) {
1364 if (!write)
1365 inode_lock(inode);
1366 fuse_sync_writes(inode);
1367 if (!write)
1368 inode_unlock(inode);
1369 }
1370
1371 io->should_dirty = !write && iter_is_iovec(iter);
1372 while (count) {
1373 size_t nres;
1374 fl_owner_t owner = current->files;
1375 size_t nbytes = min(count, nmax);
1376 err = fuse_get_user_pages(req, iter, &nbytes, write);
1377 if (err && !nbytes)
1378 break;
1379
1380 if (write) {
1381 if (!capable(CAP_FSETID)) {
1382 struct fuse_write_in *inarg;
1383
1384 inarg = &req->misc.write.in;
1385 inarg->write_flags |= FUSE_WRITE_KILL_PRIV;
1386 }
1387 nres = fuse_send_write(req, io, pos, nbytes, owner);
1388 } else {
1389 nres = fuse_send_read(req, io, pos, nbytes, owner);
1390 }
1391
1392 if (!io->async)
1393 fuse_release_user_pages(req, io->should_dirty);
1394 if (req->out.h.error) {
1395 err = req->out.h.error;
1396 break;
1397 } else if (nres > nbytes) {
1398 res = 0;
1399 err = -EIO;
1400 break;
1401 }
1402 count -= nres;
1403 res += nres;
1404 pos += nres;
1405 if (nres != nbytes)
1406 break;
1407 if (count) {
1408 fuse_put_request(fc, req);
1409 if (io->async)
1410 req = fuse_get_req_for_background(fc,
1411 iov_iter_npages(iter, fc->max_pages));
1412 else
1413 req = fuse_get_req(fc, iov_iter_npages(iter,
1414 fc->max_pages));
1415 if (IS_ERR(req))
1416 break;
1417 }
1418 }
1419 if (!IS_ERR(req))
1420 fuse_put_request(fc, req);
1421 if (res > 0)
1422 *ppos = pos;
1423
1424 return res > 0 ? res : err;
1425}
1426EXPORT_SYMBOL_GPL(fuse_direct_io);
1427
1428static ssize_t __fuse_direct_read(struct fuse_io_priv *io,
1429 struct iov_iter *iter,
1430 loff_t *ppos)
1431{
1432 ssize_t res;
1433 struct inode *inode = file_inode(io->iocb->ki_filp);
1434
1435 res = fuse_direct_io(io, iter, ppos, 0);
1436
1437 fuse_invalidate_atime(inode);
1438
1439 return res;
1440}
1441
1442static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
1443
1444static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to)
1445{
1446 ssize_t res;
1447
1448 if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {
1449 res = fuse_direct_IO(iocb, to);
1450 } else {
1451 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1452
1453 res = __fuse_direct_read(&io, to, &iocb->ki_pos);
1454 }
1455
1456 return res;
1457}
1458
1459static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
1460{
1461 struct inode *inode = file_inode(iocb->ki_filp);
1462 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1463 ssize_t res;
1464
1465
1466 inode_lock(inode);
1467 res = generic_write_checks(iocb, from);
1468 if (res > 0) {
1469 if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {
1470 res = fuse_direct_IO(iocb, from);
1471 } else {
1472 res = fuse_direct_io(&io, from, &iocb->ki_pos,
1473 FUSE_DIO_WRITE);
1474 }
1475 }
1476 fuse_invalidate_attr(inode);
1477 if (res > 0)
1478 fuse_write_update_size(inode, iocb->ki_pos);
1479 inode_unlock(inode);
1480
1481 return res;
1482}
1483
1484static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
1485{
1486 struct file *file = iocb->ki_filp;
1487 struct fuse_file *ff = file->private_data;
1488
1489 if (is_bad_inode(file_inode(file)))
1490 return -EIO;
1491
1492 if (!(ff->open_flags & FOPEN_DIRECT_IO))
1493 return fuse_cache_read_iter(iocb, to);
1494 else
1495 return fuse_direct_read_iter(iocb, to);
1496}
1497
1498static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
1499{
1500 struct file *file = iocb->ki_filp;
1501 struct fuse_file *ff = file->private_data;
1502
1503 if (is_bad_inode(file_inode(file)))
1504 return -EIO;
1505
1506 if (!(ff->open_flags & FOPEN_DIRECT_IO))
1507 return fuse_cache_write_iter(iocb, from);
1508 else
1509 return fuse_direct_write_iter(iocb, from);
1510}
1511
1512static void fuse_writepage_free(struct fuse_conn *fc, struct fuse_req *req)
1513{
1514 int i;
1515
1516 for (i = 0; i < req->num_pages; i++)
1517 __free_page(req->pages[i]);
1518
1519 if (req->ff)
1520 fuse_file_put(req->ff, false, false);
1521}
1522
1523static void fuse_writepage_finish(struct fuse_conn *fc, struct fuse_req *req)
1524{
1525 struct inode *inode = req->inode;
1526 struct fuse_inode *fi = get_fuse_inode(inode);
1527 struct backing_dev_info *bdi = inode_to_bdi(inode);
1528 int i;
1529
1530 list_del(&req->writepages_entry);
1531 for (i = 0; i < req->num_pages; i++) {
1532 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
1533 dec_node_page_state(req->pages[i], NR_WRITEBACK_TEMP);
1534 wb_writeout_inc(&bdi->wb);
1535 }
1536 wake_up(&fi->page_waitq);
1537}
1538
1539
1540static void fuse_send_writepage(struct fuse_conn *fc, struct fuse_req *req,
1541 loff_t size)
1542__releases(fi->lock)
1543__acquires(fi->lock)
1544{
1545 struct fuse_req *aux, *next;
1546 struct fuse_inode *fi = get_fuse_inode(req->inode);
1547 struct fuse_write_in *inarg = &req->misc.write.in;
1548 __u64 data_size = req->num_pages * PAGE_SIZE;
1549 bool queued;
1550
1551 if (inarg->offset + data_size <= size) {
1552 inarg->size = data_size;
1553 } else if (inarg->offset < size) {
1554 inarg->size = size - inarg->offset;
1555 } else {
1556
1557 goto out_free;
1558 }
1559
1560 req->in.args[1].size = inarg->size;
1561 queued = fuse_request_queue_background(fc, req);
1562
1563 if (unlikely(!queued))
1564 goto out_free;
1565
1566 fi->writectr++;
1567 return;
1568
1569 out_free:
1570 fuse_writepage_finish(fc, req);
1571 spin_unlock(&fi->lock);
1572
1573
1574 for (aux = req->misc.write.next; aux; aux = next) {
1575 next = aux->misc.write.next;
1576 aux->misc.write.next = NULL;
1577 fuse_writepage_free(fc, aux);
1578 fuse_put_request(fc, aux);
1579 }
1580
1581 fuse_writepage_free(fc, req);
1582 fuse_put_request(fc, req);
1583 spin_lock(&fi->lock);
1584}
1585
1586
1587
1588
1589
1590
1591
1592void fuse_flush_writepages(struct inode *inode)
1593__releases(fi->lock)
1594__acquires(fi->lock)
1595{
1596 struct fuse_conn *fc = get_fuse_conn(inode);
1597 struct fuse_inode *fi = get_fuse_inode(inode);
1598 loff_t crop = i_size_read(inode);
1599 struct fuse_req *req;
1600
1601 while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) {
1602 req = list_entry(fi->queued_writes.next, struct fuse_req, list);
1603 list_del_init(&req->list);
1604 fuse_send_writepage(fc, req, crop);
1605 }
1606}
1607
1608static void fuse_writepage_end(struct fuse_conn *fc, struct fuse_req *req)
1609{
1610 struct inode *inode = req->inode;
1611 struct fuse_inode *fi = get_fuse_inode(inode);
1612
1613 mapping_set_error(inode->i_mapping, req->out.h.error);
1614 spin_lock(&fi->lock);
1615 while (req->misc.write.next) {
1616 struct fuse_conn *fc = get_fuse_conn(inode);
1617 struct fuse_write_in *inarg = &req->misc.write.in;
1618 struct fuse_req *next = req->misc.write.next;
1619 req->misc.write.next = next->misc.write.next;
1620 next->misc.write.next = NULL;
1621 next->ff = fuse_file_get(req->ff);
1622 list_add(&next->writepages_entry, &fi->writepages);
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647 fuse_send_writepage(fc, next, inarg->offset + inarg->size);
1648 }
1649 fi->writectr--;
1650 fuse_writepage_finish(fc, req);
1651 spin_unlock(&fi->lock);
1652 fuse_writepage_free(fc, req);
1653}
1654
1655static struct fuse_file *__fuse_write_file_get(struct fuse_conn *fc,
1656 struct fuse_inode *fi)
1657{
1658 struct fuse_file *ff = NULL;
1659
1660 spin_lock(&fi->lock);
1661 if (!list_empty(&fi->write_files)) {
1662 ff = list_entry(fi->write_files.next, struct fuse_file,
1663 write_entry);
1664 fuse_file_get(ff);
1665 }
1666 spin_unlock(&fi->lock);
1667
1668 return ff;
1669}
1670
1671static struct fuse_file *fuse_write_file_get(struct fuse_conn *fc,
1672 struct fuse_inode *fi)
1673{
1674 struct fuse_file *ff = __fuse_write_file_get(fc, fi);
1675 WARN_ON(!ff);
1676 return ff;
1677}
1678
1679int fuse_write_inode(struct inode *inode, struct writeback_control *wbc)
1680{
1681 struct fuse_conn *fc = get_fuse_conn(inode);
1682 struct fuse_inode *fi = get_fuse_inode(inode);
1683 struct fuse_file *ff;
1684 int err;
1685
1686 ff = __fuse_write_file_get(fc, fi);
1687 err = fuse_flush_times(inode, ff);
1688 if (ff)
1689 fuse_file_put(ff, false, false);
1690
1691 return err;
1692}
1693
1694static int fuse_writepage_locked(struct page *page)
1695{
1696 struct address_space *mapping = page->mapping;
1697 struct inode *inode = mapping->host;
1698 struct fuse_conn *fc = get_fuse_conn(inode);
1699 struct fuse_inode *fi = get_fuse_inode(inode);
1700 struct fuse_req *req;
1701 struct page *tmp_page;
1702 int error = -ENOMEM;
1703
1704 set_page_writeback(page);
1705
1706 req = fuse_request_alloc_nofs(1);
1707 if (!req)
1708 goto err;
1709
1710
1711 __set_bit(FR_BACKGROUND, &req->flags);
1712 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
1713 if (!tmp_page)
1714 goto err_free;
1715
1716 error = -EIO;
1717 req->ff = fuse_write_file_get(fc, fi);
1718 if (!req->ff)
1719 goto err_nofile;
1720
1721 fuse_write_fill(req, req->ff, page_offset(page), 0);
1722
1723 copy_highpage(tmp_page, page);
1724 req->misc.write.in.write_flags |= FUSE_WRITE_CACHE;
1725 req->misc.write.next = NULL;
1726 req->in.argpages = 1;
1727 req->num_pages = 1;
1728 req->pages[0] = tmp_page;
1729 req->page_descs[0].offset = 0;
1730 req->page_descs[0].length = PAGE_SIZE;
1731 req->end = fuse_writepage_end;
1732 req->inode = inode;
1733
1734 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
1735 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
1736
1737 spin_lock(&fi->lock);
1738 list_add(&req->writepages_entry, &fi->writepages);
1739 list_add_tail(&req->list, &fi->queued_writes);
1740 fuse_flush_writepages(inode);
1741 spin_unlock(&fi->lock);
1742
1743 end_page_writeback(page);
1744
1745 return 0;
1746
1747err_nofile:
1748 __free_page(tmp_page);
1749err_free:
1750 fuse_request_free(req);
1751err:
1752 mapping_set_error(page->mapping, error);
1753 end_page_writeback(page);
1754 return error;
1755}
1756
1757static int fuse_writepage(struct page *page, struct writeback_control *wbc)
1758{
1759 int err;
1760
1761 if (fuse_page_is_writeback(page->mapping->host, page->index)) {
1762
1763
1764
1765
1766
1767 WARN_ON(wbc->sync_mode == WB_SYNC_ALL);
1768
1769 redirty_page_for_writepage(wbc, page);
1770 return 0;
1771 }
1772
1773 err = fuse_writepage_locked(page);
1774 unlock_page(page);
1775
1776 return err;
1777}
1778
1779struct fuse_fill_wb_data {
1780 struct fuse_req *req;
1781 struct fuse_file *ff;
1782 struct inode *inode;
1783 struct page **orig_pages;
1784};
1785
1786static void fuse_writepages_send(struct fuse_fill_wb_data *data)
1787{
1788 struct fuse_req *req = data->req;
1789 struct inode *inode = data->inode;
1790 struct fuse_inode *fi = get_fuse_inode(inode);
1791 int num_pages = req->num_pages;
1792 int i;
1793
1794 req->ff = fuse_file_get(data->ff);
1795 spin_lock(&fi->lock);
1796 list_add_tail(&req->list, &fi->queued_writes);
1797 fuse_flush_writepages(inode);
1798 spin_unlock(&fi->lock);
1799
1800 for (i = 0; i < num_pages; i++)
1801 end_page_writeback(data->orig_pages[i]);
1802}
1803
1804
1805
1806
1807
1808
1809
1810
1811static bool fuse_writepage_in_flight(struct fuse_req *new_req,
1812 struct page *page)
1813{
1814 struct fuse_conn *fc = get_fuse_conn(new_req->inode);
1815 struct fuse_inode *fi = get_fuse_inode(new_req->inode);
1816 struct fuse_req *tmp;
1817 struct fuse_req *old_req;
1818
1819 WARN_ON(new_req->num_pages != 0);
1820
1821 spin_lock(&fi->lock);
1822 list_del(&new_req->writepages_entry);
1823 old_req = fuse_find_writeback(fi, page->index, page->index);
1824 if (!old_req) {
1825 list_add(&new_req->writepages_entry, &fi->writepages);
1826 spin_unlock(&fi->lock);
1827 return false;
1828 }
1829
1830 new_req->num_pages = 1;
1831 for (tmp = old_req->misc.write.next; tmp; tmp = tmp->misc.write.next) {
1832 pgoff_t curr_index;
1833
1834 WARN_ON(tmp->inode != new_req->inode);
1835 curr_index = tmp->misc.write.in.offset >> PAGE_SHIFT;
1836 if (curr_index == page->index) {
1837 WARN_ON(tmp->num_pages != 1);
1838 WARN_ON(!test_bit(FR_PENDING, &tmp->flags));
1839 swap(tmp->pages[0], new_req->pages[0]);
1840 break;
1841 }
1842 }
1843
1844 if (!tmp) {
1845 new_req->misc.write.next = old_req->misc.write.next;
1846 old_req->misc.write.next = new_req;
1847 }
1848
1849 spin_unlock(&fi->lock);
1850
1851 if (tmp) {
1852 struct backing_dev_info *bdi = inode_to_bdi(new_req->inode);
1853
1854 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
1855 dec_node_page_state(new_req->pages[0], NR_WRITEBACK_TEMP);
1856 wb_writeout_inc(&bdi->wb);
1857 fuse_writepage_free(fc, new_req);
1858 fuse_request_free(new_req);
1859 }
1860
1861 return true;
1862}
1863
1864static int fuse_writepages_fill(struct page *page,
1865 struct writeback_control *wbc, void *_data)
1866{
1867 struct fuse_fill_wb_data *data = _data;
1868 struct fuse_req *req = data->req;
1869 struct inode *inode = data->inode;
1870 struct fuse_inode *fi = get_fuse_inode(inode);
1871 struct fuse_conn *fc = get_fuse_conn(inode);
1872 struct page *tmp_page;
1873 bool is_writeback;
1874 int err;
1875
1876 if (!data->ff) {
1877 err = -EIO;
1878 data->ff = fuse_write_file_get(fc, get_fuse_inode(inode));
1879 if (!data->ff)
1880 goto out_unlock;
1881 }
1882
1883
1884
1885
1886
1887
1888
1889 is_writeback = fuse_page_is_writeback(inode, page->index);
1890
1891 if (req && req->num_pages &&
1892 (is_writeback || req->num_pages == fc->max_pages ||
1893 (req->num_pages + 1) * PAGE_SIZE > fc->max_write ||
1894 data->orig_pages[req->num_pages - 1]->index + 1 != page->index)) {
1895 fuse_writepages_send(data);
1896 data->req = NULL;
1897 } else if (req && req->num_pages == req->max_pages) {
1898 if (!fuse_req_realloc_pages(fc, req, GFP_NOFS)) {
1899 fuse_writepages_send(data);
1900 req = data->req = NULL;
1901 }
1902 }
1903
1904 err = -ENOMEM;
1905 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
1906 if (!tmp_page)
1907 goto out_unlock;
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922 if (data->req == NULL) {
1923 struct fuse_inode *fi = get_fuse_inode(inode);
1924
1925 err = -ENOMEM;
1926 req = fuse_request_alloc_nofs(FUSE_REQ_INLINE_PAGES);
1927 if (!req) {
1928 __free_page(tmp_page);
1929 goto out_unlock;
1930 }
1931
1932 fuse_write_fill(req, data->ff, page_offset(page), 0);
1933 req->misc.write.in.write_flags |= FUSE_WRITE_CACHE;
1934 req->misc.write.next = NULL;
1935 req->in.argpages = 1;
1936 __set_bit(FR_BACKGROUND, &req->flags);
1937 req->num_pages = 0;
1938 req->end = fuse_writepage_end;
1939 req->inode = inode;
1940
1941 spin_lock(&fi->lock);
1942 list_add(&req->writepages_entry, &fi->writepages);
1943 spin_unlock(&fi->lock);
1944
1945 data->req = req;
1946 }
1947 set_page_writeback(page);
1948
1949 copy_highpage(tmp_page, page);
1950 req->pages[req->num_pages] = tmp_page;
1951 req->page_descs[req->num_pages].offset = 0;
1952 req->page_descs[req->num_pages].length = PAGE_SIZE;
1953
1954 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
1955 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
1956
1957 err = 0;
1958 if (is_writeback && fuse_writepage_in_flight(req, page)) {
1959 end_page_writeback(page);
1960 data->req = NULL;
1961 goto out_unlock;
1962 }
1963 data->orig_pages[req->num_pages] = page;
1964
1965
1966
1967
1968
1969 spin_lock(&fi->lock);
1970 req->num_pages++;
1971 spin_unlock(&fi->lock);
1972
1973out_unlock:
1974 unlock_page(page);
1975
1976 return err;
1977}
1978
1979static int fuse_writepages(struct address_space *mapping,
1980 struct writeback_control *wbc)
1981{
1982 struct inode *inode = mapping->host;
1983 struct fuse_conn *fc = get_fuse_conn(inode);
1984 struct fuse_fill_wb_data data;
1985 int err;
1986
1987 err = -EIO;
1988 if (is_bad_inode(inode))
1989 goto out;
1990
1991 data.inode = inode;
1992 data.req = NULL;
1993 data.ff = NULL;
1994
1995 err = -ENOMEM;
1996 data.orig_pages = kcalloc(fc->max_pages,
1997 sizeof(struct page *),
1998 GFP_NOFS);
1999 if (!data.orig_pages)
2000 goto out;
2001
2002 err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data);
2003 if (data.req) {
2004
2005 BUG_ON(!data.req->num_pages);
2006 fuse_writepages_send(&data);
2007 err = 0;
2008 }
2009 if (data.ff)
2010 fuse_file_put(data.ff, false, false);
2011
2012 kfree(data.orig_pages);
2013out:
2014 return err;
2015}
2016
2017
2018
2019
2020
2021static int fuse_write_begin(struct file *file, struct address_space *mapping,
2022 loff_t pos, unsigned len, unsigned flags,
2023 struct page **pagep, void **fsdata)
2024{
2025 pgoff_t index = pos >> PAGE_SHIFT;
2026 struct fuse_conn *fc = get_fuse_conn(file_inode(file));
2027 struct page *page;
2028 loff_t fsize;
2029 int err = -ENOMEM;
2030
2031 WARN_ON(!fc->writeback_cache);
2032
2033 page = grab_cache_page_write_begin(mapping, index, flags);
2034 if (!page)
2035 goto error;
2036
2037 fuse_wait_on_page_writeback(mapping->host, page->index);
2038
2039 if (PageUptodate(page) || len == PAGE_SIZE)
2040 goto success;
2041
2042
2043
2044
2045 fsize = i_size_read(mapping->host);
2046 if (fsize <= (pos & PAGE_MASK)) {
2047 size_t off = pos & ~PAGE_MASK;
2048 if (off)
2049 zero_user_segment(page, 0, off);
2050 goto success;
2051 }
2052 err = fuse_do_readpage(file, page);
2053 if (err)
2054 goto cleanup;
2055success:
2056 *pagep = page;
2057 return 0;
2058
2059cleanup:
2060 unlock_page(page);
2061 put_page(page);
2062error:
2063 return err;
2064}
2065
2066static int fuse_write_end(struct file *file, struct address_space *mapping,
2067 loff_t pos, unsigned len, unsigned copied,
2068 struct page *page, void *fsdata)
2069{
2070 struct inode *inode = page->mapping->host;
2071
2072
2073 if (!copied)
2074 goto unlock;
2075
2076 if (!PageUptodate(page)) {
2077
2078 size_t endoff = (pos + copied) & ~PAGE_MASK;
2079 if (endoff)
2080 zero_user_segment(page, endoff, PAGE_SIZE);
2081 SetPageUptodate(page);
2082 }
2083
2084 fuse_write_update_size(inode, pos + copied);
2085 set_page_dirty(page);
2086
2087unlock:
2088 unlock_page(page);
2089 put_page(page);
2090
2091 return copied;
2092}
2093
2094static int fuse_launder_page(struct page *page)
2095{
2096 int err = 0;
2097 if (clear_page_dirty_for_io(page)) {
2098 struct inode *inode = page->mapping->host;
2099 err = fuse_writepage_locked(page);
2100 if (!err)
2101 fuse_wait_on_page_writeback(inode, page->index);
2102 }
2103 return err;
2104}
2105
2106
2107
2108
2109
2110static void fuse_vma_close(struct vm_area_struct *vma)
2111{
2112 filemap_write_and_wait(vma->vm_file->f_mapping);
2113}
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf)
2131{
2132 struct page *page = vmf->page;
2133 struct inode *inode = file_inode(vmf->vma->vm_file);
2134
2135 file_update_time(vmf->vma->vm_file);
2136 lock_page(page);
2137 if (page->mapping != inode->i_mapping) {
2138 unlock_page(page);
2139 return VM_FAULT_NOPAGE;
2140 }
2141
2142 fuse_wait_on_page_writeback(inode, page->index);
2143 return VM_FAULT_LOCKED;
2144}
2145
2146static const struct vm_operations_struct fuse_file_vm_ops = {
2147 .close = fuse_vma_close,
2148 .fault = filemap_fault,
2149 .map_pages = filemap_map_pages,
2150 .page_mkwrite = fuse_page_mkwrite,
2151};
2152
2153static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
2154{
2155 struct fuse_file *ff = file->private_data;
2156
2157 if (ff->open_flags & FOPEN_DIRECT_IO) {
2158
2159 if (vma->vm_flags & VM_MAYSHARE)
2160 return -ENODEV;
2161
2162 invalidate_inode_pages2(file->f_mapping);
2163
2164 return generic_file_mmap(file, vma);
2165 }
2166
2167 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2168 fuse_link_write_file(file);
2169
2170 file_accessed(file);
2171 vma->vm_ops = &fuse_file_vm_ops;
2172 return 0;
2173}
2174
2175static int convert_fuse_file_lock(struct fuse_conn *fc,
2176 const struct fuse_file_lock *ffl,
2177 struct file_lock *fl)
2178{
2179 switch (ffl->type) {
2180 case F_UNLCK:
2181 break;
2182
2183 case F_RDLCK:
2184 case F_WRLCK:
2185 if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX ||
2186 ffl->end < ffl->start)
2187 return -EIO;
2188
2189 fl->fl_start = ffl->start;
2190 fl->fl_end = ffl->end;
2191
2192
2193
2194
2195
2196 rcu_read_lock();
2197 fl->fl_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns);
2198 rcu_read_unlock();
2199 break;
2200
2201 default:
2202 return -EIO;
2203 }
2204 fl->fl_type = ffl->type;
2205 return 0;
2206}
2207
2208static void fuse_lk_fill(struct fuse_args *args, struct file *file,
2209 const struct file_lock *fl, int opcode, pid_t pid,
2210 int flock, struct fuse_lk_in *inarg)
2211{
2212 struct inode *inode = file_inode(file);
2213 struct fuse_conn *fc = get_fuse_conn(inode);
2214 struct fuse_file *ff = file->private_data;
2215
2216 memset(inarg, 0, sizeof(*inarg));
2217 inarg->fh = ff->fh;
2218 inarg->owner = fuse_lock_owner_id(fc, fl->fl_owner);
2219 inarg->lk.start = fl->fl_start;
2220 inarg->lk.end = fl->fl_end;
2221 inarg->lk.type = fl->fl_type;
2222 inarg->lk.pid = pid;
2223 if (flock)
2224 inarg->lk_flags |= FUSE_LK_FLOCK;
2225 args->in.h.opcode = opcode;
2226 args->in.h.nodeid = get_node_id(inode);
2227 args->in.numargs = 1;
2228 args->in.args[0].size = sizeof(*inarg);
2229 args->in.args[0].value = inarg;
2230}
2231
2232static int fuse_getlk(struct file *file, struct file_lock *fl)
2233{
2234 struct inode *inode = file_inode(file);
2235 struct fuse_conn *fc = get_fuse_conn(inode);
2236 FUSE_ARGS(args);
2237 struct fuse_lk_in inarg;
2238 struct fuse_lk_out outarg;
2239 int err;
2240
2241 fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg);
2242 args.out.numargs = 1;
2243 args.out.args[0].size = sizeof(outarg);
2244 args.out.args[0].value = &outarg;
2245 err = fuse_simple_request(fc, &args);
2246 if (!err)
2247 err = convert_fuse_file_lock(fc, &outarg.lk, fl);
2248
2249 return err;
2250}
2251
2252static int fuse_setlk(struct file *file, struct file_lock *fl, int flock)
2253{
2254 struct inode *inode = file_inode(file);
2255 struct fuse_conn *fc = get_fuse_conn(inode);
2256 FUSE_ARGS(args);
2257 struct fuse_lk_in inarg;
2258 int opcode = (fl->fl_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK;
2259 struct pid *pid = fl->fl_type != F_UNLCK ? task_tgid(current) : NULL;
2260 pid_t pid_nr = pid_nr_ns(pid, fc->pid_ns);
2261 int err;
2262
2263 if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
2264
2265 return -ENOLCK;
2266 }
2267
2268
2269 if ((fl->fl_flags & FL_CLOSE_POSIX) == FL_CLOSE_POSIX)
2270 return 0;
2271
2272 fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg);
2273 err = fuse_simple_request(fc, &args);
2274
2275
2276 if (err == -EINTR)
2277 err = -ERESTARTSYS;
2278
2279 return err;
2280}
2281
2282static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl)
2283{
2284 struct inode *inode = file_inode(file);
2285 struct fuse_conn *fc = get_fuse_conn(inode);
2286 int err;
2287
2288 if (cmd == F_CANCELLK) {
2289 err = 0;
2290 } else if (cmd == F_GETLK) {
2291 if (fc->no_lock) {
2292 posix_test_lock(file, fl);
2293 err = 0;
2294 } else
2295 err = fuse_getlk(file, fl);
2296 } else {
2297 if (fc->no_lock)
2298 err = posix_lock_file(file, fl, NULL);
2299 else
2300 err = fuse_setlk(file, fl, 0);
2301 }
2302 return err;
2303}
2304
2305static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl)
2306{
2307 struct inode *inode = file_inode(file);
2308 struct fuse_conn *fc = get_fuse_conn(inode);
2309 int err;
2310
2311 if (fc->no_flock) {
2312 err = locks_lock_file_wait(file, fl);
2313 } else {
2314 struct fuse_file *ff = file->private_data;
2315
2316
2317 ff->flock = true;
2318 err = fuse_setlk(file, fl, 1);
2319 }
2320
2321 return err;
2322}
2323
2324static sector_t fuse_bmap(struct address_space *mapping, sector_t block)
2325{
2326 struct inode *inode = mapping->host;
2327 struct fuse_conn *fc = get_fuse_conn(inode);
2328 FUSE_ARGS(args);
2329 struct fuse_bmap_in inarg;
2330 struct fuse_bmap_out outarg;
2331 int err;
2332
2333 if (!inode->i_sb->s_bdev || fc->no_bmap)
2334 return 0;
2335
2336 memset(&inarg, 0, sizeof(inarg));
2337 inarg.block = block;
2338 inarg.blocksize = inode->i_sb->s_blocksize;
2339 args.in.h.opcode = FUSE_BMAP;
2340 args.in.h.nodeid = get_node_id(inode);
2341 args.in.numargs = 1;
2342 args.in.args[0].size = sizeof(inarg);
2343 args.in.args[0].value = &inarg;
2344 args.out.numargs = 1;
2345 args.out.args[0].size = sizeof(outarg);
2346 args.out.args[0].value = &outarg;
2347 err = fuse_simple_request(fc, &args);
2348 if (err == -ENOSYS)
2349 fc->no_bmap = 1;
2350
2351 return err ? 0 : outarg.block;
2352}
2353
2354static loff_t fuse_lseek(struct file *file, loff_t offset, int whence)
2355{
2356 struct inode *inode = file->f_mapping->host;
2357 struct fuse_conn *fc = get_fuse_conn(inode);
2358 struct fuse_file *ff = file->private_data;
2359 FUSE_ARGS(args);
2360 struct fuse_lseek_in inarg = {
2361 .fh = ff->fh,
2362 .offset = offset,
2363 .whence = whence
2364 };
2365 struct fuse_lseek_out outarg;
2366 int err;
2367
2368 if (fc->no_lseek)
2369 goto fallback;
2370
2371 args.in.h.opcode = FUSE_LSEEK;
2372 args.in.h.nodeid = ff->nodeid;
2373 args.in.numargs = 1;
2374 args.in.args[0].size = sizeof(inarg);
2375 args.in.args[0].value = &inarg;
2376 args.out.numargs = 1;
2377 args.out.args[0].size = sizeof(outarg);
2378 args.out.args[0].value = &outarg;
2379 err = fuse_simple_request(fc, &args);
2380 if (err) {
2381 if (err == -ENOSYS) {
2382 fc->no_lseek = 1;
2383 goto fallback;
2384 }
2385 return err;
2386 }
2387
2388 return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes);
2389
2390fallback:
2391 err = fuse_update_attributes(inode, file);
2392 if (!err)
2393 return generic_file_llseek(file, offset, whence);
2394 else
2395 return err;
2396}
2397
2398static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
2399{
2400 loff_t retval;
2401 struct inode *inode = file_inode(file);
2402
2403 switch (whence) {
2404 case SEEK_SET:
2405 case SEEK_CUR:
2406
2407 retval = generic_file_llseek(file, offset, whence);
2408 break;
2409 case SEEK_END:
2410 inode_lock(inode);
2411 retval = fuse_update_attributes(inode, file);
2412 if (!retval)
2413 retval = generic_file_llseek(file, offset, whence);
2414 inode_unlock(inode);
2415 break;
2416 case SEEK_HOLE:
2417 case SEEK_DATA:
2418 inode_lock(inode);
2419 retval = fuse_lseek(file, offset, whence);
2420 inode_unlock(inode);
2421 break;
2422 default:
2423 retval = -EINVAL;
2424 }
2425
2426 return retval;
2427}
2428
2429
2430
2431
2432
2433
2434
2435static int fuse_copy_ioctl_iovec_old(struct iovec *dst, void *src,
2436 size_t transferred, unsigned count,
2437 bool is_compat)
2438{
2439#ifdef CONFIG_COMPAT
2440 if (count * sizeof(struct compat_iovec) == transferred) {
2441 struct compat_iovec *ciov = src;
2442 unsigned i;
2443
2444
2445
2446
2447
2448
2449 if (!is_compat)
2450 return -EINVAL;
2451
2452 for (i = 0; i < count; i++) {
2453 dst[i].iov_base = compat_ptr(ciov[i].iov_base);
2454 dst[i].iov_len = ciov[i].iov_len;
2455 }
2456 return 0;
2457 }
2458#endif
2459
2460 if (count * sizeof(struct iovec) != transferred)
2461 return -EIO;
2462
2463 memcpy(dst, src, transferred);
2464 return 0;
2465}
2466
2467
2468static int fuse_verify_ioctl_iov(struct fuse_conn *fc, struct iovec *iov,
2469 size_t count)
2470{
2471 size_t n;
2472 u32 max = fc->max_pages << PAGE_SHIFT;
2473
2474 for (n = 0; n < count; n++, iov++) {
2475 if (iov->iov_len > (size_t) max)
2476 return -ENOMEM;
2477 max -= iov->iov_len;
2478 }
2479 return 0;
2480}
2481
2482static int fuse_copy_ioctl_iovec(struct fuse_conn *fc, struct iovec *dst,
2483 void *src, size_t transferred, unsigned count,
2484 bool is_compat)
2485{
2486 unsigned i;
2487 struct fuse_ioctl_iovec *fiov = src;
2488
2489 if (fc->minor < 16) {
2490 return fuse_copy_ioctl_iovec_old(dst, src, transferred,
2491 count, is_compat);
2492 }
2493
2494 if (count * sizeof(struct fuse_ioctl_iovec) != transferred)
2495 return -EIO;
2496
2497 for (i = 0; i < count; i++) {
2498
2499 if (fiov[i].base != (unsigned long) fiov[i].base ||
2500 fiov[i].len != (unsigned long) fiov[i].len)
2501 return -EIO;
2502
2503 dst[i].iov_base = (void __user *) (unsigned long) fiov[i].base;
2504 dst[i].iov_len = (size_t) fiov[i].len;
2505
2506#ifdef CONFIG_COMPAT
2507 if (is_compat &&
2508 (ptr_to_compat(dst[i].iov_base) != fiov[i].base ||
2509 (compat_size_t) dst[i].iov_len != fiov[i].len))
2510 return -EIO;
2511#endif
2512 }
2513
2514 return 0;
2515}
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
2565 unsigned int flags)
2566{
2567 struct fuse_file *ff = file->private_data;
2568 struct fuse_conn *fc = ff->fc;
2569 struct fuse_ioctl_in inarg = {
2570 .fh = ff->fh,
2571 .cmd = cmd,
2572 .arg = arg,
2573 .flags = flags
2574 };
2575 struct fuse_ioctl_out outarg;
2576 struct fuse_req *req = NULL;
2577 struct page **pages = NULL;
2578 struct iovec *iov_page = NULL;
2579 struct iovec *in_iov = NULL, *out_iov = NULL;
2580 unsigned int in_iovs = 0, out_iovs = 0, num_pages = 0, max_pages;
2581 size_t in_size, out_size, transferred, c;
2582 int err, i;
2583 struct iov_iter ii;
2584
2585#if BITS_PER_LONG == 32
2586 inarg.flags |= FUSE_IOCTL_32BIT;
2587#else
2588 if (flags & FUSE_IOCTL_COMPAT) {
2589 inarg.flags |= FUSE_IOCTL_32BIT;
2590#ifdef CONFIG_X86_X32
2591 if (in_x32_syscall())
2592 inarg.flags |= FUSE_IOCTL_COMPAT_X32;
2593#endif
2594 }
2595#endif
2596
2597
2598 BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE);
2599
2600 err = -ENOMEM;
2601 pages = kcalloc(fc->max_pages, sizeof(pages[0]), GFP_KERNEL);
2602 iov_page = (struct iovec *) __get_free_page(GFP_KERNEL);
2603 if (!pages || !iov_page)
2604 goto out;
2605
2606
2607
2608
2609
2610 if (!(flags & FUSE_IOCTL_UNRESTRICTED)) {
2611 struct iovec *iov = iov_page;
2612
2613 iov->iov_base = (void __user *)arg;
2614 iov->iov_len = _IOC_SIZE(cmd);
2615
2616 if (_IOC_DIR(cmd) & _IOC_WRITE) {
2617 in_iov = iov;
2618 in_iovs = 1;
2619 }
2620
2621 if (_IOC_DIR(cmd) & _IOC_READ) {
2622 out_iov = iov;
2623 out_iovs = 1;
2624 }
2625 }
2626
2627 retry:
2628 inarg.in_size = in_size = iov_length(in_iov, in_iovs);
2629 inarg.out_size = out_size = iov_length(out_iov, out_iovs);
2630
2631
2632
2633
2634
2635 out_size = max_t(size_t, out_size, PAGE_SIZE);
2636 max_pages = DIV_ROUND_UP(max(in_size, out_size), PAGE_SIZE);
2637
2638
2639 err = -ENOMEM;
2640 if (max_pages > fc->max_pages)
2641 goto out;
2642 while (num_pages < max_pages) {
2643 pages[num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
2644 if (!pages[num_pages])
2645 goto out;
2646 num_pages++;
2647 }
2648
2649 req = fuse_get_req(fc, num_pages);
2650 if (IS_ERR(req)) {
2651 err = PTR_ERR(req);
2652 req = NULL;
2653 goto out;
2654 }
2655 memcpy(req->pages, pages, sizeof(req->pages[0]) * num_pages);
2656 req->num_pages = num_pages;
2657 fuse_page_descs_length_init(req, 0, req->num_pages);
2658
2659
2660 req->in.h.opcode = FUSE_IOCTL;
2661 req->in.h.nodeid = ff->nodeid;
2662 req->in.numargs = 1;
2663 req->in.args[0].size = sizeof(inarg);
2664 req->in.args[0].value = &inarg;
2665 if (in_size) {
2666 req->in.numargs++;
2667 req->in.args[1].size = in_size;
2668 req->in.argpages = 1;
2669
2670 err = -EFAULT;
2671 iov_iter_init(&ii, WRITE, in_iov, in_iovs, in_size);
2672 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= num_pages); i++) {
2673 c = copy_page_from_iter(pages[i], 0, PAGE_SIZE, &ii);
2674 if (c != PAGE_SIZE && iov_iter_count(&ii))
2675 goto out;
2676 }
2677 }
2678
2679 req->out.numargs = 2;
2680 req->out.args[0].size = sizeof(outarg);
2681 req->out.args[0].value = &outarg;
2682 req->out.args[1].size = out_size;
2683 req->out.argpages = 1;
2684 req->out.argvar = 1;
2685
2686 fuse_request_send(fc, req);
2687 err = req->out.h.error;
2688 transferred = req->out.args[1].size;
2689 fuse_put_request(fc, req);
2690 req = NULL;
2691 if (err)
2692 goto out;
2693
2694
2695 if (outarg.flags & FUSE_IOCTL_RETRY) {
2696 void *vaddr;
2697
2698
2699 err = -EIO;
2700 if (!(flags & FUSE_IOCTL_UNRESTRICTED))
2701 goto out;
2702
2703 in_iovs = outarg.in_iovs;
2704 out_iovs = outarg.out_iovs;
2705
2706
2707
2708
2709
2710 err = -ENOMEM;
2711 if (in_iovs > FUSE_IOCTL_MAX_IOV ||
2712 out_iovs > FUSE_IOCTL_MAX_IOV ||
2713 in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV)
2714 goto out;
2715
2716 vaddr = kmap_atomic(pages[0]);
2717 err = fuse_copy_ioctl_iovec(fc, iov_page, vaddr,
2718 transferred, in_iovs + out_iovs,
2719 (flags & FUSE_IOCTL_COMPAT) != 0);
2720 kunmap_atomic(vaddr);
2721 if (err)
2722 goto out;
2723
2724 in_iov = iov_page;
2725 out_iov = in_iov + in_iovs;
2726
2727 err = fuse_verify_ioctl_iov(fc, in_iov, in_iovs);
2728 if (err)
2729 goto out;
2730
2731 err = fuse_verify_ioctl_iov(fc, out_iov, out_iovs);
2732 if (err)
2733 goto out;
2734
2735 goto retry;
2736 }
2737
2738 err = -EIO;
2739 if (transferred > inarg.out_size)
2740 goto out;
2741
2742 err = -EFAULT;
2743 iov_iter_init(&ii, READ, out_iov, out_iovs, transferred);
2744 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= num_pages); i++) {
2745 c = copy_page_to_iter(pages[i], 0, PAGE_SIZE, &ii);
2746 if (c != PAGE_SIZE && iov_iter_count(&ii))
2747 goto out;
2748 }
2749 err = 0;
2750 out:
2751 if (req)
2752 fuse_put_request(fc, req);
2753 free_page((unsigned long) iov_page);
2754 while (num_pages)
2755 __free_page(pages[--num_pages]);
2756 kfree(pages);
2757
2758 return err ? err : outarg.result;
2759}
2760EXPORT_SYMBOL_GPL(fuse_do_ioctl);
2761
2762long fuse_ioctl_common(struct file *file, unsigned int cmd,
2763 unsigned long arg, unsigned int flags)
2764{
2765 struct inode *inode = file_inode(file);
2766 struct fuse_conn *fc = get_fuse_conn(inode);
2767
2768 if (!fuse_allow_current_process(fc))
2769 return -EACCES;
2770
2771 if (is_bad_inode(inode))
2772 return -EIO;
2773
2774 return fuse_do_ioctl(file, cmd, arg, flags);
2775}
2776
2777static long fuse_file_ioctl(struct file *file, unsigned int cmd,
2778 unsigned long arg)
2779{
2780 return fuse_ioctl_common(file, cmd, arg, 0);
2781}
2782
2783static long fuse_file_compat_ioctl(struct file *file, unsigned int cmd,
2784 unsigned long arg)
2785{
2786 return fuse_ioctl_common(file, cmd, arg, FUSE_IOCTL_COMPAT);
2787}
2788
2789
2790
2791
2792
2793
2794static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
2795 struct rb_node **parent_out)
2796{
2797 struct rb_node **link = &fc->polled_files.rb_node;
2798 struct rb_node *last = NULL;
2799
2800 while (*link) {
2801 struct fuse_file *ff;
2802
2803 last = *link;
2804 ff = rb_entry(last, struct fuse_file, polled_node);
2805
2806 if (kh < ff->kh)
2807 link = &last->rb_left;
2808 else if (kh > ff->kh)
2809 link = &last->rb_right;
2810 else
2811 return link;
2812 }
2813
2814 if (parent_out)
2815 *parent_out = last;
2816 return link;
2817}
2818
2819
2820
2821
2822
2823
2824
2825static void fuse_register_polled_file(struct fuse_conn *fc,
2826 struct fuse_file *ff)
2827{
2828 spin_lock(&fc->lock);
2829 if (RB_EMPTY_NODE(&ff->polled_node)) {
2830 struct rb_node **link, *uninitialized_var(parent);
2831
2832 link = fuse_find_polled_node(fc, ff->kh, &parent);
2833 BUG_ON(*link);
2834 rb_link_node(&ff->polled_node, parent, link);
2835 rb_insert_color(&ff->polled_node, &fc->polled_files);
2836 }
2837 spin_unlock(&fc->lock);
2838}
2839
2840__poll_t fuse_file_poll(struct file *file, poll_table *wait)
2841{
2842 struct fuse_file *ff = file->private_data;
2843 struct fuse_conn *fc = ff->fc;
2844 struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
2845 struct fuse_poll_out outarg;
2846 FUSE_ARGS(args);
2847 int err;
2848
2849 if (fc->no_poll)
2850 return DEFAULT_POLLMASK;
2851
2852 poll_wait(file, &ff->poll_wait, wait);
2853 inarg.events = mangle_poll(poll_requested_events(wait));
2854
2855
2856
2857
2858
2859 if (waitqueue_active(&ff->poll_wait)) {
2860 inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
2861 fuse_register_polled_file(fc, ff);
2862 }
2863
2864 args.in.h.opcode = FUSE_POLL;
2865 args.in.h.nodeid = ff->nodeid;
2866 args.in.numargs = 1;
2867 args.in.args[0].size = sizeof(inarg);
2868 args.in.args[0].value = &inarg;
2869 args.out.numargs = 1;
2870 args.out.args[0].size = sizeof(outarg);
2871 args.out.args[0].value = &outarg;
2872 err = fuse_simple_request(fc, &args);
2873
2874 if (!err)
2875 return demangle_poll(outarg.revents);
2876 if (err == -ENOSYS) {
2877 fc->no_poll = 1;
2878 return DEFAULT_POLLMASK;
2879 }
2880 return EPOLLERR;
2881}
2882EXPORT_SYMBOL_GPL(fuse_file_poll);
2883
2884
2885
2886
2887
2888int fuse_notify_poll_wakeup(struct fuse_conn *fc,
2889 struct fuse_notify_poll_wakeup_out *outarg)
2890{
2891 u64 kh = outarg->kh;
2892 struct rb_node **link;
2893
2894 spin_lock(&fc->lock);
2895
2896 link = fuse_find_polled_node(fc, kh, NULL);
2897 if (*link) {
2898 struct fuse_file *ff;
2899
2900 ff = rb_entry(*link, struct fuse_file, polled_node);
2901 wake_up_interruptible_sync(&ff->poll_wait);
2902 }
2903
2904 spin_unlock(&fc->lock);
2905 return 0;
2906}
2907
2908static void fuse_do_truncate(struct file *file)
2909{
2910 struct inode *inode = file->f_mapping->host;
2911 struct iattr attr;
2912
2913 attr.ia_valid = ATTR_SIZE;
2914 attr.ia_size = i_size_read(inode);
2915
2916 attr.ia_file = file;
2917 attr.ia_valid |= ATTR_FILE;
2918
2919 fuse_do_setattr(file_dentry(file), &attr, file);
2920}
2921
2922static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off)
2923{
2924 return round_up(off, fc->max_pages << PAGE_SHIFT);
2925}
2926
2927static ssize_t
2928fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
2929{
2930 DECLARE_COMPLETION_ONSTACK(wait);
2931 ssize_t ret = 0;
2932 struct file *file = iocb->ki_filp;
2933 struct fuse_file *ff = file->private_data;
2934 bool async_dio = ff->fc->async_dio;
2935 loff_t pos = 0;
2936 struct inode *inode;
2937 loff_t i_size;
2938 size_t count = iov_iter_count(iter);
2939 loff_t offset = iocb->ki_pos;
2940 struct fuse_io_priv *io;
2941
2942 pos = offset;
2943 inode = file->f_mapping->host;
2944 i_size = i_size_read(inode);
2945
2946 if ((iov_iter_rw(iter) == READ) && (offset > i_size))
2947 return 0;
2948
2949
2950 if (async_dio && iov_iter_rw(iter) != WRITE && offset + count > i_size) {
2951 if (offset >= i_size)
2952 return 0;
2953 iov_iter_truncate(iter, fuse_round_up(ff->fc, i_size - offset));
2954 count = iov_iter_count(iter);
2955 }
2956
2957 io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL);
2958 if (!io)
2959 return -ENOMEM;
2960 spin_lock_init(&io->lock);
2961 kref_init(&io->refcnt);
2962 io->reqs = 1;
2963 io->bytes = -1;
2964 io->size = 0;
2965 io->offset = offset;
2966 io->write = (iov_iter_rw(iter) == WRITE);
2967 io->err = 0;
2968
2969
2970
2971
2972 io->async = async_dio;
2973 io->iocb = iocb;
2974 io->blocking = is_sync_kiocb(iocb);
2975
2976
2977
2978
2979
2980 if ((offset + count > i_size) && iov_iter_rw(iter) == WRITE)
2981 io->blocking = true;
2982
2983 if (io->async && io->blocking) {
2984
2985
2986
2987
2988 kref_get(&io->refcnt);
2989 io->done = &wait;
2990 }
2991
2992 if (iov_iter_rw(iter) == WRITE) {
2993 ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE);
2994 fuse_invalidate_attr(inode);
2995 } else {
2996 ret = __fuse_direct_read(io, iter, &pos);
2997 }
2998
2999 if (io->async) {
3000 bool blocking = io->blocking;
3001
3002 fuse_aio_complete(io, ret < 0 ? ret : 0, -1);
3003
3004
3005 if (!blocking)
3006 return -EIOCBQUEUED;
3007
3008 wait_for_completion(&wait);
3009 ret = fuse_get_res_by_io(io);
3010 }
3011
3012 kref_put(&io->refcnt, fuse_io_release);
3013
3014 if (iov_iter_rw(iter) == WRITE) {
3015 if (ret > 0)
3016 fuse_write_update_size(inode, pos);
3017 else if (ret < 0 && offset + count > i_size)
3018 fuse_do_truncate(file);
3019 }
3020
3021 return ret;
3022}
3023
3024static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end)
3025{
3026 int err = filemap_write_and_wait_range(inode->i_mapping, start, end);
3027
3028 if (!err)
3029 fuse_sync_writes(inode);
3030
3031 return err;
3032}
3033
3034static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
3035 loff_t length)
3036{
3037 struct fuse_file *ff = file->private_data;
3038 struct inode *inode = file_inode(file);
3039 struct fuse_inode *fi = get_fuse_inode(inode);
3040 struct fuse_conn *fc = ff->fc;
3041 FUSE_ARGS(args);
3042 struct fuse_fallocate_in inarg = {
3043 .fh = ff->fh,
3044 .offset = offset,
3045 .length = length,
3046 .mode = mode
3047 };
3048 int err;
3049 bool lock_inode = !(mode & FALLOC_FL_KEEP_SIZE) ||
3050 (mode & FALLOC_FL_PUNCH_HOLE);
3051
3052 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
3053 return -EOPNOTSUPP;
3054
3055 if (fc->no_fallocate)
3056 return -EOPNOTSUPP;
3057
3058 if (lock_inode) {
3059 inode_lock(inode);
3060 if (mode & FALLOC_FL_PUNCH_HOLE) {
3061 loff_t endbyte = offset + length - 1;
3062
3063 err = fuse_writeback_range(inode, offset, endbyte);
3064 if (err)
3065 goto out;
3066 }
3067 }
3068
3069 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
3070 offset + length > i_size_read(inode)) {
3071 err = inode_newsize_ok(inode, offset + length);
3072 if (err)
3073 goto out;
3074 }
3075
3076 if (!(mode & FALLOC_FL_KEEP_SIZE))
3077 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
3078
3079 args.in.h.opcode = FUSE_FALLOCATE;
3080 args.in.h.nodeid = ff->nodeid;
3081 args.in.numargs = 1;
3082 args.in.args[0].size = sizeof(inarg);
3083 args.in.args[0].value = &inarg;
3084 err = fuse_simple_request(fc, &args);
3085 if (err == -ENOSYS) {
3086 fc->no_fallocate = 1;
3087 err = -EOPNOTSUPP;
3088 }
3089 if (err)
3090 goto out;
3091
3092
3093 if (!(mode & FALLOC_FL_KEEP_SIZE)) {
3094 bool changed = fuse_write_update_size(inode, offset + length);
3095
3096 if (changed && fc->writeback_cache)
3097 file_update_time(file);
3098 }
3099
3100 if (mode & FALLOC_FL_PUNCH_HOLE)
3101 truncate_pagecache_range(inode, offset, offset + length - 1);
3102
3103 fuse_invalidate_attr(inode);
3104
3105out:
3106 if (!(mode & FALLOC_FL_KEEP_SIZE))
3107 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
3108
3109 if (lock_inode)
3110 inode_unlock(inode);
3111
3112 return err;
3113}
3114
3115static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in,
3116 struct file *file_out, loff_t pos_out,
3117 size_t len, unsigned int flags)
3118{
3119 struct fuse_file *ff_in = file_in->private_data;
3120 struct fuse_file *ff_out = file_out->private_data;
3121 struct inode *inode_in = file_inode(file_in);
3122 struct inode *inode_out = file_inode(file_out);
3123 struct fuse_inode *fi_out = get_fuse_inode(inode_out);
3124 struct fuse_conn *fc = ff_in->fc;
3125 FUSE_ARGS(args);
3126 struct fuse_copy_file_range_in inarg = {
3127 .fh_in = ff_in->fh,
3128 .off_in = pos_in,
3129 .nodeid_out = ff_out->nodeid,
3130 .fh_out = ff_out->fh,
3131 .off_out = pos_out,
3132 .len = len,
3133 .flags = flags
3134 };
3135 struct fuse_write_out outarg;
3136 ssize_t err;
3137
3138
3139 bool is_unstable = (!fc->writeback_cache) &&
3140 ((pos_out + len) > inode_out->i_size);
3141
3142 if (fc->no_copy_file_range)
3143 return -EOPNOTSUPP;
3144
3145 if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
3146 return -EXDEV;
3147
3148 if (fc->writeback_cache) {
3149 inode_lock(inode_in);
3150 err = fuse_writeback_range(inode_in, pos_in, pos_in + len);
3151 inode_unlock(inode_in);
3152 if (err)
3153 return err;
3154 }
3155
3156 inode_lock(inode_out);
3157
3158 err = file_modified(file_out);
3159 if (err)
3160 goto out;
3161
3162 if (fc->writeback_cache) {
3163 err = fuse_writeback_range(inode_out, pos_out, pos_out + len);
3164 if (err)
3165 goto out;
3166 }
3167
3168 if (is_unstable)
3169 set_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3170
3171 args.in.h.opcode = FUSE_COPY_FILE_RANGE;
3172 args.in.h.nodeid = ff_in->nodeid;
3173 args.in.numargs = 1;
3174 args.in.args[0].size = sizeof(inarg);
3175 args.in.args[0].value = &inarg;
3176 args.out.numargs = 1;
3177 args.out.args[0].size = sizeof(outarg);
3178 args.out.args[0].value = &outarg;
3179 err = fuse_simple_request(fc, &args);
3180 if (err == -ENOSYS) {
3181 fc->no_copy_file_range = 1;
3182 err = -EOPNOTSUPP;
3183 }
3184 if (err)
3185 goto out;
3186
3187 if (fc->writeback_cache) {
3188 fuse_write_update_size(inode_out, pos_out + outarg.size);
3189 file_update_time(file_out);
3190 }
3191
3192 fuse_invalidate_attr(inode_out);
3193
3194 err = outarg.size;
3195out:
3196 if (is_unstable)
3197 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3198
3199 inode_unlock(inode_out);
3200 file_accessed(file_in);
3201
3202 return err;
3203}
3204
3205static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off,
3206 struct file *dst_file, loff_t dst_off,
3207 size_t len, unsigned int flags)
3208{
3209 ssize_t ret;
3210
3211 ret = __fuse_copy_file_range(src_file, src_off, dst_file, dst_off,
3212 len, flags);
3213
3214 if (ret == -EOPNOTSUPP || ret == -EXDEV)
3215 ret = generic_copy_file_range(src_file, src_off, dst_file,
3216 dst_off, len, flags);
3217 return ret;
3218}
3219
3220static const struct file_operations fuse_file_operations = {
3221 .llseek = fuse_file_llseek,
3222 .read_iter = fuse_file_read_iter,
3223 .write_iter = fuse_file_write_iter,
3224 .mmap = fuse_file_mmap,
3225 .open = fuse_open,
3226 .flush = fuse_flush,
3227 .release = fuse_release,
3228 .fsync = fuse_fsync,
3229 .lock = fuse_file_lock,
3230 .flock = fuse_file_flock,
3231 .splice_read = generic_file_splice_read,
3232 .splice_write = iter_file_splice_write,
3233 .unlocked_ioctl = fuse_file_ioctl,
3234 .compat_ioctl = fuse_file_compat_ioctl,
3235 .poll = fuse_file_poll,
3236 .fallocate = fuse_file_fallocate,
3237 .copy_file_range = fuse_copy_file_range,
3238};
3239
3240static const struct address_space_operations fuse_file_aops = {
3241 .readpage = fuse_readpage,
3242 .writepage = fuse_writepage,
3243 .writepages = fuse_writepages,
3244 .launder_page = fuse_launder_page,
3245 .readpages = fuse_readpages,
3246 .set_page_dirty = __set_page_dirty_nobuffers,
3247 .bmap = fuse_bmap,
3248 .direct_IO = fuse_direct_IO,
3249 .write_begin = fuse_write_begin,
3250 .write_end = fuse_write_end,
3251};
3252
3253void fuse_init_file_inode(struct inode *inode)
3254{
3255 struct fuse_inode *fi = get_fuse_inode(inode);
3256
3257 inode->i_fop = &fuse_file_operations;
3258 inode->i_data.a_ops = &fuse_file_aops;
3259
3260 INIT_LIST_HEAD(&fi->write_files);
3261 INIT_LIST_HEAD(&fi->queued_writes);
3262 fi->writectr = 0;
3263 init_waitqueue_head(&fi->page_waitq);
3264 INIT_LIST_HEAD(&fi->writepages);
3265}
3266