1
2
3
4
5
6
7
8
9#include "fuse_i.h"
10
11#include <linux/pagemap.h>
12#include <linux/slab.h>
13#include <linux/kernel.h>
14#include <linux/sched.h>
15#include <linux/module.h>
16#include <linux/compat.h>
17#include <linux/swap.h>
18#include <linux/aio.h>
19#include <linux/falloc.h>
20
21static const struct file_operations fuse_direct_io_file_operations;
22
23static int fuse_send_open(struct fuse_conn *fc, u64 nodeid, struct file *file,
24 int opcode, struct fuse_open_out *outargp)
25{
26 struct fuse_open_in inarg;
27 struct fuse_req *req;
28 int err;
29
30 req = fuse_get_req_nopages(fc);
31 if (IS_ERR(req))
32 return PTR_ERR(req);
33
34 memset(&inarg, 0, sizeof(inarg));
35 inarg.flags = file->f_flags & ~(O_CREAT | O_EXCL | O_NOCTTY);
36 if (!fc->atomic_o_trunc)
37 inarg.flags &= ~O_TRUNC;
38 req->in.h.opcode = opcode;
39 req->in.h.nodeid = nodeid;
40 req->in.numargs = 1;
41 req->in.args[0].size = sizeof(inarg);
42 req->in.args[0].value = &inarg;
43 req->out.numargs = 1;
44 req->out.args[0].size = sizeof(*outargp);
45 req->out.args[0].value = outargp;
46 fuse_request_send(fc, req);
47 err = req->out.h.error;
48 fuse_put_request(fc, req);
49
50 return err;
51}
52
53struct fuse_file *fuse_file_alloc(struct fuse_conn *fc)
54{
55 struct fuse_file *ff;
56
57 ff = kmalloc(sizeof(struct fuse_file), GFP_KERNEL);
58 if (unlikely(!ff))
59 return NULL;
60
61 ff->fc = fc;
62 ff->reserved_req = fuse_request_alloc(0);
63 if (unlikely(!ff->reserved_req)) {
64 kfree(ff);
65 return NULL;
66 }
67
68 INIT_LIST_HEAD(&ff->write_entry);
69 atomic_set(&ff->count, 0);
70 RB_CLEAR_NODE(&ff->polled_node);
71 init_waitqueue_head(&ff->poll_wait);
72
73 spin_lock(&fc->lock);
74 ff->kh = ++fc->khctr;
75 spin_unlock(&fc->lock);
76
77 return ff;
78}
79
80void fuse_file_free(struct fuse_file *ff)
81{
82 fuse_request_free(ff->reserved_req);
83 kfree(ff);
84}
85
86struct fuse_file *fuse_file_get(struct fuse_file *ff)
87{
88 atomic_inc(&ff->count);
89 return ff;
90}
91
92static void fuse_release_async(struct work_struct *work)
93{
94 struct fuse_req *req;
95 struct fuse_conn *fc;
96 struct path path;
97
98 req = container_of(work, struct fuse_req, misc.release.work);
99 path = req->misc.release.path;
100 fc = get_fuse_conn(path.dentry->d_inode);
101
102 fuse_put_request(fc, req);
103 path_put(&path);
104}
105
106static void fuse_release_end(struct fuse_conn *fc, struct fuse_req *req)
107{
108 if (fc->destroy_req) {
109
110
111
112
113
114
115
116
117 atomic_inc(&req->count);
118 INIT_WORK(&req->misc.release.work, fuse_release_async);
119 schedule_work(&req->misc.release.work);
120 } else {
121 path_put(&req->misc.release.path);
122 }
123}
124
125static void fuse_file_put(struct fuse_file *ff, bool sync)
126{
127 if (atomic_dec_and_test(&ff->count)) {
128 struct fuse_req *req = ff->reserved_req;
129
130 if (ff->fc->no_open) {
131
132
133
134
135 req->background = 0;
136 path_put(&req->misc.release.path);
137 fuse_put_request(ff->fc, req);
138 } else if (sync) {
139 req->background = 0;
140 fuse_request_send(ff->fc, req);
141 path_put(&req->misc.release.path);
142 fuse_put_request(ff->fc, req);
143 } else {
144 req->end = fuse_release_end;
145 req->background = 1;
146 fuse_request_send_background(ff->fc, req);
147 }
148 kfree(ff);
149 }
150}
151
152int fuse_do_open(struct fuse_conn *fc, u64 nodeid, struct file *file,
153 bool isdir)
154{
155 struct fuse_file *ff;
156 int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN;
157
158 ff = fuse_file_alloc(fc);
159 if (!ff)
160 return -ENOMEM;
161
162 ff->fh = 0;
163 ff->open_flags = FOPEN_KEEP_CACHE;
164 if (!fc->no_open || isdir) {
165 struct fuse_open_out outarg;
166 int err;
167
168 err = fuse_send_open(fc, nodeid, file, opcode, &outarg);
169 if (!err) {
170 ff->fh = outarg.fh;
171 ff->open_flags = outarg.open_flags;
172
173 } else if (err != -ENOSYS || isdir) {
174 fuse_file_free(ff);
175 return err;
176 } else {
177 fc->no_open = 1;
178 }
179 }
180
181 if (isdir)
182 ff->open_flags &= ~FOPEN_DIRECT_IO;
183
184 ff->nodeid = nodeid;
185 file->private_data = fuse_file_get(ff);
186
187 return 0;
188}
189EXPORT_SYMBOL_GPL(fuse_do_open);
190
191static void fuse_link_write_file(struct file *file)
192{
193 struct inode *inode = file_inode(file);
194 struct fuse_conn *fc = get_fuse_conn(inode);
195 struct fuse_inode *fi = get_fuse_inode(inode);
196 struct fuse_file *ff = file->private_data;
197
198
199
200
201 spin_lock(&fc->lock);
202 if (list_empty(&ff->write_entry))
203 list_add(&ff->write_entry, &fi->write_files);
204 spin_unlock(&fc->lock);
205}
206
207void fuse_finish_open(struct inode *inode, struct file *file)
208{
209 struct fuse_file *ff = file->private_data;
210 struct fuse_conn *fc = get_fuse_conn(inode);
211
212 if (ff->open_flags & FOPEN_DIRECT_IO)
213 file->f_op = &fuse_direct_io_file_operations;
214 if (!(ff->open_flags & FOPEN_KEEP_CACHE))
215 invalidate_inode_pages2(inode->i_mapping);
216 if (ff->open_flags & FOPEN_NONSEEKABLE)
217 nonseekable_open(inode, file);
218 if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) {
219 struct fuse_inode *fi = get_fuse_inode(inode);
220
221 spin_lock(&fc->lock);
222 fi->attr_version = ++fc->attr_version;
223 i_size_write(inode, 0);
224 spin_unlock(&fc->lock);
225 fuse_invalidate_attr(inode);
226 if (fc->writeback_cache)
227 file_update_time(file);
228 }
229 if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache)
230 fuse_link_write_file(file);
231}
232
233int fuse_open_common(struct inode *inode, struct file *file, bool isdir)
234{
235 struct fuse_conn *fc = get_fuse_conn(inode);
236 int err;
237 bool lock_inode = (file->f_flags & O_TRUNC) &&
238 fc->atomic_o_trunc &&
239 fc->writeback_cache;
240
241 err = generic_file_open(inode, file);
242 if (err)
243 return err;
244
245 if (lock_inode)
246 mutex_lock(&inode->i_mutex);
247
248 err = fuse_do_open(fc, get_node_id(inode), file, isdir);
249
250 if (!err)
251 fuse_finish_open(inode, file);
252
253 if (lock_inode)
254 mutex_unlock(&inode->i_mutex);
255
256 return err;
257}
258
259static void fuse_prepare_release(struct fuse_file *ff, int flags, int opcode)
260{
261 struct fuse_conn *fc = ff->fc;
262 struct fuse_req *req = ff->reserved_req;
263 struct fuse_release_in *inarg = &req->misc.release.in;
264
265 spin_lock(&fc->lock);
266 list_del(&ff->write_entry);
267 if (!RB_EMPTY_NODE(&ff->polled_node))
268 rb_erase(&ff->polled_node, &fc->polled_files);
269 spin_unlock(&fc->lock);
270
271 wake_up_interruptible_all(&ff->poll_wait);
272
273 inarg->fh = ff->fh;
274 inarg->flags = flags;
275 req->in.h.opcode = opcode;
276 req->in.h.nodeid = ff->nodeid;
277 req->in.numargs = 1;
278 req->in.args[0].size = sizeof(struct fuse_release_in);
279 req->in.args[0].value = inarg;
280}
281
282void fuse_release_common(struct file *file, int opcode)
283{
284 struct fuse_file *ff;
285 struct fuse_req *req;
286
287 ff = file->private_data;
288 if (unlikely(!ff))
289 return;
290
291 req = ff->reserved_req;
292 fuse_prepare_release(ff, file->f_flags, opcode);
293
294 if (ff->flock) {
295 struct fuse_release_in *inarg = &req->misc.release.in;
296 inarg->release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
297 inarg->lock_owner = fuse_lock_owner_id(ff->fc,
298 (fl_owner_t) file);
299 }
300
301 path_get(&file->f_path);
302 req->misc.release.path = file->f_path;
303
304
305
306
307
308
309
310
311
312
313 fuse_file_put(ff, ff->fc->destroy_req != NULL);
314}
315
316static int fuse_open(struct inode *inode, struct file *file)
317{
318 return fuse_open_common(inode, file, false);
319}
320
321static int fuse_release(struct inode *inode, struct file *file)
322{
323 struct fuse_conn *fc = get_fuse_conn(inode);
324
325
326 if (fc->writeback_cache)
327 write_inode_now(inode, 1);
328
329 fuse_release_common(file, FUSE_RELEASE);
330
331
332 return 0;
333}
334
335void fuse_sync_release(struct fuse_file *ff, int flags)
336{
337 WARN_ON(atomic_read(&ff->count) > 1);
338 fuse_prepare_release(ff, flags, FUSE_RELEASE);
339 ff->reserved_req->force = 1;
340 ff->reserved_req->background = 0;
341 fuse_request_send(ff->fc, ff->reserved_req);
342 fuse_put_request(ff->fc, ff->reserved_req);
343 kfree(ff);
344}
345EXPORT_SYMBOL_GPL(fuse_sync_release);
346
347
348
349
350
351u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id)
352{
353 u32 *k = fc->scramble_key;
354 u64 v = (unsigned long) id;
355 u32 v0 = v;
356 u32 v1 = v >> 32;
357 u32 sum = 0;
358 int i;
359
360 for (i = 0; i < 32; i++) {
361 v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]);
362 sum += 0x9E3779B9;
363 v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]);
364 }
365
366 return (u64) v0 + ((u64) v1 << 32);
367}
368
369
370
371
372
373
374
375static bool fuse_range_is_writeback(struct inode *inode, pgoff_t idx_from,
376 pgoff_t idx_to)
377{
378 struct fuse_conn *fc = get_fuse_conn(inode);
379 struct fuse_inode *fi = get_fuse_inode(inode);
380 struct fuse_req *req;
381 bool found = false;
382
383 spin_lock(&fc->lock);
384 list_for_each_entry(req, &fi->writepages, writepages_entry) {
385 pgoff_t curr_index;
386
387 BUG_ON(req->inode != inode);
388 curr_index = req->misc.write.in.offset >> PAGE_CACHE_SHIFT;
389 if (idx_from < curr_index + req->num_pages &&
390 curr_index <= idx_to) {
391 found = true;
392 break;
393 }
394 }
395 spin_unlock(&fc->lock);
396
397 return found;
398}
399
400static inline bool fuse_page_is_writeback(struct inode *inode, pgoff_t index)
401{
402 return fuse_range_is_writeback(inode, index, index);
403}
404
405
406
407
408
409
410
411static int fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index)
412{
413 struct fuse_inode *fi = get_fuse_inode(inode);
414
415 wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index));
416 return 0;
417}
418
419
420
421
422
423
424
425
426
427
428static void fuse_sync_writes(struct inode *inode)
429{
430 fuse_set_nowrite(inode);
431 fuse_release_nowrite(inode);
432}
433
434static int fuse_flush(struct file *file, fl_owner_t id)
435{
436 struct inode *inode = file_inode(file);
437 struct fuse_conn *fc = get_fuse_conn(inode);
438 struct fuse_file *ff = file->private_data;
439 struct fuse_req *req;
440 struct fuse_flush_in inarg;
441 int err;
442
443 if (is_bad_inode(inode))
444 return -EIO;
445
446 if (fc->no_flush)
447 return 0;
448
449 err = write_inode_now(inode, 1);
450 if (err)
451 return err;
452
453 mutex_lock(&inode->i_mutex);
454 fuse_sync_writes(inode);
455 mutex_unlock(&inode->i_mutex);
456
457 req = fuse_get_req_nofail_nopages(fc, file);
458 memset(&inarg, 0, sizeof(inarg));
459 inarg.fh = ff->fh;
460 inarg.lock_owner = fuse_lock_owner_id(fc, id);
461 req->in.h.opcode = FUSE_FLUSH;
462 req->in.h.nodeid = get_node_id(inode);
463 req->in.numargs = 1;
464 req->in.args[0].size = sizeof(inarg);
465 req->in.args[0].value = &inarg;
466 req->force = 1;
467 fuse_request_send(fc, req);
468 err = req->out.h.error;
469 fuse_put_request(fc, req);
470 if (err == -ENOSYS) {
471 fc->no_flush = 1;
472 err = 0;
473 }
474 return err;
475}
476
477int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
478 int datasync, int isdir)
479{
480 struct inode *inode = file->f_mapping->host;
481 struct fuse_conn *fc = get_fuse_conn(inode);
482 struct fuse_file *ff = file->private_data;
483 struct fuse_req *req;
484 struct fuse_fsync_in inarg;
485 int err;
486
487 if (is_bad_inode(inode))
488 return -EIO;
489
490 mutex_lock(&inode->i_mutex);
491
492
493
494
495
496
497 err = filemap_write_and_wait_range(inode->i_mapping, start, end);
498 if (err)
499 goto out;
500
501 fuse_sync_writes(inode);
502 err = sync_inode_metadata(inode, 1);
503 if (err)
504 goto out;
505
506 if ((!isdir && fc->no_fsync) || (isdir && fc->no_fsyncdir))
507 goto out;
508
509 req = fuse_get_req_nopages(fc);
510 if (IS_ERR(req)) {
511 err = PTR_ERR(req);
512 goto out;
513 }
514
515 memset(&inarg, 0, sizeof(inarg));
516 inarg.fh = ff->fh;
517 inarg.fsync_flags = datasync ? 1 : 0;
518 req->in.h.opcode = isdir ? FUSE_FSYNCDIR : FUSE_FSYNC;
519 req->in.h.nodeid = get_node_id(inode);
520 req->in.numargs = 1;
521 req->in.args[0].size = sizeof(inarg);
522 req->in.args[0].value = &inarg;
523 fuse_request_send(fc, req);
524 err = req->out.h.error;
525 fuse_put_request(fc, req);
526 if (err == -ENOSYS) {
527 if (isdir)
528 fc->no_fsyncdir = 1;
529 else
530 fc->no_fsync = 1;
531 err = 0;
532 }
533out:
534 mutex_unlock(&inode->i_mutex);
535 return err;
536}
537
538static int fuse_fsync(struct file *file, loff_t start, loff_t end,
539 int datasync)
540{
541 return fuse_fsync_common(file, start, end, datasync, 0);
542}
543
544void fuse_read_fill(struct fuse_req *req, struct file *file, loff_t pos,
545 size_t count, int opcode)
546{
547 struct fuse_read_in *inarg = &req->misc.read.in;
548 struct fuse_file *ff = file->private_data;
549
550 inarg->fh = ff->fh;
551 inarg->offset = pos;
552 inarg->size = count;
553 inarg->flags = file->f_flags;
554 req->in.h.opcode = opcode;
555 req->in.h.nodeid = ff->nodeid;
556 req->in.numargs = 1;
557 req->in.args[0].size = sizeof(struct fuse_read_in);
558 req->in.args[0].value = inarg;
559 req->out.argvar = 1;
560 req->out.numargs = 1;
561 req->out.args[0].size = count;
562}
563
564static void fuse_release_user_pages(struct fuse_req *req, int write)
565{
566 unsigned i;
567
568 for (i = 0; i < req->num_pages; i++) {
569 struct page *page = req->pages[i];
570 if (write)
571 set_page_dirty_lock(page);
572 put_page(page);
573 }
574}
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
593{
594 int left;
595
596 spin_lock(&io->lock);
597 if (err)
598 io->err = io->err ? : err;
599 else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))
600 io->bytes = pos;
601
602 left = --io->reqs;
603 spin_unlock(&io->lock);
604
605 if (!left) {
606 long res;
607
608 if (io->err)
609 res = io->err;
610 else if (io->bytes >= 0 && io->write)
611 res = -EIO;
612 else {
613 res = io->bytes < 0 ? io->size : io->bytes;
614
615 if (!is_sync_kiocb(io->iocb)) {
616 struct inode *inode = file_inode(io->iocb->ki_filp);
617 struct fuse_conn *fc = get_fuse_conn(inode);
618 struct fuse_inode *fi = get_fuse_inode(inode);
619
620 spin_lock(&fc->lock);
621 fi->attr_version = ++fc->attr_version;
622 spin_unlock(&fc->lock);
623 }
624 }
625
626 aio_complete(io->iocb, res, 0);
627 kfree(io);
628 }
629}
630
631static void fuse_aio_complete_req(struct fuse_conn *fc, struct fuse_req *req)
632{
633 struct fuse_io_priv *io = req->io;
634 ssize_t pos = -1;
635
636 fuse_release_user_pages(req, !io->write);
637
638 if (io->write) {
639 if (req->misc.write.in.size != req->misc.write.out.size)
640 pos = req->misc.write.in.offset - io->offset +
641 req->misc.write.out.size;
642 } else {
643 if (req->misc.read.in.size != req->out.args[0].size)
644 pos = req->misc.read.in.offset - io->offset +
645 req->out.args[0].size;
646 }
647
648 fuse_aio_complete(io, req->out.h.error, pos);
649}
650
651static size_t fuse_async_req_send(struct fuse_conn *fc, struct fuse_req *req,
652 size_t num_bytes, struct fuse_io_priv *io)
653{
654 spin_lock(&io->lock);
655 io->size += num_bytes;
656 io->reqs++;
657 spin_unlock(&io->lock);
658
659 req->io = io;
660 req->end = fuse_aio_complete_req;
661
662 __fuse_get_request(req);
663 fuse_request_send_background(fc, req);
664
665 return num_bytes;
666}
667
668static size_t fuse_send_read(struct fuse_req *req, struct fuse_io_priv *io,
669 loff_t pos, size_t count, fl_owner_t owner)
670{
671 struct file *file = io->file;
672 struct fuse_file *ff = file->private_data;
673 struct fuse_conn *fc = ff->fc;
674
675 fuse_read_fill(req, file, pos, count, FUSE_READ);
676 if (owner != NULL) {
677 struct fuse_read_in *inarg = &req->misc.read.in;
678
679 inarg->read_flags |= FUSE_READ_LOCKOWNER;
680 inarg->lock_owner = fuse_lock_owner_id(fc, owner);
681 }
682
683 if (io->async)
684 return fuse_async_req_send(fc, req, count, io);
685
686 fuse_request_send(fc, req);
687 return req->out.args[0].size;
688}
689
690static void fuse_read_update_size(struct inode *inode, loff_t size,
691 u64 attr_ver)
692{
693 struct fuse_conn *fc = get_fuse_conn(inode);
694 struct fuse_inode *fi = get_fuse_inode(inode);
695
696 spin_lock(&fc->lock);
697 if (attr_ver == fi->attr_version && size < inode->i_size &&
698 !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) {
699 fi->attr_version = ++fc->attr_version;
700 i_size_write(inode, size);
701 }
702 spin_unlock(&fc->lock);
703}
704
705static void fuse_short_read(struct fuse_req *req, struct inode *inode,
706 u64 attr_ver)
707{
708 size_t num_read = req->out.args[0].size;
709 struct fuse_conn *fc = get_fuse_conn(inode);
710
711 if (fc->writeback_cache) {
712
713
714
715
716
717 int i;
718 int start_idx = num_read >> PAGE_CACHE_SHIFT;
719 size_t off = num_read & (PAGE_CACHE_SIZE - 1);
720
721 for (i = start_idx; i < req->num_pages; i++) {
722 zero_user_segment(req->pages[i], off, PAGE_CACHE_SIZE);
723 off = 0;
724 }
725 } else {
726 loff_t pos = page_offset(req->pages[0]) + num_read;
727 fuse_read_update_size(inode, pos, attr_ver);
728 }
729}
730
731static int fuse_do_readpage(struct file *file, struct page *page)
732{
733 struct fuse_io_priv io = { .async = 0, .file = file };
734 struct inode *inode = page->mapping->host;
735 struct fuse_conn *fc = get_fuse_conn(inode);
736 struct fuse_req *req;
737 size_t num_read;
738 loff_t pos = page_offset(page);
739 size_t count = PAGE_CACHE_SIZE;
740 u64 attr_ver;
741 int err;
742
743
744
745
746
747
748 fuse_wait_on_page_writeback(inode, page->index);
749
750 req = fuse_get_req(fc, 1);
751 if (IS_ERR(req))
752 return PTR_ERR(req);
753
754 attr_ver = fuse_get_attr_version(fc);
755
756 req->out.page_zeroing = 1;
757 req->out.argpages = 1;
758 req->num_pages = 1;
759 req->pages[0] = page;
760 req->page_descs[0].length = count;
761 num_read = fuse_send_read(req, &io, pos, count, NULL);
762 err = req->out.h.error;
763
764 if (!err) {
765
766
767
768 if (num_read < count)
769 fuse_short_read(req, inode, attr_ver);
770
771 SetPageUptodate(page);
772 }
773
774 fuse_put_request(fc, req);
775
776 return err;
777}
778
779static int fuse_readpage(struct file *file, struct page *page)
780{
781 struct inode *inode = page->mapping->host;
782 int err;
783
784 err = -EIO;
785 if (is_bad_inode(inode))
786 goto out;
787
788 err = fuse_do_readpage(file, page);
789 fuse_invalidate_atime(inode);
790 out:
791 unlock_page(page);
792 return err;
793}
794
795static void fuse_readpages_end(struct fuse_conn *fc, struct fuse_req *req)
796{
797 int i;
798 size_t count = req->misc.read.in.size;
799 size_t num_read = req->out.args[0].size;
800 struct address_space *mapping = NULL;
801
802 for (i = 0; mapping == NULL && i < req->num_pages; i++)
803 mapping = req->pages[i]->mapping;
804
805 if (mapping) {
806 struct inode *inode = mapping->host;
807
808
809
810
811 if (!req->out.h.error && num_read < count)
812 fuse_short_read(req, inode, req->misc.read.attr_ver);
813
814 fuse_invalidate_atime(inode);
815 }
816
817 for (i = 0; i < req->num_pages; i++) {
818 struct page *page = req->pages[i];
819 if (!req->out.h.error)
820 SetPageUptodate(page);
821 else
822 SetPageError(page);
823 unlock_page(page);
824 page_cache_release(page);
825 }
826 if (req->ff)
827 fuse_file_put(req->ff, false);
828}
829
830static void fuse_send_readpages(struct fuse_req *req, struct file *file)
831{
832 struct fuse_file *ff = file->private_data;
833 struct fuse_conn *fc = ff->fc;
834 loff_t pos = page_offset(req->pages[0]);
835 size_t count = req->num_pages << PAGE_CACHE_SHIFT;
836
837 req->out.argpages = 1;
838 req->out.page_zeroing = 1;
839 req->out.page_replace = 1;
840 fuse_read_fill(req, file, pos, count, FUSE_READ);
841 req->misc.read.attr_ver = fuse_get_attr_version(fc);
842 if (fc->async_read) {
843 req->ff = fuse_file_get(ff);
844 req->end = fuse_readpages_end;
845 fuse_request_send_background(fc, req);
846 } else {
847 fuse_request_send(fc, req);
848 fuse_readpages_end(fc, req);
849 fuse_put_request(fc, req);
850 }
851}
852
853struct fuse_fill_data {
854 struct fuse_req *req;
855 struct file *file;
856 struct inode *inode;
857 unsigned nr_pages;
858};
859
860static int fuse_readpages_fill(void *_data, struct page *page)
861{
862 struct fuse_fill_data *data = _data;
863 struct fuse_req *req = data->req;
864 struct inode *inode = data->inode;
865 struct fuse_conn *fc = get_fuse_conn(inode);
866
867 fuse_wait_on_page_writeback(inode, page->index);
868
869 if (req->num_pages &&
870 (req->num_pages == FUSE_MAX_PAGES_PER_REQ ||
871 (req->num_pages + 1) * PAGE_CACHE_SIZE > fc->max_read ||
872 req->pages[req->num_pages - 1]->index + 1 != page->index)) {
873 int nr_alloc = min_t(unsigned, data->nr_pages,
874 FUSE_MAX_PAGES_PER_REQ);
875 fuse_send_readpages(req, data->file);
876 if (fc->async_read)
877 req = fuse_get_req_for_background(fc, nr_alloc);
878 else
879 req = fuse_get_req(fc, nr_alloc);
880
881 data->req = req;
882 if (IS_ERR(req)) {
883 unlock_page(page);
884 return PTR_ERR(req);
885 }
886 }
887
888 if (WARN_ON(req->num_pages >= req->max_pages)) {
889 fuse_put_request(fc, req);
890 return -EIO;
891 }
892
893 page_cache_get(page);
894 req->pages[req->num_pages] = page;
895 req->page_descs[req->num_pages].length = PAGE_SIZE;
896 req->num_pages++;
897 data->nr_pages--;
898 return 0;
899}
900
901static int fuse_readpages(struct file *file, struct address_space *mapping,
902 struct list_head *pages, unsigned nr_pages)
903{
904 struct inode *inode = mapping->host;
905 struct fuse_conn *fc = get_fuse_conn(inode);
906 struct fuse_fill_data data;
907 int err;
908 int nr_alloc = min_t(unsigned, nr_pages, FUSE_MAX_PAGES_PER_REQ);
909
910 err = -EIO;
911 if (is_bad_inode(inode))
912 goto out;
913
914 data.file = file;
915 data.inode = inode;
916 if (fc->async_read)
917 data.req = fuse_get_req_for_background(fc, nr_alloc);
918 else
919 data.req = fuse_get_req(fc, nr_alloc);
920 data.nr_pages = nr_pages;
921 err = PTR_ERR(data.req);
922 if (IS_ERR(data.req))
923 goto out;
924
925 err = read_cache_pages(mapping, pages, fuse_readpages_fill, &data);
926 if (!err) {
927 if (data.req->num_pages)
928 fuse_send_readpages(data.req, file);
929 else
930 fuse_put_request(fc, data.req);
931 }
932out:
933 return err;
934}
935
936static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
937{
938 struct inode *inode = iocb->ki_filp->f_mapping->host;
939 struct fuse_conn *fc = get_fuse_conn(inode);
940
941
942
943
944
945
946 if (fc->auto_inval_data ||
947 (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) {
948 int err;
949 err = fuse_update_attributes(inode, NULL, iocb->ki_filp, NULL);
950 if (err)
951 return err;
952 }
953
954 return generic_file_read_iter(iocb, to);
955}
956
957static void fuse_write_fill(struct fuse_req *req, struct fuse_file *ff,
958 loff_t pos, size_t count)
959{
960 struct fuse_write_in *inarg = &req->misc.write.in;
961 struct fuse_write_out *outarg = &req->misc.write.out;
962
963 inarg->fh = ff->fh;
964 inarg->offset = pos;
965 inarg->size = count;
966 req->in.h.opcode = FUSE_WRITE;
967 req->in.h.nodeid = ff->nodeid;
968 req->in.numargs = 2;
969 if (ff->fc->minor < 9)
970 req->in.args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
971 else
972 req->in.args[0].size = sizeof(struct fuse_write_in);
973 req->in.args[0].value = inarg;
974 req->in.args[1].size = count;
975 req->out.numargs = 1;
976 req->out.args[0].size = sizeof(struct fuse_write_out);
977 req->out.args[0].value = outarg;
978}
979
980static size_t fuse_send_write(struct fuse_req *req, struct fuse_io_priv *io,
981 loff_t pos, size_t count, fl_owner_t owner)
982{
983 struct file *file = io->file;
984 struct fuse_file *ff = file->private_data;
985 struct fuse_conn *fc = ff->fc;
986 struct fuse_write_in *inarg = &req->misc.write.in;
987
988 fuse_write_fill(req, ff, pos, count);
989 inarg->flags = file->f_flags;
990 if (owner != NULL) {
991 inarg->write_flags |= FUSE_WRITE_LOCKOWNER;
992 inarg->lock_owner = fuse_lock_owner_id(fc, owner);
993 }
994
995 if (io->async)
996 return fuse_async_req_send(fc, req, count, io);
997
998 fuse_request_send(fc, req);
999 return req->misc.write.out.size;
1000}
1001
1002bool fuse_write_update_size(struct inode *inode, loff_t pos)
1003{
1004 struct fuse_conn *fc = get_fuse_conn(inode);
1005 struct fuse_inode *fi = get_fuse_inode(inode);
1006 bool ret = false;
1007
1008 spin_lock(&fc->lock);
1009 fi->attr_version = ++fc->attr_version;
1010 if (pos > inode->i_size) {
1011 i_size_write(inode, pos);
1012 ret = true;
1013 }
1014 spin_unlock(&fc->lock);
1015
1016 return ret;
1017}
1018
1019static size_t fuse_send_write_pages(struct fuse_req *req, struct file *file,
1020 struct inode *inode, loff_t pos,
1021 size_t count)
1022{
1023 size_t res;
1024 unsigned offset;
1025 unsigned i;
1026 struct fuse_io_priv io = { .async = 0, .file = file };
1027
1028 for (i = 0; i < req->num_pages; i++)
1029 fuse_wait_on_page_writeback(inode, req->pages[i]->index);
1030
1031 res = fuse_send_write(req, &io, pos, count, NULL);
1032
1033 offset = req->page_descs[0].offset;
1034 count = res;
1035 for (i = 0; i < req->num_pages; i++) {
1036 struct page *page = req->pages[i];
1037
1038 if (!req->out.h.error && !offset && count >= PAGE_CACHE_SIZE)
1039 SetPageUptodate(page);
1040
1041 if (count > PAGE_CACHE_SIZE - offset)
1042 count -= PAGE_CACHE_SIZE - offset;
1043 else
1044 count = 0;
1045 offset = 0;
1046
1047 unlock_page(page);
1048 page_cache_release(page);
1049 }
1050
1051 return res;
1052}
1053
1054static ssize_t fuse_fill_write_pages(struct fuse_req *req,
1055 struct address_space *mapping,
1056 struct iov_iter *ii, loff_t pos)
1057{
1058 struct fuse_conn *fc = get_fuse_conn(mapping->host);
1059 unsigned offset = pos & (PAGE_CACHE_SIZE - 1);
1060 size_t count = 0;
1061 int err;
1062
1063 req->in.argpages = 1;
1064 req->page_descs[0].offset = offset;
1065
1066 do {
1067 size_t tmp;
1068 struct page *page;
1069 pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1070 size_t bytes = min_t(size_t, PAGE_CACHE_SIZE - offset,
1071 iov_iter_count(ii));
1072
1073 bytes = min_t(size_t, bytes, fc->max_write - count);
1074
1075 again:
1076 err = -EFAULT;
1077 if (iov_iter_fault_in_readable(ii, bytes))
1078 break;
1079
1080 err = -ENOMEM;
1081 page = grab_cache_page_write_begin(mapping, index, 0);
1082 if (!page)
1083 break;
1084
1085 if (mapping_writably_mapped(mapping))
1086 flush_dcache_page(page);
1087
1088 tmp = iov_iter_copy_from_user_atomic(page, ii, offset, bytes);
1089 flush_dcache_page(page);
1090
1091 if (!tmp) {
1092 unlock_page(page);
1093 page_cache_release(page);
1094 bytes = min(bytes, iov_iter_single_seg_count(ii));
1095 goto again;
1096 }
1097
1098 err = 0;
1099 req->pages[req->num_pages] = page;
1100 req->page_descs[req->num_pages].length = tmp;
1101 req->num_pages++;
1102
1103 iov_iter_advance(ii, tmp);
1104 count += tmp;
1105 pos += tmp;
1106 offset += tmp;
1107 if (offset == PAGE_CACHE_SIZE)
1108 offset = 0;
1109
1110 if (!fc->big_writes)
1111 break;
1112 } while (iov_iter_count(ii) && count < fc->max_write &&
1113 req->num_pages < req->max_pages && offset == 0);
1114
1115 return count > 0 ? count : err;
1116}
1117
1118static inline unsigned fuse_wr_pages(loff_t pos, size_t len)
1119{
1120 return min_t(unsigned,
1121 ((pos + len - 1) >> PAGE_CACHE_SHIFT) -
1122 (pos >> PAGE_CACHE_SHIFT) + 1,
1123 FUSE_MAX_PAGES_PER_REQ);
1124}
1125
1126static ssize_t fuse_perform_write(struct file *file,
1127 struct address_space *mapping,
1128 struct iov_iter *ii, loff_t pos)
1129{
1130 struct inode *inode = mapping->host;
1131 struct fuse_conn *fc = get_fuse_conn(inode);
1132 struct fuse_inode *fi = get_fuse_inode(inode);
1133 int err = 0;
1134 ssize_t res = 0;
1135
1136 if (is_bad_inode(inode))
1137 return -EIO;
1138
1139 if (inode->i_size < pos + iov_iter_count(ii))
1140 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1141
1142 do {
1143 struct fuse_req *req;
1144 ssize_t count;
1145 unsigned nr_pages = fuse_wr_pages(pos, iov_iter_count(ii));
1146
1147 req = fuse_get_req(fc, nr_pages);
1148 if (IS_ERR(req)) {
1149 err = PTR_ERR(req);
1150 break;
1151 }
1152
1153 count = fuse_fill_write_pages(req, mapping, ii, pos);
1154 if (count <= 0) {
1155 err = count;
1156 } else {
1157 size_t num_written;
1158
1159 num_written = fuse_send_write_pages(req, file, inode,
1160 pos, count);
1161 err = req->out.h.error;
1162 if (!err) {
1163 res += num_written;
1164 pos += num_written;
1165
1166
1167 if (num_written != count)
1168 err = -EIO;
1169 }
1170 }
1171 fuse_put_request(fc, req);
1172 } while (!err && iov_iter_count(ii));
1173
1174 if (res > 0)
1175 fuse_write_update_size(inode, pos);
1176
1177 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1178 fuse_invalidate_attr(inode);
1179
1180 return res > 0 ? res : err;
1181}
1182
1183static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
1184{
1185 struct file *file = iocb->ki_filp;
1186 struct address_space *mapping = file->f_mapping;
1187 size_t count = iov_iter_count(from);
1188 ssize_t written = 0;
1189 ssize_t written_buffered = 0;
1190 struct inode *inode = mapping->host;
1191 ssize_t err;
1192 loff_t endbyte = 0;
1193 loff_t pos = iocb->ki_pos;
1194
1195 if (get_fuse_conn(inode)->writeback_cache) {
1196
1197 err = fuse_update_attributes(mapping->host, NULL, file, NULL);
1198 if (err)
1199 return err;
1200
1201 return generic_file_write_iter(iocb, from);
1202 }
1203
1204 mutex_lock(&inode->i_mutex);
1205
1206
1207 current->backing_dev_info = mapping->backing_dev_info;
1208
1209 err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
1210 if (err)
1211 goto out;
1212
1213 if (count == 0)
1214 goto out;
1215
1216 iov_iter_truncate(from, count);
1217 err = file_remove_suid(file);
1218 if (err)
1219 goto out;
1220
1221 err = file_update_time(file);
1222 if (err)
1223 goto out;
1224
1225 if (file->f_flags & O_DIRECT) {
1226 written = generic_file_direct_write(iocb, from, pos);
1227 if (written < 0 || !iov_iter_count(from))
1228 goto out;
1229
1230 pos += written;
1231
1232 written_buffered = fuse_perform_write(file, mapping, from, pos);
1233 if (written_buffered < 0) {
1234 err = written_buffered;
1235 goto out;
1236 }
1237 endbyte = pos + written_buffered - 1;
1238
1239 err = filemap_write_and_wait_range(file->f_mapping, pos,
1240 endbyte);
1241 if (err)
1242 goto out;
1243
1244 invalidate_mapping_pages(file->f_mapping,
1245 pos >> PAGE_CACHE_SHIFT,
1246 endbyte >> PAGE_CACHE_SHIFT);
1247
1248 written += written_buffered;
1249 iocb->ki_pos = pos + written_buffered;
1250 } else {
1251 written = fuse_perform_write(file, mapping, from, pos);
1252 if (written >= 0)
1253 iocb->ki_pos = pos + written;
1254 }
1255out:
1256 current->backing_dev_info = NULL;
1257 mutex_unlock(&inode->i_mutex);
1258
1259 return written ? written : err;
1260}
1261
1262static inline void fuse_page_descs_length_init(struct fuse_req *req,
1263 unsigned index, unsigned nr_pages)
1264{
1265 int i;
1266
1267 for (i = index; i < index + nr_pages; i++)
1268 req->page_descs[i].length = PAGE_SIZE -
1269 req->page_descs[i].offset;
1270}
1271
1272static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
1273{
1274 return (unsigned long)ii->iov->iov_base + ii->iov_offset;
1275}
1276
1277static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
1278 size_t max_size)
1279{
1280 return min(iov_iter_single_seg_count(ii), max_size);
1281}
1282
1283static int fuse_get_user_pages(struct fuse_req *req, struct iov_iter *ii,
1284 size_t *nbytesp, int write)
1285{
1286 size_t nbytes = 0;
1287
1288
1289 if (ii->type & ITER_KVEC) {
1290 unsigned long user_addr = fuse_get_user_addr(ii);
1291 size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
1292
1293 if (write)
1294 req->in.args[1].value = (void *) user_addr;
1295 else
1296 req->out.args[0].value = (void *) user_addr;
1297
1298 iov_iter_advance(ii, frag_size);
1299 *nbytesp = frag_size;
1300 return 0;
1301 }
1302
1303 while (nbytes < *nbytesp && req->num_pages < req->max_pages) {
1304 unsigned npages;
1305 size_t start;
1306 ssize_t ret = iov_iter_get_pages(ii,
1307 &req->pages[req->num_pages],
1308 *nbytesp - nbytes,
1309 req->max_pages - req->num_pages,
1310 &start);
1311 if (ret < 0)
1312 return ret;
1313
1314 iov_iter_advance(ii, ret);
1315 nbytes += ret;
1316
1317 ret += start;
1318 npages = (ret + PAGE_SIZE - 1) / PAGE_SIZE;
1319
1320 req->page_descs[req->num_pages].offset = start;
1321 fuse_page_descs_length_init(req, req->num_pages, npages);
1322
1323 req->num_pages += npages;
1324 req->page_descs[req->num_pages - 1].length -=
1325 (PAGE_SIZE - ret) & (PAGE_SIZE - 1);
1326 }
1327
1328 if (write)
1329 req->in.argpages = 1;
1330 else
1331 req->out.argpages = 1;
1332
1333 *nbytesp = nbytes;
1334
1335 return 0;
1336}
1337
1338static inline int fuse_iter_npages(const struct iov_iter *ii_p)
1339{
1340 return iov_iter_npages(ii_p, FUSE_MAX_PAGES_PER_REQ);
1341}
1342
1343ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
1344 loff_t *ppos, int flags)
1345{
1346 int write = flags & FUSE_DIO_WRITE;
1347 int cuse = flags & FUSE_DIO_CUSE;
1348 struct file *file = io->file;
1349 struct inode *inode = file->f_mapping->host;
1350 struct fuse_file *ff = file->private_data;
1351 struct fuse_conn *fc = ff->fc;
1352 size_t nmax = write ? fc->max_write : fc->max_read;
1353 loff_t pos = *ppos;
1354 size_t count = iov_iter_count(iter);
1355 pgoff_t idx_from = pos >> PAGE_CACHE_SHIFT;
1356 pgoff_t idx_to = (pos + count - 1) >> PAGE_CACHE_SHIFT;
1357 ssize_t res = 0;
1358 struct fuse_req *req;
1359
1360 if (io->async)
1361 req = fuse_get_req_for_background(fc, fuse_iter_npages(iter));
1362 else
1363 req = fuse_get_req(fc, fuse_iter_npages(iter));
1364 if (IS_ERR(req))
1365 return PTR_ERR(req);
1366
1367 if (!cuse && fuse_range_is_writeback(inode, idx_from, idx_to)) {
1368 if (!write)
1369 mutex_lock(&inode->i_mutex);
1370 fuse_sync_writes(inode);
1371 if (!write)
1372 mutex_unlock(&inode->i_mutex);
1373 }
1374
1375 while (count) {
1376 size_t nres;
1377 fl_owner_t owner = current->files;
1378 size_t nbytes = min(count, nmax);
1379 int err = fuse_get_user_pages(req, iter, &nbytes, write);
1380 if (err) {
1381 res = err;
1382 break;
1383 }
1384
1385 if (write)
1386 nres = fuse_send_write(req, io, pos, nbytes, owner);
1387 else
1388 nres = fuse_send_read(req, io, pos, nbytes, owner);
1389
1390 if (!io->async)
1391 fuse_release_user_pages(req, !write);
1392 if (req->out.h.error) {
1393 if (!res)
1394 res = req->out.h.error;
1395 break;
1396 } else if (nres > nbytes) {
1397 res = -EIO;
1398 break;
1399 }
1400 count -= nres;
1401 res += nres;
1402 pos += nres;
1403 if (nres != nbytes)
1404 break;
1405 if (count) {
1406 fuse_put_request(fc, req);
1407 if (io->async)
1408 req = fuse_get_req_for_background(fc,
1409 fuse_iter_npages(iter));
1410 else
1411 req = fuse_get_req(fc, fuse_iter_npages(iter));
1412 if (IS_ERR(req))
1413 break;
1414 }
1415 }
1416 if (!IS_ERR(req))
1417 fuse_put_request(fc, req);
1418 if (res > 0)
1419 *ppos = pos;
1420
1421 return res;
1422}
1423EXPORT_SYMBOL_GPL(fuse_direct_io);
1424
1425static ssize_t __fuse_direct_read(struct fuse_io_priv *io,
1426 struct iov_iter *iter,
1427 loff_t *ppos)
1428{
1429 ssize_t res;
1430 struct file *file = io->file;
1431 struct inode *inode = file_inode(file);
1432
1433 if (is_bad_inode(inode))
1434 return -EIO;
1435
1436 res = fuse_direct_io(io, iter, ppos, 0);
1437
1438 fuse_invalidate_attr(inode);
1439
1440 return res;
1441}
1442
1443static ssize_t fuse_direct_read(struct file *file, char __user *buf,
1444 size_t count, loff_t *ppos)
1445{
1446 struct fuse_io_priv io = { .async = 0, .file = file };
1447 struct iovec iov = { .iov_base = buf, .iov_len = count };
1448 struct iov_iter ii;
1449 iov_iter_init(&ii, READ, &iov, 1, count);
1450 return __fuse_direct_read(&io, &ii, ppos);
1451}
1452
1453static ssize_t __fuse_direct_write(struct fuse_io_priv *io,
1454 struct iov_iter *iter,
1455 loff_t *ppos)
1456{
1457 struct file *file = io->file;
1458 struct inode *inode = file_inode(file);
1459 size_t count = iov_iter_count(iter);
1460 ssize_t res;
1461
1462
1463 res = generic_write_checks(file, ppos, &count, 0);
1464 if (!res) {
1465 iov_iter_truncate(iter, count);
1466 res = fuse_direct_io(io, iter, ppos, FUSE_DIO_WRITE);
1467 }
1468
1469 fuse_invalidate_attr(inode);
1470
1471 return res;
1472}
1473
1474static ssize_t fuse_direct_write(struct file *file, const char __user *buf,
1475 size_t count, loff_t *ppos)
1476{
1477 struct iovec iov = { .iov_base = (void __user *)buf, .iov_len = count };
1478 struct inode *inode = file_inode(file);
1479 ssize_t res;
1480 struct fuse_io_priv io = { .async = 0, .file = file };
1481 struct iov_iter ii;
1482 iov_iter_init(&ii, WRITE, &iov, 1, count);
1483
1484 if (is_bad_inode(inode))
1485 return -EIO;
1486
1487
1488 mutex_lock(&inode->i_mutex);
1489 res = __fuse_direct_write(&io, &ii, ppos);
1490 if (res > 0)
1491 fuse_write_update_size(inode, *ppos);
1492 mutex_unlock(&inode->i_mutex);
1493
1494 return res;
1495}
1496
1497static void fuse_writepage_free(struct fuse_conn *fc, struct fuse_req *req)
1498{
1499 int i;
1500
1501 for (i = 0; i < req->num_pages; i++)
1502 __free_page(req->pages[i]);
1503
1504 if (req->ff)
1505 fuse_file_put(req->ff, false);
1506}
1507
1508static void fuse_writepage_finish(struct fuse_conn *fc, struct fuse_req *req)
1509{
1510 struct inode *inode = req->inode;
1511 struct fuse_inode *fi = get_fuse_inode(inode);
1512 struct backing_dev_info *bdi = inode->i_mapping->backing_dev_info;
1513 int i;
1514
1515 list_del(&req->writepages_entry);
1516 for (i = 0; i < req->num_pages; i++) {
1517 dec_bdi_stat(bdi, BDI_WRITEBACK);
1518 dec_zone_page_state(req->pages[i], NR_WRITEBACK_TEMP);
1519 bdi_writeout_inc(bdi);
1520 }
1521 wake_up(&fi->page_waitq);
1522}
1523
1524
1525static void fuse_send_writepage(struct fuse_conn *fc, struct fuse_req *req,
1526 loff_t size)
1527__releases(fc->lock)
1528__acquires(fc->lock)
1529{
1530 struct fuse_inode *fi = get_fuse_inode(req->inode);
1531 struct fuse_write_in *inarg = &req->misc.write.in;
1532 __u64 data_size = req->num_pages * PAGE_CACHE_SIZE;
1533
1534 if (!fc->connected)
1535 goto out_free;
1536
1537 if (inarg->offset + data_size <= size) {
1538 inarg->size = data_size;
1539 } else if (inarg->offset < size) {
1540 inarg->size = size - inarg->offset;
1541 } else {
1542
1543 goto out_free;
1544 }
1545
1546 req->in.args[1].size = inarg->size;
1547 fi->writectr++;
1548 fuse_request_send_background_locked(fc, req);
1549 return;
1550
1551 out_free:
1552 fuse_writepage_finish(fc, req);
1553 spin_unlock(&fc->lock);
1554 fuse_writepage_free(fc, req);
1555 fuse_put_request(fc, req);
1556 spin_lock(&fc->lock);
1557}
1558
1559
1560
1561
1562
1563
1564
1565void fuse_flush_writepages(struct inode *inode)
1566__releases(fc->lock)
1567__acquires(fc->lock)
1568{
1569 struct fuse_conn *fc = get_fuse_conn(inode);
1570 struct fuse_inode *fi = get_fuse_inode(inode);
1571 size_t crop = i_size_read(inode);
1572 struct fuse_req *req;
1573
1574 while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) {
1575 req = list_entry(fi->queued_writes.next, struct fuse_req, list);
1576 list_del_init(&req->list);
1577 fuse_send_writepage(fc, req, crop);
1578 }
1579}
1580
1581static void fuse_writepage_end(struct fuse_conn *fc, struct fuse_req *req)
1582{
1583 struct inode *inode = req->inode;
1584 struct fuse_inode *fi = get_fuse_inode(inode);
1585
1586 mapping_set_error(inode->i_mapping, req->out.h.error);
1587 spin_lock(&fc->lock);
1588 while (req->misc.write.next) {
1589 struct fuse_conn *fc = get_fuse_conn(inode);
1590 struct fuse_write_in *inarg = &req->misc.write.in;
1591 struct fuse_req *next = req->misc.write.next;
1592 req->misc.write.next = next->misc.write.next;
1593 next->misc.write.next = NULL;
1594 next->ff = fuse_file_get(req->ff);
1595 list_add(&next->writepages_entry, &fi->writepages);
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620 fuse_send_writepage(fc, next, inarg->offset + inarg->size);
1621 }
1622 fi->writectr--;
1623 fuse_writepage_finish(fc, req);
1624 spin_unlock(&fc->lock);
1625 fuse_writepage_free(fc, req);
1626}
1627
1628static struct fuse_file *__fuse_write_file_get(struct fuse_conn *fc,
1629 struct fuse_inode *fi)
1630{
1631 struct fuse_file *ff = NULL;
1632
1633 spin_lock(&fc->lock);
1634 if (!list_empty(&fi->write_files)) {
1635 ff = list_entry(fi->write_files.next, struct fuse_file,
1636 write_entry);
1637 fuse_file_get(ff);
1638 }
1639 spin_unlock(&fc->lock);
1640
1641 return ff;
1642}
1643
1644static struct fuse_file *fuse_write_file_get(struct fuse_conn *fc,
1645 struct fuse_inode *fi)
1646{
1647 struct fuse_file *ff = __fuse_write_file_get(fc, fi);
1648 WARN_ON(!ff);
1649 return ff;
1650}
1651
1652int fuse_write_inode(struct inode *inode, struct writeback_control *wbc)
1653{
1654 struct fuse_conn *fc = get_fuse_conn(inode);
1655 struct fuse_inode *fi = get_fuse_inode(inode);
1656 struct fuse_file *ff;
1657 int err;
1658
1659 ff = __fuse_write_file_get(fc, fi);
1660 err = fuse_flush_times(inode, ff);
1661 if (ff)
1662 fuse_file_put(ff, 0);
1663
1664 return err;
1665}
1666
1667static int fuse_writepage_locked(struct page *page)
1668{
1669 struct address_space *mapping = page->mapping;
1670 struct inode *inode = mapping->host;
1671 struct fuse_conn *fc = get_fuse_conn(inode);
1672 struct fuse_inode *fi = get_fuse_inode(inode);
1673 struct fuse_req *req;
1674 struct page *tmp_page;
1675 int error = -ENOMEM;
1676
1677 set_page_writeback(page);
1678
1679 req = fuse_request_alloc_nofs(1);
1680 if (!req)
1681 goto err;
1682
1683 req->background = 1;
1684 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
1685 if (!tmp_page)
1686 goto err_free;
1687
1688 error = -EIO;
1689 req->ff = fuse_write_file_get(fc, fi);
1690 if (!req->ff)
1691 goto err_nofile;
1692
1693 fuse_write_fill(req, req->ff, page_offset(page), 0);
1694
1695 copy_highpage(tmp_page, page);
1696 req->misc.write.in.write_flags |= FUSE_WRITE_CACHE;
1697 req->misc.write.next = NULL;
1698 req->in.argpages = 1;
1699 req->num_pages = 1;
1700 req->pages[0] = tmp_page;
1701 req->page_descs[0].offset = 0;
1702 req->page_descs[0].length = PAGE_SIZE;
1703 req->end = fuse_writepage_end;
1704 req->inode = inode;
1705
1706 inc_bdi_stat(mapping->backing_dev_info, BDI_WRITEBACK);
1707 inc_zone_page_state(tmp_page, NR_WRITEBACK_TEMP);
1708
1709 spin_lock(&fc->lock);
1710 list_add(&req->writepages_entry, &fi->writepages);
1711 list_add_tail(&req->list, &fi->queued_writes);
1712 fuse_flush_writepages(inode);
1713 spin_unlock(&fc->lock);
1714
1715 end_page_writeback(page);
1716
1717 return 0;
1718
1719err_nofile:
1720 __free_page(tmp_page);
1721err_free:
1722 fuse_request_free(req);
1723err:
1724 end_page_writeback(page);
1725 return error;
1726}
1727
1728static int fuse_writepage(struct page *page, struct writeback_control *wbc)
1729{
1730 int err;
1731
1732 if (fuse_page_is_writeback(page->mapping->host, page->index)) {
1733
1734
1735
1736
1737
1738 WARN_ON(wbc->sync_mode == WB_SYNC_ALL);
1739
1740 redirty_page_for_writepage(wbc, page);
1741 return 0;
1742 }
1743
1744 err = fuse_writepage_locked(page);
1745 unlock_page(page);
1746
1747 return err;
1748}
1749
1750struct fuse_fill_wb_data {
1751 struct fuse_req *req;
1752 struct fuse_file *ff;
1753 struct inode *inode;
1754 struct page **orig_pages;
1755};
1756
1757static void fuse_writepages_send(struct fuse_fill_wb_data *data)
1758{
1759 struct fuse_req *req = data->req;
1760 struct inode *inode = data->inode;
1761 struct fuse_conn *fc = get_fuse_conn(inode);
1762 struct fuse_inode *fi = get_fuse_inode(inode);
1763 int num_pages = req->num_pages;
1764 int i;
1765
1766 req->ff = fuse_file_get(data->ff);
1767 spin_lock(&fc->lock);
1768 list_add_tail(&req->list, &fi->queued_writes);
1769 fuse_flush_writepages(inode);
1770 spin_unlock(&fc->lock);
1771
1772 for (i = 0; i < num_pages; i++)
1773 end_page_writeback(data->orig_pages[i]);
1774}
1775
1776static bool fuse_writepage_in_flight(struct fuse_req *new_req,
1777 struct page *page)
1778{
1779 struct fuse_conn *fc = get_fuse_conn(new_req->inode);
1780 struct fuse_inode *fi = get_fuse_inode(new_req->inode);
1781 struct fuse_req *tmp;
1782 struct fuse_req *old_req;
1783 bool found = false;
1784 pgoff_t curr_index;
1785
1786 BUG_ON(new_req->num_pages != 0);
1787
1788 spin_lock(&fc->lock);
1789 list_del(&new_req->writepages_entry);
1790 list_for_each_entry(old_req, &fi->writepages, writepages_entry) {
1791 BUG_ON(old_req->inode != new_req->inode);
1792 curr_index = old_req->misc.write.in.offset >> PAGE_CACHE_SHIFT;
1793 if (curr_index <= page->index &&
1794 page->index < curr_index + old_req->num_pages) {
1795 found = true;
1796 break;
1797 }
1798 }
1799 if (!found) {
1800 list_add(&new_req->writepages_entry, &fi->writepages);
1801 goto out_unlock;
1802 }
1803
1804 new_req->num_pages = 1;
1805 for (tmp = old_req; tmp != NULL; tmp = tmp->misc.write.next) {
1806 BUG_ON(tmp->inode != new_req->inode);
1807 curr_index = tmp->misc.write.in.offset >> PAGE_CACHE_SHIFT;
1808 if (tmp->num_pages == 1 &&
1809 curr_index == page->index) {
1810 old_req = tmp;
1811 }
1812 }
1813
1814 if (old_req->num_pages == 1 && (old_req->state == FUSE_REQ_INIT ||
1815 old_req->state == FUSE_REQ_PENDING)) {
1816 struct backing_dev_info *bdi = page->mapping->backing_dev_info;
1817
1818 copy_highpage(old_req->pages[0], page);
1819 spin_unlock(&fc->lock);
1820
1821 dec_bdi_stat(bdi, BDI_WRITEBACK);
1822 dec_zone_page_state(page, NR_WRITEBACK_TEMP);
1823 bdi_writeout_inc(bdi);
1824 fuse_writepage_free(fc, new_req);
1825 fuse_request_free(new_req);
1826 goto out;
1827 } else {
1828 new_req->misc.write.next = old_req->misc.write.next;
1829 old_req->misc.write.next = new_req;
1830 }
1831out_unlock:
1832 spin_unlock(&fc->lock);
1833out:
1834 return found;
1835}
1836
1837static int fuse_writepages_fill(struct page *page,
1838 struct writeback_control *wbc, void *_data)
1839{
1840 struct fuse_fill_wb_data *data = _data;
1841 struct fuse_req *req = data->req;
1842 struct inode *inode = data->inode;
1843 struct fuse_conn *fc = get_fuse_conn(inode);
1844 struct page *tmp_page;
1845 bool is_writeback;
1846 int err;
1847
1848 if (!data->ff) {
1849 err = -EIO;
1850 data->ff = fuse_write_file_get(fc, get_fuse_inode(inode));
1851 if (!data->ff)
1852 goto out_unlock;
1853 }
1854
1855
1856
1857
1858
1859
1860
1861 is_writeback = fuse_page_is_writeback(inode, page->index);
1862
1863 if (req && req->num_pages &&
1864 (is_writeback || req->num_pages == FUSE_MAX_PAGES_PER_REQ ||
1865 (req->num_pages + 1) * PAGE_CACHE_SIZE > fc->max_write ||
1866 data->orig_pages[req->num_pages - 1]->index + 1 != page->index)) {
1867 fuse_writepages_send(data);
1868 data->req = NULL;
1869 }
1870 err = -ENOMEM;
1871 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
1872 if (!tmp_page)
1873 goto out_unlock;
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888 if (data->req == NULL) {
1889 struct fuse_inode *fi = get_fuse_inode(inode);
1890
1891 err = -ENOMEM;
1892 req = fuse_request_alloc_nofs(FUSE_MAX_PAGES_PER_REQ);
1893 if (!req) {
1894 __free_page(tmp_page);
1895 goto out_unlock;
1896 }
1897
1898 fuse_write_fill(req, data->ff, page_offset(page), 0);
1899 req->misc.write.in.write_flags |= FUSE_WRITE_CACHE;
1900 req->misc.write.next = NULL;
1901 req->in.argpages = 1;
1902 req->background = 1;
1903 req->num_pages = 0;
1904 req->end = fuse_writepage_end;
1905 req->inode = inode;
1906
1907 spin_lock(&fc->lock);
1908 list_add(&req->writepages_entry, &fi->writepages);
1909 spin_unlock(&fc->lock);
1910
1911 data->req = req;
1912 }
1913 set_page_writeback(page);
1914
1915 copy_highpage(tmp_page, page);
1916 req->pages[req->num_pages] = tmp_page;
1917 req->page_descs[req->num_pages].offset = 0;
1918 req->page_descs[req->num_pages].length = PAGE_SIZE;
1919
1920 inc_bdi_stat(page->mapping->backing_dev_info, BDI_WRITEBACK);
1921 inc_zone_page_state(tmp_page, NR_WRITEBACK_TEMP);
1922
1923 err = 0;
1924 if (is_writeback && fuse_writepage_in_flight(req, page)) {
1925 end_page_writeback(page);
1926 data->req = NULL;
1927 goto out_unlock;
1928 }
1929 data->orig_pages[req->num_pages] = page;
1930
1931
1932
1933
1934
1935 spin_lock(&fc->lock);
1936 req->num_pages++;
1937 spin_unlock(&fc->lock);
1938
1939out_unlock:
1940 unlock_page(page);
1941
1942 return err;
1943}
1944
1945static int fuse_writepages(struct address_space *mapping,
1946 struct writeback_control *wbc)
1947{
1948 struct inode *inode = mapping->host;
1949 struct fuse_fill_wb_data data;
1950 int err;
1951
1952 err = -EIO;
1953 if (is_bad_inode(inode))
1954 goto out;
1955
1956 data.inode = inode;
1957 data.req = NULL;
1958 data.ff = NULL;
1959
1960 err = -ENOMEM;
1961 data.orig_pages = kcalloc(FUSE_MAX_PAGES_PER_REQ,
1962 sizeof(struct page *),
1963 GFP_NOFS);
1964 if (!data.orig_pages)
1965 goto out;
1966
1967 err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data);
1968 if (data.req) {
1969
1970 BUG_ON(!data.req->num_pages);
1971 fuse_writepages_send(&data);
1972 err = 0;
1973 }
1974 if (data.ff)
1975 fuse_file_put(data.ff, false);
1976
1977 kfree(data.orig_pages);
1978out:
1979 return err;
1980}
1981
1982
1983
1984
1985
1986static int fuse_write_begin(struct file *file, struct address_space *mapping,
1987 loff_t pos, unsigned len, unsigned flags,
1988 struct page **pagep, void **fsdata)
1989{
1990 pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1991 struct fuse_conn *fc = get_fuse_conn(file->f_dentry->d_inode);
1992 struct page *page;
1993 loff_t fsize;
1994 int err = -ENOMEM;
1995
1996 WARN_ON(!fc->writeback_cache);
1997
1998 page = grab_cache_page_write_begin(mapping, index, flags);
1999 if (!page)
2000 goto error;
2001
2002 fuse_wait_on_page_writeback(mapping->host, page->index);
2003
2004 if (PageUptodate(page) || len == PAGE_CACHE_SIZE)
2005 goto success;
2006
2007
2008
2009
2010 fsize = i_size_read(mapping->host);
2011 if (fsize <= (pos & PAGE_CACHE_MASK)) {
2012 size_t off = pos & ~PAGE_CACHE_MASK;
2013 if (off)
2014 zero_user_segment(page, 0, off);
2015 goto success;
2016 }
2017 err = fuse_do_readpage(file, page);
2018 if (err)
2019 goto cleanup;
2020success:
2021 *pagep = page;
2022 return 0;
2023
2024cleanup:
2025 unlock_page(page);
2026 page_cache_release(page);
2027error:
2028 return err;
2029}
2030
2031static int fuse_write_end(struct file *file, struct address_space *mapping,
2032 loff_t pos, unsigned len, unsigned copied,
2033 struct page *page, void *fsdata)
2034{
2035 struct inode *inode = page->mapping->host;
2036
2037 if (!PageUptodate(page)) {
2038
2039 size_t endoff = (pos + copied) & ~PAGE_CACHE_MASK;
2040 if (endoff)
2041 zero_user_segment(page, endoff, PAGE_CACHE_SIZE);
2042 SetPageUptodate(page);
2043 }
2044
2045 fuse_write_update_size(inode, pos + copied);
2046 set_page_dirty(page);
2047 unlock_page(page);
2048 page_cache_release(page);
2049
2050 return copied;
2051}
2052
2053static int fuse_launder_page(struct page *page)
2054{
2055 int err = 0;
2056 if (clear_page_dirty_for_io(page)) {
2057 struct inode *inode = page->mapping->host;
2058 err = fuse_writepage_locked(page);
2059 if (!err)
2060 fuse_wait_on_page_writeback(inode, page->index);
2061 }
2062 return err;
2063}
2064
2065
2066
2067
2068
2069static void fuse_vma_close(struct vm_area_struct *vma)
2070{
2071 filemap_write_and_wait(vma->vm_file->f_mapping);
2072}
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089static int fuse_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
2090{
2091 struct page *page = vmf->page;
2092 struct inode *inode = file_inode(vma->vm_file);
2093
2094 file_update_time(vma->vm_file);
2095 lock_page(page);
2096 if (page->mapping != inode->i_mapping) {
2097 unlock_page(page);
2098 return VM_FAULT_NOPAGE;
2099 }
2100
2101 fuse_wait_on_page_writeback(inode, page->index);
2102 return VM_FAULT_LOCKED;
2103}
2104
2105static const struct vm_operations_struct fuse_file_vm_ops = {
2106 .close = fuse_vma_close,
2107 .fault = filemap_fault,
2108 .map_pages = filemap_map_pages,
2109 .page_mkwrite = fuse_page_mkwrite,
2110 .remap_pages = generic_file_remap_pages,
2111};
2112
2113static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
2114{
2115 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2116 fuse_link_write_file(file);
2117
2118 file_accessed(file);
2119 vma->vm_ops = &fuse_file_vm_ops;
2120 return 0;
2121}
2122
2123static int fuse_direct_mmap(struct file *file, struct vm_area_struct *vma)
2124{
2125
2126 if (vma->vm_flags & VM_MAYSHARE)
2127 return -ENODEV;
2128
2129 invalidate_inode_pages2(file->f_mapping);
2130
2131 return generic_file_mmap(file, vma);
2132}
2133
2134static int convert_fuse_file_lock(const struct fuse_file_lock *ffl,
2135 struct file_lock *fl)
2136{
2137 switch (ffl->type) {
2138 case F_UNLCK:
2139 break;
2140
2141 case F_RDLCK:
2142 case F_WRLCK:
2143 if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX ||
2144 ffl->end < ffl->start)
2145 return -EIO;
2146
2147 fl->fl_start = ffl->start;
2148 fl->fl_end = ffl->end;
2149 fl->fl_pid = ffl->pid;
2150 break;
2151
2152 default:
2153 return -EIO;
2154 }
2155 fl->fl_type = ffl->type;
2156 return 0;
2157}
2158
2159static void fuse_lk_fill(struct fuse_req *req, struct file *file,
2160 const struct file_lock *fl, int opcode, pid_t pid,
2161 int flock)
2162{
2163 struct inode *inode = file_inode(file);
2164 struct fuse_conn *fc = get_fuse_conn(inode);
2165 struct fuse_file *ff = file->private_data;
2166 struct fuse_lk_in *arg = &req->misc.lk_in;
2167
2168 arg->fh = ff->fh;
2169 arg->owner = fuse_lock_owner_id(fc, fl->fl_owner);
2170 arg->lk.start = fl->fl_start;
2171 arg->lk.end = fl->fl_end;
2172 arg->lk.type = fl->fl_type;
2173 arg->lk.pid = pid;
2174 if (flock)
2175 arg->lk_flags |= FUSE_LK_FLOCK;
2176 req->in.h.opcode = opcode;
2177 req->in.h.nodeid = get_node_id(inode);
2178 req->in.numargs = 1;
2179 req->in.args[0].size = sizeof(*arg);
2180 req->in.args[0].value = arg;
2181}
2182
2183static int fuse_getlk(struct file *file, struct file_lock *fl)
2184{
2185 struct inode *inode = file_inode(file);
2186 struct fuse_conn *fc = get_fuse_conn(inode);
2187 struct fuse_req *req;
2188 struct fuse_lk_out outarg;
2189 int err;
2190
2191 req = fuse_get_req_nopages(fc);
2192 if (IS_ERR(req))
2193 return PTR_ERR(req);
2194
2195 fuse_lk_fill(req, file, fl, FUSE_GETLK, 0, 0);
2196 req->out.numargs = 1;
2197 req->out.args[0].size = sizeof(outarg);
2198 req->out.args[0].value = &outarg;
2199 fuse_request_send(fc, req);
2200 err = req->out.h.error;
2201 fuse_put_request(fc, req);
2202 if (!err)
2203 err = convert_fuse_file_lock(&outarg.lk, fl);
2204
2205 return err;
2206}
2207
2208static int fuse_setlk(struct file *file, struct file_lock *fl, int flock)
2209{
2210 struct inode *inode = file_inode(file);
2211 struct fuse_conn *fc = get_fuse_conn(inode);
2212 struct fuse_req *req;
2213 int opcode = (fl->fl_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK;
2214 pid_t pid = fl->fl_type != F_UNLCK ? current->tgid : 0;
2215 int err;
2216
2217 if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
2218
2219 return -ENOLCK;
2220 }
2221
2222
2223 if (fl->fl_flags & FL_CLOSE)
2224 return 0;
2225
2226 req = fuse_get_req_nopages(fc);
2227 if (IS_ERR(req))
2228 return PTR_ERR(req);
2229
2230 fuse_lk_fill(req, file, fl, opcode, pid, flock);
2231 fuse_request_send(fc, req);
2232 err = req->out.h.error;
2233
2234 if (err == -EINTR)
2235 err = -ERESTARTSYS;
2236 fuse_put_request(fc, req);
2237 return err;
2238}
2239
2240static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl)
2241{
2242 struct inode *inode = file_inode(file);
2243 struct fuse_conn *fc = get_fuse_conn(inode);
2244 int err;
2245
2246 if (cmd == F_CANCELLK) {
2247 err = 0;
2248 } else if (cmd == F_GETLK) {
2249 if (fc->no_lock) {
2250 posix_test_lock(file, fl);
2251 err = 0;
2252 } else
2253 err = fuse_getlk(file, fl);
2254 } else {
2255 if (fc->no_lock)
2256 err = posix_lock_file(file, fl, NULL);
2257 else
2258 err = fuse_setlk(file, fl, 0);
2259 }
2260 return err;
2261}
2262
2263static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl)
2264{
2265 struct inode *inode = file_inode(file);
2266 struct fuse_conn *fc = get_fuse_conn(inode);
2267 int err;
2268
2269 if (fc->no_flock) {
2270 err = flock_lock_file_wait(file, fl);
2271 } else {
2272 struct fuse_file *ff = file->private_data;
2273
2274
2275 ff->flock = true;
2276 err = fuse_setlk(file, fl, 1);
2277 }
2278
2279 return err;
2280}
2281
2282static sector_t fuse_bmap(struct address_space *mapping, sector_t block)
2283{
2284 struct inode *inode = mapping->host;
2285 struct fuse_conn *fc = get_fuse_conn(inode);
2286 struct fuse_req *req;
2287 struct fuse_bmap_in inarg;
2288 struct fuse_bmap_out outarg;
2289 int err;
2290
2291 if (!inode->i_sb->s_bdev || fc->no_bmap)
2292 return 0;
2293
2294 req = fuse_get_req_nopages(fc);
2295 if (IS_ERR(req))
2296 return 0;
2297
2298 memset(&inarg, 0, sizeof(inarg));
2299 inarg.block = block;
2300 inarg.blocksize = inode->i_sb->s_blocksize;
2301 req->in.h.opcode = FUSE_BMAP;
2302 req->in.h.nodeid = get_node_id(inode);
2303 req->in.numargs = 1;
2304 req->in.args[0].size = sizeof(inarg);
2305 req->in.args[0].value = &inarg;
2306 req->out.numargs = 1;
2307 req->out.args[0].size = sizeof(outarg);
2308 req->out.args[0].value = &outarg;
2309 fuse_request_send(fc, req);
2310 err = req->out.h.error;
2311 fuse_put_request(fc, req);
2312 if (err == -ENOSYS)
2313 fc->no_bmap = 1;
2314
2315 return err ? 0 : outarg.block;
2316}
2317
2318static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
2319{
2320 loff_t retval;
2321 struct inode *inode = file_inode(file);
2322
2323
2324 if (whence == SEEK_CUR || whence == SEEK_SET)
2325 return generic_file_llseek(file, offset, whence);
2326
2327 mutex_lock(&inode->i_mutex);
2328 retval = fuse_update_attributes(inode, NULL, file, NULL);
2329 if (!retval)
2330 retval = generic_file_llseek(file, offset, whence);
2331 mutex_unlock(&inode->i_mutex);
2332
2333 return retval;
2334}
2335
2336static int fuse_ioctl_copy_user(struct page **pages, struct iovec *iov,
2337 unsigned int nr_segs, size_t bytes, bool to_user)
2338{
2339 struct iov_iter ii;
2340 int page_idx = 0;
2341
2342 if (!bytes)
2343 return 0;
2344
2345 iov_iter_init(&ii, to_user ? READ : WRITE, iov, nr_segs, bytes);
2346
2347 while (iov_iter_count(&ii)) {
2348 struct page *page = pages[page_idx++];
2349 size_t todo = min_t(size_t, PAGE_SIZE, iov_iter_count(&ii));
2350 void *kaddr;
2351
2352 kaddr = kmap(page);
2353
2354 while (todo) {
2355 char __user *uaddr = ii.iov->iov_base + ii.iov_offset;
2356 size_t iov_len = ii.iov->iov_len - ii.iov_offset;
2357 size_t copy = min(todo, iov_len);
2358 size_t left;
2359
2360 if (!to_user)
2361 left = copy_from_user(kaddr, uaddr, copy);
2362 else
2363 left = copy_to_user(uaddr, kaddr, copy);
2364
2365 if (unlikely(left))
2366 return -EFAULT;
2367
2368 iov_iter_advance(&ii, copy);
2369 todo -= copy;
2370 kaddr += copy;
2371 }
2372
2373 kunmap(page);
2374 }
2375
2376 return 0;
2377}
2378
2379
2380
2381
2382
2383
2384
2385static int fuse_copy_ioctl_iovec_old(struct iovec *dst, void *src,
2386 size_t transferred, unsigned count,
2387 bool is_compat)
2388{
2389#ifdef CONFIG_COMPAT
2390 if (count * sizeof(struct compat_iovec) == transferred) {
2391 struct compat_iovec *ciov = src;
2392 unsigned i;
2393
2394
2395
2396
2397
2398
2399 if (!is_compat)
2400 return -EINVAL;
2401
2402 for (i = 0; i < count; i++) {
2403 dst[i].iov_base = compat_ptr(ciov[i].iov_base);
2404 dst[i].iov_len = ciov[i].iov_len;
2405 }
2406 return 0;
2407 }
2408#endif
2409
2410 if (count * sizeof(struct iovec) != transferred)
2411 return -EIO;
2412
2413 memcpy(dst, src, transferred);
2414 return 0;
2415}
2416
2417
2418static int fuse_verify_ioctl_iov(struct iovec *iov, size_t count)
2419{
2420 size_t n;
2421 u32 max = FUSE_MAX_PAGES_PER_REQ << PAGE_SHIFT;
2422
2423 for (n = 0; n < count; n++, iov++) {
2424 if (iov->iov_len > (size_t) max)
2425 return -ENOMEM;
2426 max -= iov->iov_len;
2427 }
2428 return 0;
2429}
2430
2431static int fuse_copy_ioctl_iovec(struct fuse_conn *fc, struct iovec *dst,
2432 void *src, size_t transferred, unsigned count,
2433 bool is_compat)
2434{
2435 unsigned i;
2436 struct fuse_ioctl_iovec *fiov = src;
2437
2438 if (fc->minor < 16) {
2439 return fuse_copy_ioctl_iovec_old(dst, src, transferred,
2440 count, is_compat);
2441 }
2442
2443 if (count * sizeof(struct fuse_ioctl_iovec) != transferred)
2444 return -EIO;
2445
2446 for (i = 0; i < count; i++) {
2447
2448 if (fiov[i].base != (unsigned long) fiov[i].base ||
2449 fiov[i].len != (unsigned long) fiov[i].len)
2450 return -EIO;
2451
2452 dst[i].iov_base = (void __user *) (unsigned long) fiov[i].base;
2453 dst[i].iov_len = (size_t) fiov[i].len;
2454
2455#ifdef CONFIG_COMPAT
2456 if (is_compat &&
2457 (ptr_to_compat(dst[i].iov_base) != fiov[i].base ||
2458 (compat_size_t) dst[i].iov_len != fiov[i].len))
2459 return -EIO;
2460#endif
2461 }
2462
2463 return 0;
2464}
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
2514 unsigned int flags)
2515{
2516 struct fuse_file *ff = file->private_data;
2517 struct fuse_conn *fc = ff->fc;
2518 struct fuse_ioctl_in inarg = {
2519 .fh = ff->fh,
2520 .cmd = cmd,
2521 .arg = arg,
2522 .flags = flags
2523 };
2524 struct fuse_ioctl_out outarg;
2525 struct fuse_req *req = NULL;
2526 struct page **pages = NULL;
2527 struct iovec *iov_page = NULL;
2528 struct iovec *in_iov = NULL, *out_iov = NULL;
2529 unsigned int in_iovs = 0, out_iovs = 0, num_pages = 0, max_pages;
2530 size_t in_size, out_size, transferred;
2531 int err;
2532
2533#if BITS_PER_LONG == 32
2534 inarg.flags |= FUSE_IOCTL_32BIT;
2535#else
2536 if (flags & FUSE_IOCTL_COMPAT)
2537 inarg.flags |= FUSE_IOCTL_32BIT;
2538#endif
2539
2540
2541 BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE);
2542
2543 err = -ENOMEM;
2544 pages = kcalloc(FUSE_MAX_PAGES_PER_REQ, sizeof(pages[0]), GFP_KERNEL);
2545 iov_page = (struct iovec *) __get_free_page(GFP_KERNEL);
2546 if (!pages || !iov_page)
2547 goto out;
2548
2549
2550
2551
2552
2553 if (!(flags & FUSE_IOCTL_UNRESTRICTED)) {
2554 struct iovec *iov = iov_page;
2555
2556 iov->iov_base = (void __user *)arg;
2557 iov->iov_len = _IOC_SIZE(cmd);
2558
2559 if (_IOC_DIR(cmd) & _IOC_WRITE) {
2560 in_iov = iov;
2561 in_iovs = 1;
2562 }
2563
2564 if (_IOC_DIR(cmd) & _IOC_READ) {
2565 out_iov = iov;
2566 out_iovs = 1;
2567 }
2568 }
2569
2570 retry:
2571 inarg.in_size = in_size = iov_length(in_iov, in_iovs);
2572 inarg.out_size = out_size = iov_length(out_iov, out_iovs);
2573
2574
2575
2576
2577
2578 out_size = max_t(size_t, out_size, PAGE_SIZE);
2579 max_pages = DIV_ROUND_UP(max(in_size, out_size), PAGE_SIZE);
2580
2581
2582 err = -ENOMEM;
2583 if (max_pages > FUSE_MAX_PAGES_PER_REQ)
2584 goto out;
2585 while (num_pages < max_pages) {
2586 pages[num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
2587 if (!pages[num_pages])
2588 goto out;
2589 num_pages++;
2590 }
2591
2592 req = fuse_get_req(fc, num_pages);
2593 if (IS_ERR(req)) {
2594 err = PTR_ERR(req);
2595 req = NULL;
2596 goto out;
2597 }
2598 memcpy(req->pages, pages, sizeof(req->pages[0]) * num_pages);
2599 req->num_pages = num_pages;
2600 fuse_page_descs_length_init(req, 0, req->num_pages);
2601
2602
2603 req->in.h.opcode = FUSE_IOCTL;
2604 req->in.h.nodeid = ff->nodeid;
2605 req->in.numargs = 1;
2606 req->in.args[0].size = sizeof(inarg);
2607 req->in.args[0].value = &inarg;
2608 if (in_size) {
2609 req->in.numargs++;
2610 req->in.args[1].size = in_size;
2611 req->in.argpages = 1;
2612
2613 err = fuse_ioctl_copy_user(pages, in_iov, in_iovs, in_size,
2614 false);
2615 if (err)
2616 goto out;
2617 }
2618
2619 req->out.numargs = 2;
2620 req->out.args[0].size = sizeof(outarg);
2621 req->out.args[0].value = &outarg;
2622 req->out.args[1].size = out_size;
2623 req->out.argpages = 1;
2624 req->out.argvar = 1;
2625
2626 fuse_request_send(fc, req);
2627 err = req->out.h.error;
2628 transferred = req->out.args[1].size;
2629 fuse_put_request(fc, req);
2630 req = NULL;
2631 if (err)
2632 goto out;
2633
2634
2635 if (outarg.flags & FUSE_IOCTL_RETRY) {
2636 void *vaddr;
2637
2638
2639 err = -EIO;
2640 if (!(flags & FUSE_IOCTL_UNRESTRICTED))
2641 goto out;
2642
2643 in_iovs = outarg.in_iovs;
2644 out_iovs = outarg.out_iovs;
2645
2646
2647
2648
2649
2650 err = -ENOMEM;
2651 if (in_iovs > FUSE_IOCTL_MAX_IOV ||
2652 out_iovs > FUSE_IOCTL_MAX_IOV ||
2653 in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV)
2654 goto out;
2655
2656 vaddr = kmap_atomic(pages[0]);
2657 err = fuse_copy_ioctl_iovec(fc, iov_page, vaddr,
2658 transferred, in_iovs + out_iovs,
2659 (flags & FUSE_IOCTL_COMPAT) != 0);
2660 kunmap_atomic(vaddr);
2661 if (err)
2662 goto out;
2663
2664 in_iov = iov_page;
2665 out_iov = in_iov + in_iovs;
2666
2667 err = fuse_verify_ioctl_iov(in_iov, in_iovs);
2668 if (err)
2669 goto out;
2670
2671 err = fuse_verify_ioctl_iov(out_iov, out_iovs);
2672 if (err)
2673 goto out;
2674
2675 goto retry;
2676 }
2677
2678 err = -EIO;
2679 if (transferred > inarg.out_size)
2680 goto out;
2681
2682 err = fuse_ioctl_copy_user(pages, out_iov, out_iovs, transferred, true);
2683 out:
2684 if (req)
2685 fuse_put_request(fc, req);
2686 free_page((unsigned long) iov_page);
2687 while (num_pages)
2688 __free_page(pages[--num_pages]);
2689 kfree(pages);
2690
2691 return err ? err : outarg.result;
2692}
2693EXPORT_SYMBOL_GPL(fuse_do_ioctl);
2694
2695long fuse_ioctl_common(struct file *file, unsigned int cmd,
2696 unsigned long arg, unsigned int flags)
2697{
2698 struct inode *inode = file_inode(file);
2699 struct fuse_conn *fc = get_fuse_conn(inode);
2700
2701 if (!fuse_allow_current_process(fc))
2702 return -EACCES;
2703
2704 if (is_bad_inode(inode))
2705 return -EIO;
2706
2707 return fuse_do_ioctl(file, cmd, arg, flags);
2708}
2709
2710static long fuse_file_ioctl(struct file *file, unsigned int cmd,
2711 unsigned long arg)
2712{
2713 return fuse_ioctl_common(file, cmd, arg, 0);
2714}
2715
2716static long fuse_file_compat_ioctl(struct file *file, unsigned int cmd,
2717 unsigned long arg)
2718{
2719 return fuse_ioctl_common(file, cmd, arg, FUSE_IOCTL_COMPAT);
2720}
2721
2722
2723
2724
2725
2726
2727static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
2728 struct rb_node **parent_out)
2729{
2730 struct rb_node **link = &fc->polled_files.rb_node;
2731 struct rb_node *last = NULL;
2732
2733 while (*link) {
2734 struct fuse_file *ff;
2735
2736 last = *link;
2737 ff = rb_entry(last, struct fuse_file, polled_node);
2738
2739 if (kh < ff->kh)
2740 link = &last->rb_left;
2741 else if (kh > ff->kh)
2742 link = &last->rb_right;
2743 else
2744 return link;
2745 }
2746
2747 if (parent_out)
2748 *parent_out = last;
2749 return link;
2750}
2751
2752
2753
2754
2755
2756
2757
2758static void fuse_register_polled_file(struct fuse_conn *fc,
2759 struct fuse_file *ff)
2760{
2761 spin_lock(&fc->lock);
2762 if (RB_EMPTY_NODE(&ff->polled_node)) {
2763 struct rb_node **link, *uninitialized_var(parent);
2764
2765 link = fuse_find_polled_node(fc, ff->kh, &parent);
2766 BUG_ON(*link);
2767 rb_link_node(&ff->polled_node, parent, link);
2768 rb_insert_color(&ff->polled_node, &fc->polled_files);
2769 }
2770 spin_unlock(&fc->lock);
2771}
2772
2773unsigned fuse_file_poll(struct file *file, poll_table *wait)
2774{
2775 struct fuse_file *ff = file->private_data;
2776 struct fuse_conn *fc = ff->fc;
2777 struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
2778 struct fuse_poll_out outarg;
2779 struct fuse_req *req;
2780 int err;
2781
2782 if (fc->no_poll)
2783 return DEFAULT_POLLMASK;
2784
2785 poll_wait(file, &ff->poll_wait, wait);
2786 inarg.events = (__u32)poll_requested_events(wait);
2787
2788
2789
2790
2791
2792 if (waitqueue_active(&ff->poll_wait)) {
2793 inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
2794 fuse_register_polled_file(fc, ff);
2795 }
2796
2797 req = fuse_get_req_nopages(fc);
2798 if (IS_ERR(req))
2799 return POLLERR;
2800
2801 req->in.h.opcode = FUSE_POLL;
2802 req->in.h.nodeid = ff->nodeid;
2803 req->in.numargs = 1;
2804 req->in.args[0].size = sizeof(inarg);
2805 req->in.args[0].value = &inarg;
2806 req->out.numargs = 1;
2807 req->out.args[0].size = sizeof(outarg);
2808 req->out.args[0].value = &outarg;
2809 fuse_request_send(fc, req);
2810 err = req->out.h.error;
2811 fuse_put_request(fc, req);
2812
2813 if (!err)
2814 return outarg.revents;
2815 if (err == -ENOSYS) {
2816 fc->no_poll = 1;
2817 return DEFAULT_POLLMASK;
2818 }
2819 return POLLERR;
2820}
2821EXPORT_SYMBOL_GPL(fuse_file_poll);
2822
2823
2824
2825
2826
2827int fuse_notify_poll_wakeup(struct fuse_conn *fc,
2828 struct fuse_notify_poll_wakeup_out *outarg)
2829{
2830 u64 kh = outarg->kh;
2831 struct rb_node **link;
2832
2833 spin_lock(&fc->lock);
2834
2835 link = fuse_find_polled_node(fc, kh, NULL);
2836 if (*link) {
2837 struct fuse_file *ff;
2838
2839 ff = rb_entry(*link, struct fuse_file, polled_node);
2840 wake_up_interruptible_sync(&ff->poll_wait);
2841 }
2842
2843 spin_unlock(&fc->lock);
2844 return 0;
2845}
2846
2847static void fuse_do_truncate(struct file *file)
2848{
2849 struct inode *inode = file->f_mapping->host;
2850 struct iattr attr;
2851
2852 attr.ia_valid = ATTR_SIZE;
2853 attr.ia_size = i_size_read(inode);
2854
2855 attr.ia_file = file;
2856 attr.ia_valid |= ATTR_FILE;
2857
2858 fuse_do_setattr(inode, &attr, file);
2859}
2860
2861static inline loff_t fuse_round_up(loff_t off)
2862{
2863 return round_up(off, FUSE_MAX_PAGES_PER_REQ << PAGE_SHIFT);
2864}
2865
2866static ssize_t
2867fuse_direct_IO(int rw, struct kiocb *iocb, struct iov_iter *iter,
2868 loff_t offset)
2869{
2870 ssize_t ret = 0;
2871 struct file *file = iocb->ki_filp;
2872 struct fuse_file *ff = file->private_data;
2873 bool async_dio = ff->fc->async_dio;
2874 loff_t pos = 0;
2875 struct inode *inode;
2876 loff_t i_size;
2877 size_t count = iov_iter_count(iter);
2878 struct fuse_io_priv *io;
2879
2880 pos = offset;
2881 inode = file->f_mapping->host;
2882 i_size = i_size_read(inode);
2883
2884 if ((rw == READ) && (offset > i_size))
2885 return 0;
2886
2887
2888 if (async_dio && rw != WRITE && offset + count > i_size) {
2889 if (offset >= i_size)
2890 return 0;
2891 count = min_t(loff_t, count, fuse_round_up(i_size - offset));
2892 iov_iter_truncate(iter, count);
2893 }
2894
2895 io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL);
2896 if (!io)
2897 return -ENOMEM;
2898 spin_lock_init(&io->lock);
2899 io->reqs = 1;
2900 io->bytes = -1;
2901 io->size = 0;
2902 io->offset = offset;
2903 io->write = (rw == WRITE);
2904 io->err = 0;
2905 io->file = file;
2906
2907
2908
2909
2910 io->async = async_dio;
2911 io->iocb = iocb;
2912
2913
2914
2915
2916
2917
2918 if (!is_sync_kiocb(iocb) && (offset + count > i_size) && rw == WRITE)
2919 io->async = false;
2920
2921 if (rw == WRITE)
2922 ret = __fuse_direct_write(io, iter, &pos);
2923 else
2924 ret = __fuse_direct_read(io, iter, &pos);
2925
2926 if (io->async) {
2927 fuse_aio_complete(io, ret < 0 ? ret : 0, -1);
2928
2929
2930 if (!is_sync_kiocb(iocb))
2931 return -EIOCBQUEUED;
2932
2933 ret = wait_on_sync_kiocb(iocb);
2934 } else {
2935 kfree(io);
2936 }
2937
2938 if (rw == WRITE) {
2939 if (ret > 0)
2940 fuse_write_update_size(inode, pos);
2941 else if (ret < 0 && offset + count > i_size)
2942 fuse_do_truncate(file);
2943 }
2944
2945 return ret;
2946}
2947
2948static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
2949 loff_t length)
2950{
2951 struct fuse_file *ff = file->private_data;
2952 struct inode *inode = file->f_inode;
2953 struct fuse_inode *fi = get_fuse_inode(inode);
2954 struct fuse_conn *fc = ff->fc;
2955 struct fuse_req *req;
2956 struct fuse_fallocate_in inarg = {
2957 .fh = ff->fh,
2958 .offset = offset,
2959 .length = length,
2960 .mode = mode
2961 };
2962 int err;
2963 bool lock_inode = !(mode & FALLOC_FL_KEEP_SIZE) ||
2964 (mode & FALLOC_FL_PUNCH_HOLE);
2965
2966 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2967 return -EOPNOTSUPP;
2968
2969 if (fc->no_fallocate)
2970 return -EOPNOTSUPP;
2971
2972 if (lock_inode) {
2973 mutex_lock(&inode->i_mutex);
2974 if (mode & FALLOC_FL_PUNCH_HOLE) {
2975 loff_t endbyte = offset + length - 1;
2976 err = filemap_write_and_wait_range(inode->i_mapping,
2977 offset, endbyte);
2978 if (err)
2979 goto out;
2980
2981 fuse_sync_writes(inode);
2982 }
2983 }
2984
2985 if (!(mode & FALLOC_FL_KEEP_SIZE))
2986 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
2987
2988 req = fuse_get_req_nopages(fc);
2989 if (IS_ERR(req)) {
2990 err = PTR_ERR(req);
2991 goto out;
2992 }
2993
2994 req->in.h.opcode = FUSE_FALLOCATE;
2995 req->in.h.nodeid = ff->nodeid;
2996 req->in.numargs = 1;
2997 req->in.args[0].size = sizeof(inarg);
2998 req->in.args[0].value = &inarg;
2999 fuse_request_send(fc, req);
3000 err = req->out.h.error;
3001 if (err == -ENOSYS) {
3002 fc->no_fallocate = 1;
3003 err = -EOPNOTSUPP;
3004 }
3005 fuse_put_request(fc, req);
3006
3007 if (err)
3008 goto out;
3009
3010
3011 if (!(mode & FALLOC_FL_KEEP_SIZE)) {
3012 bool changed = fuse_write_update_size(inode, offset + length);
3013
3014 if (changed && fc->writeback_cache)
3015 file_update_time(file);
3016 }
3017
3018 if (mode & FALLOC_FL_PUNCH_HOLE)
3019 truncate_pagecache_range(inode, offset, offset + length - 1);
3020
3021 fuse_invalidate_attr(inode);
3022
3023out:
3024 if (!(mode & FALLOC_FL_KEEP_SIZE))
3025 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
3026
3027 if (lock_inode)
3028 mutex_unlock(&inode->i_mutex);
3029
3030 return err;
3031}
3032
3033static const struct file_operations fuse_file_operations = {
3034 .llseek = fuse_file_llseek,
3035 .read = new_sync_read,
3036 .read_iter = fuse_file_read_iter,
3037 .write = new_sync_write,
3038 .write_iter = fuse_file_write_iter,
3039 .mmap = fuse_file_mmap,
3040 .open = fuse_open,
3041 .flush = fuse_flush,
3042 .release = fuse_release,
3043 .fsync = fuse_fsync,
3044 .lock = fuse_file_lock,
3045 .flock = fuse_file_flock,
3046 .splice_read = generic_file_splice_read,
3047 .unlocked_ioctl = fuse_file_ioctl,
3048 .compat_ioctl = fuse_file_compat_ioctl,
3049 .poll = fuse_file_poll,
3050 .fallocate = fuse_file_fallocate,
3051};
3052
3053static const struct file_operations fuse_direct_io_file_operations = {
3054 .llseek = fuse_file_llseek,
3055 .read = fuse_direct_read,
3056 .write = fuse_direct_write,
3057 .mmap = fuse_direct_mmap,
3058 .open = fuse_open,
3059 .flush = fuse_flush,
3060 .release = fuse_release,
3061 .fsync = fuse_fsync,
3062 .lock = fuse_file_lock,
3063 .flock = fuse_file_flock,
3064 .unlocked_ioctl = fuse_file_ioctl,
3065 .compat_ioctl = fuse_file_compat_ioctl,
3066 .poll = fuse_file_poll,
3067 .fallocate = fuse_file_fallocate,
3068
3069};
3070
3071static const struct address_space_operations fuse_file_aops = {
3072 .readpage = fuse_readpage,
3073 .writepage = fuse_writepage,
3074 .writepages = fuse_writepages,
3075 .launder_page = fuse_launder_page,
3076 .readpages = fuse_readpages,
3077 .set_page_dirty = __set_page_dirty_nobuffers,
3078 .bmap = fuse_bmap,
3079 .direct_IO = fuse_direct_IO,
3080 .write_begin = fuse_write_begin,
3081 .write_end = fuse_write_end,
3082};
3083
3084void fuse_init_file_inode(struct inode *inode)
3085{
3086 inode->i_fop = &fuse_file_operations;
3087 inode->i_data.a_ops = &fuse_file_aops;
3088}
3089