1
2
3
4
5
6
7#include "fuse_i.h"
8
9#include <linux/delay.h>
10#include <linux/dax.h>
11#include <linux/uio.h>
12#include <linux/pagemap.h>
13#include <linux/pfn_t.h>
14#include <linux/iomap.h>
15#include <linux/interval_tree.h>
16
17
18
19
20
21#define FUSE_DAX_SHIFT 21
22#define FUSE_DAX_SZ (1 << FUSE_DAX_SHIFT)
23#define FUSE_DAX_PAGES (FUSE_DAX_SZ / PAGE_SIZE)
24
25
26#define FUSE_DAX_RECLAIM_CHUNK (10)
27
28
29
30
31
32
33#define FUSE_DAX_RECLAIM_THRESHOLD (20)
34
35
36struct fuse_dax_mapping {
37
38 struct inode *inode;
39
40
41 struct list_head list;
42
43
44 struct interval_tree_node itn;
45
46
47 struct list_head busy_list;
48
49
50 u64 window_offset;
51
52
53 loff_t length;
54
55
56 bool writable;
57
58
59 refcount_t refcnt;
60};
61
62
63struct fuse_inode_dax {
64
65 struct rw_semaphore sem;
66
67
68 struct rb_root_cached tree;
69 unsigned long nr;
70};
71
72struct fuse_conn_dax {
73
74 struct dax_device *dev;
75
76
77 spinlock_t lock;
78
79
80 unsigned long nr_busy_ranges;
81 struct list_head busy_ranges;
82
83
84 struct delayed_work free_work;
85
86
87 wait_queue_head_t range_waitq;
88
89
90 long nr_free_ranges;
91 struct list_head free_ranges;
92
93 unsigned long nr_ranges;
94};
95
96static inline struct fuse_dax_mapping *
97node_to_dmap(struct interval_tree_node *node)
98{
99 if (!node)
100 return NULL;
101
102 return container_of(node, struct fuse_dax_mapping, itn);
103}
104
105static struct fuse_dax_mapping *
106alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode);
107
108static void
109__kick_dmap_free_worker(struct fuse_conn_dax *fcd, unsigned long delay_ms)
110{
111 unsigned long free_threshold;
112
113
114 free_threshold = max_t(unsigned long, fcd->nr_ranges * FUSE_DAX_RECLAIM_THRESHOLD / 100,
115 1);
116 if (fcd->nr_free_ranges < free_threshold)
117 queue_delayed_work(system_long_wq, &fcd->free_work,
118 msecs_to_jiffies(delay_ms));
119}
120
121static void kick_dmap_free_worker(struct fuse_conn_dax *fcd,
122 unsigned long delay_ms)
123{
124 spin_lock(&fcd->lock);
125 __kick_dmap_free_worker(fcd, delay_ms);
126 spin_unlock(&fcd->lock);
127}
128
129static struct fuse_dax_mapping *alloc_dax_mapping(struct fuse_conn_dax *fcd)
130{
131 struct fuse_dax_mapping *dmap;
132
133 spin_lock(&fcd->lock);
134 dmap = list_first_entry_or_null(&fcd->free_ranges,
135 struct fuse_dax_mapping, list);
136 if (dmap) {
137 list_del_init(&dmap->list);
138 WARN_ON(fcd->nr_free_ranges <= 0);
139 fcd->nr_free_ranges--;
140 }
141 spin_unlock(&fcd->lock);
142
143 kick_dmap_free_worker(fcd, 0);
144 return dmap;
145}
146
147
148static void __dmap_remove_busy_list(struct fuse_conn_dax *fcd,
149 struct fuse_dax_mapping *dmap)
150{
151 list_del_init(&dmap->busy_list);
152 WARN_ON(fcd->nr_busy_ranges == 0);
153 fcd->nr_busy_ranges--;
154}
155
156static void dmap_remove_busy_list(struct fuse_conn_dax *fcd,
157 struct fuse_dax_mapping *dmap)
158{
159 spin_lock(&fcd->lock);
160 __dmap_remove_busy_list(fcd, dmap);
161 spin_unlock(&fcd->lock);
162}
163
164
165static void __dmap_add_to_free_pool(struct fuse_conn_dax *fcd,
166 struct fuse_dax_mapping *dmap)
167{
168 list_add_tail(&dmap->list, &fcd->free_ranges);
169 fcd->nr_free_ranges++;
170 wake_up(&fcd->range_waitq);
171}
172
173static void dmap_add_to_free_pool(struct fuse_conn_dax *fcd,
174 struct fuse_dax_mapping *dmap)
175{
176
177 spin_lock(&fcd->lock);
178 __dmap_add_to_free_pool(fcd, dmap);
179 spin_unlock(&fcd->lock);
180}
181
182static int fuse_setup_one_mapping(struct inode *inode, unsigned long start_idx,
183 struct fuse_dax_mapping *dmap, bool writable,
184 bool upgrade)
185{
186 struct fuse_mount *fm = get_fuse_mount(inode);
187 struct fuse_conn_dax *fcd = fm->fc->dax;
188 struct fuse_inode *fi = get_fuse_inode(inode);
189 struct fuse_setupmapping_in inarg;
190 loff_t offset = start_idx << FUSE_DAX_SHIFT;
191 FUSE_ARGS(args);
192 ssize_t err;
193
194 WARN_ON(fcd->nr_free_ranges < 0);
195
196
197 memset(&inarg, 0, sizeof(inarg));
198 inarg.foffset = offset;
199 inarg.fh = -1;
200 inarg.moffset = dmap->window_offset;
201 inarg.len = FUSE_DAX_SZ;
202 inarg.flags |= FUSE_SETUPMAPPING_FLAG_READ;
203 if (writable)
204 inarg.flags |= FUSE_SETUPMAPPING_FLAG_WRITE;
205 args.opcode = FUSE_SETUPMAPPING;
206 args.nodeid = fi->nodeid;
207 args.in_numargs = 1;
208 args.in_args[0].size = sizeof(inarg);
209 args.in_args[0].value = &inarg;
210 err = fuse_simple_request(fm, &args);
211 if (err < 0)
212 return err;
213 dmap->writable = writable;
214 if (!upgrade) {
215
216
217
218
219
220 dmap->inode = inode;
221 dmap->itn.start = dmap->itn.last = start_idx;
222
223 interval_tree_insert(&dmap->itn, &fi->dax->tree);
224 fi->dax->nr++;
225 spin_lock(&fcd->lock);
226 list_add_tail(&dmap->busy_list, &fcd->busy_ranges);
227 fcd->nr_busy_ranges++;
228 spin_unlock(&fcd->lock);
229 }
230 return 0;
231}
232
233static int fuse_send_removemapping(struct inode *inode,
234 struct fuse_removemapping_in *inargp,
235 struct fuse_removemapping_one *remove_one)
236{
237 struct fuse_inode *fi = get_fuse_inode(inode);
238 struct fuse_mount *fm = get_fuse_mount(inode);
239 FUSE_ARGS(args);
240
241 args.opcode = FUSE_REMOVEMAPPING;
242 args.nodeid = fi->nodeid;
243 args.in_numargs = 2;
244 args.in_args[0].size = sizeof(*inargp);
245 args.in_args[0].value = inargp;
246 args.in_args[1].size = inargp->count * sizeof(*remove_one);
247 args.in_args[1].value = remove_one;
248 return fuse_simple_request(fm, &args);
249}
250
251static int dmap_removemapping_list(struct inode *inode, unsigned int num,
252 struct list_head *to_remove)
253{
254 struct fuse_removemapping_one *remove_one, *ptr;
255 struct fuse_removemapping_in inarg;
256 struct fuse_dax_mapping *dmap;
257 int ret, i = 0, nr_alloc;
258
259 nr_alloc = min_t(unsigned int, num, FUSE_REMOVEMAPPING_MAX_ENTRY);
260 remove_one = kmalloc_array(nr_alloc, sizeof(*remove_one), GFP_NOFS);
261 if (!remove_one)
262 return -ENOMEM;
263
264 ptr = remove_one;
265 list_for_each_entry(dmap, to_remove, list) {
266 ptr->moffset = dmap->window_offset;
267 ptr->len = dmap->length;
268 ptr++;
269 i++;
270 num--;
271 if (i >= nr_alloc || num == 0) {
272 memset(&inarg, 0, sizeof(inarg));
273 inarg.count = i;
274 ret = fuse_send_removemapping(inode, &inarg,
275 remove_one);
276 if (ret)
277 goto out;
278 ptr = remove_one;
279 i = 0;
280 }
281 }
282out:
283 kfree(remove_one);
284 return ret;
285}
286
287
288
289
290
291static void dmap_reinit_add_to_free_pool(struct fuse_conn_dax *fcd,
292 struct fuse_dax_mapping *dmap)
293{
294 pr_debug("fuse: freeing memory range start_idx=0x%lx end_idx=0x%lx window_offset=0x%llx length=0x%llx\n",
295 dmap->itn.start, dmap->itn.last, dmap->window_offset,
296 dmap->length);
297 __dmap_remove_busy_list(fcd, dmap);
298 dmap->inode = NULL;
299 dmap->itn.start = dmap->itn.last = 0;
300 __dmap_add_to_free_pool(fcd, dmap);
301}
302
303
304
305
306
307
308
309static void inode_reclaim_dmap_range(struct fuse_conn_dax *fcd,
310 struct inode *inode,
311 loff_t start, loff_t end)
312{
313 struct fuse_inode *fi = get_fuse_inode(inode);
314 struct fuse_dax_mapping *dmap, *n;
315 int err, num = 0;
316 LIST_HEAD(to_remove);
317 unsigned long start_idx = start >> FUSE_DAX_SHIFT;
318 unsigned long end_idx = end >> FUSE_DAX_SHIFT;
319 struct interval_tree_node *node;
320
321 while (1) {
322 node = interval_tree_iter_first(&fi->dax->tree, start_idx,
323 end_idx);
324 if (!node)
325 break;
326 dmap = node_to_dmap(node);
327
328 WARN_ON(refcount_read(&dmap->refcnt) > 1);
329 interval_tree_remove(&dmap->itn, &fi->dax->tree);
330 num++;
331 list_add(&dmap->list, &to_remove);
332 }
333
334
335 if (list_empty(&to_remove))
336 return;
337
338 WARN_ON(fi->dax->nr < num);
339 fi->dax->nr -= num;
340 err = dmap_removemapping_list(inode, num, &to_remove);
341 if (err && err != -ENOTCONN) {
342 pr_warn("Failed to removemappings. start=0x%llx end=0x%llx\n",
343 start, end);
344 }
345 spin_lock(&fcd->lock);
346 list_for_each_entry_safe(dmap, n, &to_remove, list) {
347 list_del_init(&dmap->list);
348 dmap_reinit_add_to_free_pool(fcd, dmap);
349 }
350 spin_unlock(&fcd->lock);
351}
352
353static int dmap_removemapping_one(struct inode *inode,
354 struct fuse_dax_mapping *dmap)
355{
356 struct fuse_removemapping_one forget_one;
357 struct fuse_removemapping_in inarg;
358
359 memset(&inarg, 0, sizeof(inarg));
360 inarg.count = 1;
361 memset(&forget_one, 0, sizeof(forget_one));
362 forget_one.moffset = dmap->window_offset;
363 forget_one.len = dmap->length;
364
365 return fuse_send_removemapping(inode, &inarg, &forget_one);
366}
367
368
369
370
371
372
373
374void fuse_dax_inode_cleanup(struct inode *inode)
375{
376 struct fuse_conn *fc = get_fuse_conn(inode);
377 struct fuse_inode *fi = get_fuse_inode(inode);
378
379
380
381
382
383
384 inode_reclaim_dmap_range(fc->dax, inode, 0, -1);
385 WARN_ON(fi->dax->nr);
386}
387
388static void fuse_fill_iomap_hole(struct iomap *iomap, loff_t length)
389{
390 iomap->addr = IOMAP_NULL_ADDR;
391 iomap->length = length;
392 iomap->type = IOMAP_HOLE;
393}
394
395static void fuse_fill_iomap(struct inode *inode, loff_t pos, loff_t length,
396 struct iomap *iomap, struct fuse_dax_mapping *dmap,
397 unsigned int flags)
398{
399 loff_t offset, len;
400 loff_t i_size = i_size_read(inode);
401
402 offset = pos - (dmap->itn.start << FUSE_DAX_SHIFT);
403 len = min(length, dmap->length - offset);
404
405
406 if (pos + len > i_size)
407 len = i_size - pos;
408
409 if (len > 0) {
410 iomap->addr = dmap->window_offset + offset;
411 iomap->length = len;
412 if (flags & IOMAP_FAULT)
413 iomap->length = ALIGN(len, PAGE_SIZE);
414 iomap->type = IOMAP_MAPPED;
415
416
417
418
419
420 refcount_inc(&dmap->refcnt);
421
422
423 WARN_ON_ONCE(iomap->private);
424 iomap->private = dmap;
425 } else {
426
427 fuse_fill_iomap_hole(iomap, length);
428 }
429}
430
431static int fuse_setup_new_dax_mapping(struct inode *inode, loff_t pos,
432 loff_t length, unsigned int flags,
433 struct iomap *iomap)
434{
435 struct fuse_inode *fi = get_fuse_inode(inode);
436 struct fuse_conn *fc = get_fuse_conn(inode);
437 struct fuse_conn_dax *fcd = fc->dax;
438 struct fuse_dax_mapping *dmap, *alloc_dmap = NULL;
439 int ret;
440 bool writable = flags & IOMAP_WRITE;
441 unsigned long start_idx = pos >> FUSE_DAX_SHIFT;
442 struct interval_tree_node *node;
443
444
445
446
447
448
449
450
451
452
453
454 if (flags & IOMAP_FAULT) {
455 alloc_dmap = alloc_dax_mapping(fcd);
456 if (!alloc_dmap)
457 return -EAGAIN;
458 } else {
459 alloc_dmap = alloc_dax_mapping_reclaim(fcd, inode);
460 if (IS_ERR(alloc_dmap))
461 return PTR_ERR(alloc_dmap);
462 }
463
464
465 if (WARN_ON(!alloc_dmap))
466 return -EIO;
467
468
469
470
471
472 down_write(&fi->dax->sem);
473
474
475
476
477 node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
478 if (node) {
479 dmap = node_to_dmap(node);
480 fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
481 dmap_add_to_free_pool(fcd, alloc_dmap);
482 up_write(&fi->dax->sem);
483 return 0;
484 }
485
486
487 ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, alloc_dmap,
488 writable, false);
489 if (ret < 0) {
490 dmap_add_to_free_pool(fcd, alloc_dmap);
491 up_write(&fi->dax->sem);
492 return ret;
493 }
494 fuse_fill_iomap(inode, pos, length, iomap, alloc_dmap, flags);
495 up_write(&fi->dax->sem);
496 return 0;
497}
498
499static int fuse_upgrade_dax_mapping(struct inode *inode, loff_t pos,
500 loff_t length, unsigned int flags,
501 struct iomap *iomap)
502{
503 struct fuse_inode *fi = get_fuse_inode(inode);
504 struct fuse_dax_mapping *dmap;
505 int ret;
506 unsigned long idx = pos >> FUSE_DAX_SHIFT;
507 struct interval_tree_node *node;
508
509
510
511
512
513 down_write(&fi->dax->sem);
514 node = interval_tree_iter_first(&fi->dax->tree, idx, idx);
515
516
517
518
519
520
521
522 ret = -EIO;
523 if (WARN_ON(!node))
524 goto out_err;
525
526 dmap = node_to_dmap(node);
527
528
529
530
531
532 if (refcount_dec_and_test(&dmap->refcnt)) {
533
534
535
536 WARN_ON_ONCE(1);
537 }
538
539
540
541
542 if (dmap->writable) {
543 ret = 0;
544 goto out_fill_iomap;
545 }
546
547 ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, dmap, true,
548 true);
549 if (ret < 0)
550 goto out_err;
551out_fill_iomap:
552 fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
553out_err:
554 up_write(&fi->dax->sem);
555 return ret;
556}
557
558
559
560
561static int fuse_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
562 unsigned int flags, struct iomap *iomap,
563 struct iomap *srcmap)
564{
565 struct fuse_inode *fi = get_fuse_inode(inode);
566 struct fuse_conn *fc = get_fuse_conn(inode);
567 struct fuse_dax_mapping *dmap;
568 bool writable = flags & IOMAP_WRITE;
569 unsigned long start_idx = pos >> FUSE_DAX_SHIFT;
570 struct interval_tree_node *node;
571
572
573 if (WARN_ON(flags & IOMAP_REPORT))
574 return -EIO;
575
576 iomap->offset = pos;
577 iomap->flags = 0;
578 iomap->bdev = NULL;
579 iomap->dax_dev = fc->dax->dev;
580
581
582
583
584
585
586
587
588 down_read(&fi->dax->sem);
589 node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
590 if (node) {
591 dmap = node_to_dmap(node);
592 if (writable && !dmap->writable) {
593
594
595
596
597
598
599
600
601
602 refcount_inc(&dmap->refcnt);
603 up_read(&fi->dax->sem);
604 pr_debug("%s: Upgrading mapping at offset 0x%llx length 0x%llx\n",
605 __func__, pos, length);
606 return fuse_upgrade_dax_mapping(inode, pos, length,
607 flags, iomap);
608 } else {
609 fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
610 up_read(&fi->dax->sem);
611 return 0;
612 }
613 } else {
614 up_read(&fi->dax->sem);
615 pr_debug("%s: no mapping at offset 0x%llx length 0x%llx\n",
616 __func__, pos, length);
617 if (pos >= i_size_read(inode))
618 goto iomap_hole;
619
620 return fuse_setup_new_dax_mapping(inode, pos, length, flags,
621 iomap);
622 }
623
624
625
626
627
628iomap_hole:
629 fuse_fill_iomap_hole(iomap, length);
630 pr_debug("%s returning hole mapping. pos=0x%llx length_asked=0x%llx length_returned=0x%llx\n",
631 __func__, pos, length, iomap->length);
632 return 0;
633}
634
635static int fuse_iomap_end(struct inode *inode, loff_t pos, loff_t length,
636 ssize_t written, unsigned int flags,
637 struct iomap *iomap)
638{
639 struct fuse_dax_mapping *dmap = iomap->private;
640
641 if (dmap) {
642 if (refcount_dec_and_test(&dmap->refcnt)) {
643
644
645
646 WARN_ON_ONCE(1);
647 }
648 }
649
650
651
652
653 return 0;
654}
655
656static const struct iomap_ops fuse_iomap_ops = {
657 .iomap_begin = fuse_iomap_begin,
658 .iomap_end = fuse_iomap_end,
659};
660
661static void fuse_wait_dax_page(struct inode *inode)
662{
663 filemap_invalidate_unlock(inode->i_mapping);
664 schedule();
665 filemap_invalidate_lock(inode->i_mapping);
666}
667
668
669static int __fuse_dax_break_layouts(struct inode *inode, bool *retry,
670 loff_t start, loff_t end)
671{
672 struct page *page;
673
674 page = dax_layout_busy_page_range(inode->i_mapping, start, end);
675 if (!page)
676 return 0;
677
678 *retry = true;
679 return ___wait_var_event(&page->_refcount,
680 atomic_read(&page->_refcount) == 1, TASK_INTERRUPTIBLE,
681 0, 0, fuse_wait_dax_page(inode));
682}
683
684
685int fuse_dax_break_layouts(struct inode *inode, u64 dmap_start,
686 u64 dmap_end)
687{
688 bool retry;
689 int ret;
690
691 do {
692 retry = false;
693 ret = __fuse_dax_break_layouts(inode, &retry, dmap_start,
694 dmap_end);
695 } while (ret == 0 && retry);
696
697 return ret;
698}
699
700ssize_t fuse_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)
701{
702 struct inode *inode = file_inode(iocb->ki_filp);
703 ssize_t ret;
704
705 if (iocb->ki_flags & IOCB_NOWAIT) {
706 if (!inode_trylock_shared(inode))
707 return -EAGAIN;
708 } else {
709 inode_lock_shared(inode);
710 }
711
712 ret = dax_iomap_rw(iocb, to, &fuse_iomap_ops);
713 inode_unlock_shared(inode);
714
715
716 return ret;
717}
718
719static bool file_extending_write(struct kiocb *iocb, struct iov_iter *from)
720{
721 struct inode *inode = file_inode(iocb->ki_filp);
722
723 return (iov_iter_rw(from) == WRITE &&
724 ((iocb->ki_pos) >= i_size_read(inode) ||
725 (iocb->ki_pos + iov_iter_count(from) > i_size_read(inode))));
726}
727
728static ssize_t fuse_dax_direct_write(struct kiocb *iocb, struct iov_iter *from)
729{
730 struct inode *inode = file_inode(iocb->ki_filp);
731 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
732 ssize_t ret;
733
734 ret = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE);
735 if (ret < 0)
736 return ret;
737
738 fuse_invalidate_attr(inode);
739 fuse_write_update_size(inode, iocb->ki_pos);
740 return ret;
741}
742
743ssize_t fuse_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)
744{
745 struct inode *inode = file_inode(iocb->ki_filp);
746 ssize_t ret;
747
748 if (iocb->ki_flags & IOCB_NOWAIT) {
749 if (!inode_trylock(inode))
750 return -EAGAIN;
751 } else {
752 inode_lock(inode);
753 }
754
755 ret = generic_write_checks(iocb, from);
756 if (ret <= 0)
757 goto out;
758
759 ret = file_remove_privs(iocb->ki_filp);
760 if (ret)
761 goto out;
762
763
764
765
766
767 if (file_extending_write(iocb, from))
768 ret = fuse_dax_direct_write(iocb, from);
769 else
770 ret = dax_iomap_rw(iocb, from, &fuse_iomap_ops);
771
772out:
773 inode_unlock(inode);
774
775 if (ret > 0)
776 ret = generic_write_sync(iocb, ret);
777 return ret;
778}
779
780static int fuse_dax_writepages(struct address_space *mapping,
781 struct writeback_control *wbc)
782{
783
784 struct inode *inode = mapping->host;
785 struct fuse_conn *fc = get_fuse_conn(inode);
786
787 return dax_writeback_mapping_range(mapping, fc->dax->dev, wbc);
788}
789
790static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf,
791 enum page_entry_size pe_size, bool write)
792{
793 vm_fault_t ret;
794 struct inode *inode = file_inode(vmf->vma->vm_file);
795 struct super_block *sb = inode->i_sb;
796 pfn_t pfn;
797 int error = 0;
798 struct fuse_conn *fc = get_fuse_conn(inode);
799 struct fuse_conn_dax *fcd = fc->dax;
800 bool retry = false;
801
802 if (write)
803 sb_start_pagefault(sb);
804retry:
805 if (retry && !(fcd->nr_free_ranges > 0))
806 wait_event(fcd->range_waitq, (fcd->nr_free_ranges > 0));
807
808
809
810
811
812
813
814 filemap_invalidate_lock_shared(inode->i_mapping);
815 ret = dax_iomap_fault(vmf, pe_size, &pfn, &error, &fuse_iomap_ops);
816 if ((ret & VM_FAULT_ERROR) && error == -EAGAIN) {
817 error = 0;
818 retry = true;
819 filemap_invalidate_unlock_shared(inode->i_mapping);
820 goto retry;
821 }
822
823 if (ret & VM_FAULT_NEEDDSYNC)
824 ret = dax_finish_sync_fault(vmf, pe_size, pfn);
825 filemap_invalidate_unlock_shared(inode->i_mapping);
826
827 if (write)
828 sb_end_pagefault(sb);
829
830 return ret;
831}
832
833static vm_fault_t fuse_dax_fault(struct vm_fault *vmf)
834{
835 return __fuse_dax_fault(vmf, PE_SIZE_PTE,
836 vmf->flags & FAULT_FLAG_WRITE);
837}
838
839static vm_fault_t fuse_dax_huge_fault(struct vm_fault *vmf,
840 enum page_entry_size pe_size)
841{
842 return __fuse_dax_fault(vmf, pe_size, vmf->flags & FAULT_FLAG_WRITE);
843}
844
845static vm_fault_t fuse_dax_page_mkwrite(struct vm_fault *vmf)
846{
847 return __fuse_dax_fault(vmf, PE_SIZE_PTE, true);
848}
849
850static vm_fault_t fuse_dax_pfn_mkwrite(struct vm_fault *vmf)
851{
852 return __fuse_dax_fault(vmf, PE_SIZE_PTE, true);
853}
854
855static const struct vm_operations_struct fuse_dax_vm_ops = {
856 .fault = fuse_dax_fault,
857 .huge_fault = fuse_dax_huge_fault,
858 .page_mkwrite = fuse_dax_page_mkwrite,
859 .pfn_mkwrite = fuse_dax_pfn_mkwrite,
860};
861
862int fuse_dax_mmap(struct file *file, struct vm_area_struct *vma)
863{
864 file_accessed(file);
865 vma->vm_ops = &fuse_dax_vm_ops;
866 vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
867 return 0;
868}
869
870static int dmap_writeback_invalidate(struct inode *inode,
871 struct fuse_dax_mapping *dmap)
872{
873 int ret;
874 loff_t start_pos = dmap->itn.start << FUSE_DAX_SHIFT;
875 loff_t end_pos = (start_pos + FUSE_DAX_SZ - 1);
876
877 ret = filemap_fdatawrite_range(inode->i_mapping, start_pos, end_pos);
878 if (ret) {
879 pr_debug("fuse: filemap_fdatawrite_range() failed. err=%d start_pos=0x%llx, end_pos=0x%llx\n",
880 ret, start_pos, end_pos);
881 return ret;
882 }
883
884 ret = invalidate_inode_pages2_range(inode->i_mapping,
885 start_pos >> PAGE_SHIFT,
886 end_pos >> PAGE_SHIFT);
887 if (ret)
888 pr_debug("fuse: invalidate_inode_pages2_range() failed err=%d\n",
889 ret);
890
891 return ret;
892}
893
894static int reclaim_one_dmap_locked(struct inode *inode,
895 struct fuse_dax_mapping *dmap)
896{
897 int ret;
898 struct fuse_inode *fi = get_fuse_inode(inode);
899
900
901
902
903
904 ret = dmap_writeback_invalidate(inode, dmap);
905 if (ret)
906 return ret;
907
908
909 interval_tree_remove(&dmap->itn, &fi->dax->tree);
910 fi->dax->nr--;
911
912
913
914
915
916 ret = dmap_removemapping_one(inode, dmap);
917 if (ret && ret != -ENOTCONN) {
918 pr_warn("Failed to remove mapping. offset=0x%llx len=0x%llx ret=%d\n",
919 dmap->window_offset, dmap->length, ret);
920 }
921 return 0;
922}
923
924
925
926
927static struct fuse_dax_mapping *inode_lookup_first_dmap(struct inode *inode)
928{
929 struct fuse_inode *fi = get_fuse_inode(inode);
930 struct fuse_dax_mapping *dmap;
931 struct interval_tree_node *node;
932
933 for (node = interval_tree_iter_first(&fi->dax->tree, 0, -1); node;
934 node = interval_tree_iter_next(node, 0, -1)) {
935 dmap = node_to_dmap(node);
936
937 if (refcount_read(&dmap->refcnt) > 1)
938 continue;
939
940 return dmap;
941 }
942
943 return NULL;
944}
945
946
947
948
949
950static struct fuse_dax_mapping *
951inode_inline_reclaim_one_dmap(struct fuse_conn_dax *fcd, struct inode *inode,
952 bool *retry)
953{
954 struct fuse_inode *fi = get_fuse_inode(inode);
955 struct fuse_dax_mapping *dmap;
956 u64 dmap_start, dmap_end;
957 unsigned long start_idx;
958 int ret;
959 struct interval_tree_node *node;
960
961 filemap_invalidate_lock(inode->i_mapping);
962
963
964 down_read(&fi->dax->sem);
965 dmap = inode_lookup_first_dmap(inode);
966 if (dmap) {
967 start_idx = dmap->itn.start;
968 dmap_start = start_idx << FUSE_DAX_SHIFT;
969 dmap_end = dmap_start + FUSE_DAX_SZ - 1;
970 }
971 up_read(&fi->dax->sem);
972
973 if (!dmap)
974 goto out_mmap_sem;
975
976
977
978
979 ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end);
980 if (ret) {
981 pr_debug("fuse: fuse_dax_break_layouts() failed. err=%d\n",
982 ret);
983 dmap = ERR_PTR(ret);
984 goto out_mmap_sem;
985 }
986
987 down_write(&fi->dax->sem);
988 node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
989
990 if (!node) {
991 if (retry)
992 *retry = true;
993 goto out_write_dmap_sem;
994 }
995
996 dmap = node_to_dmap(node);
997
998 if (refcount_read(&dmap->refcnt) > 1) {
999 dmap = NULL;
1000 if (retry)
1001 *retry = true;
1002 goto out_write_dmap_sem;
1003 }
1004
1005 ret = reclaim_one_dmap_locked(inode, dmap);
1006 if (ret < 0) {
1007 dmap = ERR_PTR(ret);
1008 goto out_write_dmap_sem;
1009 }
1010
1011
1012 dmap_remove_busy_list(fcd, dmap);
1013 dmap->inode = NULL;
1014 dmap->itn.start = dmap->itn.last = 0;
1015
1016 pr_debug("fuse: %s: inline reclaimed memory range. inode=%p, window_offset=0x%llx, length=0x%llx\n",
1017 __func__, inode, dmap->window_offset, dmap->length);
1018
1019out_write_dmap_sem:
1020 up_write(&fi->dax->sem);
1021out_mmap_sem:
1022 filemap_invalidate_unlock(inode->i_mapping);
1023 return dmap;
1024}
1025
1026static struct fuse_dax_mapping *
1027alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode)
1028{
1029 struct fuse_dax_mapping *dmap;
1030 struct fuse_inode *fi = get_fuse_inode(inode);
1031
1032 while (1) {
1033 bool retry = false;
1034
1035 dmap = alloc_dax_mapping(fcd);
1036 if (dmap)
1037 return dmap;
1038
1039 dmap = inode_inline_reclaim_one_dmap(fcd, inode, &retry);
1040
1041
1042
1043
1044 if (dmap)
1045 return dmap;
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056 if (retry)
1057 continue;
1058
1059
1060
1061
1062
1063
1064
1065 if (!fi->dax->nr && !(fcd->nr_free_ranges > 0)) {
1066 if (wait_event_killable_exclusive(fcd->range_waitq,
1067 (fcd->nr_free_ranges > 0))) {
1068 return ERR_PTR(-EINTR);
1069 }
1070 }
1071 }
1072}
1073
1074static int lookup_and_reclaim_dmap_locked(struct fuse_conn_dax *fcd,
1075 struct inode *inode,
1076 unsigned long start_idx)
1077{
1078 int ret;
1079 struct fuse_inode *fi = get_fuse_inode(inode);
1080 struct fuse_dax_mapping *dmap;
1081 struct interval_tree_node *node;
1082
1083
1084 node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
1085
1086
1087 if (!node)
1088 return 0;
1089 dmap = node_to_dmap(node);
1090
1091
1092 if (refcount_read(&dmap->refcnt) > 1)
1093 return 0;
1094
1095 ret = reclaim_one_dmap_locked(inode, dmap);
1096 if (ret < 0)
1097 return ret;
1098
1099
1100 spin_lock(&fcd->lock);
1101 dmap_reinit_add_to_free_pool(fcd, dmap);
1102 spin_unlock(&fcd->lock);
1103 return ret;
1104}
1105
1106
1107
1108
1109
1110
1111
1112
1113static int lookup_and_reclaim_dmap(struct fuse_conn_dax *fcd,
1114 struct inode *inode,
1115 unsigned long start_idx,
1116 unsigned long end_idx)
1117{
1118 int ret;
1119 struct fuse_inode *fi = get_fuse_inode(inode);
1120 loff_t dmap_start = start_idx << FUSE_DAX_SHIFT;
1121 loff_t dmap_end = (dmap_start + FUSE_DAX_SZ) - 1;
1122
1123 filemap_invalidate_lock(inode->i_mapping);
1124 ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end);
1125 if (ret) {
1126 pr_debug("virtio_fs: fuse_dax_break_layouts() failed. err=%d\n",
1127 ret);
1128 goto out_mmap_sem;
1129 }
1130
1131 down_write(&fi->dax->sem);
1132 ret = lookup_and_reclaim_dmap_locked(fcd, inode, start_idx);
1133 up_write(&fi->dax->sem);
1134out_mmap_sem:
1135 filemap_invalidate_unlock(inode->i_mapping);
1136 return ret;
1137}
1138
1139static int try_to_free_dmap_chunks(struct fuse_conn_dax *fcd,
1140 unsigned long nr_to_free)
1141{
1142 struct fuse_dax_mapping *dmap, *pos, *temp;
1143 int ret, nr_freed = 0;
1144 unsigned long start_idx = 0, end_idx = 0;
1145 struct inode *inode = NULL;
1146
1147
1148 while (1) {
1149 if (nr_freed >= nr_to_free)
1150 break;
1151
1152 dmap = NULL;
1153 spin_lock(&fcd->lock);
1154
1155 if (!fcd->nr_busy_ranges) {
1156 spin_unlock(&fcd->lock);
1157 return 0;
1158 }
1159
1160 list_for_each_entry_safe(pos, temp, &fcd->busy_ranges,
1161 busy_list) {
1162
1163 if (refcount_read(&pos->refcnt) > 1)
1164 continue;
1165
1166 inode = igrab(pos->inode);
1167
1168
1169
1170
1171
1172 if (!inode)
1173 continue;
1174
1175
1176
1177
1178
1179 dmap = pos;
1180 list_move_tail(&dmap->busy_list, &fcd->busy_ranges);
1181 start_idx = end_idx = dmap->itn.start;
1182 break;
1183 }
1184 spin_unlock(&fcd->lock);
1185 if (!dmap)
1186 return 0;
1187
1188 ret = lookup_and_reclaim_dmap(fcd, inode, start_idx, end_idx);
1189 iput(inode);
1190 if (ret)
1191 return ret;
1192 nr_freed++;
1193 }
1194 return 0;
1195}
1196
1197static void fuse_dax_free_mem_worker(struct work_struct *work)
1198{
1199 int ret;
1200 struct fuse_conn_dax *fcd = container_of(work, struct fuse_conn_dax,
1201 free_work.work);
1202 ret = try_to_free_dmap_chunks(fcd, FUSE_DAX_RECLAIM_CHUNK);
1203 if (ret) {
1204 pr_debug("fuse: try_to_free_dmap_chunks() failed with err=%d\n",
1205 ret);
1206 }
1207
1208
1209 kick_dmap_free_worker(fcd, 1);
1210}
1211
1212static void fuse_free_dax_mem_ranges(struct list_head *mem_list)
1213{
1214 struct fuse_dax_mapping *range, *temp;
1215
1216
1217 list_for_each_entry_safe(range, temp, mem_list, list) {
1218 list_del(&range->list);
1219 if (!list_empty(&range->busy_list))
1220 list_del(&range->busy_list);
1221 kfree(range);
1222 }
1223}
1224
1225void fuse_dax_conn_free(struct fuse_conn *fc)
1226{
1227 if (fc->dax) {
1228 fuse_free_dax_mem_ranges(&fc->dax->free_ranges);
1229 kfree(fc->dax);
1230 }
1231}
1232
1233static int fuse_dax_mem_range_init(struct fuse_conn_dax *fcd)
1234{
1235 long nr_pages, nr_ranges;
1236 struct fuse_dax_mapping *range;
1237 int ret, id;
1238 size_t dax_size = -1;
1239 unsigned long i;
1240
1241 init_waitqueue_head(&fcd->range_waitq);
1242 INIT_LIST_HEAD(&fcd->free_ranges);
1243 INIT_LIST_HEAD(&fcd->busy_ranges);
1244 INIT_DELAYED_WORK(&fcd->free_work, fuse_dax_free_mem_worker);
1245
1246 id = dax_read_lock();
1247 nr_pages = dax_direct_access(fcd->dev, 0, PHYS_PFN(dax_size), NULL,
1248 NULL);
1249 dax_read_unlock(id);
1250 if (nr_pages < 0) {
1251 pr_debug("dax_direct_access() returned %ld\n", nr_pages);
1252 return nr_pages;
1253 }
1254
1255 nr_ranges = nr_pages/FUSE_DAX_PAGES;
1256 pr_debug("%s: dax mapped %ld pages. nr_ranges=%ld\n",
1257 __func__, nr_pages, nr_ranges);
1258
1259 for (i = 0; i < nr_ranges; i++) {
1260 range = kzalloc(sizeof(struct fuse_dax_mapping), GFP_KERNEL);
1261 ret = -ENOMEM;
1262 if (!range)
1263 goto out_err;
1264
1265
1266
1267
1268
1269 range->window_offset = i * FUSE_DAX_SZ;
1270 range->length = FUSE_DAX_SZ;
1271 INIT_LIST_HEAD(&range->busy_list);
1272 refcount_set(&range->refcnt, 1);
1273 list_add_tail(&range->list, &fcd->free_ranges);
1274 }
1275
1276 fcd->nr_free_ranges = nr_ranges;
1277 fcd->nr_ranges = nr_ranges;
1278 return 0;
1279out_err:
1280
1281 fuse_free_dax_mem_ranges(&fcd->free_ranges);
1282 return ret;
1283}
1284
1285int fuse_dax_conn_alloc(struct fuse_conn *fc, struct dax_device *dax_dev)
1286{
1287 struct fuse_conn_dax *fcd;
1288 int err;
1289
1290 if (!dax_dev)
1291 return 0;
1292
1293 fcd = kzalloc(sizeof(*fcd), GFP_KERNEL);
1294 if (!fcd)
1295 return -ENOMEM;
1296
1297 spin_lock_init(&fcd->lock);
1298 fcd->dev = dax_dev;
1299 err = fuse_dax_mem_range_init(fcd);
1300 if (err) {
1301 kfree(fcd);
1302 return err;
1303 }
1304
1305 fc->dax = fcd;
1306 return 0;
1307}
1308
1309bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi)
1310{
1311 struct fuse_conn *fc = get_fuse_conn_super(sb);
1312
1313 fi->dax = NULL;
1314 if (fc->dax) {
1315 fi->dax = kzalloc(sizeof(*fi->dax), GFP_KERNEL_ACCOUNT);
1316 if (!fi->dax)
1317 return false;
1318
1319 init_rwsem(&fi->dax->sem);
1320 fi->dax->tree = RB_ROOT_CACHED;
1321 }
1322
1323 return true;
1324}
1325
1326static const struct address_space_operations fuse_dax_file_aops = {
1327 .writepages = fuse_dax_writepages,
1328 .direct_IO = noop_direct_IO,
1329 .set_page_dirty = __set_page_dirty_no_writeback,
1330 .invalidatepage = noop_invalidatepage,
1331};
1332
1333void fuse_dax_inode_init(struct inode *inode)
1334{
1335 struct fuse_conn *fc = get_fuse_conn(inode);
1336
1337 if (!fc->dax)
1338 return;
1339
1340 inode->i_flags |= S_DAX;
1341 inode->i_data.a_ops = &fuse_dax_file_aops;
1342}
1343
1344bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment)
1345{
1346 if (fc->dax && (map_alignment > FUSE_DAX_SHIFT)) {
1347 pr_warn("FUSE: map_alignment %u incompatible with dax mem range size %u\n",
1348 map_alignment, FUSE_DAX_SZ);
1349 return false;
1350 }
1351 return true;
1352}
1353
1354void fuse_dax_cancel_work(struct fuse_conn *fc)
1355{
1356 struct fuse_conn_dax *fcd = fc->dax;
1357
1358 if (fcd)
1359 cancel_delayed_work_sync(&fcd->free_work);
1360
1361}
1362EXPORT_SYMBOL_GPL(fuse_dax_cancel_work);
1363