1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/atomic.h>
18#include <linux/blkdev.h>
19#include <linux/buffer_head.h>
20#include <linux/dax.h>
21#include <linux/fs.h>
22#include <linux/genhd.h>
23#include <linux/highmem.h>
24#include <linux/memcontrol.h>
25#include <linux/mm.h>
26#include <linux/mutex.h>
27#include <linux/pagevec.h>
28#include <linux/sched.h>
29#include <linux/sched/signal.h>
30#include <linux/uio.h>
31#include <linux/vmstat.h>
32#include <linux/pfn_t.h>
33#include <linux/sizes.h>
34#include <linux/mmu_notifier.h>
35#include <linux/iomap.h>
36#include "internal.h"
37
38#define CREATE_TRACE_POINTS
39#include <trace/events/fs_dax.h>
40
41static inline unsigned int pe_order(enum page_entry_size pe_size)
42{
43 if (pe_size == PE_SIZE_PTE)
44 return PAGE_SHIFT - PAGE_SHIFT;
45 if (pe_size == PE_SIZE_PMD)
46 return PMD_SHIFT - PAGE_SHIFT;
47 if (pe_size == PE_SIZE_PUD)
48 return PUD_SHIFT - PAGE_SHIFT;
49 return ~0;
50}
51
52
53#define DAX_WAIT_TABLE_BITS 12
54#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)
55
56
57#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1)
58#define PG_PMD_NR (PMD_SIZE >> PAGE_SHIFT)
59
60
61#define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT)
62
63static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
64
65static int __init init_dax_wait_table(void)
66{
67 int i;
68
69 for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
70 init_waitqueue_head(wait_table + i);
71 return 0;
72}
73fs_initcall(init_dax_wait_table);
74
75
76
77
78
79
80
81
82
83
84
85#define DAX_SHIFT (4)
86#define DAX_LOCKED (1UL << 0)
87#define DAX_PMD (1UL << 1)
88#define DAX_ZERO_PAGE (1UL << 2)
89#define DAX_EMPTY (1UL << 3)
90
91static unsigned long dax_to_pfn(void *entry)
92{
93 return xa_to_value(entry) >> DAX_SHIFT;
94}
95
96static void *dax_make_entry(pfn_t pfn, unsigned long flags)
97{
98 return xa_mk_value(flags | (pfn_t_to_pfn(pfn) << DAX_SHIFT));
99}
100
101static bool dax_is_locked(void *entry)
102{
103 return xa_to_value(entry) & DAX_LOCKED;
104}
105
106static unsigned int dax_entry_order(void *entry)
107{
108 if (xa_to_value(entry) & DAX_PMD)
109 return PMD_ORDER;
110 return 0;
111}
112
113static unsigned long dax_is_pmd_entry(void *entry)
114{
115 return xa_to_value(entry) & DAX_PMD;
116}
117
118static bool dax_is_pte_entry(void *entry)
119{
120 return !(xa_to_value(entry) & DAX_PMD);
121}
122
123static int dax_is_zero_entry(void *entry)
124{
125 return xa_to_value(entry) & DAX_ZERO_PAGE;
126}
127
128static int dax_is_empty_entry(void *entry)
129{
130 return xa_to_value(entry) & DAX_EMPTY;
131}
132
133
134
135
136struct exceptional_entry_key {
137 struct xarray *xa;
138 pgoff_t entry_start;
139};
140
141struct wait_exceptional_entry_queue {
142 wait_queue_entry_t wait;
143 struct exceptional_entry_key key;
144};
145
146static wait_queue_head_t *dax_entry_waitqueue(struct xa_state *xas,
147 void *entry, struct exceptional_entry_key *key)
148{
149 unsigned long hash;
150 unsigned long index = xas->xa_index;
151
152
153
154
155
156
157 if (dax_is_pmd_entry(entry))
158 index &= ~PG_PMD_COLOUR;
159 key->xa = xas->xa;
160 key->entry_start = index;
161
162 hash = hash_long((unsigned long)xas->xa ^ index, DAX_WAIT_TABLE_BITS);
163 return wait_table + hash;
164}
165
166static int wake_exceptional_entry_func(wait_queue_entry_t *wait,
167 unsigned int mode, int sync, void *keyp)
168{
169 struct exceptional_entry_key *key = keyp;
170 struct wait_exceptional_entry_queue *ewait =
171 container_of(wait, struct wait_exceptional_entry_queue, wait);
172
173 if (key->xa != ewait->key.xa ||
174 key->entry_start != ewait->key.entry_start)
175 return 0;
176 return autoremove_wake_function(wait, mode, sync, NULL);
177}
178
179
180
181
182
183
184static void dax_wake_entry(struct xa_state *xas, void *entry, bool wake_all)
185{
186 struct exceptional_entry_key key;
187 wait_queue_head_t *wq;
188
189 wq = dax_entry_waitqueue(xas, entry, &key);
190
191
192
193
194
195
196
197 if (waitqueue_active(wq))
198 __wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
199}
200
201
202
203
204
205
206
207
208
209static void *get_unlocked_entry(struct xa_state *xas)
210{
211 void *entry;
212 struct wait_exceptional_entry_queue ewait;
213 wait_queue_head_t *wq;
214
215 init_wait(&ewait.wait);
216 ewait.wait.func = wake_exceptional_entry_func;
217
218 for (;;) {
219 entry = xas_find_conflict(xas);
220 if (!entry || WARN_ON_ONCE(!xa_is_value(entry)) ||
221 !dax_is_locked(entry))
222 return entry;
223
224 wq = dax_entry_waitqueue(xas, entry, &ewait.key);
225 prepare_to_wait_exclusive(wq, &ewait.wait,
226 TASK_UNINTERRUPTIBLE);
227 xas_unlock_irq(xas);
228 xas_reset(xas);
229 schedule();
230 finish_wait(wq, &ewait.wait);
231 xas_lock_irq(xas);
232 }
233}
234
235
236
237
238
239
240static void wait_entry_unlocked(struct xa_state *xas, void *entry)
241{
242 struct wait_exceptional_entry_queue ewait;
243 wait_queue_head_t *wq;
244
245 init_wait(&ewait.wait);
246 ewait.wait.func = wake_exceptional_entry_func;
247
248 wq = dax_entry_waitqueue(xas, entry, &ewait.key);
249
250
251
252
253
254
255 prepare_to_wait(wq, &ewait.wait, TASK_UNINTERRUPTIBLE);
256 xas_unlock_irq(xas);
257 schedule();
258 finish_wait(wq, &ewait.wait);
259}
260
261static void put_unlocked_entry(struct xa_state *xas, void *entry)
262{
263
264 if (entry)
265 dax_wake_entry(xas, entry, false);
266}
267
268
269
270
271
272
273static void dax_unlock_entry(struct xa_state *xas, void *entry)
274{
275 void *old;
276
277 BUG_ON(dax_is_locked(entry));
278 xas_reset(xas);
279 xas_lock_irq(xas);
280 old = xas_store(xas, entry);
281 xas_unlock_irq(xas);
282 BUG_ON(!dax_is_locked(old));
283 dax_wake_entry(xas, entry, false);
284}
285
286
287
288
289static void *dax_lock_entry(struct xa_state *xas, void *entry)
290{
291 unsigned long v = xa_to_value(entry);
292 return xas_store(xas, xa_mk_value(v | DAX_LOCKED));
293}
294
295static unsigned long dax_entry_size(void *entry)
296{
297 if (dax_is_zero_entry(entry))
298 return 0;
299 else if (dax_is_empty_entry(entry))
300 return 0;
301 else if (dax_is_pmd_entry(entry))
302 return PMD_SIZE;
303 else
304 return PAGE_SIZE;
305}
306
307static unsigned long dax_end_pfn(void *entry)
308{
309 return dax_to_pfn(entry) + dax_entry_size(entry) / PAGE_SIZE;
310}
311
312
313
314
315
316#define for_each_mapped_pfn(entry, pfn) \
317 for (pfn = dax_to_pfn(entry); \
318 pfn < dax_end_pfn(entry); pfn++)
319
320
321
322
323
324
325static void dax_associate_entry(void *entry, struct address_space *mapping,
326 struct vm_area_struct *vma, unsigned long address)
327{
328 unsigned long size = dax_entry_size(entry), pfn, index;
329 int i = 0;
330
331 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
332 return;
333
334 index = linear_page_index(vma, address & ~(size - 1));
335 for_each_mapped_pfn(entry, pfn) {
336 struct page *page = pfn_to_page(pfn);
337
338 WARN_ON_ONCE(page->mapping);
339 page->mapping = mapping;
340 page->index = index + i++;
341 }
342}
343
344static void dax_disassociate_entry(void *entry, struct address_space *mapping,
345 bool trunc)
346{
347 unsigned long pfn;
348
349 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
350 return;
351
352 for_each_mapped_pfn(entry, pfn) {
353 struct page *page = pfn_to_page(pfn);
354
355 WARN_ON_ONCE(trunc && page_ref_count(page) > 1);
356 WARN_ON_ONCE(page->mapping && page->mapping != mapping);
357 page->mapping = NULL;
358 page->index = 0;
359 }
360}
361
362static struct page *dax_busy_page(void *entry)
363{
364 unsigned long pfn;
365
366 for_each_mapped_pfn(entry, pfn) {
367 struct page *page = pfn_to_page(pfn);
368
369 if (page_ref_count(page) > 1)
370 return page;
371 }
372 return NULL;
373}
374
375
376
377
378
379
380
381
382
383dax_entry_t dax_lock_page(struct page *page)
384{
385 XA_STATE(xas, NULL, 0);
386 void *entry;
387
388
389 rcu_read_lock();
390 for (;;) {
391 struct address_space *mapping = READ_ONCE(page->mapping);
392
393 entry = NULL;
394 if (!mapping || !dax_mapping(mapping))
395 break;
396
397
398
399
400
401
402
403
404 entry = (void *)~0UL;
405 if (S_ISCHR(mapping->host->i_mode))
406 break;
407
408 xas.xa = &mapping->i_pages;
409 xas_lock_irq(&xas);
410 if (mapping != page->mapping) {
411 xas_unlock_irq(&xas);
412 continue;
413 }
414 xas_set(&xas, page->index);
415 entry = xas_load(&xas);
416 if (dax_is_locked(entry)) {
417 rcu_read_unlock();
418 wait_entry_unlocked(&xas, entry);
419 rcu_read_lock();
420 continue;
421 }
422 dax_lock_entry(&xas, entry);
423 xas_unlock_irq(&xas);
424 break;
425 }
426 rcu_read_unlock();
427 return (dax_entry_t)entry;
428}
429
430void dax_unlock_page(struct page *page, dax_entry_t cookie)
431{
432 struct address_space *mapping = page->mapping;
433 XA_STATE(xas, &mapping->i_pages, page->index);
434
435 if (S_ISCHR(mapping->host->i_mode))
436 return;
437
438 dax_unlock_entry(&xas, (void *)cookie);
439}
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470static void *grab_mapping_entry(struct xa_state *xas,
471 struct address_space *mapping, unsigned long size_flag)
472{
473 unsigned long index = xas->xa_index;
474 bool pmd_downgrade = false;
475 void *entry;
476
477retry:
478 xas_lock_irq(xas);
479 entry = get_unlocked_entry(xas);
480
481 if (entry) {
482 if (!xa_is_value(entry)) {
483 xas_set_err(xas, EIO);
484 goto out_unlock;
485 }
486
487 if (size_flag & DAX_PMD) {
488 if (dax_is_pte_entry(entry)) {
489 put_unlocked_entry(xas, entry);
490 goto fallback;
491 }
492 } else {
493 if (dax_is_pmd_entry(entry) &&
494 (dax_is_zero_entry(entry) ||
495 dax_is_empty_entry(entry))) {
496 pmd_downgrade = true;
497 }
498 }
499 }
500
501 if (pmd_downgrade) {
502
503
504
505
506 dax_lock_entry(xas, entry);
507
508
509
510
511
512
513 if (dax_is_zero_entry(entry)) {
514 xas_unlock_irq(xas);
515 unmap_mapping_pages(mapping,
516 xas->xa_index & ~PG_PMD_COLOUR,
517 PG_PMD_NR, false);
518 xas_reset(xas);
519 xas_lock_irq(xas);
520 }
521
522 dax_disassociate_entry(entry, mapping, false);
523 xas_store(xas, NULL);
524 dax_wake_entry(xas, entry, true);
525 mapping->nrexceptional--;
526 entry = NULL;
527 xas_set(xas, index);
528 }
529
530 if (entry) {
531 dax_lock_entry(xas, entry);
532 } else {
533 entry = dax_make_entry(pfn_to_pfn_t(0), size_flag | DAX_EMPTY);
534 dax_lock_entry(xas, entry);
535 if (xas_error(xas))
536 goto out_unlock;
537 mapping->nrexceptional++;
538 }
539
540out_unlock:
541 xas_unlock_irq(xas);
542 if (xas_nomem(xas, mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM))
543 goto retry;
544 if (xas->xa_node == XA_ERROR(-ENOMEM))
545 return xa_mk_internal(VM_FAULT_OOM);
546 if (xas_error(xas))
547 return xa_mk_internal(VM_FAULT_SIGBUS);
548 return entry;
549fallback:
550 xas_unlock_irq(xas);
551 return xa_mk_internal(VM_FAULT_FALLBACK);
552}
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569struct page *dax_layout_busy_page(struct address_space *mapping)
570{
571 XA_STATE(xas, &mapping->i_pages, 0);
572 void *entry;
573 unsigned int scanned = 0;
574 struct page *page = NULL;
575
576
577
578
579 if (IS_ENABLED(CONFIG_FS_DAX_LIMITED))
580 return NULL;
581
582 if (!dax_mapping(mapping) || !mapping_mapped(mapping))
583 return NULL;
584
585
586
587
588
589
590
591
592
593
594
595
596
597 unmap_mapping_range(mapping, 0, 0, 1);
598
599 xas_lock_irq(&xas);
600 xas_for_each(&xas, entry, ULONG_MAX) {
601 if (WARN_ON_ONCE(!xa_is_value(entry)))
602 continue;
603 if (unlikely(dax_is_locked(entry)))
604 entry = get_unlocked_entry(&xas);
605 if (entry)
606 page = dax_busy_page(entry);
607 put_unlocked_entry(&xas, entry);
608 if (page)
609 break;
610 if (++scanned % XA_CHECK_SCHED)
611 continue;
612
613 xas_pause(&xas);
614 xas_unlock_irq(&xas);
615 cond_resched();
616 xas_lock_irq(&xas);
617 }
618 xas_unlock_irq(&xas);
619 return page;
620}
621EXPORT_SYMBOL_GPL(dax_layout_busy_page);
622
623static int __dax_invalidate_entry(struct address_space *mapping,
624 pgoff_t index, bool trunc)
625{
626 XA_STATE(xas, &mapping->i_pages, index);
627 int ret = 0;
628 void *entry;
629
630 xas_lock_irq(&xas);
631 entry = get_unlocked_entry(&xas);
632 if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
633 goto out;
634 if (!trunc &&
635 (xas_get_mark(&xas, PAGECACHE_TAG_DIRTY) ||
636 xas_get_mark(&xas, PAGECACHE_TAG_TOWRITE)))
637 goto out;
638 dax_disassociate_entry(entry, mapping, trunc);
639 xas_store(&xas, NULL);
640 mapping->nrexceptional--;
641 ret = 1;
642out:
643 put_unlocked_entry(&xas, entry);
644 xas_unlock_irq(&xas);
645 return ret;
646}
647
648
649
650
651
652int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
653{
654 int ret = __dax_invalidate_entry(mapping, index, true);
655
656
657
658
659
660
661
662
663 WARN_ON_ONCE(!ret);
664 return ret;
665}
666
667
668
669
670int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
671 pgoff_t index)
672{
673 return __dax_invalidate_entry(mapping, index, false);
674}
675
676static int copy_user_dax(struct block_device *bdev, struct dax_device *dax_dev,
677 sector_t sector, size_t size, struct page *to,
678 unsigned long vaddr)
679{
680 void *vto, *kaddr;
681 pgoff_t pgoff;
682 long rc;
683 int id;
684
685 rc = bdev_dax_pgoff(bdev, sector, size, &pgoff);
686 if (rc)
687 return rc;
688
689 id = dax_read_lock();
690 rc = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size), &kaddr, NULL);
691 if (rc < 0) {
692 dax_read_unlock(id);
693 return rc;
694 }
695 vto = kmap_atomic(to);
696 copy_user_page(vto, (void __force *)kaddr, vaddr, to);
697 kunmap_atomic(vto);
698 dax_read_unlock(id);
699 return 0;
700}
701
702
703
704
705
706
707
708
709static void *dax_insert_entry(struct xa_state *xas,
710 struct address_space *mapping, struct vm_fault *vmf,
711 void *entry, pfn_t pfn, unsigned long flags, bool dirty)
712{
713 void *new_entry = dax_make_entry(pfn, flags);
714
715 if (dirty)
716 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
717
718 if (dax_is_zero_entry(entry) && !(flags & DAX_ZERO_PAGE)) {
719 unsigned long index = xas->xa_index;
720
721 if (dax_is_pmd_entry(entry))
722 unmap_mapping_pages(mapping, index & ~PG_PMD_COLOUR,
723 PG_PMD_NR, false);
724 else
725 unmap_mapping_pages(mapping, index, 1, false);
726 }
727
728 xas_reset(xas);
729 xas_lock_irq(xas);
730 if (dax_entry_size(entry) != dax_entry_size(new_entry)) {
731 dax_disassociate_entry(entry, mapping, false);
732 dax_associate_entry(new_entry, mapping, vmf->vma, vmf->address);
733 }
734
735 if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
736
737
738
739
740
741
742
743
744 void *old = dax_lock_entry(xas, new_entry);
745 WARN_ON_ONCE(old != xa_mk_value(xa_to_value(entry) |
746 DAX_LOCKED));
747 entry = new_entry;
748 } else {
749 xas_load(xas);
750 }
751
752 if (dirty)
753 xas_set_mark(xas, PAGECACHE_TAG_DIRTY);
754
755 xas_unlock_irq(xas);
756 return entry;
757}
758
759static inline
760unsigned long pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
761{
762 unsigned long address;
763
764 address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
765 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
766 return address;
767}
768
769
770static void dax_entry_mkclean(struct address_space *mapping, pgoff_t index,
771 unsigned long pfn)
772{
773 struct vm_area_struct *vma;
774 pte_t pte, *ptep = NULL;
775 pmd_t *pmdp = NULL;
776 spinlock_t *ptl;
777
778 i_mmap_lock_read(mapping);
779 vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
780 struct mmu_notifier_range range;
781 unsigned long address;
782
783 cond_resched();
784
785 if (!(vma->vm_flags & VM_SHARED))
786 continue;
787
788 address = pgoff_address(index, vma);
789
790
791
792
793
794
795 if (follow_pte_pmd(vma->vm_mm, address, &range,
796 &ptep, &pmdp, &ptl))
797 continue;
798
799
800
801
802
803
804
805
806 if (pmdp) {
807#ifdef CONFIG_FS_DAX_PMD
808 pmd_t pmd;
809
810 if (pfn != pmd_pfn(*pmdp))
811 goto unlock_pmd;
812 if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp))
813 goto unlock_pmd;
814
815 flush_cache_page(vma, address, pfn);
816 pmd = pmdp_huge_clear_flush(vma, address, pmdp);
817 pmd = pmd_wrprotect(pmd);
818 pmd = pmd_mkclean(pmd);
819 set_pmd_at(vma->vm_mm, address, pmdp, pmd);
820unlock_pmd:
821#endif
822 spin_unlock(ptl);
823 } else {
824 if (pfn != pte_pfn(*ptep))
825 goto unlock_pte;
826 if (!pte_dirty(*ptep) && !pte_write(*ptep))
827 goto unlock_pte;
828
829 flush_cache_page(vma, address, pfn);
830 pte = ptep_clear_flush(vma, address, ptep);
831 pte = pte_wrprotect(pte);
832 pte = pte_mkclean(pte);
833 set_pte_at(vma->vm_mm, address, ptep, pte);
834unlock_pte:
835 pte_unmap_unlock(ptep, ptl);
836 }
837
838 mmu_notifier_invalidate_range_end(&range);
839 }
840 i_mmap_unlock_read(mapping);
841}
842
843static int dax_writeback_one(struct xa_state *xas, struct dax_device *dax_dev,
844 struct address_space *mapping, void *entry)
845{
846 unsigned long pfn;
847 long ret = 0;
848 size_t size;
849
850
851
852
853
854 if (WARN_ON(!xa_is_value(entry)))
855 return -EIO;
856
857 if (unlikely(dax_is_locked(entry))) {
858 void *old_entry = entry;
859
860 entry = get_unlocked_entry(xas);
861
862
863 if (!entry || WARN_ON_ONCE(!xa_is_value(entry)))
864 goto put_unlocked;
865
866
867
868
869
870 if (dax_to_pfn(old_entry) != dax_to_pfn(entry))
871 goto put_unlocked;
872 if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
873 dax_is_zero_entry(entry))) {
874 ret = -EIO;
875 goto put_unlocked;
876 }
877
878
879 if (!xas_get_mark(xas, PAGECACHE_TAG_TOWRITE))
880 goto put_unlocked;
881 }
882
883
884 dax_lock_entry(xas, entry);
885
886
887
888
889
890
891
892
893 xas_clear_mark(xas, PAGECACHE_TAG_TOWRITE);
894 xas_unlock_irq(xas);
895
896
897
898
899
900
901
902
903 pfn = dax_to_pfn(entry);
904 size = PAGE_SIZE << dax_entry_order(entry);
905
906 dax_entry_mkclean(mapping, xas->xa_index, pfn);
907 dax_flush(dax_dev, page_address(pfn_to_page(pfn)), size);
908
909
910
911
912
913
914 xas_reset(xas);
915 xas_lock_irq(xas);
916 xas_store(xas, entry);
917 xas_clear_mark(xas, PAGECACHE_TAG_DIRTY);
918 dax_wake_entry(xas, entry, false);
919
920 trace_dax_writeback_one(mapping->host, xas->xa_index,
921 size >> PAGE_SHIFT);
922 return ret;
923
924 put_unlocked:
925 put_unlocked_entry(xas, entry);
926 return ret;
927}
928
929
930
931
932
933
934int dax_writeback_mapping_range(struct address_space *mapping,
935 struct block_device *bdev, struct writeback_control *wbc)
936{
937 XA_STATE(xas, &mapping->i_pages, wbc->range_start >> PAGE_SHIFT);
938 struct inode *inode = mapping->host;
939 pgoff_t end_index = wbc->range_end >> PAGE_SHIFT;
940 struct dax_device *dax_dev;
941 void *entry;
942 int ret = 0;
943 unsigned int scanned = 0;
944
945 if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
946 return -EIO;
947
948 if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
949 return 0;
950
951 dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
952 if (!dax_dev)
953 return -EIO;
954
955 trace_dax_writeback_range(inode, xas.xa_index, end_index);
956
957 tag_pages_for_writeback(mapping, xas.xa_index, end_index);
958
959 xas_lock_irq(&xas);
960 xas_for_each_marked(&xas, entry, end_index, PAGECACHE_TAG_TOWRITE) {
961 ret = dax_writeback_one(&xas, dax_dev, mapping, entry);
962 if (ret < 0) {
963 mapping_set_error(mapping, ret);
964 break;
965 }
966 if (++scanned % XA_CHECK_SCHED)
967 continue;
968
969 xas_pause(&xas);
970 xas_unlock_irq(&xas);
971 cond_resched();
972 xas_lock_irq(&xas);
973 }
974 xas_unlock_irq(&xas);
975 put_dax(dax_dev);
976 trace_dax_writeback_range_done(inode, xas.xa_index, end_index);
977 return ret;
978}
979EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);
980
981static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
982{
983 return (iomap->addr + (pos & PAGE_MASK) - iomap->offset) >> 9;
984}
985
986static int dax_iomap_pfn(struct iomap *iomap, loff_t pos, size_t size,
987 pfn_t *pfnp)
988{
989 const sector_t sector = dax_iomap_sector(iomap, pos);
990 pgoff_t pgoff;
991 int id, rc;
992 long length;
993
994 rc = bdev_dax_pgoff(iomap->bdev, sector, size, &pgoff);
995 if (rc)
996 return rc;
997 id = dax_read_lock();
998 length = dax_direct_access(iomap->dax_dev, pgoff, PHYS_PFN(size),
999 NULL, pfnp);
1000 if (length < 0) {
1001 rc = length;
1002 goto out;
1003 }
1004 rc = -EINVAL;
1005 if (PFN_PHYS(length) < size)
1006 goto out;
1007 if (pfn_t_to_pfn(*pfnp) & (PHYS_PFN(size)-1))
1008 goto out;
1009
1010 if (length > 1 && !pfn_t_devmap(*pfnp))
1011 goto out;
1012 rc = 0;
1013out:
1014 dax_read_unlock(id);
1015 return rc;
1016}
1017
1018
1019
1020
1021
1022
1023
1024
1025static vm_fault_t dax_load_hole(struct xa_state *xas,
1026 struct address_space *mapping, void **entry,
1027 struct vm_fault *vmf)
1028{
1029 struct inode *inode = mapping->host;
1030 unsigned long vaddr = vmf->address;
1031 pfn_t pfn = pfn_to_pfn_t(my_zero_pfn(vaddr));
1032 vm_fault_t ret;
1033
1034 *entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,
1035 DAX_ZERO_PAGE, false);
1036
1037 ret = vmf_insert_mixed(vmf->vma, vaddr, pfn);
1038 trace_dax_load_hole(inode, vmf, ret);
1039 return ret;
1040}
1041
1042static bool dax_range_is_aligned(struct block_device *bdev,
1043 unsigned int offset, unsigned int length)
1044{
1045 unsigned short sector_size = bdev_logical_block_size(bdev);
1046
1047 if (!IS_ALIGNED(offset, sector_size))
1048 return false;
1049 if (!IS_ALIGNED(length, sector_size))
1050 return false;
1051
1052 return true;
1053}
1054
1055int __dax_zero_page_range(struct block_device *bdev,
1056 struct dax_device *dax_dev, sector_t sector,
1057 unsigned int offset, unsigned int size)
1058{
1059 if (dax_range_is_aligned(bdev, offset, size)) {
1060 sector_t start_sector = sector + (offset >> 9);
1061
1062 return blkdev_issue_zeroout(bdev, start_sector,
1063 size >> 9, GFP_NOFS, 0);
1064 } else {
1065 pgoff_t pgoff;
1066 long rc, id;
1067 void *kaddr;
1068
1069 rc = bdev_dax_pgoff(bdev, sector, PAGE_SIZE, &pgoff);
1070 if (rc)
1071 return rc;
1072
1073 id = dax_read_lock();
1074 rc = dax_direct_access(dax_dev, pgoff, 1, &kaddr, NULL);
1075 if (rc < 0) {
1076 dax_read_unlock(id);
1077 return rc;
1078 }
1079 memset(kaddr + offset, 0, size);
1080 dax_flush(dax_dev, kaddr + offset, size);
1081 dax_read_unlock(id);
1082 }
1083 return 0;
1084}
1085EXPORT_SYMBOL_GPL(__dax_zero_page_range);
1086
1087static loff_t
1088dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
1089 struct iomap *iomap)
1090{
1091 struct block_device *bdev = iomap->bdev;
1092 struct dax_device *dax_dev = iomap->dax_dev;
1093 struct iov_iter *iter = data;
1094 loff_t end = pos + length, done = 0;
1095 ssize_t ret = 0;
1096 size_t xfer;
1097 int id;
1098
1099 if (iov_iter_rw(iter) == READ) {
1100 end = min(end, i_size_read(inode));
1101 if (pos >= end)
1102 return 0;
1103
1104 if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
1105 return iov_iter_zero(min(length, end - pos), iter);
1106 }
1107
1108 if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
1109 return -EIO;
1110
1111
1112
1113
1114
1115
1116 if (iomap->flags & IOMAP_F_NEW) {
1117 invalidate_inode_pages2_range(inode->i_mapping,
1118 pos >> PAGE_SHIFT,
1119 (end - 1) >> PAGE_SHIFT);
1120 }
1121
1122 id = dax_read_lock();
1123 while (pos < end) {
1124 unsigned offset = pos & (PAGE_SIZE - 1);
1125 const size_t size = ALIGN(length + offset, PAGE_SIZE);
1126 const sector_t sector = dax_iomap_sector(iomap, pos);
1127 ssize_t map_len;
1128 pgoff_t pgoff;
1129 void *kaddr;
1130
1131 if (fatal_signal_pending(current)) {
1132 ret = -EINTR;
1133 break;
1134 }
1135
1136 ret = bdev_dax_pgoff(bdev, sector, size, &pgoff);
1137 if (ret)
1138 break;
1139
1140 map_len = dax_direct_access(dax_dev, pgoff, PHYS_PFN(size),
1141 &kaddr, NULL);
1142 if (map_len < 0) {
1143 ret = map_len;
1144 break;
1145 }
1146
1147 map_len = PFN_PHYS(map_len);
1148 kaddr += offset;
1149 map_len -= offset;
1150 if (map_len > end - pos)
1151 map_len = end - pos;
1152
1153
1154
1155
1156
1157
1158 if (iov_iter_rw(iter) == WRITE)
1159 xfer = dax_copy_from_iter(dax_dev, pgoff, kaddr,
1160 map_len, iter);
1161 else
1162 xfer = dax_copy_to_iter(dax_dev, pgoff, kaddr,
1163 map_len, iter);
1164
1165 pos += xfer;
1166 length -= xfer;
1167 done += xfer;
1168
1169 if (xfer == 0)
1170 ret = -EFAULT;
1171 if (xfer < map_len)
1172 break;
1173 }
1174 dax_read_unlock(id);
1175
1176 return done ? done : ret;
1177}
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189ssize_t
1190dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
1191 const struct iomap_ops *ops)
1192{
1193 struct address_space *mapping = iocb->ki_filp->f_mapping;
1194 struct inode *inode = mapping->host;
1195 loff_t pos = iocb->ki_pos, ret = 0, done = 0;
1196 unsigned flags = 0;
1197
1198 if (iov_iter_rw(iter) == WRITE) {
1199 lockdep_assert_held_exclusive(&inode->i_rwsem);
1200 flags |= IOMAP_WRITE;
1201 } else {
1202 lockdep_assert_held(&inode->i_rwsem);
1203 }
1204
1205 while (iov_iter_count(iter)) {
1206 ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
1207 iter, dax_iomap_actor);
1208 if (ret <= 0)
1209 break;
1210 pos += ret;
1211 done += ret;
1212 }
1213
1214 iocb->ki_pos += done;
1215 return done ? done : ret;
1216}
1217EXPORT_SYMBOL_GPL(dax_iomap_rw);
1218
1219static vm_fault_t dax_fault_return(int error)
1220{
1221 if (error == 0)
1222 return VM_FAULT_NOPAGE;
1223 if (error == -ENOMEM)
1224 return VM_FAULT_OOM;
1225 return VM_FAULT_SIGBUS;
1226}
1227
1228
1229
1230
1231
1232static bool dax_fault_is_synchronous(unsigned long flags,
1233 struct vm_area_struct *vma, struct iomap *iomap)
1234{
1235 return (flags & IOMAP_WRITE) && (vma->vm_flags & VM_SYNC)
1236 && (iomap->flags & IOMAP_F_DIRTY);
1237}
1238
1239static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, pfn_t *pfnp,
1240 int *iomap_errp, const struct iomap_ops *ops)
1241{
1242 struct vm_area_struct *vma = vmf->vma;
1243 struct address_space *mapping = vma->vm_file->f_mapping;
1244 XA_STATE(xas, &mapping->i_pages, vmf->pgoff);
1245 struct inode *inode = mapping->host;
1246 unsigned long vaddr = vmf->address;
1247 loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
1248 struct iomap iomap = { 0 };
1249 unsigned flags = IOMAP_FAULT;
1250 int error, major = 0;
1251 bool write = vmf->flags & FAULT_FLAG_WRITE;
1252 bool sync;
1253 vm_fault_t ret = 0;
1254 void *entry;
1255 pfn_t pfn;
1256
1257 trace_dax_pte_fault(inode, vmf, ret);
1258
1259
1260
1261
1262
1263 if (pos >= i_size_read(inode)) {
1264 ret = VM_FAULT_SIGBUS;
1265 goto out;
1266 }
1267
1268 if (write && !vmf->cow_page)
1269 flags |= IOMAP_WRITE;
1270
1271 entry = grab_mapping_entry(&xas, mapping, 0);
1272 if (xa_is_internal(entry)) {
1273 ret = xa_to_internal(entry);
1274 goto out;
1275 }
1276
1277
1278
1279
1280
1281
1282
1283 if (pmd_trans_huge(*vmf->pmd) || pmd_devmap(*vmf->pmd)) {
1284 ret = VM_FAULT_NOPAGE;
1285 goto unlock_entry;
1286 }
1287
1288
1289
1290
1291
1292
1293 error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap);
1294 if (iomap_errp)
1295 *iomap_errp = error;
1296 if (error) {
1297 ret = dax_fault_return(error);
1298 goto unlock_entry;
1299 }
1300 if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
1301 error = -EIO;
1302 goto error_finish_iomap;
1303 }
1304
1305 if (vmf->cow_page) {
1306 sector_t sector = dax_iomap_sector(&iomap, pos);
1307
1308 switch (iomap.type) {
1309 case IOMAP_HOLE:
1310 case IOMAP_UNWRITTEN:
1311 clear_user_highpage(vmf->cow_page, vaddr);
1312 break;
1313 case IOMAP_MAPPED:
1314 error = copy_user_dax(iomap.bdev, iomap.dax_dev,
1315 sector, PAGE_SIZE, vmf->cow_page, vaddr);
1316 break;
1317 default:
1318 WARN_ON_ONCE(1);
1319 error = -EIO;
1320 break;
1321 }
1322
1323 if (error)
1324 goto error_finish_iomap;
1325
1326 __SetPageUptodate(vmf->cow_page);
1327 ret = finish_fault(vmf);
1328 if (!ret)
1329 ret = VM_FAULT_DONE_COW;
1330 goto finish_iomap;
1331 }
1332
1333 sync = dax_fault_is_synchronous(flags, vma, &iomap);
1334
1335 switch (iomap.type) {
1336 case IOMAP_MAPPED:
1337 if (iomap.flags & IOMAP_F_NEW) {
1338 count_vm_event(PGMAJFAULT);
1339 count_memcg_event_mm(vma->vm_mm, PGMAJFAULT);
1340 major = VM_FAULT_MAJOR;
1341 }
1342 error = dax_iomap_pfn(&iomap, pos, PAGE_SIZE, &pfn);
1343 if (error < 0)
1344 goto error_finish_iomap;
1345
1346 entry = dax_insert_entry(&xas, mapping, vmf, entry, pfn,
1347 0, write && !sync);
1348
1349
1350
1351
1352
1353
1354
1355 if (sync) {
1356 if (WARN_ON_ONCE(!pfnp)) {
1357 error = -EIO;
1358 goto error_finish_iomap;
1359 }
1360 *pfnp = pfn;
1361 ret = VM_FAULT_NEEDDSYNC | major;
1362 goto finish_iomap;
1363 }
1364 trace_dax_insert_mapping(inode, vmf, entry);
1365 if (write)
1366 ret = vmf_insert_mixed_mkwrite(vma, vaddr, pfn);
1367 else
1368 ret = vmf_insert_mixed(vma, vaddr, pfn);
1369
1370 goto finish_iomap;
1371 case IOMAP_UNWRITTEN:
1372 case IOMAP_HOLE:
1373 if (!write) {
1374 ret = dax_load_hole(&xas, mapping, &entry, vmf);
1375 goto finish_iomap;
1376 }
1377
1378 default:
1379 WARN_ON_ONCE(1);
1380 error = -EIO;
1381 break;
1382 }
1383
1384 error_finish_iomap:
1385 ret = dax_fault_return(error);
1386 finish_iomap:
1387 if (ops->iomap_end) {
1388 int copied = PAGE_SIZE;
1389
1390 if (ret & VM_FAULT_ERROR)
1391 copied = 0;
1392
1393
1394
1395
1396
1397
1398 ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap);
1399 }
1400 unlock_entry:
1401 dax_unlock_entry(&xas, entry);
1402 out:
1403 trace_dax_pte_fault_done(inode, vmf, ret);
1404 return ret | major;
1405}
1406
1407#ifdef CONFIG_FS_DAX_PMD
1408static vm_fault_t dax_pmd_load_hole(struct xa_state *xas, struct vm_fault *vmf,
1409 struct iomap *iomap, void **entry)
1410{
1411 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1412 unsigned long pmd_addr = vmf->address & PMD_MASK;
1413 struct inode *inode = mapping->host;
1414 struct page *zero_page;
1415 spinlock_t *ptl;
1416 pmd_t pmd_entry;
1417 pfn_t pfn;
1418
1419 zero_page = mm_get_huge_zero_page(vmf->vma->vm_mm);
1420
1421 if (unlikely(!zero_page))
1422 goto fallback;
1423
1424 pfn = page_to_pfn_t(zero_page);
1425 *entry = dax_insert_entry(xas, mapping, vmf, *entry, pfn,
1426 DAX_PMD | DAX_ZERO_PAGE, false);
1427
1428 ptl = pmd_lock(vmf->vma->vm_mm, vmf->pmd);
1429 if (!pmd_none(*(vmf->pmd))) {
1430 spin_unlock(ptl);
1431 goto fallback;
1432 }
1433
1434 pmd_entry = mk_pmd(zero_page, vmf->vma->vm_page_prot);
1435 pmd_entry = pmd_mkhuge(pmd_entry);
1436 set_pmd_at(vmf->vma->vm_mm, pmd_addr, vmf->pmd, pmd_entry);
1437 spin_unlock(ptl);
1438 trace_dax_pmd_load_hole(inode, vmf, zero_page, *entry);
1439 return VM_FAULT_NOPAGE;
1440
1441fallback:
1442 trace_dax_pmd_load_hole_fallback(inode, vmf, zero_page, *entry);
1443 return VM_FAULT_FALLBACK;
1444}
1445
1446static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
1447 const struct iomap_ops *ops)
1448{
1449 struct vm_area_struct *vma = vmf->vma;
1450 struct address_space *mapping = vma->vm_file->f_mapping;
1451 XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, PMD_ORDER);
1452 unsigned long pmd_addr = vmf->address & PMD_MASK;
1453 bool write = vmf->flags & FAULT_FLAG_WRITE;
1454 bool sync;
1455 unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;
1456 struct inode *inode = mapping->host;
1457 vm_fault_t result = VM_FAULT_FALLBACK;
1458 struct iomap iomap = { 0 };
1459 pgoff_t max_pgoff;
1460 void *entry;
1461 loff_t pos;
1462 int error;
1463 pfn_t pfn;
1464
1465
1466
1467
1468
1469
1470 max_pgoff = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
1471
1472 trace_dax_pmd_fault(inode, vmf, max_pgoff, 0);
1473
1474
1475
1476
1477
1478
1479
1480 if ((vmf->pgoff & PG_PMD_COLOUR) !=
1481 ((vmf->address >> PAGE_SHIFT) & PG_PMD_COLOUR))
1482 goto fallback;
1483
1484
1485 if (write && !(vma->vm_flags & VM_SHARED))
1486 goto fallback;
1487
1488
1489 if (pmd_addr < vma->vm_start)
1490 goto fallback;
1491 if ((pmd_addr + PMD_SIZE) > vma->vm_end)
1492 goto fallback;
1493
1494 if (xas.xa_index >= max_pgoff) {
1495 result = VM_FAULT_SIGBUS;
1496 goto out;
1497 }
1498
1499
1500 if ((xas.xa_index | PG_PMD_COLOUR) >= max_pgoff)
1501 goto fallback;
1502
1503
1504
1505
1506
1507
1508
1509 entry = grab_mapping_entry(&xas, mapping, DAX_PMD);
1510 if (xa_is_internal(entry)) {
1511 result = xa_to_internal(entry);
1512 goto fallback;
1513 }
1514
1515
1516
1517
1518
1519
1520
1521 if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd) &&
1522 !pmd_devmap(*vmf->pmd)) {
1523 result = 0;
1524 goto unlock_entry;
1525 }
1526
1527
1528
1529
1530
1531
1532 pos = (loff_t)xas.xa_index << PAGE_SHIFT;
1533 error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap);
1534 if (error)
1535 goto unlock_entry;
1536
1537 if (iomap.offset + iomap.length < pos + PMD_SIZE)
1538 goto finish_iomap;
1539
1540 sync = dax_fault_is_synchronous(iomap_flags, vma, &iomap);
1541
1542 switch (iomap.type) {
1543 case IOMAP_MAPPED:
1544 error = dax_iomap_pfn(&iomap, pos, PMD_SIZE, &pfn);
1545 if (error < 0)
1546 goto finish_iomap;
1547
1548 entry = dax_insert_entry(&xas, mapping, vmf, entry, pfn,
1549 DAX_PMD, write && !sync);
1550
1551
1552
1553
1554
1555
1556
1557 if (sync) {
1558 if (WARN_ON_ONCE(!pfnp))
1559 goto finish_iomap;
1560 *pfnp = pfn;
1561 result = VM_FAULT_NEEDDSYNC;
1562 goto finish_iomap;
1563 }
1564
1565 trace_dax_pmd_insert_mapping(inode, vmf, PMD_SIZE, pfn, entry);
1566 result = vmf_insert_pfn_pmd(vma, vmf->address, vmf->pmd, pfn,
1567 write);
1568 break;
1569 case IOMAP_UNWRITTEN:
1570 case IOMAP_HOLE:
1571 if (WARN_ON_ONCE(write))
1572 break;
1573 result = dax_pmd_load_hole(&xas, vmf, &iomap, &entry);
1574 break;
1575 default:
1576 WARN_ON_ONCE(1);
1577 break;
1578 }
1579
1580 finish_iomap:
1581 if (ops->iomap_end) {
1582 int copied = PMD_SIZE;
1583
1584 if (result == VM_FAULT_FALLBACK)
1585 copied = 0;
1586
1587
1588
1589
1590
1591
1592 ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags,
1593 &iomap);
1594 }
1595 unlock_entry:
1596 dax_unlock_entry(&xas, entry);
1597 fallback:
1598 if (result == VM_FAULT_FALLBACK) {
1599 split_huge_pmd(vma, vmf->pmd, vmf->address);
1600 count_vm_event(THP_FAULT_FALLBACK);
1601 }
1602out:
1603 trace_dax_pmd_fault_done(inode, vmf, max_pgoff, result);
1604 return result;
1605}
1606#else
1607static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, pfn_t *pfnp,
1608 const struct iomap_ops *ops)
1609{
1610 return VM_FAULT_FALLBACK;
1611}
1612#endif
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627vm_fault_t dax_iomap_fault(struct vm_fault *vmf, enum page_entry_size pe_size,
1628 pfn_t *pfnp, int *iomap_errp, const struct iomap_ops *ops)
1629{
1630 switch (pe_size) {
1631 case PE_SIZE_PTE:
1632 return dax_iomap_pte_fault(vmf, pfnp, iomap_errp, ops);
1633 case PE_SIZE_PMD:
1634 return dax_iomap_pmd_fault(vmf, pfnp, ops);
1635 default:
1636 return VM_FAULT_FALLBACK;
1637 }
1638}
1639EXPORT_SYMBOL_GPL(dax_iomap_fault);
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650static vm_fault_t
1651dax_insert_pfn_mkwrite(struct vm_fault *vmf, pfn_t pfn, unsigned int order)
1652{
1653 struct address_space *mapping = vmf->vma->vm_file->f_mapping;
1654 XA_STATE_ORDER(xas, &mapping->i_pages, vmf->pgoff, order);
1655 void *entry;
1656 vm_fault_t ret;
1657
1658 xas_lock_irq(&xas);
1659 entry = get_unlocked_entry(&xas);
1660
1661 if (!entry ||
1662 (order == 0 && !dax_is_pte_entry(entry)) ||
1663 (order == PMD_ORDER && !dax_is_pmd_entry(entry))) {
1664 put_unlocked_entry(&xas, entry);
1665 xas_unlock_irq(&xas);
1666 trace_dax_insert_pfn_mkwrite_no_entry(mapping->host, vmf,
1667 VM_FAULT_NOPAGE);
1668 return VM_FAULT_NOPAGE;
1669 }
1670 xas_set_mark(&xas, PAGECACHE_TAG_DIRTY);
1671 dax_lock_entry(&xas, entry);
1672 xas_unlock_irq(&xas);
1673 if (order == 0)
1674 ret = vmf_insert_mixed_mkwrite(vmf->vma, vmf->address, pfn);
1675#ifdef CONFIG_FS_DAX_PMD
1676 else if (order == PMD_ORDER)
1677 ret = vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd,
1678 pfn, true);
1679#endif
1680 else
1681 ret = VM_FAULT_FALLBACK;
1682 dax_unlock_entry(&xas, entry);
1683 trace_dax_insert_pfn_mkwrite(mapping->host, vmf, ret);
1684 return ret;
1685}
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697vm_fault_t dax_finish_sync_fault(struct vm_fault *vmf,
1698 enum page_entry_size pe_size, pfn_t pfn)
1699{
1700 int err;
1701 loff_t start = ((loff_t)vmf->pgoff) << PAGE_SHIFT;
1702 unsigned int order = pe_order(pe_size);
1703 size_t len = PAGE_SIZE << order;
1704
1705 err = vfs_fsync_range(vmf->vma->vm_file, start, start + len - 1, 1);
1706 if (err)
1707 return VM_FAULT_SIGBUS;
1708 return dax_insert_pfn_mkwrite(vmf, pfn, order);
1709}
1710EXPORT_SYMBOL_GPL(dax_finish_sync_fault);
1711