1
2
3
4
5
6
7
8
9#include <linux/slab.h>
10#include <linux/backing-dev.h>
11#include <linux/mm.h>
12#include <linux/shm.h>
13#include <linux/mman.h>
14#include <linux/pagemap.h>
15#include <linux/swap.h>
16#include <linux/syscalls.h>
17#include <linux/capability.h>
18#include <linux/init.h>
19#include <linux/file.h>
20#include <linux/fs.h>
21#include <linux/personality.h>
22#include <linux/security.h>
23#include <linux/hugetlb.h>
24#include <linux/profile.h>
25#include <linux/module.h>
26#include <linux/mount.h>
27#include <linux/mempolicy.h>
28#include <linux/rmap.h>
29#include <linux/mmu_notifier.h>
30#include <linux/perf_event.h>
31#include <linux/audit.h>
32#include <linux/khugepaged.h>
33
34#include <asm/uaccess.h>
35#include <asm/cacheflush.h>
36#include <asm/tlb.h>
37#include <asm/mmu_context.h>
38
39#include "internal.h"
40
41#ifndef arch_mmap_check
42#define arch_mmap_check(addr, len, flags) (0)
43#endif
44
45#ifndef arch_rebalance_pgtables
46#define arch_rebalance_pgtables(addr, len) (addr)
47#endif
48
49static void unmap_region(struct mm_struct *mm,
50 struct vm_area_struct *vma, struct vm_area_struct *prev,
51 unsigned long start, unsigned long end);
52
53
54
55
56
57#undef DEBUG_MM_RB
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74pgprot_t protection_map[16] = {
75 __P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
76 __S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
77};
78
79pgprot_t vm_get_page_prot(unsigned long vm_flags)
80{
81 return __pgprot(pgprot_val(protection_map[vm_flags &
82 (VM_READ|VM_WRITE|VM_EXEC|VM_SHARED)]) |
83 pgprot_val(arch_vm_get_page_prot(vm_flags)));
84}
85EXPORT_SYMBOL(vm_get_page_prot);
86
87int sysctl_overcommit_memory = OVERCOMMIT_GUESS;
88int sysctl_overcommit_ratio = 50;
89int sysctl_max_map_count __read_mostly = DEFAULT_MAX_MAP_COUNT;
90struct percpu_counter vm_committed_as;
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin)
109{
110 unsigned long free, allowed;
111
112 vm_acct_memory(pages);
113
114
115
116
117 if (sysctl_overcommit_memory == OVERCOMMIT_ALWAYS)
118 return 0;
119
120 if (sysctl_overcommit_memory == OVERCOMMIT_GUESS) {
121 unsigned long n;
122
123 free = global_page_state(NR_FILE_PAGES);
124 free += nr_swap_pages;
125
126
127
128
129
130
131
132 free += global_page_state(NR_SLAB_RECLAIMABLE);
133
134
135
136
137 if (!cap_sys_admin)
138 free -= free / 32;
139
140 if (free > pages)
141 return 0;
142
143
144
145
146
147 n = nr_free_pages();
148
149
150
151
152 if (n <= totalreserve_pages)
153 goto error;
154 else
155 n -= totalreserve_pages;
156
157
158
159
160 if (!cap_sys_admin)
161 n -= n / 32;
162 free += n;
163
164 if (free > pages)
165 return 0;
166
167 goto error;
168 }
169
170 allowed = (totalram_pages - hugetlb_total_pages())
171 * sysctl_overcommit_ratio / 100;
172
173
174
175 if (!cap_sys_admin)
176 allowed -= allowed / 32;
177 allowed += total_swap_pages;
178
179
180
181 if (mm)
182 allowed -= mm->total_vm / 32;
183
184 if (percpu_counter_read_positive(&vm_committed_as) < allowed)
185 return 0;
186error:
187 vm_unacct_memory(pages);
188
189 return -ENOMEM;
190}
191
192
193
194
195static void __remove_shared_vm_struct(struct vm_area_struct *vma,
196 struct file *file, struct address_space *mapping)
197{
198 if (vma->vm_flags & VM_DENYWRITE)
199 atomic_inc(&file->f_path.dentry->d_inode->i_writecount);
200 if (vma->vm_flags & VM_SHARED)
201 mapping->i_mmap_writable--;
202
203 flush_dcache_mmap_lock(mapping);
204 if (unlikely(vma->vm_flags & VM_NONLINEAR))
205 list_del_init(&vma->shared.vm_set.list);
206 else
207 vma_prio_tree_remove(vma, &mapping->i_mmap);
208 flush_dcache_mmap_unlock(mapping);
209}
210
211
212
213
214
215void unlink_file_vma(struct vm_area_struct *vma)
216{
217 struct file *file = vma->vm_file;
218
219 if (file) {
220 struct address_space *mapping = file->f_mapping;
221 spin_lock(&mapping->i_mmap_lock);
222 __remove_shared_vm_struct(vma, file, mapping);
223 spin_unlock(&mapping->i_mmap_lock);
224 }
225}
226
227
228
229
230static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
231{
232 struct vm_area_struct *next = vma->vm_next;
233
234 might_sleep();
235 if (vma->vm_ops && vma->vm_ops->close)
236 vma->vm_ops->close(vma);
237 if (vma->vm_file) {
238 fput(vma->vm_file);
239 if (vma->vm_flags & VM_EXECUTABLE)
240 removed_exe_file_vma(vma->vm_mm);
241 }
242 mpol_put(vma_policy(vma));
243 kmem_cache_free(vm_area_cachep, vma);
244 return next;
245}
246
247SYSCALL_DEFINE1(brk, unsigned long, brk)
248{
249 unsigned long rlim, retval;
250 unsigned long newbrk, oldbrk;
251 struct mm_struct *mm = current->mm;
252 unsigned long min_brk;
253
254 down_write(&mm->mmap_sem);
255
256#ifdef CONFIG_COMPAT_BRK
257
258
259
260
261
262 if (current->brk_randomized)
263 min_brk = mm->start_brk;
264 else
265 min_brk = mm->end_data;
266#else
267 min_brk = mm->start_brk;
268#endif
269 if (brk < min_brk)
270 goto out;
271
272
273
274
275
276
277
278 rlim = rlimit(RLIMIT_DATA);
279 if (rlim < RLIM_INFINITY && (brk - mm->start_brk) +
280 (mm->end_data - mm->start_data) > rlim)
281 goto out;
282
283 newbrk = PAGE_ALIGN(brk);
284 oldbrk = PAGE_ALIGN(mm->brk);
285 if (oldbrk == newbrk)
286 goto set_brk;
287
288
289 if (brk <= mm->brk) {
290 if (!do_munmap(mm, newbrk, oldbrk-newbrk))
291 goto set_brk;
292 goto out;
293 }
294
295
296 if (find_vma_intersection(mm, oldbrk, newbrk+PAGE_SIZE))
297 goto out;
298
299
300 if (do_brk(oldbrk, newbrk-oldbrk) != oldbrk)
301 goto out;
302set_brk:
303 mm->brk = brk;
304out:
305 retval = mm->brk;
306 up_write(&mm->mmap_sem);
307 return retval;
308}
309
310#ifdef DEBUG_MM_RB
311static int browse_rb(struct rb_root *root)
312{
313 int i = 0, j;
314 struct rb_node *nd, *pn = NULL;
315 unsigned long prev = 0, pend = 0;
316
317 for (nd = rb_first(root); nd; nd = rb_next(nd)) {
318 struct vm_area_struct *vma;
319 vma = rb_entry(nd, struct vm_area_struct, vm_rb);
320 if (vma->vm_start < prev)
321 printk("vm_start %lx prev %lx\n", vma->vm_start, prev), i = -1;
322 if (vma->vm_start < pend)
323 printk("vm_start %lx pend %lx\n", vma->vm_start, pend);
324 if (vma->vm_start > vma->vm_end)
325 printk("vm_end %lx < vm_start %lx\n", vma->vm_end, vma->vm_start);
326 i++;
327 pn = nd;
328 prev = vma->vm_start;
329 pend = vma->vm_end;
330 }
331 j = 0;
332 for (nd = pn; nd; nd = rb_prev(nd)) {
333 j++;
334 }
335 if (i != j)
336 printk("backwards %d, forwards %d\n", j, i), i = 0;
337 return i;
338}
339
340void validate_mm(struct mm_struct *mm)
341{
342 int bug = 0;
343 int i = 0;
344 struct vm_area_struct *tmp = mm->mmap;
345 while (tmp) {
346 tmp = tmp->vm_next;
347 i++;
348 }
349 if (i != mm->map_count)
350 printk("map_count %d vm_next %d\n", mm->map_count, i), bug = 1;
351 i = browse_rb(&mm->mm_rb);
352 if (i != mm->map_count)
353 printk("map_count %d rb %d\n", mm->map_count, i), bug = 1;
354 BUG_ON(bug);
355}
356#else
357#define validate_mm(mm) do { } while (0)
358#endif
359
360static struct vm_area_struct *
361find_vma_prepare(struct mm_struct *mm, unsigned long addr,
362 struct vm_area_struct **pprev, struct rb_node ***rb_link,
363 struct rb_node ** rb_parent)
364{
365 struct vm_area_struct * vma;
366 struct rb_node ** __rb_link, * __rb_parent, * rb_prev;
367
368 __rb_link = &mm->mm_rb.rb_node;
369 rb_prev = __rb_parent = NULL;
370 vma = NULL;
371
372 while (*__rb_link) {
373 struct vm_area_struct *vma_tmp;
374
375 __rb_parent = *__rb_link;
376 vma_tmp = rb_entry(__rb_parent, struct vm_area_struct, vm_rb);
377
378 if (vma_tmp->vm_end > addr) {
379 vma = vma_tmp;
380 if (vma_tmp->vm_start <= addr)
381 break;
382 __rb_link = &__rb_parent->rb_left;
383 } else {
384 rb_prev = __rb_parent;
385 __rb_link = &__rb_parent->rb_right;
386 }
387 }
388
389 *pprev = NULL;
390 if (rb_prev)
391 *pprev = rb_entry(rb_prev, struct vm_area_struct, vm_rb);
392 *rb_link = __rb_link;
393 *rb_parent = __rb_parent;
394 return vma;
395}
396
397static inline void
398__vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
399 struct vm_area_struct *prev, struct rb_node *rb_parent)
400{
401 struct vm_area_struct *next;
402
403 vma->vm_prev = prev;
404 if (prev) {
405 next = prev->vm_next;
406 prev->vm_next = vma;
407 } else {
408 mm->mmap = vma;
409 if (rb_parent)
410 next = rb_entry(rb_parent,
411 struct vm_area_struct, vm_rb);
412 else
413 next = NULL;
414 }
415 vma->vm_next = next;
416 if (next)
417 next->vm_prev = vma;
418}
419
420void __vma_link_rb(struct mm_struct *mm, struct vm_area_struct *vma,
421 struct rb_node **rb_link, struct rb_node *rb_parent)
422{
423 rb_link_node(&vma->vm_rb, rb_parent, rb_link);
424 rb_insert_color(&vma->vm_rb, &mm->mm_rb);
425}
426
427static void __vma_link_file(struct vm_area_struct *vma)
428{
429 struct file *file;
430
431 file = vma->vm_file;
432 if (file) {
433 struct address_space *mapping = file->f_mapping;
434
435 if (vma->vm_flags & VM_DENYWRITE)
436 atomic_dec(&file->f_path.dentry->d_inode->i_writecount);
437 if (vma->vm_flags & VM_SHARED)
438 mapping->i_mmap_writable++;
439
440 flush_dcache_mmap_lock(mapping);
441 if (unlikely(vma->vm_flags & VM_NONLINEAR))
442 vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear);
443 else
444 vma_prio_tree_insert(vma, &mapping->i_mmap);
445 flush_dcache_mmap_unlock(mapping);
446 }
447}
448
449static void
450__vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
451 struct vm_area_struct *prev, struct rb_node **rb_link,
452 struct rb_node *rb_parent)
453{
454 __vma_link_list(mm, vma, prev, rb_parent);
455 __vma_link_rb(mm, vma, rb_link, rb_parent);
456}
457
458static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
459 struct vm_area_struct *prev, struct rb_node **rb_link,
460 struct rb_node *rb_parent)
461{
462 struct address_space *mapping = NULL;
463
464 if (vma->vm_file)
465 mapping = vma->vm_file->f_mapping;
466
467 if (mapping) {
468 spin_lock(&mapping->i_mmap_lock);
469 vma->vm_truncate_count = mapping->truncate_count;
470 }
471
472 __vma_link(mm, vma, prev, rb_link, rb_parent);
473 __vma_link_file(vma);
474
475 if (mapping)
476 spin_unlock(&mapping->i_mmap_lock);
477
478 mm->map_count++;
479 validate_mm(mm);
480}
481
482
483
484
485
486
487static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
488{
489 struct vm_area_struct *__vma, *prev;
490 struct rb_node **rb_link, *rb_parent;
491
492 __vma = find_vma_prepare(mm, vma->vm_start,&prev, &rb_link, &rb_parent);
493 BUG_ON(__vma && __vma->vm_start < vma->vm_end);
494 __vma_link(mm, vma, prev, rb_link, rb_parent);
495 mm->map_count++;
496}
497
498static inline void
499__vma_unlink(struct mm_struct *mm, struct vm_area_struct *vma,
500 struct vm_area_struct *prev)
501{
502 struct vm_area_struct *next = vma->vm_next;
503
504 prev->vm_next = next;
505 if (next)
506 next->vm_prev = prev;
507 rb_erase(&vma->vm_rb, &mm->mm_rb);
508 if (mm->mmap_cache == vma)
509 mm->mmap_cache = prev;
510}
511
512
513
514
515
516
517
518
519int vma_adjust(struct vm_area_struct *vma, unsigned long start,
520 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert)
521{
522 struct mm_struct *mm = vma->vm_mm;
523 struct vm_area_struct *next = vma->vm_next;
524 struct vm_area_struct *importer = NULL;
525 struct address_space *mapping = NULL;
526 struct prio_tree_root *root = NULL;
527 struct anon_vma *anon_vma = NULL;
528 struct file *file = vma->vm_file;
529 long adjust_next = 0;
530 int remove_next = 0;
531
532 if (next && !insert) {
533 struct vm_area_struct *exporter = NULL;
534
535 if (end >= next->vm_end) {
536
537
538
539
540again: remove_next = 1 + (end > next->vm_end);
541 end = next->vm_end;
542 exporter = next;
543 importer = vma;
544 } else if (end > next->vm_start) {
545
546
547
548
549 adjust_next = (end - next->vm_start) >> PAGE_SHIFT;
550 exporter = next;
551 importer = vma;
552 } else if (end < vma->vm_end) {
553
554
555
556
557
558 adjust_next = - ((vma->vm_end - end) >> PAGE_SHIFT);
559 exporter = vma;
560 importer = next;
561 }
562
563
564
565
566
567
568 if (exporter && exporter->anon_vma && !importer->anon_vma) {
569 if (anon_vma_clone(importer, exporter))
570 return -ENOMEM;
571 importer->anon_vma = exporter->anon_vma;
572 }
573 }
574
575 if (file) {
576 mapping = file->f_mapping;
577 if (!(vma->vm_flags & VM_NONLINEAR))
578 root = &mapping->i_mmap;
579 spin_lock(&mapping->i_mmap_lock);
580 if (importer &&
581 vma->vm_truncate_count != next->vm_truncate_count) {
582
583
584
585
586 importer->vm_truncate_count = 0;
587 }
588 if (insert) {
589 insert->vm_truncate_count = vma->vm_truncate_count;
590
591
592
593
594
595
596 __vma_link_file(insert);
597 }
598 }
599
600 vma_adjust_trans_huge(vma, start, end, adjust_next);
601
602
603
604
605
606
607
608 if (vma->anon_vma && (insert || importer || start != vma->vm_start)) {
609 anon_vma = vma->anon_vma;
610 anon_vma_lock(anon_vma);
611 }
612
613 if (root) {
614 flush_dcache_mmap_lock(mapping);
615 vma_prio_tree_remove(vma, root);
616 if (adjust_next)
617 vma_prio_tree_remove(next, root);
618 }
619
620 vma->vm_start = start;
621 vma->vm_end = end;
622 vma->vm_pgoff = pgoff;
623 if (adjust_next) {
624 next->vm_start += adjust_next << PAGE_SHIFT;
625 next->vm_pgoff += adjust_next;
626 }
627
628 if (root) {
629 if (adjust_next)
630 vma_prio_tree_insert(next, root);
631 vma_prio_tree_insert(vma, root);
632 flush_dcache_mmap_unlock(mapping);
633 }
634
635 if (remove_next) {
636
637
638
639
640 __vma_unlink(mm, next, vma);
641 if (file)
642 __remove_shared_vm_struct(next, file, mapping);
643 } else if (insert) {
644
645
646
647
648
649 __insert_vm_struct(mm, insert);
650 }
651
652 if (anon_vma)
653 anon_vma_unlock(anon_vma);
654 if (mapping)
655 spin_unlock(&mapping->i_mmap_lock);
656
657 if (remove_next) {
658 if (file) {
659 fput(file);
660 if (next->vm_flags & VM_EXECUTABLE)
661 removed_exe_file_vma(mm);
662 }
663 if (next->anon_vma)
664 anon_vma_merge(vma, next);
665 mm->map_count--;
666 mpol_put(vma_policy(next));
667 kmem_cache_free(vm_area_cachep, next);
668
669
670
671
672
673 if (remove_next == 2) {
674 next = vma->vm_next;
675 goto again;
676 }
677 }
678
679 validate_mm(mm);
680
681 return 0;
682}
683
684
685
686
687
688static inline int is_mergeable_vma(struct vm_area_struct *vma,
689 struct file *file, unsigned long vm_flags)
690{
691
692 if ((vma->vm_flags ^ vm_flags) & ~VM_CAN_NONLINEAR)
693 return 0;
694 if (vma->vm_file != file)
695 return 0;
696 if (vma->vm_ops && vma->vm_ops->close)
697 return 0;
698 return 1;
699}
700
701static inline int is_mergeable_anon_vma(struct anon_vma *anon_vma1,
702 struct anon_vma *anon_vma2)
703{
704 return !anon_vma1 || !anon_vma2 || (anon_vma1 == anon_vma2);
705}
706
707
708
709
710
711
712
713
714
715
716
717
718static int
719can_vma_merge_before(struct vm_area_struct *vma, unsigned long vm_flags,
720 struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
721{
722 if (is_mergeable_vma(vma, file, vm_flags) &&
723 is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {
724 if (vma->vm_pgoff == vm_pgoff)
725 return 1;
726 }
727 return 0;
728}
729
730
731
732
733
734
735
736
737static int
738can_vma_merge_after(struct vm_area_struct *vma, unsigned long vm_flags,
739 struct anon_vma *anon_vma, struct file *file, pgoff_t vm_pgoff)
740{
741 if (is_mergeable_vma(vma, file, vm_flags) &&
742 is_mergeable_anon_vma(anon_vma, vma->anon_vma)) {
743 pgoff_t vm_pglen;
744 vm_pglen = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
745 if (vma->vm_pgoff + vm_pglen == vm_pgoff)
746 return 1;
747 }
748 return 0;
749}
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780struct vm_area_struct *vma_merge(struct mm_struct *mm,
781 struct vm_area_struct *prev, unsigned long addr,
782 unsigned long end, unsigned long vm_flags,
783 struct anon_vma *anon_vma, struct file *file,
784 pgoff_t pgoff, struct mempolicy *policy)
785{
786 pgoff_t pglen = (end - addr) >> PAGE_SHIFT;
787 struct vm_area_struct *area, *next;
788 int err;
789
790
791
792
793
794 if (vm_flags & VM_SPECIAL)
795 return NULL;
796
797 if (prev)
798 next = prev->vm_next;
799 else
800 next = mm->mmap;
801 area = next;
802 if (next && next->vm_end == end)
803 next = next->vm_next;
804
805
806
807
808 if (prev && prev->vm_end == addr &&
809 mpol_equal(vma_policy(prev), policy) &&
810 can_vma_merge_after(prev, vm_flags,
811 anon_vma, file, pgoff)) {
812
813
814
815 if (next && end == next->vm_start &&
816 mpol_equal(policy, vma_policy(next)) &&
817 can_vma_merge_before(next, vm_flags,
818 anon_vma, file, pgoff+pglen) &&
819 is_mergeable_anon_vma(prev->anon_vma,
820 next->anon_vma)) {
821
822 err = vma_adjust(prev, prev->vm_start,
823 next->vm_end, prev->vm_pgoff, NULL);
824 } else
825 err = vma_adjust(prev, prev->vm_start,
826 end, prev->vm_pgoff, NULL);
827 if (err)
828 return NULL;
829 khugepaged_enter_vma_merge(prev);
830 return prev;
831 }
832
833
834
835
836 if (next && end == next->vm_start &&
837 mpol_equal(policy, vma_policy(next)) &&
838 can_vma_merge_before(next, vm_flags,
839 anon_vma, file, pgoff+pglen)) {
840 if (prev && addr < prev->vm_end)
841 err = vma_adjust(prev, prev->vm_start,
842 addr, prev->vm_pgoff, NULL);
843 else
844 err = vma_adjust(area, addr, next->vm_end,
845 next->vm_pgoff - pglen, NULL);
846 if (err)
847 return NULL;
848 khugepaged_enter_vma_merge(area);
849 return area;
850 }
851
852 return NULL;
853}
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868static int anon_vma_compatible(struct vm_area_struct *a, struct vm_area_struct *b)
869{
870 return a->vm_end == b->vm_start &&
871 mpol_equal(vma_policy(a), vma_policy(b)) &&
872 a->vm_file == b->vm_file &&
873 !((a->vm_flags ^ b->vm_flags) & ~(VM_READ|VM_WRITE|VM_EXEC)) &&
874 b->vm_pgoff == a->vm_pgoff + ((b->vm_start - a->vm_start) >> PAGE_SHIFT);
875}
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899static struct anon_vma *reusable_anon_vma(struct vm_area_struct *old, struct vm_area_struct *a, struct vm_area_struct *b)
900{
901 if (anon_vma_compatible(a, b)) {
902 struct anon_vma *anon_vma = ACCESS_ONCE(old->anon_vma);
903
904 if (anon_vma && list_is_singular(&old->anon_vma_chain))
905 return anon_vma;
906 }
907 return NULL;
908}
909
910
911
912
913
914
915
916
917
918struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *vma)
919{
920 struct anon_vma *anon_vma;
921 struct vm_area_struct *near;
922
923 near = vma->vm_next;
924 if (!near)
925 goto try_prev;
926
927 anon_vma = reusable_anon_vma(near, vma, near);
928 if (anon_vma)
929 return anon_vma;
930try_prev:
931
932
933
934
935
936
937
938 BUG_ON(find_vma_prev(vma->vm_mm, vma->vm_start, &near) != vma);
939 if (!near)
940 goto none;
941
942 anon_vma = reusable_anon_vma(near, near, vma);
943 if (anon_vma)
944 return anon_vma;
945none:
946
947
948
949
950
951
952
953
954 return NULL;
955}
956
957#ifdef CONFIG_PROC_FS
958void vm_stat_account(struct mm_struct *mm, unsigned long flags,
959 struct file *file, long pages)
960{
961 const unsigned long stack_flags
962 = VM_STACK_FLAGS & (VM_GROWSUP|VM_GROWSDOWN);
963
964 if (file) {
965 mm->shared_vm += pages;
966 if ((flags & (VM_EXEC|VM_WRITE)) == VM_EXEC)
967 mm->exec_vm += pages;
968 } else if (flags & stack_flags)
969 mm->stack_vm += pages;
970 if (flags & (VM_RESERVED|VM_IO))
971 mm->reserved_vm += pages;
972}
973#endif
974
975
976
977
978
979unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
980 unsigned long len, unsigned long prot,
981 unsigned long flags, unsigned long pgoff)
982{
983 struct mm_struct * mm = current->mm;
984 struct inode *inode;
985 unsigned int vm_flags;
986 int error;
987 unsigned long reqprot = prot;
988
989
990
991
992
993
994
995 if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC))
996 if (!(file && (file->f_path.mnt->mnt_flags & MNT_NOEXEC)))
997 prot |= PROT_EXEC;
998
999 if (!len)
1000 return -EINVAL;
1001
1002 if (!(flags & MAP_FIXED))
1003 addr = round_hint_to_min(addr);
1004
1005
1006 len = PAGE_ALIGN(len);
1007 if (!len)
1008 return -ENOMEM;
1009
1010
1011 if ((pgoff + (len >> PAGE_SHIFT)) < pgoff)
1012 return -EOVERFLOW;
1013
1014
1015 if (mm->map_count > sysctl_max_map_count)
1016 return -ENOMEM;
1017
1018
1019
1020
1021 addr = get_unmapped_area(file, addr, len, pgoff, flags);
1022 if (addr & ~PAGE_MASK)
1023 return addr;
1024
1025
1026
1027
1028
1029 vm_flags = calc_vm_prot_bits(prot) | calc_vm_flag_bits(flags) |
1030 mm->def_flags | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC;
1031
1032 if (flags & MAP_LOCKED)
1033 if (!can_do_mlock())
1034 return -EPERM;
1035
1036
1037 if (vm_flags & VM_LOCKED) {
1038 unsigned long locked, lock_limit;
1039 locked = len >> PAGE_SHIFT;
1040 locked += mm->locked_vm;
1041 lock_limit = rlimit(RLIMIT_MEMLOCK);
1042 lock_limit >>= PAGE_SHIFT;
1043 if (locked > lock_limit && !capable(CAP_IPC_LOCK))
1044 return -EAGAIN;
1045 }
1046
1047 inode = file ? file->f_path.dentry->d_inode : NULL;
1048
1049 if (file) {
1050 switch (flags & MAP_TYPE) {
1051 case MAP_SHARED:
1052 if ((prot&PROT_WRITE) && !(file->f_mode&FMODE_WRITE))
1053 return -EACCES;
1054
1055
1056
1057
1058
1059 if (IS_APPEND(inode) && (file->f_mode & FMODE_WRITE))
1060 return -EACCES;
1061
1062
1063
1064
1065 if (locks_verify_locked(inode))
1066 return -EAGAIN;
1067
1068 vm_flags |= VM_SHARED | VM_MAYSHARE;
1069 if (!(file->f_mode & FMODE_WRITE))
1070 vm_flags &= ~(VM_MAYWRITE | VM_SHARED);
1071
1072
1073 case MAP_PRIVATE:
1074 if (!(file->f_mode & FMODE_READ))
1075 return -EACCES;
1076 if (file->f_path.mnt->mnt_flags & MNT_NOEXEC) {
1077 if (vm_flags & VM_EXEC)
1078 return -EPERM;
1079 vm_flags &= ~VM_MAYEXEC;
1080 }
1081
1082 if (!file->f_op || !file->f_op->mmap)
1083 return -ENODEV;
1084 break;
1085
1086 default:
1087 return -EINVAL;
1088 }
1089 } else {
1090 switch (flags & MAP_TYPE) {
1091 case MAP_SHARED:
1092
1093
1094
1095 pgoff = 0;
1096 vm_flags |= VM_SHARED | VM_MAYSHARE;
1097 break;
1098 case MAP_PRIVATE:
1099
1100
1101
1102 pgoff = addr >> PAGE_SHIFT;
1103 break;
1104 default:
1105 return -EINVAL;
1106 }
1107 }
1108
1109 error = security_file_mmap(file, reqprot, prot, flags, addr, 0);
1110 if (error)
1111 return error;
1112
1113 return mmap_region(file, addr, len, flags, vm_flags, pgoff);
1114}
1115EXPORT_SYMBOL(do_mmap_pgoff);
1116
1117SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
1118 unsigned long, prot, unsigned long, flags,
1119 unsigned long, fd, unsigned long, pgoff)
1120{
1121 struct file *file = NULL;
1122 unsigned long retval = -EBADF;
1123
1124 if (!(flags & MAP_ANONYMOUS)) {
1125 audit_mmap_fd(fd, flags);
1126 if (unlikely(flags & MAP_HUGETLB))
1127 return -EINVAL;
1128 file = fget(fd);
1129 if (!file)
1130 goto out;
1131 } else if (flags & MAP_HUGETLB) {
1132 struct user_struct *user = NULL;
1133
1134
1135
1136
1137
1138
1139 len = ALIGN(len, huge_page_size(&default_hstate));
1140 file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, VM_NORESERVE,
1141 &user, HUGETLB_ANONHUGE_INODE);
1142 if (IS_ERR(file))
1143 return PTR_ERR(file);
1144 }
1145
1146 flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
1147
1148 down_write(¤t->mm->mmap_sem);
1149 retval = do_mmap_pgoff(file, addr, len, prot, flags, pgoff);
1150 up_write(¤t->mm->mmap_sem);
1151
1152 if (file)
1153 fput(file);
1154out:
1155 return retval;
1156}
1157
1158#ifdef __ARCH_WANT_SYS_OLD_MMAP
1159struct mmap_arg_struct {
1160 unsigned long addr;
1161 unsigned long len;
1162 unsigned long prot;
1163 unsigned long flags;
1164 unsigned long fd;
1165 unsigned long offset;
1166};
1167
1168SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg)
1169{
1170 struct mmap_arg_struct a;
1171
1172 if (copy_from_user(&a, arg, sizeof(a)))
1173 return -EFAULT;
1174 if (a.offset & ~PAGE_MASK)
1175 return -EINVAL;
1176
1177 return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd,
1178 a.offset >> PAGE_SHIFT);
1179}
1180#endif
1181
1182
1183
1184
1185
1186
1187
1188int vma_wants_writenotify(struct vm_area_struct *vma)
1189{
1190 unsigned int vm_flags = vma->vm_flags;
1191
1192
1193 if ((vm_flags & (VM_WRITE|VM_SHARED)) != ((VM_WRITE|VM_SHARED)))
1194 return 0;
1195
1196
1197 if (vma->vm_ops && vma->vm_ops->page_mkwrite)
1198 return 1;
1199
1200
1201 if (pgprot_val(vma->vm_page_prot) !=
1202 pgprot_val(vm_get_page_prot(vm_flags)))
1203 return 0;
1204
1205
1206 if (vm_flags & (VM_PFNMAP|VM_INSERTPAGE))
1207 return 0;
1208
1209
1210 return vma->vm_file && vma->vm_file->f_mapping &&
1211 mapping_cap_account_dirty(vma->vm_file->f_mapping);
1212}
1213
1214
1215
1216
1217
1218static inline int accountable_mapping(struct file *file, unsigned int vm_flags)
1219{
1220
1221
1222
1223
1224 if (file && is_file_hugepages(file))
1225 return 0;
1226
1227 return (vm_flags & (VM_NORESERVE | VM_SHARED | VM_WRITE)) == VM_WRITE;
1228}
1229
1230unsigned long mmap_region(struct file *file, unsigned long addr,
1231 unsigned long len, unsigned long flags,
1232 unsigned int vm_flags, unsigned long pgoff)
1233{
1234 struct mm_struct *mm = current->mm;
1235 struct vm_area_struct *vma, *prev;
1236 int correct_wcount = 0;
1237 int error;
1238 struct rb_node **rb_link, *rb_parent;
1239 unsigned long charged = 0;
1240 struct inode *inode = file ? file->f_path.dentry->d_inode : NULL;
1241
1242
1243 error = -ENOMEM;
1244munmap_back:
1245 vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
1246 if (vma && vma->vm_start < addr + len) {
1247 if (do_munmap(mm, addr, len))
1248 return -ENOMEM;
1249 goto munmap_back;
1250 }
1251
1252
1253 if (!may_expand_vm(mm, len >> PAGE_SHIFT))
1254 return -ENOMEM;
1255
1256
1257
1258
1259
1260 if ((flags & MAP_NORESERVE)) {
1261
1262 if (sysctl_overcommit_memory != OVERCOMMIT_NEVER)
1263 vm_flags |= VM_NORESERVE;
1264
1265
1266 if (file && is_file_hugepages(file))
1267 vm_flags |= VM_NORESERVE;
1268 }
1269
1270
1271
1272
1273 if (accountable_mapping(file, vm_flags)) {
1274 charged = len >> PAGE_SHIFT;
1275 if (security_vm_enough_memory(charged))
1276 return -ENOMEM;
1277 vm_flags |= VM_ACCOUNT;
1278 }
1279
1280
1281
1282
1283 vma = vma_merge(mm, prev, addr, addr + len, vm_flags, NULL, file, pgoff, NULL);
1284 if (vma)
1285 goto out;
1286
1287
1288
1289
1290
1291
1292 vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
1293 if (!vma) {
1294 error = -ENOMEM;
1295 goto unacct_error;
1296 }
1297
1298 vma->vm_mm = mm;
1299 vma->vm_start = addr;
1300 vma->vm_end = addr + len;
1301 vma->vm_flags = vm_flags;
1302 vma->vm_page_prot = vm_get_page_prot(vm_flags);
1303 vma->vm_pgoff = pgoff;
1304 INIT_LIST_HEAD(&vma->anon_vma_chain);
1305
1306 if (file) {
1307 error = -EINVAL;
1308 if (vm_flags & (VM_GROWSDOWN|VM_GROWSUP))
1309 goto free_vma;
1310 if (vm_flags & VM_DENYWRITE) {
1311 error = deny_write_access(file);
1312 if (error)
1313 goto free_vma;
1314 correct_wcount = 1;
1315 }
1316 vma->vm_file = file;
1317 get_file(file);
1318 error = file->f_op->mmap(file, vma);
1319 if (error)
1320 goto unmap_and_free_vma;
1321 if (vm_flags & VM_EXECUTABLE)
1322 added_exe_file_vma(mm);
1323
1324
1325
1326
1327
1328
1329 addr = vma->vm_start;
1330 pgoff = vma->vm_pgoff;
1331 vm_flags = vma->vm_flags;
1332 } else if (vm_flags & VM_SHARED) {
1333 error = shmem_zero_setup(vma);
1334 if (error)
1335 goto free_vma;
1336 }
1337
1338 if (vma_wants_writenotify(vma)) {
1339 pgprot_t pprot = vma->vm_page_prot;
1340
1341
1342
1343
1344
1345
1346
1347
1348 vma->vm_page_prot = vm_get_page_prot(vm_flags & ~VM_SHARED);
1349 if (pgprot_val(pprot) == pgprot_val(pgprot_noncached(pprot)))
1350 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
1351 }
1352
1353 vma_link(mm, vma, prev, rb_link, rb_parent);
1354 file = vma->vm_file;
1355
1356
1357 if (correct_wcount)
1358 atomic_inc(&inode->i_writecount);
1359out:
1360 perf_event_mmap(vma);
1361
1362 mm->total_vm += len >> PAGE_SHIFT;
1363 vm_stat_account(mm, vm_flags, file, len >> PAGE_SHIFT);
1364 if (vm_flags & VM_LOCKED) {
1365 if (!mlock_vma_pages_range(vma, addr, addr + len))
1366 mm->locked_vm += (len >> PAGE_SHIFT);
1367 } else if ((flags & MAP_POPULATE) && !(flags & MAP_NONBLOCK))
1368 make_pages_present(addr, addr + len);
1369 return addr;
1370
1371unmap_and_free_vma:
1372 if (correct_wcount)
1373 atomic_inc(&inode->i_writecount);
1374 vma->vm_file = NULL;
1375 fput(file);
1376
1377
1378 unmap_region(mm, vma, prev, vma->vm_start, vma->vm_end);
1379 charged = 0;
1380free_vma:
1381 kmem_cache_free(vm_area_cachep, vma);
1382unacct_error:
1383 if (charged)
1384 vm_unacct_memory(charged);
1385 return error;
1386}
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399#ifndef HAVE_ARCH_UNMAPPED_AREA
1400unsigned long
1401arch_get_unmapped_area(struct file *filp, unsigned long addr,
1402 unsigned long len, unsigned long pgoff, unsigned long flags)
1403{
1404 struct mm_struct *mm = current->mm;
1405 struct vm_area_struct *vma;
1406 unsigned long start_addr;
1407
1408 if (len > TASK_SIZE)
1409 return -ENOMEM;
1410
1411 if (flags & MAP_FIXED)
1412 return addr;
1413
1414 if (addr) {
1415 addr = PAGE_ALIGN(addr);
1416 vma = find_vma(mm, addr);
1417 if (TASK_SIZE - len >= addr &&
1418 (!vma || addr + len <= vma->vm_start))
1419 return addr;
1420 }
1421 if (len > mm->cached_hole_size) {
1422 start_addr = addr = mm->free_area_cache;
1423 } else {
1424 start_addr = addr = TASK_UNMAPPED_BASE;
1425 mm->cached_hole_size = 0;
1426 }
1427
1428full_search:
1429 for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
1430
1431 if (TASK_SIZE - len < addr) {
1432
1433
1434
1435
1436 if (start_addr != TASK_UNMAPPED_BASE) {
1437 addr = TASK_UNMAPPED_BASE;
1438 start_addr = addr;
1439 mm->cached_hole_size = 0;
1440 goto full_search;
1441 }
1442 return -ENOMEM;
1443 }
1444 if (!vma || addr + len <= vma->vm_start) {
1445
1446
1447
1448 mm->free_area_cache = addr + len;
1449 return addr;
1450 }
1451 if (addr + mm->cached_hole_size < vma->vm_start)
1452 mm->cached_hole_size = vma->vm_start - addr;
1453 addr = vma->vm_end;
1454 }
1455}
1456#endif
1457
1458void arch_unmap_area(struct mm_struct *mm, unsigned long addr)
1459{
1460
1461
1462
1463 if (addr >= TASK_UNMAPPED_BASE && addr < mm->free_area_cache) {
1464 mm->free_area_cache = addr;
1465 mm->cached_hole_size = ~0UL;
1466 }
1467}
1468
1469
1470
1471
1472
1473#ifndef HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
1474unsigned long
1475arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0,
1476 const unsigned long len, const unsigned long pgoff,
1477 const unsigned long flags)
1478{
1479 struct vm_area_struct *vma;
1480 struct mm_struct *mm = current->mm;
1481 unsigned long addr = addr0;
1482
1483
1484 if (len > TASK_SIZE)
1485 return -ENOMEM;
1486
1487 if (flags & MAP_FIXED)
1488 return addr;
1489
1490
1491 if (addr) {
1492 addr = PAGE_ALIGN(addr);
1493 vma = find_vma(mm, addr);
1494 if (TASK_SIZE - len >= addr &&
1495 (!vma || addr + len <= vma->vm_start))
1496 return addr;
1497 }
1498
1499
1500 if (len <= mm->cached_hole_size) {
1501 mm->cached_hole_size = 0;
1502 mm->free_area_cache = mm->mmap_base;
1503 }
1504
1505
1506 addr = mm->free_area_cache;
1507
1508
1509 if (addr > len) {
1510 vma = find_vma(mm, addr-len);
1511 if (!vma || addr <= vma->vm_start)
1512
1513 return (mm->free_area_cache = addr-len);
1514 }
1515
1516 if (mm->mmap_base < len)
1517 goto bottomup;
1518
1519 addr = mm->mmap_base-len;
1520
1521 do {
1522
1523
1524
1525
1526
1527 vma = find_vma(mm, addr);
1528 if (!vma || addr+len <= vma->vm_start)
1529
1530 return (mm->free_area_cache = addr);
1531
1532
1533 if (addr + mm->cached_hole_size < vma->vm_start)
1534 mm->cached_hole_size = vma->vm_start - addr;
1535
1536
1537 addr = vma->vm_start-len;
1538 } while (len < vma->vm_start);
1539
1540bottomup:
1541
1542
1543
1544
1545
1546
1547 mm->cached_hole_size = ~0UL;
1548 mm->free_area_cache = TASK_UNMAPPED_BASE;
1549 addr = arch_get_unmapped_area(filp, addr0, len, pgoff, flags);
1550
1551
1552
1553 mm->free_area_cache = mm->mmap_base;
1554 mm->cached_hole_size = ~0UL;
1555
1556 return addr;
1557}
1558#endif
1559
1560void arch_unmap_area_topdown(struct mm_struct *mm, unsigned long addr)
1561{
1562
1563
1564
1565 if (addr > mm->free_area_cache)
1566 mm->free_area_cache = addr;
1567
1568
1569 if (mm->free_area_cache > mm->mmap_base)
1570 mm->free_area_cache = mm->mmap_base;
1571}
1572
1573unsigned long
1574get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
1575 unsigned long pgoff, unsigned long flags)
1576{
1577 unsigned long (*get_area)(struct file *, unsigned long,
1578 unsigned long, unsigned long, unsigned long);
1579
1580 unsigned long error = arch_mmap_check(addr, len, flags);
1581 if (error)
1582 return error;
1583
1584
1585 if (len > TASK_SIZE)
1586 return -ENOMEM;
1587
1588 get_area = current->mm->get_unmapped_area;
1589 if (file && file->f_op && file->f_op->get_unmapped_area)
1590 get_area = file->f_op->get_unmapped_area;
1591 addr = get_area(file, addr, len, pgoff, flags);
1592 if (IS_ERR_VALUE(addr))
1593 return addr;
1594
1595 if (addr > TASK_SIZE - len)
1596 return -ENOMEM;
1597 if (addr & ~PAGE_MASK)
1598 return -EINVAL;
1599
1600 return arch_rebalance_pgtables(addr, len);
1601}
1602
1603EXPORT_SYMBOL(get_unmapped_area);
1604
1605
1606struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
1607{
1608 struct vm_area_struct *vma = NULL;
1609
1610 if (mm) {
1611
1612
1613 vma = mm->mmap_cache;
1614 if (!(vma && vma->vm_end > addr && vma->vm_start <= addr)) {
1615 struct rb_node * rb_node;
1616
1617 rb_node = mm->mm_rb.rb_node;
1618 vma = NULL;
1619
1620 while (rb_node) {
1621 struct vm_area_struct * vma_tmp;
1622
1623 vma_tmp = rb_entry(rb_node,
1624 struct vm_area_struct, vm_rb);
1625
1626 if (vma_tmp->vm_end > addr) {
1627 vma = vma_tmp;
1628 if (vma_tmp->vm_start <= addr)
1629 break;
1630 rb_node = rb_node->rb_left;
1631 } else
1632 rb_node = rb_node->rb_right;
1633 }
1634 if (vma)
1635 mm->mmap_cache = vma;
1636 }
1637 }
1638 return vma;
1639}
1640
1641EXPORT_SYMBOL(find_vma);
1642
1643
1644struct vm_area_struct *
1645find_vma_prev(struct mm_struct *mm, unsigned long addr,
1646 struct vm_area_struct **pprev)
1647{
1648 struct vm_area_struct *vma = NULL, *prev = NULL;
1649 struct rb_node *rb_node;
1650 if (!mm)
1651 goto out;
1652
1653
1654 vma = mm->mmap;
1655
1656
1657 rb_node = mm->mm_rb.rb_node;
1658
1659 while (rb_node) {
1660 struct vm_area_struct *vma_tmp;
1661 vma_tmp = rb_entry(rb_node, struct vm_area_struct, vm_rb);
1662
1663 if (addr < vma_tmp->vm_end) {
1664 rb_node = rb_node->rb_left;
1665 } else {
1666 prev = vma_tmp;
1667 if (!prev->vm_next || (addr < prev->vm_next->vm_end))
1668 break;
1669 rb_node = rb_node->rb_right;
1670 }
1671 }
1672
1673out:
1674 *pprev = prev;
1675 return prev ? prev->vm_next : vma;
1676}
1677
1678
1679
1680
1681
1682
1683static int acct_stack_growth(struct vm_area_struct *vma, unsigned long size, unsigned long grow)
1684{
1685 struct mm_struct *mm = vma->vm_mm;
1686 struct rlimit *rlim = current->signal->rlim;
1687 unsigned long new_start;
1688
1689
1690 if (!may_expand_vm(mm, grow))
1691 return -ENOMEM;
1692
1693
1694 if (size > ACCESS_ONCE(rlim[RLIMIT_STACK].rlim_cur))
1695 return -ENOMEM;
1696
1697
1698 if (vma->vm_flags & VM_LOCKED) {
1699 unsigned long locked;
1700 unsigned long limit;
1701 locked = mm->locked_vm + grow;
1702 limit = ACCESS_ONCE(rlim[RLIMIT_MEMLOCK].rlim_cur);
1703 limit >>= PAGE_SHIFT;
1704 if (locked > limit && !capable(CAP_IPC_LOCK))
1705 return -ENOMEM;
1706 }
1707
1708
1709 new_start = (vma->vm_flags & VM_GROWSUP) ? vma->vm_start :
1710 vma->vm_end - size;
1711 if (is_hugepage_only_range(vma->vm_mm, new_start, size))
1712 return -EFAULT;
1713
1714
1715
1716
1717
1718 if (security_vm_enough_memory_mm(mm, grow))
1719 return -ENOMEM;
1720
1721
1722 mm->total_vm += grow;
1723 if (vma->vm_flags & VM_LOCKED)
1724 mm->locked_vm += grow;
1725 vm_stat_account(mm, vma->vm_flags, vma->vm_file, grow);
1726 return 0;
1727}
1728
1729#if defined(CONFIG_STACK_GROWSUP) || defined(CONFIG_IA64)
1730
1731
1732
1733
1734int expand_upwards(struct vm_area_struct *vma, unsigned long address)
1735{
1736 int error;
1737
1738 if (!(vma->vm_flags & VM_GROWSUP))
1739 return -EFAULT;
1740
1741
1742
1743
1744
1745 if (unlikely(anon_vma_prepare(vma)))
1746 return -ENOMEM;
1747 vma_lock_anon_vma(vma);
1748
1749
1750
1751
1752
1753
1754
1755 if (address < PAGE_ALIGN(address+4))
1756 address = PAGE_ALIGN(address+4);
1757 else {
1758 vma_unlock_anon_vma(vma);
1759 return -ENOMEM;
1760 }
1761 error = 0;
1762
1763
1764 if (address > vma->vm_end) {
1765 unsigned long size, grow;
1766
1767 size = address - vma->vm_start;
1768 grow = (address - vma->vm_end) >> PAGE_SHIFT;
1769
1770 error = -ENOMEM;
1771 if (vma->vm_pgoff + (size >> PAGE_SHIFT) >= vma->vm_pgoff) {
1772 error = acct_stack_growth(vma, size, grow);
1773 if (!error) {
1774 vma->vm_end = address;
1775 perf_event_mmap(vma);
1776 }
1777 }
1778 }
1779 vma_unlock_anon_vma(vma);
1780 khugepaged_enter_vma_merge(vma);
1781 return error;
1782}
1783#endif
1784
1785
1786
1787
1788static int expand_downwards(struct vm_area_struct *vma,
1789 unsigned long address)
1790{
1791 int error;
1792
1793
1794
1795
1796
1797 if (unlikely(anon_vma_prepare(vma)))
1798 return -ENOMEM;
1799
1800 address &= PAGE_MASK;
1801 error = security_file_mmap(NULL, 0, 0, 0, address, 1);
1802 if (error)
1803 return error;
1804
1805 vma_lock_anon_vma(vma);
1806
1807
1808
1809
1810
1811
1812
1813
1814 if (address < vma->vm_start) {
1815 unsigned long size, grow;
1816
1817 size = vma->vm_end - address;
1818 grow = (vma->vm_start - address) >> PAGE_SHIFT;
1819
1820 error = -ENOMEM;
1821 if (grow <= vma->vm_pgoff) {
1822 error = acct_stack_growth(vma, size, grow);
1823 if (!error) {
1824 vma->vm_start = address;
1825 vma->vm_pgoff -= grow;
1826 perf_event_mmap(vma);
1827 }
1828 }
1829 }
1830 vma_unlock_anon_vma(vma);
1831 khugepaged_enter_vma_merge(vma);
1832 return error;
1833}
1834
1835int expand_stack_downwards(struct vm_area_struct *vma, unsigned long address)
1836{
1837 return expand_downwards(vma, address);
1838}
1839
1840#ifdef CONFIG_STACK_GROWSUP
1841int expand_stack(struct vm_area_struct *vma, unsigned long address)
1842{
1843 return expand_upwards(vma, address);
1844}
1845
1846struct vm_area_struct *
1847find_extend_vma(struct mm_struct *mm, unsigned long addr)
1848{
1849 struct vm_area_struct *vma, *prev;
1850
1851 addr &= PAGE_MASK;
1852 vma = find_vma_prev(mm, addr, &prev);
1853 if (vma && (vma->vm_start <= addr))
1854 return vma;
1855 if (!prev || expand_stack(prev, addr))
1856 return NULL;
1857 if (prev->vm_flags & VM_LOCKED) {
1858 mlock_vma_pages_range(prev, addr, prev->vm_end);
1859 }
1860 return prev;
1861}
1862#else
1863int expand_stack(struct vm_area_struct *vma, unsigned long address)
1864{
1865 return expand_downwards(vma, address);
1866}
1867
1868struct vm_area_struct *
1869find_extend_vma(struct mm_struct * mm, unsigned long addr)
1870{
1871 struct vm_area_struct * vma;
1872 unsigned long start;
1873
1874 addr &= PAGE_MASK;
1875 vma = find_vma(mm,addr);
1876 if (!vma)
1877 return NULL;
1878 if (vma->vm_start <= addr)
1879 return vma;
1880 if (!(vma->vm_flags & VM_GROWSDOWN))
1881 return NULL;
1882 start = vma->vm_start;
1883 if (expand_stack(vma, addr))
1884 return NULL;
1885 if (vma->vm_flags & VM_LOCKED) {
1886 mlock_vma_pages_range(vma, addr, start);
1887 }
1888 return vma;
1889}
1890#endif
1891
1892
1893
1894
1895
1896
1897
1898static void remove_vma_list(struct mm_struct *mm, struct vm_area_struct *vma)
1899{
1900
1901 update_hiwater_vm(mm);
1902 do {
1903 long nrpages = vma_pages(vma);
1904
1905 mm->total_vm -= nrpages;
1906 vm_stat_account(mm, vma->vm_flags, vma->vm_file, -nrpages);
1907 vma = remove_vma(vma);
1908 } while (vma);
1909 validate_mm(mm);
1910}
1911
1912
1913
1914
1915
1916
1917static void unmap_region(struct mm_struct *mm,
1918 struct vm_area_struct *vma, struct vm_area_struct *prev,
1919 unsigned long start, unsigned long end)
1920{
1921 struct vm_area_struct *next = prev? prev->vm_next: mm->mmap;
1922 struct mmu_gather *tlb;
1923 unsigned long nr_accounted = 0;
1924
1925 lru_add_drain();
1926 tlb = tlb_gather_mmu(mm, 0);
1927 update_hiwater_rss(mm);
1928 unmap_vmas(&tlb, vma, start, end, &nr_accounted, NULL);
1929 vm_unacct_memory(nr_accounted);
1930 free_pgtables(tlb, vma, prev? prev->vm_end: FIRST_USER_ADDRESS,
1931 next? next->vm_start: 0);
1932 tlb_finish_mmu(tlb, start, end);
1933}
1934
1935
1936
1937
1938
1939static void
1940detach_vmas_to_be_unmapped(struct mm_struct *mm, struct vm_area_struct *vma,
1941 struct vm_area_struct *prev, unsigned long end)
1942{
1943 struct vm_area_struct **insertion_point;
1944 struct vm_area_struct *tail_vma = NULL;
1945 unsigned long addr;
1946
1947 insertion_point = (prev ? &prev->vm_next : &mm->mmap);
1948 vma->vm_prev = NULL;
1949 do {
1950 rb_erase(&vma->vm_rb, &mm->mm_rb);
1951 mm->map_count--;
1952 tail_vma = vma;
1953 vma = vma->vm_next;
1954 } while (vma && vma->vm_start < end);
1955 *insertion_point = vma;
1956 if (vma)
1957 vma->vm_prev = prev;
1958 tail_vma->vm_next = NULL;
1959 if (mm->unmap_area == arch_unmap_area)
1960 addr = prev ? prev->vm_end : mm->mmap_base;
1961 else
1962 addr = vma ? vma->vm_start : mm->mmap_base;
1963 mm->unmap_area(mm, addr);
1964 mm->mmap_cache = NULL;
1965}
1966
1967
1968
1969
1970
1971static int __split_vma(struct mm_struct * mm, struct vm_area_struct * vma,
1972 unsigned long addr, int new_below)
1973{
1974 struct mempolicy *pol;
1975 struct vm_area_struct *new;
1976 int err = -ENOMEM;
1977
1978 if (is_vm_hugetlb_page(vma) && (addr &
1979 ~(huge_page_mask(hstate_vma(vma)))))
1980 return -EINVAL;
1981
1982 new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
1983 if (!new)
1984 goto out_err;
1985
1986
1987 *new = *vma;
1988
1989 INIT_LIST_HEAD(&new->anon_vma_chain);
1990
1991 if (new_below)
1992 new->vm_end = addr;
1993 else {
1994 new->vm_start = addr;
1995 new->vm_pgoff += ((addr - vma->vm_start) >> PAGE_SHIFT);
1996 }
1997
1998 pol = mpol_dup(vma_policy(vma));
1999 if (IS_ERR(pol)) {
2000 err = PTR_ERR(pol);
2001 goto out_free_vma;
2002 }
2003 vma_set_policy(new, pol);
2004
2005 if (anon_vma_clone(new, vma))
2006 goto out_free_mpol;
2007
2008 if (new->vm_file) {
2009 get_file(new->vm_file);
2010 if (vma->vm_flags & VM_EXECUTABLE)
2011 added_exe_file_vma(mm);
2012 }
2013
2014 if (new->vm_ops && new->vm_ops->open)
2015 new->vm_ops->open(new);
2016
2017 if (new_below)
2018 err = vma_adjust(vma, addr, vma->vm_end, vma->vm_pgoff +
2019 ((addr - new->vm_start) >> PAGE_SHIFT), new);
2020 else
2021 err = vma_adjust(vma, vma->vm_start, addr, vma->vm_pgoff, new);
2022
2023
2024 if (!err)
2025 return 0;
2026
2027
2028 if (new->vm_ops && new->vm_ops->close)
2029 new->vm_ops->close(new);
2030 if (new->vm_file) {
2031 if (vma->vm_flags & VM_EXECUTABLE)
2032 removed_exe_file_vma(mm);
2033 fput(new->vm_file);
2034 }
2035 unlink_anon_vmas(new);
2036 out_free_mpol:
2037 mpol_put(pol);
2038 out_free_vma:
2039 kmem_cache_free(vm_area_cachep, new);
2040 out_err:
2041 return err;
2042}
2043
2044
2045
2046
2047
2048int split_vma(struct mm_struct *mm, struct vm_area_struct *vma,
2049 unsigned long addr, int new_below)
2050{
2051 if (mm->map_count >= sysctl_max_map_count)
2052 return -ENOMEM;
2053
2054 return __split_vma(mm, vma, addr, new_below);
2055}
2056
2057
2058
2059
2060
2061
2062int do_munmap(struct mm_struct *mm, unsigned long start, size_t len)
2063{
2064 unsigned long end;
2065 struct vm_area_struct *vma, *prev, *last;
2066
2067 if ((start & ~PAGE_MASK) || start > TASK_SIZE || len > TASK_SIZE-start)
2068 return -EINVAL;
2069
2070 if ((len = PAGE_ALIGN(len)) == 0)
2071 return -EINVAL;
2072
2073
2074 vma = find_vma_prev(mm, start, &prev);
2075 if (!vma)
2076 return 0;
2077
2078
2079
2080 end = start + len;
2081 if (vma->vm_start >= end)
2082 return 0;
2083
2084
2085
2086
2087
2088
2089
2090
2091 if (start > vma->vm_start) {
2092 int error;
2093
2094
2095
2096
2097
2098
2099 if (end < vma->vm_end && mm->map_count >= sysctl_max_map_count)
2100 return -ENOMEM;
2101
2102 error = __split_vma(mm, vma, start, 0);
2103 if (error)
2104 return error;
2105 prev = vma;
2106 }
2107
2108
2109 last = find_vma(mm, end);
2110 if (last && end > last->vm_start) {
2111 int error = __split_vma(mm, last, end, 1);
2112 if (error)
2113 return error;
2114 }
2115 vma = prev? prev->vm_next: mm->mmap;
2116
2117
2118
2119
2120 if (mm->locked_vm) {
2121 struct vm_area_struct *tmp = vma;
2122 while (tmp && tmp->vm_start < end) {
2123 if (tmp->vm_flags & VM_LOCKED) {
2124 mm->locked_vm -= vma_pages(tmp);
2125 munlock_vma_pages_all(tmp);
2126 }
2127 tmp = tmp->vm_next;
2128 }
2129 }
2130
2131
2132
2133
2134 detach_vmas_to_be_unmapped(mm, vma, prev, end);
2135 unmap_region(mm, vma, prev, start, end);
2136
2137
2138 remove_vma_list(mm, vma);
2139
2140 return 0;
2141}
2142
2143EXPORT_SYMBOL(do_munmap);
2144
2145SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len)
2146{
2147 int ret;
2148 struct mm_struct *mm = current->mm;
2149
2150 profile_munmap(addr);
2151
2152 down_write(&mm->mmap_sem);
2153 ret = do_munmap(mm, addr, len);
2154 up_write(&mm->mmap_sem);
2155 return ret;
2156}
2157
2158static inline void verify_mm_writelocked(struct mm_struct *mm)
2159{
2160#ifdef CONFIG_DEBUG_VM
2161 if (unlikely(down_read_trylock(&mm->mmap_sem))) {
2162 WARN_ON(1);
2163 up_read(&mm->mmap_sem);
2164 }
2165#endif
2166}
2167
2168
2169
2170
2171
2172
2173unsigned long do_brk(unsigned long addr, unsigned long len)
2174{
2175 struct mm_struct * mm = current->mm;
2176 struct vm_area_struct * vma, * prev;
2177 unsigned long flags;
2178 struct rb_node ** rb_link, * rb_parent;
2179 pgoff_t pgoff = addr >> PAGE_SHIFT;
2180 int error;
2181
2182 len = PAGE_ALIGN(len);
2183 if (!len)
2184 return addr;
2185
2186 error = security_file_mmap(NULL, 0, 0, 0, addr, 1);
2187 if (error)
2188 return error;
2189
2190 flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags;
2191
2192 error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED);
2193 if (error & ~PAGE_MASK)
2194 return error;
2195
2196
2197
2198
2199 if (mm->def_flags & VM_LOCKED) {
2200 unsigned long locked, lock_limit;
2201 locked = len >> PAGE_SHIFT;
2202 locked += mm->locked_vm;
2203 lock_limit = rlimit(RLIMIT_MEMLOCK);
2204 lock_limit >>= PAGE_SHIFT;
2205 if (locked > lock_limit && !capable(CAP_IPC_LOCK))
2206 return -EAGAIN;
2207 }
2208
2209
2210
2211
2212
2213 verify_mm_writelocked(mm);
2214
2215
2216
2217
2218 munmap_back:
2219 vma = find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
2220 if (vma && vma->vm_start < addr + len) {
2221 if (do_munmap(mm, addr, len))
2222 return -ENOMEM;
2223 goto munmap_back;
2224 }
2225
2226
2227 if (!may_expand_vm(mm, len >> PAGE_SHIFT))
2228 return -ENOMEM;
2229
2230 if (mm->map_count > sysctl_max_map_count)
2231 return -ENOMEM;
2232
2233 if (security_vm_enough_memory(len >> PAGE_SHIFT))
2234 return -ENOMEM;
2235
2236
2237 vma = vma_merge(mm, prev, addr, addr + len, flags,
2238 NULL, NULL, pgoff, NULL);
2239 if (vma)
2240 goto out;
2241
2242
2243
2244
2245 vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
2246 if (!vma) {
2247 vm_unacct_memory(len >> PAGE_SHIFT);
2248 return -ENOMEM;
2249 }
2250
2251 INIT_LIST_HEAD(&vma->anon_vma_chain);
2252 vma->vm_mm = mm;
2253 vma->vm_start = addr;
2254 vma->vm_end = addr + len;
2255 vma->vm_pgoff = pgoff;
2256 vma->vm_flags = flags;
2257 vma->vm_page_prot = vm_get_page_prot(flags);
2258 vma_link(mm, vma, prev, rb_link, rb_parent);
2259out:
2260 perf_event_mmap(vma);
2261 mm->total_vm += len >> PAGE_SHIFT;
2262 if (flags & VM_LOCKED) {
2263 if (!mlock_vma_pages_range(vma, addr, addr + len))
2264 mm->locked_vm += (len >> PAGE_SHIFT);
2265 }
2266 return addr;
2267}
2268
2269EXPORT_SYMBOL(do_brk);
2270
2271
2272void exit_mmap(struct mm_struct *mm)
2273{
2274 struct mmu_gather *tlb;
2275 struct vm_area_struct *vma;
2276 unsigned long nr_accounted = 0;
2277 unsigned long end;
2278
2279
2280 mmu_notifier_release(mm);
2281
2282 if (mm->locked_vm) {
2283 vma = mm->mmap;
2284 while (vma) {
2285 if (vma->vm_flags & VM_LOCKED)
2286 munlock_vma_pages_all(vma);
2287 vma = vma->vm_next;
2288 }
2289 }
2290
2291 arch_exit_mmap(mm);
2292
2293 vma = mm->mmap;
2294 if (!vma)
2295 return;
2296
2297 lru_add_drain();
2298 flush_cache_mm(mm);
2299 tlb = tlb_gather_mmu(mm, 1);
2300
2301
2302 end = unmap_vmas(&tlb, vma, 0, -1, &nr_accounted, NULL);
2303 vm_unacct_memory(nr_accounted);
2304
2305 free_pgtables(tlb, vma, FIRST_USER_ADDRESS, 0);
2306 tlb_finish_mmu(tlb, 0, end);
2307
2308
2309
2310
2311
2312 while (vma)
2313 vma = remove_vma(vma);
2314
2315 BUG_ON(mm->nr_ptes > (FIRST_USER_ADDRESS+PMD_SIZE-1)>>PMD_SHIFT);
2316}
2317
2318
2319
2320
2321
2322int insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma)
2323{
2324 struct vm_area_struct * __vma, * prev;
2325 struct rb_node ** rb_link, * rb_parent;
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339 if (!vma->vm_file) {
2340 BUG_ON(vma->anon_vma);
2341 vma->vm_pgoff = vma->vm_start >> PAGE_SHIFT;
2342 }
2343 __vma = find_vma_prepare(mm,vma->vm_start,&prev,&rb_link,&rb_parent);
2344 if (__vma && __vma->vm_start < vma->vm_end)
2345 return -ENOMEM;
2346 if ((vma->vm_flags & VM_ACCOUNT) &&
2347 security_vm_enough_memory_mm(mm, vma_pages(vma)))
2348 return -ENOMEM;
2349 vma_link(mm, vma, prev, rb_link, rb_parent);
2350 return 0;
2351}
2352
2353
2354
2355
2356
2357struct vm_area_struct *copy_vma(struct vm_area_struct **vmap,
2358 unsigned long addr, unsigned long len, pgoff_t pgoff)
2359{
2360 struct vm_area_struct *vma = *vmap;
2361 unsigned long vma_start = vma->vm_start;
2362 struct mm_struct *mm = vma->vm_mm;
2363 struct vm_area_struct *new_vma, *prev;
2364 struct rb_node **rb_link, *rb_parent;
2365 struct mempolicy *pol;
2366
2367
2368
2369
2370
2371 if (!vma->vm_file && !vma->anon_vma)
2372 pgoff = addr >> PAGE_SHIFT;
2373
2374 find_vma_prepare(mm, addr, &prev, &rb_link, &rb_parent);
2375 new_vma = vma_merge(mm, prev, addr, addr + len, vma->vm_flags,
2376 vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma));
2377 if (new_vma) {
2378
2379
2380
2381 if (vma_start >= new_vma->vm_start &&
2382 vma_start < new_vma->vm_end)
2383 *vmap = new_vma;
2384 } else {
2385 new_vma = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL);
2386 if (new_vma) {
2387 *new_vma = *vma;
2388 pol = mpol_dup(vma_policy(vma));
2389 if (IS_ERR(pol))
2390 goto out_free_vma;
2391 INIT_LIST_HEAD(&new_vma->anon_vma_chain);
2392 if (anon_vma_clone(new_vma, vma))
2393 goto out_free_mempol;
2394 vma_set_policy(new_vma, pol);
2395 new_vma->vm_start = addr;
2396 new_vma->vm_end = addr + len;
2397 new_vma->vm_pgoff = pgoff;
2398 if (new_vma->vm_file) {
2399 get_file(new_vma->vm_file);
2400 if (vma->vm_flags & VM_EXECUTABLE)
2401 added_exe_file_vma(mm);
2402 }
2403 if (new_vma->vm_ops && new_vma->vm_ops->open)
2404 new_vma->vm_ops->open(new_vma);
2405 vma_link(mm, new_vma, prev, rb_link, rb_parent);
2406 }
2407 }
2408 return new_vma;
2409
2410 out_free_mempol:
2411 mpol_put(pol);
2412 out_free_vma:
2413 kmem_cache_free(vm_area_cachep, new_vma);
2414 return NULL;
2415}
2416
2417
2418
2419
2420
2421int may_expand_vm(struct mm_struct *mm, unsigned long npages)
2422{
2423 unsigned long cur = mm->total_vm;
2424 unsigned long lim;
2425
2426 lim = rlimit(RLIMIT_AS) >> PAGE_SHIFT;
2427
2428 if (cur + npages > lim)
2429 return 0;
2430 return 1;
2431}
2432
2433
2434static int special_mapping_fault(struct vm_area_struct *vma,
2435 struct vm_fault *vmf)
2436{
2437 pgoff_t pgoff;
2438 struct page **pages;
2439
2440
2441
2442
2443
2444
2445
2446 pgoff = vmf->pgoff - vma->vm_pgoff;
2447
2448 for (pages = vma->vm_private_data; pgoff && *pages; ++pages)
2449 pgoff--;
2450
2451 if (*pages) {
2452 struct page *page = *pages;
2453 get_page(page);
2454 vmf->page = page;
2455 return 0;
2456 }
2457
2458 return VM_FAULT_SIGBUS;
2459}
2460
2461
2462
2463
2464static void special_mapping_close(struct vm_area_struct *vma)
2465{
2466}
2467
2468static const struct vm_operations_struct special_mapping_vmops = {
2469 .close = special_mapping_close,
2470 .fault = special_mapping_fault,
2471};
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482int install_special_mapping(struct mm_struct *mm,
2483 unsigned long addr, unsigned long len,
2484 unsigned long vm_flags, struct page **pages)
2485{
2486 int ret;
2487 struct vm_area_struct *vma;
2488
2489 vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL);
2490 if (unlikely(vma == NULL))
2491 return -ENOMEM;
2492
2493 INIT_LIST_HEAD(&vma->anon_vma_chain);
2494 vma->vm_mm = mm;
2495 vma->vm_start = addr;
2496 vma->vm_end = addr + len;
2497
2498 vma->vm_flags = vm_flags | mm->def_flags | VM_DONTEXPAND;
2499 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
2500
2501 vma->vm_ops = &special_mapping_vmops;
2502 vma->vm_private_data = pages;
2503
2504 ret = security_file_mmap(NULL, 0, 0, 0, vma->vm_start, 1);
2505 if (ret)
2506 goto out;
2507
2508 ret = insert_vm_struct(mm, vma);
2509 if (ret)
2510 goto out;
2511
2512 mm->total_vm += len >> PAGE_SHIFT;
2513
2514 perf_event_mmap(vma);
2515
2516 return 0;
2517
2518out:
2519 kmem_cache_free(vm_area_cachep, vma);
2520 return ret;
2521}
2522
2523static DEFINE_MUTEX(mm_all_locks_mutex);
2524
2525static void vm_lock_anon_vma(struct mm_struct *mm, struct anon_vma *anon_vma)
2526{
2527 if (!test_bit(0, (unsigned long *) &anon_vma->root->head.next)) {
2528
2529
2530
2531
2532 spin_lock_nest_lock(&anon_vma->root->lock, &mm->mmap_sem);
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542 if (__test_and_set_bit(0, (unsigned long *)
2543 &anon_vma->root->head.next))
2544 BUG();
2545 }
2546}
2547
2548static void vm_lock_mapping(struct mm_struct *mm, struct address_space *mapping)
2549{
2550 if (!test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560 if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
2561 BUG();
2562 spin_lock_nest_lock(&mapping->i_mmap_lock, &mm->mmap_sem);
2563 }
2564}
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598int mm_take_all_locks(struct mm_struct *mm)
2599{
2600 struct vm_area_struct *vma;
2601 struct anon_vma_chain *avc;
2602 int ret = -EINTR;
2603
2604 BUG_ON(down_read_trylock(&mm->mmap_sem));
2605
2606 mutex_lock(&mm_all_locks_mutex);
2607
2608 for (vma = mm->mmap; vma; vma = vma->vm_next) {
2609 if (signal_pending(current))
2610 goto out_unlock;
2611 if (vma->vm_file && vma->vm_file->f_mapping)
2612 vm_lock_mapping(mm, vma->vm_file->f_mapping);
2613 }
2614
2615 for (vma = mm->mmap; vma; vma = vma->vm_next) {
2616 if (signal_pending(current))
2617 goto out_unlock;
2618 if (vma->anon_vma)
2619 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
2620 vm_lock_anon_vma(mm, avc->anon_vma);
2621 }
2622
2623 ret = 0;
2624
2625out_unlock:
2626 if (ret)
2627 mm_drop_all_locks(mm);
2628
2629 return ret;
2630}
2631
2632static void vm_unlock_anon_vma(struct anon_vma *anon_vma)
2633{
2634 if (test_bit(0, (unsigned long *) &anon_vma->root->head.next)) {
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647 if (!__test_and_clear_bit(0, (unsigned long *)
2648 &anon_vma->root->head.next))
2649 BUG();
2650 anon_vma_unlock(anon_vma);
2651 }
2652}
2653
2654static void vm_unlock_mapping(struct address_space *mapping)
2655{
2656 if (test_bit(AS_MM_ALL_LOCKS, &mapping->flags)) {
2657
2658
2659
2660
2661 spin_unlock(&mapping->i_mmap_lock);
2662 if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
2663 &mapping->flags))
2664 BUG();
2665 }
2666}
2667
2668
2669
2670
2671
2672void mm_drop_all_locks(struct mm_struct *mm)
2673{
2674 struct vm_area_struct *vma;
2675 struct anon_vma_chain *avc;
2676
2677 BUG_ON(down_read_trylock(&mm->mmap_sem));
2678 BUG_ON(!mutex_is_locked(&mm_all_locks_mutex));
2679
2680 for (vma = mm->mmap; vma; vma = vma->vm_next) {
2681 if (vma->anon_vma)
2682 list_for_each_entry(avc, &vma->anon_vma_chain, same_vma)
2683 vm_unlock_anon_vma(avc->anon_vma);
2684 if (vma->vm_file && vma->vm_file->f_mapping)
2685 vm_unlock_mapping(vma->vm_file->f_mapping);
2686 }
2687
2688 mutex_unlock(&mm_all_locks_mutex);
2689}
2690
2691
2692
2693
2694void __init mmap_init(void)
2695{
2696 int ret;
2697
2698 ret = percpu_counter_init(&vm_committed_as, 0);
2699 VM_BUG_ON(ret);
2700}
2701