1
2#include <linux/mm.h>
3#include <linux/vmacache.h>
4#include <linux/hugetlb.h>
5#include <linux/huge_mm.h>
6#include <linux/mount.h>
7#include <linux/seq_file.h>
8#include <linux/highmem.h>
9#include <linux/ptrace.h>
10#include <linux/slab.h>
11#include <linux/pagemap.h>
12#include <linux/mempolicy.h>
13#include <linux/rmap.h>
14#include <linux/swap.h>
15#include <linux/sched/mm.h>
16#include <linux/swapops.h>
17#include <linux/mmu_notifier.h>
18#include <linux/page_idle.h>
19#include <linux/shmem_fs.h>
20#include <linux/uaccess.h>
21#include <linux/pkeys.h>
22
23#include <asm/elf.h>
24#include <asm/tlb.h>
25#include <asm/tlbflush.h>
26#include "internal.h"
27
28#define SEQ_PUT_DEC(str, val) \
29 seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8)
30void task_mem(struct seq_file *m, struct mm_struct *mm)
31{
32 unsigned long text, lib, swap, anon, file, shmem;
33 unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
34
35 anon = get_mm_counter(mm, MM_ANONPAGES);
36 file = get_mm_counter(mm, MM_FILEPAGES);
37 shmem = get_mm_counter(mm, MM_SHMEMPAGES);
38
39
40
41
42
43
44
45
46 hiwater_vm = total_vm = mm->total_vm;
47 if (hiwater_vm < mm->hiwater_vm)
48 hiwater_vm = mm->hiwater_vm;
49 hiwater_rss = total_rss = anon + file + shmem;
50 if (hiwater_rss < mm->hiwater_rss)
51 hiwater_rss = mm->hiwater_rss;
52
53
54 text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK);
55 text = min(text, mm->exec_vm << PAGE_SHIFT);
56 lib = (mm->exec_vm << PAGE_SHIFT) - text;
57
58 swap = get_mm_counter(mm, MM_SWAPENTS);
59 SEQ_PUT_DEC("VmPeak:\t", hiwater_vm);
60 SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm);
61 SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm);
62 SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm));
63 SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss);
64 SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss);
65 SEQ_PUT_DEC(" kB\nRssAnon:\t", anon);
66 SEQ_PUT_DEC(" kB\nRssFile:\t", file);
67 SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem);
68 SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm);
69 SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm);
70 seq_put_decimal_ull_width(m,
71 " kB\nVmExe:\t", text >> 10, 8);
72 seq_put_decimal_ull_width(m,
73 " kB\nVmLib:\t", lib >> 10, 8);
74 seq_put_decimal_ull_width(m,
75 " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8);
76 SEQ_PUT_DEC(" kB\nVmSwap:\t", swap);
77 seq_puts(m, " kB\n");
78 hugetlb_report_usage(m, mm);
79}
80#undef SEQ_PUT_DEC
81
82unsigned long task_vsize(struct mm_struct *mm)
83{
84 return PAGE_SIZE * mm->total_vm;
85}
86
87unsigned long task_statm(struct mm_struct *mm,
88 unsigned long *shared, unsigned long *text,
89 unsigned long *data, unsigned long *resident)
90{
91 *shared = get_mm_counter(mm, MM_FILEPAGES) +
92 get_mm_counter(mm, MM_SHMEMPAGES);
93 *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
94 >> PAGE_SHIFT;
95 *data = mm->data_vm + mm->stack_vm;
96 *resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
97 return mm->total_vm;
98}
99
100#ifdef CONFIG_NUMA
101
102
103
104static void hold_task_mempolicy(struct proc_maps_private *priv)
105{
106 struct task_struct *task = priv->task;
107
108 task_lock(task);
109 priv->task_mempolicy = get_task_policy(task);
110 mpol_get(priv->task_mempolicy);
111 task_unlock(task);
112}
113static void release_task_mempolicy(struct proc_maps_private *priv)
114{
115 mpol_put(priv->task_mempolicy);
116}
117#else
118static void hold_task_mempolicy(struct proc_maps_private *priv)
119{
120}
121static void release_task_mempolicy(struct proc_maps_private *priv)
122{
123}
124#endif
125
126static void vma_stop(struct proc_maps_private *priv)
127{
128 struct mm_struct *mm = priv->mm;
129
130 release_task_mempolicy(priv);
131 up_read(&mm->mmap_sem);
132 mmput(mm);
133}
134
135static struct vm_area_struct *
136m_next_vma(struct proc_maps_private *priv, struct vm_area_struct *vma)
137{
138 if (vma == priv->tail_vma)
139 return NULL;
140 return vma->vm_next ?: priv->tail_vma;
141}
142
143static void m_cache_vma(struct seq_file *m, struct vm_area_struct *vma)
144{
145 if (m->count < m->size)
146 m->version = m_next_vma(m->private, vma) ? vma->vm_end : -1UL;
147}
148
149static void *m_start(struct seq_file *m, loff_t *ppos)
150{
151 struct proc_maps_private *priv = m->private;
152 unsigned long last_addr = m->version;
153 struct mm_struct *mm;
154 struct vm_area_struct *vma;
155 unsigned int pos = *ppos;
156
157
158 if (last_addr == -1UL)
159 return NULL;
160
161 priv->task = get_proc_task(priv->inode);
162 if (!priv->task)
163 return ERR_PTR(-ESRCH);
164
165 mm = priv->mm;
166 if (!mm || !mmget_not_zero(mm))
167 return NULL;
168
169 if (down_read_killable(&mm->mmap_sem)) {
170 mmput(mm);
171 return ERR_PTR(-EINTR);
172 }
173
174 hold_task_mempolicy(priv);
175 priv->tail_vma = get_gate_vma(mm);
176
177 if (last_addr) {
178 vma = find_vma(mm, last_addr - 1);
179 if (vma && vma->vm_start <= last_addr)
180 vma = m_next_vma(priv, vma);
181 if (vma)
182 return vma;
183 }
184
185 m->version = 0;
186 if (pos < mm->map_count) {
187 for (vma = mm->mmap; pos; pos--) {
188 m->version = vma->vm_start;
189 vma = vma->vm_next;
190 }
191 return vma;
192 }
193
194
195 if (pos == mm->map_count && priv->tail_vma)
196 return priv->tail_vma;
197
198 vma_stop(priv);
199 return NULL;
200}
201
202static void *m_next(struct seq_file *m, void *v, loff_t *pos)
203{
204 struct proc_maps_private *priv = m->private;
205 struct vm_area_struct *next;
206
207 (*pos)++;
208 next = m_next_vma(priv, v);
209 if (!next)
210 vma_stop(priv);
211 return next;
212}
213
214static void m_stop(struct seq_file *m, void *v)
215{
216 struct proc_maps_private *priv = m->private;
217
218 if (!IS_ERR_OR_NULL(v))
219 vma_stop(priv);
220 if (priv->task) {
221 put_task_struct(priv->task);
222 priv->task = NULL;
223 }
224}
225
226static int proc_maps_open(struct inode *inode, struct file *file,
227 const struct seq_operations *ops, int psize)
228{
229 struct proc_maps_private *priv = __seq_open_private(file, ops, psize);
230
231 if (!priv)
232 return -ENOMEM;
233
234 priv->inode = inode;
235 priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
236 if (IS_ERR(priv->mm)) {
237 int err = PTR_ERR(priv->mm);
238
239 seq_release_private(inode, file);
240 return err;
241 }
242
243 return 0;
244}
245
246static int proc_map_release(struct inode *inode, struct file *file)
247{
248 struct seq_file *seq = file->private_data;
249 struct proc_maps_private *priv = seq->private;
250
251 if (priv->mm)
252 mmdrop(priv->mm);
253
254 return seq_release_private(inode, file);
255}
256
257static int do_maps_open(struct inode *inode, struct file *file,
258 const struct seq_operations *ops)
259{
260 return proc_maps_open(inode, file, ops,
261 sizeof(struct proc_maps_private));
262}
263
264
265
266
267
268static int is_stack(struct vm_area_struct *vma)
269{
270
271
272
273
274
275 return vma->vm_start <= vma->vm_mm->start_stack &&
276 vma->vm_end >= vma->vm_mm->start_stack;
277}
278
279static void show_vma_header_prefix(struct seq_file *m,
280 unsigned long start, unsigned long end,
281 vm_flags_t flags, unsigned long long pgoff,
282 dev_t dev, unsigned long ino)
283{
284 seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
285 seq_put_hex_ll(m, NULL, start, 8);
286 seq_put_hex_ll(m, "-", end, 8);
287 seq_putc(m, ' ');
288 seq_putc(m, flags & VM_READ ? 'r' : '-');
289 seq_putc(m, flags & VM_WRITE ? 'w' : '-');
290 seq_putc(m, flags & VM_EXEC ? 'x' : '-');
291 seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p');
292 seq_put_hex_ll(m, " ", pgoff, 8);
293 seq_put_hex_ll(m, " ", MAJOR(dev), 2);
294 seq_put_hex_ll(m, ":", MINOR(dev), 2);
295 seq_put_decimal_ull(m, " ", ino);
296 seq_putc(m, ' ');
297}
298
299static void
300show_map_vma(struct seq_file *m, struct vm_area_struct *vma)
301{
302 struct mm_struct *mm = vma->vm_mm;
303 struct file *file = vma->vm_file;
304 vm_flags_t flags = vma->vm_flags;
305 unsigned long ino = 0;
306 unsigned long long pgoff = 0;
307 unsigned long start, end;
308 dev_t dev = 0;
309 const char *name = NULL;
310
311 if (file) {
312 struct inode *inode = file_inode(vma->vm_file);
313 dev = inode->i_sb->s_dev;
314 ino = inode->i_ino;
315 pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
316 }
317
318 start = vma->vm_start;
319 end = vma->vm_end;
320 show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino);
321
322
323
324
325
326 if (file) {
327 seq_pad(m, ' ');
328 seq_file_path(m, file, "\n");
329 goto done;
330 }
331
332 if (vma->vm_ops && vma->vm_ops->name) {
333 name = vma->vm_ops->name(vma);
334 if (name)
335 goto done;
336 }
337
338 name = arch_vma_name(vma);
339 if (!name) {
340 if (!mm) {
341 name = "[vdso]";
342 goto done;
343 }
344
345 if (vma->vm_start <= mm->brk &&
346 vma->vm_end >= mm->start_brk) {
347 name = "[heap]";
348 goto done;
349 }
350
351 if (is_stack(vma))
352 name = "[stack]";
353 }
354
355done:
356 if (name) {
357 seq_pad(m, ' ');
358 seq_puts(m, name);
359 }
360 seq_putc(m, '\n');
361}
362
363static int show_map(struct seq_file *m, void *v)
364{
365 show_map_vma(m, v);
366 m_cache_vma(m, v);
367 return 0;
368}
369
370static const struct seq_operations proc_pid_maps_op = {
371 .start = m_start,
372 .next = m_next,
373 .stop = m_stop,
374 .show = show_map
375};
376
377static int pid_maps_open(struct inode *inode, struct file *file)
378{
379 return do_maps_open(inode, file, &proc_pid_maps_op);
380}
381
382const struct file_operations proc_pid_maps_operations = {
383 .open = pid_maps_open,
384 .read = seq_read,
385 .llseek = seq_lseek,
386 .release = proc_map_release,
387};
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406#define PSS_SHIFT 12
407
408#ifdef CONFIG_PROC_PAGE_MONITOR
409struct mem_size_stats {
410 unsigned long resident;
411 unsigned long shared_clean;
412 unsigned long shared_dirty;
413 unsigned long private_clean;
414 unsigned long private_dirty;
415 unsigned long referenced;
416 unsigned long anonymous;
417 unsigned long lazyfree;
418 unsigned long anonymous_thp;
419 unsigned long shmem_thp;
420 unsigned long swap;
421 unsigned long shared_hugetlb;
422 unsigned long private_hugetlb;
423 u64 pss;
424 u64 pss_anon;
425 u64 pss_file;
426 u64 pss_shmem;
427 u64 pss_locked;
428 u64 swap_pss;
429 bool check_shmem_swap;
430};
431
432static void smaps_page_accumulate(struct mem_size_stats *mss,
433 struct page *page, unsigned long size, unsigned long pss,
434 bool dirty, bool locked, bool private)
435{
436 mss->pss += pss;
437
438 if (PageAnon(page))
439 mss->pss_anon += pss;
440 else if (PageSwapBacked(page))
441 mss->pss_shmem += pss;
442 else
443 mss->pss_file += pss;
444
445 if (locked)
446 mss->pss_locked += pss;
447
448 if (dirty || PageDirty(page)) {
449 if (private)
450 mss->private_dirty += size;
451 else
452 mss->shared_dirty += size;
453 } else {
454 if (private)
455 mss->private_clean += size;
456 else
457 mss->shared_clean += size;
458 }
459}
460
461static void smaps_account(struct mem_size_stats *mss, struct page *page,
462 bool compound, bool young, bool dirty, bool locked)
463{
464 int i, nr = compound ? 1 << compound_order(page) : 1;
465 unsigned long size = nr * PAGE_SIZE;
466
467
468
469
470
471 if (PageAnon(page)) {
472 mss->anonymous += size;
473 if (!PageSwapBacked(page) && !dirty && !PageDirty(page))
474 mss->lazyfree += size;
475 }
476
477 mss->resident += size;
478
479 if (young || page_is_young(page) || PageReferenced(page))
480 mss->referenced += size;
481
482
483
484
485
486
487
488
489
490 if (page_count(page) == 1) {
491 smaps_page_accumulate(mss, page, size, size << PSS_SHIFT, dirty,
492 locked, true);
493 return;
494 }
495 for (i = 0; i < nr; i++, page++) {
496 int mapcount = page_mapcount(page);
497 unsigned long pss = PAGE_SIZE << PSS_SHIFT;
498 if (mapcount >= 2)
499 pss /= mapcount;
500 smaps_page_accumulate(mss, page, PAGE_SIZE, pss, dirty, locked,
501 mapcount < 2);
502 }
503}
504
505#ifdef CONFIG_SHMEM
506static int smaps_pte_hole(unsigned long addr, unsigned long end,
507 struct mm_walk *walk)
508{
509 struct mem_size_stats *mss = walk->private;
510
511 mss->swap += shmem_partial_swap_usage(
512 walk->vma->vm_file->f_mapping, addr, end);
513
514 return 0;
515}
516#endif
517
518static void smaps_pte_entry(pte_t *pte, unsigned long addr,
519 struct mm_walk *walk)
520{
521 struct mem_size_stats *mss = walk->private;
522 struct vm_area_struct *vma = walk->vma;
523 bool locked = !!(vma->vm_flags & VM_LOCKED);
524 struct page *page = NULL;
525
526 if (pte_present(*pte)) {
527 page = vm_normal_page(vma, addr, *pte);
528 } else if (is_swap_pte(*pte)) {
529 swp_entry_t swpent = pte_to_swp_entry(*pte);
530
531 if (!non_swap_entry(swpent)) {
532 int mapcount;
533
534 mss->swap += PAGE_SIZE;
535 mapcount = swp_swapcount(swpent);
536 if (mapcount >= 2) {
537 u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT;
538
539 do_div(pss_delta, mapcount);
540 mss->swap_pss += pss_delta;
541 } else {
542 mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
543 }
544 } else if (is_migration_entry(swpent))
545 page = migration_entry_to_page(swpent);
546 else if (is_device_private_entry(swpent))
547 page = device_private_entry_to_page(swpent);
548 } else if (unlikely(IS_ENABLED(CONFIG_SHMEM) && mss->check_shmem_swap
549 && pte_none(*pte))) {
550 page = find_get_entry(vma->vm_file->f_mapping,
551 linear_page_index(vma, addr));
552 if (!page)
553 return;
554
555 if (xa_is_value(page))
556 mss->swap += PAGE_SIZE;
557 else
558 put_page(page);
559
560 return;
561 }
562
563 if (!page)
564 return;
565
566 smaps_account(mss, page, false, pte_young(*pte), pte_dirty(*pte), locked);
567}
568
569#ifdef CONFIG_TRANSPARENT_HUGEPAGE
570static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
571 struct mm_walk *walk)
572{
573 struct mem_size_stats *mss = walk->private;
574 struct vm_area_struct *vma = walk->vma;
575 bool locked = !!(vma->vm_flags & VM_LOCKED);
576 struct page *page;
577
578
579 page = follow_trans_huge_pmd(vma, addr, pmd, FOLL_DUMP);
580 if (IS_ERR_OR_NULL(page))
581 return;
582 if (PageAnon(page))
583 mss->anonymous_thp += HPAGE_PMD_SIZE;
584 else if (PageSwapBacked(page))
585 mss->shmem_thp += HPAGE_PMD_SIZE;
586 else if (is_zone_device_page(page))
587 ;
588 else
589 VM_BUG_ON_PAGE(1, page);
590 smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd), locked);
591}
592#else
593static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
594 struct mm_walk *walk)
595{
596}
597#endif
598
599static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
600 struct mm_walk *walk)
601{
602 struct vm_area_struct *vma = walk->vma;
603 pte_t *pte;
604 spinlock_t *ptl;
605
606 ptl = pmd_trans_huge_lock(pmd, vma);
607 if (ptl) {
608 if (pmd_present(*pmd))
609 smaps_pmd_entry(pmd, addr, walk);
610 spin_unlock(ptl);
611 goto out;
612 }
613
614 if (pmd_trans_unstable(pmd))
615 goto out;
616
617
618
619
620
621 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
622 for (; addr != end; pte++, addr += PAGE_SIZE)
623 smaps_pte_entry(pte, addr, walk);
624 pte_unmap_unlock(pte - 1, ptl);
625out:
626 cond_resched();
627 return 0;
628}
629
630static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
631{
632
633
634
635 static const char mnemonics[BITS_PER_LONG][2] = {
636
637
638
639 [0 ... (BITS_PER_LONG-1)] = "??",
640
641 [ilog2(VM_READ)] = "rd",
642 [ilog2(VM_WRITE)] = "wr",
643 [ilog2(VM_EXEC)] = "ex",
644 [ilog2(VM_SHARED)] = "sh",
645 [ilog2(VM_MAYREAD)] = "mr",
646 [ilog2(VM_MAYWRITE)] = "mw",
647 [ilog2(VM_MAYEXEC)] = "me",
648 [ilog2(VM_MAYSHARE)] = "ms",
649 [ilog2(VM_GROWSDOWN)] = "gd",
650 [ilog2(VM_PFNMAP)] = "pf",
651 [ilog2(VM_DENYWRITE)] = "dw",
652#ifdef CONFIG_X86_INTEL_MPX
653 [ilog2(VM_MPX)] = "mp",
654#endif
655 [ilog2(VM_LOCKED)] = "lo",
656 [ilog2(VM_IO)] = "io",
657 [ilog2(VM_SEQ_READ)] = "sr",
658 [ilog2(VM_RAND_READ)] = "rr",
659 [ilog2(VM_DONTCOPY)] = "dc",
660 [ilog2(VM_DONTEXPAND)] = "de",
661 [ilog2(VM_ACCOUNT)] = "ac",
662 [ilog2(VM_NORESERVE)] = "nr",
663 [ilog2(VM_HUGETLB)] = "ht",
664 [ilog2(VM_SYNC)] = "sf",
665 [ilog2(VM_ARCH_1)] = "ar",
666 [ilog2(VM_WIPEONFORK)] = "wf",
667 [ilog2(VM_DONTDUMP)] = "dd",
668#ifdef CONFIG_MEM_SOFT_DIRTY
669 [ilog2(VM_SOFTDIRTY)] = "sd",
670#endif
671 [ilog2(VM_MIXEDMAP)] = "mm",
672 [ilog2(VM_HUGEPAGE)] = "hg",
673 [ilog2(VM_NOHUGEPAGE)] = "nh",
674 [ilog2(VM_MERGEABLE)] = "mg",
675 [ilog2(VM_UFFD_MISSING)]= "um",
676 [ilog2(VM_UFFD_WP)] = "uw",
677#ifdef CONFIG_ARCH_HAS_PKEYS
678
679 [ilog2(VM_PKEY_BIT0)] = "",
680 [ilog2(VM_PKEY_BIT1)] = "",
681 [ilog2(VM_PKEY_BIT2)] = "",
682 [ilog2(VM_PKEY_BIT3)] = "",
683#if VM_PKEY_BIT4
684 [ilog2(VM_PKEY_BIT4)] = "",
685#endif
686#endif
687 };
688 size_t i;
689
690 seq_puts(m, "VmFlags: ");
691 for (i = 0; i < BITS_PER_LONG; i++) {
692 if (!mnemonics[i][0])
693 continue;
694 if (vma->vm_flags & (1UL << i)) {
695 seq_putc(m, mnemonics[i][0]);
696 seq_putc(m, mnemonics[i][1]);
697 seq_putc(m, ' ');
698 }
699 }
700 seq_putc(m, '\n');
701}
702
703#ifdef CONFIG_HUGETLB_PAGE
704static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
705 unsigned long addr, unsigned long end,
706 struct mm_walk *walk)
707{
708 struct mem_size_stats *mss = walk->private;
709 struct vm_area_struct *vma = walk->vma;
710 struct page *page = NULL;
711
712 if (pte_present(*pte)) {
713 page = vm_normal_page(vma, addr, *pte);
714 } else if (is_swap_pte(*pte)) {
715 swp_entry_t swpent = pte_to_swp_entry(*pte);
716
717 if (is_migration_entry(swpent))
718 page = migration_entry_to_page(swpent);
719 else if (is_device_private_entry(swpent))
720 page = device_private_entry_to_page(swpent);
721 }
722 if (page) {
723 int mapcount = page_mapcount(page);
724
725 if (mapcount >= 2)
726 mss->shared_hugetlb += huge_page_size(hstate_vma(vma));
727 else
728 mss->private_hugetlb += huge_page_size(hstate_vma(vma));
729 }
730 return 0;
731}
732#endif
733
734static void smap_gather_stats(struct vm_area_struct *vma,
735 struct mem_size_stats *mss)
736{
737 struct mm_walk smaps_walk = {
738 .pmd_entry = smaps_pte_range,
739#ifdef CONFIG_HUGETLB_PAGE
740 .hugetlb_entry = smaps_hugetlb_range,
741#endif
742 .mm = vma->vm_mm,
743 };
744
745 smaps_walk.private = mss;
746
747#ifdef CONFIG_SHMEM
748
749 mss->check_shmem_swap = false;
750 if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
751
752
753
754
755
756
757
758
759
760
761 unsigned long shmem_swapped = shmem_swap_usage(vma);
762
763 if (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
764 !(vma->vm_flags & VM_WRITE)) {
765 mss->swap += shmem_swapped;
766 } else {
767 mss->check_shmem_swap = true;
768 smaps_walk.pte_hole = smaps_pte_hole;
769 }
770 }
771#endif
772
773 walk_page_vma(vma, &smaps_walk);
774}
775
776#define SEQ_PUT_DEC(str, val) \
777 seq_put_decimal_ull_width(m, str, (val) >> 10, 8)
778
779
780static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,
781 bool rollup_mode)
782{
783 SEQ_PUT_DEC("Rss: ", mss->resident);
784 SEQ_PUT_DEC(" kB\nPss: ", mss->pss >> PSS_SHIFT);
785 if (rollup_mode) {
786
787
788
789
790 SEQ_PUT_DEC(" kB\nPss_Anon: ",
791 mss->pss_anon >> PSS_SHIFT);
792 SEQ_PUT_DEC(" kB\nPss_File: ",
793 mss->pss_file >> PSS_SHIFT);
794 SEQ_PUT_DEC(" kB\nPss_Shmem: ",
795 mss->pss_shmem >> PSS_SHIFT);
796 }
797 SEQ_PUT_DEC(" kB\nShared_Clean: ", mss->shared_clean);
798 SEQ_PUT_DEC(" kB\nShared_Dirty: ", mss->shared_dirty);
799 SEQ_PUT_DEC(" kB\nPrivate_Clean: ", mss->private_clean);
800 SEQ_PUT_DEC(" kB\nPrivate_Dirty: ", mss->private_dirty);
801 SEQ_PUT_DEC(" kB\nReferenced: ", mss->referenced);
802 SEQ_PUT_DEC(" kB\nAnonymous: ", mss->anonymous);
803 SEQ_PUT_DEC(" kB\nLazyFree: ", mss->lazyfree);
804 SEQ_PUT_DEC(" kB\nAnonHugePages: ", mss->anonymous_thp);
805 SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp);
806 SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb);
807 seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ",
808 mss->private_hugetlb >> 10, 7);
809 SEQ_PUT_DEC(" kB\nSwap: ", mss->swap);
810 SEQ_PUT_DEC(" kB\nSwapPss: ",
811 mss->swap_pss >> PSS_SHIFT);
812 SEQ_PUT_DEC(" kB\nLocked: ",
813 mss->pss_locked >> PSS_SHIFT);
814 seq_puts(m, " kB\n");
815}
816
817static int show_smap(struct seq_file *m, void *v)
818{
819 struct vm_area_struct *vma = v;
820 struct mem_size_stats mss;
821
822 memset(&mss, 0, sizeof(mss));
823
824 smap_gather_stats(vma, &mss);
825
826 show_map_vma(m, vma);
827
828 SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start);
829 SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma));
830 SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma));
831 seq_puts(m, " kB\n");
832
833 __show_smap(m, &mss, false);
834
835 seq_printf(m, "THPeligible: %d\n",
836 transparent_hugepage_enabled(vma));
837
838 if (arch_pkeys_enabled())
839 seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma));
840 show_smap_vma_flags(m, vma);
841
842 m_cache_vma(m, vma);
843
844 return 0;
845}
846
847static int show_smaps_rollup(struct seq_file *m, void *v)
848{
849 struct proc_maps_private *priv = m->private;
850 struct mem_size_stats mss;
851 struct mm_struct *mm;
852 struct vm_area_struct *vma;
853 unsigned long last_vma_end = 0;
854 int ret = 0;
855
856 priv->task = get_proc_task(priv->inode);
857 if (!priv->task)
858 return -ESRCH;
859
860 mm = priv->mm;
861 if (!mm || !mmget_not_zero(mm)) {
862 ret = -ESRCH;
863 goto out_put_task;
864 }
865
866 memset(&mss, 0, sizeof(mss));
867
868 ret = down_read_killable(&mm->mmap_sem);
869 if (ret)
870 goto out_put_mm;
871
872 hold_task_mempolicy(priv);
873
874 for (vma = priv->mm->mmap; vma; vma = vma->vm_next) {
875 smap_gather_stats(vma, &mss);
876 last_vma_end = vma->vm_end;
877 }
878
879 show_vma_header_prefix(m, priv->mm->mmap->vm_start,
880 last_vma_end, 0, 0, 0, 0);
881 seq_pad(m, ' ');
882 seq_puts(m, "[rollup]\n");
883
884 __show_smap(m, &mss, true);
885
886 release_task_mempolicy(priv);
887 up_read(&mm->mmap_sem);
888
889out_put_mm:
890 mmput(mm);
891out_put_task:
892 put_task_struct(priv->task);
893 priv->task = NULL;
894
895 return ret;
896}
897#undef SEQ_PUT_DEC
898
899static const struct seq_operations proc_pid_smaps_op = {
900 .start = m_start,
901 .next = m_next,
902 .stop = m_stop,
903 .show = show_smap
904};
905
906static int pid_smaps_open(struct inode *inode, struct file *file)
907{
908 return do_maps_open(inode, file, &proc_pid_smaps_op);
909}
910
911static int smaps_rollup_open(struct inode *inode, struct file *file)
912{
913 int ret;
914 struct proc_maps_private *priv;
915
916 priv = kzalloc(sizeof(*priv), GFP_KERNEL_ACCOUNT);
917 if (!priv)
918 return -ENOMEM;
919
920 ret = single_open(file, show_smaps_rollup, priv);
921 if (ret)
922 goto out_free;
923
924 priv->inode = inode;
925 priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
926 if (IS_ERR(priv->mm)) {
927 ret = PTR_ERR(priv->mm);
928
929 single_release(inode, file);
930 goto out_free;
931 }
932
933 return 0;
934
935out_free:
936 kfree(priv);
937 return ret;
938}
939
940static int smaps_rollup_release(struct inode *inode, struct file *file)
941{
942 struct seq_file *seq = file->private_data;
943 struct proc_maps_private *priv = seq->private;
944
945 if (priv->mm)
946 mmdrop(priv->mm);
947
948 kfree(priv);
949 return single_release(inode, file);
950}
951
952const struct file_operations proc_pid_smaps_operations = {
953 .open = pid_smaps_open,
954 .read = seq_read,
955 .llseek = seq_lseek,
956 .release = proc_map_release,
957};
958
959const struct file_operations proc_pid_smaps_rollup_operations = {
960 .open = smaps_rollup_open,
961 .read = seq_read,
962 .llseek = seq_lseek,
963 .release = smaps_rollup_release,
964};
965
966enum clear_refs_types {
967 CLEAR_REFS_ALL = 1,
968 CLEAR_REFS_ANON,
969 CLEAR_REFS_MAPPED,
970 CLEAR_REFS_SOFT_DIRTY,
971 CLEAR_REFS_MM_HIWATER_RSS,
972 CLEAR_REFS_LAST,
973};
974
975struct clear_refs_private {
976 enum clear_refs_types type;
977};
978
979#ifdef CONFIG_MEM_SOFT_DIRTY
980static inline void clear_soft_dirty(struct vm_area_struct *vma,
981 unsigned long addr, pte_t *pte)
982{
983
984
985
986
987
988
989 pte_t ptent = *pte;
990
991 if (pte_present(ptent)) {
992 pte_t old_pte;
993
994 old_pte = ptep_modify_prot_start(vma, addr, pte);
995 ptent = pte_wrprotect(old_pte);
996 ptent = pte_clear_soft_dirty(ptent);
997 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent);
998 } else if (is_swap_pte(ptent)) {
999 ptent = pte_swp_clear_soft_dirty(ptent);
1000 set_pte_at(vma->vm_mm, addr, pte, ptent);
1001 }
1002}
1003#else
1004static inline void clear_soft_dirty(struct vm_area_struct *vma,
1005 unsigned long addr, pte_t *pte)
1006{
1007}
1008#endif
1009
1010#if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
1011static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1012 unsigned long addr, pmd_t *pmdp)
1013{
1014 pmd_t old, pmd = *pmdp;
1015
1016 if (pmd_present(pmd)) {
1017
1018 old = pmdp_invalidate(vma, addr, pmdp);
1019 if (pmd_dirty(old))
1020 pmd = pmd_mkdirty(pmd);
1021 if (pmd_young(old))
1022 pmd = pmd_mkyoung(pmd);
1023
1024 pmd = pmd_wrprotect(pmd);
1025 pmd = pmd_clear_soft_dirty(pmd);
1026
1027 set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1028 } else if (is_migration_entry(pmd_to_swp_entry(pmd))) {
1029 pmd = pmd_swp_clear_soft_dirty(pmd);
1030 set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1031 }
1032}
1033#else
1034static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1035 unsigned long addr, pmd_t *pmdp)
1036{
1037}
1038#endif
1039
1040static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
1041 unsigned long end, struct mm_walk *walk)
1042{
1043 struct clear_refs_private *cp = walk->private;
1044 struct vm_area_struct *vma = walk->vma;
1045 pte_t *pte, ptent;
1046 spinlock_t *ptl;
1047 struct page *page;
1048
1049 ptl = pmd_trans_huge_lock(pmd, vma);
1050 if (ptl) {
1051 if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1052 clear_soft_dirty_pmd(vma, addr, pmd);
1053 goto out;
1054 }
1055
1056 if (!pmd_present(*pmd))
1057 goto out;
1058
1059 page = pmd_page(*pmd);
1060
1061
1062 pmdp_test_and_clear_young(vma, addr, pmd);
1063 test_and_clear_page_young(page);
1064 ClearPageReferenced(page);
1065out:
1066 spin_unlock(ptl);
1067 return 0;
1068 }
1069
1070 if (pmd_trans_unstable(pmd))
1071 return 0;
1072
1073 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1074 for (; addr != end; pte++, addr += PAGE_SIZE) {
1075 ptent = *pte;
1076
1077 if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1078 clear_soft_dirty(vma, addr, pte);
1079 continue;
1080 }
1081
1082 if (!pte_present(ptent))
1083 continue;
1084
1085 page = vm_normal_page(vma, addr, ptent);
1086 if (!page)
1087 continue;
1088
1089
1090 ptep_test_and_clear_young(vma, addr, pte);
1091 test_and_clear_page_young(page);
1092 ClearPageReferenced(page);
1093 }
1094 pte_unmap_unlock(pte - 1, ptl);
1095 cond_resched();
1096 return 0;
1097}
1098
1099static int clear_refs_test_walk(unsigned long start, unsigned long end,
1100 struct mm_walk *walk)
1101{
1102 struct clear_refs_private *cp = walk->private;
1103 struct vm_area_struct *vma = walk->vma;
1104
1105 if (vma->vm_flags & VM_PFNMAP)
1106 return 1;
1107
1108
1109
1110
1111
1112
1113
1114 if (cp->type == CLEAR_REFS_ANON && vma->vm_file)
1115 return 1;
1116 if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)
1117 return 1;
1118 return 0;
1119}
1120
1121static ssize_t clear_refs_write(struct file *file, const char __user *buf,
1122 size_t count, loff_t *ppos)
1123{
1124 struct task_struct *task;
1125 char buffer[PROC_NUMBUF];
1126 struct mm_struct *mm;
1127 struct vm_area_struct *vma;
1128 enum clear_refs_types type;
1129 struct mmu_gather tlb;
1130 int itype;
1131 int rv;
1132
1133 memset(buffer, 0, sizeof(buffer));
1134 if (count > sizeof(buffer) - 1)
1135 count = sizeof(buffer) - 1;
1136 if (copy_from_user(buffer, buf, count))
1137 return -EFAULT;
1138 rv = kstrtoint(strstrip(buffer), 10, &itype);
1139 if (rv < 0)
1140 return rv;
1141 type = (enum clear_refs_types)itype;
1142 if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
1143 return -EINVAL;
1144
1145 task = get_proc_task(file_inode(file));
1146 if (!task)
1147 return -ESRCH;
1148 mm = get_task_mm(task);
1149 if (mm) {
1150 struct mmu_notifier_range range;
1151 struct clear_refs_private cp = {
1152 .type = type,
1153 };
1154 struct mm_walk clear_refs_walk = {
1155 .pmd_entry = clear_refs_pte_range,
1156 .test_walk = clear_refs_test_walk,
1157 .mm = mm,
1158 .private = &cp,
1159 };
1160
1161 if (type == CLEAR_REFS_MM_HIWATER_RSS) {
1162 if (down_write_killable(&mm->mmap_sem)) {
1163 count = -EINTR;
1164 goto out_mm;
1165 }
1166
1167
1168
1169
1170
1171 reset_mm_hiwater_rss(mm);
1172 up_write(&mm->mmap_sem);
1173 goto out_mm;
1174 }
1175
1176 if (down_read_killable(&mm->mmap_sem)) {
1177 count = -EINTR;
1178 goto out_mm;
1179 }
1180 tlb_gather_mmu(&tlb, mm, 0, -1);
1181 if (type == CLEAR_REFS_SOFT_DIRTY) {
1182 for (vma = mm->mmap; vma; vma = vma->vm_next) {
1183 if (!(vma->vm_flags & VM_SOFTDIRTY))
1184 continue;
1185 up_read(&mm->mmap_sem);
1186 if (down_write_killable(&mm->mmap_sem)) {
1187 count = -EINTR;
1188 goto out_mm;
1189 }
1190
1191
1192
1193
1194
1195 if (!mmget_still_valid(mm)) {
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205 up_write(&mm->mmap_sem);
1206 goto out_mm;
1207 }
1208 for (vma = mm->mmap; vma; vma = vma->vm_next) {
1209 vma->vm_flags &= ~VM_SOFTDIRTY;
1210 vma_set_page_prot(vma);
1211 }
1212 downgrade_write(&mm->mmap_sem);
1213 break;
1214 }
1215
1216 mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY,
1217 0, NULL, mm, 0, -1UL);
1218 mmu_notifier_invalidate_range_start(&range);
1219 }
1220 walk_page_range(0, mm->highest_vm_end, &clear_refs_walk);
1221 if (type == CLEAR_REFS_SOFT_DIRTY)
1222 mmu_notifier_invalidate_range_end(&range);
1223 tlb_finish_mmu(&tlb, 0, -1);
1224 up_read(&mm->mmap_sem);
1225out_mm:
1226 mmput(mm);
1227 }
1228 put_task_struct(task);
1229
1230 return count;
1231}
1232
1233const struct file_operations proc_clear_refs_operations = {
1234 .write = clear_refs_write,
1235 .llseek = noop_llseek,
1236};
1237
1238typedef struct {
1239 u64 pme;
1240} pagemap_entry_t;
1241
1242struct pagemapread {
1243 int pos, len;
1244 pagemap_entry_t *buffer;
1245 bool show_pfn;
1246};
1247
1248#define PAGEMAP_WALK_SIZE (PMD_SIZE)
1249#define PAGEMAP_WALK_MASK (PMD_MASK)
1250
1251#define PM_ENTRY_BYTES sizeof(pagemap_entry_t)
1252#define PM_PFRAME_BITS 55
1253#define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0)
1254#define PM_SOFT_DIRTY BIT_ULL(55)
1255#define PM_MMAP_EXCLUSIVE BIT_ULL(56)
1256#define PM_FILE BIT_ULL(61)
1257#define PM_SWAP BIT_ULL(62)
1258#define PM_PRESENT BIT_ULL(63)
1259
1260#define PM_END_OF_BUFFER 1
1261
1262static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
1263{
1264 return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
1265}
1266
1267static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
1268 struct pagemapread *pm)
1269{
1270 pm->buffer[pm->pos++] = *pme;
1271 if (pm->pos >= pm->len)
1272 return PM_END_OF_BUFFER;
1273 return 0;
1274}
1275
1276static int pagemap_pte_hole(unsigned long start, unsigned long end,
1277 struct mm_walk *walk)
1278{
1279 struct pagemapread *pm = walk->private;
1280 unsigned long addr = start;
1281 int err = 0;
1282
1283 while (addr < end) {
1284 struct vm_area_struct *vma = find_vma(walk->mm, addr);
1285 pagemap_entry_t pme = make_pme(0, 0);
1286
1287 unsigned long hole_end;
1288
1289 if (vma)
1290 hole_end = min(end, vma->vm_start);
1291 else
1292 hole_end = end;
1293
1294 for (; addr < hole_end; addr += PAGE_SIZE) {
1295 err = add_to_pagemap(addr, &pme, pm);
1296 if (err)
1297 goto out;
1298 }
1299
1300 if (!vma)
1301 break;
1302
1303
1304 if (vma->vm_flags & VM_SOFTDIRTY)
1305 pme = make_pme(0, PM_SOFT_DIRTY);
1306 for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {
1307 err = add_to_pagemap(addr, &pme, pm);
1308 if (err)
1309 goto out;
1310 }
1311 }
1312out:
1313 return err;
1314}
1315
1316static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
1317 struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1318{
1319 u64 frame = 0, flags = 0;
1320 struct page *page = NULL;
1321
1322 if (pte_present(pte)) {
1323 if (pm->show_pfn)
1324 frame = pte_pfn(pte);
1325 flags |= PM_PRESENT;
1326 page = vm_normal_page(vma, addr, pte);
1327 if (pte_soft_dirty(pte))
1328 flags |= PM_SOFT_DIRTY;
1329 } else if (is_swap_pte(pte)) {
1330 swp_entry_t entry;
1331 if (pte_swp_soft_dirty(pte))
1332 flags |= PM_SOFT_DIRTY;
1333 entry = pte_to_swp_entry(pte);
1334 if (pm->show_pfn)
1335 frame = swp_type(entry) |
1336 (swp_offset(entry) << MAX_SWAPFILES_SHIFT);
1337 flags |= PM_SWAP;
1338 if (is_migration_entry(entry))
1339 page = migration_entry_to_page(entry);
1340
1341 if (is_device_private_entry(entry))
1342 page = device_private_entry_to_page(entry);
1343 }
1344
1345 if (page && !PageAnon(page))
1346 flags |= PM_FILE;
1347 if (page && page_mapcount(page) == 1)
1348 flags |= PM_MMAP_EXCLUSIVE;
1349 if (vma->vm_flags & VM_SOFTDIRTY)
1350 flags |= PM_SOFT_DIRTY;
1351
1352 return make_pme(frame, flags);
1353}
1354
1355static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
1356 struct mm_walk *walk)
1357{
1358 struct vm_area_struct *vma = walk->vma;
1359 struct pagemapread *pm = walk->private;
1360 spinlock_t *ptl;
1361 pte_t *pte, *orig_pte;
1362 int err = 0;
1363
1364#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1365 ptl = pmd_trans_huge_lock(pmdp, vma);
1366 if (ptl) {
1367 u64 flags = 0, frame = 0;
1368 pmd_t pmd = *pmdp;
1369 struct page *page = NULL;
1370
1371 if (vma->vm_flags & VM_SOFTDIRTY)
1372 flags |= PM_SOFT_DIRTY;
1373
1374 if (pmd_present(pmd)) {
1375 page = pmd_page(pmd);
1376
1377 flags |= PM_PRESENT;
1378 if (pmd_soft_dirty(pmd))
1379 flags |= PM_SOFT_DIRTY;
1380 if (pm->show_pfn)
1381 frame = pmd_pfn(pmd) +
1382 ((addr & ~PMD_MASK) >> PAGE_SHIFT);
1383 }
1384#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1385 else if (is_swap_pmd(pmd)) {
1386 swp_entry_t entry = pmd_to_swp_entry(pmd);
1387 unsigned long offset;
1388
1389 if (pm->show_pfn) {
1390 offset = swp_offset(entry) +
1391 ((addr & ~PMD_MASK) >> PAGE_SHIFT);
1392 frame = swp_type(entry) |
1393 (offset << MAX_SWAPFILES_SHIFT);
1394 }
1395 flags |= PM_SWAP;
1396 if (pmd_swp_soft_dirty(pmd))
1397 flags |= PM_SOFT_DIRTY;
1398 VM_BUG_ON(!is_pmd_migration_entry(pmd));
1399 page = migration_entry_to_page(entry);
1400 }
1401#endif
1402
1403 if (page && page_mapcount(page) == 1)
1404 flags |= PM_MMAP_EXCLUSIVE;
1405
1406 for (; addr != end; addr += PAGE_SIZE) {
1407 pagemap_entry_t pme = make_pme(frame, flags);
1408
1409 err = add_to_pagemap(addr, &pme, pm);
1410 if (err)
1411 break;
1412 if (pm->show_pfn) {
1413 if (flags & PM_PRESENT)
1414 frame++;
1415 else if (flags & PM_SWAP)
1416 frame += (1 << MAX_SWAPFILES_SHIFT);
1417 }
1418 }
1419 spin_unlock(ptl);
1420 return err;
1421 }
1422
1423 if (pmd_trans_unstable(pmdp))
1424 return 0;
1425#endif
1426
1427
1428
1429
1430
1431 orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);
1432 for (; addr < end; pte++, addr += PAGE_SIZE) {
1433 pagemap_entry_t pme;
1434
1435 pme = pte_to_pagemap_entry(pm, vma, addr, *pte);
1436 err = add_to_pagemap(addr, &pme, pm);
1437 if (err)
1438 break;
1439 }
1440 pte_unmap_unlock(orig_pte, ptl);
1441
1442 cond_resched();
1443
1444 return err;
1445}
1446
1447#ifdef CONFIG_HUGETLB_PAGE
1448
1449static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
1450 unsigned long addr, unsigned long end,
1451 struct mm_walk *walk)
1452{
1453 struct pagemapread *pm = walk->private;
1454 struct vm_area_struct *vma = walk->vma;
1455 u64 flags = 0, frame = 0;
1456 int err = 0;
1457 pte_t pte;
1458
1459 if (vma->vm_flags & VM_SOFTDIRTY)
1460 flags |= PM_SOFT_DIRTY;
1461
1462 pte = huge_ptep_get(ptep);
1463 if (pte_present(pte)) {
1464 struct page *page = pte_page(pte);
1465
1466 if (!PageAnon(page))
1467 flags |= PM_FILE;
1468
1469 if (page_mapcount(page) == 1)
1470 flags |= PM_MMAP_EXCLUSIVE;
1471
1472 flags |= PM_PRESENT;
1473 if (pm->show_pfn)
1474 frame = pte_pfn(pte) +
1475 ((addr & ~hmask) >> PAGE_SHIFT);
1476 }
1477
1478 for (; addr != end; addr += PAGE_SIZE) {
1479 pagemap_entry_t pme = make_pme(frame, flags);
1480
1481 err = add_to_pagemap(addr, &pme, pm);
1482 if (err)
1483 return err;
1484 if (pm->show_pfn && (flags & PM_PRESENT))
1485 frame++;
1486 }
1487
1488 cond_resched();
1489
1490 return err;
1491}
1492#endif
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520static ssize_t pagemap_read(struct file *file, char __user *buf,
1521 size_t count, loff_t *ppos)
1522{
1523 struct mm_struct *mm = file->private_data;
1524 struct pagemapread pm;
1525 struct mm_walk pagemap_walk = {};
1526 unsigned long src;
1527 unsigned long svpfn;
1528 unsigned long start_vaddr;
1529 unsigned long end_vaddr;
1530 int ret = 0, copied = 0;
1531
1532 if (!mm || !mmget_not_zero(mm))
1533 goto out;
1534
1535 ret = -EINVAL;
1536
1537 if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
1538 goto out_mm;
1539
1540 ret = 0;
1541 if (!count)
1542 goto out_mm;
1543
1544
1545 pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN);
1546
1547 pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
1548 pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL);
1549 ret = -ENOMEM;
1550 if (!pm.buffer)
1551 goto out_mm;
1552
1553 pagemap_walk.pmd_entry = pagemap_pmd_range;
1554 pagemap_walk.pte_hole = pagemap_pte_hole;
1555#ifdef CONFIG_HUGETLB_PAGE
1556 pagemap_walk.hugetlb_entry = pagemap_hugetlb_range;
1557#endif
1558 pagemap_walk.mm = mm;
1559 pagemap_walk.private = ±
1560
1561 src = *ppos;
1562 svpfn = src / PM_ENTRY_BYTES;
1563 start_vaddr = svpfn << PAGE_SHIFT;
1564 end_vaddr = mm->task_size;
1565
1566
1567 if (svpfn > mm->task_size >> PAGE_SHIFT)
1568 start_vaddr = end_vaddr;
1569
1570
1571
1572
1573
1574
1575
1576 ret = 0;
1577 while (count && (start_vaddr < end_vaddr)) {
1578 int len;
1579 unsigned long end;
1580
1581 pm.pos = 0;
1582 end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
1583
1584 if (end < start_vaddr || end > end_vaddr)
1585 end = end_vaddr;
1586 ret = down_read_killable(&mm->mmap_sem);
1587 if (ret)
1588 goto out_free;
1589 ret = walk_page_range(start_vaddr, end, &pagemap_walk);
1590 up_read(&mm->mmap_sem);
1591 start_vaddr = end;
1592
1593 len = min(count, PM_ENTRY_BYTES * pm.pos);
1594 if (copy_to_user(buf, pm.buffer, len)) {
1595 ret = -EFAULT;
1596 goto out_free;
1597 }
1598 copied += len;
1599 buf += len;
1600 count -= len;
1601 }
1602 *ppos += copied;
1603 if (!ret || ret == PM_END_OF_BUFFER)
1604 ret = copied;
1605
1606out_free:
1607 kfree(pm.buffer);
1608out_mm:
1609 mmput(mm);
1610out:
1611 return ret;
1612}
1613
1614static int pagemap_open(struct inode *inode, struct file *file)
1615{
1616 struct mm_struct *mm;
1617
1618 mm = proc_mem_open(inode, PTRACE_MODE_READ);
1619 if (IS_ERR(mm))
1620 return PTR_ERR(mm);
1621 file->private_data = mm;
1622 return 0;
1623}
1624
1625static int pagemap_release(struct inode *inode, struct file *file)
1626{
1627 struct mm_struct *mm = file->private_data;
1628
1629 if (mm)
1630 mmdrop(mm);
1631 return 0;
1632}
1633
1634const struct file_operations proc_pagemap_operations = {
1635 .llseek = mem_lseek,
1636 .read = pagemap_read,
1637 .open = pagemap_open,
1638 .release = pagemap_release,
1639};
1640#endif
1641
1642#ifdef CONFIG_NUMA
1643
1644struct numa_maps {
1645 unsigned long pages;
1646 unsigned long anon;
1647 unsigned long active;
1648 unsigned long writeback;
1649 unsigned long mapcount_max;
1650 unsigned long dirty;
1651 unsigned long swapcache;
1652 unsigned long node[MAX_NUMNODES];
1653};
1654
1655struct numa_maps_private {
1656 struct proc_maps_private proc_maps;
1657 struct numa_maps md;
1658};
1659
1660static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
1661 unsigned long nr_pages)
1662{
1663 int count = page_mapcount(page);
1664
1665 md->pages += nr_pages;
1666 if (pte_dirty || PageDirty(page))
1667 md->dirty += nr_pages;
1668
1669 if (PageSwapCache(page))
1670 md->swapcache += nr_pages;
1671
1672 if (PageActive(page) || PageUnevictable(page))
1673 md->active += nr_pages;
1674
1675 if (PageWriteback(page))
1676 md->writeback += nr_pages;
1677
1678 if (PageAnon(page))
1679 md->anon += nr_pages;
1680
1681 if (count > md->mapcount_max)
1682 md->mapcount_max = count;
1683
1684 md->node[page_to_nid(page)] += nr_pages;
1685}
1686
1687static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
1688 unsigned long addr)
1689{
1690 struct page *page;
1691 int nid;
1692
1693 if (!pte_present(pte))
1694 return NULL;
1695
1696 page = vm_normal_page(vma, addr, pte);
1697 if (!page)
1698 return NULL;
1699
1700 if (PageReserved(page))
1701 return NULL;
1702
1703 nid = page_to_nid(page);
1704 if (!node_isset(nid, node_states[N_MEMORY]))
1705 return NULL;
1706
1707 return page;
1708}
1709
1710#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1711static struct page *can_gather_numa_stats_pmd(pmd_t pmd,
1712 struct vm_area_struct *vma,
1713 unsigned long addr)
1714{
1715 struct page *page;
1716 int nid;
1717
1718 if (!pmd_present(pmd))
1719 return NULL;
1720
1721 page = vm_normal_page_pmd(vma, addr, pmd);
1722 if (!page)
1723 return NULL;
1724
1725 if (PageReserved(page))
1726 return NULL;
1727
1728 nid = page_to_nid(page);
1729 if (!node_isset(nid, node_states[N_MEMORY]))
1730 return NULL;
1731
1732 return page;
1733}
1734#endif
1735
1736static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
1737 unsigned long end, struct mm_walk *walk)
1738{
1739 struct numa_maps *md = walk->private;
1740 struct vm_area_struct *vma = walk->vma;
1741 spinlock_t *ptl;
1742 pte_t *orig_pte;
1743 pte_t *pte;
1744
1745#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1746 ptl = pmd_trans_huge_lock(pmd, vma);
1747 if (ptl) {
1748 struct page *page;
1749
1750 page = can_gather_numa_stats_pmd(*pmd, vma, addr);
1751 if (page)
1752 gather_stats(page, md, pmd_dirty(*pmd),
1753 HPAGE_PMD_SIZE/PAGE_SIZE);
1754 spin_unlock(ptl);
1755 return 0;
1756 }
1757
1758 if (pmd_trans_unstable(pmd))
1759 return 0;
1760#endif
1761 orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
1762 do {
1763 struct page *page = can_gather_numa_stats(*pte, vma, addr);
1764 if (!page)
1765 continue;
1766 gather_stats(page, md, pte_dirty(*pte), 1);
1767
1768 } while (pte++, addr += PAGE_SIZE, addr != end);
1769 pte_unmap_unlock(orig_pte, ptl);
1770 cond_resched();
1771 return 0;
1772}
1773#ifdef CONFIG_HUGETLB_PAGE
1774static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1775 unsigned long addr, unsigned long end, struct mm_walk *walk)
1776{
1777 pte_t huge_pte = huge_ptep_get(pte);
1778 struct numa_maps *md;
1779 struct page *page;
1780
1781 if (!pte_present(huge_pte))
1782 return 0;
1783
1784 page = pte_page(huge_pte);
1785 if (!page)
1786 return 0;
1787
1788 md = walk->private;
1789 gather_stats(page, md, pte_dirty(huge_pte), 1);
1790 return 0;
1791}
1792
1793#else
1794static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1795 unsigned long addr, unsigned long end, struct mm_walk *walk)
1796{
1797 return 0;
1798}
1799#endif
1800
1801
1802
1803
1804static int show_numa_map(struct seq_file *m, void *v)
1805{
1806 struct numa_maps_private *numa_priv = m->private;
1807 struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
1808 struct vm_area_struct *vma = v;
1809 struct numa_maps *md = &numa_priv->md;
1810 struct file *file = vma->vm_file;
1811 struct mm_struct *mm = vma->vm_mm;
1812 struct mm_walk walk = {
1813 .hugetlb_entry = gather_hugetlb_stats,
1814 .pmd_entry = gather_pte_stats,
1815 .private = md,
1816 .mm = mm,
1817 };
1818 struct mempolicy *pol;
1819 char buffer[64];
1820 int nid;
1821
1822 if (!mm)
1823 return 0;
1824
1825
1826 memset(md, 0, sizeof(*md));
1827
1828 pol = __get_vma_policy(vma, vma->vm_start);
1829 if (pol) {
1830 mpol_to_str(buffer, sizeof(buffer), pol);
1831 mpol_cond_put(pol);
1832 } else {
1833 mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
1834 }
1835
1836 seq_printf(m, "%08lx %s", vma->vm_start, buffer);
1837
1838 if (file) {
1839 seq_puts(m, " file=");
1840 seq_file_path(m, file, "\n\t= ");
1841 } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
1842 seq_puts(m, " heap");
1843 } else if (is_stack(vma)) {
1844 seq_puts(m, " stack");
1845 }
1846
1847 if (is_vm_hugetlb_page(vma))
1848 seq_puts(m, " huge");
1849
1850
1851 walk_page_vma(vma, &walk);
1852
1853 if (!md->pages)
1854 goto out;
1855
1856 if (md->anon)
1857 seq_printf(m, " anon=%lu", md->anon);
1858
1859 if (md->dirty)
1860 seq_printf(m, " dirty=%lu", md->dirty);
1861
1862 if (md->pages != md->anon && md->pages != md->dirty)
1863 seq_printf(m, " mapped=%lu", md->pages);
1864
1865 if (md->mapcount_max > 1)
1866 seq_printf(m, " mapmax=%lu", md->mapcount_max);
1867
1868 if (md->swapcache)
1869 seq_printf(m, " swapcache=%lu", md->swapcache);
1870
1871 if (md->active < md->pages && !is_vm_hugetlb_page(vma))
1872 seq_printf(m, " active=%lu", md->active);
1873
1874 if (md->writeback)
1875 seq_printf(m, " writeback=%lu", md->writeback);
1876
1877 for_each_node_state(nid, N_MEMORY)
1878 if (md->node[nid])
1879 seq_printf(m, " N%d=%lu", nid, md->node[nid]);
1880
1881 seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);
1882out:
1883 seq_putc(m, '\n');
1884 m_cache_vma(m, vma);
1885 return 0;
1886}
1887
1888static const struct seq_operations proc_pid_numa_maps_op = {
1889 .start = m_start,
1890 .next = m_next,
1891 .stop = m_stop,
1892 .show = show_numa_map,
1893};
1894
1895static int pid_numa_maps_open(struct inode *inode, struct file *file)
1896{
1897 return proc_maps_open(inode, file, &proc_pid_numa_maps_op,
1898 sizeof(struct numa_maps_private));
1899}
1900
1901const struct file_operations proc_pid_numa_maps_operations = {
1902 .open = pid_numa_maps_open,
1903 .read = seq_read,
1904 .llseek = seq_lseek,
1905 .release = proc_map_release,
1906};
1907
1908#endif
1909