1
2
3
4
5
6#include <linux/pagewalk.h>
7#include <linux/pgtable.h>
8#include <asm/tlbflush.h>
9#include <asm/bitops.h>
10#include <asm/set_memory.h>
11
12struct pageattr_masks {
13 pgprot_t set_mask;
14 pgprot_t clear_mask;
15};
16
17static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk)
18{
19 struct pageattr_masks *masks = walk->private;
20 unsigned long new_val = val;
21
22 new_val &= ~(pgprot_val(masks->clear_mask));
23 new_val |= (pgprot_val(masks->set_mask));
24
25 return new_val;
26}
27
28static int pageattr_pgd_entry(pgd_t *pgd, unsigned long addr,
29 unsigned long next, struct mm_walk *walk)
30{
31 pgd_t val = READ_ONCE(*pgd);
32
33 if (pgd_leaf(val)) {
34 val = __pgd(set_pageattr_masks(pgd_val(val), walk));
35 set_pgd(pgd, val);
36 }
37
38 return 0;
39}
40
41static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr,
42 unsigned long next, struct mm_walk *walk)
43{
44 p4d_t val = READ_ONCE(*p4d);
45
46 if (p4d_leaf(val)) {
47 val = __p4d(set_pageattr_masks(p4d_val(val), walk));
48 set_p4d(p4d, val);
49 }
50
51 return 0;
52}
53
54static int pageattr_pud_entry(pud_t *pud, unsigned long addr,
55 unsigned long next, struct mm_walk *walk)
56{
57 pud_t val = READ_ONCE(*pud);
58
59 if (pud_leaf(val)) {
60 val = __pud(set_pageattr_masks(pud_val(val), walk));
61 set_pud(pud, val);
62 }
63
64 return 0;
65}
66
67static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr,
68 unsigned long next, struct mm_walk *walk)
69{
70 pmd_t val = READ_ONCE(*pmd);
71
72 if (pmd_leaf(val)) {
73 val = __pmd(set_pageattr_masks(pmd_val(val), walk));
74 set_pmd(pmd, val);
75 }
76
77 return 0;
78}
79
80static int pageattr_pte_entry(pte_t *pte, unsigned long addr,
81 unsigned long next, struct mm_walk *walk)
82{
83 pte_t val = READ_ONCE(*pte);
84
85 val = __pte(set_pageattr_masks(pte_val(val), walk));
86 set_pte(pte, val);
87
88 return 0;
89}
90
91static int pageattr_pte_hole(unsigned long addr, unsigned long next,
92 int depth, struct mm_walk *walk)
93{
94
95 return 0;
96}
97
98static const struct mm_walk_ops pageattr_ops = {
99 .pgd_entry = pageattr_pgd_entry,
100 .p4d_entry = pageattr_p4d_entry,
101 .pud_entry = pageattr_pud_entry,
102 .pmd_entry = pageattr_pmd_entry,
103 .pte_entry = pageattr_pte_entry,
104 .pte_hole = pageattr_pte_hole,
105};
106
107static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask,
108 pgprot_t clear_mask)
109{
110 int ret;
111 unsigned long start = addr;
112 unsigned long end = start + PAGE_SIZE * numpages;
113 struct pageattr_masks masks = {
114 .set_mask = set_mask,
115 .clear_mask = clear_mask
116 };
117
118 if (!numpages)
119 return 0;
120
121 mmap_read_lock(&init_mm);
122 ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL,
123 &masks);
124 mmap_read_unlock(&init_mm);
125
126 flush_tlb_kernel_range(start, end);
127
128 return ret;
129}
130
131int set_memory_ro(unsigned long addr, int numpages)
132{
133 return __set_memory(addr, numpages, __pgprot(_PAGE_READ),
134 __pgprot(_PAGE_WRITE));
135}
136
137int set_memory_rw(unsigned long addr, int numpages)
138{
139 return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE),
140 __pgprot(0));
141}
142
143int set_memory_x(unsigned long addr, int numpages)
144{
145 return __set_memory(addr, numpages, __pgprot(_PAGE_EXEC), __pgprot(0));
146}
147
148int set_memory_nx(unsigned long addr, int numpages)
149{
150 return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC));
151}
152
153int set_direct_map_invalid_noflush(struct page *page)
154{
155 int ret;
156 unsigned long start = (unsigned long)page_address(page);
157 unsigned long end = start + PAGE_SIZE;
158 struct pageattr_masks masks = {
159 .set_mask = __pgprot(0),
160 .clear_mask = __pgprot(_PAGE_PRESENT)
161 };
162
163 mmap_read_lock(&init_mm);
164 ret = walk_page_range(&init_mm, start, end, &pageattr_ops, &masks);
165 mmap_read_unlock(&init_mm);
166
167 return ret;
168}
169
170int set_direct_map_default_noflush(struct page *page)
171{
172 int ret;
173 unsigned long start = (unsigned long)page_address(page);
174 unsigned long end = start + PAGE_SIZE;
175 struct pageattr_masks masks = {
176 .set_mask = PAGE_KERNEL,
177 .clear_mask = __pgprot(0)
178 };
179
180 mmap_read_lock(&init_mm);
181 ret = walk_page_range(&init_mm, start, end, &pageattr_ops, &masks);
182 mmap_read_unlock(&init_mm);
183
184 return ret;
185}
186
187void __kernel_map_pages(struct page *page, int numpages, int enable)
188{
189 if (!debug_pagealloc_enabled())
190 return;
191
192 if (enable)
193 __set_memory((unsigned long)page_address(page), numpages,
194 __pgprot(_PAGE_PRESENT), __pgprot(0));
195 else
196 __set_memory((unsigned long)page_address(page), numpages,
197 __pgprot(0), __pgprot(_PAGE_PRESENT));
198}
199
