1
2
3
4
5#include <linux/kernel.h>
6#include <linux/mm.h>
7#include <linux/module.h>
8#include <linux/sched.h>
9#include <linux/vmalloc.h>
10
11#include <asm/cacheflush.h>
12#include <asm/set_memory.h>
13#include <asm/tlbflush.h>
14
15struct page_change_data {
16 pgprot_t set_mask;
17 pgprot_t clear_mask;
18};
19
20bool rodata_full __ro_after_init = IS_ENABLED(CONFIG_RODATA_FULL_DEFAULT_ENABLED);
21
22static int change_page_range(pte_t *ptep, unsigned long addr, void *data)
23{
24 struct page_change_data *cdata = data;
25 pte_t pte = READ_ONCE(*ptep);
26
27 pte = clear_pte_bit(pte, cdata->clear_mask);
28 pte = set_pte_bit(pte, cdata->set_mask);
29
30 set_pte(ptep, pte);
31 return 0;
32}
33
34
35
36
37static int __change_memory_common(unsigned long start, unsigned long size,
38 pgprot_t set_mask, pgprot_t clear_mask)
39{
40 struct page_change_data data;
41 int ret;
42
43 data.set_mask = set_mask;
44 data.clear_mask = clear_mask;
45
46 ret = apply_to_page_range(&init_mm, start, size, change_page_range,
47 &data);
48
49 flush_tlb_kernel_range(start, start + size);
50 return ret;
51}
52
53static int change_memory_common(unsigned long addr, int numpages,
54 pgprot_t set_mask, pgprot_t clear_mask)
55{
56 unsigned long start = addr;
57 unsigned long size = PAGE_SIZE * numpages;
58 unsigned long end = start + size;
59 struct vm_struct *area;
60 int i;
61
62 if (!PAGE_ALIGNED(addr)) {
63 start &= PAGE_MASK;
64 end = start + size;
65 WARN_ON_ONCE(1);
66 }
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81 area = find_vm_area((void *)addr);
82 if (!area ||
83 end > (unsigned long)area->addr + area->size ||
84 !(area->flags & VM_ALLOC))
85 return -EINVAL;
86
87 if (!numpages)
88 return 0;
89
90
91
92
93
94 if (rodata_full && (pgprot_val(set_mask) == PTE_RDONLY ||
95 pgprot_val(clear_mask) == PTE_RDONLY)) {
96 for (i = 0; i < area->nr_pages; i++) {
97 __change_memory_common((u64)page_address(area->pages[i]),
98 PAGE_SIZE, set_mask, clear_mask);
99 }
100 }
101
102
103
104
105
106 vm_unmap_aliases();
107
108 return __change_memory_common(start, size, set_mask, clear_mask);
109}
110
111int set_memory_ro(unsigned long addr, int numpages)
112{
113 return change_memory_common(addr, numpages,
114 __pgprot(PTE_RDONLY),
115 __pgprot(PTE_WRITE));
116}
117
118int set_memory_rw(unsigned long addr, int numpages)
119{
120 return change_memory_common(addr, numpages,
121 __pgprot(PTE_WRITE),
122 __pgprot(PTE_RDONLY));
123}
124
125int set_memory_nx(unsigned long addr, int numpages)
126{
127 return change_memory_common(addr, numpages,
128 __pgprot(PTE_PXN),
129 __pgprot(PTE_MAYBE_GP));
130}
131
132int set_memory_x(unsigned long addr, int numpages)
133{
134 return change_memory_common(addr, numpages,
135 __pgprot(PTE_MAYBE_GP),
136 __pgprot(PTE_PXN));
137}
138
139int set_memory_valid(unsigned long addr, int numpages, int enable)
140{
141 if (enable)
142 return __change_memory_common(addr, PAGE_SIZE * numpages,
143 __pgprot(PTE_VALID),
144 __pgprot(0));
145 else
146 return __change_memory_common(addr, PAGE_SIZE * numpages,
147 __pgprot(0),
148 __pgprot(PTE_VALID));
149}
150
151int set_direct_map_invalid_noflush(struct page *page)
152{
153 struct page_change_data data = {
154 .set_mask = __pgprot(0),
155 .clear_mask = __pgprot(PTE_VALID),
156 };
157
158 if (!rodata_full)
159 return 0;
160
161 return apply_to_page_range(&init_mm,
162 (unsigned long)page_address(page),
163 PAGE_SIZE, change_page_range, &data);
164}
165
166int set_direct_map_default_noflush(struct page *page)
167{
168 struct page_change_data data = {
169 .set_mask = __pgprot(PTE_VALID | PTE_WRITE),
170 .clear_mask = __pgprot(PTE_RDONLY),
171 };
172
173 if (!rodata_full)
174 return 0;
175
176 return apply_to_page_range(&init_mm,
177 (unsigned long)page_address(page),
178 PAGE_SIZE, change_page_range, &data);
179}
180
181void __kernel_map_pages(struct page *page, int numpages, int enable)
182{
183 if (!debug_pagealloc_enabled() && !rodata_full)
184 return;
185
186 set_memory_valid((unsigned long)page_address(page), numpages, enable);
187}
188
189
190
191
192
193
194
195
196
197
198bool kernel_page_present(struct page *page)
199{
200 pgd_t *pgdp;
201 p4d_t *p4dp;
202 pud_t *pudp, pud;
203 pmd_t *pmdp, pmd;
204 pte_t *ptep;
205 unsigned long addr = (unsigned long)page_address(page);
206
207 if (!debug_pagealloc_enabled() && !rodata_full)
208 return true;
209
210 pgdp = pgd_offset_k(addr);
211 if (pgd_none(READ_ONCE(*pgdp)))
212 return false;
213
214 p4dp = p4d_offset(pgdp, addr);
215 if (p4d_none(READ_ONCE(*p4dp)))
216 return false;
217
218 pudp = pud_offset(p4dp, addr);
219 pud = READ_ONCE(*pudp);
220 if (pud_none(pud))
221 return false;
222 if (pud_sect(pud))
223 return true;
224
225 pmdp = pmd_offset(pudp, addr);
226 pmd = READ_ONCE(*pmdp);
227 if (pmd_none(pmd))
228 return false;
229 if (pmd_sect(pmd))
230 return true;
231
232 ptep = pte_offset_kernel(pmdp, addr);
233 return pte_valid(READ_ONCE(*ptep));
234}
235