1
2
3
4
5
6
7
8
9
10
11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12
13#include <linux/mm.h>
14#include <linux/slab.h>
15#include <linux/sched.h>
16#include <linux/sched/task.h>
17#include <linux/sched/task_stack.h>
18#include <linux/thread_info.h>
19#include <linux/atomic.h>
20#include <linux/jump_label.h>
21#include <asm/sections.h>
22
23
24
25
26
27
28
29
30
31
32
33static noinline int check_stack_object(const void *obj, unsigned long len)
34{
35 const void * const stack = task_stack_page(current);
36 const void * const stackend = stack + THREAD_SIZE;
37 int ret;
38
39
40 if (obj + len <= stack || stackend <= obj)
41 return NOT_STACK;
42
43
44
45
46
47
48 if (obj < stack || stackend < obj + len)
49 return BAD_STACK;
50
51
52 ret = arch_within_stack_frames(stack, stackend, obj, len);
53 if (ret)
54 return ret;
55
56 return GOOD_STACK;
57}
58
59
60
61
62
63
64
65
66
67
68
69
70
71void usercopy_warn(const char *name, const char *detail, bool to_user,
72 unsigned long offset, unsigned long len)
73{
74 WARN_ONCE(1, "Bad or missing usercopy whitelist? Kernel memory %s attempt detected %s %s%s%s%s (offset %lu, size %lu)!\n",
75 to_user ? "exposure" : "overwrite",
76 to_user ? "from" : "to",
77 name ? : "unknown?!",
78 detail ? " '" : "", detail ? : "", detail ? "'" : "",
79 offset, len);
80}
81
82void __noreturn usercopy_abort(const char *name, const char *detail,
83 bool to_user, unsigned long offset,
84 unsigned long len)
85{
86 pr_emerg("Kernel memory %s attempt detected %s %s%s%s%s (offset %lu, size %lu)!\n",
87 to_user ? "exposure" : "overwrite",
88 to_user ? "from" : "to",
89 name ? : "unknown?!",
90 detail ? " '" : "", detail ? : "", detail ? "'" : "",
91 offset, len);
92
93
94
95
96
97
98 BUG();
99}
100
101
102static bool overlaps(const unsigned long ptr, unsigned long n,
103 unsigned long low, unsigned long high)
104{
105 const unsigned long check_low = ptr;
106 unsigned long check_high = check_low + n;
107
108
109 if (check_low >= high || check_high <= low)
110 return false;
111
112 return true;
113}
114
115
116static inline void check_kernel_text_object(const unsigned long ptr,
117 unsigned long n, bool to_user)
118{
119 unsigned long textlow = (unsigned long)_stext;
120 unsigned long texthigh = (unsigned long)_etext;
121 unsigned long textlow_linear, texthigh_linear;
122
123 if (overlaps(ptr, n, textlow, texthigh))
124 usercopy_abort("kernel text", NULL, to_user, ptr - textlow, n);
125
126
127
128
129
130
131
132
133
134 textlow_linear = (unsigned long)lm_alias(textlow);
135
136 if (textlow_linear == textlow)
137 return;
138
139
140 texthigh_linear = (unsigned long)lm_alias(texthigh);
141 if (overlaps(ptr, n, textlow_linear, texthigh_linear))
142 usercopy_abort("linear kernel text", NULL, to_user,
143 ptr - textlow_linear, n);
144}
145
146static inline void check_bogus_address(const unsigned long ptr, unsigned long n,
147 bool to_user)
148{
149
150 if (ptr + n < ptr)
151 usercopy_abort("wrapped address", NULL, to_user, 0, ptr + n);
152
153
154 if (ZERO_OR_NULL_PTR(ptr))
155 usercopy_abort("null address", NULL, to_user, ptr, n);
156}
157
158
159static inline void check_page_span(const void *ptr, unsigned long n,
160 struct page *page, bool to_user)
161{
162#ifdef CONFIG_HARDENED_USERCOPY_PAGESPAN
163 const void *end = ptr + n - 1;
164 struct page *endpage;
165 bool is_reserved, is_cma;
166
167
168
169
170
171
172
173
174 if (ptr >= (const void *)__start_rodata &&
175 end <= (const void *)__end_rodata) {
176 if (!to_user)
177 usercopy_abort("rodata", NULL, to_user, 0, n);
178 return;
179 }
180
181
182 if (ptr >= (const void *)_sdata && end <= (const void *)_edata)
183 return;
184
185
186 if (ptr >= (const void *)__bss_start &&
187 end <= (const void *)__bss_stop)
188 return;
189
190
191 if (likely(((unsigned long)ptr & (unsigned long)PAGE_MASK) ==
192 ((unsigned long)end & (unsigned long)PAGE_MASK)))
193 return;
194
195
196 endpage = virt_to_head_page(end);
197 if (likely(endpage == page))
198 return;
199
200
201
202
203
204
205 is_reserved = PageReserved(page);
206 is_cma = is_migrate_cma_page(page);
207 if (!is_reserved && !is_cma)
208 usercopy_abort("spans multiple pages", NULL, to_user, 0, n);
209
210 for (ptr += PAGE_SIZE; ptr <= end; ptr += PAGE_SIZE) {
211 page = virt_to_head_page(ptr);
212 if (is_reserved && !PageReserved(page))
213 usercopy_abort("spans Reserved and non-Reserved pages",
214 NULL, to_user, 0, n);
215 if (is_cma && !is_migrate_cma_page(page))
216 usercopy_abort("spans CMA and non-CMA pages", NULL,
217 to_user, 0, n);
218 }
219#endif
220}
221
222static inline void check_heap_object(const void *ptr, unsigned long n,
223 bool to_user)
224{
225 struct page *page;
226
227 if (!virt_addr_valid(ptr))
228 return;
229
230 page = virt_to_head_page(ptr);
231
232 if (PageSlab(page)) {
233
234 __check_heap_object(ptr, n, page, to_user);
235 } else {
236
237 check_page_span(ptr, n, page, to_user);
238 }
239}
240
241static DEFINE_STATIC_KEY_FALSE_RO(bypass_usercopy_checks);
242
243
244
245
246
247
248
249
250void __check_object_size(const void *ptr, unsigned long n, bool to_user)
251{
252 if (static_branch_unlikely(&bypass_usercopy_checks))
253 return;
254
255
256 if (!n)
257 return;
258
259
260 check_bogus_address((const unsigned long)ptr, n, to_user);
261
262
263 switch (check_stack_object(ptr, n)) {
264 case NOT_STACK:
265
266 break;
267 case GOOD_FRAME:
268 case GOOD_STACK:
269
270
271
272
273
274 return;
275 default:
276 usercopy_abort("process stack", NULL, to_user, 0, n);
277 }
278
279
280 check_heap_object(ptr, n, to_user);
281
282
283 check_kernel_text_object((const unsigned long)ptr, n, to_user);
284}
285EXPORT_SYMBOL(__check_object_size);
286
287static bool enable_checks __initdata = true;
288
289static int __init parse_hardened_usercopy(char *str)
290{
291 return strtobool(str, &enable_checks);
292}
293
294__setup("hardened_usercopy=", parse_hardened_usercopy);
295
296static int __init set_hardened_usercopy(void)
297{
298 if (enable_checks == false)
299 static_branch_enable(&bypass_usercopy_checks);
300 return 1;
301}
302
303late_initcall(set_hardened_usercopy);
304