1#ifndef _ASM_X86_UACCESS_H
2#define _ASM_X86_UACCESS_H
3
4
5
6#include <linux/errno.h>
7#include <linux/compiler.h>
8#include <linux/thread_info.h>
9#include <linux/string.h>
10#include <asm/asm.h>
11#include <asm/page.h>
12#include <asm/smap.h>
13
14#define VERIFY_READ 0
15#define VERIFY_WRITE 1
16
17
18
19
20
21
22
23
24
25#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) })
26
27#define KERNEL_DS MAKE_MM_SEG(-1UL)
28#define USER_DS MAKE_MM_SEG(TASK_SIZE_MAX)
29
30#define get_ds() (KERNEL_DS)
31#define get_fs() (current_thread_info()->addr_limit)
32#define set_fs(x) (current_thread_info()->addr_limit = (x))
33
34#define segment_eq(a, b) ((a).seg == (b).seg)
35
36#define user_addr_max() (current_thread_info()->addr_limit.seg)
37#define __addr_ok(addr) \
38 ((unsigned long __force)(addr) < user_addr_max())
39
40
41
42
43
44
45
46
47
48
49
50#define __range_not_ok(addr, size, limit) \
51({ \
52 unsigned long flag, roksum; \
53 __chk_user_ptr(addr); \
54 asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" \
55 : "=&r" (flag), "=r" (roksum) \
56 : "1" (addr), "g" ((long)(size)), \
57 "rm" (limit)); \
58 flag; \
59})
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80#define access_ok(type, addr, size) \
81 (likely(__range_not_ok(addr, size, user_addr_max()) == 0))
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97struct exception_table_entry {
98 int insn, fixup;
99};
100
101#define ARCH_HAS_SORT_EXTABLE
102#define ARCH_HAS_SEARCH_EXTABLE
103
104extern int fixup_exception(struct pt_regs *regs);
105extern int early_fixup_exception(unsigned long *ip);
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122extern int __get_user_1(void);
123extern int __get_user_2(void);
124extern int __get_user_4(void);
125extern int __get_user_8(void);
126extern int __get_user_bad(void);
127
128
129
130
131
132#define __inttype(x) \
133__typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162#define get_user(x, ptr) \
163({ \
164 int __ret_gu; \
165 register __inttype(*(ptr)) __val_gu asm("%edx"); \
166 __chk_user_ptr(ptr); \
167 might_fault(); \
168 asm volatile("call __get_user_%P3" \
169 : "=a" (__ret_gu), "=r" (__val_gu) \
170 : "0" (ptr), "i" (sizeof(*(ptr)))); \
171 (x) = (__typeof__(*(ptr))) __val_gu; \
172 __ret_gu; \
173})
174
175#define __put_user_x(size, x, ptr, __ret_pu) \
176 asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
177 : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
178
179
180
181#ifdef CONFIG_X86_32
182#define __put_user_asm_u64(x, addr, err, errret) \
183 asm volatile(ASM_STAC "\n" \
184 "1: movl %%eax,0(%2)\n" \
185 "2: movl %%edx,4(%2)\n" \
186 "3: " ASM_CLAC "\n" \
187 ".section .fixup,\"ax\"\n" \
188 "4: movl %3,%0\n" \
189 " jmp 3b\n" \
190 ".previous\n" \
191 _ASM_EXTABLE(1b, 4b) \
192 _ASM_EXTABLE(2b, 4b) \
193 : "=r" (err) \
194 : "A" (x), "r" (addr), "i" (errret), "0" (err))
195
196#define __put_user_asm_ex_u64(x, addr) \
197 asm volatile(ASM_STAC "\n" \
198 "1: movl %%eax,0(%1)\n" \
199 "2: movl %%edx,4(%1)\n" \
200 "3: " ASM_CLAC "\n" \
201 _ASM_EXTABLE_EX(1b, 2b) \
202 _ASM_EXTABLE_EX(2b, 3b) \
203 : : "A" (x), "r" (addr))
204
205#define __put_user_x8(x, ptr, __ret_pu) \
206 asm volatile("call __put_user_8" : "=a" (__ret_pu) \
207 : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
208#else
209#define __put_user_asm_u64(x, ptr, retval, errret) \
210 __put_user_asm(x, ptr, retval, "q", "", "er", errret)
211#define __put_user_asm_ex_u64(x, addr) \
212 __put_user_asm_ex(x, addr, "q", "", "er")
213#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
214#endif
215
216extern void __put_user_bad(void);
217
218
219
220
221
222extern void __put_user_1(void);
223extern void __put_user_2(void);
224extern void __put_user_4(void);
225extern void __put_user_8(void);
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243#define put_user(x, ptr) \
244({ \
245 int __ret_pu; \
246 __typeof__(*(ptr)) __pu_val; \
247 __chk_user_ptr(ptr); \
248 might_fault(); \
249 __pu_val = x; \
250 switch (sizeof(*(ptr))) { \
251 case 1: \
252 __put_user_x(1, __pu_val, ptr, __ret_pu); \
253 break; \
254 case 2: \
255 __put_user_x(2, __pu_val, ptr, __ret_pu); \
256 break; \
257 case 4: \
258 __put_user_x(4, __pu_val, ptr, __ret_pu); \
259 break; \
260 case 8: \
261 __put_user_x8(__pu_val, ptr, __ret_pu); \
262 break; \
263 default: \
264 __put_user_x(X, __pu_val, ptr, __ret_pu); \
265 break; \
266 } \
267 __ret_pu; \
268})
269
270#define __put_user_size(x, ptr, size, retval, errret) \
271do { \
272 retval = 0; \
273 __chk_user_ptr(ptr); \
274 switch (size) { \
275 case 1: \
276 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \
277 break; \
278 case 2: \
279 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \
280 break; \
281 case 4: \
282 __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \
283 break; \
284 case 8: \
285 __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval, \
286 errret); \
287 break; \
288 default: \
289 __put_user_bad(); \
290 } \
291} while (0)
292
293#define __put_user_size_ex(x, ptr, size) \
294do { \
295 __chk_user_ptr(ptr); \
296 switch (size) { \
297 case 1: \
298 __put_user_asm_ex(x, ptr, "b", "b", "iq"); \
299 break; \
300 case 2: \
301 __put_user_asm_ex(x, ptr, "w", "w", "ir"); \
302 break; \
303 case 4: \
304 __put_user_asm_ex(x, ptr, "l", "k", "ir"); \
305 break; \
306 case 8: \
307 __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr); \
308 break; \
309 default: \
310 __put_user_bad(); \
311 } \
312} while (0)
313
314#ifdef CONFIG_X86_32
315#define __get_user_asm_u64(x, ptr, retval, errret) (x) = __get_user_bad()
316#define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad()
317#else
318#define __get_user_asm_u64(x, ptr, retval, errret) \
319 __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
320#define __get_user_asm_ex_u64(x, ptr) \
321 __get_user_asm_ex(x, ptr, "q", "", "=r")
322#endif
323
324#define __get_user_size(x, ptr, size, retval, errret) \
325do { \
326 retval = 0; \
327 __chk_user_ptr(ptr); \
328 switch (size) { \
329 case 1: \
330 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \
331 break; \
332 case 2: \
333 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \
334 break; \
335 case 4: \
336 __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \
337 break; \
338 case 8: \
339 __get_user_asm_u64(x, ptr, retval, errret); \
340 break; \
341 default: \
342 (x) = __get_user_bad(); \
343 } \
344} while (0)
345
346#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
347 asm volatile(ASM_STAC "\n" \
348 "1: mov"itype" %2,%"rtype"1\n" \
349 "2: " ASM_CLAC "\n" \
350 ".section .fixup,\"ax\"\n" \
351 "3: mov %3,%0\n" \
352 " xor"itype" %"rtype"1,%"rtype"1\n" \
353 " jmp 2b\n" \
354 ".previous\n" \
355 _ASM_EXTABLE(1b, 3b) \
356 : "=r" (err), ltype(x) \
357 : "m" (__m(addr)), "i" (errret), "0" (err))
358
359#define __get_user_size_ex(x, ptr, size) \
360do { \
361 __chk_user_ptr(ptr); \
362 switch (size) { \
363 case 1: \
364 __get_user_asm_ex(x, ptr, "b", "b", "=q"); \
365 break; \
366 case 2: \
367 __get_user_asm_ex(x, ptr, "w", "w", "=r"); \
368 break; \
369 case 4: \
370 __get_user_asm_ex(x, ptr, "l", "k", "=r"); \
371 break; \
372 case 8: \
373 __get_user_asm_ex_u64(x, ptr); \
374 break; \
375 default: \
376 (x) = __get_user_bad(); \
377 } \
378} while (0)
379
380#define __get_user_asm_ex(x, addr, itype, rtype, ltype) \
381 asm volatile("1: mov"itype" %1,%"rtype"0\n" \
382 "2:\n" \
383 _ASM_EXTABLE_EX(1b, 2b) \
384 : ltype(x) : "m" (__m(addr)))
385
386#define __put_user_nocheck(x, ptr, size) \
387({ \
388 int __pu_err; \
389 __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
390 __pu_err; \
391})
392
393#define __get_user_nocheck(x, ptr, size) \
394({ \
395 int __gu_err; \
396 unsigned long __gu_val; \
397 __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \
398 (x) = (__force __typeof__(*(ptr)))__gu_val; \
399 __gu_err; \
400})
401
402
403struct __large_struct { unsigned long buf[100]; };
404#define __m(x) (*(struct __large_struct __user *)(x))
405
406
407
408
409
410
411#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
412 asm volatile(ASM_STAC "\n" \
413 "1: mov"itype" %"rtype"1,%2\n" \
414 "2: " ASM_CLAC "\n" \
415 ".section .fixup,\"ax\"\n" \
416 "3: mov %3,%0\n" \
417 " jmp 2b\n" \
418 ".previous\n" \
419 _ASM_EXTABLE(1b, 3b) \
420 : "=r"(err) \
421 : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
422
423#define __put_user_asm_ex(x, addr, itype, rtype, ltype) \
424 asm volatile("1: mov"itype" %"rtype"0,%1\n" \
425 "2:\n" \
426 _ASM_EXTABLE_EX(1b, 2b) \
427 : : ltype(x), "m" (__m(addr)))
428
429
430
431
432#define uaccess_try do { \
433 current_thread_info()->uaccess_err = 0; \
434 stac(); \
435 barrier();
436
437#define uaccess_catch(err) \
438 clac(); \
439 (err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0); \
440} while (0)
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463#define __get_user(x, ptr) \
464 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486#define __put_user(x, ptr) \
487 __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
488
489#define __get_user_unaligned __get_user
490#define __put_user_unaligned __put_user
491
492
493
494
495
496
497
498
499#define get_user_try uaccess_try
500#define get_user_catch(err) uaccess_catch(err)
501
502#define get_user_ex(x, ptr) do { \
503 unsigned long __gue_val; \
504 __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \
505 (x) = (__force __typeof__(*(ptr)))__gue_val; \
506} while (0)
507
508#define put_user_try uaccess_try
509#define put_user_catch(err) uaccess_catch(err)
510
511#define put_user_ex(x, ptr) \
512 __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
513
514extern unsigned long
515copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
516extern __must_check long
517strncpy_from_user(char *dst, const char __user *src, long count);
518
519extern __must_check long strlen_user(const char __user *str);
520extern __must_check long strnlen_user(const char __user *str, long n);
521
522unsigned long __must_check clear_user(void __user *mem, unsigned long len);
523unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
524
525
526
527
528#ifdef CONFIG_X86_INTEL_USERCOPY
529extern struct movsl_mask {
530 int mask;
531} ____cacheline_aligned_in_smp movsl_mask;
532#endif
533
534#define ARCH_HAS_NOCACHE_UACCESS 1
535
536#ifdef CONFIG_X86_32
537# include <asm/uaccess_32.h>
538#else
539# include <asm/uaccess_64.h>
540#endif
541
542#endif
543
544