1#ifndef _ASM_X86_UACCESS_H
2#define _ASM_X86_UACCESS_H
3
4
5
6#include <linux/errno.h>
7#include <linux/compiler.h>
8#include <linux/thread_info.h>
9#include <linux/prefetch.h>
10#include <linux/string.h>
11#include <asm/asm.h>
12#include <asm/page.h>
13
14#define VERIFY_READ 0
15#define VERIFY_WRITE 1
16
17
18
19
20
21
22
23
24
25#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) })
26
27#define KERNEL_DS MAKE_MM_SEG(-1UL)
28#define USER_DS MAKE_MM_SEG(TASK_SIZE_MAX)
29
30#define get_ds() (KERNEL_DS)
31#define get_fs() (current_thread_info()->addr_limit)
32#define set_fs(x) (current_thread_info()->addr_limit = (x))
33
34#define segment_eq(a, b) ((a).seg == (b).seg)
35
36#define __addr_ok(addr) \
37 ((unsigned long __force)(addr) < \
38 (current_thread_info()->addr_limit.seg))
39
40
41
42
43
44
45
46
47
48
49
50#define __range_not_ok(addr, size) \
51({ \
52 unsigned long flag, roksum; \
53 __chk_user_ptr(addr); \
54 asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" \
55 : "=&r" (flag), "=r" (roksum) \
56 : "1" (addr), "g" ((long)(size)), \
57 "rm" (current_thread_info()->addr_limit.seg)); \
58 flag; \
59})
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80#define access_ok(type, addr, size) (likely(__range_not_ok(addr, size) == 0))
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95struct exception_table_entry {
96 unsigned long insn, fixup;
97};
98
99extern int fixup_exception(struct pt_regs *regs);
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116extern int __get_user_1(void);
117extern int __get_user_2(void);
118extern int __get_user_4(void);
119extern int __get_user_8(void);
120extern int __get_user_bad(void);
121
122#define __get_user_x(size, ret, x, ptr) \
123 asm volatile("call __get_user_" #size \
124 : "=a" (ret), "=d" (x) \
125 : "0" (ptr)) \
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147#ifdef CONFIG_X86_32
148#define __get_user_8(__ret_gu, __val_gu, ptr) \
149 __get_user_x(X, __ret_gu, __val_gu, ptr)
150#else
151#define __get_user_8(__ret_gu, __val_gu, ptr) \
152 __get_user_x(8, __ret_gu, __val_gu, ptr)
153#endif
154
155#define get_user(x, ptr) \
156({ \
157 int __ret_gu; \
158 unsigned long __val_gu; \
159 __chk_user_ptr(ptr); \
160 might_fault(); \
161 switch (sizeof(*(ptr))) { \
162 case 1: \
163 __get_user_x(1, __ret_gu, __val_gu, ptr); \
164 break; \
165 case 2: \
166 __get_user_x(2, __ret_gu, __val_gu, ptr); \
167 break; \
168 case 4: \
169 __get_user_x(4, __ret_gu, __val_gu, ptr); \
170 break; \
171 case 8: \
172 __get_user_8(__ret_gu, __val_gu, ptr); \
173 break; \
174 default: \
175 __get_user_x(X, __ret_gu, __val_gu, ptr); \
176 break; \
177 } \
178 (x) = (__typeof__(*(ptr)))__val_gu; \
179 __ret_gu; \
180})
181
182#define __put_user_x(size, x, ptr, __ret_pu) \
183 asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
184 : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
185
186
187
188#ifdef CONFIG_X86_32
189#define __put_user_asm_u64(x, addr, err, errret) \
190 asm volatile("1: movl %%eax,0(%2)\n" \
191 "2: movl %%edx,4(%2)\n" \
192 "3:\n" \
193 ".section .fixup,\"ax\"\n" \
194 "4: movl %3,%0\n" \
195 " jmp 3b\n" \
196 ".previous\n" \
197 _ASM_EXTABLE(1b, 4b) \
198 _ASM_EXTABLE(2b, 4b) \
199 : "=r" (err) \
200 : "A" (x), "r" (addr), "i" (errret), "0" (err))
201
202#define __put_user_asm_ex_u64(x, addr) \
203 asm volatile("1: movl %%eax,0(%1)\n" \
204 "2: movl %%edx,4(%1)\n" \
205 "3:\n" \
206 _ASM_EXTABLE(1b, 2b - 1b) \
207 _ASM_EXTABLE(2b, 3b - 2b) \
208 : : "A" (x), "r" (addr))
209
210#define __put_user_x8(x, ptr, __ret_pu) \
211 asm volatile("call __put_user_8" : "=a" (__ret_pu) \
212 : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
213#else
214#define __put_user_asm_u64(x, ptr, retval, errret) \
215 __put_user_asm(x, ptr, retval, "q", "", "er", errret)
216#define __put_user_asm_ex_u64(x, addr) \
217 __put_user_asm_ex(x, addr, "q", "", "er")
218#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
219#endif
220
221extern void __put_user_bad(void);
222
223
224
225
226
227extern void __put_user_1(void);
228extern void __put_user_2(void);
229extern void __put_user_4(void);
230extern void __put_user_8(void);
231
232#ifdef CONFIG_X86_WP_WORKS_OK
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250#define put_user(x, ptr) \
251({ \
252 int __ret_pu; \
253 __typeof__(*(ptr)) __pu_val; \
254 __chk_user_ptr(ptr); \
255 might_fault(); \
256 __pu_val = x; \
257 switch (sizeof(*(ptr))) { \
258 case 1: \
259 __put_user_x(1, __pu_val, ptr, __ret_pu); \
260 break; \
261 case 2: \
262 __put_user_x(2, __pu_val, ptr, __ret_pu); \
263 break; \
264 case 4: \
265 __put_user_x(4, __pu_val, ptr, __ret_pu); \
266 break; \
267 case 8: \
268 __put_user_x8(__pu_val, ptr, __ret_pu); \
269 break; \
270 default: \
271 __put_user_x(X, __pu_val, ptr, __ret_pu); \
272 break; \
273 } \
274 __ret_pu; \
275})
276
277#define __put_user_size(x, ptr, size, retval, errret) \
278do { \
279 retval = 0; \
280 __chk_user_ptr(ptr); \
281 switch (size) { \
282 case 1: \
283 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \
284 break; \
285 case 2: \
286 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \
287 break; \
288 case 4: \
289 __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \
290 break; \
291 case 8: \
292 __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval, \
293 errret); \
294 break; \
295 default: \
296 __put_user_bad(); \
297 } \
298} while (0)
299
300#define __put_user_size_ex(x, ptr, size) \
301do { \
302 __chk_user_ptr(ptr); \
303 switch (size) { \
304 case 1: \
305 __put_user_asm_ex(x, ptr, "b", "b", "iq"); \
306 break; \
307 case 2: \
308 __put_user_asm_ex(x, ptr, "w", "w", "ir"); \
309 break; \
310 case 4: \
311 __put_user_asm_ex(x, ptr, "l", "k", "ir"); \
312 break; \
313 case 8: \
314 __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr); \
315 break; \
316 default: \
317 __put_user_bad(); \
318 } \
319} while (0)
320
321#else
322
323#define __put_user_size(x, ptr, size, retval, errret) \
324do { \
325 __typeof__(*(ptr))__pus_tmp = x; \
326 retval = 0; \
327 \
328 if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, size) != 0)) \
329 retval = errret; \
330} while (0)
331
332#define put_user(x, ptr) \
333({ \
334 int __ret_pu; \
335 __typeof__(*(ptr))__pus_tmp = x; \
336 __ret_pu = 0; \
337 if (unlikely(__copy_to_user_ll(ptr, &__pus_tmp, \
338 sizeof(*(ptr))) != 0)) \
339 __ret_pu = -EFAULT; \
340 __ret_pu; \
341})
342#endif
343
344#ifdef CONFIG_X86_32
345#define __get_user_asm_u64(x, ptr, retval, errret) (x) = __get_user_bad()
346#define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad()
347#else
348#define __get_user_asm_u64(x, ptr, retval, errret) \
349 __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
350#define __get_user_asm_ex_u64(x, ptr) \
351 __get_user_asm_ex(x, ptr, "q", "", "=r")
352#endif
353
354#define __get_user_size(x, ptr, size, retval, errret) \
355do { \
356 retval = 0; \
357 __chk_user_ptr(ptr); \
358 switch (size) { \
359 case 1: \
360 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \
361 break; \
362 case 2: \
363 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \
364 break; \
365 case 4: \
366 __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \
367 break; \
368 case 8: \
369 __get_user_asm_u64(x, ptr, retval, errret); \
370 break; \
371 default: \
372 (x) = __get_user_bad(); \
373 } \
374} while (0)
375
376#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
377 asm volatile("1: mov"itype" %2,%"rtype"1\n" \
378 "2:\n" \
379 ".section .fixup,\"ax\"\n" \
380 "3: mov %3,%0\n" \
381 " xor"itype" %"rtype"1,%"rtype"1\n" \
382 " jmp 2b\n" \
383 ".previous\n" \
384 _ASM_EXTABLE(1b, 3b) \
385 : "=r" (err), ltype(x) \
386 : "m" (__m(addr)), "i" (errret), "0" (err))
387
388#define __get_user_size_ex(x, ptr, size) \
389do { \
390 __chk_user_ptr(ptr); \
391 switch (size) { \
392 case 1: \
393 __get_user_asm_ex(x, ptr, "b", "b", "=q"); \
394 break; \
395 case 2: \
396 __get_user_asm_ex(x, ptr, "w", "w", "=r"); \
397 break; \
398 case 4: \
399 __get_user_asm_ex(x, ptr, "l", "k", "=r"); \
400 break; \
401 case 8: \
402 __get_user_asm_ex_u64(x, ptr); \
403 break; \
404 default: \
405 (x) = __get_user_bad(); \
406 } \
407} while (0)
408
409#define __get_user_asm_ex(x, addr, itype, rtype, ltype) \
410 asm volatile("1: mov"itype" %1,%"rtype"0\n" \
411 "2:\n" \
412 _ASM_EXTABLE(1b, 2b - 1b) \
413 : ltype(x) : "m" (__m(addr)))
414
415#define __put_user_nocheck(x, ptr, size) \
416({ \
417 int __pu_err; \
418 __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
419 __pu_err; \
420})
421
422#define __get_user_nocheck(x, ptr, size) \
423({ \
424 int __gu_err; \
425 unsigned long __gu_val; \
426 __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \
427 (x) = (__force __typeof__(*(ptr)))__gu_val; \
428 __gu_err; \
429})
430
431
432struct __large_struct { unsigned long buf[100]; };
433#define __m(x) (*(struct __large_struct __user *)(x))
434
435
436
437
438
439
440#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
441 asm volatile("1: mov"itype" %"rtype"1,%2\n" \
442 "2:\n" \
443 ".section .fixup,\"ax\"\n" \
444 "3: mov %3,%0\n" \
445 " jmp 2b\n" \
446 ".previous\n" \
447 _ASM_EXTABLE(1b, 3b) \
448 : "=r"(err) \
449 : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
450
451#define __put_user_asm_ex(x, addr, itype, rtype, ltype) \
452 asm volatile("1: mov"itype" %"rtype"0,%1\n" \
453 "2:\n" \
454 _ASM_EXTABLE(1b, 2b - 1b) \
455 : : ltype(x), "m" (__m(addr)))
456
457
458
459
460#define uaccess_try do { \
461 int prev_err = current_thread_info()->uaccess_err; \
462 current_thread_info()->uaccess_err = 0; \
463 barrier();
464
465#define uaccess_catch(err) \
466 (err) |= current_thread_info()->uaccess_err; \
467 current_thread_info()->uaccess_err = prev_err; \
468} while (0)
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491#define __get_user(x, ptr) \
492 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514#define __put_user(x, ptr) \
515 __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
516
517#define __get_user_unaligned __get_user
518#define __put_user_unaligned __put_user
519
520
521
522
523
524
525
526
527#define get_user_try uaccess_try
528#define get_user_catch(err) uaccess_catch(err)
529
530#define get_user_ex(x, ptr) do { \
531 unsigned long __gue_val; \
532 __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \
533 (x) = (__force __typeof__(*(ptr)))__gue_val; \
534} while (0)
535
536#ifdef CONFIG_X86_WP_WORKS_OK
537
538#define put_user_try uaccess_try
539#define put_user_catch(err) uaccess_catch(err)
540
541#define put_user_ex(x, ptr) \
542 __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
543
544#else
545
546#define put_user_try do { \
547 int __uaccess_err = 0;
548
549#define put_user_catch(err) \
550 (err) |= __uaccess_err; \
551} while (0)
552
553#define put_user_ex(x, ptr) do { \
554 __uaccess_err |= __put_user(x, ptr); \
555} while (0)
556
557#endif
558
559
560
561
562#ifdef CONFIG_X86_INTEL_USERCOPY
563extern struct movsl_mask {
564 int mask;
565} ____cacheline_aligned_in_smp movsl_mask;
566#endif
567
568#define ARCH_HAS_NOCACHE_UACCESS 1
569
570#ifdef CONFIG_X86_32
571# include "uaccess_32.h"
572#else
573# define ARCH_HAS_SEARCH_EXTABLE
574# include "uaccess_64.h"
575#endif
576
577#endif
578
579