1#ifndef _ASM_X86_UACCESS_H
2#define _ASM_X86_UACCESS_H
3
4
5
6#include <linux/errno.h>
7#include <linux/compiler.h>
8#include <linux/thread_info.h>
9#include <linux/string.h>
10#include <asm/asm.h>
11#include <asm/page.h>
12#include <asm/smap.h>
13
14#define VERIFY_READ 0
15#define VERIFY_WRITE 1
16
17
18
19
20
21
22
23
24
25#define MAKE_MM_SEG(s) ((mm_segment_t) { (s) })
26
27#define KERNEL_DS MAKE_MM_SEG(-1UL)
28#define USER_DS MAKE_MM_SEG(TASK_SIZE_MAX)
29
30#define get_ds() (KERNEL_DS)
31#define get_fs() (current_thread_info()->addr_limit)
32#define set_fs(x) (current_thread_info()->addr_limit = (x))
33
34#define segment_eq(a, b) ((a).seg == (b).seg)
35
36#define user_addr_max() (current_thread_info()->addr_limit.seg)
37#define __addr_ok(addr) \
38 ((unsigned long __force)(addr) < user_addr_max())
39
40
41
42
43
44static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, unsigned long limit)
45{
46
47
48
49
50
51
52
53 if (__builtin_constant_p(size))
54 return unlikely(addr > limit - size);
55
56
57 addr += size;
58 if (unlikely(addr < size))
59 return true;
60 return unlikely(addr > limit);
61}
62
63#define __range_not_ok(addr, size, limit) \
64({ \
65 __chk_user_ptr(addr); \
66 __chk_range_not_ok((unsigned long __force)(addr), size, limit); \
67})
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89#define access_ok(type, addr, size) \
90 likely(!__range_not_ok(addr, size, user_addr_max()))
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106struct exception_table_entry {
107 int insn, fixup;
108};
109
110#define ARCH_HAS_SORT_EXTABLE
111#define ARCH_HAS_SEARCH_EXTABLE
112
113extern int fixup_exception(struct pt_regs *regs);
114extern int early_fixup_exception(unsigned long *ip);
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131extern int __get_user_1(void);
132extern int __get_user_2(void);
133extern int __get_user_4(void);
134extern int __get_user_8(void);
135extern int __get_user_bad(void);
136
137#define __uaccess_begin() stac()
138#define __uaccess_end() clac()
139
140
141
142
143
144#define __inttype(x) \
145__typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178#define get_user(x, ptr) \
179({ \
180 int __ret_gu; \
181 register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX); \
182 __chk_user_ptr(ptr); \
183 might_fault(); \
184 asm volatile("call __get_user_%P3" \
185 : "=a" (__ret_gu), "=r" (__val_gu) \
186 : "0" (ptr), "i" (sizeof(*(ptr)))); \
187 (x) = (__force __typeof__(*(ptr))) __val_gu; \
188 __builtin_expect(__ret_gu, 0); \
189})
190
191#define __put_user_x(size, x, ptr, __ret_pu) \
192 asm volatile("call __put_user_" #size : "=a" (__ret_pu) \
193 : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
194
195
196
197#ifdef CONFIG_X86_32
198#define __put_user_asm_u64(x, addr, err, errret) \
199 asm volatile("\n" \
200 "1: movl %%eax,0(%2)\n" \
201 "2: movl %%edx,4(%2)\n" \
202 "3:" \
203 ".section .fixup,\"ax\"\n" \
204 "4: movl %3,%0\n" \
205 " jmp 3b\n" \
206 ".previous\n" \
207 _ASM_EXTABLE(1b, 4b) \
208 _ASM_EXTABLE(2b, 4b) \
209 : "=r" (err) \
210 : "A" (x), "r" (addr), "i" (errret), "0" (err))
211
212#define __put_user_asm_ex_u64(x, addr) \
213 asm volatile("\n" \
214 "1: movl %%eax,0(%1)\n" \
215 "2: movl %%edx,4(%1)\n" \
216 "3:" \
217 _ASM_EXTABLE_EX(1b, 2b) \
218 _ASM_EXTABLE_EX(2b, 3b) \
219 : : "A" (x), "r" (addr))
220
221#define __put_user_x8(x, ptr, __ret_pu) \
222 asm volatile("call __put_user_8" : "=a" (__ret_pu) \
223 : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx")
224#else
225#define __put_user_asm_u64(x, ptr, retval, errret) \
226 __put_user_asm(x, ptr, retval, "q", "", "er", errret)
227#define __put_user_asm_ex_u64(x, addr) \
228 __put_user_asm_ex(x, addr, "q", "", "er")
229#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu)
230#endif
231
232extern void __put_user_bad(void);
233
234
235
236
237
238extern void __put_user_1(void);
239extern void __put_user_2(void);
240extern void __put_user_4(void);
241extern void __put_user_8(void);
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260#define put_user(x, ptr) \
261({ \
262 int __ret_pu; \
263 __typeof__(*(ptr)) __pu_val; \
264 __chk_user_ptr(ptr); \
265 might_fault(); \
266 __pu_val = x; \
267 switch (sizeof(*(ptr))) { \
268 case 1: \
269 __put_user_x(1, __pu_val, ptr, __ret_pu); \
270 break; \
271 case 2: \
272 __put_user_x(2, __pu_val, ptr, __ret_pu); \
273 break; \
274 case 4: \
275 __put_user_x(4, __pu_val, ptr, __ret_pu); \
276 break; \
277 case 8: \
278 __put_user_x8(__pu_val, ptr, __ret_pu); \
279 break; \
280 default: \
281 __put_user_x(X, __pu_val, ptr, __ret_pu); \
282 break; \
283 } \
284 __builtin_expect(__ret_pu, 0); \
285})
286
287#define __put_user_size(x, ptr, size, retval, errret) \
288do { \
289 retval = 0; \
290 __chk_user_ptr(ptr); \
291 switch (size) { \
292 case 1: \
293 __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \
294 break; \
295 case 2: \
296 __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \
297 break; \
298 case 4: \
299 __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \
300 break; \
301 case 8: \
302 __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval, \
303 errret); \
304 break; \
305 default: \
306 __put_user_bad(); \
307 } \
308} while (0)
309
310
311
312
313
314#define __put_user_size_ex(x, ptr, size) \
315do { \
316 __chk_user_ptr(ptr); \
317 switch (size) { \
318 case 1: \
319 __put_user_asm_ex(x, ptr, "b", "b", "iq"); \
320 break; \
321 case 2: \
322 __put_user_asm_ex(x, ptr, "w", "w", "ir"); \
323 break; \
324 case 4: \
325 __put_user_asm_ex(x, ptr, "l", "k", "ir"); \
326 break; \
327 case 8: \
328 __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr); \
329 break; \
330 default: \
331 __put_user_bad(); \
332 } \
333} while (0)
334
335#ifdef CONFIG_X86_32
336#define __get_user_asm_u64(x, ptr, retval, errret) (x) = __get_user_bad()
337#define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad()
338#else
339#define __get_user_asm_u64(x, ptr, retval, errret) \
340 __get_user_asm(x, ptr, retval, "q", "", "=r", errret)
341#define __get_user_asm_ex_u64(x, ptr) \
342 __get_user_asm_ex(x, ptr, "q", "", "=r")
343#endif
344
345#define __get_user_size(x, ptr, size, retval, errret) \
346do { \
347 retval = 0; \
348 __chk_user_ptr(ptr); \
349 switch (size) { \
350 case 1: \
351 __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \
352 break; \
353 case 2: \
354 __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \
355 break; \
356 case 4: \
357 __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \
358 break; \
359 case 8: \
360 __get_user_asm_u64(x, ptr, retval, errret); \
361 break; \
362 default: \
363 (x) = __get_user_bad(); \
364 } \
365} while (0)
366
367#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \
368 asm volatile("\n" \
369 "1: mov"itype" %2,%"rtype"1\n" \
370 "2:\n" \
371 ".section .fixup,\"ax\"\n" \
372 "3: mov %3,%0\n" \
373 " xor"itype" %"rtype"1,%"rtype"1\n" \
374 " jmp 2b\n" \
375 ".previous\n" \
376 _ASM_EXTABLE(1b, 3b) \
377 : "=r" (err), ltype(x) \
378 : "m" (__m(addr)), "i" (errret), "0" (err))
379
380
381
382
383
384#define __get_user_size_ex(x, ptr, size) \
385do { \
386 __chk_user_ptr(ptr); \
387 switch (size) { \
388 case 1: \
389 __get_user_asm_ex(x, ptr, "b", "b", "=q"); \
390 break; \
391 case 2: \
392 __get_user_asm_ex(x, ptr, "w", "w", "=r"); \
393 break; \
394 case 4: \
395 __get_user_asm_ex(x, ptr, "l", "k", "=r"); \
396 break; \
397 case 8: \
398 __get_user_asm_ex_u64(x, ptr); \
399 break; \
400 default: \
401 (x) = __get_user_bad(); \
402 } \
403} while (0)
404
405#define __get_user_asm_ex(x, addr, itype, rtype, ltype) \
406 asm volatile("1: mov"itype" %1,%"rtype"0\n" \
407 "2:\n" \
408 _ASM_EXTABLE_EX(1b, 2b) \
409 : ltype(x) : "m" (__m(addr)))
410
411#define __put_user_nocheck(x, ptr, size) \
412({ \
413 int __pu_err; \
414 __uaccess_begin(); \
415 __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \
416 __uaccess_end(); \
417 __builtin_expect(__pu_err, 0); \
418})
419
420#define __get_user_nocheck(x, ptr, size) \
421({ \
422 int __gu_err; \
423 unsigned long __gu_val; \
424 __uaccess_begin(); \
425 __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \
426 __uaccess_end(); \
427 (x) = (__force __typeof__(*(ptr)))__gu_val; \
428 __builtin_expect(__gu_err, 0); \
429})
430
431
432struct __large_struct { unsigned long buf[100]; };
433#define __m(x) (*(struct __large_struct __user *)(x))
434
435
436
437
438
439
440#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \
441 asm volatile("\n" \
442 "1: mov"itype" %"rtype"1,%2\n" \
443 "2:\n" \
444 ".section .fixup,\"ax\"\n" \
445 "3: mov %3,%0\n" \
446 " jmp 2b\n" \
447 ".previous\n" \
448 _ASM_EXTABLE(1b, 3b) \
449 : "=r"(err) \
450 : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err))
451
452#define __put_user_asm_ex(x, addr, itype, rtype, ltype) \
453 asm volatile("1: mov"itype" %"rtype"0,%1\n" \
454 "2:\n" \
455 _ASM_EXTABLE_EX(1b, 2b) \
456 : : ltype(x), "m" (__m(addr)))
457
458
459
460
461#define uaccess_try do { \
462 current_thread_info()->uaccess_err = 0; \
463 __uaccess_begin(); \
464 barrier();
465
466#define uaccess_catch(err) \
467 __uaccess_end(); \
468 (err) |= (current_thread_info()->uaccess_err ? -EFAULT : 0); \
469} while (0)
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493#define __get_user(x, ptr) \
494 __get_user_nocheck((x), (ptr), sizeof(*(ptr)))
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517#define __put_user(x, ptr) \
518 __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
519
520#define __get_user_unaligned __get_user
521#define __put_user_unaligned __put_user
522
523
524
525
526
527
528
529
530#define get_user_try uaccess_try
531#define get_user_catch(err) uaccess_catch(err)
532
533#define get_user_ex(x, ptr) do { \
534 unsigned long __gue_val; \
535 __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \
536 (x) = (__force __typeof__(*(ptr)))__gue_val; \
537} while (0)
538
539#define put_user_try uaccess_try
540#define put_user_catch(err) uaccess_catch(err)
541
542#define put_user_ex(x, ptr) \
543 __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
544
545extern unsigned long
546copy_from_user_nmi(void *to, const void __user *from, unsigned long n);
547extern __must_check long
548strncpy_from_user(char *dst, const char __user *src, long count);
549
550extern __must_check long strlen_user(const char __user *str);
551extern __must_check long strnlen_user(const char __user *str, long n);
552
553unsigned long __must_check clear_user(void __user *mem, unsigned long len);
554unsigned long __must_check __clear_user(void __user *mem, unsigned long len);
555
556extern void __cmpxchg_wrong_size(void)
557 __compiletime_error("Bad argument size for cmpxchg");
558
559#define __user_atomic_cmpxchg_inatomic(uval, ptr, old, new, size) \
560({ \
561 int __ret = 0; \
562 __typeof__(ptr) __uval = (uval); \
563 __typeof__(*(ptr)) __old = (old); \
564 __typeof__(*(ptr)) __new = (new); \
565 __uaccess_begin(); \
566 switch (size) { \
567 case 1: \
568 { \
569 asm volatile("\n" \
570 "1:\t" LOCK_PREFIX "cmpxchgb %4, %2\n" \
571 "2:\n" \
572 "\t.section .fixup, \"ax\"\n" \
573 "3:\tmov %3, %0\n" \
574 "\tjmp 2b\n" \
575 "\t.previous\n" \
576 _ASM_EXTABLE(1b, 3b) \
577 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
578 : "i" (-EFAULT), "q" (__new), "1" (__old) \
579 : "memory" \
580 ); \
581 break; \
582 } \
583 case 2: \
584 { \
585 asm volatile("\n" \
586 "1:\t" LOCK_PREFIX "cmpxchgw %4, %2\n" \
587 "2:\n" \
588 "\t.section .fixup, \"ax\"\n" \
589 "3:\tmov %3, %0\n" \
590 "\tjmp 2b\n" \
591 "\t.previous\n" \
592 _ASM_EXTABLE(1b, 3b) \
593 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
594 : "i" (-EFAULT), "r" (__new), "1" (__old) \
595 : "memory" \
596 ); \
597 break; \
598 } \
599 case 4: \
600 { \
601 asm volatile("\n" \
602 "1:\t" LOCK_PREFIX "cmpxchgl %4, %2\n" \
603 "2:\n" \
604 "\t.section .fixup, \"ax\"\n" \
605 "3:\tmov %3, %0\n" \
606 "\tjmp 2b\n" \
607 "\t.previous\n" \
608 _ASM_EXTABLE(1b, 3b) \
609 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
610 : "i" (-EFAULT), "r" (__new), "1" (__old) \
611 : "memory" \
612 ); \
613 break; \
614 } \
615 case 8: \
616 { \
617 if (!IS_ENABLED(CONFIG_X86_64)) \
618 __cmpxchg_wrong_size(); \
619 \
620 asm volatile("\n" \
621 "1:\t" LOCK_PREFIX "cmpxchgq %4, %2\n" \
622 "2:\n" \
623 "\t.section .fixup, \"ax\"\n" \
624 "3:\tmov %3, %0\n" \
625 "\tjmp 2b\n" \
626 "\t.previous\n" \
627 _ASM_EXTABLE(1b, 3b) \
628 : "+r" (__ret), "=a" (__old), "+m" (*(ptr)) \
629 : "i" (-EFAULT), "r" (__new), "1" (__old) \
630 : "memory" \
631 ); \
632 break; \
633 } \
634 default: \
635 __cmpxchg_wrong_size(); \
636 } \
637 __uaccess_end(); \
638 *__uval = __old; \
639 __ret; \
640})
641
642#define user_atomic_cmpxchg_inatomic(uval, ptr, old, new) \
643({ \
644 access_ok(VERIFY_WRITE, (ptr), sizeof(*(ptr))) ? \
645 __user_atomic_cmpxchg_inatomic((uval), (ptr), \
646 (old), (new), sizeof(*(ptr))) : \
647 -EFAULT; \
648})
649
650
651
652
653#ifdef CONFIG_X86_INTEL_USERCOPY
654extern struct movsl_mask {
655 int mask;
656} ____cacheline_aligned_in_smp movsl_mask;
657#endif
658
659#define ARCH_HAS_NOCACHE_UACCESS 1
660
661#ifdef CONFIG_X86_32
662# include <asm/uaccess_32.h>
663#else
664# include <asm/uaccess_64.h>
665#endif
666
667unsigned long __must_check _copy_from_user(void *to, const void __user *from,
668 unsigned n);
669unsigned long __must_check _copy_to_user(void __user *to, const void *from,
670 unsigned n);
671
672#ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS
673# define copy_user_diag __compiletime_error
674#else
675# define copy_user_diag __compiletime_warning
676#endif
677
678extern void copy_user_diag("copy_from_user() buffer size is too small")
679copy_from_user_overflow(void);
680extern void copy_user_diag("copy_to_user() buffer size is too small")
681copy_to_user_overflow(void) __asm__("copy_from_user_overflow");
682
683#undef copy_user_diag
684
685#ifdef CONFIG_DEBUG_STRICT_USER_COPY_CHECKS
686
687extern void
688__compiletime_warning("copy_from_user() buffer size is not provably correct")
689__copy_from_user_overflow(void) __asm__("copy_from_user_overflow");
690#define __copy_from_user_overflow(size, count) __copy_from_user_overflow()
691
692extern void
693__compiletime_warning("copy_to_user() buffer size is not provably correct")
694__copy_to_user_overflow(void) __asm__("copy_from_user_overflow");
695#define __copy_to_user_overflow(size, count) __copy_to_user_overflow()
696
697#else
698
699static inline void
700__copy_from_user_overflow(int size, unsigned long count)
701{
702 WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
703}
704
705#define __copy_to_user_overflow __copy_from_user_overflow
706
707#endif
708
709static inline unsigned long __must_check
710copy_from_user(void *to, const void __user *from, unsigned long n)
711{
712 int sz = __compiletime_object_size(to);
713
714 might_fault();
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734 if (likely(sz < 0 || sz >= n))
735 n = _copy_from_user(to, from, n);
736 else if(__builtin_constant_p(n))
737 copy_from_user_overflow();
738 else
739 __copy_from_user_overflow(sz, n);
740
741 return n;
742}
743
744static inline unsigned long __must_check
745copy_to_user(void __user *to, const void *from, unsigned long n)
746{
747 int sz = __compiletime_object_size(from);
748
749 might_fault();
750
751
752 if (likely(sz < 0 || sz >= n))
753 n = _copy_to_user(to, from, n);
754 else if(__builtin_constant_p(n))
755 copy_to_user_overflow();
756 else
757 __copy_to_user_overflow(sz, n);
758
759 return n;
760}
761
762#undef __copy_from_user_overflow
763#undef __copy_to_user_overflow
764
765
766
767
768
769
770
771
772#define __copy_from_user_nmi __copy_from_user_inatomic
773
774
775
776
777
778
779
780#define user_access_begin() __uaccess_begin()
781#define user_access_end() __uaccess_end()
782
783#define unsafe_put_user(x, ptr) \
784({ \
785 int __pu_err; \
786 __put_user_size((x), (ptr), sizeof(*(ptr)), __pu_err, -EFAULT); \
787 __builtin_expect(__pu_err, 0); \
788})
789
790#define unsafe_get_user(x, ptr) \
791({ \
792 int __gu_err; \
793 unsigned long __gu_val; \
794 __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err, -EFAULT); \
795 (x) = (__force __typeof__(*(ptr)))__gu_val; \
796 __builtin_expect(__gu_err, 0); \
797})
798
799#endif
800
801