LXR linux/arch/x86/lib/copy_user

   1/*
   2 * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com>
   3 * Copyright 2002 Andi Kleen, SuSE Labs.
   4 * Subject to the GNU Public License v2.
   5 *
   6 * Functions to copy from and to user space.
   7 */
   8
   9#include <linux/linkage.h>
  10#include <asm/dwarf2.h>
  11#include <asm/current.h>
  12#include <asm/asm-offsets.h>
  13#include <asm/thread_info.h>
  14#include <asm/cpufeature.h>
  15#include <asm/alternative-asm.h>
  16#include <asm/asm.h>
  17#include <asm/smap.h>
  18
  19        .macro ALIGN_DESTINATION
  20        /* check for bad alignment of destination */
  21        movl %edi,%ecx
  22        andl $7,%ecx
  23        jz 102f                         /* already aligned */
  24        subl $8,%ecx
  25        negl %ecx
  26        subl %ecx,%edx
  27100:    movb (%rsi),%al
  28101:    movb %al,(%rdi)
  29        incq %rsi
  30        incq %rdi
  31        decl %ecx
  32        jnz 100b
  33102:
  34        .section .fixup,"ax"
  35103:    addl %ecx,%edx                  /* ecx is zerorest also */
  36        jmp copy_user_handle_tail
  37        .previous
  38
  39        _ASM_EXTABLE(100b,103b)
  40        _ASM_EXTABLE(101b,103b)
  41        .endm
  42
  43/* Standard copy_to_user with segment limit checking */
  44ENTRY(_copy_to_user)
  45        CFI_STARTPROC
  46        GET_THREAD_INFO(%rax)
  47        movq %rdi,%rcx
  48        addq %rdx,%rcx
  49        jc bad_to_user
  50        cmpq TI_addr_limit(%rax),%rcx
  51        ja bad_to_user
  52        ALTERNATIVE_2 "jmp copy_user_generic_unrolled",         \
  53                      "jmp copy_user_generic_string",           \
  54                      X86_FEATURE_REP_GOOD,                     \
  55                      "jmp copy_user_enhanced_fast_string",     \
  56                      X86_FEATURE_ERMS
  57        CFI_ENDPROC
  58ENDPROC(_copy_to_user)
  59
  60/* Standard copy_from_user with segment limit checking */
  61ENTRY(_copy_from_user)
  62        CFI_STARTPROC
  63        GET_THREAD_INFO(%rax)
  64        movq %rsi,%rcx
  65        addq %rdx,%rcx
  66        jc bad_from_user
  67        cmpq TI_addr_limit(%rax),%rcx
  68        ja bad_from_user
  69        ALTERNATIVE_2 "jmp copy_user_generic_unrolled",         \
  70                      "jmp copy_user_generic_string",           \
  71                      X86_FEATURE_REP_GOOD,                     \
  72                      "jmp copy_user_enhanced_fast_string",     \
  73                      X86_FEATURE_ERMS
  74        CFI_ENDPROC
  75ENDPROC(_copy_from_user)
  76
  77        .section .fixup,"ax"
  78        /* must zero dest */
  79ENTRY(bad_from_user)
  80bad_from_user:
  81        CFI_STARTPROC
  82        movl %edx,%ecx
  83        xorl %eax,%eax
  84        rep
  85        stosb
  86bad_to_user:
  87        movl %edx,%eax
  88        ret
  89        CFI_ENDPROC
  90ENDPROC(bad_from_user)
  91        .previous
  92
  93/*
  94 * copy_user_generic_unrolled - memory copy with exception handling.
  95 * This version is for CPUs like P4 that don't have efficient micro
  96 * code for rep movsq
  97 *
  98 * Input:
  99 * rdi destination
 100 * rsi source
 101 * rdx count
 102 *
 103 * Output:
 104 * eax uncopied bytes or 0 if successful.
 105 */
 106ENTRY(copy_user_generic_unrolled)
 107        CFI_STARTPROC
 108        ASM_STAC
 109        cmpl $8,%edx
 110        jb 20f          /* less then 8 bytes, go to byte copy loop */
 111        ALIGN_DESTINATION
 112        movl %edx,%ecx
 113        andl $63,%edx
 114        shrl $6,%ecx
 115        jz 17f
 1161:      movq (%rsi),%r8
 1172:      movq 1*8(%rsi),%r9
 1183:      movq 2*8(%rsi),%r10
 1194:      movq 3*8(%rsi),%r11
 1205:      movq %r8,(%rdi)
 1216:      movq %r9,1*8(%rdi)
 1227:      movq %r10,2*8(%rdi)
 1238:      movq %r11,3*8(%rdi)
 1249:      movq 4*8(%rsi),%r8
 12510:     movq 5*8(%rsi),%r9
 12611:     movq 6*8(%rsi),%r10
 12712:     movq 7*8(%rsi),%r11
 12813:     movq %r8,4*8(%rdi)
 12914:     movq %r9,5*8(%rdi)
 13015:     movq %r10,6*8(%rdi)
 13116:     movq %r11,7*8(%rdi)
 132        leaq 64(%rsi),%rsi
 133        leaq 64(%rdi),%rdi
 134        decl %ecx
 135        jnz 1b
 13617:     movl %edx,%ecx
 137        andl $7,%edx
 138        shrl $3,%ecx
 139        jz 20f
 14018:     movq (%rsi),%r8
 14119:     movq %r8,(%rdi)
 142        leaq 8(%rsi),%rsi
 143        leaq 8(%rdi),%rdi
 144        decl %ecx
 145        jnz 18b
 14620:     andl %edx,%edx
 147        jz 23f
 148        movl %edx,%ecx
 14921:     movb (%rsi),%al
 15022:     movb %al,(%rdi)
 151        incq %rsi
 152        incq %rdi
 153        decl %ecx
 154        jnz 21b
 15523:     xor %eax,%eax
 156        ASM_CLAC
 157        ret
 158
 159        .section .fixup,"ax"
 16030:     shll $6,%ecx
 161        addl %ecx,%edx
 162        jmp 60f
 16340:     leal (%rdx,%rcx,8),%edx
 164        jmp 60f
 16550:     movl %ecx,%edx
 16660:     jmp copy_user_handle_tail /* ecx is zerorest also */
 167        .previous
 168
 169        _ASM_EXTABLE(1b,30b)
 170        _ASM_EXTABLE(2b,30b)
 171        _ASM_EXTABLE(3b,30b)
 172        _ASM_EXTABLE(4b,30b)
 173        _ASM_EXTABLE(5b,30b)
 174        _ASM_EXTABLE(6b,30b)
 175        _ASM_EXTABLE(7b,30b)
 176        _ASM_EXTABLE(8b,30b)
 177        _ASM_EXTABLE(9b,30b)
 178        _ASM_EXTABLE(10b,30b)
 179        _ASM_EXTABLE(11b,30b)
 180        _ASM_EXTABLE(12b,30b)
 181        _ASM_EXTABLE(13b,30b)
 182        _ASM_EXTABLE(14b,30b)
 183        _ASM_EXTABLE(15b,30b)
 184        _ASM_EXTABLE(16b,30b)
 185        _ASM_EXTABLE(18b,40b)
 186        _ASM_EXTABLE(19b,40b)
 187        _ASM_EXTABLE(21b,50b)
 188        _ASM_EXTABLE(22b,50b)
 189        CFI_ENDPROC
 190ENDPROC(copy_user_generic_unrolled)
 191
 192/* Some CPUs run faster using the string copy instructions.
 193 * This is also a lot simpler. Use them when possible.
 194 *
 195 * Only 4GB of copy is supported. This shouldn't be a problem
 196 * because the kernel normally only writes from/to page sized chunks
 197 * even if user space passed a longer buffer.
 198 * And more would be dangerous because both Intel and AMD have
 199 * errata with rep movsq > 4GB. If someone feels the need to fix
 200 * this please consider this.
 201 *
 202 * Input:
 203 * rdi destination
 204 * rsi source
 205 * rdx count
 206 *
 207 * Output:
 208 * eax uncopied bytes or 0 if successful.
 209 */
 210ENTRY(copy_user_generic_string)
 211        CFI_STARTPROC
 212        ASM_STAC
 213        cmpl $8,%edx
 214        jb 2f           /* less than 8 bytes, go to byte copy loop */
 215        ALIGN_DESTINATION
 216        movl %edx,%ecx
 217        shrl $3,%ecx
 218        andl $7,%edx
 2191:      rep
 220        movsq
 2212:      movl %edx,%ecx
 2223:      rep
 223        movsb
 224        xorl %eax,%eax
 225        ASM_CLAC
 226        ret
 227
 228        .section .fixup,"ax"
 22911:     leal (%rdx,%rcx,8),%ecx
 23012:     movl %ecx,%edx          /* ecx is zerorest also */
 231        jmp copy_user_handle_tail
 232        .previous
 233
 234        _ASM_EXTABLE(1b,11b)
 235        _ASM_EXTABLE(3b,12b)
 236        CFI_ENDPROC
 237ENDPROC(copy_user_generic_string)
 238
 239/*
 240 * Some CPUs are adding enhanced REP MOVSB/STOSB instructions.
 241 * It's recommended to use enhanced REP MOVSB/STOSB if it's enabled.
 242 *
 243 * Input:
 244 * rdi destination
 245 * rsi source
 246 * rdx count
 247 *
 248 * Output:
 249 * eax uncopied bytes or 0 if successful.
 250 */
 251ENTRY(copy_user_enhanced_fast_string)
 252        CFI_STARTPROC
 253        ASM_STAC
 254        movl %edx,%ecx
 2551:      rep
 256        movsb
 257        xorl %eax,%eax
 258        ASM_CLAC
 259        ret
 260
 261        .section .fixup,"ax"
 26212:     movl %ecx,%edx          /* ecx is zerorest also */
 263        jmp copy_user_handle_tail
 264        .previous
 265
 266        _ASM_EXTABLE(1b,12b)
 267        CFI_ENDPROC
 268ENDPROC(copy_user_enhanced_fast_string)
 269