LXR linux/arch/x86/lib/csum-copy

   1/*
   2 * Copyright 2002,2003 Andi Kleen, SuSE Labs.
   3 *      
   4 * This file is subject to the terms and conditions of the GNU General Public
   5 * License.  See the file COPYING in the main directory of this archive
   6 * for more details. No warranty for anything given at all.
   7 */
   8#include <linux/linkage.h>
   9#include <asm/dwarf2.h>
  10#include <asm/errno.h>
  11
  12/*
  13 * Checksum copy with exception handling.
  14 * On exceptions src_err_ptr or dst_err_ptr is set to -EFAULT and the 
  15 * destination is zeroed.
  16 * 
  17 * Input
  18 * rdi  source
  19 * rsi  destination
  20 * edx  len (32bit)
  21 * ecx  sum (32bit) 
  22 * r8   src_err_ptr (int)
  23 * r9   dst_err_ptr (int)
  24 *
  25 * Output
  26 * eax  64bit sum. undefined in case of exception.
  27 * 
  28 * Wrappers need to take care of valid exception sum and zeroing.                
  29 * They also should align source or destination to 8 bytes.
  30 */
  31
  32        .macro source
  3310:
  34        .section __ex_table,"a"
  35        .align 8
  36        .quad 10b,.Lbad_source
  37        .previous
  38        .endm
  39                
  40        .macro dest
  4120:
  42        .section __ex_table,"a"
  43        .align 8
  44        .quad 20b,.Lbad_dest
  45        .previous
  46        .endm
  47                        
  48        .macro ignore L=.Lignore
  4930:
  50        .section __ex_table,"a"
  51        .align 8
  52        .quad 30b,\L
  53        .previous
  54        .endm
  55        
  56                                
  57ENTRY(csum_partial_copy_generic)
  58        CFI_STARTPROC
  59        cmpl     $3*64,%edx
  60        jle      .Lignore
  61
  62.Lignore:               
  63        subq  $7*8,%rsp
  64        CFI_ADJUST_CFA_OFFSET 7*8
  65        movq  %rbx,2*8(%rsp)
  66        CFI_REL_OFFSET rbx, 2*8
  67        movq  %r12,3*8(%rsp)
  68        CFI_REL_OFFSET r12, 3*8
  69        movq  %r14,4*8(%rsp)
  70        CFI_REL_OFFSET r14, 4*8
  71        movq  %r13,5*8(%rsp)
  72        CFI_REL_OFFSET r13, 5*8
  73        movq  %rbp,6*8(%rsp)
  74        CFI_REL_OFFSET rbp, 6*8
  75
  76        movq  %r8,(%rsp)
  77        movq  %r9,1*8(%rsp)
  78        
  79        movl  %ecx,%eax
  80        movl  %edx,%ecx
  81
  82        xorl  %r9d,%r9d
  83        movq  %rcx,%r12
  84
  85        shrq  $6,%r12
  86        jz    .Lhandle_tail       /* < 64 */
  87
  88        clc
  89        
  90        /* main loop. clear in 64 byte blocks */
  91        /* r9: zero, r8: temp2, rbx: temp1, rax: sum, rcx: saved length */
  92        /* r11: temp3, rdx: temp4, r12 loopcnt */
  93        /* r10: temp5, rbp: temp6, r14 temp7, r13 temp8 */
  94        .p2align 4
  95.Lloop:
  96        source
  97        movq  (%rdi),%rbx
  98        source
  99        movq  8(%rdi),%r8
 100        source
 101        movq  16(%rdi),%r11
 102        source
 103        movq  24(%rdi),%rdx
 104
 105        source
 106        movq  32(%rdi),%r10
 107        source
 108        movq  40(%rdi),%rbp
 109        source
 110        movq  48(%rdi),%r14
 111        source
 112        movq  56(%rdi),%r13
 113                
 114        ignore 2f
 115        prefetcht0 5*64(%rdi)
 1162:                                                      
 117        adcq  %rbx,%rax
 118        adcq  %r8,%rax
 119        adcq  %r11,%rax
 120        adcq  %rdx,%rax
 121        adcq  %r10,%rax
 122        adcq  %rbp,%rax
 123        adcq  %r14,%rax
 124        adcq  %r13,%rax
 125
 126        decl %r12d
 127        
 128        dest
 129        movq %rbx,(%rsi)
 130        dest
 131        movq %r8,8(%rsi)
 132        dest
 133        movq %r11,16(%rsi)
 134        dest
 135        movq %rdx,24(%rsi)
 136
 137        dest
 138        movq %r10,32(%rsi)
 139        dest
 140        movq %rbp,40(%rsi)
 141        dest
 142        movq %r14,48(%rsi)
 143        dest
 144        movq %r13,56(%rsi)
 145        
 1463:
 147        
 148        leaq 64(%rdi),%rdi
 149        leaq 64(%rsi),%rsi
 150
 151        jnz   .Lloop
 152
 153        adcq  %r9,%rax
 154
 155        /* do last upto 56 bytes */
 156.Lhandle_tail:
 157        /* ecx: count */
 158        movl %ecx,%r10d
 159        andl $63,%ecx
 160        shrl $3,%ecx
 161        jz       .Lfold
 162        clc
 163        .p2align 4
 164.Lloop_8:       
 165        source
 166        movq (%rdi),%rbx
 167        adcq %rbx,%rax
 168        decl %ecx
 169        dest
 170        movq %rbx,(%rsi)
 171        leaq 8(%rsi),%rsi /* preserve carry */
 172        leaq 8(%rdi),%rdi
 173        jnz     .Lloop_8
 174        adcq %r9,%rax   /* add in carry */
 175
 176.Lfold:
 177        /* reduce checksum to 32bits */
 178        movl %eax,%ebx
 179        shrq $32,%rax
 180        addl %ebx,%eax
 181        adcl %r9d,%eax
 182
 183        /* do last upto 6 bytes */      
 184.Lhandle_7:
 185        movl %r10d,%ecx
 186        andl $7,%ecx
 187        shrl $1,%ecx
 188        jz   .Lhandle_1
 189        movl $2,%edx
 190        xorl %ebx,%ebx
 191        clc  
 192        .p2align 4
 193.Lloop_1:       
 194        source
 195        movw (%rdi),%bx
 196        adcl %ebx,%eax
 197        decl %ecx
 198        dest
 199        movw %bx,(%rsi)
 200        leaq 2(%rdi),%rdi
 201        leaq 2(%rsi),%rsi
 202        jnz .Lloop_1
 203        adcl %r9d,%eax  /* add in carry */
 204        
 205        /* handle last odd byte */
 206.Lhandle_1:
 207        testl $1,%r10d
 208        jz    .Lende
 209        xorl  %ebx,%ebx
 210        source
 211        movb (%rdi),%bl
 212        dest
 213        movb %bl,(%rsi)
 214        addl %ebx,%eax
 215        adcl %r9d,%eax          /* carry */
 216                        
 217        CFI_REMEMBER_STATE
 218.Lende:
 219        movq 2*8(%rsp),%rbx
 220        CFI_RESTORE rbx
 221        movq 3*8(%rsp),%r12
 222        CFI_RESTORE r12
 223        movq 4*8(%rsp),%r14
 224        CFI_RESTORE r14
 225        movq 5*8(%rsp),%r13
 226        CFI_RESTORE r13
 227        movq 6*8(%rsp),%rbp
 228        CFI_RESTORE rbp
 229        addq $7*8,%rsp
 230        CFI_ADJUST_CFA_OFFSET -7*8
 231        ret
 232        CFI_RESTORE_STATE
 233
 234        /* Exception handlers. Very simple, zeroing is done in the wrappers */
 235.Lbad_source:
 236        movq (%rsp),%rax
 237        testq %rax,%rax
 238        jz   .Lende
 239        movl $-EFAULT,(%rax)
 240        jmp  .Lende
 241        
 242.Lbad_dest:
 243        movq 8(%rsp),%rax
 244        testq %rax,%rax
 245        jz   .Lende     
 246        movl $-EFAULT,(%rax)
 247        jmp .Lende
 248        CFI_ENDPROC
 249ENDPROC(csum_partial_copy_generic)
 250