LXR linux/arch/powerpc/lib/copyuser

   1/*
   2 * This program is free software; you can redistribute it and/or modify
   3 * it under the terms of the GNU General Public License as published by
   4 * the Free Software Foundation; either version 2 of the License, or
   5 * (at your option) any later version.
   6 *
   7 * This program is distributed in the hope that it will be useful,
   8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
   9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10 * GNU General Public License for more details.
  11 *
  12 * You should have received a copy of the GNU General Public License
  13 * along with this program; if not, write to the Free Software
  14 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  15 *
  16 * Copyright (C) IBM Corporation, 2011
  17 *
  18 * Author: Anton Blanchard <anton@au.ibm.com>
  19 */
  20#include <asm/ppc_asm.h>
  21
  22#ifdef __BIG_ENDIAN__
  23#define LVS(VRT,RA,RB)          lvsl    VRT,RA,RB
  24#define VPERM(VRT,VRA,VRB,VRC)  vperm   VRT,VRA,VRB,VRC
  25#else
  26#define LVS(VRT,RA,RB)          lvsr    VRT,RA,RB
  27#define VPERM(VRT,VRA,VRB,VRC)  vperm   VRT,VRB,VRA,VRC
  28#endif
  29
  30        .macro err1
  31100:
  32        EX_TABLE(100b,.Ldo_err1)
  33        .endm
  34
  35        .macro err2
  36200:
  37        EX_TABLE(200b,.Ldo_err2)
  38        .endm
  39
  40#ifdef CONFIG_ALTIVEC
  41        .macro err3
  42300:
  43        EX_TABLE(300b,.Ldo_err3)
  44        .endm
  45
  46        .macro err4
  47400:
  48        EX_TABLE(400b,.Ldo_err4)
  49        .endm
  50
  51
  52.Ldo_err4:
  53        ld      r16,STK_REG(R16)(r1)
  54        ld      r15,STK_REG(R15)(r1)
  55        ld      r14,STK_REG(R14)(r1)
  56.Ldo_err3:
  57        bl      exit_vmx_usercopy
  58        ld      r0,STACKFRAMESIZE+16(r1)
  59        mtlr    r0
  60        b       .Lexit
  61#endif /* CONFIG_ALTIVEC */
  62
  63.Ldo_err2:
  64        ld      r22,STK_REG(R22)(r1)
  65        ld      r21,STK_REG(R21)(r1)
  66        ld      r20,STK_REG(R20)(r1)
  67        ld      r19,STK_REG(R19)(r1)
  68        ld      r18,STK_REG(R18)(r1)
  69        ld      r17,STK_REG(R17)(r1)
  70        ld      r16,STK_REG(R16)(r1)
  71        ld      r15,STK_REG(R15)(r1)
  72        ld      r14,STK_REG(R14)(r1)
  73.Lexit:
  74        addi    r1,r1,STACKFRAMESIZE
  75.Ldo_err1:
  76        ld      r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
  77        ld      r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
  78        ld      r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
  79        b       __copy_tofrom_user_base
  80
  81
  82_GLOBAL(__copy_tofrom_user_power7)
  83#ifdef CONFIG_ALTIVEC
  84        cmpldi  r5,16
  85        cmpldi  cr1,r5,3328
  86
  87        std     r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
  88        std     r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
  89        std     r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
  90
  91        blt     .Lshort_copy
  92        bge     cr1,.Lvmx_copy
  93#else
  94        cmpldi  r5,16
  95
  96        std     r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
  97        std     r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
  98        std     r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
  99
 100        blt     .Lshort_copy
 101#endif
 102
 103.Lnonvmx_copy:
 104        /* Get the source 8B aligned */
 105        neg     r6,r4
 106        mtocrf  0x01,r6
 107        clrldi  r6,r6,(64-3)
 108
 109        bf      cr7*4+3,1f
 110err1;   lbz     r0,0(r4)
 111        addi    r4,r4,1
 112err1;   stb     r0,0(r3)
 113        addi    r3,r3,1
 114
 1151:      bf      cr7*4+2,2f
 116err1;   lhz     r0,0(r4)
 117        addi    r4,r4,2
 118err1;   sth     r0,0(r3)
 119        addi    r3,r3,2
 120
 1212:      bf      cr7*4+1,3f
 122err1;   lwz     r0,0(r4)
 123        addi    r4,r4,4
 124err1;   stw     r0,0(r3)
 125        addi    r3,r3,4
 126
 1273:      sub     r5,r5,r6
 128        cmpldi  r5,128
 129        blt     5f
 130
 131        mflr    r0
 132        stdu    r1,-STACKFRAMESIZE(r1)
 133        std     r14,STK_REG(R14)(r1)
 134        std     r15,STK_REG(R15)(r1)
 135        std     r16,STK_REG(R16)(r1)
 136        std     r17,STK_REG(R17)(r1)
 137        std     r18,STK_REG(R18)(r1)
 138        std     r19,STK_REG(R19)(r1)
 139        std     r20,STK_REG(R20)(r1)
 140        std     r21,STK_REG(R21)(r1)
 141        std     r22,STK_REG(R22)(r1)
 142        std     r0,STACKFRAMESIZE+16(r1)
 143
 144        srdi    r6,r5,7
 145        mtctr   r6
 146
 147        /* Now do cacheline (128B) sized loads and stores. */
 148        .align  5
 1494:
 150err2;   ld      r0,0(r4)
 151err2;   ld      r6,8(r4)
 152err2;   ld      r7,16(r4)
 153err2;   ld      r8,24(r4)
 154err2;   ld      r9,32(r4)
 155err2;   ld      r10,40(r4)
 156err2;   ld      r11,48(r4)
 157err2;   ld      r12,56(r4)
 158err2;   ld      r14,64(r4)
 159err2;   ld      r15,72(r4)
 160err2;   ld      r16,80(r4)
 161err2;   ld      r17,88(r4)
 162err2;   ld      r18,96(r4)
 163err2;   ld      r19,104(r4)
 164err2;   ld      r20,112(r4)
 165err2;   ld      r21,120(r4)
 166        addi    r4,r4,128
 167err2;   std     r0,0(r3)
 168err2;   std     r6,8(r3)
 169err2;   std     r7,16(r3)
 170err2;   std     r8,24(r3)
 171err2;   std     r9,32(r3)
 172err2;   std     r10,40(r3)
 173err2;   std     r11,48(r3)
 174err2;   std     r12,56(r3)
 175err2;   std     r14,64(r3)
 176err2;   std     r15,72(r3)
 177err2;   std     r16,80(r3)
 178err2;   std     r17,88(r3)
 179err2;   std     r18,96(r3)
 180err2;   std     r19,104(r3)
 181err2;   std     r20,112(r3)
 182err2;   std     r21,120(r3)
 183        addi    r3,r3,128
 184        bdnz    4b
 185
 186        clrldi  r5,r5,(64-7)
 187
 188        ld      r14,STK_REG(R14)(r1)
 189        ld      r15,STK_REG(R15)(r1)
 190        ld      r16,STK_REG(R16)(r1)
 191        ld      r17,STK_REG(R17)(r1)
 192        ld      r18,STK_REG(R18)(r1)
 193        ld      r19,STK_REG(R19)(r1)
 194        ld      r20,STK_REG(R20)(r1)
 195        ld      r21,STK_REG(R21)(r1)
 196        ld      r22,STK_REG(R22)(r1)
 197        addi    r1,r1,STACKFRAMESIZE
 198
 199        /* Up to 127B to go */
 2005:      srdi    r6,r5,4
 201        mtocrf  0x01,r6
 202
 2036:      bf      cr7*4+1,7f
 204err1;   ld      r0,0(r4)
 205err1;   ld      r6,8(r4)
 206err1;   ld      r7,16(r4)
 207err1;   ld      r8,24(r4)
 208err1;   ld      r9,32(r4)
 209err1;   ld      r10,40(r4)
 210err1;   ld      r11,48(r4)
 211err1;   ld      r12,56(r4)
 212        addi    r4,r4,64
 213err1;   std     r0,0(r3)
 214err1;   std     r6,8(r3)
 215err1;   std     r7,16(r3)
 216err1;   std     r8,24(r3)
 217err1;   std     r9,32(r3)
 218err1;   std     r10,40(r3)
 219err1;   std     r11,48(r3)
 220err1;   std     r12,56(r3)
 221        addi    r3,r3,64
 222
 223        /* Up to 63B to go */
 2247:      bf      cr7*4+2,8f
 225err1;   ld      r0,0(r4)
 226err1;   ld      r6,8(r4)
 227err1;   ld      r7,16(r4)
 228err1;   ld      r8,24(r4)
 229        addi    r4,r4,32
 230err1;   std     r0,0(r3)
 231err1;   std     r6,8(r3)
 232err1;   std     r7,16(r3)
 233err1;   std     r8,24(r3)
 234        addi    r3,r3,32
 235
 236        /* Up to 31B to go */
 2378:      bf      cr7*4+3,9f
 238err1;   ld      r0,0(r4)
 239err1;   ld      r6,8(r4)
 240        addi    r4,r4,16
 241err1;   std     r0,0(r3)
 242err1;   std     r6,8(r3)
 243        addi    r3,r3,16
 244
 2459:      clrldi  r5,r5,(64-4)
 246
 247        /* Up to 15B to go */
 248.Lshort_copy:
 249        mtocrf  0x01,r5
 250        bf      cr7*4+0,12f
 251err1;   lwz     r0,0(r4)        /* Less chance of a reject with word ops */
 252err1;   lwz     r6,4(r4)
 253        addi    r4,r4,8
 254err1;   stw     r0,0(r3)
 255err1;   stw     r6,4(r3)
 256        addi    r3,r3,8
 257
 25812:     bf      cr7*4+1,13f
 259err1;   lwz     r0,0(r4)
 260        addi    r4,r4,4
 261err1;   stw     r0,0(r3)
 262        addi    r3,r3,4
 263
 26413:     bf      cr7*4+2,14f
 265err1;   lhz     r0,0(r4)
 266        addi    r4,r4,2
 267err1;   sth     r0,0(r3)
 268        addi    r3,r3,2
 269
 27014:     bf      cr7*4+3,15f
 271err1;   lbz     r0,0(r4)
 272err1;   stb     r0,0(r3)
 273
 27415:     li      r3,0
 275        blr
 276
 277.Lunwind_stack_nonvmx_copy:
 278        addi    r1,r1,STACKFRAMESIZE
 279        b       .Lnonvmx_copy
 280
 281#ifdef CONFIG_ALTIVEC
 282.Lvmx_copy:
 283        mflr    r0
 284        std     r0,16(r1)
 285        stdu    r1,-STACKFRAMESIZE(r1)
 286        bl      enter_vmx_usercopy
 287        cmpwi   cr1,r3,0
 288        ld      r0,STACKFRAMESIZE+16(r1)
 289        ld      r3,STK_REG(R31)(r1)
 290        ld      r4,STK_REG(R30)(r1)
 291        ld      r5,STK_REG(R29)(r1)
 292        mtlr    r0
 293
 294        /*
 295         * We prefetch both the source and destination using enhanced touch
 296         * instructions. We use a stream ID of 0 for the load side and
 297         * 1 for the store side.
 298         */
 299        clrrdi  r6,r4,7
 300        clrrdi  r9,r3,7
 301        ori     r9,r9,1         /* stream=1 */
 302
 303        srdi    r7,r5,7         /* length in cachelines, capped at 0x3FF */
 304        cmpldi  r7,0x3FF
 305        ble     1f
 306        li      r7,0x3FF
 3071:      lis     r0,0x0E00       /* depth=7 */
 308        sldi    r7,r7,7
 309        or      r7,r7,r0
 310        ori     r10,r7,1        /* stream=1 */
 311
 312        lis     r8,0x8000       /* GO=1 */
 313        clrldi  r8,r8,32
 314
 315        /* setup read stream 0 */
 316        dcbt    0,r6,0b01000   /* addr from */
 317        dcbt    0,r7,0b01010   /* length and depth from */
 318        /* setup write stream 1 */
 319        dcbtst  0,r9,0b01000   /* addr to */
 320        dcbtst  0,r10,0b01010  /* length and depth to */
 321        eieio
 322        dcbt    0,r8,0b01010    /* all streams GO */
 323
 324        beq     cr1,.Lunwind_stack_nonvmx_copy
 325
 326        /*
 327         * If source and destination are not relatively aligned we use a
 328         * slower permute loop.
 329         */
 330        xor     r6,r4,r3
 331        rldicl. r6,r6,0,(64-4)
 332        bne     .Lvmx_unaligned_copy
 333
 334        /* Get the destination 16B aligned */
 335        neg     r6,r3
 336        mtocrf  0x01,r6
 337        clrldi  r6,r6,(64-4)
 338
 339        bf      cr7*4+3,1f
 340err3;   lbz     r0,0(r4)
 341        addi    r4,r4,1
 342err3;   stb     r0,0(r3)
 343        addi    r3,r3,1
 344
 3451:      bf      cr7*4+2,2f
 346err3;   lhz     r0,0(r4)
 347        addi    r4,r4,2
 348err3;   sth     r0,0(r3)
 349        addi    r3,r3,2
 350
 3512:      bf      cr7*4+1,3f
 352err3;   lwz     r0,0(r4)
 353        addi    r4,r4,4
 354err3;   stw     r0,0(r3)
 355        addi    r3,r3,4
 356
 3573:      bf      cr7*4+0,4f
 358err3;   ld      r0,0(r4)
 359        addi    r4,r4,8
 360err3;   std     r0,0(r3)
 361        addi    r3,r3,8
 362
 3634:      sub     r5,r5,r6
 364
 365        /* Get the desination 128B aligned */
 366        neg     r6,r3
 367        srdi    r7,r6,4
 368        mtocrf  0x01,r7
 369        clrldi  r6,r6,(64-7)
 370
 371        li      r9,16
 372        li      r10,32
 373        li      r11,48
 374
 375        bf      cr7*4+3,5f
 376err3;   lvx     v1,0,r4
 377        addi    r4,r4,16
 378err3;   stvx    v1,0,r3
 379        addi    r3,r3,16
 380
 3815:      bf      cr7*4+2,6f
 382err3;   lvx     v1,0,r4
 383err3;   lvx     v0,r4,r9
 384        addi    r4,r4,32
 385err3;   stvx    v1,0,r3
 386err3;   stvx    v0,r3,r9
 387        addi    r3,r3,32
 388
 3896:      bf      cr7*4+1,7f
 390err3;   lvx     v3,0,r4
 391err3;   lvx     v2,r4,r9
 392err3;   lvx     v1,r4,r10
 393err3;   lvx     v0,r4,r11
 394        addi    r4,r4,64
 395err3;   stvx    v3,0,r3
 396err3;   stvx    v2,r3,r9
 397err3;   stvx    v1,r3,r10
 398err3;   stvx    v0,r3,r11
 399        addi    r3,r3,64
 400
 4017:      sub     r5,r5,r6
 402        srdi    r6,r5,7
 403
 404        std     r14,STK_REG(R14)(r1)
 405        std     r15,STK_REG(R15)(r1)
 406        std     r16,STK_REG(R16)(r1)
 407
 408        li      r12,64
 409        li      r14,80
 410        li      r15,96
 411        li      r16,112
 412
 413        mtctr   r6
 414
 415        /*
 416         * Now do cacheline sized loads and stores. By this stage the
 417         * cacheline stores are also cacheline aligned.
 418         */
 419        .align  5
 4208:
 421err4;   lvx     v7,0,r4
 422err4;   lvx     v6,r4,r9
 423err4;   lvx     v5,r4,r10
 424err4;   lvx     v4,r4,r11
 425err4;   lvx     v3,r4,r12
 426err4;   lvx     v2,r4,r14
 427err4;   lvx     v1,r4,r15
 428err4;   lvx     v0,r4,r16
 429        addi    r4,r4,128
 430err4;   stvx    v7,0,r3
 431err4;   stvx    v6,r3,r9
 432err4;   stvx    v5,r3,r10
 433err4;   stvx    v4,r3,r11
 434err4;   stvx    v3,r3,r12
 435err4;   stvx    v2,r3,r14
 436err4;   stvx    v1,r3,r15
 437err4;   stvx    v0,r3,r16
 438        addi    r3,r3,128
 439        bdnz    8b
 440
 441        ld      r14,STK_REG(R14)(r1)
 442        ld      r15,STK_REG(R15)(r1)
 443        ld      r16,STK_REG(R16)(r1)
 444
 445        /* Up to 127B to go */
 446        clrldi  r5,r5,(64-7)
 447        srdi    r6,r5,4
 448        mtocrf  0x01,r6
 449
 450        bf      cr7*4+1,9f
 451err3;   lvx     v3,0,r4
 452err3;   lvx     v2,r4,r9
 453err3;   lvx     v1,r4,r10
 454err3;   lvx     v0,r4,r11
 455        addi    r4,r4,64
 456err3;   stvx    v3,0,r3
 457err3;   stvx    v2,r3,r9
 458err3;   stvx    v1,r3,r10
 459err3;   stvx    v0,r3,r11
 460        addi    r3,r3,64
 461
 4629:      bf      cr7*4+2,10f
 463err3;   lvx     v1,0,r4
 464err3;   lvx     v0,r4,r9
 465        addi    r4,r4,32
 466err3;   stvx    v1,0,r3
 467err3;   stvx    v0,r3,r9
 468        addi    r3,r3,32
 469
 47010:     bf      cr7*4+3,11f
 471err3;   lvx     v1,0,r4
 472        addi    r4,r4,16
 473err3;   stvx    v1,0,r3
 474        addi    r3,r3,16
 475
 476        /* Up to 15B to go */
 47711:     clrldi  r5,r5,(64-4)
 478        mtocrf  0x01,r5
 479        bf      cr7*4+0,12f
 480err3;   ld      r0,0(r4)
 481        addi    r4,r4,8
 482err3;   std     r0,0(r3)
 483        addi    r3,r3,8
 484
 48512:     bf      cr7*4+1,13f
 486err3;   lwz     r0,0(r4)
 487        addi    r4,r4,4
 488err3;   stw     r0,0(r3)
 489        addi    r3,r3,4
 490
 49113:     bf      cr7*4+2,14f
 492err3;   lhz     r0,0(r4)
 493        addi    r4,r4,2
 494err3;   sth     r0,0(r3)
 495        addi    r3,r3,2
 496
 49714:     bf      cr7*4+3,15f
 498err3;   lbz     r0,0(r4)
 499err3;   stb     r0,0(r3)
 500
 50115:     addi    r1,r1,STACKFRAMESIZE
 502        b       exit_vmx_usercopy       /* tail call optimise */
 503
 504.Lvmx_unaligned_copy:
 505        /* Get the destination 16B aligned */
 506        neg     r6,r3
 507        mtocrf  0x01,r6
 508        clrldi  r6,r6,(64-4)
 509
 510        bf      cr7*4+3,1f
 511err3;   lbz     r0,0(r4)
 512        addi    r4,r4,1
 513err3;   stb     r0,0(r3)
 514        addi    r3,r3,1
 515
 5161:      bf      cr7*4+2,2f
 517err3;   lhz     r0,0(r4)
 518        addi    r4,r4,2
 519err3;   sth     r0,0(r3)
 520        addi    r3,r3,2
 521
 5222:      bf      cr7*4+1,3f
 523err3;   lwz     r0,0(r4)
 524        addi    r4,r4,4
 525err3;   stw     r0,0(r3)
 526        addi    r3,r3,4
 527
 5283:      bf      cr7*4+0,4f
 529err3;   lwz     r0,0(r4)        /* Less chance of a reject with word ops */
 530err3;   lwz     r7,4(r4)
 531        addi    r4,r4,8
 532err3;   stw     r0,0(r3)
 533err3;   stw     r7,4(r3)
 534        addi    r3,r3,8
 535
 5364:      sub     r5,r5,r6
 537
 538        /* Get the desination 128B aligned */
 539        neg     r6,r3
 540        srdi    r7,r6,4
 541        mtocrf  0x01,r7
 542        clrldi  r6,r6,(64-7)
 543
 544        li      r9,16
 545        li      r10,32
 546        li      r11,48
 547
 548        LVS(v16,0,r4)           /* Setup permute control vector */
 549err3;   lvx     v0,0,r4
 550        addi    r4,r4,16
 551
 552        bf      cr7*4+3,5f
 553err3;   lvx     v1,0,r4
 554        VPERM(v8,v0,v1,v16)
 555        addi    r4,r4,16
 556err3;   stvx    v8,0,r3
 557        addi    r3,r3,16
 558        vor     v0,v1,v1
 559
 5605:      bf      cr7*4+2,6f
 561err3;   lvx     v1,0,r4
 562        VPERM(v8,v0,v1,v16)
 563err3;   lvx     v0,r4,r9
 564        VPERM(v9,v1,v0,v16)
 565        addi    r4,r4,32
 566err3;   stvx    v8,0,r3
 567err3;   stvx    v9,r3,r9
 568        addi    r3,r3,32
 569
 5706:      bf      cr7*4+1,7f
 571err3;   lvx     v3,0,r4
 572        VPERM(v8,v0,v3,v16)
 573err3;   lvx     v2,r4,r9
 574        VPERM(v9,v3,v2,v16)
 575err3;   lvx     v1,r4,r10
 576        VPERM(v10,v2,v1,v16)
 577err3;   lvx     v0,r4,r11
 578        VPERM(v11,v1,v0,v16)
 579        addi    r4,r4,64
 580err3;   stvx    v8,0,r3
 581err3;   stvx    v9,r3,r9
 582err3;   stvx    v10,r3,r10
 583err3;   stvx    v11,r3,r11
 584        addi    r3,r3,64
 585
 5867:      sub     r5,r5,r6
 587        srdi    r6,r5,7
 588
 589        std     r14,STK_REG(R14)(r1)
 590        std     r15,STK_REG(R15)(r1)
 591        std     r16,STK_REG(R16)(r1)
 592
 593        li      r12,64
 594        li      r14,80
 595        li      r15,96
 596        li      r16,112
 597
 598        mtctr   r6
 599
 600        /*
 601         * Now do cacheline sized loads and stores. By this stage the
 602         * cacheline stores are also cacheline aligned.
 603         */
 604        .align  5
 6058:
 606err4;   lvx     v7,0,r4
 607        VPERM(v8,v0,v7,v16)
 608err4;   lvx     v6,r4,r9
 609        VPERM(v9,v7,v6,v16)
 610err4;   lvx     v5,r4,r10
 611        VPERM(v10,v6,v5,v16)
 612err4;   lvx     v4,r4,r11
 613        VPERM(v11,v5,v4,v16)
 614err4;   lvx     v3,r4,r12
 615        VPERM(v12,v4,v3,v16)
 616err4;   lvx     v2,r4,r14
 617        VPERM(v13,v3,v2,v16)
 618err4;   lvx     v1,r4,r15
 619        VPERM(v14,v2,v1,v16)
 620err4;   lvx     v0,r4,r16
 621        VPERM(v15,v1,v0,v16)
 622        addi    r4,r4,128
 623err4;   stvx    v8,0,r3
 624err4;   stvx    v9,r3,r9
 625err4;   stvx    v10,r3,r10
 626err4;   stvx    v11,r3,r11
 627err4;   stvx    v12,r3,r12
 628err4;   stvx    v13,r3,r14
 629err4;   stvx    v14,r3,r15
 630err4;   stvx    v15,r3,r16
 631        addi    r3,r3,128
 632        bdnz    8b
 633
 634        ld      r14,STK_REG(R14)(r1)
 635        ld      r15,STK_REG(R15)(r1)
 636        ld      r16,STK_REG(R16)(r1)
 637
 638        /* Up to 127B to go */
 639        clrldi  r5,r5,(64-7)
 640        srdi    r6,r5,4
 641        mtocrf  0x01,r6
 642
 643        bf      cr7*4+1,9f
 644err3;   lvx     v3,0,r4
 645        VPERM(v8,v0,v3,v16)
 646err3;   lvx     v2,r4,r9
 647        VPERM(v9,v3,v2,v16)
 648err3;   lvx     v1,r4,r10
 649        VPERM(v10,v2,v1,v16)
 650err3;   lvx     v0,r4,r11
 651        VPERM(v11,v1,v0,v16)
 652        addi    r4,r4,64
 653err3;   stvx    v8,0,r3
 654err3;   stvx    v9,r3,r9
 655err3;   stvx    v10,r3,r10
 656err3;   stvx    v11,r3,r11
 657        addi    r3,r3,64
 658
 6599:      bf      cr7*4+2,10f
 660err3;   lvx     v1,0,r4
 661        VPERM(v8,v0,v1,v16)
 662err3;   lvx     v0,r4,r9
 663        VPERM(v9,v1,v0,v16)
 664        addi    r4,r4,32
 665err3;   stvx    v8,0,r3
 666err3;   stvx    v9,r3,r9
 667        addi    r3,r3,32
 668
 66910:     bf      cr7*4+3,11f
 670err3;   lvx     v1,0,r4
 671        VPERM(v8,v0,v1,v16)
 672        addi    r4,r4,16
 673err3;   stvx    v8,0,r3
 674        addi    r3,r3,16
 675
 676        /* Up to 15B to go */
 67711:     clrldi  r5,r5,(64-4)
 678        addi    r4,r4,-16       /* Unwind the +16 load offset */
 679        mtocrf  0x01,r5
 680        bf      cr7*4+0,12f
 681err3;   lwz     r0,0(r4)        /* Less chance of a reject with word ops */
 682err3;   lwz     r6,4(r4)
 683        addi    r4,r4,8
 684err3;   stw     r0,0(r3)
 685err3;   stw     r6,4(r3)
 686        addi    r3,r3,8
 687
 68812:     bf      cr7*4+1,13f
 689err3;   lwz     r0,0(r4)
 690        addi    r4,r4,4
 691err3;   stw     r0,0(r3)
 692        addi    r3,r3,4
 693
 69413:     bf      cr7*4+2,14f
 695err3;   lhz     r0,0(r4)
 696        addi    r4,r4,2
 697err3;   sth     r0,0(r3)
 698        addi    r3,r3,2
 699
 70014:     bf      cr7*4+3,15f
 701err3;   lbz     r0,0(r4)
 702err3;   stb     r0,0(r3)
 703
 70415:     addi    r1,r1,STACKFRAMESIZE
 705        b       exit_vmx_usercopy       /* tail call optimise */
 706#endif /* CONFIG_ALTIVEC */
 707