LXR linux/arch/mips/lib/csum

   1/*
   2 * This file is subject to the terms and conditions of the GNU General Public
   3 * License.  See the file "COPYING" in the main directory of this archive
   4 * for more details.
   5 *
   6 * Quick'n'dirty IP checksum ...
   7 *
   8 * Copyright (C) 1998, 1999 Ralf Baechle
   9 * Copyright (C) 1999 Silicon Graphics, Inc.
  10 * Copyright (C) 2007  Maciej W. Rozycki
  11 */
  12#include <linux/errno.h>
  13#include <asm/asm.h>
  14#include <asm/asm-offsets.h>
  15#include <asm/regdef.h>
  16
  17#ifdef CONFIG_64BIT
  18/*
  19 * As we are sharing code base with the mips32 tree (which use the o32 ABI
  20 * register definitions). We need to redefine the register definitions from
  21 * the n64 ABI register naming to the o32 ABI register naming.
  22 */
  23#undef t0
  24#undef t1
  25#undef t2
  26#undef t3
  27#define t0      $8
  28#define t1      $9
  29#define t2      $10
  30#define t3      $11
  31#define t4      $12
  32#define t5      $13
  33#define t6      $14
  34#define t7      $15
  35
  36#define USE_DOUBLE
  37#endif
  38
  39#ifdef USE_DOUBLE
  40
  41#define LOAD   ld
  42#define LOAD32 lwu
  43#define ADD    daddu
  44#define NBYTES 8
  45
  46#else
  47
  48#define LOAD   lw
  49#define LOAD32 lw
  50#define ADD    addu
  51#define NBYTES 4
  52
  53#endif /* USE_DOUBLE */
  54
  55#define UNIT(unit)  ((unit)*NBYTES)
  56
  57#define ADDC(sum,reg)                                           \
  58        ADD     sum, reg;                                       \
  59        sltu    v1, sum, reg;                                   \
  60        ADD     sum, v1;                                        \
  61
  62#define ADDC32(sum,reg)                                         \
  63        addu    sum, reg;                                       \
  64        sltu    v1, sum, reg;                                   \
  65        addu    sum, v1;                                        \
  66
  67#define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3)    \
  68        LOAD    _t0, (offset + UNIT(0))(src);                   \
  69        LOAD    _t1, (offset + UNIT(1))(src);                   \
  70        LOAD    _t2, (offset + UNIT(2))(src);                   \
  71        LOAD    _t3, (offset + UNIT(3))(src);                   \
  72        ADDC(sum, _t0);                                         \
  73        ADDC(sum, _t1);                                         \
  74        ADDC(sum, _t2);                                         \
  75        ADDC(sum, _t3)
  76
  77#ifdef USE_DOUBLE
  78#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3)     \
  79        CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3)
  80#else
  81#define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3)     \
  82        CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3);   \
  83        CSUM_BIGCHUNK1(src, offset + 0x10, sum, _t0, _t1, _t2, _t3)
  84#endif
  85
  86/*
  87 * a0: source address
  88 * a1: length of the area to checksum
  89 * a2: partial checksum
  90 */
  91
  92#define src a0
  93#define sum v0
  94
  95        .text
  96        .set    noreorder
  97        .align  5
  98LEAF(csum_partial)
  99        move    sum, zero
 100        move    t7, zero
 101
 102        sltiu   t8, a1, 0x8
 103        bnez    t8, .Lsmall_csumcpy             /* < 8 bytes to copy */
 104         move   t2, a1
 105
 106        andi    t7, src, 0x1                    /* odd buffer? */
 107
 108.Lhword_align:
 109        beqz    t7, .Lword_align
 110         andi   t8, src, 0x2
 111
 112        lbu     t0, (src)
 113        LONG_SUBU       a1, a1, 0x1
 114#ifdef __MIPSEL__
 115        sll     t0, t0, 8
 116#endif
 117        ADDC(sum, t0)
 118        PTR_ADDU        src, src, 0x1
 119        andi    t8, src, 0x2
 120
 121.Lword_align:
 122        beqz    t8, .Ldword_align
 123         sltiu  t8, a1, 56
 124
 125        lhu     t0, (src)
 126        LONG_SUBU       a1, a1, 0x2
 127        ADDC(sum, t0)
 128        sltiu   t8, a1, 56
 129        PTR_ADDU        src, src, 0x2
 130
 131.Ldword_align:
 132        bnez    t8, .Ldo_end_words
 133         move   t8, a1
 134
 135        andi    t8, src, 0x4
 136        beqz    t8, .Lqword_align
 137         andi   t8, src, 0x8
 138
 139        LOAD32  t0, 0x00(src)
 140        LONG_SUBU       a1, a1, 0x4
 141        ADDC(sum, t0)
 142        PTR_ADDU        src, src, 0x4
 143        andi    t8, src, 0x8
 144
 145.Lqword_align:
 146        beqz    t8, .Loword_align
 147         andi   t8, src, 0x10
 148
 149#ifdef USE_DOUBLE
 150        ld      t0, 0x00(src)
 151        LONG_SUBU       a1, a1, 0x8
 152        ADDC(sum, t0)
 153#else
 154        lw      t0, 0x00(src)
 155        lw      t1, 0x04(src)
 156        LONG_SUBU       a1, a1, 0x8
 157        ADDC(sum, t0)
 158        ADDC(sum, t1)
 159#endif
 160        PTR_ADDU        src, src, 0x8
 161        andi    t8, src, 0x10
 162
 163.Loword_align:
 164        beqz    t8, .Lbegin_movement
 165         LONG_SRL       t8, a1, 0x7
 166
 167#ifdef USE_DOUBLE
 168        ld      t0, 0x00(src)
 169        ld      t1, 0x08(src)
 170        ADDC(sum, t0)
 171        ADDC(sum, t1)
 172#else
 173        CSUM_BIGCHUNK1(src, 0x00, sum, t0, t1, t3, t4)
 174#endif
 175        LONG_SUBU       a1, a1, 0x10
 176        PTR_ADDU        src, src, 0x10
 177        LONG_SRL        t8, a1, 0x7
 178
 179.Lbegin_movement:
 180        beqz    t8, 1f
 181         andi   t2, a1, 0x40
 182
 183.Lmove_128bytes:
 184        CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
 185        CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
 186        CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4)
 187        CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4)
 188        LONG_SUBU       t8, t8, 0x01
 189        .set    reorder                         /* DADDI_WAR */
 190        PTR_ADDU        src, src, 0x80
 191        bnez    t8, .Lmove_128bytes
 192        .set    noreorder
 193
 1941:
 195        beqz    t2, 1f
 196         andi   t2, a1, 0x20
 197
 198.Lmove_64bytes:
 199        CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
 200        CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
 201        PTR_ADDU        src, src, 0x40
 202
 2031:
 204        beqz    t2, .Ldo_end_words
 205         andi   t8, a1, 0x1c
 206
 207.Lmove_32bytes:
 208        CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
 209        andi    t8, a1, 0x1c
 210        PTR_ADDU        src, src, 0x20
 211
 212.Ldo_end_words:
 213        beqz    t8, .Lsmall_csumcpy
 214         andi   t2, a1, 0x3
 215        LONG_SRL        t8, t8, 0x2
 216
 217.Lend_words:
 218        LOAD32  t0, (src)
 219        LONG_SUBU       t8, t8, 0x1
 220        ADDC(sum, t0)
 221        .set    reorder                         /* DADDI_WAR */
 222        PTR_ADDU        src, src, 0x4
 223        bnez    t8, .Lend_words
 224        .set    noreorder
 225
 226/* unknown src alignment and < 8 bytes to go  */
 227.Lsmall_csumcpy:
 228        move    a1, t2
 229
 230        andi    t0, a1, 4
 231        beqz    t0, 1f
 232         andi   t0, a1, 2
 233
 234        /* Still a full word to go  */
 235        ulw     t1, (src)
 236        PTR_ADDIU       src, 4
 237#ifdef USE_DOUBLE
 238        dsll    t1, t1, 32                      /* clear lower 32bit */
 239#endif
 240        ADDC(sum, t1)
 241
 2421:      move    t1, zero
 243        beqz    t0, 1f
 244         andi   t0, a1, 1
 245
 246        /* Still a halfword to go  */
 247        ulhu    t1, (src)
 248        PTR_ADDIU       src, 2
 249
 2501:      beqz    t0, 1f
 251         sll    t1, t1, 16
 252
 253        lbu     t2, (src)
 254         nop
 255
 256#ifdef __MIPSEB__
 257        sll     t2, t2, 8
 258#endif
 259        or      t1, t2
 260
 2611:      ADDC(sum, t1)
 262
 263        /* fold checksum */
 264#ifdef USE_DOUBLE
 265        dsll32  v1, sum, 0
 266        daddu   sum, v1
 267        sltu    v1, sum, v1
 268        dsra32  sum, sum, 0
 269        addu    sum, v1
 270#endif
 271
 272        /* odd buffer alignment? */
 273#ifdef CONFIG_CPU_MIPSR2
 274        wsbh    v1, sum
 275        movn    sum, v1, t7
 276#else
 277        beqz    t7, 1f                  /* odd buffer alignment? */
 278         lui    v1, 0x00ff
 279        addu    v1, 0x00ff
 280        and     t0, sum, v1
 281        sll     t0, t0, 8
 282        srl     sum, sum, 8
 283        and     sum, sum, v1
 284        or      sum, sum, t0
 2851:
 286#endif
 287        .set    reorder
 288        /* Add the passed partial csum.  */
 289        ADDC32(sum, a2)
 290        jr      ra
 291        .set    noreorder
 292        END(csum_partial)
 293
 294
 295/*
 296 * checksum and copy routines based on memcpy.S
 297 *
 298 *      csum_partial_copy_nocheck(src, dst, len, sum)
 299 *      __csum_partial_copy_user(src, dst, len, sum, errp)
 300 *
 301 * See "Spec" in memcpy.S for details.  Unlike __copy_user, all
 302 * function in this file use the standard calling convention.
 303 */
 304
 305#define src a0
 306#define dst a1
 307#define len a2
 308#define psum a3
 309#define sum v0
 310#define odd t8
 311#define errptr t9
 312
 313/*
 314 * The exception handler for loads requires that:
 315 *  1- AT contain the address of the byte just past the end of the source
 316 *     of the copy,
 317 *  2- src_entry <= src < AT, and
 318 *  3- (dst - src) == (dst_entry - src_entry),
 319 * The _entry suffix denotes values when __copy_user was called.
 320 *
 321 * (1) is set up up by __csum_partial_copy_from_user and maintained by
 322 *      not writing AT in __csum_partial_copy
 323 * (2) is met by incrementing src by the number of bytes copied
 324 * (3) is met by not doing loads between a pair of increments of dst and src
 325 *
 326 * The exception handlers for stores stores -EFAULT to errptr and return.
 327 * These handlers do not need to overwrite any data.
 328 */
 329
 330#define EXC(inst_reg,addr,handler)              \
 3319:      inst_reg, addr;                         \
 332        .section __ex_table,"a";                \
 333        PTR     9b, handler;                    \
 334        .previous
 335
 336#ifdef USE_DOUBLE
 337
 338#define LOAD   ld
 339#define LOADL  ldl
 340#define LOADR  ldr
 341#define STOREL sdl
 342#define STORER sdr
 343#define STORE  sd
 344#define ADD    daddu
 345#define SUB    dsubu
 346#define SRL    dsrl
 347#define SLL    dsll
 348#define SLLV   dsllv
 349#define SRLV   dsrlv
 350#define NBYTES 8
 351#define LOG_NBYTES 3
 352
 353#else
 354
 355#define LOAD   lw
 356#define LOADL  lwl
 357#define LOADR  lwr
 358#define STOREL swl
 359#define STORER swr
 360#define STORE  sw
 361#define ADD    addu
 362#define SUB    subu
 363#define SRL    srl
 364#define SLL    sll
 365#define SLLV   sllv
 366#define SRLV   srlv
 367#define NBYTES 4
 368#define LOG_NBYTES 2
 369
 370#endif /* USE_DOUBLE */
 371
 372#ifdef CONFIG_CPU_LITTLE_ENDIAN
 373#define LDFIRST LOADR
 374#define LDREST  LOADL
 375#define STFIRST STORER
 376#define STREST  STOREL
 377#define SHIFT_DISCARD SLLV
 378#define SHIFT_DISCARD_REVERT SRLV
 379#else
 380#define LDFIRST LOADL
 381#define LDREST  LOADR
 382#define STFIRST STOREL
 383#define STREST  STORER
 384#define SHIFT_DISCARD SRLV
 385#define SHIFT_DISCARD_REVERT SLLV
 386#endif
 387
 388#define FIRST(unit) ((unit)*NBYTES)
 389#define REST(unit)  (FIRST(unit)+NBYTES-1)
 390
 391#define ADDRMASK (NBYTES-1)
 392
 393#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
 394        .set    noat
 395#else
 396        .set    at=v1
 397#endif
 398
 399LEAF(__csum_partial_copy_user)
 400        PTR_ADDU        AT, src, len    /* See (1) above. */
 401#ifdef CONFIG_64BIT
 402        move    errptr, a4
 403#else
 404        lw      errptr, 16(sp)
 405#endif
 406FEXPORT(csum_partial_copy_nocheck)
 407        move    sum, zero
 408        move    odd, zero
 409        /*
 410         * Note: dst & src may be unaligned, len may be 0
 411         * Temps
 412         */
 413        /*
 414         * The "issue break"s below are very approximate.
 415         * Issue delays for dcache fills will perturb the schedule, as will
 416         * load queue full replay traps, etc.
 417         *
 418         * If len < NBYTES use byte operations.
 419         */
 420        sltu    t2, len, NBYTES
 421        and     t1, dst, ADDRMASK
 422        bnez    t2, .Lcopy_bytes_checklen
 423         and    t0, src, ADDRMASK
 424        andi    odd, dst, 0x1                   /* odd buffer? */
 425        bnez    t1, .Ldst_unaligned
 426         nop
 427        bnez    t0, .Lsrc_unaligned_dst_aligned
 428        /*
 429         * use delay slot for fall-through
 430         * src and dst are aligned; need to compute rem
 431         */
 432.Lboth_aligned:
 433         SRL    t0, len, LOG_NBYTES+3    # +3 for 8 units/iter
 434        beqz    t0, .Lcleanup_both_aligned # len < 8*NBYTES
 435         nop
 436        SUB     len, 8*NBYTES           # subtract here for bgez loop
 437        .align  4
 4381:
 439EXC(    LOAD    t0, UNIT(0)(src),       .Ll_exc)
 440EXC(    LOAD    t1, UNIT(1)(src),       .Ll_exc_copy)
 441EXC(    LOAD    t2, UNIT(2)(src),       .Ll_exc_copy)
 442EXC(    LOAD    t3, UNIT(3)(src),       .Ll_exc_copy)
 443EXC(    LOAD    t4, UNIT(4)(src),       .Ll_exc_copy)
 444EXC(    LOAD    t5, UNIT(5)(src),       .Ll_exc_copy)
 445EXC(    LOAD    t6, UNIT(6)(src),       .Ll_exc_copy)
 446EXC(    LOAD    t7, UNIT(7)(src),       .Ll_exc_copy)
 447        SUB     len, len, 8*NBYTES
 448        ADD     src, src, 8*NBYTES
 449EXC(    STORE   t0, UNIT(0)(dst),       .Ls_exc)
 450        ADDC(sum, t0)
 451EXC(    STORE   t1, UNIT(1)(dst),       .Ls_exc)
 452        ADDC(sum, t1)
 453EXC(    STORE   t2, UNIT(2)(dst),       .Ls_exc)
 454        ADDC(sum, t2)
 455EXC(    STORE   t3, UNIT(3)(dst),       .Ls_exc)
 456        ADDC(sum, t3)
 457EXC(    STORE   t4, UNIT(4)(dst),       .Ls_exc)
 458        ADDC(sum, t4)
 459EXC(    STORE   t5, UNIT(5)(dst),       .Ls_exc)
 460        ADDC(sum, t5)
 461EXC(    STORE   t6, UNIT(6)(dst),       .Ls_exc)
 462        ADDC(sum, t6)
 463EXC(    STORE   t7, UNIT(7)(dst),       .Ls_exc)
 464        ADDC(sum, t7)
 465        .set    reorder                         /* DADDI_WAR */
 466        ADD     dst, dst, 8*NBYTES
 467        bgez    len, 1b
 468        .set    noreorder
 469        ADD     len, 8*NBYTES           # revert len (see above)
 470
 471        /*
 472         * len == the number of bytes left to copy < 8*NBYTES
 473         */
 474.Lcleanup_both_aligned:
 475#define rem t7
 476        beqz    len, .Ldone
 477         sltu   t0, len, 4*NBYTES
 478        bnez    t0, .Lless_than_4units
 479         and    rem, len, (NBYTES-1)    # rem = len % NBYTES
 480        /*
 481         * len >= 4*NBYTES
 482         */
 483EXC(    LOAD    t0, UNIT(0)(src),       .Ll_exc)
 484EXC(    LOAD    t1, UNIT(1)(src),       .Ll_exc_copy)
 485EXC(    LOAD    t2, UNIT(2)(src),       .Ll_exc_copy)
 486EXC(    LOAD    t3, UNIT(3)(src),       .Ll_exc_copy)
 487        SUB     len, len, 4*NBYTES
 488        ADD     src, src, 4*NBYTES
 489EXC(    STORE   t0, UNIT(0)(dst),       .Ls_exc)
 490        ADDC(sum, t0)
 491EXC(    STORE   t1, UNIT(1)(dst),       .Ls_exc)
 492        ADDC(sum, t1)
 493EXC(    STORE   t2, UNIT(2)(dst),       .Ls_exc)
 494        ADDC(sum, t2)
 495EXC(    STORE   t3, UNIT(3)(dst),       .Ls_exc)
 496        ADDC(sum, t3)
 497        .set    reorder                         /* DADDI_WAR */
 498        ADD     dst, dst, 4*NBYTES
 499        beqz    len, .Ldone
 500        .set    noreorder
 501.Lless_than_4units:
 502        /*
 503         * rem = len % NBYTES
 504         */
 505        beq     rem, len, .Lcopy_bytes
 506         nop
 5071:
 508EXC(    LOAD    t0, 0(src),             .Ll_exc)
 509        ADD     src, src, NBYTES
 510        SUB     len, len, NBYTES
 511EXC(    STORE   t0, 0(dst),             .Ls_exc)
 512        ADDC(sum, t0)
 513        .set    reorder                         /* DADDI_WAR */
 514        ADD     dst, dst, NBYTES
 515        bne     rem, len, 1b
 516        .set    noreorder
 517
 518        /*
 519         * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
 520         * A loop would do only a byte at a time with possible branch
 521         * mispredicts.  Can't do an explicit LOAD dst,mask,or,STORE
 522         * because can't assume read-access to dst.  Instead, use
 523         * STREST dst, which doesn't require read access to dst.
 524         *
 525         * This code should perform better than a simple loop on modern,
 526         * wide-issue mips processors because the code has fewer branches and
 527         * more instruction-level parallelism.
 528         */
 529#define bits t2
 530        beqz    len, .Ldone
 531         ADD    t1, dst, len    # t1 is just past last byte of dst
 532        li      bits, 8*NBYTES
 533        SLL     rem, len, 3     # rem = number of bits to keep
 534EXC(    LOAD    t0, 0(src),             .Ll_exc)
 535        SUB     bits, bits, rem # bits = number of bits to discard
 536        SHIFT_DISCARD t0, t0, bits
 537EXC(    STREST  t0, -1(t1),             .Ls_exc)
 538        SHIFT_DISCARD_REVERT t0, t0, bits
 539        .set reorder
 540        ADDC(sum, t0)
 541        b       .Ldone
 542        .set noreorder
 543.Ldst_unaligned:
 544        /*
 545         * dst is unaligned
 546         * t0 = src & ADDRMASK
 547         * t1 = dst & ADDRMASK; T1 > 0
 548         * len >= NBYTES
 549         *
 550         * Copy enough bytes to align dst
 551         * Set match = (src and dst have same alignment)
 552         */
 553#define match rem
 554EXC(    LDFIRST t3, FIRST(0)(src),      .Ll_exc)
 555        ADD     t2, zero, NBYTES
 556EXC(    LDREST  t3, REST(0)(src),       .Ll_exc_copy)
 557        SUB     t2, t2, t1      # t2 = number of bytes copied
 558        xor     match, t0, t1
 559EXC(    STFIRST t3, FIRST(0)(dst),      .Ls_exc)
 560        SLL     t4, t1, 3               # t4 = number of bits to discard
 561        SHIFT_DISCARD t3, t3, t4
 562        /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
 563        ADDC(sum, t3)
 564        beq     len, t2, .Ldone
 565         SUB    len, len, t2
 566        ADD     dst, dst, t2
 567        beqz    match, .Lboth_aligned
 568         ADD    src, src, t2
 569
 570.Lsrc_unaligned_dst_aligned:
 571        SRL     t0, len, LOG_NBYTES+2    # +2 for 4 units/iter
 572        beqz    t0, .Lcleanup_src_unaligned
 573         and    rem, len, (4*NBYTES-1)   # rem = len % 4*NBYTES
 5741:
 575/*
 576 * Avoid consecutive LD*'s to the same register since some mips
 577 * implementations can't issue them in the same cycle.
 578 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
 579 * are to the same unit (unless src is aligned, but it's not).
 580 */
 581EXC(    LDFIRST t0, FIRST(0)(src),      .Ll_exc)
 582EXC(    LDFIRST t1, FIRST(1)(src),      .Ll_exc_copy)
 583        SUB     len, len, 4*NBYTES
 584EXC(    LDREST  t0, REST(0)(src),       .Ll_exc_copy)
 585EXC(    LDREST  t1, REST(1)(src),       .Ll_exc_copy)
 586EXC(    LDFIRST t2, FIRST(2)(src),      .Ll_exc_copy)
 587EXC(    LDFIRST t3, FIRST(3)(src),      .Ll_exc_copy)
 588EXC(    LDREST  t2, REST(2)(src),       .Ll_exc_copy)
 589EXC(    LDREST  t3, REST(3)(src),       .Ll_exc_copy)
 590        ADD     src, src, 4*NBYTES
 591#ifdef CONFIG_CPU_SB1
 592        nop                             # improves slotting
 593#endif
 594EXC(    STORE   t0, UNIT(0)(dst),       .Ls_exc)
 595        ADDC(sum, t0)
 596EXC(    STORE   t1, UNIT(1)(dst),       .Ls_exc)
 597        ADDC(sum, t1)
 598EXC(    STORE   t2, UNIT(2)(dst),       .Ls_exc)
 599        ADDC(sum, t2)
 600EXC(    STORE   t3, UNIT(3)(dst),       .Ls_exc)
 601        ADDC(sum, t3)
 602        .set    reorder                         /* DADDI_WAR */
 603        ADD     dst, dst, 4*NBYTES
 604        bne     len, rem, 1b
 605        .set    noreorder
 606
 607.Lcleanup_src_unaligned:
 608        beqz    len, .Ldone
 609         and    rem, len, NBYTES-1  # rem = len % NBYTES
 610        beq     rem, len, .Lcopy_bytes
 611         nop
 6121:
 613EXC(    LDFIRST t0, FIRST(0)(src),      .Ll_exc)
 614EXC(    LDREST  t0, REST(0)(src),       .Ll_exc_copy)
 615        ADD     src, src, NBYTES
 616        SUB     len, len, NBYTES
 617EXC(    STORE   t0, 0(dst),             .Ls_exc)
 618        ADDC(sum, t0)
 619        .set    reorder                         /* DADDI_WAR */
 620        ADD     dst, dst, NBYTES
 621        bne     len, rem, 1b
 622        .set    noreorder
 623
 624.Lcopy_bytes_checklen:
 625        beqz    len, .Ldone
 626         nop
 627.Lcopy_bytes:
 628        /* 0 < len < NBYTES  */
 629#ifdef CONFIG_CPU_LITTLE_ENDIAN
 630#define SHIFT_START 0
 631#define SHIFT_INC 8
 632#else
 633#define SHIFT_START 8*(NBYTES-1)
 634#define SHIFT_INC -8
 635#endif
 636        move    t2, zero        # partial word
 637        li      t3, SHIFT_START # shift
 638/* use .Ll_exc_copy here to return correct sum on fault */
 639#define COPY_BYTE(N)                    \
 640EXC(    lbu     t0, N(src), .Ll_exc_copy);      \
 641        SUB     len, len, 1;            \
 642EXC(    sb      t0, N(dst), .Ls_exc);   \
 643        SLLV    t0, t0, t3;             \
 644        addu    t3, SHIFT_INC;          \
 645        beqz    len, .Lcopy_bytes_done; \
 646         or     t2, t0
 647
 648        COPY_BYTE(0)
 649        COPY_BYTE(1)
 650#ifdef USE_DOUBLE
 651        COPY_BYTE(2)
 652        COPY_BYTE(3)
 653        COPY_BYTE(4)
 654        COPY_BYTE(5)
 655#endif
 656EXC(    lbu     t0, NBYTES-2(src), .Ll_exc_copy)
 657        SUB     len, len, 1
 658EXC(    sb      t0, NBYTES-2(dst), .Ls_exc)
 659        SLLV    t0, t0, t3
 660        or      t2, t0
 661.Lcopy_bytes_done:
 662        ADDC(sum, t2)
 663.Ldone:
 664        /* fold checksum */
 665#ifdef USE_DOUBLE
 666        dsll32  v1, sum, 0
 667        daddu   sum, v1
 668        sltu    v1, sum, v1
 669        dsra32  sum, sum, 0
 670        addu    sum, v1
 671#endif
 672
 673#ifdef CONFIG_CPU_MIPSR2
 674        wsbh    v1, sum
 675        movn    sum, v1, odd
 676#else
 677        beqz    odd, 1f                 /* odd buffer alignment? */
 678         lui    v1, 0x00ff
 679        addu    v1, 0x00ff
 680        and     t0, sum, v1
 681        sll     t0, t0, 8
 682        srl     sum, sum, 8
 683        and     sum, sum, v1
 684        or      sum, sum, t0
 6851:
 686#endif
 687        .set reorder
 688        ADDC32(sum, psum)
 689        jr      ra
 690        .set noreorder
 691
 692.Ll_exc_copy:
 693        /*
 694         * Copy bytes from src until faulting load address (or until a
 695         * lb faults)
 696         *
 697         * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
 698         * may be more than a byte beyond the last address.
 699         * Hence, the lb below may get an exception.
 700         *
 701         * Assumes src < THREAD_BUADDR($28)
 702         */
 703        LOAD    t0, TI_TASK($28)
 704         li     t2, SHIFT_START
 705        LOAD    t0, THREAD_BUADDR(t0)
 7061:
 707EXC(    lbu     t1, 0(src),     .Ll_exc)
 708        ADD     src, src, 1
 709        sb      t1, 0(dst)      # can't fault -- we're copy_from_user
 710        SLLV    t1, t1, t2
 711        addu    t2, SHIFT_INC
 712        ADDC(sum, t1)
 713        .set    reorder                         /* DADDI_WAR */
 714        ADD     dst, dst, 1
 715        bne     src, t0, 1b
 716        .set    noreorder
 717.Ll_exc:
 718        LOAD    t0, TI_TASK($28)
 719         nop
 720        LOAD    t0, THREAD_BUADDR(t0)   # t0 is just past last good address
 721         nop
 722        SUB     len, AT, t0             # len number of uncopied bytes
 723        /*
 724         * Here's where we rely on src and dst being incremented in tandem,
 725         *   See (3) above.
 726         * dst += (fault addr - src) to put dst at first byte to clear
 727         */
 728        ADD     dst, t0                 # compute start address in a1
 729        SUB     dst, src
 730        /*
 731         * Clear len bytes starting at dst.  Can't call __bzero because it
 732         * might modify len.  An inefficient loop for these rare times...
 733         */
 734        .set    reorder                         /* DADDI_WAR */
 735        SUB     src, len, 1
 736        beqz    len, .Ldone
 737        .set    noreorder
 7381:      sb      zero, 0(dst)
 739        ADD     dst, dst, 1
 740        .set    push
 741        .set    noat
 742#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
 743        bnez    src, 1b
 744         SUB    src, src, 1
 745#else
 746        li      v1, 1
 747        bnez    src, 1b
 748         SUB    src, src, v1
 749#endif
 750        li      v1, -EFAULT
 751        b       .Ldone
 752         sw     v1, (errptr)
 753
 754.Ls_exc:
 755        li      v0, -1 /* invalid checksum */
 756        li      v1, -EFAULT
 757        jr      ra
 758         sw     v1, (errptr)
 759        .set    pop
 760        END(__csum_partial_copy_user)
 761