LXR linux/arch/mips/cavium-octeon/octeon-memcpy.S

   1/*
   2 * This file is subject to the terms and conditions of the GNU General Public
   3 * License.  See the file "COPYING" in the main directory of this archive
   4 * for more details.
   5 *
   6 * Unified implementation of memcpy, memmove and the __copy_user backend.
   7 *
   8 * Copyright (C) 1998, 99, 2000, 01, 2002 Ralf Baechle (ralf@gnu.org)
   9 * Copyright (C) 1999, 2000, 01, 2002 Silicon Graphics, Inc.
  10 * Copyright (C) 2002 Broadcom, Inc.
  11 *   memcpy/copy_user author: Mark Vandevoorde
  12 *
  13 * Mnemonic names for arguments to memcpy/__copy_user
  14 */
  15
  16#include <asm/asm.h>
  17#include <asm/asm-offsets.h>
  18#include <asm/regdef.h>
  19
  20#define dst a0
  21#define src a1
  22#define len a2
  23
  24/*
  25 * Spec
  26 *
  27 * memcpy copies len bytes from src to dst and sets v0 to dst.
  28 * It assumes that
  29 *   - src and dst don't overlap
  30 *   - src is readable
  31 *   - dst is writable
  32 * memcpy uses the standard calling convention
  33 *
  34 * __copy_user copies up to len bytes from src to dst and sets a2 (len) to
  35 * the number of uncopied bytes due to an exception caused by a read or write.
  36 * __copy_user assumes that src and dst don't overlap, and that the call is
  37 * implementing one of the following:
  38 *   copy_to_user
  39 *     - src is readable  (no exceptions when reading src)
  40 *   copy_from_user
  41 *     - dst is writable  (no exceptions when writing dst)
  42 * __copy_user uses a non-standard calling convention; see
  43 * arch/mips/include/asm/uaccess.h
  44 *
  45 * When an exception happens on a load, the handler must
  46 # ensure that all of the destination buffer is overwritten to prevent
  47 * leaking information to user mode programs.
  48 */
  49
  50/*
  51 * Implementation
  52 */
  53
  54/*
  55 * The exception handler for loads requires that:
  56 *  1- AT contain the address of the byte just past the end of the source
  57 *     of the copy,
  58 *  2- src_entry <= src < AT, and
  59 *  3- (dst - src) == (dst_entry - src_entry),
  60 * The _entry suffix denotes values when __copy_user was called.
  61 *
  62 * (1) is set up up by uaccess.h and maintained by not writing AT in copy_user
  63 * (2) is met by incrementing src by the number of bytes copied
  64 * (3) is met by not doing loads between a pair of increments of dst and src
  65 *
  66 * The exception handlers for stores adjust len (if necessary) and return.
  67 * These handlers do not need to overwrite any data.
  68 *
  69 * For __rmemcpy and memmove an exception is always a kernel bug, therefore
  70 * they're not protected.
  71 */
  72
  73#define EXC(inst_reg,addr,handler)              \
  749:      inst_reg, addr;                         \
  75        .section __ex_table,"a";                \
  76        PTR     9b, handler;                    \
  77        .previous
  78
  79/*
  80 * Only on the 64-bit kernel we can made use of 64-bit registers.
  81 */
  82#ifdef CONFIG_64BIT
  83#define USE_DOUBLE
  84#endif
  85
  86#ifdef USE_DOUBLE
  87
  88#define LOAD   ld
  89#define LOADL  ldl
  90#define LOADR  ldr
  91#define STOREL sdl
  92#define STORER sdr
  93#define STORE  sd
  94#define ADD    daddu
  95#define SUB    dsubu
  96#define SRL    dsrl
  97#define SRA    dsra
  98#define SLL    dsll
  99#define SLLV   dsllv
 100#define SRLV   dsrlv
 101#define NBYTES 8
 102#define LOG_NBYTES 3
 103
 104/*
 105 * As we are sharing code base with the mips32 tree (which use the o32 ABI
 106 * register definitions). We need to redefine the register definitions from
 107 * the n64 ABI register naming to the o32 ABI register naming.
 108 */
 109#undef t0
 110#undef t1
 111#undef t2
 112#undef t3
 113#define t0      $8
 114#define t1      $9
 115#define t2      $10
 116#define t3      $11
 117#define t4      $12
 118#define t5      $13
 119#define t6      $14
 120#define t7      $15
 121
 122#else
 123
 124#define LOAD   lw
 125#define LOADL  lwl
 126#define LOADR  lwr
 127#define STOREL swl
 128#define STORER swr
 129#define STORE  sw
 130#define ADD    addu
 131#define SUB    subu
 132#define SRL    srl
 133#define SLL    sll
 134#define SRA    sra
 135#define SLLV   sllv
 136#define SRLV   srlv
 137#define NBYTES 4
 138#define LOG_NBYTES 2
 139
 140#endif /* USE_DOUBLE */
 141
 142#ifdef CONFIG_CPU_LITTLE_ENDIAN
 143#define LDFIRST LOADR
 144#define LDREST  LOADL
 145#define STFIRST STORER
 146#define STREST  STOREL
 147#define SHIFT_DISCARD SLLV
 148#else
 149#define LDFIRST LOADL
 150#define LDREST  LOADR
 151#define STFIRST STOREL
 152#define STREST  STORER
 153#define SHIFT_DISCARD SRLV
 154#endif
 155
 156#define FIRST(unit) ((unit)*NBYTES)
 157#define REST(unit)  (FIRST(unit)+NBYTES-1)
 158#define UNIT(unit)  FIRST(unit)
 159
 160#define ADDRMASK (NBYTES-1)
 161
 162        .text
 163        .set    noreorder
 164        .set    noat
 165
 166/*
 167 * A combined memcpy/__copy_user
 168 * __copy_user sets len to 0 for success; else to an upper bound of
 169 * the number of uncopied bytes.
 170 * memcpy sets v0 to dst.
 171 */
 172        .align  5
 173LEAF(memcpy)                                    /* a0=dst a1=src a2=len */
 174        move    v0, dst                         /* return value */
 175__memcpy:
 176FEXPORT(__copy_user)
 177        /*
 178         * Note: dst & src may be unaligned, len may be 0
 179         * Temps
 180         */
 181        #
 182        # Octeon doesn't care if the destination is unaligned. The hardware
 183        # can fix it faster than we can special case the assembly.
 184        #
 185        pref    0, 0(src)
 186        sltu    t0, len, NBYTES         # Check if < 1 word
 187        bnez    t0, copy_bytes_checklen
 188         and    t0, src, ADDRMASK       # Check if src unaligned
 189        bnez    t0, src_unaligned
 190         sltu   t0, len, 4*NBYTES       # Check if < 4 words
 191        bnez    t0, less_than_4units
 192         sltu   t0, len, 8*NBYTES       # Check if < 8 words
 193        bnez    t0, less_than_8units
 194         sltu   t0, len, 16*NBYTES      # Check if < 16 words
 195        bnez    t0, cleanup_both_aligned
 196         sltu   t0, len, 128+1          # Check if len < 129
 197        bnez    t0, 1f                  # Skip prefetch if len is too short
 198         sltu   t0, len, 256+1          # Check if len < 257
 199        bnez    t0, 1f                  # Skip prefetch if len is too short
 200         pref   0, 128(src)             # We must not prefetch invalid addresses
 201        #
 202        # This is where we loop if there is more than 128 bytes left
 2032:      pref    0, 256(src)             # We must not prefetch invalid addresses
 204        #
 205        # This is where we loop if we can't prefetch anymore
 2061:
 207EXC(    LOAD    t0, UNIT(0)(src),       l_exc)
 208EXC(    LOAD    t1, UNIT(1)(src),       l_exc_copy)
 209EXC(    LOAD    t2, UNIT(2)(src),       l_exc_copy)
 210EXC(    LOAD    t3, UNIT(3)(src),       l_exc_copy)
 211        SUB     len, len, 16*NBYTES
 212EXC(    STORE   t0, UNIT(0)(dst),       s_exc_p16u)
 213EXC(    STORE   t1, UNIT(1)(dst),       s_exc_p15u)
 214EXC(    STORE   t2, UNIT(2)(dst),       s_exc_p14u)
 215EXC(    STORE   t3, UNIT(3)(dst),       s_exc_p13u)
 216EXC(    LOAD    t0, UNIT(4)(src),       l_exc_copy)
 217EXC(    LOAD    t1, UNIT(5)(src),       l_exc_copy)
 218EXC(    LOAD    t2, UNIT(6)(src),       l_exc_copy)
 219EXC(    LOAD    t3, UNIT(7)(src),       l_exc_copy)
 220EXC(    STORE   t0, UNIT(4)(dst),       s_exc_p12u)
 221EXC(    STORE   t1, UNIT(5)(dst),       s_exc_p11u)
 222EXC(    STORE   t2, UNIT(6)(dst),       s_exc_p10u)
 223        ADD     src, src, 16*NBYTES
 224EXC(    STORE   t3, UNIT(7)(dst),       s_exc_p9u)
 225        ADD     dst, dst, 16*NBYTES
 226EXC(    LOAD    t0, UNIT(-8)(src),      l_exc_copy)
 227EXC(    LOAD    t1, UNIT(-7)(src),      l_exc_copy)
 228EXC(    LOAD    t2, UNIT(-6)(src),      l_exc_copy)
 229EXC(    LOAD    t3, UNIT(-5)(src),      l_exc_copy)
 230EXC(    STORE   t0, UNIT(-8)(dst),      s_exc_p8u)
 231EXC(    STORE   t1, UNIT(-7)(dst),      s_exc_p7u)
 232EXC(    STORE   t2, UNIT(-6)(dst),      s_exc_p6u)
 233EXC(    STORE   t3, UNIT(-5)(dst),      s_exc_p5u)
 234EXC(    LOAD    t0, UNIT(-4)(src),      l_exc_copy)
 235EXC(    LOAD    t1, UNIT(-3)(src),      l_exc_copy)
 236EXC(    LOAD    t2, UNIT(-2)(src),      l_exc_copy)
 237EXC(    LOAD    t3, UNIT(-1)(src),      l_exc_copy)
 238EXC(    STORE   t0, UNIT(-4)(dst),      s_exc_p4u)
 239EXC(    STORE   t1, UNIT(-3)(dst),      s_exc_p3u)
 240EXC(    STORE   t2, UNIT(-2)(dst),      s_exc_p2u)
 241EXC(    STORE   t3, UNIT(-1)(dst),      s_exc_p1u)
 242        sltu    t0, len, 256+1          # See if we can prefetch more
 243        beqz    t0, 2b
 244         sltu   t0, len, 128            # See if we can loop more time
 245        beqz    t0, 1b
 246         nop
 247        #
 248        # Jump here if there are less than 16*NBYTES left.
 249        #
 250cleanup_both_aligned:
 251        beqz    len, done
 252         sltu   t0, len, 8*NBYTES
 253        bnez    t0, less_than_8units
 254         nop
 255EXC(    LOAD    t0, UNIT(0)(src),       l_exc)
 256EXC(    LOAD    t1, UNIT(1)(src),       l_exc_copy)
 257EXC(    LOAD    t2, UNIT(2)(src),       l_exc_copy)
 258EXC(    LOAD    t3, UNIT(3)(src),       l_exc_copy)
 259        SUB     len, len, 8*NBYTES
 260EXC(    STORE   t0, UNIT(0)(dst),       s_exc_p8u)
 261EXC(    STORE   t1, UNIT(1)(dst),       s_exc_p7u)
 262EXC(    STORE   t2, UNIT(2)(dst),       s_exc_p6u)
 263EXC(    STORE   t3, UNIT(3)(dst),       s_exc_p5u)
 264EXC(    LOAD    t0, UNIT(4)(src),       l_exc_copy)
 265EXC(    LOAD    t1, UNIT(5)(src),       l_exc_copy)
 266EXC(    LOAD    t2, UNIT(6)(src),       l_exc_copy)
 267EXC(    LOAD    t3, UNIT(7)(src),       l_exc_copy)
 268EXC(    STORE   t0, UNIT(4)(dst),       s_exc_p4u)
 269EXC(    STORE   t1, UNIT(5)(dst),       s_exc_p3u)
 270EXC(    STORE   t2, UNIT(6)(dst),       s_exc_p2u)
 271EXC(    STORE   t3, UNIT(7)(dst),       s_exc_p1u)
 272        ADD     src, src, 8*NBYTES
 273        beqz    len, done
 274         ADD    dst, dst, 8*NBYTES
 275        #
 276        # Jump here if there are less than 8*NBYTES left.
 277        #
 278less_than_8units:
 279        sltu    t0, len, 4*NBYTES
 280        bnez    t0, less_than_4units
 281         nop
 282EXC(    LOAD    t0, UNIT(0)(src),       l_exc)
 283EXC(    LOAD    t1, UNIT(1)(src),       l_exc_copy)
 284EXC(    LOAD    t2, UNIT(2)(src),       l_exc_copy)
 285EXC(    LOAD    t3, UNIT(3)(src),       l_exc_copy)
 286        SUB     len, len, 4*NBYTES
 287EXC(    STORE   t0, UNIT(0)(dst),       s_exc_p4u)
 288EXC(    STORE   t1, UNIT(1)(dst),       s_exc_p3u)
 289EXC(    STORE   t2, UNIT(2)(dst),       s_exc_p2u)
 290EXC(    STORE   t3, UNIT(3)(dst),       s_exc_p1u)
 291        ADD     src, src, 4*NBYTES
 292        beqz    len, done
 293         ADD    dst, dst, 4*NBYTES
 294        #
 295        # Jump here if there are less than 4*NBYTES left. This means
 296        # we may need to copy up to 3 NBYTES words.
 297        #
 298less_than_4units:
 299        sltu    t0, len, 1*NBYTES
 300        bnez    t0, copy_bytes_checklen
 301         nop
 302        #
 303        # 1) Copy NBYTES, then check length again
 304        #
 305EXC(    LOAD    t0, 0(src),             l_exc)
 306        SUB     len, len, NBYTES
 307        sltu    t1, len, 8
 308EXC(    STORE   t0, 0(dst),             s_exc_p1u)
 309        ADD     src, src, NBYTES
 310        bnez    t1, copy_bytes_checklen
 311         ADD    dst, dst, NBYTES
 312        #
 313        # 2) Copy NBYTES, then check length again
 314        #
 315EXC(    LOAD    t0, 0(src),             l_exc)
 316        SUB     len, len, NBYTES
 317        sltu    t1, len, 8
 318EXC(    STORE   t0, 0(dst),             s_exc_p1u)
 319        ADD     src, src, NBYTES
 320        bnez    t1, copy_bytes_checklen
 321         ADD    dst, dst, NBYTES
 322        #
 323        # 3) Copy NBYTES, then check length again
 324        #
 325EXC(    LOAD    t0, 0(src),             l_exc)
 326        SUB     len, len, NBYTES
 327        ADD     src, src, NBYTES
 328        ADD     dst, dst, NBYTES
 329        b copy_bytes_checklen
 330EXC(     STORE  t0, -8(dst),            s_exc_p1u)
 331
 332src_unaligned:
 333#define rem t8
 334        SRL     t0, len, LOG_NBYTES+2    # +2 for 4 units/iter
 335        beqz    t0, cleanup_src_unaligned
 336         and    rem, len, (4*NBYTES-1)   # rem = len % 4*NBYTES
 3371:
 338/*
 339 * Avoid consecutive LD*'s to the same register since some mips
 340 * implementations can't issue them in the same cycle.
 341 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
 342 * are to the same unit (unless src is aligned, but it's not).
 343 */
 344EXC(    LDFIRST t0, FIRST(0)(src),      l_exc)
 345EXC(    LDFIRST t1, FIRST(1)(src),      l_exc_copy)
 346        SUB     len, len, 4*NBYTES
 347EXC(    LDREST  t0, REST(0)(src),       l_exc_copy)
 348EXC(    LDREST  t1, REST(1)(src),       l_exc_copy)
 349EXC(    LDFIRST t2, FIRST(2)(src),      l_exc_copy)
 350EXC(    LDFIRST t3, FIRST(3)(src),      l_exc_copy)
 351EXC(    LDREST  t2, REST(2)(src),       l_exc_copy)
 352EXC(    LDREST  t3, REST(3)(src),       l_exc_copy)
 353        ADD     src, src, 4*NBYTES
 354EXC(    STORE   t0, UNIT(0)(dst),       s_exc_p4u)
 355EXC(    STORE   t1, UNIT(1)(dst),       s_exc_p3u)
 356EXC(    STORE   t2, UNIT(2)(dst),       s_exc_p2u)
 357EXC(    STORE   t3, UNIT(3)(dst),       s_exc_p1u)
 358        bne     len, rem, 1b
 359         ADD    dst, dst, 4*NBYTES
 360
 361cleanup_src_unaligned:
 362        beqz    len, done
 363         and    rem, len, NBYTES-1  # rem = len % NBYTES
 364        beq     rem, len, copy_bytes
 365         nop
 3661:
 367EXC(    LDFIRST t0, FIRST(0)(src),      l_exc)
 368EXC(    LDREST  t0, REST(0)(src),       l_exc_copy)
 369        SUB     len, len, NBYTES
 370EXC(    STORE   t0, 0(dst),             s_exc_p1u)
 371        ADD     src, src, NBYTES
 372        bne     len, rem, 1b
 373         ADD    dst, dst, NBYTES
 374
 375copy_bytes_checklen:
 376        beqz    len, done
 377         nop
 378copy_bytes:
 379        /* 0 < len < NBYTES  */
 380#define COPY_BYTE(N)                    \
 381EXC(    lb      t0, N(src), l_exc);     \
 382        SUB     len, len, 1;            \
 383        beqz    len, done;              \
 384EXC(     sb     t0, N(dst), s_exc_p1)
 385
 386        COPY_BYTE(0)
 387        COPY_BYTE(1)
 388#ifdef USE_DOUBLE
 389        COPY_BYTE(2)
 390        COPY_BYTE(3)
 391        COPY_BYTE(4)
 392        COPY_BYTE(5)
 393#endif
 394EXC(    lb      t0, NBYTES-2(src), l_exc)
 395        SUB     len, len, 1
 396        jr      ra
 397EXC(     sb     t0, NBYTES-2(dst), s_exc_p1)
 398done:
 399        jr      ra
 400         nop
 401        END(memcpy)
 402
 403l_exc_copy:
 404        /*
 405         * Copy bytes from src until faulting load address (or until a
 406         * lb faults)
 407         *
 408         * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
 409         * may be more than a byte beyond the last address.
 410         * Hence, the lb below may get an exception.
 411         *
 412         * Assumes src < THREAD_BUADDR($28)
 413         */
 414        LOAD    t0, TI_TASK($28)
 415         nop
 416        LOAD    t0, THREAD_BUADDR(t0)
 4171:
 418EXC(    lb      t1, 0(src),     l_exc)
 419        ADD     src, src, 1
 420        sb      t1, 0(dst)      # can't fault -- we're copy_from_user
 421        bne     src, t0, 1b
 422         ADD    dst, dst, 1
 423l_exc:
 424        LOAD    t0, TI_TASK($28)
 425         nop
 426        LOAD    t0, THREAD_BUADDR(t0)   # t0 is just past last good address
 427         nop
 428        SUB     len, AT, t0             # len number of uncopied bytes
 429        /*
 430         * Here's where we rely on src and dst being incremented in tandem,
 431         *   See (3) above.
 432         * dst += (fault addr - src) to put dst at first byte to clear
 433         */
 434        ADD     dst, t0                 # compute start address in a1
 435        SUB     dst, src
 436        /*
 437         * Clear len bytes starting at dst.  Can't call __bzero because it
 438         * might modify len.  An inefficient loop for these rare times...
 439         */
 440        beqz    len, done
 441         SUB    src, len, 1
 4421:      sb      zero, 0(dst)
 443        ADD     dst, dst, 1
 444        bnez    src, 1b
 445         SUB    src, src, 1
 446        jr      ra
 447         nop
 448
 449
 450#define SEXC(n)                         \
 451s_exc_p ## n ## u:                      \
 452        jr      ra;                     \
 453         ADD    len, len, n*NBYTES
 454
 455SEXC(16)
 456SEXC(15)
 457SEXC(14)
 458SEXC(13)
 459SEXC(12)
 460SEXC(11)
 461SEXC(10)
 462SEXC(9)
 463SEXC(8)
 464SEXC(7)
 465SEXC(6)
 466SEXC(5)
 467SEXC(4)
 468SEXC(3)
 469SEXC(2)
 470SEXC(1)
 471
 472s_exc_p1:
 473        jr      ra
 474         ADD    len, len, 1
 475s_exc:
 476        jr      ra
 477         nop
 478
 479        .align  5
 480LEAF(memmove)
 481        ADD     t0, a0, a2
 482        ADD     t1, a1, a2
 483        sltu    t0, a1, t0                      # dst + len <= src -> memcpy
 484        sltu    t1, a0, t1                      # dst >= src + len -> memcpy
 485        and     t0, t1
 486        beqz    t0, __memcpy
 487         move   v0, a0                          /* return value */
 488        beqz    a2, r_out
 489        END(memmove)
 490
 491        /* fall through to __rmemcpy */
 492LEAF(__rmemcpy)                                 /* a0=dst a1=src a2=len */
 493         sltu   t0, a1, a0
 494        beqz    t0, r_end_bytes_up              # src >= dst
 495         nop
 496        ADD     a0, a2                          # dst = dst + len
 497        ADD     a1, a2                          # src = src + len
 498
 499r_end_bytes:
 500        lb      t0, -1(a1)
 501        SUB     a2, a2, 0x1
 502        sb      t0, -1(a0)
 503        SUB     a1, a1, 0x1
 504        bnez    a2, r_end_bytes
 505         SUB    a0, a0, 0x1
 506
 507r_out:
 508        jr      ra
 509         move   a2, zero
 510
 511r_end_bytes_up:
 512        lb      t0, (a1)
 513        SUB     a2, a2, 0x1
 514        sb      t0, (a0)
 515        ADD     a1, a1, 0x1
 516        bnez    a2, r_end_bytes_up
 517         ADD    a0, a0, 0x1
 518
 519        jr      ra
 520         move   a2, zero
 521        END(__rmemcpy)
 522