LXR linux/arch/arm64/lib/memcpy.S

   1/* SPDX-License-Identifier: GPL-2.0-only */
   2/*
   3 * Copyright (c) 2012-2021, Arm Limited.
   4 *
   5 * Adapted from the original at:
   6 * https://github.com/ARM-software/optimized-routines/blob/afd6244a1f8d9229/string/aarch64/memcpy.S
   7 */
   8
   9#include <linux/linkage.h>
  10#include <asm/assembler.h>
  11
  12/* Assumptions:
  13 *
  14 * ARMv8-a, AArch64, unaligned accesses.
  15 *
  16 */
  17
  18#define L(label) .L ## label
  19
  20#define dstin   x0
  21#define src     x1
  22#define count   x2
  23#define dst     x3
  24#define srcend  x4
  25#define dstend  x5
  26#define A_l     x6
  27#define A_lw    w6
  28#define A_h     x7
  29#define B_l     x8
  30#define B_lw    w8
  31#define B_h     x9
  32#define C_l     x10
  33#define C_lw    w10
  34#define C_h     x11
  35#define D_l     x12
  36#define D_h     x13
  37#define E_l     x14
  38#define E_h     x15
  39#define F_l     x16
  40#define F_h     x17
  41#define G_l     count
  42#define G_h     dst
  43#define H_l     src
  44#define H_h     srcend
  45#define tmp1    x14
  46
  47/* This implementation handles overlaps and supports both memcpy and memmove
  48   from a single entry point.  It uses unaligned accesses and branchless
  49   sequences to keep the code small, simple and improve performance.
  50
  51   Copies are split into 3 main cases: small copies of up to 32 bytes, medium
  52   copies of up to 128 bytes, and large copies.  The overhead of the overlap
  53   check is negligible since it is only required for large copies.
  54
  55   Large copies use a software pipelined loop processing 64 bytes per iteration.
  56   The destination pointer is 16-byte aligned to minimize unaligned accesses.
  57   The loop tail is handled by always copying 64 bytes from the end.
  58*/
  59
  60SYM_FUNC_START_ALIAS(__memmove)
  61SYM_FUNC_START_WEAK_ALIAS_PI(memmove)
  62SYM_FUNC_START_ALIAS(__memcpy)
  63SYM_FUNC_START_WEAK_PI(memcpy)
  64        add     srcend, src, count
  65        add     dstend, dstin, count
  66        cmp     count, 128
  67        b.hi    L(copy_long)
  68        cmp     count, 32
  69        b.hi    L(copy32_128)
  70
  71        /* Small copies: 0..32 bytes.  */
  72        cmp     count, 16
  73        b.lo    L(copy16)
  74        ldp     A_l, A_h, [src]
  75        ldp     D_l, D_h, [srcend, -16]
  76        stp     A_l, A_h, [dstin]
  77        stp     D_l, D_h, [dstend, -16]
  78        ret
  79
  80        /* Copy 8-15 bytes.  */
  81L(copy16):
  82        tbz     count, 3, L(copy8)
  83        ldr     A_l, [src]
  84        ldr     A_h, [srcend, -8]
  85        str     A_l, [dstin]
  86        str     A_h, [dstend, -8]
  87        ret
  88
  89        .p2align 3
  90        /* Copy 4-7 bytes.  */
  91L(copy8):
  92        tbz     count, 2, L(copy4)
  93        ldr     A_lw, [src]
  94        ldr     B_lw, [srcend, -4]
  95        str     A_lw, [dstin]
  96        str     B_lw, [dstend, -4]
  97        ret
  98
  99        /* Copy 0..3 bytes using a branchless sequence.  */
 100L(copy4):
 101        cbz     count, L(copy0)
 102        lsr     tmp1, count, 1
 103        ldrb    A_lw, [src]
 104        ldrb    C_lw, [srcend, -1]
 105        ldrb    B_lw, [src, tmp1]
 106        strb    A_lw, [dstin]
 107        strb    B_lw, [dstin, tmp1]
 108        strb    C_lw, [dstend, -1]
 109L(copy0):
 110        ret
 111
 112        .p2align 4
 113        /* Medium copies: 33..128 bytes.  */
 114L(copy32_128):
 115        ldp     A_l, A_h, [src]
 116        ldp     B_l, B_h, [src, 16]
 117        ldp     C_l, C_h, [srcend, -32]
 118        ldp     D_l, D_h, [srcend, -16]
 119        cmp     count, 64
 120        b.hi    L(copy128)
 121        stp     A_l, A_h, [dstin]
 122        stp     B_l, B_h, [dstin, 16]
 123        stp     C_l, C_h, [dstend, -32]
 124        stp     D_l, D_h, [dstend, -16]
 125        ret
 126
 127        .p2align 4
 128        /* Copy 65..128 bytes.  */
 129L(copy128):
 130        ldp     E_l, E_h, [src, 32]
 131        ldp     F_l, F_h, [src, 48]
 132        cmp     count, 96
 133        b.ls    L(copy96)
 134        ldp     G_l, G_h, [srcend, -64]
 135        ldp     H_l, H_h, [srcend, -48]
 136        stp     G_l, G_h, [dstend, -64]
 137        stp     H_l, H_h, [dstend, -48]
 138L(copy96):
 139        stp     A_l, A_h, [dstin]
 140        stp     B_l, B_h, [dstin, 16]
 141        stp     E_l, E_h, [dstin, 32]
 142        stp     F_l, F_h, [dstin, 48]
 143        stp     C_l, C_h, [dstend, -32]
 144        stp     D_l, D_h, [dstend, -16]
 145        ret
 146
 147        .p2align 4
 148        /* Copy more than 128 bytes.  */
 149L(copy_long):
 150        /* Use backwards copy if there is an overlap.  */
 151        sub     tmp1, dstin, src
 152        cbz     tmp1, L(copy0)
 153        cmp     tmp1, count
 154        b.lo    L(copy_long_backwards)
 155
 156        /* Copy 16 bytes and then align dst to 16-byte alignment.  */
 157
 158        ldp     D_l, D_h, [src]
 159        and     tmp1, dstin, 15
 160        bic     dst, dstin, 15
 161        sub     src, src, tmp1
 162        add     count, count, tmp1      /* Count is now 16 too large.  */
 163        ldp     A_l, A_h, [src, 16]
 164        stp     D_l, D_h, [dstin]
 165        ldp     B_l, B_h, [src, 32]
 166        ldp     C_l, C_h, [src, 48]
 167        ldp     D_l, D_h, [src, 64]!
 168        subs    count, count, 128 + 16  /* Test and readjust count.  */
 169        b.ls    L(copy64_from_end)
 170
 171L(loop64):
 172        stp     A_l, A_h, [dst, 16]
 173        ldp     A_l, A_h, [src, 16]
 174        stp     B_l, B_h, [dst, 32]
 175        ldp     B_l, B_h, [src, 32]
 176        stp     C_l, C_h, [dst, 48]
 177        ldp     C_l, C_h, [src, 48]
 178        stp     D_l, D_h, [dst, 64]!
 179        ldp     D_l, D_h, [src, 64]!
 180        subs    count, count, 64
 181        b.hi    L(loop64)
 182
 183        /* Write the last iteration and copy 64 bytes from the end.  */
 184L(copy64_from_end):
 185        ldp     E_l, E_h, [srcend, -64]
 186        stp     A_l, A_h, [dst, 16]
 187        ldp     A_l, A_h, [srcend, -48]
 188        stp     B_l, B_h, [dst, 32]
 189        ldp     B_l, B_h, [srcend, -32]
 190        stp     C_l, C_h, [dst, 48]
 191        ldp     C_l, C_h, [srcend, -16]
 192        stp     D_l, D_h, [dst, 64]
 193        stp     E_l, E_h, [dstend, -64]
 194        stp     A_l, A_h, [dstend, -48]
 195        stp     B_l, B_h, [dstend, -32]
 196        stp     C_l, C_h, [dstend, -16]
 197        ret
 198
 199        .p2align 4
 200
 201        /* Large backwards copy for overlapping copies.
 202           Copy 16 bytes and then align dst to 16-byte alignment.  */
 203L(copy_long_backwards):
 204        ldp     D_l, D_h, [srcend, -16]
 205        and     tmp1, dstend, 15
 206        sub     srcend, srcend, tmp1
 207        sub     count, count, tmp1
 208        ldp     A_l, A_h, [srcend, -16]
 209        stp     D_l, D_h, [dstend, -16]
 210        ldp     B_l, B_h, [srcend, -32]
 211        ldp     C_l, C_h, [srcend, -48]
 212        ldp     D_l, D_h, [srcend, -64]!
 213        sub     dstend, dstend, tmp1
 214        subs    count, count, 128
 215        b.ls    L(copy64_from_start)
 216
 217L(loop64_backwards):
 218        stp     A_l, A_h, [dstend, -16]
 219        ldp     A_l, A_h, [srcend, -16]
 220        stp     B_l, B_h, [dstend, -32]
 221        ldp     B_l, B_h, [srcend, -32]
 222        stp     C_l, C_h, [dstend, -48]
 223        ldp     C_l, C_h, [srcend, -48]
 224        stp     D_l, D_h, [dstend, -64]!
 225        ldp     D_l, D_h, [srcend, -64]!
 226        subs    count, count, 64
 227        b.hi    L(loop64_backwards)
 228
 229        /* Write the last iteration and copy 64 bytes from the start.  */
 230L(copy64_from_start):
 231        ldp     G_l, G_h, [src, 48]
 232        stp     A_l, A_h, [dstend, -16]
 233        ldp     A_l, A_h, [src, 32]
 234        stp     B_l, B_h, [dstend, -32]
 235        ldp     B_l, B_h, [src, 16]
 236        stp     C_l, C_h, [dstend, -48]
 237        ldp     C_l, C_h, [src]
 238        stp     D_l, D_h, [dstend, -64]
 239        stp     G_l, G_h, [dstin, 48]
 240        stp     A_l, A_h, [dstin, 32]
 241        stp     B_l, B_h, [dstin, 16]
 242        stp     C_l, C_h, [dstin]
 243        ret
 244
 245SYM_FUNC_END_PI(memcpy)
 246EXPORT_SYMBOL(memcpy)
 247SYM_FUNC_END_ALIAS(__memcpy)
 248EXPORT_SYMBOL(__memcpy)
 249SYM_FUNC_END_ALIAS_PI(memmove)
 250EXPORT_SYMBOL(memmove)
 251SYM_FUNC_END_ALIAS(__memmove)
 252EXPORT_SYMBOL(__memmove)
 253