LXR linux/arch/arm64/lib/strcmp.S

   1/* SPDX-License-Identifier: GPL-2.0-only */
   2/*
   3 * Copyright (c) 2012-2022, Arm Limited.
   4 *
   5 * Adapted from the original at:
   6 * https://github.com/ARM-software/optimized-routines/blob/189dfefe37d54c5b/string/aarch64/strcmp.S
   7 */
   8
   9#include <linux/linkage.h>
  10#include <asm/assembler.h>
  11
  12/* Assumptions:
  13 *
  14 * ARMv8-a, AArch64.
  15 * MTE compatible.
  16 */
  17
  18#define L(label) .L ## label
  19
  20#define REP8_01 0x0101010101010101
  21#define REP8_7f 0x7f7f7f7f7f7f7f7f
  22
  23#define src1            x0
  24#define src2            x1
  25#define result          x0
  26
  27#define data1           x2
  28#define data1w          w2
  29#define data2           x3
  30#define data2w          w3
  31#define has_nul         x4
  32#define diff            x5
  33#define off1            x5
  34#define syndrome        x6
  35#define tmp             x6
  36#define data3           x7
  37#define zeroones        x8
  38#define shift           x9
  39#define off2            x10
  40
  41/* On big-endian early bytes are at MSB and on little-endian LSB.
  42   LS_FW means shifting towards early bytes.  */
  43#ifdef __AARCH64EB__
  44# define LS_FW lsl
  45#else
  46# define LS_FW lsr
  47#endif
  48
  49/* NUL detection works on the principle that (X - 1) & (~X) & 0x80
  50   (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
  51   can be done in parallel across the entire word.
  52   Since carry propagation makes 0x1 bytes before a NUL byte appear
  53   NUL too in big-endian, byte-reverse the data before the NUL check.  */
  54
  55
  56SYM_FUNC_START(__pi_strcmp)
  57        sub     off2, src2, src1
  58        mov     zeroones, REP8_01
  59        and     tmp, src1, 7
  60        tst     off2, 7
  61        b.ne    L(misaligned8)
  62        cbnz    tmp, L(mutual_align)
  63
  64        .p2align 4
  65
  66L(loop_aligned):
  67        ldr     data2, [src1, off2]
  68        ldr     data1, [src1], 8
  69L(start_realigned):
  70#ifdef __AARCH64EB__
  71        rev     tmp, data1
  72        sub     has_nul, tmp, zeroones
  73        orr     tmp, tmp, REP8_7f
  74#else
  75        sub     has_nul, data1, zeroones
  76        orr     tmp, data1, REP8_7f
  77#endif
  78        bics    has_nul, has_nul, tmp   /* Non-zero if NUL terminator.  */
  79        ccmp    data1, data2, 0, eq
  80        b.eq    L(loop_aligned)
  81#ifdef __AARCH64EB__
  82        rev     has_nul, has_nul
  83#endif
  84        eor     diff, data1, data2
  85        orr     syndrome, diff, has_nul
  86L(end):
  87#ifndef __AARCH64EB__
  88        rev     syndrome, syndrome
  89        rev     data1, data1
  90        rev     data2, data2
  91#endif
  92        clz     shift, syndrome
  93        /* The most-significant-non-zero bit of the syndrome marks either the
  94           first bit that is different, or the top bit of the first zero byte.
  95           Shifting left now will bring the critical information into the
  96           top bits.  */
  97        lsl     data1, data1, shift
  98        lsl     data2, data2, shift
  99        /* But we need to zero-extend (char is unsigned) the value and then
 100           perform a signed 32-bit subtraction.  */
 101        lsr     data1, data1, 56
 102        sub     result, data1, data2, lsr 56
 103        ret
 104
 105        .p2align 4
 106
 107L(mutual_align):
 108        /* Sources are mutually aligned, but are not currently at an
 109           alignment boundary.  Round down the addresses and then mask off
 110           the bytes that precede the start point.  */
 111        bic     src1, src1, 7
 112        ldr     data2, [src1, off2]
 113        ldr     data1, [src1], 8
 114        neg     shift, src2, lsl 3      /* Bits to alignment -64.  */
 115        mov     tmp, -1
 116        LS_FW   tmp, tmp, shift
 117        orr     data1, data1, tmp
 118        orr     data2, data2, tmp
 119        b       L(start_realigned)
 120
 121L(misaligned8):
 122        /* Align SRC1 to 8 bytes and then compare 8 bytes at a time, always
 123           checking to make sure that we don't access beyond the end of SRC2.  */
 124        cbz     tmp, L(src1_aligned)
 125L(do_misaligned):
 126        ldrb    data1w, [src1], 1
 127        ldrb    data2w, [src2], 1
 128        cmp     data1w, 0
 129        ccmp    data1w, data2w, 0, ne   /* NZCV = 0b0000.  */
 130        b.ne    L(done)
 131        tst     src1, 7
 132        b.ne    L(do_misaligned)
 133
 134L(src1_aligned):
 135        neg     shift, src2, lsl 3
 136        bic     src2, src2, 7
 137        ldr     data3, [src2], 8
 138#ifdef __AARCH64EB__
 139        rev     data3, data3
 140#endif
 141        lsr     tmp, zeroones, shift
 142        orr     data3, data3, tmp
 143        sub     has_nul, data3, zeroones
 144        orr     tmp, data3, REP8_7f
 145        bics    has_nul, has_nul, tmp
 146        b.ne    L(tail)
 147
 148        sub     off1, src2, src1
 149
 150        .p2align 4
 151
 152L(loop_unaligned):
 153        ldr     data3, [src1, off1]
 154        ldr     data2, [src1, off2]
 155#ifdef __AARCH64EB__
 156        rev     data3, data3
 157#endif
 158        sub     has_nul, data3, zeroones
 159        orr     tmp, data3, REP8_7f
 160        ldr     data1, [src1], 8
 161        bics    has_nul, has_nul, tmp
 162        ccmp    data1, data2, 0, eq
 163        b.eq    L(loop_unaligned)
 164
 165        lsl     tmp, has_nul, shift
 166#ifdef __AARCH64EB__
 167        rev     tmp, tmp
 168#endif
 169        eor     diff, data1, data2
 170        orr     syndrome, diff, tmp
 171        cbnz    syndrome, L(end)
 172L(tail):
 173        ldr     data1, [src1]
 174        neg     shift, shift
 175        lsr     data2, data3, shift
 176        lsr     has_nul, has_nul, shift
 177#ifdef __AARCH64EB__
 178        rev     data2, data2
 179        rev     has_nul, has_nul
 180#endif
 181        eor     diff, data1, data2
 182        orr     syndrome, diff, has_nul
 183        b       L(end)
 184
 185L(done):
 186        sub     result, data1, data2
 187        ret
 188SYM_FUNC_END(__pi_strcmp)
 189SYM_FUNC_ALIAS_WEAK(strcmp, __pi_strcmp)
 190EXPORT_SYMBOL_NOKASAN(strcmp)
 191