LXR linux/arch/arm64/lib/crc32.S

   1/* SPDX-License-Identifier: GPL-2.0-only */
   2/*
   3 * Accelerated CRC32(C) using AArch64 CRC instructions
   4 *
   5 * Copyright (C) 2016 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
   6 */
   7
   8#include <linux/linkage.h>
   9#include <asm/alternative.h>
  10#include <asm/assembler.h>
  11
  12        .cpu            generic+crc
  13
  14        .macro          __crc32, c
  15        cmp             x2, #16
  16        b.lt            8f                      // less than 16 bytes
  17
  18        and             x7, x2, #0x1f
  19        and             x2, x2, #~0x1f
  20        cbz             x7, 32f                 // multiple of 32 bytes
  21
  22        and             x8, x7, #0xf
  23        ldp             x3, x4, [x1]
  24        add             x8, x8, x1
  25        add             x1, x1, x7
  26        ldp             x5, x6, [x8]
  27CPU_BE( rev             x3, x3          )
  28CPU_BE( rev             x4, x4          )
  29CPU_BE( rev             x5, x5          )
  30CPU_BE( rev             x6, x6          )
  31
  32        tst             x7, #8
  33        crc32\c\()x     w8, w0, x3
  34        csel            x3, x3, x4, eq
  35        csel            w0, w0, w8, eq
  36        tst             x7, #4
  37        lsr             x4, x3, #32
  38        crc32\c\()w     w8, w0, w3
  39        csel            x3, x3, x4, eq
  40        csel            w0, w0, w8, eq
  41        tst             x7, #2
  42        lsr             w4, w3, #16
  43        crc32\c\()h     w8, w0, w3
  44        csel            w3, w3, w4, eq
  45        csel            w0, w0, w8, eq
  46        tst             x7, #1
  47        crc32\c\()b     w8, w0, w3
  48        csel            w0, w0, w8, eq
  49        tst             x7, #16
  50        crc32\c\()x     w8, w0, x5
  51        crc32\c\()x     w8, w8, x6
  52        csel            w0, w0, w8, eq
  53        cbz             x2, 0f
  54
  5532:     ldp             x3, x4, [x1], #32
  56        sub             x2, x2, #32
  57        ldp             x5, x6, [x1, #-16]
  58CPU_BE( rev             x3, x3          )
  59CPU_BE( rev             x4, x4          )
  60CPU_BE( rev             x5, x5          )
  61CPU_BE( rev             x6, x6          )
  62        crc32\c\()x     w0, w0, x3
  63        crc32\c\()x     w0, w0, x4
  64        crc32\c\()x     w0, w0, x5
  65        crc32\c\()x     w0, w0, x6
  66        cbnz            x2, 32b
  670:      ret
  68
  698:      tbz             x2, #3, 4f
  70        ldr             x3, [x1], #8
  71CPU_BE( rev             x3, x3          )
  72        crc32\c\()x     w0, w0, x3
  734:      tbz             x2, #2, 2f
  74        ldr             w3, [x1], #4
  75CPU_BE( rev             w3, w3          )
  76        crc32\c\()w     w0, w0, w3
  772:      tbz             x2, #1, 1f
  78        ldrh            w3, [x1], #2
  79CPU_BE( rev16           w3, w3          )
  80        crc32\c\()h     w0, w0, w3
  811:      tbz             x2, #0, 0f
  82        ldrb            w3, [x1]
  83        crc32\c\()b     w0, w0, w3
  840:      ret
  85        .endm
  86
  87        .align          5
  88ENTRY(crc32_le)
  89alternative_if_not ARM64_HAS_CRC32
  90        b               crc32_le_base
  91alternative_else_nop_endif
  92        __crc32
  93ENDPROC(crc32_le)
  94
  95        .align          5
  96ENTRY(__crc32c_le)
  97alternative_if_not ARM64_HAS_CRC32
  98        b               __crc32c_le_base
  99alternative_else_nop_endif
 100        __crc32         c
 101ENDPROC(__crc32c_le)
 102