LXR linux/arch/arm64/lib/crc32.S

   1/*
   2 * Accelerated CRC32(C) using AArch64 CRC instructions
   3 *
   4 * Copyright (C) 2016 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License version 2 as
   8 * published by the Free Software Foundation.
   9 */
  10
  11#include <linux/linkage.h>
  12#include <asm/alternative.h>
  13#include <asm/assembler.h>
  14
  15        .cpu            generic+crc
  16
  17        .macro          __crc32, c
  18        cmp             x2, #16
  19        b.lt            8f                      // less than 16 bytes
  20
  21        and             x7, x2, #0x1f
  22        and             x2, x2, #~0x1f
  23        cbz             x7, 32f                 // multiple of 32 bytes
  24
  25        and             x8, x7, #0xf
  26        ldp             x3, x4, [x1]
  27        add             x8, x8, x1
  28        add             x1, x1, x7
  29        ldp             x5, x6, [x8]
  30CPU_BE( rev             x3, x3          )
  31CPU_BE( rev             x4, x4          )
  32CPU_BE( rev             x5, x5          )
  33CPU_BE( rev             x6, x6          )
  34
  35        tst             x7, #8
  36        crc32\c\()x     w8, w0, x3
  37        csel            x3, x3, x4, eq
  38        csel            w0, w0, w8, eq
  39        tst             x7, #4
  40        lsr             x4, x3, #32
  41        crc32\c\()w     w8, w0, w3
  42        csel            x3, x3, x4, eq
  43        csel            w0, w0, w8, eq
  44        tst             x7, #2
  45        lsr             w4, w3, #16
  46        crc32\c\()h     w8, w0, w3
  47        csel            w3, w3, w4, eq
  48        csel            w0, w0, w8, eq
  49        tst             x7, #1
  50        crc32\c\()b     w8, w0, w3
  51        csel            w0, w0, w8, eq
  52        tst             x7, #16
  53        crc32\c\()x     w8, w0, x5
  54        crc32\c\()x     w8, w8, x6
  55        csel            w0, w0, w8, eq
  56        cbz             x2, 0f
  57
  5832:     ldp             x3, x4, [x1], #32
  59        sub             x2, x2, #32
  60        ldp             x5, x6, [x1, #-16]
  61CPU_BE( rev             x3, x3          )
  62CPU_BE( rev             x4, x4          )
  63CPU_BE( rev             x5, x5          )
  64CPU_BE( rev             x6, x6          )
  65        crc32\c\()x     w0, w0, x3
  66        crc32\c\()x     w0, w0, x4
  67        crc32\c\()x     w0, w0, x5
  68        crc32\c\()x     w0, w0, x6
  69        cbnz            x2, 32b
  700:      ret
  71
  728:      tbz             x2, #3, 4f
  73        ldr             x3, [x1], #8
  74CPU_BE( rev             x3, x3          )
  75        crc32\c\()x     w0, w0, x3
  764:      tbz             x2, #2, 2f
  77        ldr             w3, [x1], #4
  78CPU_BE( rev             w3, w3          )
  79        crc32\c\()w     w0, w0, w3
  802:      tbz             x2, #1, 1f
  81        ldrh            w3, [x1], #2
  82CPU_BE( rev16           w3, w3          )
  83        crc32\c\()h     w0, w0, w3
  841:      tbz             x2, #0, 0f
  85        ldrb            w3, [x1]
  86        crc32\c\()b     w0, w0, w3
  870:      ret
  88        .endm
  89
  90        .align          5
  91ENTRY(crc32_le)
  92alternative_if_not ARM64_HAS_CRC32
  93        b               crc32_le_base
  94alternative_else_nop_endif
  95        __crc32
  96ENDPROC(crc32_le)
  97
  98        .align          5
  99ENTRY(__crc32c_le)
 100alternative_if_not ARM64_HAS_CRC32
 101        b               __crc32c_le_base
 102alternative_else_nop_endif
 103        __crc32         c
 104ENDPROC(__crc32c_le)
 105