LXR linux/arch/xtensa/lib/checksum.S

   1/* SPDX-License-Identifier: GPL-2.0-or-later */
   2/*
   3 * INET         An implementation of the TCP/IP protocol suite for the LINUX
   4 *              operating system.  INET is implemented using the  BSD Socket
   5 *              interface as the means of communication with the user level.
   6 *
   7 *              IP/TCP/UDP checksumming routines
   8 *
   9 * Xtensa version:  Copyright (C) 2001 Tensilica, Inc. by Kevin Chea
  10 *                  Optimized by Joe Taylor
  11 */
  12
  13#include <linux/errno.h>
  14#include <linux/linkage.h>
  15#include <asm/asmmacro.h>
  16#include <asm/core.h>
  17
  18/*
  19 * computes a partial checksum, e.g. for TCP/UDP fragments
  20 */
  21
  22/*
  23 * unsigned int csum_partial(const unsigned char *buf, int len,
  24 *                           unsigned int sum);
  25 *    a2 = buf
  26 *    a3 = len
  27 *    a4 = sum
  28 *
  29 * This function assumes 2- or 4-byte alignment.  Other alignments will fail!
  30 */
  31
  32/* ONES_ADD converts twos-complement math to ones-complement. */
  33#define ONES_ADD(sum, val)        \
  34        add     sum, sum, val   ; \
  35        bgeu    sum, val, 99f   ; \
  36        addi    sum, sum, 1     ; \
  3799:                             ;
  38
  39.text
  40ENTRY(csum_partial)
  41
  42        /*
  43         * Experiments with Ethernet and SLIP connections show that buf
  44         * is aligned on either a 2-byte or 4-byte boundary.
  45         */
  46        abi_entry_default
  47        extui   a5, a2, 0, 2
  48        bnez    a5, 8f          /* branch if 2-byte aligned */
  49        /* Fall-through on common case, 4-byte alignment */
  501:
  51        srli    a5, a3, 5       /* 32-byte chunks */
  52#if XCHAL_HAVE_LOOPS
  53        loopgtz a5, 2f
  54#else
  55        beqz    a5, 2f
  56        slli    a5, a5, 5
  57        add     a5, a5, a2      /* a5 = end of last 32-byte chunk */
  58.Loop1:
  59#endif
  60        l32i    a6, a2, 0
  61        l32i    a7, a2, 4
  62        ONES_ADD(a4, a6)
  63        ONES_ADD(a4, a7)
  64        l32i    a6, a2, 8
  65        l32i    a7, a2, 12
  66        ONES_ADD(a4, a6)
  67        ONES_ADD(a4, a7)
  68        l32i    a6, a2, 16
  69        l32i    a7, a2, 20
  70        ONES_ADD(a4, a6)
  71        ONES_ADD(a4, a7)
  72        l32i    a6, a2, 24
  73        l32i    a7, a2, 28
  74        ONES_ADD(a4, a6)
  75        ONES_ADD(a4, a7)
  76        addi    a2, a2, 4*8
  77#if !XCHAL_HAVE_LOOPS
  78        blt     a2, a5, .Loop1
  79#endif
  802:
  81        extui   a5, a3, 2, 3    /* remaining 4-byte chunks */
  82#if XCHAL_HAVE_LOOPS
  83        loopgtz a5, 3f
  84#else
  85        beqz    a5, 3f
  86        slli    a5, a5, 2
  87        add     a5, a5, a2      /* a5 = end of last 4-byte chunk */
  88.Loop2:
  89#endif
  90        l32i    a6, a2, 0
  91        ONES_ADD(a4, a6)
  92        addi    a2, a2, 4
  93#if !XCHAL_HAVE_LOOPS
  94        blt     a2, a5, .Loop2
  95#endif
  963:
  97        _bbci.l a3, 1, 5f       /* remaining 2-byte chunk */
  98        l16ui   a6, a2, 0
  99        ONES_ADD(a4, a6)
 100        addi    a2, a2, 2
 1015:
 102        _bbci.l a3, 0, 7f       /* remaining 1-byte chunk */
 1036:      l8ui    a6, a2, 0
 104#ifdef __XTENSA_EB__
 105        slli    a6, a6, 8       /* load byte into bits 8..15 */
 106#endif
 107        ONES_ADD(a4, a6)
 1087:
 109        mov     a2, a4
 110        abi_ret_default
 111
 112        /* uncommon case, buf is 2-byte aligned */
 1138:
 114        beqz    a3, 7b          /* branch if len == 0 */
 115        beqi    a3, 1, 6b       /* branch if len == 1 */
 116
 117        extui   a5, a2, 0, 1
 118        bnez    a5, 8f          /* branch if 1-byte aligned */
 119
 120        l16ui   a6, a2, 0       /* common case, len >= 2 */
 121        ONES_ADD(a4, a6)
 122        addi    a2, a2, 2       /* adjust buf */
 123        addi    a3, a3, -2      /* adjust len */
 124        j       1b              /* now buf is 4-byte aligned */
 125
 126        /* case: odd-byte aligned, len > 1
 127         * This case is dog slow, so don't give us an odd address.
 128         * (I don't think this ever happens, but just in case.)
 129         */
 1308:
 131        srli    a5, a3, 2       /* 4-byte chunks */
 132#if XCHAL_HAVE_LOOPS
 133        loopgtz a5, 2f
 134#else
 135        beqz    a5, 2f
 136        slli    a5, a5, 2
 137        add     a5, a5, a2      /* a5 = end of last 4-byte chunk */
 138.Loop3:
 139#endif
 140        l8ui    a6, a2, 0       /* bits 24..31 */
 141        l16ui   a7, a2, 1       /* bits  8..23 */
 142        l8ui    a8, a2, 3       /* bits  0.. 8 */
 143#ifdef  __XTENSA_EB__
 144        slli    a6, a6, 24
 145#else
 146        slli    a8, a8, 24
 147#endif
 148        slli    a7, a7, 8
 149        or      a7, a7, a6
 150        or      a7, a7, a8
 151        ONES_ADD(a4, a7)
 152        addi    a2, a2, 4
 153#if !XCHAL_HAVE_LOOPS
 154        blt     a2, a5, .Loop3
 155#endif
 1562:
 157        _bbci.l a3, 1, 3f       /* remaining 2-byte chunk, still odd addr */
 158        l8ui    a6, a2, 0
 159        l8ui    a7, a2, 1
 160#ifdef  __XTENSA_EB__
 161        slli    a6, a6, 8
 162#else
 163        slli    a7, a7, 8
 164#endif
 165        or      a7, a7, a6
 166        ONES_ADD(a4, a7)
 167        addi    a2, a2, 2
 1683:
 169        j       5b              /* branch to handle the remaining byte */
 170
 171ENDPROC(csum_partial)
 172
 173/*
 174 * Copy from ds while checksumming, otherwise like csum_partial
 175 */
 176
 177/*
 178unsigned int csum_partial_copy_generic (const char *src, char *dst, int len)
 179        a2  = src
 180        a3  = dst
 181        a4  = len
 182        a5  = sum
 183        a8  = temp
 184        a9  = temp
 185        a10 = temp
 186
 187    This function is optimized for 4-byte aligned addresses.  Other
 188    alignments work, but not nearly as efficiently.
 189 */
 190
 191ENTRY(csum_partial_copy_generic)
 192
 193        abi_entry_default
 194        movi    a5, -1
 195        or      a10, a2, a3
 196
 197        /* We optimize the following alignment tests for the 4-byte
 198        aligned case.  Two bbsi.l instructions might seem more optimal
 199        (commented out below).  However, both labels 5: and 3: are out
 200        of the imm8 range, so the assembler relaxes them into
 201        equivalent bbci.l, j combinations, which is actually
 202        slower. */
 203
 204        extui   a9, a10, 0, 2
 205        beqz    a9, 1f          /* branch if both are 4-byte aligned */
 206        bbsi.l  a10, 0, 5f      /* branch if one address is odd */
 207        j       3f              /* one address is 2-byte aligned */
 208
 209/*      _bbsi.l a10, 0, 5f */   /* branch if odd address */
 210/*      _bbsi.l a10, 1, 3f */   /* branch if 2-byte-aligned address */
 211
 2121:
 213        /* src and dst are both 4-byte aligned */
 214        srli    a10, a4, 5      /* 32-byte chunks */
 215#if XCHAL_HAVE_LOOPS
 216        loopgtz a10, 2f
 217#else
 218        beqz    a10, 2f
 219        slli    a10, a10, 5
 220        add     a10, a10, a2    /* a10 = end of last 32-byte src chunk */
 221.Loop5:
 222#endif
 223EX(10f) l32i    a9, a2, 0
 224EX(10f) l32i    a8, a2, 4
 225EX(10f) s32i    a9, a3, 0
 226EX(10f) s32i    a8, a3, 4
 227        ONES_ADD(a5, a9)
 228        ONES_ADD(a5, a8)
 229EX(10f) l32i    a9, a2, 8
 230EX(10f) l32i    a8, a2, 12
 231EX(10f) s32i    a9, a3, 8
 232EX(10f) s32i    a8, a3, 12
 233        ONES_ADD(a5, a9)
 234        ONES_ADD(a5, a8)
 235EX(10f) l32i    a9, a2, 16
 236EX(10f) l32i    a8, a2, 20
 237EX(10f) s32i    a9, a3, 16
 238EX(10f) s32i    a8, a3, 20
 239        ONES_ADD(a5, a9)
 240        ONES_ADD(a5, a8)
 241EX(10f) l32i    a9, a2, 24
 242EX(10f) l32i    a8, a2, 28
 243EX(10f) s32i    a9, a3, 24
 244EX(10f) s32i    a8, a3, 28
 245        ONES_ADD(a5, a9)
 246        ONES_ADD(a5, a8)
 247        addi    a2, a2, 32
 248        addi    a3, a3, 32
 249#if !XCHAL_HAVE_LOOPS
 250        blt     a2, a10, .Loop5
 251#endif
 2522:
 253        extui   a10, a4, 2, 3   /* remaining 4-byte chunks */
 254        extui   a4, a4, 0, 2    /* reset len for general-case, 2-byte chunks */
 255#if XCHAL_HAVE_LOOPS
 256        loopgtz a10, 3f
 257#else
 258        beqz    a10, 3f
 259        slli    a10, a10, 2
 260        add     a10, a10, a2    /* a10 = end of last 4-byte src chunk */
 261.Loop6:
 262#endif
 263EX(10f) l32i    a9, a2, 0
 264EX(10f) s32i    a9, a3, 0
 265        ONES_ADD(a5, a9)
 266        addi    a2, a2, 4
 267        addi    a3, a3, 4
 268#if !XCHAL_HAVE_LOOPS
 269        blt     a2, a10, .Loop6
 270#endif
 2713:
 272        /*
 273        Control comes to here in two cases: (1) It may fall through
 274        to here from the 4-byte alignment case to process, at most,
 275        one 2-byte chunk.  (2) It branches to here from above if
 276        either src or dst is 2-byte aligned, and we process all bytes
 277        here, except for perhaps a trailing odd byte.  It's
 278        inefficient, so align your addresses to 4-byte boundaries.
 279
 280        a2 = src
 281        a3 = dst
 282        a4 = len
 283        a5 = sum
 284        */
 285        srli    a10, a4, 1      /* 2-byte chunks */
 286#if XCHAL_HAVE_LOOPS
 287        loopgtz a10, 4f
 288#else
 289        beqz    a10, 4f
 290        slli    a10, a10, 1
 291        add     a10, a10, a2    /* a10 = end of last 2-byte src chunk */
 292.Loop7:
 293#endif
 294EX(10f) l16ui   a9, a2, 0
 295EX(10f) s16i    a9, a3, 0
 296        ONES_ADD(a5, a9)
 297        addi    a2, a2, 2
 298        addi    a3, a3, 2
 299#if !XCHAL_HAVE_LOOPS
 300        blt     a2, a10, .Loop7
 301#endif
 3024:
 303        /* This section processes a possible trailing odd byte. */
 304        _bbci.l a4, 0, 8f       /* 1-byte chunk */
 305EX(10f) l8ui    a9, a2, 0
 306EX(10f) s8i     a9, a3, 0
 307#ifdef __XTENSA_EB__
 308        slli    a9, a9, 8       /* shift byte to bits 8..15 */
 309#endif
 310        ONES_ADD(a5, a9)
 3118:
 312        mov     a2, a5
 313        abi_ret_default
 314
 3155:
 316        /* Control branch to here when either src or dst is odd.  We
 317        process all bytes using 8-bit accesses.  Grossly inefficient,
 318        so don't feed us an odd address. */
 319
 320        srli    a10, a4, 1      /* handle in pairs for 16-bit csum */
 321#if XCHAL_HAVE_LOOPS
 322        loopgtz a10, 6f
 323#else
 324        beqz    a10, 6f
 325        slli    a10, a10, 1
 326        add     a10, a10, a2    /* a10 = end of last odd-aligned, 2-byte src chunk */
 327.Loop8:
 328#endif
 329EX(10f) l8ui    a9, a2, 0
 330EX(10f) l8ui    a8, a2, 1
 331EX(10f) s8i     a9, a3, 0
 332EX(10f) s8i     a8, a3, 1
 333#ifdef __XTENSA_EB__
 334        slli    a9, a9, 8       /* combine into a single 16-bit value */
 335#else                           /* for checksum computation */
 336        slli    a8, a8, 8
 337#endif
 338        or      a9, a9, a8
 339        ONES_ADD(a5, a9)
 340        addi    a2, a2, 2
 341        addi    a3, a3, 2
 342#if !XCHAL_HAVE_LOOPS
 343        blt     a2, a10, .Loop8
 344#endif
 3456:
 346        j       4b              /* process the possible trailing odd byte */
 347
 348ENDPROC(csum_partial_copy_generic)
 349
 350
 351# Exception handler:
 352.section .fixup, "ax"
 35310:
 354        movi    a2, 0
 355        abi_ret_default
 356
 357.previous
 358