linux/arch/sparc/lib/csum_copy.S
<<
>>
Prefs 
   1/* SPDX-License-Identifier: GPL-2.0 */
   2/* csum_copy.S: Checksum+copy code for sparc64
   3 *
   4 * Copyright (C) 2005 David S. Miller <davem@davemloft.net>
   5 */
   6
   7#include <asm/export.h>
   8
   9#ifdef __KERNEL__
  10#define GLOBAL_SPARE    %g7
  11#else
  12#define GLOBAL_SPARE    %g5
  13#endif
  14
  15#ifndef EX_LD
  16#define EX_LD(x)        x
  17#endif
  18
  19#ifndef EX_ST
  20#define EX_ST(x)        x
  21#endif
  22
  23#ifndef EX_RETVAL
  24#define EX_RETVAL(x)    x
  25#endif
  26
  27#ifndef LOAD
  28#define LOAD(type,addr,dest)    type [addr], dest
  29#endif
  30
  31#ifndef STORE
  32#define STORE(type,src,addr)    type src, [addr]
  33#endif
  34
  35#ifndef FUNC_NAME
  36#define FUNC_NAME       csum_partial_copy_nocheck
  37#endif
  38
  39        .register       %g2, #scratch
  40        .register       %g3, #scratch
  41
  42        .text
  43
  4490:
  45        /* We checked for zero length already, so there must be
  46         * at least one byte.
  47         */
  48        be,pt           %icc, 1f
  49         nop
  50        EX_LD(LOAD(ldub, %o0 + 0x00, %o4))
  51        add             %o0, 1, %o0
  52        sub             %o2, 1, %o2
  53        EX_ST(STORE(stb, %o4, %o1 + 0x00))
  54        add             %o1, 1, %o1
  551:      andcc           %o0, 0x2, %g0
  56        be,pn           %icc, 80f
  57         cmp            %o2, 2
  58        blu,pn          %icc, 60f
  59         nop
  60        EX_LD(LOAD(lduh, %o0 + 0x00, %o5))
  61        add             %o0, 2, %o0
  62        sub             %o2, 2, %o2
  63        EX_ST(STORE(sth, %o5, %o1 + 0x00))
  64        add             %o1, 2, %o1
  65        ba,pt           %xcc, 80f
  66         add            %o5, %o4, %o4
  67
  68        .globl          FUNC_NAME
  69        .type           FUNC_NAME,#function
  70        EXPORT_SYMBOL(FUNC_NAME)
  71FUNC_NAME:              /* %o0=src, %o1=dst, %o2=len, %o3=sum */
  72        LOAD(prefetch, %o0 + 0x000, #n_reads)
  73        xor             %o0, %o1, %g1
  74        clr             %o4
  75        andcc           %g1, 0x3, %g0
  76        bne,pn          %icc, 95f
  77         LOAD(prefetch, %o0 + 0x040, #n_reads)
  78        
  79        brz,pn          %o2, 70f
  80         andcc          %o0, 0x3, %g0
  81
  82        /* We "remember" whether the lowest bit in the address
  83         * was set in GLOBAL_SPARE.  Because if it is, we have to swap
  84         * upper and lower 8 bit fields of the sum we calculate.
  85        */
  86        bne,pn          %icc, 90b
  87         andcc          %o0, 0x1, GLOBAL_SPARE
  88
  8980:
  90        LOAD(prefetch, %o0 + 0x080, #n_reads)
  91        andncc          %o2, 0x3f, %g3
  92
  93        LOAD(prefetch, %o0 + 0x0c0, #n_reads)
  94        sub             %o2, %g3, %o2
  95        brz,pn          %g3, 2f
  96         LOAD(prefetch, %o0 + 0x100, #n_reads)
  97
  98        /* So that we don't need to use the non-pairing
  99         * add-with-carry instructions we accumulate 32-bit
 100         * values into a 64-bit register.  At the end of the
 101         * loop we fold it down to 32-bits and so on.
 102         */
 103        ba,pt           %xcc, 1f
 104        LOAD(prefetch, %o0 + 0x140, #n_reads)
 105
 106        .align          32
 1071:      EX_LD(LOAD(lduw, %o0 + 0x00, %o5))
 108        EX_LD(LOAD(lduw, %o0 + 0x04, %g1))
 109        EX_LD(LOAD(lduw, %o0 + 0x08, %g2))
 110        add             %o4, %o5, %o4
 111        EX_ST(STORE(stw, %o5, %o1 + 0x00))
 112        EX_LD(LOAD(lduw, %o0 + 0x0c, %o5))
 113        add             %o4, %g1, %o4
 114        EX_ST(STORE(stw, %g1, %o1 + 0x04))
 115        EX_LD(LOAD(lduw, %o0 + 0x10, %g1))
 116        add             %o4, %g2, %o4
 117        EX_ST(STORE(stw, %g2, %o1 + 0x08))
 118        EX_LD(LOAD(lduw, %o0 + 0x14, %g2))
 119        add             %o4, %o5, %o4
 120        EX_ST(STORE(stw, %o5, %o1 + 0x0c))
 121        EX_LD(LOAD(lduw, %o0 + 0x18, %o5))
 122        add             %o4, %g1, %o4
 123        EX_ST(STORE(stw, %g1, %o1 + 0x10))
 124        EX_LD(LOAD(lduw, %o0 + 0x1c, %g1))
 125        add             %o4, %g2, %o4
 126        EX_ST(STORE(stw, %g2, %o1 + 0x14))
 127        EX_LD(LOAD(lduw, %o0 + 0x20, %g2))
 128        add             %o4, %o5, %o4
 129        EX_ST(STORE(stw, %o5, %o1 + 0x18))
 130        EX_LD(LOAD(lduw, %o0 + 0x24, %o5))
 131        add             %o4, %g1, %o4
 132        EX_ST(STORE(stw, %g1, %o1 + 0x1c))
 133        EX_LD(LOAD(lduw, %o0 + 0x28, %g1))
 134        add             %o4, %g2, %o4
 135        EX_ST(STORE(stw, %g2, %o1 + 0x20))
 136        EX_LD(LOAD(lduw, %o0 + 0x2c, %g2))
 137        add             %o4, %o5, %o4
 138        EX_ST(STORE(stw, %o5, %o1 + 0x24))
 139        EX_LD(LOAD(lduw, %o0 + 0x30, %o5))
 140        add             %o4, %g1, %o4
 141        EX_ST(STORE(stw, %g1, %o1 + 0x28))
 142        EX_LD(LOAD(lduw, %o0 + 0x34, %g1))
 143        add             %o4, %g2, %o4
 144        EX_ST(STORE(stw, %g2, %o1 + 0x2c))
 145        EX_LD(LOAD(lduw, %o0 + 0x38, %g2))
 146        add             %o4, %o5, %o4
 147        EX_ST(STORE(stw, %o5, %o1 + 0x30))
 148        EX_LD(LOAD(lduw, %o0 + 0x3c, %o5))
 149        add             %o4, %g1, %o4
 150        EX_ST(STORE(stw, %g1, %o1 + 0x34))
 151        LOAD(prefetch, %o0 + 0x180, #n_reads)
 152        add             %o4, %g2, %o4
 153        EX_ST(STORE(stw, %g2, %o1 + 0x38))
 154        subcc           %g3, 0x40, %g3
 155        add             %o0, 0x40, %o0
 156        add             %o4, %o5, %o4
 157        EX_ST(STORE(stw, %o5, %o1 + 0x3c))
 158        bne,pt          %icc, 1b
 159         add            %o1, 0x40, %o1
 160
 1612:      and             %o2, 0x3c, %g3
 162        brz,pn          %g3, 2f
 163         sub            %o2, %g3, %o2
 1641:      EX_LD(LOAD(lduw, %o0 + 0x00, %o5))
 165        subcc           %g3, 0x4, %g3
 166        add             %o0, 0x4, %o0
 167        add             %o4, %o5, %o4
 168        EX_ST(STORE(stw, %o5, %o1 + 0x00))
 169        bne,pt          %icc, 1b
 170         add            %o1, 0x4, %o1
 171
 1722:
 173        /* fold 64-->32 */
 174        srlx            %o4, 32, %o5
 175        srl             %o4, 0, %o4
 176        add             %o4, %o5, %o4
 177        srlx            %o4, 32, %o5
 178        srl             %o4, 0, %o4
 179        add             %o4, %o5, %o4
 180
 181        /* fold 32-->16 */
 182        sethi           %hi(0xffff0000), %g1
 183        srl             %o4, 16, %o5
 184        andn            %o4, %g1, %g2
 185        add             %o5, %g2, %o4
 186        srl             %o4, 16, %o5
 187        andn            %o4, %g1, %g2
 188        add             %o5, %g2, %o4
 189
 19060:
 191        /* %o4 has the 16-bit sum we have calculated so-far.  */
 192        cmp             %o2, 2
 193        blu,pt          %icc, 1f
 194         nop
 195        EX_LD(LOAD(lduh, %o0 + 0x00, %o5))
 196        sub             %o2, 2, %o2
 197        add             %o0, 2, %o0
 198        add             %o4, %o5, %o4
 199        EX_ST(STORE(sth, %o5, %o1 + 0x00))
 200        add             %o1, 0x2, %o1
 2011:      brz,pt          %o2, 1f
 202         nop
 203        EX_LD(LOAD(ldub, %o0 + 0x00, %o5))
 204        sub             %o2, 1, %o2
 205        add             %o0, 1, %o0
 206        EX_ST(STORE(stb, %o5, %o1 + 0x00))
 207        sllx            %o5, 8, %o5
 208        add             %o1, 1, %o1
 209        add             %o4, %o5, %o4
 2101:
 211        /* fold 32-->16 */
 212        sethi           %hi(0xffff0000), %g1
 213        srl             %o4, 16, %o5
 214        andn            %o4, %g1, %g2
 215        add             %o5, %g2, %o4
 216        srl             %o4, 16, %o5
 217        andn            %o4, %g1, %g2
 218        add             %o5, %g2, %o4
 219
 2201:      brz,pt          GLOBAL_SPARE, 1f
 221         nop
 222
 223        /* We started with an odd byte, byte-swap the result.  */
 224        srl             %o4, 8, %o5
 225        and             %o4, 0xff, %g1
 226        sll             %g1, 8, %g1
 227        or              %o5, %g1, %o4
 228
 2291:      addcc           %o3, %o4, %o3
 230        addc            %g0, %o3, %o3
 231
 23270:
 233        retl
 234         srl            %o3, 0, %o0
 235
 23695:     mov             0, GLOBAL_SPARE
 237        brlez,pn        %o2, 4f
 238         andcc          %o0, 1, %o5             
 239        be,a,pt         %icc, 1f
 240         srl            %o2, 1, %g1             
 241        sub             %o2, 1, %o2     
 242        EX_LD(LOAD(ldub, %o0, GLOBAL_SPARE))
 243        add             %o0, 1, %o0     
 244        EX_ST(STORE(stb, GLOBAL_SPARE, %o1))
 245        srl             %o2, 1, %g1
 246        add             %o1, 1, %o1
 2471:      brz,a,pn        %g1, 3f
 248         andcc          %o2, 1, %g0
 249        andcc           %o0, 2, %g0     
 250        be,a,pt         %icc, 1f
 251         srl            %g1, 1, %g1
 252        EX_LD(LOAD(lduh, %o0, %o4))
 253        sub             %o2, 2, %o2     
 254        srl             %o4, 8, %g2
 255        sub             %g1, 1, %g1     
 256        EX_ST(STORE(stb, %g2, %o1))
 257        add             %o4, GLOBAL_SPARE, GLOBAL_SPARE
 258        EX_ST(STORE(stb, %o4, %o1 + 1))
 259        add             %o0, 2, %o0     
 260        srl             %g1, 1, %g1
 261        add             %o1, 2, %o1
 2621:      brz,a,pn        %g1, 2f         
 263         andcc          %o2, 2, %g0
 264        EX_LD(LOAD(lduw, %o0, %o4))
 2655:      srl             %o4, 24, %g2
 266        srl             %o4, 16, %g3
 267        EX_ST(STORE(stb, %g2, %o1))
 268        srl             %o4, 8, %g2
 269        EX_ST(STORE(stb, %g3, %o1 + 1))
 270        add             %o0, 4, %o0
 271        EX_ST(STORE(stb, %g2, %o1 + 2))
 272        addcc           %o4, GLOBAL_SPARE, GLOBAL_SPARE
 273        EX_ST(STORE(stb, %o4, %o1 + 3))
 274        addc            GLOBAL_SPARE, %g0, GLOBAL_SPARE
 275        add             %o1, 4, %o1
 276        subcc           %g1, 1, %g1
 277        bne,a,pt        %icc, 5b
 278         EX_LD(LOAD(lduw, %o0, %o4))
 279        sll             GLOBAL_SPARE, 16, %g2
 280        srl             GLOBAL_SPARE, 16, GLOBAL_SPARE
 281        srl             %g2, 16, %g2
 282        andcc           %o2, 2, %g0
 283        add             %g2, GLOBAL_SPARE, GLOBAL_SPARE 
 2842:      be,a,pt         %icc, 3f                
 285         andcc          %o2, 1, %g0
 286        EX_LD(LOAD(lduh, %o0, %o4))
 287        andcc           %o2, 1, %g0
 288        srl             %o4, 8, %g2
 289        add             %o0, 2, %o0     
 290        EX_ST(STORE(stb, %g2, %o1))
 291        add             GLOBAL_SPARE, %o4, GLOBAL_SPARE
 292        EX_ST(STORE(stb, %o4, %o1 + 1))
 293        add             %o1, 2, %o1
 2943:      be,a,pt         %icc, 1f                
 295         sll            GLOBAL_SPARE, 16, %o4
 296        EX_LD(LOAD(ldub, %o0, %g2))
 297        sll             %g2, 8, %o4     
 298        EX_ST(STORE(stb, %g2, %o1))
 299        add             GLOBAL_SPARE, %o4, GLOBAL_SPARE
 300        sll             GLOBAL_SPARE, 16, %o4
 3011:      addcc           %o4, GLOBAL_SPARE, GLOBAL_SPARE
 302        srl             GLOBAL_SPARE, 16, %o4
 303        addc            %g0, %o4, GLOBAL_SPARE
 304        brz,pt          %o5, 4f
 305         srl            GLOBAL_SPARE, 8, %o4
 306        and             GLOBAL_SPARE, 0xff, %g2
 307        and             %o4, 0xff, %o4
 308        sll             %g2, 8, %g2
 309        or              %g2, %o4, GLOBAL_SPARE
 3104:      addcc           %o3, GLOBAL_SPARE, %o3
 311        addc            %g0, %o3, %o0
 312        retl
 313         srl            %o0, 0, %o0
 314        .size           FUNC_NAME, .-FUNC_NAME
 315
Hosted by Missing Link Electronics. The original LXR software by the LXR community, this experimental version by lxr@linux.no.