LXR linux/arch/alpha/lib/csum_partial

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * csum_partial_copy - do IP checksumming and copy
   4 *
   5 * (C) Copyright 1996 Linus Torvalds
   6 * accelerated versions (and 21264 assembly versions ) contributed by
   7 *      Rick Gorton     <rick.gorton@alpha-processor.com>
   8 *
   9 * Don't look at this too closely - you'll go mad. The things
  10 * we do for performance..
  11 */
  12
  13#include <linux/types.h>
  14#include <linux/string.h>
  15#include <linux/uaccess.h>
  16#include <net/checksum.h>
  17
  18
  19#define ldq_u(x,y) \
  20__asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
  21
  22#define stq_u(x,y) \
  23__asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
  24
  25#define extql(x,y,z) \
  26__asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  27
  28#define extqh(x,y,z) \
  29__asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  30
  31#define mskql(x,y,z) \
  32__asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  33
  34#define mskqh(x,y,z) \
  35__asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  36
  37#define insql(x,y,z) \
  38__asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  39
  40#define insqh(x,y,z) \
  41__asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  42
  43#define __get_word(insn,x,ptr)                          \
  44({                                                      \
  45        long __guu_err;                                 \
  46        __asm__ __volatile__(                           \
  47        "1:     "#insn" %0,%2\n"                        \
  48        "2:\n"                                          \
  49        EXC(1b,2b,%0,%1)                                \
  50                : "=r"(x), "=r"(__guu_err)              \
  51                : "m"(__m(ptr)), "1"(0));               \
  52        __guu_err;                                      \
  53})
  54
  55static inline unsigned short from64to16(unsigned long x)
  56{
  57        /* Using extract instructions is a bit more efficient
  58           than the original shift/bitmask version.  */
  59
  60        union {
  61                unsigned long   ul;
  62                unsigned int    ui[2];
  63                unsigned short  us[4];
  64        } in_v, tmp_v, out_v;
  65
  66        in_v.ul = x;
  67        tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
  68
  69        /* Since the bits of tmp_v.sh[3] are going to always be zero,
  70           we don't have to bother to add that in.  */
  71        out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
  72                        + (unsigned long) tmp_v.us[2];
  73
  74        /* Similarly, out_v.us[2] is always zero for the final add.  */
  75        return out_v.us[0] + out_v.us[1];
  76}
  77
  78
  79
  80/*
  81 * Ok. This isn't fun, but this is the EASY case.
  82 */
  83static inline unsigned long
  84csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
  85                         long len)
  86{
  87        unsigned long checksum = ~0U;
  88        unsigned long carry = 0;
  89
  90        while (len >= 0) {
  91                unsigned long word;
  92                if (__get_word(ldq, word, src))
  93                        return 0;
  94                checksum += carry;
  95                src++;
  96                checksum += word;
  97                len -= 8;
  98                carry = checksum < word;
  99                *dst = word;
 100                dst++;
 101        }
 102        len += 8;
 103        checksum += carry;
 104        if (len) {
 105                unsigned long word, tmp;
 106                if (__get_word(ldq, word, src))
 107                        return 0;
 108                tmp = *dst;
 109                mskql(word, len, word);
 110                checksum += word;
 111                mskqh(tmp, len, tmp);
 112                carry = checksum < word;
 113                *dst = word | tmp;
 114                checksum += carry;
 115        }
 116        return checksum;
 117}
 118
 119/*
 120 * This is even less fun, but this is still reasonably
 121 * easy.
 122 */
 123static inline unsigned long
 124csum_partial_cfu_dest_aligned(const unsigned long __user *src,
 125                              unsigned long *dst,
 126                              unsigned long soff,
 127                              long len)
 128{
 129        unsigned long first;
 130        unsigned long word, carry;
 131        unsigned long lastsrc = 7+len+(unsigned long)src;
 132        unsigned long checksum = ~0U;
 133
 134        if (__get_word(ldq_u, first,src))
 135                return 0;
 136        carry = 0;
 137        while (len >= 0) {
 138                unsigned long second;
 139
 140                if (__get_word(ldq_u, second, src+1))
 141                        return 0;
 142                extql(first, soff, word);
 143                len -= 8;
 144                src++;
 145                extqh(second, soff, first);
 146                checksum += carry;
 147                word |= first;
 148                first = second;
 149                checksum += word;
 150                *dst = word;
 151                dst++;
 152                carry = checksum < word;
 153        }
 154        len += 8;
 155        checksum += carry;
 156        if (len) {
 157                unsigned long tmp;
 158                unsigned long second;
 159                if (__get_word(ldq_u, second, lastsrc))
 160                        return 0;
 161                tmp = *dst;
 162                extql(first, soff, word);
 163                extqh(second, soff, first);
 164                word |= first;
 165                mskql(word, len, word);
 166                checksum += word;
 167                mskqh(tmp, len, tmp);
 168                carry = checksum < word;
 169                *dst = word | tmp;
 170                checksum += carry;
 171        }
 172        return checksum;
 173}
 174
 175/*
 176 * This is slightly less fun than the above..
 177 */
 178static inline unsigned long
 179csum_partial_cfu_src_aligned(const unsigned long __user *src,
 180                             unsigned long *dst,
 181                             unsigned long doff,
 182                             long len,
 183                             unsigned long partial_dest)
 184{
 185        unsigned long carry = 0;
 186        unsigned long word;
 187        unsigned long second_dest;
 188        unsigned long checksum = ~0U;
 189
 190        mskql(partial_dest, doff, partial_dest);
 191        while (len >= 0) {
 192                if (__get_word(ldq, word, src))
 193                        return 0;
 194                len -= 8;
 195                insql(word, doff, second_dest);
 196                checksum += carry;
 197                stq_u(partial_dest | second_dest, dst);
 198                src++;
 199                checksum += word;
 200                insqh(word, doff, partial_dest);
 201                carry = checksum < word;
 202                dst++;
 203        }
 204        len += 8;
 205        if (len) {
 206                checksum += carry;
 207                if (__get_word(ldq, word, src))
 208                        return 0;
 209                mskql(word, len, word);
 210                len -= 8;
 211                checksum += word;
 212                insql(word, doff, second_dest);
 213                len += doff;
 214                carry = checksum < word;
 215                partial_dest |= second_dest;
 216                if (len >= 0) {
 217                        stq_u(partial_dest, dst);
 218                        if (!len) goto out;
 219                        dst++;
 220                        insqh(word, doff, partial_dest);
 221                }
 222                doff = len;
 223        }
 224        ldq_u(second_dest, dst);
 225        mskqh(second_dest, doff, second_dest);
 226        stq_u(partial_dest | second_dest, dst);
 227out:
 228        checksum += carry;
 229        return checksum;
 230}
 231
 232/*
 233 * This is so totally un-fun that it's frightening. Don't
 234 * look at this too closely, you'll go blind.
 235 */
 236static inline unsigned long
 237csum_partial_cfu_unaligned(const unsigned long __user * src,
 238                           unsigned long * dst,
 239                           unsigned long soff, unsigned long doff,
 240                           long len, unsigned long partial_dest)
 241{
 242        unsigned long carry = 0;
 243        unsigned long first;
 244        unsigned long lastsrc;
 245        unsigned long checksum = ~0U;
 246
 247        if (__get_word(ldq_u, first, src))
 248                return 0;
 249        lastsrc = 7+len+(unsigned long)src;
 250        mskql(partial_dest, doff, partial_dest);
 251        while (len >= 0) {
 252                unsigned long second, word;
 253                unsigned long second_dest;
 254
 255                if (__get_word(ldq_u, second, src+1))
 256                        return 0;
 257                extql(first, soff, word);
 258                checksum += carry;
 259                len -= 8;
 260                extqh(second, soff, first);
 261                src++;
 262                word |= first;
 263                first = second;
 264                insql(word, doff, second_dest);
 265                checksum += word;
 266                stq_u(partial_dest | second_dest, dst);
 267                carry = checksum < word;
 268                insqh(word, doff, partial_dest);
 269                dst++;
 270        }
 271        len += doff;
 272        checksum += carry;
 273        if (len >= 0) {
 274                unsigned long second, word;
 275                unsigned long second_dest;
 276
 277                if (__get_word(ldq_u, second, lastsrc))
 278                        return 0;
 279                extql(first, soff, word);
 280                extqh(second, soff, first);
 281                word |= first;
 282                first = second;
 283                mskql(word, len-doff, word);
 284                checksum += word;
 285                insql(word, doff, second_dest);
 286                carry = checksum < word;
 287                stq_u(partial_dest | second_dest, dst);
 288                if (len) {
 289                        ldq_u(second_dest, dst+1);
 290                        insqh(word, doff, partial_dest);
 291                        mskqh(second_dest, len, second_dest);
 292                        stq_u(partial_dest | second_dest, dst+1);
 293                }
 294                checksum += carry;
 295        } else {
 296                unsigned long second, word;
 297                unsigned long second_dest;
 298
 299                if (__get_word(ldq_u, second, lastsrc))
 300                        return 0;
 301                extql(first, soff, word);
 302                extqh(second, soff, first);
 303                word |= first;
 304                ldq_u(second_dest, dst);
 305                mskql(word, len-doff, word);
 306                checksum += word;
 307                mskqh(second_dest, len, second_dest);
 308                carry = checksum < word;
 309                insql(word, doff, word);
 310                stq_u(partial_dest | word | second_dest, dst);
 311                checksum += carry;
 312        }
 313        return checksum;
 314}
 315
 316static __wsum __csum_and_copy(const void __user *src, void *dst, int len)
 317{
 318        unsigned long soff = 7 & (unsigned long) src;
 319        unsigned long doff = 7 & (unsigned long) dst;
 320        unsigned long checksum;
 321
 322        if (!doff) {
 323                if (!soff)
 324                        checksum = csum_partial_cfu_aligned(
 325                                (const unsigned long __user *) src,
 326                                (unsigned long *) dst, len-8);
 327                else
 328                        checksum = csum_partial_cfu_dest_aligned(
 329                                (const unsigned long __user *) src,
 330                                (unsigned long *) dst,
 331                                soff, len-8);
 332        } else {
 333                unsigned long partial_dest;
 334                ldq_u(partial_dest, dst);
 335                if (!soff)
 336                        checksum = csum_partial_cfu_src_aligned(
 337                                (const unsigned long __user *) src,
 338                                (unsigned long *) dst,
 339                                doff, len-8, partial_dest);
 340                else
 341                        checksum = csum_partial_cfu_unaligned(
 342                                (const unsigned long __user *) src,
 343                                (unsigned long *) dst,
 344                                soff, doff, len-8, partial_dest);
 345        }
 346        return (__force __wsum)from64to16 (checksum);
 347}
 348
 349__wsum
 350csum_and_copy_from_user(const void __user *src, void *dst, int len)
 351{
 352        if (!access_ok(src, len))
 353                return 0;
 354        return __csum_and_copy(src, dst, len);
 355}
 356EXPORT_SYMBOL(csum_and_copy_from_user);
 357
 358__wsum
 359csum_partial_copy_nocheck(const void *src, void *dst, int len)
 360{
 361        return __csum_and_copy((__force const void __user *)src,
 362                                                dst, len);
 363}
 364EXPORT_SYMBOL(csum_partial_copy_nocheck);
 365