LXR linux/lib/crypto/poly1305-donna64.c

   1// SPDX-License-Identifier: GPL-2.0 OR MIT
   2/*
   3 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
   4 *
   5 * This is based in part on Andrew Moon's poly1305-donna, which is in the
   6 * public domain.
   7 */
   8
   9#include <linux/kernel.h>
  10#include <asm/unaligned.h>
  11#include <crypto/internal/poly1305.h>
  12
  13typedef __uint128_t u128;
  14
  15void poly1305_core_setkey(struct poly1305_core_key *key,
  16                          const u8 raw_key[POLY1305_BLOCK_SIZE])
  17{
  18        u64 t0, t1;
  19
  20        /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
  21        t0 = get_unaligned_le64(&raw_key[0]);
  22        t1 = get_unaligned_le64(&raw_key[8]);
  23
  24        key->key.r64[0] = t0 & 0xffc0fffffffULL;
  25        key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL;
  26        key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL;
  27
  28        /* s = 20*r */
  29        key->precomputed_s.r64[0] = key->key.r64[1] * 20;
  30        key->precomputed_s.r64[1] = key->key.r64[2] * 20;
  31}
  32EXPORT_SYMBOL(poly1305_core_setkey);
  33
  34void poly1305_core_blocks(struct poly1305_state *state,
  35                          const struct poly1305_core_key *key, const void *src,
  36                          unsigned int nblocks, u32 hibit)
  37{
  38        const u8 *input = src;
  39        u64 hibit64;
  40        u64 r0, r1, r2;
  41        u64 s1, s2;
  42        u64 h0, h1, h2;
  43        u64 c;
  44        u128 d0, d1, d2, d;
  45
  46        if (!nblocks)
  47                return;
  48
  49        hibit64 = ((u64)hibit) << 40;
  50
  51        r0 = key->key.r64[0];
  52        r1 = key->key.r64[1];
  53        r2 = key->key.r64[2];
  54
  55        h0 = state->h64[0];
  56        h1 = state->h64[1];
  57        h2 = state->h64[2];
  58
  59        s1 = key->precomputed_s.r64[0];
  60        s2 = key->precomputed_s.r64[1];
  61
  62        do {
  63                u64 t0, t1;
  64
  65                /* h += m[i] */
  66                t0 = get_unaligned_le64(&input[0]);
  67                t1 = get_unaligned_le64(&input[8]);
  68
  69                h0 += t0 & 0xfffffffffffULL;
  70                h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL;
  71                h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64;
  72
  73                /* h *= r */
  74                d0 = (u128)h0 * r0;
  75                d = (u128)h1 * s2;
  76                d0 += d;
  77                d = (u128)h2 * s1;
  78                d0 += d;
  79                d1 = (u128)h0 * r1;
  80                d = (u128)h1 * r0;
  81                d1 += d;
  82                d = (u128)h2 * s2;
  83                d1 += d;
  84                d2 = (u128)h0 * r2;
  85                d = (u128)h1 * r1;
  86                d2 += d;
  87                d = (u128)h2 * r0;
  88                d2 += d;
  89
  90                /* (partial) h %= p */
  91                c = (u64)(d0 >> 44);
  92                h0 = (u64)d0 & 0xfffffffffffULL;
  93                d1 += c;
  94                c = (u64)(d1 >> 44);
  95                h1 = (u64)d1 & 0xfffffffffffULL;
  96                d2 += c;
  97                c = (u64)(d2 >> 42);
  98                h2 = (u64)d2 & 0x3ffffffffffULL;
  99                h0 += c * 5;
 100                c = h0 >> 44;
 101                h0 = h0 & 0xfffffffffffULL;
 102                h1 += c;
 103
 104                input += POLY1305_BLOCK_SIZE;
 105        } while (--nblocks);
 106
 107        state->h64[0] = h0;
 108        state->h64[1] = h1;
 109        state->h64[2] = h2;
 110}
 111EXPORT_SYMBOL(poly1305_core_blocks);
 112
 113void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
 114                        void *dst)
 115{
 116        u8 *mac = dst;
 117        u64 h0, h1, h2, c;
 118        u64 g0, g1, g2;
 119        u64 t0, t1;
 120
 121        /* fully carry h */
 122        h0 = state->h64[0];
 123        h1 = state->h64[1];
 124        h2 = state->h64[2];
 125
 126        c = h1 >> 44;
 127        h1 &= 0xfffffffffffULL;
 128        h2 += c;
 129        c = h2 >> 42;
 130        h2 &= 0x3ffffffffffULL;
 131        h0 += c * 5;
 132        c = h0 >> 44;
 133        h0 &= 0xfffffffffffULL;
 134        h1 += c;
 135        c = h1 >> 44;
 136        h1 &= 0xfffffffffffULL;
 137        h2 += c;
 138        c = h2 >> 42;
 139        h2 &= 0x3ffffffffffULL;
 140        h0 += c * 5;
 141        c = h0 >> 44;
 142        h0 &= 0xfffffffffffULL;
 143        h1 += c;
 144
 145        /* compute h + -p */
 146        g0 = h0 + 5;
 147        c  = g0 >> 44;
 148        g0 &= 0xfffffffffffULL;
 149        g1 = h1 + c;
 150        c  = g1 >> 44;
 151        g1 &= 0xfffffffffffULL;
 152        g2 = h2 + c - (1ULL << 42);
 153
 154        /* select h if h < p, or h + -p if h >= p */
 155        c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1;
 156        g0 &= c;
 157        g1 &= c;
 158        g2 &= c;
 159        c  = ~c;
 160        h0 = (h0 & c) | g0;
 161        h1 = (h1 & c) | g1;
 162        h2 = (h2 & c) | g2;
 163
 164        if (likely(nonce)) {
 165                /* h = (h + nonce) */
 166                t0 = ((u64)nonce[1] << 32) | nonce[0];
 167                t1 = ((u64)nonce[3] << 32) | nonce[2];
 168
 169                h0 += t0 & 0xfffffffffffULL;
 170                c = h0 >> 44;
 171                h0 &= 0xfffffffffffULL;
 172                h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c;
 173                c = h1 >> 44;
 174                h1 &= 0xfffffffffffULL;
 175                h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c;
 176                h2 &= 0x3ffffffffffULL;
 177        }
 178
 179        /* mac = h % (2^128) */
 180        h0 = h0 | (h1 << 44);
 181        h1 = (h1 >> 20) | (h2 << 24);
 182
 183        put_unaligned_le64(h0, &mac[0]);
 184        put_unaligned_le64(h1, &mac[8]);
 185}
 186EXPORT_SYMBOL(poly1305_core_emit);
 187