LXR linux/arch/x86/crypto/glue

   1/*
   2 * Shared glue code for 128bit block ciphers
   3 *
   4 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
   5 *
   6 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
   7 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
   8 * CTR part based on code (crypto/ctr.c) by:
   9 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
  10 *
  11 * This program is free software; you can redistribute it and/or modify
  12 * it under the terms of the GNU General Public License as published by
  13 * the Free Software Foundation; either version 2 of the License, or
  14 * (at your option) any later version.
  15 *
  16 * This program is distributed in the hope that it will be useful,
  17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 * GNU General Public License for more details.
  20 *
  21 * You should have received a copy of the GNU General Public License
  22 * along with this program; if not, write to the Free Software
  23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  24 * USA
  25 *
  26 */
  27
  28#include <linux/module.h>
  29#include <crypto/b128ops.h>
  30#include <crypto/lrw.h>
  31#include <crypto/xts.h>
  32#include <asm/crypto/glue_helper.h>
  33#include <crypto/scatterwalk.h>
  34
  35static int __glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
  36                                   struct blkcipher_desc *desc,
  37                                   struct blkcipher_walk *walk)
  38{
  39        void *ctx = crypto_blkcipher_ctx(desc->tfm);
  40        const unsigned int bsize = 128 / 8;
  41        unsigned int nbytes, i, func_bytes;
  42        bool fpu_enabled = false;
  43        int err;
  44
  45        err = blkcipher_walk_virt(desc, walk);
  46
  47        while ((nbytes = walk->nbytes)) {
  48                u8 *wsrc = walk->src.virt.addr;
  49                u8 *wdst = walk->dst.virt.addr;
  50
  51                fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
  52                                             desc, fpu_enabled, nbytes);
  53
  54                for (i = 0; i < gctx->num_funcs; i++) {
  55                        func_bytes = bsize * gctx->funcs[i].num_blocks;
  56
  57                        /* Process multi-block batch */
  58                        if (nbytes >= func_bytes) {
  59                                do {
  60                                        gctx->funcs[i].fn_u.ecb(ctx, wdst,
  61                                                                wsrc);
  62
  63                                        wsrc += func_bytes;
  64                                        wdst += func_bytes;
  65                                        nbytes -= func_bytes;
  66                                } while (nbytes >= func_bytes);
  67
  68                                if (nbytes < bsize)
  69                                        goto done;
  70                        }
  71                }
  72
  73done:
  74                err = blkcipher_walk_done(desc, walk, nbytes);
  75        }
  76
  77        glue_fpu_end(fpu_enabled);
  78        return err;
  79}
  80
  81int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
  82                          struct blkcipher_desc *desc, struct scatterlist *dst,
  83                          struct scatterlist *src, unsigned int nbytes)
  84{
  85        struct blkcipher_walk walk;
  86
  87        blkcipher_walk_init(&walk, dst, src, nbytes);
  88        return __glue_ecb_crypt_128bit(gctx, desc, &walk);
  89}
  90EXPORT_SYMBOL_GPL(glue_ecb_crypt_128bit);
  91
  92static unsigned int __glue_cbc_encrypt_128bit(const common_glue_func_t fn,
  93                                              struct blkcipher_desc *desc,
  94                                              struct blkcipher_walk *walk)
  95{
  96        void *ctx = crypto_blkcipher_ctx(desc->tfm);
  97        const unsigned int bsize = 128 / 8;
  98        unsigned int nbytes = walk->nbytes;
  99        u128 *src = (u128 *)walk->src.virt.addr;
 100        u128 *dst = (u128 *)walk->dst.virt.addr;
 101        u128 *iv = (u128 *)walk->iv;
 102
 103        do {
 104                u128_xor(dst, src, iv);
 105                fn(ctx, (u8 *)dst, (u8 *)dst);
 106                iv = dst;
 107
 108                src += 1;
 109                dst += 1;
 110                nbytes -= bsize;
 111        } while (nbytes >= bsize);
 112
 113        *(u128 *)walk->iv = *iv;
 114        return nbytes;
 115}
 116
 117int glue_cbc_encrypt_128bit(const common_glue_func_t fn,
 118                            struct blkcipher_desc *desc,
 119                            struct scatterlist *dst,
 120                            struct scatterlist *src, unsigned int nbytes)
 121{
 122        struct blkcipher_walk walk;
 123        int err;
 124
 125        blkcipher_walk_init(&walk, dst, src, nbytes);
 126        err = blkcipher_walk_virt(desc, &walk);
 127
 128        while ((nbytes = walk.nbytes)) {
 129                nbytes = __glue_cbc_encrypt_128bit(fn, desc, &walk);
 130                err = blkcipher_walk_done(desc, &walk, nbytes);
 131        }
 132
 133        return err;
 134}
 135EXPORT_SYMBOL_GPL(glue_cbc_encrypt_128bit);
 136
 137static unsigned int
 138__glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
 139                          struct blkcipher_desc *desc,
 140                          struct blkcipher_walk *walk)
 141{
 142        void *ctx = crypto_blkcipher_ctx(desc->tfm);
 143        const unsigned int bsize = 128 / 8;
 144        unsigned int nbytes = walk->nbytes;
 145        u128 *src = (u128 *)walk->src.virt.addr;
 146        u128 *dst = (u128 *)walk->dst.virt.addr;
 147        u128 last_iv;
 148        unsigned int num_blocks, func_bytes;
 149        unsigned int i;
 150
 151        /* Start of the last block. */
 152        src += nbytes / bsize - 1;
 153        dst += nbytes / bsize - 1;
 154
 155        last_iv = *src;
 156
 157        for (i = 0; i < gctx->num_funcs; i++) {
 158                num_blocks = gctx->funcs[i].num_blocks;
 159                func_bytes = bsize * num_blocks;
 160
 161                /* Process multi-block batch */
 162                if (nbytes >= func_bytes) {
 163                        do {
 164                                nbytes -= func_bytes - bsize;
 165                                src -= num_blocks - 1;
 166                                dst -= num_blocks - 1;
 167
 168                                gctx->funcs[i].fn_u.cbc(ctx, dst, src);
 169
 170                                nbytes -= bsize;
 171                                if (nbytes < bsize)
 172                                        goto done;
 173
 174                                u128_xor(dst, dst, src - 1);
 175                                src -= 1;
 176                                dst -= 1;
 177                        } while (nbytes >= func_bytes);
 178
 179                        if (nbytes < bsize)
 180                                goto done;
 181                }
 182        }
 183
 184done:
 185        u128_xor(dst, dst, (u128 *)walk->iv);
 186        *(u128 *)walk->iv = last_iv;
 187
 188        return nbytes;
 189}
 190
 191int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
 192                            struct blkcipher_desc *desc,
 193                            struct scatterlist *dst,
 194                            struct scatterlist *src, unsigned int nbytes)
 195{
 196        const unsigned int bsize = 128 / 8;
 197        bool fpu_enabled = false;
 198        struct blkcipher_walk walk;
 199        int err;
 200
 201        blkcipher_walk_init(&walk, dst, src, nbytes);
 202        err = blkcipher_walk_virt(desc, &walk);
 203
 204        while ((nbytes = walk.nbytes)) {
 205                fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
 206                                             desc, fpu_enabled, nbytes);
 207                nbytes = __glue_cbc_decrypt_128bit(gctx, desc, &walk);
 208                err = blkcipher_walk_done(desc, &walk, nbytes);
 209        }
 210
 211        glue_fpu_end(fpu_enabled);
 212        return err;
 213}
 214EXPORT_SYMBOL_GPL(glue_cbc_decrypt_128bit);
 215
 216static void glue_ctr_crypt_final_128bit(const common_glue_ctr_func_t fn_ctr,
 217                                        struct blkcipher_desc *desc,
 218                                        struct blkcipher_walk *walk)
 219{
 220        void *ctx = crypto_blkcipher_ctx(desc->tfm);
 221        u8 *src = (u8 *)walk->src.virt.addr;
 222        u8 *dst = (u8 *)walk->dst.virt.addr;
 223        unsigned int nbytes = walk->nbytes;
 224        le128 ctrblk;
 225        u128 tmp;
 226
 227        be128_to_le128(&ctrblk, (be128 *)walk->iv);
 228
 229        memcpy(&tmp, src, nbytes);
 230        fn_ctr(ctx, &tmp, &tmp, &ctrblk);
 231        memcpy(dst, &tmp, nbytes);
 232
 233        le128_to_be128((be128 *)walk->iv, &ctrblk);
 234}
 235
 236static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
 237                                            struct blkcipher_desc *desc,
 238                                            struct blkcipher_walk *walk)
 239{
 240        const unsigned int bsize = 128 / 8;
 241        void *ctx = crypto_blkcipher_ctx(desc->tfm);
 242        unsigned int nbytes = walk->nbytes;
 243        u128 *src = (u128 *)walk->src.virt.addr;
 244        u128 *dst = (u128 *)walk->dst.virt.addr;
 245        le128 ctrblk;
 246        unsigned int num_blocks, func_bytes;
 247        unsigned int i;
 248
 249        be128_to_le128(&ctrblk, (be128 *)walk->iv);
 250
 251        /* Process multi-block batch */
 252        for (i = 0; i < gctx->num_funcs; i++) {
 253                num_blocks = gctx->funcs[i].num_blocks;
 254                func_bytes = bsize * num_blocks;
 255
 256                if (nbytes >= func_bytes) {
 257                        do {
 258                                gctx->funcs[i].fn_u.ctr(ctx, dst, src, &ctrblk);
 259
 260                                src += num_blocks;
 261                                dst += num_blocks;
 262                                nbytes -= func_bytes;
 263                        } while (nbytes >= func_bytes);
 264
 265                        if (nbytes < bsize)
 266                                goto done;
 267                }
 268        }
 269
 270done:
 271        le128_to_be128((be128 *)walk->iv, &ctrblk);
 272        return nbytes;
 273}
 274
 275int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
 276                          struct blkcipher_desc *desc, struct scatterlist *dst,
 277                          struct scatterlist *src, unsigned int nbytes)
 278{
 279        const unsigned int bsize = 128 / 8;
 280        bool fpu_enabled = false;
 281        struct blkcipher_walk walk;
 282        int err;
 283
 284        blkcipher_walk_init(&walk, dst, src, nbytes);
 285        err = blkcipher_walk_virt_block(desc, &walk, bsize);
 286
 287        while ((nbytes = walk.nbytes) >= bsize) {
 288                fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
 289                                             desc, fpu_enabled, nbytes);
 290                nbytes = __glue_ctr_crypt_128bit(gctx, desc, &walk);
 291                err = blkcipher_walk_done(desc, &walk, nbytes);
 292        }
 293
 294        glue_fpu_end(fpu_enabled);
 295
 296        if (walk.nbytes) {
 297                glue_ctr_crypt_final_128bit(
 298                        gctx->funcs[gctx->num_funcs - 1].fn_u.ctr, desc, &walk);
 299                err = blkcipher_walk_done(desc, &walk, 0);
 300        }
 301
 302        return err;
 303}
 304EXPORT_SYMBOL_GPL(glue_ctr_crypt_128bit);
 305
 306static unsigned int __glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
 307                                            void *ctx,
 308                                            struct blkcipher_desc *desc,
 309                                            struct blkcipher_walk *walk)
 310{
 311        const unsigned int bsize = 128 / 8;
 312        unsigned int nbytes = walk->nbytes;
 313        u128 *src = (u128 *)walk->src.virt.addr;
 314        u128 *dst = (u128 *)walk->dst.virt.addr;
 315        unsigned int num_blocks, func_bytes;
 316        unsigned int i;
 317
 318        /* Process multi-block batch */
 319        for (i = 0; i < gctx->num_funcs; i++) {
 320                num_blocks = gctx->funcs[i].num_blocks;
 321                func_bytes = bsize * num_blocks;
 322
 323                if (nbytes >= func_bytes) {
 324                        do {
 325                                gctx->funcs[i].fn_u.xts(ctx, dst, src,
 326                                                        (le128 *)walk->iv);
 327
 328                                src += num_blocks;
 329                                dst += num_blocks;
 330                                nbytes -= func_bytes;
 331                        } while (nbytes >= func_bytes);
 332
 333                        if (nbytes < bsize)
 334                                goto done;
 335                }
 336        }
 337
 338done:
 339        return nbytes;
 340}
 341
 342/* for implementations implementing faster XTS IV generator */
 343int glue_xts_crypt_128bit(const struct common_glue_ctx *gctx,
 344                          struct blkcipher_desc *desc, struct scatterlist *dst,
 345                          struct scatterlist *src, unsigned int nbytes,
 346                          void (*tweak_fn)(void *ctx, u8 *dst, const u8 *src),
 347                          void *tweak_ctx, void *crypt_ctx)
 348{
 349        const unsigned int bsize = 128 / 8;
 350        bool fpu_enabled = false;
 351        struct blkcipher_walk walk;
 352        int err;
 353
 354        blkcipher_walk_init(&walk, dst, src, nbytes);
 355
 356        err = blkcipher_walk_virt(desc, &walk);
 357        nbytes = walk.nbytes;
 358        if (!nbytes)
 359                return err;
 360
 361        /* set minimum length to bsize, for tweak_fn */
 362        fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
 363                                     desc, fpu_enabled,
 364                                     nbytes < bsize ? bsize : nbytes);
 365
 366        /* calculate first value of T */
 367        tweak_fn(tweak_ctx, walk.iv, walk.iv);
 368
 369        while (nbytes) {
 370                nbytes = __glue_xts_crypt_128bit(gctx, crypt_ctx, desc, &walk);
 371
 372                err = blkcipher_walk_done(desc, &walk, nbytes);
 373                nbytes = walk.nbytes;
 374        }
 375
 376        glue_fpu_end(fpu_enabled);
 377
 378        return err;
 379}
 380EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit);
 381
 382void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv,
 383                               common_glue_func_t fn)
 384{
 385        le128 ivblk = *iv;
 386
 387        /* generate next IV */
 388        le128_gf128mul_x_ble(iv, &ivblk);
 389
 390        /* CC <- T xor C */
 391        u128_xor(dst, src, (u128 *)&ivblk);
 392
 393        /* PP <- D(Key2,CC) */
 394        fn(ctx, (u8 *)dst, (u8 *)dst);
 395
 396        /* P <- T xor PP */
 397        u128_xor(dst, dst, (u128 *)&ivblk);
 398}
 399EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one);
 400
 401MODULE_LICENSE("GPL");
 402