LXR linux/arch/x86/crypto/twofish-x86_64-asm

   1/*
   2 * Twofish Cipher 3-way parallel algorithm (x86_64)
   3 *
   4 * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation; either version 2 of the License, or
   9 * (at your option) any later version.
  10 *
  11 * This program is distributed in the hope that it will be useful,
  12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 * GNU General Public License for more details.
  15 *
  16 * You should have received a copy of the GNU General Public License
  17 * along with this program; if not, write to the Free Software
  18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307
  19 * USA
  20 *
  21 */
  22
  23#include <linux/linkage.h>
  24
  25.file "twofish-x86_64-asm-3way.S"
  26.text
  27
  28/* structure of crypto context */
  29#define s0      0
  30#define s1      1024
  31#define s2      2048
  32#define s3      3072
  33#define w       4096
  34#define k       4128
  35
  36/**********************************************************************
  37  3-way twofish
  38 **********************************************************************/
  39#define CTX %rdi
  40#define RIO %rdx
  41
  42#define RAB0 %rax
  43#define RAB1 %rbx
  44#define RAB2 %rcx
  45
  46#define RAB0d %eax
  47#define RAB1d %ebx
  48#define RAB2d %ecx
  49
  50#define RAB0bh %ah
  51#define RAB1bh %bh
  52#define RAB2bh %ch
  53
  54#define RAB0bl %al
  55#define RAB1bl %bl
  56#define RAB2bl %cl
  57
  58#define RCD0 %r8
  59#define RCD1 %r9
  60#define RCD2 %r10
  61
  62#define RCD0d %r8d
  63#define RCD1d %r9d
  64#define RCD2d %r10d
  65
  66#define RX0 %rbp
  67#define RX1 %r11
  68#define RX2 %r12
  69
  70#define RX0d %ebp
  71#define RX1d %r11d
  72#define RX2d %r12d
  73
  74#define RY0 %r13
  75#define RY1 %r14
  76#define RY2 %r15
  77
  78#define RY0d %r13d
  79#define RY1d %r14d
  80#define RY2d %r15d
  81
  82#define RT0 %rdx
  83#define RT1 %rsi
  84
  85#define RT0d %edx
  86#define RT1d %esi
  87
  88#define do16bit_ror(rot, op1, op2, T0, T1, tmp1, tmp2, ab, dst) \
  89        movzbl ab ## bl,                tmp2 ## d; \
  90        movzbl ab ## bh,                tmp1 ## d; \
  91        rorq $(rot),                    ab; \
  92        op1##l T0(CTX, tmp2, 4),        dst ## d; \
  93        op2##l T1(CTX, tmp1, 4),        dst ## d;
  94
  95/*
  96 * Combined G1 & G2 function. Reordered with help of rotates to have moves
  97 * at begining.
  98 */
  99#define g1g2_3(ab, cd, Tx0, Tx1, Tx2, Tx3, Ty0, Ty1, Ty2, Ty3, x, y) \
 100        /* G1,1 && G2,1 */ \
 101        do16bit_ror(32, mov, xor, Tx0, Tx1, RT0, x ## 0, ab ## 0, x ## 0); \
 102        do16bit_ror(48, mov, xor, Ty1, Ty2, RT0, y ## 0, ab ## 0, y ## 0); \
 103        \
 104        do16bit_ror(32, mov, xor, Tx0, Tx1, RT0, x ## 1, ab ## 1, x ## 1); \
 105        do16bit_ror(48, mov, xor, Ty1, Ty2, RT0, y ## 1, ab ## 1, y ## 1); \
 106        \
 107        do16bit_ror(32, mov, xor, Tx0, Tx1, RT0, x ## 2, ab ## 2, x ## 2); \
 108        do16bit_ror(48, mov, xor, Ty1, Ty2, RT0, y ## 2, ab ## 2, y ## 2); \
 109        \
 110        /* G1,2 && G2,2 */ \
 111        do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 0, x ## 0); \
 112        do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 0, y ## 0); \
 113        xchgq cd ## 0, ab ## 0; \
 114        \
 115        do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 1, x ## 1); \
 116        do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 1, y ## 1); \
 117        xchgq cd ## 1, ab ## 1; \
 118        \
 119        do16bit_ror(32, xor, xor, Tx2, Tx3, RT0, RT1, ab ## 2, x ## 2); \
 120        do16bit_ror(16, xor, xor, Ty3, Ty0, RT0, RT1, ab ## 2, y ## 2); \
 121        xchgq cd ## 2, ab ## 2;
 122
 123#define enc_round_end(ab, x, y, n) \
 124        addl y ## d,                    x ## d; \
 125        addl x ## d,                    y ## d; \
 126        addl k+4*(2*(n))(CTX),          x ## d; \
 127        xorl ab ## d,                   x ## d; \
 128        addl k+4*(2*(n)+1)(CTX),        y ## d; \
 129        shrq $32,                       ab; \
 130        roll $1,                        ab ## d; \
 131        xorl y ## d,                    ab ## d; \
 132        shlq $32,                       ab; \
 133        rorl $1,                        x ## d; \
 134        orq x,                          ab;
 135
 136#define dec_round_end(ba, x, y, n) \
 137        addl y ## d,                    x ## d; \
 138        addl x ## d,                    y ## d; \
 139        addl k+4*(2*(n))(CTX),          x ## d; \
 140        addl k+4*(2*(n)+1)(CTX),        y ## d; \
 141        xorl ba ## d,                   y ## d; \
 142        shrq $32,                       ba; \
 143        roll $1,                        ba ## d; \
 144        xorl x ## d,                    ba ## d; \
 145        shlq $32,                       ba; \
 146        rorl $1,                        y ## d; \
 147        orq y,                          ba;
 148
 149#define encrypt_round3(ab, cd, n) \
 150        g1g2_3(ab, cd, s0, s1, s2, s3, s0, s1, s2, s3, RX, RY); \
 151        \
 152        enc_round_end(ab ## 0, RX0, RY0, n); \
 153        enc_round_end(ab ## 1, RX1, RY1, n); \
 154        enc_round_end(ab ## 2, RX2, RY2, n);
 155
 156#define decrypt_round3(ba, dc, n) \
 157        g1g2_3(ba, dc, s1, s2, s3, s0, s3, s0, s1, s2, RY, RX); \
 158        \
 159        dec_round_end(ba ## 0, RX0, RY0, n); \
 160        dec_round_end(ba ## 1, RX1, RY1, n); \
 161        dec_round_end(ba ## 2, RX2, RY2, n);
 162
 163#define encrypt_cycle3(ab, cd, n) \
 164        encrypt_round3(ab, cd, n*2); \
 165        encrypt_round3(ab, cd, (n*2)+1);
 166
 167#define decrypt_cycle3(ba, dc, n) \
 168        decrypt_round3(ba, dc, (n*2)+1); \
 169        decrypt_round3(ba, dc, (n*2));
 170
 171#define inpack3(in, n, xy, m) \
 172        movq 4*(n)(in),                 xy ## 0; \
 173        xorq w+4*m(CTX),                xy ## 0; \
 174        \
 175        movq 4*(4+(n))(in),             xy ## 1; \
 176        xorq w+4*m(CTX),                xy ## 1; \
 177        \
 178        movq 4*(8+(n))(in),             xy ## 2; \
 179        xorq w+4*m(CTX),                xy ## 2;
 180
 181#define outunpack3(op, out, n, xy, m) \
 182        xorq w+4*m(CTX),                xy ## 0; \
 183        op ## q xy ## 0,                4*(n)(out); \
 184        \
 185        xorq w+4*m(CTX),                xy ## 1; \
 186        op ## q xy ## 1,                4*(4+(n))(out); \
 187        \
 188        xorq w+4*m(CTX),                xy ## 2; \
 189        op ## q xy ## 2,                4*(8+(n))(out);
 190
 191#define inpack_enc3() \
 192        inpack3(RIO, 0, RAB, 0); \
 193        inpack3(RIO, 2, RCD, 2);
 194
 195#define outunpack_enc3(op) \
 196        outunpack3(op, RIO, 2, RAB, 6); \
 197        outunpack3(op, RIO, 0, RCD, 4);
 198
 199#define inpack_dec3() \
 200        inpack3(RIO, 0, RAB, 4); \
 201        rorq $32,                       RAB0; \
 202        rorq $32,                       RAB1; \
 203        rorq $32,                       RAB2; \
 204        inpack3(RIO, 2, RCD, 6); \
 205        rorq $32,                       RCD0; \
 206        rorq $32,                       RCD1; \
 207        rorq $32,                       RCD2;
 208
 209#define outunpack_dec3() \
 210        rorq $32,                       RCD0; \
 211        rorq $32,                       RCD1; \
 212        rorq $32,                       RCD2; \
 213        outunpack3(mov, RIO, 0, RCD, 0); \
 214        rorq $32,                       RAB0; \
 215        rorq $32,                       RAB1; \
 216        rorq $32,                       RAB2; \
 217        outunpack3(mov, RIO, 2, RAB, 2);
 218
 219ENTRY(__twofish_enc_blk_3way)
 220        /* input:
 221         *      %rdi: ctx, CTX
 222         *      %rsi: dst
 223         *      %rdx: src, RIO
 224         *      %rcx: bool, if true: xor output
 225         */
 226        pushq %r15;
 227        pushq %r14;
 228        pushq %r13;
 229        pushq %r12;
 230        pushq %rbp;
 231        pushq %rbx;
 232
 233        pushq %rcx; /* bool xor */
 234        pushq %rsi; /* dst */
 235
 236        inpack_enc3();
 237
 238        encrypt_cycle3(RAB, RCD, 0);
 239        encrypt_cycle3(RAB, RCD, 1);
 240        encrypt_cycle3(RAB, RCD, 2);
 241        encrypt_cycle3(RAB, RCD, 3);
 242        encrypt_cycle3(RAB, RCD, 4);
 243        encrypt_cycle3(RAB, RCD, 5);
 244        encrypt_cycle3(RAB, RCD, 6);
 245        encrypt_cycle3(RAB, RCD, 7);
 246
 247        popq RIO; /* dst */
 248        popq %rbp; /* bool xor */
 249
 250        testb %bpl, %bpl;
 251        jnz .L__enc_xor3;
 252
 253        outunpack_enc3(mov);
 254
 255        popq %rbx;
 256        popq %rbp;
 257        popq %r12;
 258        popq %r13;
 259        popq %r14;
 260        popq %r15;
 261        ret;
 262
 263.L__enc_xor3:
 264        outunpack_enc3(xor);
 265
 266        popq %rbx;
 267        popq %rbp;
 268        popq %r12;
 269        popq %r13;
 270        popq %r14;
 271        popq %r15;
 272        ret;
 273ENDPROC(__twofish_enc_blk_3way)
 274
 275ENTRY(twofish_dec_blk_3way)
 276        /* input:
 277         *      %rdi: ctx, CTX
 278         *      %rsi: dst
 279         *      %rdx: src, RIO
 280         */
 281        pushq %r15;
 282        pushq %r14;
 283        pushq %r13;
 284        pushq %r12;
 285        pushq %rbp;
 286        pushq %rbx;
 287
 288        pushq %rsi; /* dst */
 289
 290        inpack_dec3();
 291
 292        decrypt_cycle3(RAB, RCD, 7);
 293        decrypt_cycle3(RAB, RCD, 6);
 294        decrypt_cycle3(RAB, RCD, 5);
 295        decrypt_cycle3(RAB, RCD, 4);
 296        decrypt_cycle3(RAB, RCD, 3);
 297        decrypt_cycle3(RAB, RCD, 2);
 298        decrypt_cycle3(RAB, RCD, 1);
 299        decrypt_cycle3(RAB, RCD, 0);
 300
 301        popq RIO; /* dst */
 302
 303        outunpack_dec3();
 304
 305        popq %rbx;
 306        popq %rbp;
 307        popq %r12;
 308        popq %r13;
 309        popq %r14;
 310        popq %r15;
 311        ret;
 312ENDPROC(twofish_dec_blk_3way)
 313