LXR linux/arch/x86/crypto/aesni-intel

   1/*
   2 * Implement AES algorithm in Intel AES-NI instructions.
   3 *
   4 * The white paper of AES-NI instructions can be downloaded from:
   5 *   http://softwarecommunity.intel.com/isn/downloads/intelavx/AES-Instructions-Set_WP.pdf
   6 *
   7 * Copyright (C) 2008, Intel Corp.
   8 *    Author: Huang Ying <ying.huang@intel.com>
   9 *            Vinodh Gopal <vinodh.gopal@intel.com>
  10 *            Kahraman Akdemir
  11 *
  12 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
  13 * interface for 64-bit kernels.
  14 *    Authors: Erdinc Ozturk (erdinc.ozturk@intel.com)
  15 *             Aidan O'Mahony (aidan.o.mahony@intel.com)
  16 *             Adrian Hoban <adrian.hoban@intel.com>
  17 *             James Guilford (james.guilford@intel.com)
  18 *             Gabriele Paoloni <gabriele.paoloni@intel.com>
  19 *             Tadeusz Struk (tadeusz.struk@intel.com)
  20 *             Wajdi Feghali (wajdi.k.feghali@intel.com)
  21 *    Copyright (c) 2010, Intel Corporation.
  22 *
  23 * Ported x86_64 version to x86:
  24 *    Author: Mathias Krause <minipli@googlemail.com>
  25 *
  26 * This program is free software; you can redistribute it and/or modify
  27 * it under the terms of the GNU General Public License as published by
  28 * the Free Software Foundation; either version 2 of the License, or
  29 * (at your option) any later version.
  30 */
  31
  32#include <linux/linkage.h>
  33#include <asm/inst.h>
  34
  35#ifdef __x86_64__
  36.data
  37.align 16
  38.Lgf128mul_x_ble_mask:
  39        .octa 0x00000000000000010000000000000087
  40
  41POLY:   .octa 0xC2000000000000000000000000000001
  42TWOONE: .octa 0x00000001000000000000000000000001
  43
  44# order of these constants should not change.
  45# more specifically, ALL_F should follow SHIFT_MASK,
  46# and ZERO should follow ALL_F
  47
  48SHUF_MASK:  .octa 0x000102030405060708090A0B0C0D0E0F
  49MASK1:      .octa 0x0000000000000000ffffffffffffffff
  50MASK2:      .octa 0xffffffffffffffff0000000000000000
  51SHIFT_MASK: .octa 0x0f0e0d0c0b0a09080706050403020100
  52ALL_F:      .octa 0xffffffffffffffffffffffffffffffff
  53ZERO:       .octa 0x00000000000000000000000000000000
  54ONE:        .octa 0x00000000000000000000000000000001
  55F_MIN_MASK: .octa 0xf1f2f3f4f5f6f7f8f9fafbfcfdfeff0
  56dec:        .octa 0x1
  57enc:        .octa 0x2
  58
  59
  60.text
  61
  62
  63#define STACK_OFFSET    8*3
  64#define HashKey         16*0    // store HashKey <<1 mod poly here
  65#define HashKey_2       16*1    // store HashKey^2 <<1 mod poly here
  66#define HashKey_3       16*2    // store HashKey^3 <<1 mod poly here
  67#define HashKey_4       16*3    // store HashKey^4 <<1 mod poly here
  68#define HashKey_k       16*4    // store XOR of High 64 bits and Low 64
  69                                // bits of  HashKey <<1 mod poly here
  70                                //(for Karatsuba purposes)
  71#define HashKey_2_k     16*5    // store XOR of High 64 bits and Low 64
  72                                // bits of  HashKey^2 <<1 mod poly here
  73                                // (for Karatsuba purposes)
  74#define HashKey_3_k     16*6    // store XOR of High 64 bits and Low 64
  75                                // bits of  HashKey^3 <<1 mod poly here
  76                                // (for Karatsuba purposes)
  77#define HashKey_4_k     16*7    // store XOR of High 64 bits and Low 64
  78                                // bits of  HashKey^4 <<1 mod poly here
  79                                // (for Karatsuba purposes)
  80#define VARIABLE_OFFSET 16*8
  81
  82#define arg1 rdi
  83#define arg2 rsi
  84#define arg3 rdx
  85#define arg4 rcx
  86#define arg5 r8
  87#define arg6 r9
  88#define arg7 STACK_OFFSET+8(%r14)
  89#define arg8 STACK_OFFSET+16(%r14)
  90#define arg9 STACK_OFFSET+24(%r14)
  91#define arg10 STACK_OFFSET+32(%r14)
  92#endif
  93
  94
  95#define STATE1  %xmm0
  96#define STATE2  %xmm4
  97#define STATE3  %xmm5
  98#define STATE4  %xmm6
  99#define STATE   STATE1
 100#define IN1     %xmm1
 101#define IN2     %xmm7
 102#define IN3     %xmm8
 103#define IN4     %xmm9
 104#define IN      IN1
 105#define KEY     %xmm2
 106#define IV      %xmm3
 107
 108#define BSWAP_MASK %xmm10
 109#define CTR     %xmm11
 110#define INC     %xmm12
 111
 112#define GF128MUL_MASK %xmm10
 113
 114#ifdef __x86_64__
 115#define AREG    %rax
 116#define KEYP    %rdi
 117#define OUTP    %rsi
 118#define UKEYP   OUTP
 119#define INP     %rdx
 120#define LEN     %rcx
 121#define IVP     %r8
 122#define KLEN    %r9d
 123#define T1      %r10
 124#define TKEYP   T1
 125#define T2      %r11
 126#define TCTR_LOW T2
 127#else
 128#define AREG    %eax
 129#define KEYP    %edi
 130#define OUTP    AREG
 131#define UKEYP   OUTP
 132#define INP     %edx
 133#define LEN     %esi
 134#define IVP     %ebp
 135#define KLEN    %ebx
 136#define T1      %ecx
 137#define TKEYP   T1
 138#endif
 139
 140
 141#ifdef __x86_64__
 142/* GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
 143*
 144*
 145* Input: A and B (128-bits each, bit-reflected)
 146* Output: C = A*B*x mod poly, (i.e. >>1 )
 147* To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input
 148* GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly.
 149*
 150*/
 151.macro GHASH_MUL GH HK TMP1 TMP2 TMP3 TMP4 TMP5
 152        movdqa    \GH, \TMP1
 153        pshufd    $78, \GH, \TMP2
 154        pshufd    $78, \HK, \TMP3
 155        pxor      \GH, \TMP2            # TMP2 = a1+a0
 156        pxor      \HK, \TMP3            # TMP3 = b1+b0
 157        PCLMULQDQ 0x11, \HK, \TMP1     # TMP1 = a1*b1
 158        PCLMULQDQ 0x00, \HK, \GH       # GH = a0*b0
 159        PCLMULQDQ 0x00, \TMP3, \TMP2   # TMP2 = (a0+a1)*(b1+b0)
 160        pxor      \GH, \TMP2
 161        pxor      \TMP1, \TMP2          # TMP2 = (a0*b0)+(a1*b0)
 162        movdqa    \TMP2, \TMP3
 163        pslldq    $8, \TMP3             # left shift TMP3 2 DWs
 164        psrldq    $8, \TMP2             # right shift TMP2 2 DWs
 165        pxor      \TMP3, \GH
 166        pxor      \TMP2, \TMP1          # TMP2:GH holds the result of GH*HK
 167
 168        # first phase of the reduction
 169
 170        movdqa    \GH, \TMP2
 171        movdqa    \GH, \TMP3
 172        movdqa    \GH, \TMP4            # copy GH into TMP2,TMP3 and TMP4
 173                                        # in in order to perform
 174                                        # independent shifts
 175        pslld     $31, \TMP2            # packed right shift <<31
 176        pslld     $30, \TMP3            # packed right shift <<30
 177        pslld     $25, \TMP4            # packed right shift <<25
 178        pxor      \TMP3, \TMP2          # xor the shifted versions
 179        pxor      \TMP4, \TMP2
 180        movdqa    \TMP2, \TMP5
 181        psrldq    $4, \TMP5             # right shift TMP5 1 DW
 182        pslldq    $12, \TMP2            # left shift TMP2 3 DWs
 183        pxor      \TMP2, \GH
 184
 185        # second phase of the reduction
 186
 187        movdqa    \GH,\TMP2             # copy GH into TMP2,TMP3 and TMP4
 188                                        # in in order to perform
 189                                        # independent shifts
 190        movdqa    \GH,\TMP3
 191        movdqa    \GH,\TMP4
 192        psrld     $1,\TMP2              # packed left shift >>1
 193        psrld     $2,\TMP3              # packed left shift >>2
 194        psrld     $7,\TMP4              # packed left shift >>7
 195        pxor      \TMP3,\TMP2           # xor the shifted versions
 196        pxor      \TMP4,\TMP2
 197        pxor      \TMP5, \TMP2
 198        pxor      \TMP2, \GH
 199        pxor      \TMP1, \GH            # result is in TMP1
 200.endm
 201
 202/*
 203* if a = number of total plaintext bytes
 204* b = floor(a/16)
 205* num_initial_blocks = b mod 4
 206* encrypt the initial num_initial_blocks blocks and apply ghash on
 207* the ciphertext
 208* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers
 209* are clobbered
 210* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified
 211*/
 212
 213
 214.macro INITIAL_BLOCKS_DEC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
 215XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
 216        mov        arg7, %r10           # %r10 = AAD
 217        mov        arg8, %r12           # %r12 = aadLen
 218        mov        %r12, %r11
 219        pxor       %xmm\i, %xmm\i
 220_get_AAD_loop\num_initial_blocks\operation:
 221        movd       (%r10), \TMP1
 222        pslldq     $12, \TMP1
 223        psrldq     $4, %xmm\i
 224        pxor       \TMP1, %xmm\i
 225        add        $4, %r10
 226        sub        $4, %r12
 227        jne        _get_AAD_loop\num_initial_blocks\operation
 228        cmp        $16, %r11
 229        je         _get_AAD_loop2_done\num_initial_blocks\operation
 230        mov        $16, %r12
 231_get_AAD_loop2\num_initial_blocks\operation:
 232        psrldq     $4, %xmm\i
 233        sub        $4, %r12
 234        cmp        %r11, %r12
 235        jne        _get_AAD_loop2\num_initial_blocks\operation
 236_get_AAD_loop2_done\num_initial_blocks\operation:
 237        movdqa     SHUF_MASK(%rip), %xmm14
 238        PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
 239
 240        xor        %r11, %r11 # initialise the data pointer offset as zero
 241
 242        # start AES for num_initial_blocks blocks
 243
 244        mov        %arg5, %rax                      # %rax = *Y0
 245        movdqu     (%rax), \XMM0                    # XMM0 = Y0
 246        movdqa     SHUF_MASK(%rip), %xmm14
 247        PSHUFB_XMM   %xmm14, \XMM0
 248
 249.if (\i == 5) || (\i == 6) || (\i == 7)
 250.irpc index, \i_seq
 251        paddd      ONE(%rip), \XMM0                 # INCR Y0
 252        movdqa     \XMM0, %xmm\index
 253        movdqa     SHUF_MASK(%rip), %xmm14
 254        PSHUFB_XMM   %xmm14, %xmm\index      # perform a 16 byte swap
 255
 256.endr
 257.irpc index, \i_seq
 258        pxor       16*0(%arg1), %xmm\index
 259.endr
 260.irpc index, \i_seq
 261        movaps 0x10(%rdi), \TMP1
 262        AESENC     \TMP1, %xmm\index          # Round 1
 263.endr
 264.irpc index, \i_seq
 265        movaps 0x20(%arg1), \TMP1
 266        AESENC     \TMP1, %xmm\index          # Round 2
 267.endr
 268.irpc index, \i_seq
 269        movaps 0x30(%arg1), \TMP1
 270        AESENC     \TMP1, %xmm\index          # Round 2
 271.endr
 272.irpc index, \i_seq
 273        movaps 0x40(%arg1), \TMP1
 274        AESENC     \TMP1, %xmm\index          # Round 2
 275.endr
 276.irpc index, \i_seq
 277        movaps 0x50(%arg1), \TMP1
 278        AESENC     \TMP1, %xmm\index          # Round 2
 279.endr
 280.irpc index, \i_seq
 281        movaps 0x60(%arg1), \TMP1
 282        AESENC     \TMP1, %xmm\index          # Round 2
 283.endr
 284.irpc index, \i_seq
 285        movaps 0x70(%arg1), \TMP1
 286        AESENC     \TMP1, %xmm\index          # Round 2
 287.endr
 288.irpc index, \i_seq
 289        movaps 0x80(%arg1), \TMP1
 290        AESENC     \TMP1, %xmm\index          # Round 2
 291.endr
 292.irpc index, \i_seq
 293        movaps 0x90(%arg1), \TMP1
 294        AESENC     \TMP1, %xmm\index          # Round 2
 295.endr
 296.irpc index, \i_seq
 297        movaps 0xa0(%arg1), \TMP1
 298        AESENCLAST \TMP1, %xmm\index         # Round 10
 299.endr
 300.irpc index, \i_seq
 301        movdqu     (%arg3 , %r11, 1), \TMP1
 302        pxor       \TMP1, %xmm\index
 303        movdqu     %xmm\index, (%arg2 , %r11, 1)
 304        # write back plaintext/ciphertext for num_initial_blocks
 305        add        $16, %r11
 306
 307        movdqa     \TMP1, %xmm\index
 308        movdqa     SHUF_MASK(%rip), %xmm14
 309        PSHUFB_XMM         %xmm14, %xmm\index
 310
 311                # prepare plaintext/ciphertext for GHASH computation
 312.endr
 313.endif
 314        GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 315        # apply GHASH on num_initial_blocks blocks
 316
 317.if \i == 5
 318        pxor       %xmm5, %xmm6
 319        GHASH_MUL  %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 320        pxor       %xmm6, %xmm7
 321        GHASH_MUL  %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 322        pxor       %xmm7, %xmm8
 323        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 324.elseif \i == 6
 325        pxor       %xmm6, %xmm7
 326        GHASH_MUL  %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 327        pxor       %xmm7, %xmm8
 328        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 329.elseif \i == 7
 330        pxor       %xmm7, %xmm8
 331        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 332.endif
 333        cmp        $64, %r13
 334        jl      _initial_blocks_done\num_initial_blocks\operation
 335        # no need for precomputed values
 336/*
 337*
 338* Precomputations for HashKey parallel with encryption of first 4 blocks.
 339* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
 340*/
 341        paddd      ONE(%rip), \XMM0              # INCR Y0
 342        movdqa     \XMM0, \XMM1
 343        movdqa     SHUF_MASK(%rip), %xmm14
 344        PSHUFB_XMM  %xmm14, \XMM1        # perform a 16 byte swap
 345
 346        paddd      ONE(%rip), \XMM0              # INCR Y0
 347        movdqa     \XMM0, \XMM2
 348        movdqa     SHUF_MASK(%rip), %xmm14
 349        PSHUFB_XMM  %xmm14, \XMM2        # perform a 16 byte swap
 350
 351        paddd      ONE(%rip), \XMM0              # INCR Y0
 352        movdqa     \XMM0, \XMM3
 353        movdqa     SHUF_MASK(%rip), %xmm14
 354        PSHUFB_XMM %xmm14, \XMM3        # perform a 16 byte swap
 355
 356        paddd      ONE(%rip), \XMM0              # INCR Y0
 357        movdqa     \XMM0, \XMM4
 358        movdqa     SHUF_MASK(%rip), %xmm14
 359        PSHUFB_XMM %xmm14, \XMM4        # perform a 16 byte swap
 360
 361        pxor       16*0(%arg1), \XMM1
 362        pxor       16*0(%arg1), \XMM2
 363        pxor       16*0(%arg1), \XMM3
 364        pxor       16*0(%arg1), \XMM4
 365        movdqa     \TMP3, \TMP5
 366        pshufd     $78, \TMP3, \TMP1
 367        pxor       \TMP3, \TMP1
 368        movdqa     \TMP1, HashKey_k(%rsp)
 369        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
 370# TMP5 = HashKey^2<<1 (mod poly)
 371        movdqa     \TMP5, HashKey_2(%rsp)
 372# HashKey_2 = HashKey^2<<1 (mod poly)
 373        pshufd     $78, \TMP5, \TMP1
 374        pxor       \TMP5, \TMP1
 375        movdqa     \TMP1, HashKey_2_k(%rsp)
 376.irpc index, 1234 # do 4 rounds
 377        movaps 0x10*\index(%arg1), \TMP1
 378        AESENC     \TMP1, \XMM1
 379        AESENC     \TMP1, \XMM2
 380        AESENC     \TMP1, \XMM3
 381        AESENC     \TMP1, \XMM4
 382.endr
 383        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
 384# TMP5 = HashKey^3<<1 (mod poly)
 385        movdqa     \TMP5, HashKey_3(%rsp)
 386        pshufd     $78, \TMP5, \TMP1
 387        pxor       \TMP5, \TMP1
 388        movdqa     \TMP1, HashKey_3_k(%rsp)
 389.irpc index, 56789 # do next 5 rounds
 390        movaps 0x10*\index(%arg1), \TMP1
 391        AESENC     \TMP1, \XMM1
 392        AESENC     \TMP1, \XMM2
 393        AESENC     \TMP1, \XMM3
 394        AESENC     \TMP1, \XMM4
 395.endr
 396        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
 397# TMP5 = HashKey^3<<1 (mod poly)
 398        movdqa     \TMP5, HashKey_4(%rsp)
 399        pshufd     $78, \TMP5, \TMP1
 400        pxor       \TMP5, \TMP1
 401        movdqa     \TMP1, HashKey_4_k(%rsp)
 402        movaps 0xa0(%arg1), \TMP2
 403        AESENCLAST \TMP2, \XMM1
 404        AESENCLAST \TMP2, \XMM2
 405        AESENCLAST \TMP2, \XMM3
 406        AESENCLAST \TMP2, \XMM4
 407        movdqu     16*0(%arg3 , %r11 , 1), \TMP1
 408        pxor       \TMP1, \XMM1
 409        movdqu     \XMM1, 16*0(%arg2 , %r11 , 1)
 410        movdqa     \TMP1, \XMM1
 411        movdqu     16*1(%arg3 , %r11 , 1), \TMP1
 412        pxor       \TMP1, \XMM2
 413        movdqu     \XMM2, 16*1(%arg2 , %r11 , 1)
 414        movdqa     \TMP1, \XMM2
 415        movdqu     16*2(%arg3 , %r11 , 1), \TMP1
 416        pxor       \TMP1, \XMM3
 417        movdqu     \XMM3, 16*2(%arg2 , %r11 , 1)
 418        movdqa     \TMP1, \XMM3
 419        movdqu     16*3(%arg3 , %r11 , 1), \TMP1
 420        pxor       \TMP1, \XMM4
 421        movdqu     \XMM4, 16*3(%arg2 , %r11 , 1)
 422        movdqa     \TMP1, \XMM4
 423        add        $64, %r11
 424        movdqa     SHUF_MASK(%rip), %xmm14
 425        PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
 426        pxor       \XMMDst, \XMM1
 427# combine GHASHed value with the corresponding ciphertext
 428        movdqa     SHUF_MASK(%rip), %xmm14
 429        PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
 430        movdqa     SHUF_MASK(%rip), %xmm14
 431        PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
 432        movdqa     SHUF_MASK(%rip), %xmm14
 433        PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
 434
 435_initial_blocks_done\num_initial_blocks\operation:
 436
 437.endm
 438
 439
 440/*
 441* if a = number of total plaintext bytes
 442* b = floor(a/16)
 443* num_initial_blocks = b mod 4
 444* encrypt the initial num_initial_blocks blocks and apply ghash on
 445* the ciphertext
 446* %r10, %r11, %r12, %rax, %xmm5, %xmm6, %xmm7, %xmm8, %xmm9 registers
 447* are clobbered
 448* arg1, %arg2, %arg3, %r14 are used as a pointer only, not modified
 449*/
 450
 451
 452.macro INITIAL_BLOCKS_ENC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \
 453XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation
 454        mov        arg7, %r10           # %r10 = AAD
 455        mov        arg8, %r12           # %r12 = aadLen
 456        mov        %r12, %r11
 457        pxor       %xmm\i, %xmm\i
 458_get_AAD_loop\num_initial_blocks\operation:
 459        movd       (%r10), \TMP1
 460        pslldq     $12, \TMP1
 461        psrldq     $4, %xmm\i
 462        pxor       \TMP1, %xmm\i
 463        add        $4, %r10
 464        sub        $4, %r12
 465        jne        _get_AAD_loop\num_initial_blocks\operation
 466        cmp        $16, %r11
 467        je         _get_AAD_loop2_done\num_initial_blocks\operation
 468        mov        $16, %r12
 469_get_AAD_loop2\num_initial_blocks\operation:
 470        psrldq     $4, %xmm\i
 471        sub        $4, %r12
 472        cmp        %r11, %r12
 473        jne        _get_AAD_loop2\num_initial_blocks\operation
 474_get_AAD_loop2_done\num_initial_blocks\operation:
 475        movdqa     SHUF_MASK(%rip), %xmm14
 476        PSHUFB_XMM   %xmm14, %xmm\i # byte-reflect the AAD data
 477
 478        xor        %r11, %r11 # initialise the data pointer offset as zero
 479
 480        # start AES for num_initial_blocks blocks
 481
 482        mov        %arg5, %rax                      # %rax = *Y0
 483        movdqu     (%rax), \XMM0                    # XMM0 = Y0
 484        movdqa     SHUF_MASK(%rip), %xmm14
 485        PSHUFB_XMM   %xmm14, \XMM0
 486
 487.if (\i == 5) || (\i == 6) || (\i == 7)
 488.irpc index, \i_seq
 489        paddd      ONE(%rip), \XMM0                 # INCR Y0
 490        movdqa     \XMM0, %xmm\index
 491        movdqa     SHUF_MASK(%rip), %xmm14
 492        PSHUFB_XMM   %xmm14, %xmm\index      # perform a 16 byte swap
 493
 494.endr
 495.irpc index, \i_seq
 496        pxor       16*0(%arg1), %xmm\index
 497.endr
 498.irpc index, \i_seq
 499        movaps 0x10(%rdi), \TMP1
 500        AESENC     \TMP1, %xmm\index          # Round 1
 501.endr
 502.irpc index, \i_seq
 503        movaps 0x20(%arg1), \TMP1
 504        AESENC     \TMP1, %xmm\index          # Round 2
 505.endr
 506.irpc index, \i_seq
 507        movaps 0x30(%arg1), \TMP1
 508        AESENC     \TMP1, %xmm\index          # Round 2
 509.endr
 510.irpc index, \i_seq
 511        movaps 0x40(%arg1), \TMP1
 512        AESENC     \TMP1, %xmm\index          # Round 2
 513.endr
 514.irpc index, \i_seq
 515        movaps 0x50(%arg1), \TMP1
 516        AESENC     \TMP1, %xmm\index          # Round 2
 517.endr
 518.irpc index, \i_seq
 519        movaps 0x60(%arg1), \TMP1
 520        AESENC     \TMP1, %xmm\index          # Round 2
 521.endr
 522.irpc index, \i_seq
 523        movaps 0x70(%arg1), \TMP1
 524        AESENC     \TMP1, %xmm\index          # Round 2
 525.endr
 526.irpc index, \i_seq
 527        movaps 0x80(%arg1), \TMP1
 528        AESENC     \TMP1, %xmm\index          # Round 2
 529.endr
 530.irpc index, \i_seq
 531        movaps 0x90(%arg1), \TMP1
 532        AESENC     \TMP1, %xmm\index          # Round 2
 533.endr
 534.irpc index, \i_seq
 535        movaps 0xa0(%arg1), \TMP1
 536        AESENCLAST \TMP1, %xmm\index         # Round 10
 537.endr
 538.irpc index, \i_seq
 539        movdqu     (%arg3 , %r11, 1), \TMP1
 540        pxor       \TMP1, %xmm\index
 541        movdqu     %xmm\index, (%arg2 , %r11, 1)
 542        # write back plaintext/ciphertext for num_initial_blocks
 543        add        $16, %r11
 544
 545        movdqa     SHUF_MASK(%rip), %xmm14
 546        PSHUFB_XMM         %xmm14, %xmm\index
 547
 548                # prepare plaintext/ciphertext for GHASH computation
 549.endr
 550.endif
 551        GHASH_MUL  %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 552        # apply GHASH on num_initial_blocks blocks
 553
 554.if \i == 5
 555        pxor       %xmm5, %xmm6
 556        GHASH_MUL  %xmm6, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 557        pxor       %xmm6, %xmm7
 558        GHASH_MUL  %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 559        pxor       %xmm7, %xmm8
 560        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 561.elseif \i == 6
 562        pxor       %xmm6, %xmm7
 563        GHASH_MUL  %xmm7, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 564        pxor       %xmm7, %xmm8
 565        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 566.elseif \i == 7
 567        pxor       %xmm7, %xmm8
 568        GHASH_MUL  %xmm8, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1
 569.endif
 570        cmp        $64, %r13
 571        jl      _initial_blocks_done\num_initial_blocks\operation
 572        # no need for precomputed values
 573/*
 574*
 575* Precomputations for HashKey parallel with encryption of first 4 blocks.
 576* Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
 577*/
 578        paddd      ONE(%rip), \XMM0              # INCR Y0
 579        movdqa     \XMM0, \XMM1
 580        movdqa     SHUF_MASK(%rip), %xmm14
 581        PSHUFB_XMM  %xmm14, \XMM1        # perform a 16 byte swap
 582
 583        paddd      ONE(%rip), \XMM0              # INCR Y0
 584        movdqa     \XMM0, \XMM2
 585        movdqa     SHUF_MASK(%rip), %xmm14
 586        PSHUFB_XMM  %xmm14, \XMM2        # perform a 16 byte swap
 587
 588        paddd      ONE(%rip), \XMM0              # INCR Y0
 589        movdqa     \XMM0, \XMM3
 590        movdqa     SHUF_MASK(%rip), %xmm14
 591        PSHUFB_XMM %xmm14, \XMM3        # perform a 16 byte swap
 592
 593        paddd      ONE(%rip), \XMM0              # INCR Y0
 594        movdqa     \XMM0, \XMM4
 595        movdqa     SHUF_MASK(%rip), %xmm14
 596        PSHUFB_XMM %xmm14, \XMM4        # perform a 16 byte swap
 597
 598        pxor       16*0(%arg1), \XMM1
 599        pxor       16*0(%arg1), \XMM2
 600        pxor       16*0(%arg1), \XMM3
 601        pxor       16*0(%arg1), \XMM4
 602        movdqa     \TMP3, \TMP5
 603        pshufd     $78, \TMP3, \TMP1
 604        pxor       \TMP3, \TMP1
 605        movdqa     \TMP1, HashKey_k(%rsp)
 606        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
 607# TMP5 = HashKey^2<<1 (mod poly)
 608        movdqa     \TMP5, HashKey_2(%rsp)
 609# HashKey_2 = HashKey^2<<1 (mod poly)
 610        pshufd     $78, \TMP5, \TMP1
 611        pxor       \TMP5, \TMP1
 612        movdqa     \TMP1, HashKey_2_k(%rsp)
 613.irpc index, 1234 # do 4 rounds
 614        movaps 0x10*\index(%arg1), \TMP1
 615        AESENC     \TMP1, \XMM1
 616        AESENC     \TMP1, \XMM2
 617        AESENC     \TMP1, \XMM3
 618        AESENC     \TMP1, \XMM4
 619.endr
 620        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
 621# TMP5 = HashKey^3<<1 (mod poly)
 622        movdqa     \TMP5, HashKey_3(%rsp)
 623        pshufd     $78, \TMP5, \TMP1
 624        pxor       \TMP5, \TMP1
 625        movdqa     \TMP1, HashKey_3_k(%rsp)
 626.irpc index, 56789 # do next 5 rounds
 627        movaps 0x10*\index(%arg1), \TMP1
 628        AESENC     \TMP1, \XMM1
 629        AESENC     \TMP1, \XMM2
 630        AESENC     \TMP1, \XMM3
 631        AESENC     \TMP1, \XMM4
 632.endr
 633        GHASH_MUL  \TMP5, \TMP3, \TMP1, \TMP2, \TMP4, \TMP6, \TMP7
 634# TMP5 = HashKey^3<<1 (mod poly)
 635        movdqa     \TMP5, HashKey_4(%rsp)
 636        pshufd     $78, \TMP5, \TMP1
 637        pxor       \TMP5, \TMP1
 638        movdqa     \TMP1, HashKey_4_k(%rsp)
 639        movaps 0xa0(%arg1), \TMP2
 640        AESENCLAST \TMP2, \XMM1
 641        AESENCLAST \TMP2, \XMM2
 642        AESENCLAST \TMP2, \XMM3
 643        AESENCLAST \TMP2, \XMM4
 644        movdqu     16*0(%arg3 , %r11 , 1), \TMP1
 645        pxor       \TMP1, \XMM1
 646        movdqu     16*1(%arg3 , %r11 , 1), \TMP1
 647        pxor       \TMP1, \XMM2
 648        movdqu     16*2(%arg3 , %r11 , 1), \TMP1
 649        pxor       \TMP1, \XMM3
 650        movdqu     16*3(%arg3 , %r11 , 1), \TMP1
 651        pxor       \TMP1, \XMM4
 652        movdqu     \XMM1, 16*0(%arg2 , %r11 , 1)
 653        movdqu     \XMM2, 16*1(%arg2 , %r11 , 1)
 654        movdqu     \XMM3, 16*2(%arg2 , %r11 , 1)
 655        movdqu     \XMM4, 16*3(%arg2 , %r11 , 1)
 656
 657        add        $64, %r11
 658        movdqa     SHUF_MASK(%rip), %xmm14
 659        PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap
 660        pxor       \XMMDst, \XMM1
 661# combine GHASHed value with the corresponding ciphertext
 662        movdqa     SHUF_MASK(%rip), %xmm14
 663        PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap
 664        movdqa     SHUF_MASK(%rip), %xmm14
 665        PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap
 666        movdqa     SHUF_MASK(%rip), %xmm14
 667        PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap
 668
 669_initial_blocks_done\num_initial_blocks\operation:
 670
 671.endm
 672
 673/*
 674* encrypt 4 blocks at a time
 675* ghash the 4 previously encrypted ciphertext blocks
 676* arg1, %arg2, %arg3 are used as pointers only, not modified
 677* %r11 is the data offset value
 678*/
 679.macro GHASH_4_ENCRYPT_4_PARALLEL_ENC TMP1 TMP2 TMP3 TMP4 TMP5 \
 680TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
 681
 682        movdqa    \XMM1, \XMM5
 683        movdqa    \XMM2, \XMM6
 684        movdqa    \XMM3, \XMM7
 685        movdqa    \XMM4, \XMM8
 686
 687        movdqa    SHUF_MASK(%rip), %xmm15
 688        # multiply TMP5 * HashKey using karatsuba
 689
 690        movdqa    \XMM5, \TMP4
 691        pshufd    $78, \XMM5, \TMP6
 692        pxor      \XMM5, \TMP6
 693        paddd     ONE(%rip), \XMM0              # INCR CNT
 694        movdqa    HashKey_4(%rsp), \TMP5
 695        PCLMULQDQ 0x11, \TMP5, \TMP4           # TMP4 = a1*b1
 696        movdqa    \XMM0, \XMM1
 697        paddd     ONE(%rip), \XMM0              # INCR CNT
 698        movdqa    \XMM0, \XMM2
 699        paddd     ONE(%rip), \XMM0              # INCR CNT
 700        movdqa    \XMM0, \XMM3
 701        paddd     ONE(%rip), \XMM0              # INCR CNT
 702        movdqa    \XMM0, \XMM4
 703        PSHUFB_XMM %xmm15, \XMM1        # perform a 16 byte swap
 704        PCLMULQDQ 0x00, \TMP5, \XMM5           # XMM5 = a0*b0
 705        PSHUFB_XMM %xmm15, \XMM2        # perform a 16 byte swap
 706        PSHUFB_XMM %xmm15, \XMM3        # perform a 16 byte swap
 707        PSHUFB_XMM %xmm15, \XMM4        # perform a 16 byte swap
 708
 709        pxor      (%arg1), \XMM1
 710        pxor      (%arg1), \XMM2
 711        pxor      (%arg1), \XMM3
 712        pxor      (%arg1), \XMM4
 713        movdqa    HashKey_4_k(%rsp), \TMP5
 714        PCLMULQDQ 0x00, \TMP5, \TMP6           # TMP6 = (a1+a0)*(b1+b0)
 715        movaps 0x10(%arg1), \TMP1
 716        AESENC    \TMP1, \XMM1              # Round 1
 717        AESENC    \TMP1, \XMM2
 718        AESENC    \TMP1, \XMM3
 719        AESENC    \TMP1, \XMM4
 720        movaps 0x20(%arg1), \TMP1
 721        AESENC    \TMP1, \XMM1              # Round 2
 722        AESENC    \TMP1, \XMM2
 723        AESENC    \TMP1, \XMM3
 724        AESENC    \TMP1, \XMM4
 725        movdqa    \XMM6, \TMP1
 726        pshufd    $78, \XMM6, \TMP2
 727        pxor      \XMM6, \TMP2
 728        movdqa    HashKey_3(%rsp), \TMP5
 729        PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1 * b1
 730        movaps 0x30(%arg1), \TMP3
 731        AESENC    \TMP3, \XMM1              # Round 3
 732        AESENC    \TMP3, \XMM2
 733        AESENC    \TMP3, \XMM3
 734        AESENC    \TMP3, \XMM4
 735        PCLMULQDQ 0x00, \TMP5, \XMM6           # XMM6 = a0*b0
 736        movaps 0x40(%arg1), \TMP3
 737        AESENC    \TMP3, \XMM1              # Round 4
 738        AESENC    \TMP3, \XMM2
 739        AESENC    \TMP3, \XMM3
 740        AESENC    \TMP3, \XMM4
 741        movdqa    HashKey_3_k(%rsp), \TMP5
 742        PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)
 743        movaps 0x50(%arg1), \TMP3
 744        AESENC    \TMP3, \XMM1              # Round 5
 745        AESENC    \TMP3, \XMM2
 746        AESENC    \TMP3, \XMM3
 747        AESENC    \TMP3, \XMM4
 748        pxor      \TMP1, \TMP4
 749# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
 750        pxor      \XMM6, \XMM5
 751        pxor      \TMP2, \TMP6
 752        movdqa    \XMM7, \TMP1
 753        pshufd    $78, \XMM7, \TMP2
 754        pxor      \XMM7, \TMP2
 755        movdqa    HashKey_2(%rsp ), \TMP5
 756
 757        # Multiply TMP5 * HashKey using karatsuba
 758
 759        PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1*b1
 760        movaps 0x60(%arg1), \TMP3
 761        AESENC    \TMP3, \XMM1              # Round 6
 762        AESENC    \TMP3, \XMM2
 763        AESENC    \TMP3, \XMM3
 764        AESENC    \TMP3, \XMM4
 765        PCLMULQDQ 0x00, \TMP5, \XMM7           # XMM7 = a0*b0
 766        movaps 0x70(%arg1), \TMP3
 767        AESENC    \TMP3, \XMM1             # Round 7
 768        AESENC    \TMP3, \XMM2
 769        AESENC    \TMP3, \XMM3
 770        AESENC    \TMP3, \XMM4
 771        movdqa    HashKey_2_k(%rsp), \TMP5
 772        PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)
 773        movaps 0x80(%arg1), \TMP3
 774        AESENC    \TMP3, \XMM1             # Round 8
 775        AESENC    \TMP3, \XMM2
 776        AESENC    \TMP3, \XMM3
 777        AESENC    \TMP3, \XMM4
 778        pxor      \TMP1, \TMP4
 779# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
 780        pxor      \XMM7, \XMM5
 781        pxor      \TMP2, \TMP6
 782
 783        # Multiply XMM8 * HashKey
 784        # XMM8 and TMP5 hold the values for the two operands
 785
 786        movdqa    \XMM8, \TMP1
 787        pshufd    $78, \XMM8, \TMP2
 788        pxor      \XMM8, \TMP2
 789        movdqa    HashKey(%rsp), \TMP5
 790        PCLMULQDQ 0x11, \TMP5, \TMP1          # TMP1 = a1*b1
 791        movaps 0x90(%arg1), \TMP3
 792        AESENC    \TMP3, \XMM1            # Round 9
 793        AESENC    \TMP3, \XMM2
 794        AESENC    \TMP3, \XMM3
 795        AESENC    \TMP3, \XMM4
 796        PCLMULQDQ 0x00, \TMP5, \XMM8          # XMM8 = a0*b0
 797        movaps 0xa0(%arg1), \TMP3
 798        AESENCLAST \TMP3, \XMM1           # Round 10
 799        AESENCLAST \TMP3, \XMM2
 800        AESENCLAST \TMP3, \XMM3
 801        AESENCLAST \TMP3, \XMM4
 802        movdqa    HashKey_k(%rsp), \TMP5
 803        PCLMULQDQ 0x00, \TMP5, \TMP2          # TMP2 = (a1+a0)*(b1+b0)
 804        movdqu    (%arg3,%r11,1), \TMP3
 805        pxor      \TMP3, \XMM1                 # Ciphertext/Plaintext XOR EK
 806        movdqu    16(%arg3,%r11,1), \TMP3
 807        pxor      \TMP3, \XMM2                 # Ciphertext/Plaintext XOR EK
 808        movdqu    32(%arg3,%r11,1), \TMP3
 809        pxor      \TMP3, \XMM3                 # Ciphertext/Plaintext XOR EK
 810        movdqu    48(%arg3,%r11,1), \TMP3
 811        pxor      \TMP3, \XMM4                 # Ciphertext/Plaintext XOR EK
 812        movdqu    \XMM1, (%arg2,%r11,1)        # Write to the ciphertext buffer
 813        movdqu    \XMM2, 16(%arg2,%r11,1)      # Write to the ciphertext buffer
 814        movdqu    \XMM3, 32(%arg2,%r11,1)      # Write to the ciphertext buffer
 815        movdqu    \XMM4, 48(%arg2,%r11,1)      # Write to the ciphertext buffer
 816        PSHUFB_XMM %xmm15, \XMM1        # perform a 16 byte swap
 817        PSHUFB_XMM %xmm15, \XMM2        # perform a 16 byte swap
 818        PSHUFB_XMM %xmm15, \XMM3        # perform a 16 byte swap
 819        PSHUFB_XMM %xmm15, \XMM4        # perform a 16 byte swap
 820
 821        pxor      \TMP4, \TMP1
 822        pxor      \XMM8, \XMM5
 823        pxor      \TMP6, \TMP2
 824        pxor      \TMP1, \TMP2
 825        pxor      \XMM5, \TMP2
 826        movdqa    \TMP2, \TMP3
 827        pslldq    $8, \TMP3                    # left shift TMP3 2 DWs
 828        psrldq    $8, \TMP2                    # right shift TMP2 2 DWs
 829        pxor      \TMP3, \XMM5
 830        pxor      \TMP2, \TMP1    # accumulate the results in TMP1:XMM5
 831
 832        # first phase of reduction
 833
 834        movdqa    \XMM5, \TMP2
 835        movdqa    \XMM5, \TMP3
 836        movdqa    \XMM5, \TMP4
 837# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently
 838        pslld     $31, \TMP2                   # packed right shift << 31
 839        pslld     $30, \TMP3                   # packed right shift << 30
 840        pslld     $25, \TMP4                   # packed right shift << 25
 841        pxor      \TMP3, \TMP2                 # xor the shifted versions
 842        pxor      \TMP4, \TMP2
 843        movdqa    \TMP2, \TMP5
 844        psrldq    $4, \TMP5                    # right shift T5 1 DW
 845        pslldq    $12, \TMP2                   # left shift T2 3 DWs
 846        pxor      \TMP2, \XMM5
 847
 848        # second phase of reduction
 849
 850        movdqa    \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4
 851        movdqa    \XMM5,\TMP3
 852        movdqa    \XMM5,\TMP4
 853        psrld     $1, \TMP2                    # packed left shift >>1
 854        psrld     $2, \TMP3                    # packed left shift >>2
 855        psrld     $7, \TMP4                    # packed left shift >>7
 856        pxor      \TMP3,\TMP2                  # xor the shifted versions
 857        pxor      \TMP4,\TMP2
 858        pxor      \TMP5, \TMP2
 859        pxor      \TMP2, \XMM5
 860        pxor      \TMP1, \XMM5                 # result is in TMP1
 861
 862        pxor      \XMM5, \XMM1
 863.endm
 864
 865/*
 866* decrypt 4 blocks at a time
 867* ghash the 4 previously decrypted ciphertext blocks
 868* arg1, %arg2, %arg3 are used as pointers only, not modified
 869* %r11 is the data offset value
 870*/
 871.macro GHASH_4_ENCRYPT_4_PARALLEL_DEC TMP1 TMP2 TMP3 TMP4 TMP5 \
 872TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation
 873
 874        movdqa    \XMM1, \XMM5
 875        movdqa    \XMM2, \XMM6
 876        movdqa    \XMM3, \XMM7
 877        movdqa    \XMM4, \XMM8
 878
 879        movdqa    SHUF_MASK(%rip), %xmm15
 880        # multiply TMP5 * HashKey using karatsuba
 881
 882        movdqa    \XMM5, \TMP4
 883        pshufd    $78, \XMM5, \TMP6
 884        pxor      \XMM5, \TMP6
 885        paddd     ONE(%rip), \XMM0              # INCR CNT
 886        movdqa    HashKey_4(%rsp), \TMP5
 887        PCLMULQDQ 0x11, \TMP5, \TMP4           # TMP4 = a1*b1
 888        movdqa    \XMM0, \XMM1
 889        paddd     ONE(%rip), \XMM0              # INCR CNT
 890        movdqa    \XMM0, \XMM2
 891        paddd     ONE(%rip), \XMM0              # INCR CNT
 892        movdqa    \XMM0, \XMM3
 893        paddd     ONE(%rip), \XMM0              # INCR CNT
 894        movdqa    \XMM0, \XMM4
 895        PSHUFB_XMM %xmm15, \XMM1        # perform a 16 byte swap
 896        PCLMULQDQ 0x00, \TMP5, \XMM5           # XMM5 = a0*b0
 897        PSHUFB_XMM %xmm15, \XMM2        # perform a 16 byte swap
 898        PSHUFB_XMM %xmm15, \XMM3        # perform a 16 byte swap
 899        PSHUFB_XMM %xmm15, \XMM4        # perform a 16 byte swap
 900
 901        pxor      (%arg1), \XMM1
 902        pxor      (%arg1), \XMM2
 903        pxor      (%arg1), \XMM3
 904        pxor      (%arg1), \XMM4
 905        movdqa    HashKey_4_k(%rsp), \TMP5
 906        PCLMULQDQ 0x00, \TMP5, \TMP6           # TMP6 = (a1+a0)*(b1+b0)
 907        movaps 0x10(%arg1), \TMP1
 908        AESENC    \TMP1, \XMM1              # Round 1
 909        AESENC    \TMP1, \XMM2
 910        AESENC    \TMP1, \XMM3
 911        AESENC    \TMP1, \XMM4
 912        movaps 0x20(%arg1), \TMP1
 913        AESENC    \TMP1, \XMM1              # Round 2
 914        AESENC    \TMP1, \XMM2
 915        AESENC    \TMP1, \XMM3
 916        AESENC    \TMP1, \XMM4
 917        movdqa    \XMM6, \TMP1
 918        pshufd    $78, \XMM6, \TMP2
 919        pxor      \XMM6, \TMP2
 920        movdqa    HashKey_3(%rsp), \TMP5
 921        PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1 * b1
 922        movaps 0x30(%arg1), \TMP3
 923        AESENC    \TMP3, \XMM1              # Round 3
 924        AESENC    \TMP3, \XMM2
 925        AESENC    \TMP3, \XMM3
 926        AESENC    \TMP3, \XMM4
 927        PCLMULQDQ 0x00, \TMP5, \XMM6           # XMM6 = a0*b0
 928        movaps 0x40(%arg1), \TMP3
 929        AESENC    \TMP3, \XMM1              # Round 4
 930        AESENC    \TMP3, \XMM2
 931        AESENC    \TMP3, \XMM3
 932        AESENC    \TMP3, \XMM4
 933        movdqa    HashKey_3_k(%rsp), \TMP5
 934        PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)
 935        movaps 0x50(%arg1), \TMP3
 936        AESENC    \TMP3, \XMM1              # Round 5
 937        AESENC    \TMP3, \XMM2
 938        AESENC    \TMP3, \XMM3
 939        AESENC    \TMP3, \XMM4
 940        pxor      \TMP1, \TMP4
 941# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
 942        pxor      \XMM6, \XMM5
 943        pxor      \TMP2, \TMP6
 944        movdqa    \XMM7, \TMP1
 945        pshufd    $78, \XMM7, \TMP2
 946        pxor      \XMM7, \TMP2
 947        movdqa    HashKey_2(%rsp ), \TMP5
 948
 949        # Multiply TMP5 * HashKey using karatsuba
 950
 951        PCLMULQDQ 0x11, \TMP5, \TMP1           # TMP1 = a1*b1
 952        movaps 0x60(%arg1), \TMP3
 953        AESENC    \TMP3, \XMM1              # Round 6
 954        AESENC    \TMP3, \XMM2
 955        AESENC    \TMP3, \XMM3
 956        AESENC    \TMP3, \XMM4
 957        PCLMULQDQ 0x00, \TMP5, \XMM7           # XMM7 = a0*b0
 958        movaps 0x70(%arg1), \TMP3
 959        AESENC    \TMP3, \XMM1             # Round 7
 960        AESENC    \TMP3, \XMM2
 961        AESENC    \TMP3, \XMM3
 962        AESENC    \TMP3, \XMM4
 963        movdqa    HashKey_2_k(%rsp), \TMP5
 964        PCLMULQDQ 0x00, \TMP5, \TMP2           # TMP2 = (a1+a0)*(b1+b0)
 965        movaps 0x80(%arg1), \TMP3
 966        AESENC    \TMP3, \XMM1             # Round 8
 967        AESENC    \TMP3, \XMM2
 968        AESENC    \TMP3, \XMM3
 969        AESENC    \TMP3, \XMM4
 970        pxor      \TMP1, \TMP4
 971# accumulate the results in TMP4:XMM5, TMP6 holds the middle part
 972        pxor      \XMM7, \XMM5
 973        pxor      \TMP2, \TMP6
 974
 975        # Multiply XMM8 * HashKey
 976        # XMM8 and TMP5 hold the values for the two operands
 977
 978        movdqa    \XMM8, \TMP1
 979        pshufd    $78, \XMM8, \TMP2
 980        pxor      \XMM8, \TMP2
 981        movdqa    HashKey(%rsp), \TMP5
 982        PCLMULQDQ 0x11, \TMP5, \TMP1          # TMP1 = a1*b1
 983        movaps 0x90(%arg1), \TMP3
 984        AESENC    \TMP3, \XMM1            # Round 9
 985        AESENC    \TMP3, \XMM2
 986        AESENC    \TMP3, \XMM3
 987        AESENC    \TMP3, \XMM4
 988        PCLMULQDQ 0x00, \TMP5, \XMM8          # XMM8 = a0*b0
 989        movaps 0xa0(%arg1), \TMP3
 990        AESENCLAST \TMP3, \XMM1           # Round 10
 991        AESENCLAST \TMP3, \XMM2
 992        AESENCLAST \TMP3, \XMM3
 993        AESENCLAST \TMP3, \XMM4
 994        movdqa    HashKey_k(%rsp), \TMP5
 995        PCLMULQDQ 0x00, \TMP5, \TMP2          # TMP2 = (a1+a0)*(b1+b0)
 996        movdqu    (%arg3,%r11,1), \TMP3
 997        pxor      \TMP3, \XMM1                 # Ciphertext/Plaintext XOR EK
 998        movdqu    \XMM1, (%arg2,%r11,1)        # Write to plaintext buffer
 999        movdqa    \TMP3, \XMM1
1000        movdqu    16(%arg3,%r11,1), \TMP3

1001        pxor      \TMP3, \XMM2                 # Ciphertext/Plaintext XOR EK
1002        movdqu    \XMM2, 16(%arg2,%r11,1)      # Write to plaintext buffer
1003        movdqa    \TMP3, \XMM2
1004        movdqu    32(%arg3,%r11,1), \TMP3
1005        pxor      \TMP3, \XMM3                 # Ciphertext/Plaintext XOR EK
1006        movdqu    \XMM3, 32(%arg2,%r11,1)      # Write to plaintext buffer
1007        movdqa    \TMP3, \XMM3
1008        movdqu    48(%arg3,%r11,1), \TMP3
1009        pxor      \TMP3, \XMM4                 # Ciphertext/Plaintext XOR EK
1010        movdqu    \XMM4, 48(%arg2,%r11,1)      # Write to plaintext buffer
1011        movdqa    \TMP3, \XMM4
1012        PSHUFB_XMM %xmm15, \XMM1        # perform a 16 byte swap
1013        PSHUFB_XMM %xmm15, \XMM2        # perform a 16 byte swap
1014        PSHUFB_XMM %xmm15, \XMM3        # perform a 16 byte swap
1015        PSHUFB_XMM %xmm15, \XMM4        # perform a 16 byte swap
1016
1017        pxor      \TMP4, \TMP1
1018        pxor      \XMM8, \XMM5
1019        pxor      \TMP6, \TMP2
1020        pxor      \TMP1, \TMP2
1021        pxor      \XMM5, \TMP2
1022        movdqa    \TMP2, \TMP3
1023        pslldq    $8, \TMP3                    # left shift TMP3 2 DWs
1024        psrldq    $8, \TMP2                    # right shift TMP2 2 DWs
1025        pxor      \TMP3, \XMM5
1026        pxor      \TMP2, \TMP1    # accumulate the results in TMP1:XMM5
1027
1028        # first phase of reduction
1029
1030        movdqa    \XMM5, \TMP2
1031        movdqa    \XMM5, \TMP3
1032        movdqa    \XMM5, \TMP4
1033# move XMM5 into TMP2, TMP3, TMP4 in order to perform shifts independently
1034        pslld     $31, \TMP2                   # packed right shift << 31
1035        pslld     $30, \TMP3                   # packed right shift << 30
1036        pslld     $25, \TMP4                   # packed right shift << 25
1037        pxor      \TMP3, \TMP2                 # xor the shifted versions
1038        pxor      \TMP4, \TMP2
1039        movdqa    \TMP2, \TMP5
1040        psrldq    $4, \TMP5                    # right shift T5 1 DW
1041        pslldq    $12, \TMP2                   # left shift T2 3 DWs
1042        pxor      \TMP2, \XMM5
1043
1044        # second phase of reduction
1045
1046        movdqa    \XMM5,\TMP2 # make 3 copies of XMM5 into TMP2, TMP3, TMP4
1047        movdqa    \XMM5,\TMP3
1048        movdqa    \XMM5,\TMP4
1049        psrld     $1, \TMP2                    # packed left shift >>1
1050        psrld     $2, \TMP3                    # packed left shift >>2
1051        psrld     $7, \TMP4                    # packed left shift >>7
1052        pxor      \TMP3,\TMP2                  # xor the shifted versions
1053        pxor      \TMP4,\TMP2
1054        pxor      \TMP5, \TMP2
1055        pxor      \TMP2, \XMM5
1056        pxor      \TMP1, \XMM5                 # result is in TMP1
1057
1058        pxor      \XMM5, \XMM1
1059.endm
1060
1061/* GHASH the last 4 ciphertext blocks. */
1062.macro  GHASH_LAST_4 TMP1 TMP2 TMP3 TMP4 TMP5 TMP6 \
1063TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst
1064
1065        # Multiply TMP6 * HashKey (using Karatsuba)
1066
1067        movdqa    \XMM1, \TMP6
1068        pshufd    $78, \XMM1, \TMP2
1069        pxor      \XMM1, \TMP2
1070        movdqa    HashKey_4(%rsp), \TMP5
1071        PCLMULQDQ 0x11, \TMP5, \TMP6       # TMP6 = a1*b1
1072        PCLMULQDQ 0x00, \TMP5, \XMM1       # XMM1 = a0*b0
1073        movdqa    HashKey_4_k(%rsp), \TMP4
1074        PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)
1075        movdqa    \XMM1, \XMMDst
1076        movdqa    \TMP2, \XMM1              # result in TMP6, XMMDst, XMM1
1077
1078        # Multiply TMP1 * HashKey (using Karatsuba)
1079
1080        movdqa    \XMM2, \TMP1
1081        pshufd    $78, \XMM2, \TMP2
1082        pxor      \XMM2, \TMP2
1083        movdqa    HashKey_3(%rsp), \TMP5
1084        PCLMULQDQ 0x11, \TMP5, \TMP1       # TMP1 = a1*b1
1085        PCLMULQDQ 0x00, \TMP5, \XMM2       # XMM2 = a0*b0
1086        movdqa    HashKey_3_k(%rsp), \TMP4
1087        PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)
1088        pxor      \TMP1, \TMP6
1089        pxor      \XMM2, \XMMDst
1090        pxor      \TMP2, \XMM1
1091# results accumulated in TMP6, XMMDst, XMM1
1092
1093        # Multiply TMP1 * HashKey (using Karatsuba)
1094
1095        movdqa    \XMM3, \TMP1
1096        pshufd    $78, \XMM3, \TMP2
1097        pxor      \XMM3, \TMP2
1098        movdqa    HashKey_2(%rsp), \TMP5
1099        PCLMULQDQ 0x11, \TMP5, \TMP1       # TMP1 = a1*b1
1100        PCLMULQDQ 0x00, \TMP5, \XMM3       # XMM3 = a0*b0
1101        movdqa    HashKey_2_k(%rsp), \TMP4
1102        PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)
1103        pxor      \TMP1, \TMP6
1104        pxor      \XMM3, \XMMDst
1105        pxor      \TMP2, \XMM1   # results accumulated in TMP6, XMMDst, XMM1
1106
1107        # Multiply TMP1 * HashKey (using Karatsuba)
1108        movdqa    \XMM4, \TMP1
1109        pshufd    $78, \XMM4, \TMP2
1110        pxor      \XMM4, \TMP2
1111        movdqa    HashKey(%rsp), \TMP5
1112        PCLMULQDQ 0x11, \TMP5, \TMP1        # TMP1 = a1*b1
1113        PCLMULQDQ 0x00, \TMP5, \XMM4       # XMM4 = a0*b0
1114        movdqa    HashKey_k(%rsp), \TMP4
1115        PCLMULQDQ 0x00, \TMP4, \TMP2       # TMP2 = (a1+a0)*(b1+b0)
1116        pxor      \TMP1, \TMP6
1117        pxor      \XMM4, \XMMDst
1118        pxor      \XMM1, \TMP2
1119        pxor      \TMP6, \TMP2
1120        pxor      \XMMDst, \TMP2
1121        # middle section of the temp results combined as in karatsuba algorithm
1122        movdqa    \TMP2, \TMP4
1123        pslldq    $8, \TMP4                 # left shift TMP4 2 DWs
1124        psrldq    $8, \TMP2                 # right shift TMP2 2 DWs
1125        pxor      \TMP4, \XMMDst
1126        pxor      \TMP2, \TMP6
1127# TMP6:XMMDst holds the result of the accumulated carry-less multiplications
1128        # first phase of the reduction
1129        movdqa    \XMMDst, \TMP2
1130        movdqa    \XMMDst, \TMP3
1131        movdqa    \XMMDst, \TMP4
1132# move XMMDst into TMP2, TMP3, TMP4 in order to perform 3 shifts independently
1133        pslld     $31, \TMP2                # packed right shifting << 31
1134        pslld     $30, \TMP3                # packed right shifting << 30
1135        pslld     $25, \TMP4                # packed right shifting << 25
1136        pxor      \TMP3, \TMP2              # xor the shifted versions
1137        pxor      \TMP4, \TMP2
1138        movdqa    \TMP2, \TMP7
1139        psrldq    $4, \TMP7                 # right shift TMP7 1 DW
1140        pslldq    $12, \TMP2                # left shift TMP2 3 DWs
1141        pxor      \TMP2, \XMMDst
1142
1143        # second phase of the reduction
1144        movdqa    \XMMDst, \TMP2
1145        # make 3 copies of XMMDst for doing 3 shift operations
1146        movdqa    \XMMDst, \TMP3
1147        movdqa    \XMMDst, \TMP4
1148        psrld     $1, \TMP2                 # packed left shift >> 1
1149        psrld     $2, \TMP3                 # packed left shift >> 2
1150        psrld     $7, \TMP4                 # packed left shift >> 7
1151        pxor      \TMP3, \TMP2              # xor the shifted versions
1152        pxor      \TMP4, \TMP2
1153        pxor      \TMP7, \TMP2
1154        pxor      \TMP2, \XMMDst
1155        pxor      \TMP6, \XMMDst            # reduced result is in XMMDst
1156.endm
1157
1158/* Encryption of a single block done*/
1159.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1
1160
1161        pxor    (%arg1), \XMM0
1162        movaps 16(%arg1), \TMP1
1163        AESENC  \TMP1, \XMM0
1164        movaps 32(%arg1), \TMP1
1165        AESENC  \TMP1, \XMM0
1166        movaps 48(%arg1), \TMP1
1167        AESENC  \TMP1, \XMM0
1168        movaps 64(%arg1), \TMP1
1169        AESENC  \TMP1, \XMM0
1170        movaps 80(%arg1), \TMP1
1171        AESENC  \TMP1, \XMM0
1172        movaps 96(%arg1), \TMP1
1173        AESENC  \TMP1, \XMM0
1174        movaps 112(%arg1), \TMP1
1175        AESENC  \TMP1, \XMM0
1176        movaps 128(%arg1), \TMP1
1177        AESENC  \TMP1, \XMM0
1178        movaps 144(%arg1), \TMP1
1179        AESENC  \TMP1, \XMM0
1180        movaps 160(%arg1), \TMP1
1181        AESENCLAST      \TMP1, \XMM0
1182.endm
1183
1184
1185/*****************************************************************************
1186* void aesni_gcm_dec(void *aes_ctx,    // AES Key schedule. Starts on a 16 byte boundary.
1187*                   u8 *out,           // Plaintext output. Encrypt in-place is allowed.
1188*                   const u8 *in,      // Ciphertext input
1189*                   u64 plaintext_len, // Length of data in bytes for decryption.
1190*                   u8 *iv,            // Pre-counter block j0: 4 byte salt (from Security Association)
1191*                                      // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload)
1192*                                      // concatenated with 0x00000001. 16-byte aligned pointer.
1193*                   u8 *hash_subkey,   // H, the Hash sub key input. Data starts on a 16-byte boundary.
1194*                   const u8 *aad,     // Additional Authentication Data (AAD)
1195*                   u64 aad_len,       // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes
1196*                   u8  *auth_tag,     // Authenticated Tag output. The driver will compare this to the
1197*                                      // given authentication tag and only return the plaintext if they match.
1198*                   u64 auth_tag_len); // Authenticated Tag Length in bytes. Valid values are 16
1199*                                      // (most likely), 12 or 8.
1200*
1201* Assumptions:
1202*
1203* keys:
1204*       keys are pre-expanded and aligned to 16 bytes. we are using the first
1205*       set of 11 keys in the data structure void *aes_ctx
1206*
1207* iv:
1208*       0                   1                   2                   3
1209*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1210*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1211*       |                             Salt  (From the SA)               |
1212*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1213*       |                     Initialization Vector                     |
1214*       |         (This is the sequence number from IPSec header)       |
1215*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1216*       |                              0x1                              |
1217*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1218*
1219*
1220*
1221* AAD:
1222*       AAD padded to 128 bits with 0
1223*       for example, assume AAD is a u32 vector
1224*
1225*       if AAD is 8 bytes:
1226*       AAD[3] = {A0, A1};
1227*       padded AAD in xmm register = {A1 A0 0 0}
1228*
1229*       0                   1                   2                   3
1230*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1231*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1232*       |                               SPI (A1)                        |
1233*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1234*       |                     32-bit Sequence Number (A0)               |
1235*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1236*       |                              0x0                              |
1237*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1238*
1239*                                       AAD Format with 32-bit Sequence Number
1240*
1241*       if AAD is 12 bytes:
1242*       AAD[3] = {A0, A1, A2};
1243*       padded AAD in xmm register = {A2 A1 A0 0}
1244*
1245*       0                   1                   2                   3
1246*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1247*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1248*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1249*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1250*       |                               SPI (A2)                        |
1251*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1252*       |                 64-bit Extended Sequence Number {A1,A0}       |
1253*       |                                                               |
1254*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1255*       |                              0x0                              |
1256*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1257*
1258*                        AAD Format with 64-bit Extended Sequence Number
1259*
1260* aadLen:
1261*       from the definition of the spec, aadLen can only be 8 or 12 bytes.
1262*       The code supports 16 too but for other sizes, the code will fail.
1263*
1264* TLen:
1265*       from the definition of the spec, TLen can only be 8, 12 or 16 bytes.
1266*       For other sizes, the code will fail.
1267*
1268* poly = x^128 + x^127 + x^126 + x^121 + 1
1269*
1270*****************************************************************************/
1271ENTRY(aesni_gcm_dec)
1272        push    %r12
1273        push    %r13
1274        push    %r14
1275        mov     %rsp, %r14
1276/*
1277* states of %xmm registers %xmm6:%xmm15 not saved
1278* all %xmm registers are clobbered
1279*/
1280        sub     $VARIABLE_OFFSET, %rsp
1281        and     $~63, %rsp                        # align rsp to 64 bytes
1282        mov     %arg6, %r12
1283        movdqu  (%r12), %xmm13                    # %xmm13 = HashKey
1284        movdqa  SHUF_MASK(%rip), %xmm2
1285        PSHUFB_XMM %xmm2, %xmm13
1286
1287
1288# Precompute HashKey<<1 (mod poly) from the hash key (required for GHASH)
1289
1290        movdqa  %xmm13, %xmm2
1291        psllq   $1, %xmm13
1292        psrlq   $63, %xmm2
1293        movdqa  %xmm2, %xmm1
1294        pslldq  $8, %xmm2
1295        psrldq  $8, %xmm1
1296        por     %xmm2, %xmm13
1297
1298        # Reduction
1299
1300        pshufd  $0x24, %xmm1, %xmm2
1301        pcmpeqd TWOONE(%rip), %xmm2
1302        pand    POLY(%rip), %xmm2
1303        pxor    %xmm2, %xmm13     # %xmm13 holds the HashKey<<1 (mod poly)
1304
1305
1306        # Decrypt first few blocks
1307
1308        movdqa %xmm13, HashKey(%rsp)           # store HashKey<<1 (mod poly)
1309        mov %arg4, %r13    # save the number of bytes of plaintext/ciphertext
1310        and $-16, %r13                      # %r13 = %r13 - (%r13 mod 16)
1311        mov %r13, %r12
1312        and $(3<<4), %r12
1313        jz _initial_num_blocks_is_0_decrypt
1314        cmp $(2<<4), %r12
1315        jb _initial_num_blocks_is_1_decrypt
1316        je _initial_num_blocks_is_2_decrypt
1317_initial_num_blocks_is_3_decrypt:
1318        INITIAL_BLOCKS_DEC 3, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1319%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, dec
1320        sub     $48, %r13
1321        jmp     _initial_blocks_decrypted
1322_initial_num_blocks_is_2_decrypt:
1323        INITIAL_BLOCKS_DEC      2, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1324%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, dec
1325        sub     $32, %r13
1326        jmp     _initial_blocks_decrypted
1327_initial_num_blocks_is_1_decrypt:
1328        INITIAL_BLOCKS_DEC      1, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1329%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, dec
1330        sub     $16, %r13
1331        jmp     _initial_blocks_decrypted
1332_initial_num_blocks_is_0_decrypt:
1333        INITIAL_BLOCKS_DEC      0, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1334%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, dec
1335_initial_blocks_decrypted:
1336        cmp     $0, %r13
1337        je      _zero_cipher_left_decrypt
1338        sub     $64, %r13
1339        je      _four_cipher_left_decrypt
1340_decrypt_by_4:
1341        GHASH_4_ENCRYPT_4_PARALLEL_DEC  %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \
1342%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, dec
1343        add     $64, %r11
1344        sub     $64, %r13
1345        jne     _decrypt_by_4
1346_four_cipher_left_decrypt:
1347        GHASH_LAST_4    %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \
1348%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8
1349_zero_cipher_left_decrypt:
1350        mov     %arg4, %r13
1351        and     $15, %r13                               # %r13 = arg4 (mod 16)
1352        je      _multiple_of_16_bytes_decrypt
1353
1354        # Handle the last <16 byte block separately
1355
1356        paddd ONE(%rip), %xmm0         # increment CNT to get Yn
1357        movdqa SHUF_MASK(%rip), %xmm10
1358        PSHUFB_XMM %xmm10, %xmm0
1359
1360        ENCRYPT_SINGLE_BLOCK  %xmm0, %xmm1    # E(K, Yn)
1361        sub $16, %r11
1362        add %r13, %r11
1363        movdqu (%arg3,%r11,1), %xmm1   # receive the last <16 byte block
1364        lea SHIFT_MASK+16(%rip), %r12
1365        sub %r13, %r12
1366# adjust the shuffle mask pointer to be able to shift 16-%r13 bytes
1367# (%r13 is the number of bytes in plaintext mod 16)
1368        movdqu (%r12), %xmm2           # get the appropriate shuffle mask
1369        PSHUFB_XMM %xmm2, %xmm1            # right shift 16-%r13 butes
1370
1371        movdqa  %xmm1, %xmm2
1372        pxor %xmm1, %xmm0            # Ciphertext XOR E(K, Yn)
1373        movdqu ALL_F-SHIFT_MASK(%r12), %xmm1
1374        # get the appropriate mask to mask out top 16-%r13 bytes of %xmm0
1375        pand %xmm1, %xmm0            # mask out top 16-%r13 bytes of %xmm0
1376        pand    %xmm1, %xmm2
1377        movdqa SHUF_MASK(%rip), %xmm10
1378        PSHUFB_XMM %xmm10 ,%xmm2
1379
1380        pxor %xmm2, %xmm8
1381        GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
1382                  # GHASH computation for the last <16 byte block
1383        sub %r13, %r11
1384        add $16, %r11
1385
1386        # output %r13 bytes
1387        MOVQ_R64_XMM    %xmm0, %rax
1388        cmp     $8, %r13
1389        jle     _less_than_8_bytes_left_decrypt
1390        mov     %rax, (%arg2 , %r11, 1)
1391        add     $8, %r11
1392        psrldq  $8, %xmm0
1393        MOVQ_R64_XMM    %xmm0, %rax
1394        sub     $8, %r13
1395_less_than_8_bytes_left_decrypt:
1396        mov     %al,  (%arg2, %r11, 1)
1397        add     $1, %r11
1398        shr     $8, %rax
1399        sub     $1, %r13
1400        jne     _less_than_8_bytes_left_decrypt
1401_multiple_of_16_bytes_decrypt:
1402        mov     arg8, %r12                # %r13 = aadLen (number of bytes)
1403        shl     $3, %r12                  # convert into number of bits
1404        movd    %r12d, %xmm15             # len(A) in %xmm15
1405        shl     $3, %arg4                 # len(C) in bits (*128)
1406        MOVQ_R64_XMM    %arg4, %xmm1
1407        pslldq  $8, %xmm15                # %xmm15 = len(A)||0x0000000000000000
1408        pxor    %xmm1, %xmm15             # %xmm15 = len(A)||len(C)
1409        pxor    %xmm15, %xmm8
1410        GHASH_MUL       %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
1411                 # final GHASH computation
1412        movdqa SHUF_MASK(%rip), %xmm10
1413        PSHUFB_XMM %xmm10, %xmm8
1414
1415        mov     %arg5, %rax               # %rax = *Y0
1416        movdqu  (%rax), %xmm0             # %xmm0 = Y0
1417        ENCRYPT_SINGLE_BLOCK    %xmm0,  %xmm1     # E(K, Y0)
1418        pxor    %xmm8, %xmm0
1419_return_T_decrypt:
1420        mov     arg9, %r10                # %r10 = authTag
1421        mov     arg10, %r11               # %r11 = auth_tag_len
1422        cmp     $16, %r11
1423        je      _T_16_decrypt
1424        cmp     $12, %r11
1425        je      _T_12_decrypt
1426_T_8_decrypt:
1427        MOVQ_R64_XMM    %xmm0, %rax
1428        mov     %rax, (%r10)
1429        jmp     _return_T_done_decrypt
1430_T_12_decrypt:
1431        MOVQ_R64_XMM    %xmm0, %rax
1432        mov     %rax, (%r10)
1433        psrldq  $8, %xmm0
1434        movd    %xmm0, %eax
1435        mov     %eax, 8(%r10)
1436        jmp     _return_T_done_decrypt
1437_T_16_decrypt:
1438        movdqu  %xmm0, (%r10)
1439_return_T_done_decrypt:
1440        mov     %r14, %rsp
1441        pop     %r14
1442        pop     %r13
1443        pop     %r12
1444        ret
1445ENDPROC(aesni_gcm_dec)
1446
1447
1448/*****************************************************************************
1449* void aesni_gcm_enc(void *aes_ctx,      // AES Key schedule. Starts on a 16 byte boundary.
1450*                    u8 *out,            // Ciphertext output. Encrypt in-place is allowed.
1451*                    const u8 *in,       // Plaintext input
1452*                    u64 plaintext_len,  // Length of data in bytes for encryption.
1453*                    u8 *iv,             // Pre-counter block j0: 4 byte salt (from Security Association)
1454*                                        // concatenated with 8 byte Initialisation Vector (from IPSec ESP Payload)
1455*                                        // concatenated with 0x00000001. 16-byte aligned pointer.
1456*                    u8 *hash_subkey,    // H, the Hash sub key input. Data starts on a 16-byte boundary.
1457*                    const u8 *aad,      // Additional Authentication Data (AAD)
1458*                    u64 aad_len,        // Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 bytes
1459*                    u8 *auth_tag,       // Authenticated Tag output.
1460*                    u64 auth_tag_len);  // Authenticated Tag Length in bytes. Valid values are 16 (most likely),
1461*                                        // 12 or 8.
1462*
1463* Assumptions:
1464*
1465* keys:
1466*       keys are pre-expanded and aligned to 16 bytes. we are using the
1467*       first set of 11 keys in the data structure void *aes_ctx
1468*
1469*
1470* iv:
1471*       0                   1                   2                   3
1472*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1473*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1474*       |                             Salt  (From the SA)               |
1475*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1476*       |                     Initialization Vector                     |
1477*       |         (This is the sequence number from IPSec header)       |
1478*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1479*       |                              0x1                              |
1480*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1481*
1482*
1483*
1484* AAD:
1485*       AAD padded to 128 bits with 0
1486*       for example, assume AAD is a u32 vector
1487*
1488*       if AAD is 8 bytes:
1489*       AAD[3] = {A0, A1};
1490*       padded AAD in xmm register = {A1 A0 0 0}
1491*
1492*       0                   1                   2                   3
1493*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1494*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1495*       |                               SPI (A1)                        |
1496*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1497*       |                     32-bit Sequence Number (A0)               |
1498*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1499*       |                              0x0                              |
1500*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1501*
1502*                                 AAD Format with 32-bit Sequence Number
1503*
1504*       if AAD is 12 bytes:
1505*       AAD[3] = {A0, A1, A2};
1506*       padded AAD in xmm register = {A2 A1 A0 0}
1507*
1508*       0                   1                   2                   3
1509*       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
1510*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1511*       |                               SPI (A2)                        |
1512*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1513*       |                 64-bit Extended Sequence Number {A1,A0}       |
1514*       |                                                               |
1515*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1516*       |                              0x0                              |
1517*       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
1518*
1519*                         AAD Format with 64-bit Extended Sequence Number
1520*
1521* aadLen:
1522*       from the definition of the spec, aadLen can only be 8 or 12 bytes.
1523*       The code supports 16 too but for other sizes, the code will fail.
1524*
1525* TLen:
1526*       from the definition of the spec, TLen can only be 8, 12 or 16 bytes.
1527*       For other sizes, the code will fail.
1528*
1529* poly = x^128 + x^127 + x^126 + x^121 + 1
1530***************************************************************************/
1531ENTRY(aesni_gcm_enc)
1532        push    %r12
1533        push    %r13
1534        push    %r14
1535        mov     %rsp, %r14
1536#
1537# states of %xmm registers %xmm6:%xmm15 not saved
1538# all %xmm registers are clobbered
1539#
1540        sub     $VARIABLE_OFFSET, %rsp
1541        and     $~63, %rsp
1542        mov     %arg6, %r12
1543        movdqu  (%r12), %xmm13
1544        movdqa  SHUF_MASK(%rip), %xmm2
1545        PSHUFB_XMM %xmm2, %xmm13
1546
1547
1548# precompute HashKey<<1 mod poly from the HashKey (required for GHASH)
1549
1550        movdqa  %xmm13, %xmm2
1551        psllq   $1, %xmm13
1552        psrlq   $63, %xmm2
1553        movdqa  %xmm2, %xmm1
1554        pslldq  $8, %xmm2
1555        psrldq  $8, %xmm1
1556        por     %xmm2, %xmm13
1557
1558        # reduce HashKey<<1
1559
1560        pshufd  $0x24, %xmm1, %xmm2
1561        pcmpeqd TWOONE(%rip), %xmm2
1562        pand    POLY(%rip), %xmm2
1563        pxor    %xmm2, %xmm13
1564        movdqa  %xmm13, HashKey(%rsp)
1565        mov     %arg4, %r13            # %xmm13 holds HashKey<<1 (mod poly)
1566        and     $-16, %r13
1567        mov     %r13, %r12
1568
1569        # Encrypt first few blocks
1570
1571        and     $(3<<4), %r12
1572        jz      _initial_num_blocks_is_0_encrypt
1573        cmp     $(2<<4), %r12
1574        jb      _initial_num_blocks_is_1_encrypt
1575        je      _initial_num_blocks_is_2_encrypt
1576_initial_num_blocks_is_3_encrypt:
1577        INITIAL_BLOCKS_ENC      3, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1578%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 5, 678, enc
1579        sub     $48, %r13
1580        jmp     _initial_blocks_encrypted
1581_initial_num_blocks_is_2_encrypt:
1582        INITIAL_BLOCKS_ENC      2, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1583%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 6, 78, enc
1584        sub     $32, %r13
1585        jmp     _initial_blocks_encrypted
1586_initial_num_blocks_is_1_encrypt:
1587        INITIAL_BLOCKS_ENC      1, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1588%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 7, 8, enc
1589        sub     $16, %r13
1590        jmp     _initial_blocks_encrypted
1591_initial_num_blocks_is_0_encrypt:
1592        INITIAL_BLOCKS_ENC      0, %xmm9, %xmm10, %xmm13, %xmm11, %xmm12, %xmm0, \
1593%xmm1, %xmm2, %xmm3, %xmm4, %xmm8, %xmm5, %xmm6, 8, 0, enc
1594_initial_blocks_encrypted:
1595
1596        # Main loop - Encrypt remaining blocks
1597
1598        cmp     $0, %r13
1599        je      _zero_cipher_left_encrypt
1600        sub     $64, %r13
1601        je      _four_cipher_left_encrypt
1602_encrypt_by_4_encrypt:
1603        GHASH_4_ENCRYPT_4_PARALLEL_ENC  %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, \
1604%xmm14, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, enc
1605        add     $64, %r11
1606        sub     $64, %r13
1607        jne     _encrypt_by_4_encrypt
1608_four_cipher_left_encrypt:
1609        GHASH_LAST_4    %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, \
1610%xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm8
1611_zero_cipher_left_encrypt:
1612        mov     %arg4, %r13
1613        and     $15, %r13                       # %r13 = arg4 (mod 16)
1614        je      _multiple_of_16_bytes_encrypt
1615
1616         # Handle the last <16 Byte block separately
1617        paddd ONE(%rip), %xmm0                # INCR CNT to get Yn
1618        movdqa SHUF_MASK(%rip), %xmm10
1619        PSHUFB_XMM %xmm10, %xmm0
1620
1621
1622        ENCRYPT_SINGLE_BLOCK    %xmm0, %xmm1        # Encrypt(K, Yn)
1623        sub $16, %r11
1624        add %r13, %r11
1625        movdqu (%arg3,%r11,1), %xmm1     # receive the last <16 byte blocks
1626        lea SHIFT_MASK+16(%rip), %r12
1627        sub %r13, %r12
1628        # adjust the shuffle mask pointer to be able to shift 16-r13 bytes
1629        # (%r13 is the number of bytes in plaintext mod 16)
1630        movdqu  (%r12), %xmm2           # get the appropriate shuffle mask
1631        PSHUFB_XMM      %xmm2, %xmm1            # shift right 16-r13 byte
1632        pxor    %xmm1, %xmm0            # Plaintext XOR Encrypt(K, Yn)
1633        movdqu  ALL_F-SHIFT_MASK(%r12), %xmm1
1634        # get the appropriate mask to mask out top 16-r13 bytes of xmm0
1635        pand    %xmm1, %xmm0            # mask out top 16-r13 bytes of xmm0
1636        movdqa SHUF_MASK(%rip), %xmm10
1637        PSHUFB_XMM %xmm10,%xmm0
1638
1639        pxor    %xmm0, %xmm8
1640        GHASH_MUL %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
1641        # GHASH computation for the last <16 byte block
1642        sub     %r13, %r11
1643        add     $16, %r11
1644
1645        movdqa SHUF_MASK(%rip), %xmm10
1646        PSHUFB_XMM %xmm10, %xmm0
1647
1648        # shuffle xmm0 back to output as ciphertext
1649
1650        # Output %r13 bytes
1651        MOVQ_R64_XMM %xmm0, %rax
1652        cmp $8, %r13
1653        jle _less_than_8_bytes_left_encrypt
1654        mov %rax, (%arg2 , %r11, 1)
1655        add $8, %r11
1656        psrldq $8, %xmm0
1657        MOVQ_R64_XMM %xmm0, %rax
1658        sub $8, %r13
1659_less_than_8_bytes_left_encrypt:
1660        mov %al,  (%arg2, %r11, 1)
1661        add $1, %r11
1662        shr $8, %rax
1663        sub $1, %r13
1664        jne _less_than_8_bytes_left_encrypt
1665_multiple_of_16_bytes_encrypt:
1666        mov     arg8, %r12    # %r12 = addLen (number of bytes)
1667        shl     $3, %r12
1668        movd    %r12d, %xmm15       # len(A) in %xmm15
1669        shl     $3, %arg4               # len(C) in bits (*128)
1670        MOVQ_R64_XMM    %arg4, %xmm1
1671        pslldq  $8, %xmm15          # %xmm15 = len(A)||0x0000000000000000
1672        pxor    %xmm1, %xmm15       # %xmm15 = len(A)||len(C)
1673        pxor    %xmm15, %xmm8
1674        GHASH_MUL       %xmm8, %xmm13, %xmm9, %xmm10, %xmm11, %xmm5, %xmm6
1675        # final GHASH computation
1676        movdqa SHUF_MASK(%rip), %xmm10
1677        PSHUFB_XMM %xmm10, %xmm8         # perform a 16 byte swap
1678
1679        mov     %arg5, %rax                    # %rax  = *Y0
1680        movdqu  (%rax), %xmm0                  # %xmm0 = Y0
1681        ENCRYPT_SINGLE_BLOCK    %xmm0, %xmm15         # Encrypt(K, Y0)
1682        pxor    %xmm8, %xmm0
1683_return_T_encrypt:
1684        mov     arg9, %r10                     # %r10 = authTag
1685        mov     arg10, %r11                    # %r11 = auth_tag_len
1686        cmp     $16, %r11
1687        je      _T_16_encrypt
1688        cmp     $12, %r11
1689        je      _T_12_encrypt
1690_T_8_encrypt:
1691        MOVQ_R64_XMM    %xmm0, %rax
1692        mov     %rax, (%r10)
1693        jmp     _return_T_done_encrypt
1694_T_12_encrypt:
1695        MOVQ_R64_XMM    %xmm0, %rax
1696        mov     %rax, (%r10)
1697        psrldq  $8, %xmm0
1698        movd    %xmm0, %eax
1699        mov     %eax, 8(%r10)
1700        jmp     _return_T_done_encrypt
1701_T_16_encrypt:
1702        movdqu  %xmm0, (%r10)
1703_return_T_done_encrypt:
1704        mov     %r14, %rsp
1705        pop     %r14
1706        pop     %r13
1707        pop     %r12
1708        ret
1709ENDPROC(aesni_gcm_enc)
1710
1711#endif
1712
1713
1714.align 4
1715_key_expansion_128:
1716_key_expansion_256a:
1717        pshufd $0b11111111, %xmm1, %xmm1
1718        shufps $0b00010000, %xmm0, %xmm4
1719        pxor %xmm4, %xmm0
1720        shufps $0b10001100, %xmm0, %xmm4
1721        pxor %xmm4, %xmm0
1722        pxor %xmm1, %xmm0
1723        movaps %xmm0, (TKEYP)
1724        add $0x10, TKEYP
1725        ret
1726ENDPROC(_key_expansion_128)
1727ENDPROC(_key_expansion_256a)
1728
1729.align 4
1730_key_expansion_192a:
1731        pshufd $0b01010101, %xmm1, %xmm1
1732        shufps $0b00010000, %xmm0, %xmm4
1733        pxor %xmm4, %xmm0
1734        shufps $0b10001100, %xmm0, %xmm4
1735        pxor %xmm4, %xmm0
1736        pxor %xmm1, %xmm0
1737
1738        movaps %xmm2, %xmm5
1739        movaps %xmm2, %xmm6
1740        pslldq $4, %xmm5
1741        pshufd $0b11111111, %xmm0, %xmm3
1742        pxor %xmm3, %xmm2
1743        pxor %xmm5, %xmm2
1744
1745        movaps %xmm0, %xmm1
1746        shufps $0b01000100, %xmm0, %xmm6
1747        movaps %xmm6, (TKEYP)
1748        shufps $0b01001110, %xmm2, %xmm1
1749        movaps %xmm1, 0x10(TKEYP)
1750        add $0x20, TKEYP
1751        ret
1752ENDPROC(_key_expansion_192a)
1753
1754.align 4
1755_key_expansion_192b:
1756        pshufd $0b01010101, %xmm1, %xmm1
1757        shufps $0b00010000, %xmm0, %xmm4
1758        pxor %xmm4, %xmm0
1759        shufps $0b10001100, %xmm0, %xmm4
1760        pxor %xmm4, %xmm0
1761        pxor %xmm1, %xmm0
1762
1763        movaps %xmm2, %xmm5
1764        pslldq $4, %xmm5
1765        pshufd $0b11111111, %xmm0, %xmm3
1766        pxor %xmm3, %xmm2
1767        pxor %xmm5, %xmm2
1768
1769        movaps %xmm0, (TKEYP)
1770        add $0x10, TKEYP
1771        ret
1772ENDPROC(_key_expansion_192b)
1773
1774.align 4
1775_key_expansion_256b:
1776        pshufd $0b10101010, %xmm1, %xmm1
1777        shufps $0b00010000, %xmm2, %xmm4
1778        pxor %xmm4, %xmm2
1779        shufps $0b10001100, %xmm2, %xmm4
1780        pxor %xmm4, %xmm2
1781        pxor %xmm1, %xmm2
1782        movaps %xmm2, (TKEYP)
1783        add $0x10, TKEYP
1784        ret
1785ENDPROC(_key_expansion_256b)
1786
1787/*
1788 * int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
1789 *                   unsigned int key_len)
1790 */
1791ENTRY(aesni_set_key)
1792#ifndef __x86_64__
1793        pushl KEYP
1794        movl 8(%esp), KEYP              # ctx
1795        movl 12(%esp), UKEYP            # in_key
1796        movl 16(%esp), %edx             # key_len
1797#endif
1798        movups (UKEYP), %xmm0           # user key (first 16 bytes)
1799        movaps %xmm0, (KEYP)
1800        lea 0x10(KEYP), TKEYP           # key addr
1801        movl %edx, 480(KEYP)
1802        pxor %xmm4, %xmm4               # xmm4 is assumed 0 in _key_expansion_x
1803        cmp $24, %dl
1804        jb .Lenc_key128
1805        je .Lenc_key192
1806        movups 0x10(UKEYP), %xmm2       # other user key
1807        movaps %xmm2, (TKEYP)
1808        add $0x10, TKEYP
1809        AESKEYGENASSIST 0x1 %xmm2 %xmm1         # round 1
1810        call _key_expansion_256a
1811        AESKEYGENASSIST 0x1 %xmm0 %xmm1
1812        call _key_expansion_256b
1813        AESKEYGENASSIST 0x2 %xmm2 %xmm1         # round 2
1814        call _key_expansion_256a
1815        AESKEYGENASSIST 0x2 %xmm0 %xmm1
1816        call _key_expansion_256b
1817        AESKEYGENASSIST 0x4 %xmm2 %xmm1         # round 3
1818        call _key_expansion_256a
1819        AESKEYGENASSIST 0x4 %xmm0 %xmm1
1820        call _key_expansion_256b
1821        AESKEYGENASSIST 0x8 %xmm2 %xmm1         # round 4
1822        call _key_expansion_256a
1823        AESKEYGENASSIST 0x8 %xmm0 %xmm1
1824        call _key_expansion_256b
1825        AESKEYGENASSIST 0x10 %xmm2 %xmm1        # round 5
1826        call _key_expansion_256a
1827        AESKEYGENASSIST 0x10 %xmm0 %xmm1
1828        call _key_expansion_256b
1829        AESKEYGENASSIST 0x20 %xmm2 %xmm1        # round 6
1830        call _key_expansion_256a
1831        AESKEYGENASSIST 0x20 %xmm0 %xmm1
1832        call _key_expansion_256b
1833        AESKEYGENASSIST 0x40 %xmm2 %xmm1        # round 7
1834        call _key_expansion_256a
1835        jmp .Ldec_key
1836.Lenc_key192:
1837        movq 0x10(UKEYP), %xmm2         # other user key
1838        AESKEYGENASSIST 0x1 %xmm2 %xmm1         # round 1
1839        call _key_expansion_192a
1840        AESKEYGENASSIST 0x2 %xmm2 %xmm1         # round 2
1841        call _key_expansion_192b
1842        AESKEYGENASSIST 0x4 %xmm2 %xmm1         # round 3
1843        call _key_expansion_192a
1844        AESKEYGENASSIST 0x8 %xmm2 %xmm1         # round 4
1845        call _key_expansion_192b
1846        AESKEYGENASSIST 0x10 %xmm2 %xmm1        # round 5
1847        call _key_expansion_192a
1848        AESKEYGENASSIST 0x20 %xmm2 %xmm1        # round 6
1849        call _key_expansion_192b
1850        AESKEYGENASSIST 0x40 %xmm2 %xmm1        # round 7
1851        call _key_expansion_192a
1852        AESKEYGENASSIST 0x80 %xmm2 %xmm1        # round 8
1853        call _key_expansion_192b
1854        jmp .Ldec_key
1855.Lenc_key128:
1856        AESKEYGENASSIST 0x1 %xmm0 %xmm1         # round 1
1857        call _key_expansion_128
1858        AESKEYGENASSIST 0x2 %xmm0 %xmm1         # round 2
1859        call _key_expansion_128
1860        AESKEYGENASSIST 0x4 %xmm0 %xmm1         # round 3
1861        call _key_expansion_128
1862        AESKEYGENASSIST 0x8 %xmm0 %xmm1         # round 4
1863        call _key_expansion_128
1864        AESKEYGENASSIST 0x10 %xmm0 %xmm1        # round 5
1865        call _key_expansion_128
1866        AESKEYGENASSIST 0x20 %xmm0 %xmm1        # round 6
1867        call _key_expansion_128
1868        AESKEYGENASSIST 0x40 %xmm0 %xmm1        # round 7
1869        call _key_expansion_128
1870        AESKEYGENASSIST 0x80 %xmm0 %xmm1        # round 8
1871        call _key_expansion_128
1872        AESKEYGENASSIST 0x1b %xmm0 %xmm1        # round 9
1873        call _key_expansion_128
1874        AESKEYGENASSIST 0x36 %xmm0 %xmm1        # round 10
1875        call _key_expansion_128
1876.Ldec_key:
1877        sub $0x10, TKEYP
1878        movaps (KEYP), %xmm0
1879        movaps (TKEYP), %xmm1
1880        movaps %xmm0, 240(TKEYP)
1881        movaps %xmm1, 240(KEYP)
1882        add $0x10, KEYP
1883        lea 240-16(TKEYP), UKEYP
1884.align 4
1885.Ldec_key_loop:
1886        movaps (KEYP), %xmm0
1887        AESIMC %xmm0 %xmm1
1888        movaps %xmm1, (UKEYP)
1889        add $0x10, KEYP
1890        sub $0x10, UKEYP
1891        cmp TKEYP, KEYP
1892        jb .Ldec_key_loop
1893        xor AREG, AREG
1894#ifndef __x86_64__
1895        popl KEYP
1896#endif
1897        ret
1898ENDPROC(aesni_set_key)
1899
1900/*
1901 * void aesni_enc(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
1902 */
1903ENTRY(aesni_enc)
1904#ifndef __x86_64__
1905        pushl KEYP
1906        pushl KLEN
1907        movl 12(%esp), KEYP
1908        movl 16(%esp), OUTP
1909        movl 20(%esp), INP
1910#endif
1911        movl 480(KEYP), KLEN            # key length
1912        movups (INP), STATE             # input
1913        call _aesni_enc1
1914        movups STATE, (OUTP)            # output
1915#ifndef __x86_64__
1916        popl KLEN
1917        popl KEYP
1918#endif
1919        ret
1920ENDPROC(aesni_enc)
1921
1922/*
1923 * _aesni_enc1:         internal ABI
1924 * input:
1925 *      KEYP:           key struct pointer
1926 *      KLEN:           round count
1927 *      STATE:          initial state (input)
1928 * output:
1929 *      STATE:          finial state (output)
1930 * changed:
1931 *      KEY
1932 *      TKEYP (T1)
1933 */
1934.align 4
1935_aesni_enc1:
1936        movaps (KEYP), KEY              # key
1937        mov KEYP, TKEYP
1938        pxor KEY, STATE         # round 0
1939        add $0x30, TKEYP
1940        cmp $24, KLEN
1941        jb .Lenc128
1942        lea 0x20(TKEYP), TKEYP
1943        je .Lenc192
1944        add $0x20, TKEYP
1945        movaps -0x60(TKEYP), KEY
1946        AESENC KEY STATE
1947        movaps -0x50(TKEYP), KEY
1948        AESENC KEY STATE
1949.align 4
1950.Lenc192:
1951        movaps -0x40(TKEYP), KEY
1952        AESENC KEY STATE
1953        movaps -0x30(TKEYP), KEY
1954        AESENC KEY STATE
1955.align 4
1956.Lenc128:
1957        movaps -0x20(TKEYP), KEY
1958        AESENC KEY STATE
1959        movaps -0x10(TKEYP), KEY
1960        AESENC KEY STATE
1961        movaps (TKEYP), KEY
1962        AESENC KEY STATE
1963        movaps 0x10(TKEYP), KEY
1964        AESENC KEY STATE
1965        movaps 0x20(TKEYP), KEY
1966        AESENC KEY STATE
1967        movaps 0x30(TKEYP), KEY
1968        AESENC KEY STATE
1969        movaps 0x40(TKEYP), KEY
1970        AESENC KEY STATE
1971        movaps 0x50(TKEYP), KEY
1972        AESENC KEY STATE
1973        movaps 0x60(TKEYP), KEY
1974        AESENC KEY STATE
1975        movaps 0x70(TKEYP), KEY
1976        AESENCLAST KEY STATE
1977        ret
1978ENDPROC(_aesni_enc1)
1979
1980/*
1981 * _aesni_enc4: internal ABI
1982 * input:
1983 *      KEYP:           key struct pointer
1984 *      KLEN:           round count
1985 *      STATE1:         initial state (input)
1986 *      STATE2
1987 *      STATE3
1988 *      STATE4
1989 * output:
1990 *      STATE1:         finial state (output)
1991 *      STATE2
1992 *      STATE3
1993 *      STATE4
1994 * changed:
1995 *      KEY
1996 *      TKEYP (T1)
1997 */
1998.align 4
1999_aesni_enc4:
2000        movaps (KEYP), KEY              # key

2001        mov KEYP, TKEYP
2002        pxor KEY, STATE1                # round 0
2003        pxor KEY, STATE2
2004        pxor KEY, STATE3
2005        pxor KEY, STATE4
2006        add $0x30, TKEYP
2007        cmp $24, KLEN
2008        jb .L4enc128
2009        lea 0x20(TKEYP), TKEYP
2010        je .L4enc192
2011        add $0x20, TKEYP
2012        movaps -0x60(TKEYP), KEY
2013        AESENC KEY STATE1
2014        AESENC KEY STATE2
2015        AESENC KEY STATE3
2016        AESENC KEY STATE4
2017        movaps -0x50(TKEYP), KEY
2018        AESENC KEY STATE1
2019        AESENC KEY STATE2
2020        AESENC KEY STATE3
2021        AESENC KEY STATE4
2022#.align 4
2023.L4enc192:
2024        movaps -0x40(TKEYP), KEY
2025        AESENC KEY STATE1
2026        AESENC KEY STATE2
2027        AESENC KEY STATE3
2028        AESENC KEY STATE4
2029        movaps -0x30(TKEYP), KEY
2030        AESENC KEY STATE1
2031        AESENC KEY STATE2
2032        AESENC KEY STATE3
2033        AESENC KEY STATE4
2034#.align 4
2035.L4enc128:
2036        movaps -0x20(TKEYP), KEY
2037        AESENC KEY STATE1
2038        AESENC KEY STATE2
2039        AESENC KEY STATE3
2040        AESENC KEY STATE4
2041        movaps -0x10(TKEYP), KEY
2042        AESENC KEY STATE1
2043        AESENC KEY STATE2
2044        AESENC KEY STATE3
2045        AESENC KEY STATE4
2046        movaps (TKEYP), KEY
2047        AESENC KEY STATE1
2048        AESENC KEY STATE2
2049        AESENC KEY STATE3
2050        AESENC KEY STATE4
2051        movaps 0x10(TKEYP), KEY
2052        AESENC KEY STATE1
2053        AESENC KEY STATE2
2054        AESENC KEY STATE3
2055        AESENC KEY STATE4
2056        movaps 0x20(TKEYP), KEY
2057        AESENC KEY STATE1
2058        AESENC KEY STATE2
2059        AESENC KEY STATE3
2060        AESENC KEY STATE4
2061        movaps 0x30(TKEYP), KEY
2062        AESENC KEY STATE1
2063        AESENC KEY STATE2
2064        AESENC KEY STATE3
2065        AESENC KEY STATE4
2066        movaps 0x40(TKEYP), KEY
2067        AESENC KEY STATE1
2068        AESENC KEY STATE2
2069        AESENC KEY STATE3
2070        AESENC KEY STATE4
2071        movaps 0x50(TKEYP), KEY
2072        AESENC KEY STATE1
2073        AESENC KEY STATE2
2074        AESENC KEY STATE3
2075        AESENC KEY STATE4
2076        movaps 0x60(TKEYP), KEY
2077        AESENC KEY STATE1
2078        AESENC KEY STATE2
2079        AESENC KEY STATE3
2080        AESENC KEY STATE4
2081        movaps 0x70(TKEYP), KEY
2082        AESENCLAST KEY STATE1           # last round
2083        AESENCLAST KEY STATE2
2084        AESENCLAST KEY STATE3
2085        AESENCLAST KEY STATE4
2086        ret
2087ENDPROC(_aesni_enc4)
2088
2089/*
2090 * void aesni_dec (struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src)
2091 */
2092ENTRY(aesni_dec)
2093#ifndef __x86_64__
2094        pushl KEYP
2095        pushl KLEN
2096        movl 12(%esp), KEYP
2097        movl 16(%esp), OUTP
2098        movl 20(%esp), INP
2099#endif
2100        mov 480(KEYP), KLEN             # key length
2101        add $240, KEYP
2102        movups (INP), STATE             # input
2103        call _aesni_dec1
2104        movups STATE, (OUTP)            #output
2105#ifndef __x86_64__
2106        popl KLEN
2107        popl KEYP
2108#endif
2109        ret
2110ENDPROC(aesni_dec)
2111
2112/*
2113 * _aesni_dec1:         internal ABI
2114 * input:
2115 *      KEYP:           key struct pointer
2116 *      KLEN:           key length
2117 *      STATE:          initial state (input)
2118 * output:
2119 *      STATE:          finial state (output)
2120 * changed:
2121 *      KEY
2122 *      TKEYP (T1)
2123 */
2124.align 4
2125_aesni_dec1:
2126        movaps (KEYP), KEY              # key
2127        mov KEYP, TKEYP
2128        pxor KEY, STATE         # round 0
2129        add $0x30, TKEYP
2130        cmp $24, KLEN
2131        jb .Ldec128
2132        lea 0x20(TKEYP), TKEYP
2133        je .Ldec192
2134        add $0x20, TKEYP
2135        movaps -0x60(TKEYP), KEY
2136        AESDEC KEY STATE
2137        movaps -0x50(TKEYP), KEY
2138        AESDEC KEY STATE
2139.align 4
2140.Ldec192:
2141        movaps -0x40(TKEYP), KEY
2142        AESDEC KEY STATE
2143        movaps -0x30(TKEYP), KEY
2144        AESDEC KEY STATE
2145.align 4
2146.Ldec128:
2147        movaps -0x20(TKEYP), KEY
2148        AESDEC KEY STATE
2149        movaps -0x10(TKEYP), KEY
2150        AESDEC KEY STATE
2151        movaps (TKEYP), KEY
2152        AESDEC KEY STATE
2153        movaps 0x10(TKEYP), KEY
2154        AESDEC KEY STATE
2155        movaps 0x20(TKEYP), KEY
2156        AESDEC KEY STATE
2157        movaps 0x30(TKEYP), KEY
2158        AESDEC KEY STATE
2159        movaps 0x40(TKEYP), KEY
2160        AESDEC KEY STATE
2161        movaps 0x50(TKEYP), KEY
2162        AESDEC KEY STATE
2163        movaps 0x60(TKEYP), KEY
2164        AESDEC KEY STATE
2165        movaps 0x70(TKEYP), KEY
2166        AESDECLAST KEY STATE
2167        ret
2168ENDPROC(_aesni_dec1)
2169
2170/*
2171 * _aesni_dec4: internal ABI
2172 * input:
2173 *      KEYP:           key struct pointer
2174 *      KLEN:           key length
2175 *      STATE1:         initial state (input)
2176 *      STATE2
2177 *      STATE3
2178 *      STATE4
2179 * output:
2180 *      STATE1:         finial state (output)
2181 *      STATE2
2182 *      STATE3
2183 *      STATE4
2184 * changed:
2185 *      KEY
2186 *      TKEYP (T1)
2187 */
2188.align 4
2189_aesni_dec4:
2190        movaps (KEYP), KEY              # key
2191        mov KEYP, TKEYP
2192        pxor KEY, STATE1                # round 0
2193        pxor KEY, STATE2
2194        pxor KEY, STATE3
2195        pxor KEY, STATE4
2196        add $0x30, TKEYP
2197        cmp $24, KLEN
2198        jb .L4dec128
2199        lea 0x20(TKEYP), TKEYP
2200        je .L4dec192
2201        add $0x20, TKEYP
2202        movaps -0x60(TKEYP), KEY
2203        AESDEC KEY STATE1
2204        AESDEC KEY STATE2
2205        AESDEC KEY STATE3
2206        AESDEC KEY STATE4
2207        movaps -0x50(TKEYP), KEY
2208        AESDEC KEY STATE1
2209        AESDEC KEY STATE2
2210        AESDEC KEY STATE3
2211        AESDEC KEY STATE4
2212.align 4
2213.L4dec192:
2214        movaps -0x40(TKEYP), KEY
2215        AESDEC KEY STATE1
2216        AESDEC KEY STATE2
2217        AESDEC KEY STATE3
2218        AESDEC KEY STATE4
2219        movaps -0x30(TKEYP), KEY
2220        AESDEC KEY STATE1
2221        AESDEC KEY STATE2
2222        AESDEC KEY STATE3
2223        AESDEC KEY STATE4
2224.align 4
2225.L4dec128:
2226        movaps -0x20(TKEYP), KEY
2227        AESDEC KEY STATE1
2228        AESDEC KEY STATE2
2229        AESDEC KEY STATE3
2230        AESDEC KEY STATE4
2231        movaps -0x10(TKEYP), KEY
2232        AESDEC KEY STATE1
2233        AESDEC KEY STATE2
2234        AESDEC KEY STATE3
2235        AESDEC KEY STATE4
2236        movaps (TKEYP), KEY
2237        AESDEC KEY STATE1
2238        AESDEC KEY STATE2
2239        AESDEC KEY STATE3
2240        AESDEC KEY STATE4
2241        movaps 0x10(TKEYP), KEY
2242        AESDEC KEY STATE1
2243        AESDEC KEY STATE2
2244        AESDEC KEY STATE3
2245        AESDEC KEY STATE4
2246        movaps 0x20(TKEYP), KEY
2247        AESDEC KEY STATE1
2248        AESDEC KEY STATE2
2249        AESDEC KEY STATE3
2250        AESDEC KEY STATE4
2251        movaps 0x30(TKEYP), KEY
2252        AESDEC KEY STATE1
2253        AESDEC KEY STATE2
2254        AESDEC KEY STATE3
2255        AESDEC KEY STATE4
2256        movaps 0x40(TKEYP), KEY
2257        AESDEC KEY STATE1
2258        AESDEC KEY STATE2
2259        AESDEC KEY STATE3
2260        AESDEC KEY STATE4
2261        movaps 0x50(TKEYP), KEY
2262        AESDEC KEY STATE1
2263        AESDEC KEY STATE2
2264        AESDEC KEY STATE3
2265        AESDEC KEY STATE4
2266        movaps 0x60(TKEYP), KEY
2267        AESDEC KEY STATE1
2268        AESDEC KEY STATE2
2269        AESDEC KEY STATE3
2270        AESDEC KEY STATE4
2271        movaps 0x70(TKEYP), KEY
2272        AESDECLAST KEY STATE1           # last round
2273        AESDECLAST KEY STATE2
2274        AESDECLAST KEY STATE3
2275        AESDECLAST KEY STATE4
2276        ret
2277ENDPROC(_aesni_dec4)
2278
2279/*
2280 * void aesni_ecb_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
2281 *                    size_t len)
2282 */
2283ENTRY(aesni_ecb_enc)
2284#ifndef __x86_64__
2285        pushl LEN
2286        pushl KEYP
2287        pushl KLEN
2288        movl 16(%esp), KEYP
2289        movl 20(%esp), OUTP
2290        movl 24(%esp), INP
2291        movl 28(%esp), LEN
2292#endif
2293        test LEN, LEN           # check length
2294        jz .Lecb_enc_ret
2295        mov 480(KEYP), KLEN
2296        cmp $16, LEN
2297        jb .Lecb_enc_ret
2298        cmp $64, LEN
2299        jb .Lecb_enc_loop1
2300.align 4
2301.Lecb_enc_loop4:
2302        movups (INP), STATE1
2303        movups 0x10(INP), STATE2
2304        movups 0x20(INP), STATE3
2305        movups 0x30(INP), STATE4
2306        call _aesni_enc4
2307        movups STATE1, (OUTP)
2308        movups STATE2, 0x10(OUTP)
2309        movups STATE3, 0x20(OUTP)
2310        movups STATE4, 0x30(OUTP)
2311        sub $64, LEN
2312        add $64, INP
2313        add $64, OUTP
2314        cmp $64, LEN
2315        jge .Lecb_enc_loop4
2316        cmp $16, LEN
2317        jb .Lecb_enc_ret
2318.align 4
2319.Lecb_enc_loop1:
2320        movups (INP), STATE1
2321        call _aesni_enc1
2322        movups STATE1, (OUTP)
2323        sub $16, LEN
2324        add $16, INP
2325        add $16, OUTP
2326        cmp $16, LEN
2327        jge .Lecb_enc_loop1
2328.Lecb_enc_ret:
2329#ifndef __x86_64__
2330        popl KLEN
2331        popl KEYP
2332        popl LEN
2333#endif
2334        ret
2335ENDPROC(aesni_ecb_enc)
2336
2337/*
2338 * void aesni_ecb_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
2339 *                    size_t len);
2340 */
2341ENTRY(aesni_ecb_dec)
2342#ifndef __x86_64__
2343        pushl LEN
2344        pushl KEYP
2345        pushl KLEN
2346        movl 16(%esp), KEYP
2347        movl 20(%esp), OUTP
2348        movl 24(%esp), INP
2349        movl 28(%esp), LEN
2350#endif
2351        test LEN, LEN
2352        jz .Lecb_dec_ret
2353        mov 480(KEYP), KLEN
2354        add $240, KEYP
2355        cmp $16, LEN
2356        jb .Lecb_dec_ret
2357        cmp $64, LEN
2358        jb .Lecb_dec_loop1
2359.align 4
2360.Lecb_dec_loop4:
2361        movups (INP), STATE1
2362        movups 0x10(INP), STATE2
2363        movups 0x20(INP), STATE3
2364        movups 0x30(INP), STATE4
2365        call _aesni_dec4
2366        movups STATE1, (OUTP)
2367        movups STATE2, 0x10(OUTP)
2368        movups STATE3, 0x20(OUTP)
2369        movups STATE4, 0x30(OUTP)
2370        sub $64, LEN
2371        add $64, INP
2372        add $64, OUTP
2373        cmp $64, LEN
2374        jge .Lecb_dec_loop4
2375        cmp $16, LEN
2376        jb .Lecb_dec_ret
2377.align 4
2378.Lecb_dec_loop1:
2379        movups (INP), STATE1
2380        call _aesni_dec1
2381        movups STATE1, (OUTP)
2382        sub $16, LEN
2383        add $16, INP
2384        add $16, OUTP
2385        cmp $16, LEN
2386        jge .Lecb_dec_loop1
2387.Lecb_dec_ret:
2388#ifndef __x86_64__
2389        popl KLEN
2390        popl KEYP
2391        popl LEN
2392#endif
2393        ret
2394ENDPROC(aesni_ecb_dec)
2395
2396/*
2397 * void aesni_cbc_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
2398 *                    size_t len, u8 *iv)
2399 */
2400ENTRY(aesni_cbc_enc)
2401#ifndef __x86_64__
2402        pushl IVP
2403        pushl LEN
2404        pushl KEYP
2405        pushl KLEN
2406        movl 20(%esp), KEYP
2407        movl 24(%esp), OUTP
2408        movl 28(%esp), INP
2409        movl 32(%esp), LEN
2410        movl 36(%esp), IVP
2411#endif
2412        cmp $16, LEN
2413        jb .Lcbc_enc_ret
2414        mov 480(KEYP), KLEN
2415        movups (IVP), STATE     # load iv as initial state
2416.align 4
2417.Lcbc_enc_loop:
2418        movups (INP), IN        # load input
2419        pxor IN, STATE
2420        call _aesni_enc1
2421        movups STATE, (OUTP)    # store output
2422        sub $16, LEN
2423        add $16, INP
2424        add $16, OUTP
2425        cmp $16, LEN
2426        jge .Lcbc_enc_loop
2427        movups STATE, (IVP)
2428.Lcbc_enc_ret:
2429#ifndef __x86_64__
2430        popl KLEN
2431        popl KEYP
2432        popl LEN
2433        popl IVP
2434#endif
2435        ret
2436ENDPROC(aesni_cbc_enc)
2437
2438/*
2439 * void aesni_cbc_dec(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
2440 *                    size_t len, u8 *iv)
2441 */
2442ENTRY(aesni_cbc_dec)
2443#ifndef __x86_64__
2444        pushl IVP
2445        pushl LEN
2446        pushl KEYP
2447        pushl KLEN
2448        movl 20(%esp), KEYP
2449        movl 24(%esp), OUTP
2450        movl 28(%esp), INP
2451        movl 32(%esp), LEN
2452        movl 36(%esp), IVP
2453#endif
2454        cmp $16, LEN
2455        jb .Lcbc_dec_just_ret
2456        mov 480(KEYP), KLEN
2457        add $240, KEYP
2458        movups (IVP), IV
2459        cmp $64, LEN
2460        jb .Lcbc_dec_loop1
2461.align 4
2462.Lcbc_dec_loop4:
2463        movups (INP), IN1
2464        movaps IN1, STATE1
2465        movups 0x10(INP), IN2
2466        movaps IN2, STATE2
2467#ifdef __x86_64__
2468        movups 0x20(INP), IN3
2469        movaps IN3, STATE3
2470        movups 0x30(INP), IN4
2471        movaps IN4, STATE4
2472#else
2473        movups 0x20(INP), IN1
2474        movaps IN1, STATE3
2475        movups 0x30(INP), IN2
2476        movaps IN2, STATE4
2477#endif
2478        call _aesni_dec4
2479        pxor IV, STATE1
2480#ifdef __x86_64__
2481        pxor IN1, STATE2
2482        pxor IN2, STATE3
2483        pxor IN3, STATE4
2484        movaps IN4, IV
2485#else
2486        pxor IN1, STATE4
2487        movaps IN2, IV
2488        movups (INP), IN1
2489        pxor IN1, STATE2
2490        movups 0x10(INP), IN2
2491        pxor IN2, STATE3
2492#endif
2493        movups STATE1, (OUTP)
2494        movups STATE2, 0x10(OUTP)
2495        movups STATE3, 0x20(OUTP)
2496        movups STATE4, 0x30(OUTP)
2497        sub $64, LEN
2498        add $64, INP
2499        add $64, OUTP
2500        cmp $64, LEN
2501        jge .Lcbc_dec_loop4
2502        cmp $16, LEN
2503        jb .Lcbc_dec_ret
2504.align 4
2505.Lcbc_dec_loop1:
2506        movups (INP), IN
2507        movaps IN, STATE
2508        call _aesni_dec1
2509        pxor IV, STATE
2510        movups STATE, (OUTP)
2511        movaps IN, IV
2512        sub $16, LEN
2513        add $16, INP
2514        add $16, OUTP
2515        cmp $16, LEN
2516        jge .Lcbc_dec_loop1
2517.Lcbc_dec_ret:
2518        movups IV, (IVP)
2519.Lcbc_dec_just_ret:
2520#ifndef __x86_64__
2521        popl KLEN
2522        popl KEYP
2523        popl LEN
2524        popl IVP
2525#endif
2526        ret
2527ENDPROC(aesni_cbc_dec)
2528
2529#ifdef __x86_64__
2530.align 16
2531.Lbswap_mask:
2532        .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
2533
2534/*
2535 * _aesni_inc_init:     internal ABI
2536 *      setup registers used by _aesni_inc
2537 * input:
2538 *      IV
2539 * output:
2540 *      CTR:    == IV, in little endian
2541 *      TCTR_LOW: == lower qword of CTR
2542 *      INC:    == 1, in little endian
2543 *      BSWAP_MASK == endian swapping mask
2544 */
2545.align 4
2546_aesni_inc_init:
2547        movaps .Lbswap_mask, BSWAP_MASK
2548        movaps IV, CTR
2549        PSHUFB_XMM BSWAP_MASK CTR
2550        mov $1, TCTR_LOW
2551        MOVQ_R64_XMM TCTR_LOW INC
2552        MOVQ_R64_XMM CTR TCTR_LOW
2553        ret
2554ENDPROC(_aesni_inc_init)
2555
2556/*
2557 * _aesni_inc:          internal ABI
2558 *      Increase IV by 1, IV is in big endian
2559 * input:
2560 *      IV
2561 *      CTR:    == IV, in little endian
2562 *      TCTR_LOW: == lower qword of CTR
2563 *      INC:    == 1, in little endian
2564 *      BSWAP_MASK == endian swapping mask
2565 * output:
2566 *      IV:     Increase by 1
2567 * changed:
2568 *      CTR:    == output IV, in little endian
2569 *      TCTR_LOW: == lower qword of CTR
2570 */
2571.align 4
2572_aesni_inc:
2573        paddq INC, CTR
2574        add $1, TCTR_LOW
2575        jnc .Linc_low
2576        pslldq $8, INC
2577        paddq INC, CTR
2578        psrldq $8, INC
2579.Linc_low:
2580        movaps CTR, IV
2581        PSHUFB_XMM BSWAP_MASK IV
2582        ret
2583ENDPROC(_aesni_inc)
2584
2585/*
2586 * void aesni_ctr_enc(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
2587 *                    size_t len, u8 *iv)
2588 */
2589ENTRY(aesni_ctr_enc)
2590        cmp $16, LEN
2591        jb .Lctr_enc_just_ret
2592        mov 480(KEYP), KLEN
2593        movups (IVP), IV
2594        call _aesni_inc_init
2595        cmp $64, LEN
2596        jb .Lctr_enc_loop1
2597.align 4
2598.Lctr_enc_loop4:
2599        movaps IV, STATE1
2600        call _aesni_inc
2601        movups (INP), IN1
2602        movaps IV, STATE2
2603        call _aesni_inc
2604        movups 0x10(INP), IN2
2605        movaps IV, STATE3
2606        call _aesni_inc
2607        movups 0x20(INP), IN3
2608        movaps IV, STATE4
2609        call _aesni_inc
2610        movups 0x30(INP), IN4
2611        call _aesni_enc4
2612        pxor IN1, STATE1
2613        movups STATE1, (OUTP)
2614        pxor IN2, STATE2
2615        movups STATE2, 0x10(OUTP)
2616        pxor IN3, STATE3
2617        movups STATE3, 0x20(OUTP)
2618        pxor IN4, STATE4
2619        movups STATE4, 0x30(OUTP)
2620        sub $64, LEN
2621        add $64, INP
2622        add $64, OUTP
2623        cmp $64, LEN
2624        jge .Lctr_enc_loop4
2625        cmp $16, LEN
2626        jb .Lctr_enc_ret
2627.align 4
2628.Lctr_enc_loop1:
2629        movaps IV, STATE
2630        call _aesni_inc
2631        movups (INP), IN
2632        call _aesni_enc1
2633        pxor IN, STATE
2634        movups STATE, (OUTP)
2635        sub $16, LEN
2636        add $16, INP
2637        add $16, OUTP
2638        cmp $16, LEN
2639        jge .Lctr_enc_loop1
2640.Lctr_enc_ret:
2641        movups IV, (IVP)
2642.Lctr_enc_just_ret:
2643        ret
2644ENDPROC(aesni_ctr_enc)
2645
2646/*
2647 * _aesni_gf128mul_x_ble:               internal ABI
2648 *      Multiply in GF(2^128) for XTS IVs
2649 * input:
2650 *      IV:     current IV
2651 *      GF128MUL_MASK == mask with 0x87 and 0x01
2652 * output:
2653 *      IV:     next IV
2654 * changed:
2655 *      CTR:    == temporary value
2656 */
2657#define _aesni_gf128mul_x_ble() \
2658        pshufd $0x13, IV, CTR; \
2659        paddq IV, IV; \
2660        psrad $31, CTR; \
2661        pand GF128MUL_MASK, CTR; \
2662        pxor CTR, IV;
2663
2664/*
2665 * void aesni_xts_crypt8(struct crypto_aes_ctx *ctx, const u8 *dst, u8 *src,
2666 *                       bool enc, u8 *iv)
2667 */
2668ENTRY(aesni_xts_crypt8)
2669        cmpb $0, %cl
2670        movl $0, %ecx
2671        movl $240, %r10d
2672        leaq _aesni_enc4, %r11
2673        leaq _aesni_dec4, %rax
2674        cmovel %r10d, %ecx
2675        cmoveq %rax, %r11
2676
2677        movdqa .Lgf128mul_x_ble_mask, GF128MUL_MASK
2678        movups (IVP), IV
2679
2680        mov 480(KEYP), KLEN
2681        addq %rcx, KEYP
2682
2683        movdqa IV, STATE1
2684        movdqu 0x00(INP), INC
2685        pxor INC, STATE1
2686        movdqu IV, 0x00(OUTP)
2687
2688        _aesni_gf128mul_x_ble()
2689        movdqa IV, STATE2
2690        movdqu 0x10(INP), INC
2691        pxor INC, STATE2
2692        movdqu IV, 0x10(OUTP)
2693
2694        _aesni_gf128mul_x_ble()
2695        movdqa IV, STATE3
2696        movdqu 0x20(INP), INC
2697        pxor INC, STATE3
2698        movdqu IV, 0x20(OUTP)
2699
2700        _aesni_gf128mul_x_ble()
2701        movdqa IV, STATE4
2702        movdqu 0x30(INP), INC
2703        pxor INC, STATE4
2704        movdqu IV, 0x30(OUTP)
2705
2706        call *%r11
2707
2708        movdqu 0x00(OUTP), INC
2709        pxor INC, STATE1
2710        movdqu STATE1, 0x00(OUTP)
2711
2712        _aesni_gf128mul_x_ble()
2713        movdqa IV, STATE1
2714        movdqu 0x40(INP), INC
2715        pxor INC, STATE1
2716        movdqu IV, 0x40(OUTP)
2717
2718        movdqu 0x10(OUTP), INC
2719        pxor INC, STATE2
2720        movdqu STATE2, 0x10(OUTP)
2721
2722        _aesni_gf128mul_x_ble()
2723        movdqa IV, STATE2
2724        movdqu 0x50(INP), INC
2725        pxor INC, STATE2
2726        movdqu IV, 0x50(OUTP)
2727
2728        movdqu 0x20(OUTP), INC
2729        pxor INC, STATE3
2730        movdqu STATE3, 0x20(OUTP)
2731
2732        _aesni_gf128mul_x_ble()
2733        movdqa IV, STATE3
2734        movdqu 0x60(INP), INC
2735        pxor INC, STATE3
2736        movdqu IV, 0x60(OUTP)
2737
2738        movdqu 0x30(OUTP), INC
2739        pxor INC, STATE4
2740        movdqu STATE4, 0x30(OUTP)
2741
2742        _aesni_gf128mul_x_ble()
2743        movdqa IV, STATE4
2744        movdqu 0x70(INP), INC
2745        pxor INC, STATE4
2746        movdqu IV, 0x70(OUTP)
2747
2748        _aesni_gf128mul_x_ble()
2749        movups IV, (IVP)
2750
2751        call *%r11
2752
2753        movdqu 0x40(OUTP), INC
2754        pxor INC, STATE1
2755        movdqu STATE1, 0x40(OUTP)
2756
2757        movdqu 0x50(OUTP), INC
2758        pxor INC, STATE2
2759        movdqu STATE2, 0x50(OUTP)
2760
2761        movdqu 0x60(OUTP), INC
2762        pxor INC, STATE3
2763        movdqu STATE3, 0x60(OUTP)
2764
2765        movdqu 0x70(OUTP), INC
2766        pxor INC, STATE4
2767        movdqu STATE4, 0x70(OUTP)
2768
2769        ret
2770ENDPROC(aesni_xts_crypt8)
2771
2772#endif
2773