LXR linux/arch/x86/crypto/aesni-intel_avx-x86

   1########################################################################
   2# Copyright (c) 2013, Intel Corporation
   3#
   4# This software is available to you under a choice of one of two
   5# licenses.  You may choose to be licensed under the terms of the GNU
   6# General Public License (GPL) Version 2, available from the file
   7# COPYING in the main directory of this source tree, or the
   8# OpenIB.org BSD license below:
   9#
  10# Redistribution and use in source and binary forms, with or without
  11# modification, are permitted provided that the following conditions are
  12# met:
  13#
  14# * Redistributions of source code must retain the above copyright
  15#   notice, this list of conditions and the following disclaimer.
  16#
  17# * Redistributions in binary form must reproduce the above copyright
  18#   notice, this list of conditions and the following disclaimer in the
  19#   documentation and/or other materials provided with the
  20#   distribution.
  21#
  22# * Neither the name of the Intel Corporation nor the names of its
  23#   contributors may be used to endorse or promote products derived from
  24#   this software without specific prior written permission.
  25#
  26#
  27# THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY
  28# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  29# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  30# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR
  31# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  32# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  33# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES# LOSS OF USE, DATA, OR
  34# PROFITS# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  35# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  36# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  37# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  38########################################################################
  39##
  40## Authors:
  41##      Erdinc Ozturk <erdinc.ozturk@intel.com>
  42##      Vinodh Gopal <vinodh.gopal@intel.com>
  43##      James Guilford <james.guilford@intel.com>
  44##      Tim Chen <tim.c.chen@linux.intel.com>
  45##
  46## References:
  47##       This code was derived and highly optimized from the code described in paper:
  48##               Vinodh Gopal et. al. Optimized Galois-Counter-Mode Implementation
  49##                      on Intel Architecture Processors. August, 2010
  50##       The details of the implementation is explained in:
  51##               Erdinc Ozturk et. al. Enabling High-Performance Galois-Counter-Mode
  52##                      on Intel Architecture Processors. October, 2012.
  53##
  54## Assumptions:
  55##
  56##
  57##
  58## iv:
  59##       0                   1                   2                   3
  60##       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
  61##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  62##       |                             Salt  (From the SA)               |
  63##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  64##       |                     Initialization Vector                     |
  65##       |         (This is the sequence number from IPSec header)       |
  66##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  67##       |                              0x1                              |
  68##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  69##
  70##
  71##
  72## AAD:
  73##       AAD padded to 128 bits with 0
  74##       for example, assume AAD is a u32 vector
  75##
  76##       if AAD is 8 bytes:
  77##       AAD[3] = {A0, A1}#
  78##       padded AAD in xmm register = {A1 A0 0 0}
  79##
  80##       0                   1                   2                   3
  81##       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
  82##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  83##       |                               SPI (A1)                        |
  84##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  85##       |                     32-bit Sequence Number (A0)               |
  86##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  87##       |                              0x0                              |
  88##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  89##
  90##                                       AAD Format with 32-bit Sequence Number
  91##
  92##       if AAD is 12 bytes:
  93##       AAD[3] = {A0, A1, A2}#
  94##       padded AAD in xmm register = {A2 A1 A0 0}
  95##
  96##       0                   1                   2                   3
  97##       0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
  98##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  99##       |                               SPI (A2)                        |
 100##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 101##       |                 64-bit Extended Sequence Number {A1,A0}       |
 102##       |                                                               |
 103##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 104##       |                              0x0                              |
 105##       +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 106##
 107##        AAD Format with 64-bit Extended Sequence Number
 108##
 109##
 110## aadLen:
 111##       from the definition of the spec, aadLen can only be 8 or 12 bytes.
 112##       The code additionally supports aadLen of length 16 bytes.
 113##
 114## TLen:
 115##       from the definition of the spec, TLen can only be 8, 12 or 16 bytes.
 116##
 117## poly = x^128 + x^127 + x^126 + x^121 + 1
 118## throughout the code, one tab and two tab indentations are used. one tab is
 119## for GHASH part, two tabs is for AES part.
 120##
 121
 122#include <linux/linkage.h>
 123#include <asm/inst.h>
 124
 125# constants in mergeable sections, linker can reorder and merge
 126.section        .rodata.cst16.POLY, "aM", @progbits, 16
 127.align 16
 128POLY:            .octa     0xC2000000000000000000000000000001
 129
 130.section        .rodata.cst16.POLY2, "aM", @progbits, 16
 131.align 16
 132POLY2:           .octa     0xC20000000000000000000001C2000000
 133
 134.section        .rodata.cst16.TWOONE, "aM", @progbits, 16
 135.align 16
 136TWOONE:          .octa     0x00000001000000000000000000000001
 137
 138.section        .rodata.cst16.SHUF_MASK, "aM", @progbits, 16
 139.align 16
 140SHUF_MASK:       .octa     0x000102030405060708090A0B0C0D0E0F
 141
 142.section        .rodata.cst16.ONE, "aM", @progbits, 16
 143.align 16
 144ONE:             .octa     0x00000000000000000000000000000001
 145
 146.section        .rodata.cst16.ONEf, "aM", @progbits, 16
 147.align 16
 148ONEf:            .octa     0x01000000000000000000000000000000
 149
 150# order of these constants should not change.
 151# more specifically, ALL_F should follow SHIFT_MASK, and zero should follow ALL_F
 152.section        .rodata, "a", @progbits
 153.align 16
 154SHIFT_MASK:      .octa     0x0f0e0d0c0b0a09080706050403020100
 155ALL_F:           .octa     0xffffffffffffffffffffffffffffffff
 156                 .octa     0x00000000000000000000000000000000
 157
 158.section .rodata
 159.align 16
 160.type aad_shift_arr, @object
 161.size aad_shift_arr, 272
 162aad_shift_arr:
 163        .octa     0xffffffffffffffffffffffffffffffff
 164        .octa     0xffffffffffffffffffffffffffffff0C
 165        .octa     0xffffffffffffffffffffffffffff0D0C
 166        .octa     0xffffffffffffffffffffffffff0E0D0C
 167        .octa     0xffffffffffffffffffffffff0F0E0D0C
 168        .octa     0xffffffffffffffffffffff0C0B0A0908
 169        .octa     0xffffffffffffffffffff0D0C0B0A0908
 170        .octa     0xffffffffffffffffff0E0D0C0B0A0908
 171        .octa     0xffffffffffffffff0F0E0D0C0B0A0908
 172        .octa     0xffffffffffffff0C0B0A090807060504
 173        .octa     0xffffffffffff0D0C0B0A090807060504
 174        .octa     0xffffffffff0E0D0C0B0A090807060504
 175        .octa     0xffffffff0F0E0D0C0B0A090807060504
 176        .octa     0xffffff0C0B0A09080706050403020100
 177        .octa     0xffff0D0C0B0A09080706050403020100
 178        .octa     0xff0E0D0C0B0A09080706050403020100
 179        .octa     0x0F0E0D0C0B0A09080706050403020100
 180
 181
 182.text
 183
 184
 185##define the fields of the gcm aes context
 186#{
 187#        u8 expanded_keys[16*11] store expanded keys
 188#        u8 shifted_hkey_1[16]   store HashKey <<1 mod poly here
 189#        u8 shifted_hkey_2[16]   store HashKey^2 <<1 mod poly here
 190#        u8 shifted_hkey_3[16]   store HashKey^3 <<1 mod poly here
 191#        u8 shifted_hkey_4[16]   store HashKey^4 <<1 mod poly here
 192#        u8 shifted_hkey_5[16]   store HashKey^5 <<1 mod poly here
 193#        u8 shifted_hkey_6[16]   store HashKey^6 <<1 mod poly here
 194#        u8 shifted_hkey_7[16]   store HashKey^7 <<1 mod poly here
 195#        u8 shifted_hkey_8[16]   store HashKey^8 <<1 mod poly here
 196#        u8 shifted_hkey_1_k[16] store XOR HashKey <<1 mod poly here (for Karatsuba purposes)
 197#        u8 shifted_hkey_2_k[16] store XOR HashKey^2 <<1 mod poly here (for Karatsuba purposes)
 198#        u8 shifted_hkey_3_k[16] store XOR HashKey^3 <<1 mod poly here (for Karatsuba purposes)
 199#        u8 shifted_hkey_4_k[16] store XOR HashKey^4 <<1 mod poly here (for Karatsuba purposes)
 200#        u8 shifted_hkey_5_k[16] store XOR HashKey^5 <<1 mod poly here (for Karatsuba purposes)
 201#        u8 shifted_hkey_6_k[16] store XOR HashKey^6 <<1 mod poly here (for Karatsuba purposes)
 202#        u8 shifted_hkey_7_k[16] store XOR HashKey^7 <<1 mod poly here (for Karatsuba purposes)
 203#        u8 shifted_hkey_8_k[16] store XOR HashKey^8 <<1 mod poly here (for Karatsuba purposes)
 204#} gcm_ctx#
 205
 206HashKey        = 16*11   # store HashKey <<1 mod poly here
 207HashKey_2      = 16*12   # store HashKey^2 <<1 mod poly here
 208HashKey_3      = 16*13   # store HashKey^3 <<1 mod poly here
 209HashKey_4      = 16*14   # store HashKey^4 <<1 mod poly here
 210HashKey_5      = 16*15   # store HashKey^5 <<1 mod poly here
 211HashKey_6      = 16*16   # store HashKey^6 <<1 mod poly here
 212HashKey_7      = 16*17   # store HashKey^7 <<1 mod poly here
 213HashKey_8      = 16*18   # store HashKey^8 <<1 mod poly here
 214HashKey_k      = 16*19   # store XOR of HashKey <<1 mod poly here (for Karatsuba purposes)
 215HashKey_2_k    = 16*20   # store XOR of HashKey^2 <<1 mod poly here (for Karatsuba purposes)
 216HashKey_3_k    = 16*21   # store XOR of HashKey^3 <<1 mod poly here (for Karatsuba purposes)
 217HashKey_4_k    = 16*22   # store XOR of HashKey^4 <<1 mod poly here (for Karatsuba purposes)
 218HashKey_5_k    = 16*23   # store XOR of HashKey^5 <<1 mod poly here (for Karatsuba purposes)
 219HashKey_6_k    = 16*24   # store XOR of HashKey^6 <<1 mod poly here (for Karatsuba purposes)
 220HashKey_7_k    = 16*25   # store XOR of HashKey^7 <<1 mod poly here (for Karatsuba purposes)
 221HashKey_8_k    = 16*26   # store XOR of HashKey^8 <<1 mod poly here (for Karatsuba purposes)
 222
 223#define arg1 %rdi
 224#define arg2 %rsi
 225#define arg3 %rdx
 226#define arg4 %rcx
 227#define arg5 %r8
 228#define arg6 %r9
 229#define arg7 STACK_OFFSET+8*1(%r14)
 230#define arg8 STACK_OFFSET+8*2(%r14)
 231#define arg9 STACK_OFFSET+8*3(%r14)
 232
 233i = 0
 234j = 0
 235
 236out_order = 0
 237in_order = 1
 238DEC = 0
 239ENC = 1
 240
 241.macro define_reg r n
 242reg_\r = %xmm\n
 243.endm
 244
 245.macro setreg
 246.altmacro
 247define_reg i %i
 248define_reg j %j
 249.noaltmacro
 250.endm
 251
 252# need to push 4 registers into stack to maintain
 253STACK_OFFSET = 8*4
 254
 255TMP1 =   16*0    # Temporary storage for AAD
 256TMP2 =   16*1    # Temporary storage for AES State 2 (State 1 is stored in an XMM register)
 257TMP3 =   16*2    # Temporary storage for AES State 3
 258TMP4 =   16*3    # Temporary storage for AES State 4
 259TMP5 =   16*4    # Temporary storage for AES State 5
 260TMP6 =   16*5    # Temporary storage for AES State 6
 261TMP7 =   16*6    # Temporary storage for AES State 7
 262TMP8 =   16*7    # Temporary storage for AES State 8
 263
 264VARIABLE_OFFSET = 16*8
 265
 266################################
 267# Utility Macros
 268################################
 269
 270# Encryption of a single block
 271.macro ENCRYPT_SINGLE_BLOCK XMM0
 272                vpxor    (arg1), \XMM0, \XMM0
 273                i = 1
 274                setreg
 275.rep 9
 276                vaesenc  16*i(arg1), \XMM0, \XMM0
 277                i = (i+1)
 278                setreg
 279.endr
 280                vaesenclast 16*10(arg1), \XMM0, \XMM0
 281.endm
 282
 283#ifdef CONFIG_AS_AVX
 284###############################################################################
 285# GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
 286# Input: A and B (128-bits each, bit-reflected)
 287# Output: C = A*B*x mod poly, (i.e. >>1 )
 288# To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input
 289# GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly.
 290###############################################################################
 291.macro  GHASH_MUL_AVX GH HK T1 T2 T3 T4 T5
 292
 293        vpshufd         $0b01001110, \GH, \T2
 294        vpshufd         $0b01001110, \HK, \T3
 295        vpxor           \GH     , \T2, \T2      # T2 = (a1+a0)
 296        vpxor           \HK     , \T3, \T3      # T3 = (b1+b0)
 297
 298        vpclmulqdq      $0x11, \HK, \GH, \T1    # T1 = a1*b1
 299        vpclmulqdq      $0x00, \HK, \GH, \GH    # GH = a0*b0
 300        vpclmulqdq      $0x00, \T3, \T2, \T2    # T2 = (a1+a0)*(b1+b0)
 301        vpxor           \GH, \T2,\T2
 302        vpxor           \T1, \T2,\T2            # T2 = a0*b1+a1*b0
 303
 304        vpslldq         $8, \T2,\T3             # shift-L T3 2 DWs
 305        vpsrldq         $8, \T2,\T2             # shift-R T2 2 DWs
 306        vpxor           \T3, \GH, \GH
 307        vpxor           \T2, \T1, \T1           # <T1:GH> = GH x HK
 308
 309        #first phase of the reduction
 310        vpslld  $31, \GH, \T2                   # packed right shifting << 31
 311        vpslld  $30, \GH, \T3                   # packed right shifting shift << 30
 312        vpslld  $25, \GH, \T4                   # packed right shifting shift << 25
 313
 314        vpxor   \T3, \T2, \T2                   # xor the shifted versions
 315        vpxor   \T4, \T2, \T2
 316
 317        vpsrldq $4, \T2, \T5                    # shift-R T5 1 DW
 318
 319        vpslldq $12, \T2, \T2                   # shift-L T2 3 DWs
 320        vpxor   \T2, \GH, \GH                   # first phase of the reduction complete
 321
 322        #second phase of the reduction
 323
 324        vpsrld  $1,\GH, \T2                     # packed left shifting >> 1
 325        vpsrld  $2,\GH, \T3                     # packed left shifting >> 2
 326        vpsrld  $7,\GH, \T4                     # packed left shifting >> 7
 327        vpxor   \T3, \T2, \T2                   # xor the shifted versions
 328        vpxor   \T4, \T2, \T2
 329
 330        vpxor   \T5, \T2, \T2
 331        vpxor   \T2, \GH, \GH
 332        vpxor   \T1, \GH, \GH                   # the result is in GH
 333
 334
 335.endm
 336
 337.macro PRECOMPUTE_AVX HK T1 T2 T3 T4 T5 T6
 338
 339        # Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
 340        vmovdqa  \HK, \T5
 341
 342        vpshufd  $0b01001110, \T5, \T1
 343        vpxor    \T5, \T1, \T1
 344        vmovdqa  \T1, HashKey_k(arg1)
 345
 346        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^2<<1 mod poly
 347        vmovdqa  \T5, HashKey_2(arg1)                    #  [HashKey_2] = HashKey^2<<1 mod poly
 348        vpshufd  $0b01001110, \T5, \T1
 349        vpxor    \T5, \T1, \T1
 350        vmovdqa  \T1, HashKey_2_k(arg1)
 351
 352        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^3<<1 mod poly
 353        vmovdqa  \T5, HashKey_3(arg1)
 354        vpshufd  $0b01001110, \T5, \T1
 355        vpxor    \T5, \T1, \T1
 356        vmovdqa  \T1, HashKey_3_k(arg1)
 357
 358        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^4<<1 mod poly
 359        vmovdqa  \T5, HashKey_4(arg1)
 360        vpshufd  $0b01001110, \T5, \T1
 361        vpxor    \T5, \T1, \T1
 362        vmovdqa  \T1, HashKey_4_k(arg1)
 363
 364        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^5<<1 mod poly
 365        vmovdqa  \T5, HashKey_5(arg1)
 366        vpshufd  $0b01001110, \T5, \T1
 367        vpxor    \T5, \T1, \T1
 368        vmovdqa  \T1, HashKey_5_k(arg1)
 369
 370        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^6<<1 mod poly
 371        vmovdqa  \T5, HashKey_6(arg1)
 372        vpshufd  $0b01001110, \T5, \T1
 373        vpxor    \T5, \T1, \T1
 374        vmovdqa  \T1, HashKey_6_k(arg1)
 375
 376        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^7<<1 mod poly
 377        vmovdqa  \T5, HashKey_7(arg1)
 378        vpshufd  $0b01001110, \T5, \T1
 379        vpxor    \T5, \T1, \T1
 380        vmovdqa  \T1, HashKey_7_k(arg1)
 381
 382        GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2  #  T5 = HashKey^8<<1 mod poly
 383        vmovdqa  \T5, HashKey_8(arg1)
 384        vpshufd  $0b01001110, \T5, \T1
 385        vpxor    \T5, \T1, \T1
 386        vmovdqa  \T1, HashKey_8_k(arg1)
 387
 388.endm
 389
 390## if a = number of total plaintext bytes
 391## b = floor(a/16)
 392## num_initial_blocks = b mod 4#
 393## encrypt the initial num_initial_blocks blocks and apply ghash on the ciphertext
 394## r10, r11, r12, rax are clobbered
 395## arg1, arg2, arg3, r14 are used as a pointer only, not modified
 396
 397.macro INITIAL_BLOCKS_AVX num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC
 398        i = (8-\num_initial_blocks)
 399        j = 0
 400        setreg
 401
 402        mov     arg6, %r10                      # r10 = AAD
 403        mov     arg7, %r12                      # r12 = aadLen
 404
 405
 406        mov     %r12, %r11
 407
 408        vpxor   reg_j, reg_j, reg_j
 409        vpxor   reg_i, reg_i, reg_i
 410        cmp     $16, %r11
 411        jl      _get_AAD_rest8\@
 412_get_AAD_blocks\@:
 413        vmovdqu (%r10), reg_i
 414        vpshufb SHUF_MASK(%rip), reg_i, reg_i
 415        vpxor   reg_i, reg_j, reg_j
 416        GHASH_MUL_AVX       reg_j, \T2, \T1, \T3, \T4, \T5, \T6
 417        add     $16, %r10
 418        sub     $16, %r12
 419        sub     $16, %r11
 420        cmp     $16, %r11
 421        jge     _get_AAD_blocks\@
 422        vmovdqu reg_j, reg_i
 423        cmp     $0, %r11
 424        je      _get_AAD_done\@
 425
 426        vpxor   reg_i, reg_i, reg_i
 427
 428        /* read the last <16B of AAD. since we have at least 4B of
 429        data right after the AAD (the ICV, and maybe some CT), we can
 430        read 4B/8B blocks safely, and then get rid of the extra stuff */
 431_get_AAD_rest8\@:
 432        cmp     $4, %r11
 433        jle     _get_AAD_rest4\@
 434        movq    (%r10), \T1
 435        add     $8, %r10
 436        sub     $8, %r11
 437        vpslldq $8, \T1, \T1
 438        vpsrldq $8, reg_i, reg_i
 439        vpxor   \T1, reg_i, reg_i
 440        jmp     _get_AAD_rest8\@
 441_get_AAD_rest4\@:
 442        cmp     $0, %r11
 443        jle      _get_AAD_rest0\@
 444        mov     (%r10), %eax
 445        movq    %rax, \T1
 446        add     $4, %r10
 447        sub     $4, %r11
 448        vpslldq $12, \T1, \T1
 449        vpsrldq $4, reg_i, reg_i
 450        vpxor   \T1, reg_i, reg_i
 451_get_AAD_rest0\@:
 452        /* finalize: shift out the extra bytes we read, and align
 453        left. since pslldq can only shift by an immediate, we use
 454        vpshufb and an array of shuffle masks */
 455        movq    %r12, %r11
 456        salq    $4, %r11
 457        movdqu  aad_shift_arr(%r11), \T1
 458        vpshufb \T1, reg_i, reg_i
 459_get_AAD_rest_final\@:
 460        vpshufb SHUF_MASK(%rip), reg_i, reg_i
 461        vpxor   reg_j, reg_i, reg_i
 462        GHASH_MUL_AVX       reg_i, \T2, \T1, \T3, \T4, \T5, \T6
 463
 464_get_AAD_done\@:
 465        # initialize the data pointer offset as zero
 466        xor     %r11, %r11
 467
 468        # start AES for num_initial_blocks blocks
 469        mov     arg5, %rax                     # rax = *Y0
 470        vmovdqu (%rax), \CTR                   # CTR = Y0
 471        vpshufb SHUF_MASK(%rip), \CTR, \CTR
 472
 473
 474        i = (9-\num_initial_blocks)
 475        setreg
 476.rep \num_initial_blocks
 477                vpaddd  ONE(%rip), \CTR, \CTR           # INCR Y0
 478                vmovdqa \CTR, reg_i
 479                vpshufb SHUF_MASK(%rip), reg_i, reg_i   # perform a 16Byte swap
 480        i = (i+1)
 481        setreg
 482.endr
 483
 484        vmovdqa  (arg1), \T_key
 485        i = (9-\num_initial_blocks)
 486        setreg
 487.rep \num_initial_blocks
 488                vpxor   \T_key, reg_i, reg_i
 489        i = (i+1)
 490        setreg
 491.endr
 492
 493        j = 1
 494        setreg
 495.rep 9
 496        vmovdqa  16*j(arg1), \T_key
 497        i = (9-\num_initial_blocks)
 498        setreg
 499.rep \num_initial_blocks
 500        vaesenc \T_key, reg_i, reg_i
 501        i = (i+1)
 502        setreg
 503.endr
 504
 505        j = (j+1)
 506        setreg
 507.endr
 508
 509
 510        vmovdqa  16*10(arg1), \T_key
 511        i = (9-\num_initial_blocks)
 512        setreg
 513.rep \num_initial_blocks
 514        vaesenclast      \T_key, reg_i, reg_i
 515        i = (i+1)
 516        setreg
 517.endr
 518
 519        i = (9-\num_initial_blocks)
 520        setreg
 521.rep \num_initial_blocks
 522                vmovdqu (arg3, %r11), \T1
 523                vpxor   \T1, reg_i, reg_i
 524                vmovdqu reg_i, (arg2 , %r11)           # write back ciphertext for num_initial_blocks blocks
 525                add     $16, %r11
 526.if  \ENC_DEC == DEC
 527                vmovdqa \T1, reg_i
 528.endif
 529                vpshufb SHUF_MASK(%rip), reg_i, reg_i  # prepare ciphertext for GHASH computations
 530        i = (i+1)
 531        setreg
 532.endr
 533
 534
 535        i = (8-\num_initial_blocks)
 536        j = (9-\num_initial_blocks)
 537        setreg
 538
 539.rep \num_initial_blocks
 540        vpxor    reg_i, reg_j, reg_j
 541        GHASH_MUL_AVX       reg_j, \T2, \T1, \T3, \T4, \T5, \T6 # apply GHASH on num_initial_blocks blocks
 542        i = (i+1)
 543        j = (j+1)
 544        setreg
 545.endr
 546        # XMM8 has the combined result here
 547
 548        vmovdqa  \XMM8, TMP1(%rsp)
 549        vmovdqa  \XMM8, \T3
 550
 551        cmp     $128, %r13
 552        jl      _initial_blocks_done\@                  # no need for precomputed constants
 553
 554###############################################################################
 555# Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
 556                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
 557                vmovdqa  \CTR, \XMM1
 558                vpshufb  SHUF_MASK(%rip), \XMM1, \XMM1  # perform a 16Byte swap
 559
 560                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
 561                vmovdqa  \CTR, \XMM2
 562                vpshufb  SHUF_MASK(%rip), \XMM2, \XMM2  # perform a 16Byte swap
 563
 564                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
 565                vmovdqa  \CTR, \XMM3
 566                vpshufb  SHUF_MASK(%rip), \XMM3, \XMM3  # perform a 16Byte swap
 567
 568                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
 569                vmovdqa  \CTR, \XMM4
 570                vpshufb  SHUF_MASK(%rip), \XMM4, \XMM4  # perform a 16Byte swap
 571
 572                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
 573                vmovdqa  \CTR, \XMM5
 574                vpshufb  SHUF_MASK(%rip), \XMM5, \XMM5  # perform a 16Byte swap
 575
 576                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
 577                vmovdqa  \CTR, \XMM6
 578                vpshufb  SHUF_MASK(%rip), \XMM6, \XMM6  # perform a 16Byte swap
 579
 580                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
 581                vmovdqa  \CTR, \XMM7
 582                vpshufb  SHUF_MASK(%rip), \XMM7, \XMM7  # perform a 16Byte swap
 583
 584                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
 585                vmovdqa  \CTR, \XMM8
 586                vpshufb  SHUF_MASK(%rip), \XMM8, \XMM8  # perform a 16Byte swap
 587
 588                vmovdqa  (arg1), \T_key
 589                vpxor    \T_key, \XMM1, \XMM1
 590                vpxor    \T_key, \XMM2, \XMM2
 591                vpxor    \T_key, \XMM3, \XMM3
 592                vpxor    \T_key, \XMM4, \XMM4
 593                vpxor    \T_key, \XMM5, \XMM5
 594                vpxor    \T_key, \XMM6, \XMM6
 595                vpxor    \T_key, \XMM7, \XMM7
 596                vpxor    \T_key, \XMM8, \XMM8
 597
 598                i = 1
 599                setreg
 600.rep    9       # do 9 rounds
 601                vmovdqa  16*i(arg1), \T_key
 602                vaesenc  \T_key, \XMM1, \XMM1
 603                vaesenc  \T_key, \XMM2, \XMM2
 604                vaesenc  \T_key, \XMM3, \XMM3
 605                vaesenc  \T_key, \XMM4, \XMM4
 606                vaesenc  \T_key, \XMM5, \XMM5
 607                vaesenc  \T_key, \XMM6, \XMM6
 608                vaesenc  \T_key, \XMM7, \XMM7
 609                vaesenc  \T_key, \XMM8, \XMM8
 610                i = (i+1)
 611                setreg
 612.endr
 613
 614
 615                vmovdqa  16*i(arg1), \T_key
 616                vaesenclast  \T_key, \XMM1, \XMM1
 617                vaesenclast  \T_key, \XMM2, \XMM2
 618                vaesenclast  \T_key, \XMM3, \XMM3
 619                vaesenclast  \T_key, \XMM4, \XMM4
 620                vaesenclast  \T_key, \XMM5, \XMM5
 621                vaesenclast  \T_key, \XMM6, \XMM6
 622                vaesenclast  \T_key, \XMM7, \XMM7
 623                vaesenclast  \T_key, \XMM8, \XMM8
 624
 625                vmovdqu  (arg3, %r11), \T1
 626                vpxor    \T1, \XMM1, \XMM1
 627                vmovdqu  \XMM1, (arg2 , %r11)
 628                .if   \ENC_DEC == DEC
 629                vmovdqa  \T1, \XMM1
 630                .endif
 631
 632                vmovdqu  16*1(arg3, %r11), \T1
 633                vpxor    \T1, \XMM2, \XMM2
 634                vmovdqu  \XMM2, 16*1(arg2 , %r11)
 635                .if   \ENC_DEC == DEC
 636                vmovdqa  \T1, \XMM2
 637                .endif
 638
 639                vmovdqu  16*2(arg3, %r11), \T1
 640                vpxor    \T1, \XMM3, \XMM3
 641                vmovdqu  \XMM3, 16*2(arg2 , %r11)
 642                .if   \ENC_DEC == DEC
 643                vmovdqa  \T1, \XMM3
 644                .endif
 645
 646                vmovdqu  16*3(arg3, %r11), \T1
 647                vpxor    \T1, \XMM4, \XMM4
 648                vmovdqu  \XMM4, 16*3(arg2 , %r11)
 649                .if   \ENC_DEC == DEC
 650                vmovdqa  \T1, \XMM4
 651                .endif
 652
 653                vmovdqu  16*4(arg3, %r11), \T1
 654                vpxor    \T1, \XMM5, \XMM5
 655                vmovdqu  \XMM5, 16*4(arg2 , %r11)
 656                .if   \ENC_DEC == DEC
 657                vmovdqa  \T1, \XMM5
 658                .endif
 659
 660                vmovdqu  16*5(arg3, %r11), \T1
 661                vpxor    \T1, \XMM6, \XMM6
 662                vmovdqu  \XMM6, 16*5(arg2 , %r11)
 663                .if   \ENC_DEC == DEC
 664                vmovdqa  \T1, \XMM6
 665                .endif
 666
 667                vmovdqu  16*6(arg3, %r11), \T1
 668                vpxor    \T1, \XMM7, \XMM7
 669                vmovdqu  \XMM7, 16*6(arg2 , %r11)
 670                .if   \ENC_DEC == DEC
 671                vmovdqa  \T1, \XMM7
 672                .endif
 673
 674                vmovdqu  16*7(arg3, %r11), \T1
 675                vpxor    \T1, \XMM8, \XMM8
 676                vmovdqu  \XMM8, 16*7(arg2 , %r11)
 677                .if   \ENC_DEC == DEC
 678                vmovdqa  \T1, \XMM8
 679                .endif
 680
 681                add     $128, %r11
 682
 683                vpshufb  SHUF_MASK(%rip), \XMM1, \XMM1     # perform a 16Byte swap
 684                vpxor    TMP1(%rsp), \XMM1, \XMM1          # combine GHASHed value with the corresponding ciphertext
 685                vpshufb  SHUF_MASK(%rip), \XMM2, \XMM2     # perform a 16Byte swap
 686                vpshufb  SHUF_MASK(%rip), \XMM3, \XMM3     # perform a 16Byte swap
 687                vpshufb  SHUF_MASK(%rip), \XMM4, \XMM4     # perform a 16Byte swap
 688                vpshufb  SHUF_MASK(%rip), \XMM5, \XMM5     # perform a 16Byte swap
 689                vpshufb  SHUF_MASK(%rip), \XMM6, \XMM6     # perform a 16Byte swap
 690                vpshufb  SHUF_MASK(%rip), \XMM7, \XMM7     # perform a 16Byte swap
 691                vpshufb  SHUF_MASK(%rip), \XMM8, \XMM8     # perform a 16Byte swap
 692
 693###############################################################################
 694
 695_initial_blocks_done\@:
 696
 697.endm
 698
 699# encrypt 8 blocks at a time
 700# ghash the 8 previously encrypted ciphertext blocks
 701# arg1, arg2, arg3 are used as pointers only, not modified
 702# r11 is the data offset value
 703.macro GHASH_8_ENCRYPT_8_PARALLEL_AVX T1 T2 T3 T4 T5 T6 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T7 loop_idx ENC_DEC
 704
 705        vmovdqa \XMM1, \T2
 706        vmovdqa \XMM2, TMP2(%rsp)
 707        vmovdqa \XMM3, TMP3(%rsp)
 708        vmovdqa \XMM4, TMP4(%rsp)
 709        vmovdqa \XMM5, TMP5(%rsp)
 710        vmovdqa \XMM6, TMP6(%rsp)
 711        vmovdqa \XMM7, TMP7(%rsp)
 712        vmovdqa \XMM8, TMP8(%rsp)
 713
 714.if \loop_idx == in_order
 715                vpaddd  ONE(%rip), \CTR, \XMM1           # INCR CNT
 716                vpaddd  ONE(%rip), \XMM1, \XMM2
 717                vpaddd  ONE(%rip), \XMM2, \XMM3
 718                vpaddd  ONE(%rip), \XMM3, \XMM4
 719                vpaddd  ONE(%rip), \XMM4, \XMM5
 720                vpaddd  ONE(%rip), \XMM5, \XMM6
 721                vpaddd  ONE(%rip), \XMM6, \XMM7
 722                vpaddd  ONE(%rip), \XMM7, \XMM8
 723                vmovdqa \XMM8, \CTR
 724
 725                vpshufb SHUF_MASK(%rip), \XMM1, \XMM1    # perform a 16Byte swap
 726                vpshufb SHUF_MASK(%rip), \XMM2, \XMM2    # perform a 16Byte swap
 727                vpshufb SHUF_MASK(%rip), \XMM3, \XMM3    # perform a 16Byte swap
 728                vpshufb SHUF_MASK(%rip), \XMM4, \XMM4    # perform a 16Byte swap
 729                vpshufb SHUF_MASK(%rip), \XMM5, \XMM5    # perform a 16Byte swap
 730                vpshufb SHUF_MASK(%rip), \XMM6, \XMM6    # perform a 16Byte swap
 731                vpshufb SHUF_MASK(%rip), \XMM7, \XMM7    # perform a 16Byte swap
 732                vpshufb SHUF_MASK(%rip), \XMM8, \XMM8    # perform a 16Byte swap
 733.else
 734                vpaddd  ONEf(%rip), \CTR, \XMM1           # INCR CNT
 735                vpaddd  ONEf(%rip), \XMM1, \XMM2
 736                vpaddd  ONEf(%rip), \XMM2, \XMM3
 737                vpaddd  ONEf(%rip), \XMM3, \XMM4
 738                vpaddd  ONEf(%rip), \XMM4, \XMM5
 739                vpaddd  ONEf(%rip), \XMM5, \XMM6
 740                vpaddd  ONEf(%rip), \XMM6, \XMM7
 741                vpaddd  ONEf(%rip), \XMM7, \XMM8
 742                vmovdqa \XMM8, \CTR
 743.endif
 744
 745
 746        #######################################################################
 747
 748                vmovdqu (arg1), \T1
 749                vpxor   \T1, \XMM1, \XMM1
 750                vpxor   \T1, \XMM2, \XMM2
 751                vpxor   \T1, \XMM3, \XMM3
 752                vpxor   \T1, \XMM4, \XMM4
 753                vpxor   \T1, \XMM5, \XMM5
 754                vpxor   \T1, \XMM6, \XMM6
 755                vpxor   \T1, \XMM7, \XMM7
 756                vpxor   \T1, \XMM8, \XMM8
 757
 758        #######################################################################
 759
 760
 761
 762
 763
 764                vmovdqu 16*1(arg1), \T1
 765                vaesenc \T1, \XMM1, \XMM1
 766                vaesenc \T1, \XMM2, \XMM2
 767                vaesenc \T1, \XMM3, \XMM3
 768                vaesenc \T1, \XMM4, \XMM4
 769                vaesenc \T1, \XMM5, \XMM5
 770                vaesenc \T1, \XMM6, \XMM6
 771                vaesenc \T1, \XMM7, \XMM7
 772                vaesenc \T1, \XMM8, \XMM8
 773
 774                vmovdqu 16*2(arg1), \T1
 775                vaesenc \T1, \XMM1, \XMM1
 776                vaesenc \T1, \XMM2, \XMM2
 777                vaesenc \T1, \XMM3, \XMM3
 778                vaesenc \T1, \XMM4, \XMM4
 779                vaesenc \T1, \XMM5, \XMM5
 780                vaesenc \T1, \XMM6, \XMM6
 781                vaesenc \T1, \XMM7, \XMM7
 782                vaesenc \T1, \XMM8, \XMM8
 783
 784
 785        #######################################################################
 786
 787        vmovdqa         HashKey_8(arg1), \T5
 788        vpclmulqdq      $0x11, \T5, \T2, \T4             # T4 = a1*b1
 789        vpclmulqdq      $0x00, \T5, \T2, \T7             # T7 = a0*b0
 790
 791        vpshufd         $0b01001110, \T2, \T6
 792        vpxor           \T2, \T6, \T6
 793
 794        vmovdqa         HashKey_8_k(arg1), \T5
 795        vpclmulqdq      $0x00, \T5, \T6, \T6
 796
 797                vmovdqu 16*3(arg1), \T1
 798                vaesenc \T1, \XMM1, \XMM1
 799                vaesenc \T1, \XMM2, \XMM2
 800                vaesenc \T1, \XMM3, \XMM3
 801                vaesenc \T1, \XMM4, \XMM4
 802                vaesenc \T1, \XMM5, \XMM5
 803                vaesenc \T1, \XMM6, \XMM6
 804                vaesenc \T1, \XMM7, \XMM7
 805                vaesenc \T1, \XMM8, \XMM8
 806
 807        vmovdqa         TMP2(%rsp), \T1
 808        vmovdqa         HashKey_7(arg1), \T5
 809        vpclmulqdq      $0x11, \T5, \T1, \T3
 810        vpxor           \T3, \T4, \T4
 811        vpclmulqdq      $0x00, \T5, \T1, \T3
 812        vpxor           \T3, \T7, \T7
 813
 814        vpshufd         $0b01001110, \T1, \T3
 815        vpxor           \T1, \T3, \T3
 816        vmovdqa         HashKey_7_k(arg1), \T5
 817        vpclmulqdq      $0x10, \T5, \T3, \T3
 818        vpxor           \T3, \T6, \T6
 819
 820                vmovdqu 16*4(arg1), \T1
 821                vaesenc \T1, \XMM1, \XMM1
 822                vaesenc \T1, \XMM2, \XMM2
 823                vaesenc \T1, \XMM3, \XMM3
 824                vaesenc \T1, \XMM4, \XMM4
 825                vaesenc \T1, \XMM5, \XMM5
 826                vaesenc \T1, \XMM6, \XMM6
 827                vaesenc \T1, \XMM7, \XMM7
 828                vaesenc \T1, \XMM8, \XMM8
 829
 830        #######################################################################
 831
 832        vmovdqa         TMP3(%rsp), \T1
 833        vmovdqa         HashKey_6(arg1), \T5
 834        vpclmulqdq      $0x11, \T5, \T1, \T3
 835        vpxor           \T3, \T4, \T4
 836        vpclmulqdq      $0x00, \T5, \T1, \T3
 837        vpxor           \T3, \T7, \T7
 838
 839        vpshufd         $0b01001110, \T1, \T3
 840        vpxor           \T1, \T3, \T3
 841        vmovdqa         HashKey_6_k(arg1), \T5
 842        vpclmulqdq      $0x10, \T5, \T3, \T3
 843        vpxor           \T3, \T6, \T6
 844
 845                vmovdqu 16*5(arg1), \T1
 846                vaesenc \T1, \XMM1, \XMM1
 847                vaesenc \T1, \XMM2, \XMM2
 848                vaesenc \T1, \XMM3, \XMM3
 849                vaesenc \T1, \XMM4, \XMM4
 850                vaesenc \T1, \XMM5, \XMM5
 851                vaesenc \T1, \XMM6, \XMM6
 852                vaesenc \T1, \XMM7, \XMM7
 853                vaesenc \T1, \XMM8, \XMM8
 854
 855        vmovdqa         TMP4(%rsp), \T1
 856        vmovdqa         HashKey_5(arg1), \T5
 857        vpclmulqdq      $0x11, \T5, \T1, \T3
 858        vpxor           \T3, \T4, \T4
 859        vpclmulqdq      $0x00, \T5, \T1, \T3
 860        vpxor           \T3, \T7, \T7
 861
 862        vpshufd         $0b01001110, \T1, \T3
 863        vpxor           \T1, \T3, \T3
 864        vmovdqa         HashKey_5_k(arg1), \T5
 865        vpclmulqdq      $0x10, \T5, \T3, \T3
 866        vpxor           \T3, \T6, \T6
 867
 868                vmovdqu 16*6(arg1), \T1
 869                vaesenc \T1, \XMM1, \XMM1
 870                vaesenc \T1, \XMM2, \XMM2
 871                vaesenc \T1, \XMM3, \XMM3
 872                vaesenc \T1, \XMM4, \XMM4
 873                vaesenc \T1, \XMM5, \XMM5
 874                vaesenc \T1, \XMM6, \XMM6
 875                vaesenc \T1, \XMM7, \XMM7
 876                vaesenc \T1, \XMM8, \XMM8
 877
 878
 879        vmovdqa         TMP5(%rsp), \T1
 880        vmovdqa         HashKey_4(arg1), \T5
 881        vpclmulqdq      $0x11, \T5, \T1, \T3
 882        vpxor           \T3, \T4, \T4
 883        vpclmulqdq      $0x00, \T5, \T1, \T3
 884        vpxor           \T3, \T7, \T7
 885
 886        vpshufd         $0b01001110, \T1, \T3
 887        vpxor           \T1, \T3, \T3
 888        vmovdqa         HashKey_4_k(arg1), \T5
 889        vpclmulqdq      $0x10, \T5, \T3, \T3
 890        vpxor           \T3, \T6, \T6
 891
 892                vmovdqu 16*7(arg1), \T1
 893                vaesenc \T1, \XMM1, \XMM1
 894                vaesenc \T1, \XMM2, \XMM2
 895                vaesenc \T1, \XMM3, \XMM3
 896                vaesenc \T1, \XMM4, \XMM4
 897                vaesenc \T1, \XMM5, \XMM5
 898                vaesenc \T1, \XMM6, \XMM6
 899                vaesenc \T1, \XMM7, \XMM7
 900                vaesenc \T1, \XMM8, \XMM8
 901
 902        vmovdqa         TMP6(%rsp), \T1
 903        vmovdqa         HashKey_3(arg1), \T5
 904        vpclmulqdq      $0x11, \T5, \T1, \T3
 905        vpxor           \T3, \T4, \T4
 906        vpclmulqdq      $0x00, \T5, \T1, \T3
 907        vpxor           \T3, \T7, \T7
 908
 909        vpshufd         $0b01001110, \T1, \T3
 910        vpxor           \T1, \T3, \T3
 911        vmovdqa         HashKey_3_k(arg1), \T5
 912        vpclmulqdq      $0x10, \T5, \T3, \T3
 913        vpxor           \T3, \T6, \T6
 914
 915
 916                vmovdqu 16*8(arg1), \T1
 917                vaesenc \T1, \XMM1, \XMM1
 918                vaesenc \T1, \XMM2, \XMM2
 919                vaesenc \T1, \XMM3, \XMM3
 920                vaesenc \T1, \XMM4, \XMM4
 921                vaesenc \T1, \XMM5, \XMM5
 922                vaesenc \T1, \XMM6, \XMM6
 923                vaesenc \T1, \XMM7, \XMM7
 924                vaesenc \T1, \XMM8, \XMM8
 925
 926        vmovdqa         TMP7(%rsp), \T1
 927        vmovdqa         HashKey_2(arg1), \T5
 928        vpclmulqdq      $0x11, \T5, \T1, \T3
 929        vpxor           \T3, \T4, \T4
 930        vpclmulqdq      $0x00, \T5, \T1, \T3
 931        vpxor           \T3, \T7, \T7
 932
 933        vpshufd         $0b01001110, \T1, \T3
 934        vpxor           \T1, \T3, \T3
 935        vmovdqa         HashKey_2_k(arg1), \T5
 936        vpclmulqdq      $0x10, \T5, \T3, \T3
 937        vpxor           \T3, \T6, \T6
 938
 939        #######################################################################
 940
 941                vmovdqu 16*9(arg1), \T5
 942                vaesenc \T5, \XMM1, \XMM1
 943                vaesenc \T5, \XMM2, \XMM2
 944                vaesenc \T5, \XMM3, \XMM3
 945                vaesenc \T5, \XMM4, \XMM4
 946                vaesenc \T5, \XMM5, \XMM5
 947                vaesenc \T5, \XMM6, \XMM6
 948                vaesenc \T5, \XMM7, \XMM7
 949                vaesenc \T5, \XMM8, \XMM8
 950
 951        vmovdqa         TMP8(%rsp), \T1
 952        vmovdqa         HashKey(arg1), \T5
 953        vpclmulqdq      $0x11, \T5, \T1, \T3
 954        vpxor           \T3, \T4, \T4
 955        vpclmulqdq      $0x00, \T5, \T1, \T3
 956        vpxor           \T3, \T7, \T7
 957
 958        vpshufd         $0b01001110, \T1, \T3
 959        vpxor           \T1, \T3, \T3
 960        vmovdqa         HashKey_k(arg1), \T5
 961        vpclmulqdq      $0x10, \T5, \T3, \T3
 962        vpxor           \T3, \T6, \T6
 963
 964        vpxor           \T4, \T6, \T6
 965        vpxor           \T7, \T6, \T6
 966
 967                vmovdqu 16*10(arg1), \T5
 968
 969        i = 0
 970        j = 1
 971        setreg
 972.rep 8
 973                vpxor   16*i(arg3, %r11), \T5, \T2
 974                .if \ENC_DEC == ENC
 975                vaesenclast     \T2, reg_j, reg_j
 976                .else
 977                vaesenclast     \T2, reg_j, \T3
 978                vmovdqu 16*i(arg3, %r11), reg_j
 979                vmovdqu \T3, 16*i(arg2, %r11)
 980                .endif
 981        i = (i+1)
 982        j = (j+1)
 983        setreg
 984.endr
 985        #######################################################################
 986
 987
 988        vpslldq $8, \T6, \T3                            # shift-L T3 2 DWs
 989        vpsrldq $8, \T6, \T6                            # shift-R T2 2 DWs
 990        vpxor   \T3, \T7, \T7
 991        vpxor   \T4, \T6, \T6                           # accumulate the results in T6:T7
 992
 993
 994
 995        #######################################################################
 996        #first phase of the reduction
 997        #######################################################################
 998        vpslld  $31, \T7, \T2                           # packed right shifting << 31
 999        vpslld  $30, \T7, \T3                           # packed right shifting shift << 30
1000        vpslld  $25, \T7, \T4                           # packed right shifting shift << 25

1001
1002        vpxor   \T3, \T2, \T2                           # xor the shifted versions
1003        vpxor   \T4, \T2, \T2
1004
1005        vpsrldq $4, \T2, \T1                            # shift-R T1 1 DW
1006
1007        vpslldq $12, \T2, \T2                           # shift-L T2 3 DWs
1008        vpxor   \T2, \T7, \T7                           # first phase of the reduction complete
1009        #######################################################################
1010                .if \ENC_DEC == ENC
1011                vmovdqu  \XMM1, 16*0(arg2,%r11)         # Write to the Ciphertext buffer
1012                vmovdqu  \XMM2, 16*1(arg2,%r11)         # Write to the Ciphertext buffer
1013                vmovdqu  \XMM3, 16*2(arg2,%r11)         # Write to the Ciphertext buffer
1014                vmovdqu  \XMM4, 16*3(arg2,%r11)         # Write to the Ciphertext buffer
1015                vmovdqu  \XMM5, 16*4(arg2,%r11)         # Write to the Ciphertext buffer
1016                vmovdqu  \XMM6, 16*5(arg2,%r11)         # Write to the Ciphertext buffer
1017                vmovdqu  \XMM7, 16*6(arg2,%r11)         # Write to the Ciphertext buffer
1018                vmovdqu  \XMM8, 16*7(arg2,%r11)         # Write to the Ciphertext buffer
1019                .endif
1020
1021        #######################################################################
1022        #second phase of the reduction
1023        vpsrld  $1, \T7, \T2                            # packed left shifting >> 1
1024        vpsrld  $2, \T7, \T3                            # packed left shifting >> 2
1025        vpsrld  $7, \T7, \T4                            # packed left shifting >> 7
1026        vpxor   \T3, \T2, \T2                           # xor the shifted versions
1027        vpxor   \T4, \T2, \T2
1028
1029        vpxor   \T1, \T2, \T2
1030        vpxor   \T2, \T7, \T7
1031        vpxor   \T7, \T6, \T6                           # the result is in T6
1032        #######################################################################
1033
1034                vpshufb SHUF_MASK(%rip), \XMM1, \XMM1   # perform a 16Byte swap
1035                vpshufb SHUF_MASK(%rip), \XMM2, \XMM2   # perform a 16Byte swap
1036                vpshufb SHUF_MASK(%rip), \XMM3, \XMM3   # perform a 16Byte swap
1037                vpshufb SHUF_MASK(%rip), \XMM4, \XMM4   # perform a 16Byte swap
1038                vpshufb SHUF_MASK(%rip), \XMM5, \XMM5   # perform a 16Byte swap
1039                vpshufb SHUF_MASK(%rip), \XMM6, \XMM6   # perform a 16Byte swap
1040                vpshufb SHUF_MASK(%rip), \XMM7, \XMM7   # perform a 16Byte swap
1041                vpshufb SHUF_MASK(%rip), \XMM8, \XMM8   # perform a 16Byte swap
1042
1043
1044        vpxor   \T6, \XMM1, \XMM1
1045
1046
1047
1048.endm
1049
1050
1051# GHASH the last 4 ciphertext blocks.
1052.macro  GHASH_LAST_8_AVX T1 T2 T3 T4 T5 T6 T7 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8
1053
1054        ## Karatsuba Method
1055
1056
1057        vpshufd         $0b01001110, \XMM1, \T2
1058        vpxor           \XMM1, \T2, \T2
1059        vmovdqa         HashKey_8(arg1), \T5
1060        vpclmulqdq      $0x11, \T5, \XMM1, \T6
1061        vpclmulqdq      $0x00, \T5, \XMM1, \T7
1062
1063        vmovdqa         HashKey_8_k(arg1), \T3
1064        vpclmulqdq      $0x00, \T3, \T2, \XMM1
1065
1066        ######################
1067
1068        vpshufd         $0b01001110, \XMM2, \T2
1069        vpxor           \XMM2, \T2, \T2
1070        vmovdqa         HashKey_7(arg1), \T5
1071        vpclmulqdq      $0x11, \T5, \XMM2, \T4
1072        vpxor           \T4, \T6, \T6
1073
1074        vpclmulqdq      $0x00, \T5, \XMM2, \T4
1075        vpxor           \T4, \T7, \T7
1076
1077        vmovdqa         HashKey_7_k(arg1), \T3
1078        vpclmulqdq      $0x00, \T3, \T2, \T2
1079        vpxor           \T2, \XMM1, \XMM1
1080
1081        ######################
1082
1083        vpshufd         $0b01001110, \XMM3, \T2
1084        vpxor           \XMM3, \T2, \T2
1085        vmovdqa         HashKey_6(arg1), \T5
1086        vpclmulqdq      $0x11, \T5, \XMM3, \T4
1087        vpxor           \T4, \T6, \T6
1088
1089        vpclmulqdq      $0x00, \T5, \XMM3, \T4
1090        vpxor           \T4, \T7, \T7
1091
1092        vmovdqa         HashKey_6_k(arg1), \T3
1093        vpclmulqdq      $0x00, \T3, \T2, \T2
1094        vpxor           \T2, \XMM1, \XMM1
1095
1096        ######################
1097
1098        vpshufd         $0b01001110, \XMM4, \T2
1099        vpxor           \XMM4, \T2, \T2
1100        vmovdqa         HashKey_5(arg1), \T5
1101        vpclmulqdq      $0x11, \T5, \XMM4, \T4
1102        vpxor           \T4, \T6, \T6
1103
1104        vpclmulqdq      $0x00, \T5, \XMM4, \T4
1105        vpxor           \T4, \T7, \T7
1106
1107        vmovdqa         HashKey_5_k(arg1), \T3
1108        vpclmulqdq      $0x00, \T3, \T2, \T2
1109        vpxor           \T2, \XMM1, \XMM1
1110
1111        ######################
1112
1113        vpshufd         $0b01001110, \XMM5, \T2
1114        vpxor           \XMM5, \T2, \T2
1115        vmovdqa         HashKey_4(arg1), \T5
1116        vpclmulqdq      $0x11, \T5, \XMM5, \T4
1117        vpxor           \T4, \T6, \T6
1118
1119        vpclmulqdq      $0x00, \T5, \XMM5, \T4
1120        vpxor           \T4, \T7, \T7
1121
1122        vmovdqa         HashKey_4_k(arg1), \T3
1123        vpclmulqdq      $0x00, \T3, \T2, \T2
1124        vpxor           \T2, \XMM1, \XMM1
1125
1126        ######################
1127
1128        vpshufd         $0b01001110, \XMM6, \T2
1129        vpxor           \XMM6, \T2, \T2
1130        vmovdqa         HashKey_3(arg1), \T5
1131        vpclmulqdq      $0x11, \T5, \XMM6, \T4
1132        vpxor           \T4, \T6, \T6
1133
1134        vpclmulqdq      $0x00, \T5, \XMM6, \T4
1135        vpxor           \T4, \T7, \T7
1136
1137        vmovdqa         HashKey_3_k(arg1), \T3
1138        vpclmulqdq      $0x00, \T3, \T2, \T2
1139        vpxor           \T2, \XMM1, \XMM1
1140
1141        ######################
1142
1143        vpshufd         $0b01001110, \XMM7, \T2
1144        vpxor           \XMM7, \T2, \T2
1145        vmovdqa         HashKey_2(arg1), \T5
1146        vpclmulqdq      $0x11, \T5, \XMM7, \T4
1147        vpxor           \T4, \T6, \T6
1148
1149        vpclmulqdq      $0x00, \T5, \XMM7, \T4
1150        vpxor           \T4, \T7, \T7
1151
1152        vmovdqa         HashKey_2_k(arg1), \T3
1153        vpclmulqdq      $0x00, \T3, \T2, \T2
1154        vpxor           \T2, \XMM1, \XMM1
1155
1156        ######################
1157
1158        vpshufd         $0b01001110, \XMM8, \T2
1159        vpxor           \XMM8, \T2, \T2
1160        vmovdqa         HashKey(arg1), \T5
1161        vpclmulqdq      $0x11, \T5, \XMM8, \T4
1162        vpxor           \T4, \T6, \T6
1163
1164        vpclmulqdq      $0x00, \T5, \XMM8, \T4
1165        vpxor           \T4, \T7, \T7
1166
1167        vmovdqa         HashKey_k(arg1), \T3
1168        vpclmulqdq      $0x00, \T3, \T2, \T2
1169
1170        vpxor           \T2, \XMM1, \XMM1
1171        vpxor           \T6, \XMM1, \XMM1
1172        vpxor           \T7, \XMM1, \T2
1173
1174
1175
1176
1177        vpslldq $8, \T2, \T4
1178        vpsrldq $8, \T2, \T2
1179
1180        vpxor   \T4, \T7, \T7
1181        vpxor   \T2, \T6, \T6   # <T6:T7> holds the result of
1182                                # the accumulated carry-less multiplications
1183
1184        #######################################################################
1185        #first phase of the reduction
1186        vpslld  $31, \T7, \T2   # packed right shifting << 31
1187        vpslld  $30, \T7, \T3   # packed right shifting shift << 30
1188        vpslld  $25, \T7, \T4   # packed right shifting shift << 25
1189
1190        vpxor   \T3, \T2, \T2   # xor the shifted versions
1191        vpxor   \T4, \T2, \T2
1192
1193        vpsrldq $4, \T2, \T1    # shift-R T1 1 DW
1194
1195        vpslldq $12, \T2, \T2   # shift-L T2 3 DWs
1196        vpxor   \T2, \T7, \T7   # first phase of the reduction complete
1197        #######################################################################
1198
1199
1200        #second phase of the reduction
1201        vpsrld  $1, \T7, \T2    # packed left shifting >> 1
1202        vpsrld  $2, \T7, \T3    # packed left shifting >> 2
1203        vpsrld  $7, \T7, \T4    # packed left shifting >> 7
1204        vpxor   \T3, \T2, \T2   # xor the shifted versions
1205        vpxor   \T4, \T2, \T2
1206
1207        vpxor   \T1, \T2, \T2
1208        vpxor   \T2, \T7, \T7
1209        vpxor   \T7, \T6, \T6   # the result is in T6
1210
1211.endm
1212
1213
1214# combined for GCM encrypt and decrypt functions
1215# clobbering all xmm registers
1216# clobbering r10, r11, r12, r13, r14, r15
1217.macro  GCM_ENC_DEC_AVX     ENC_DEC
1218
1219        #the number of pushes must equal STACK_OFFSET
1220        push    %r12
1221        push    %r13
1222        push    %r14
1223        push    %r15
1224
1225        mov     %rsp, %r14
1226
1227
1228
1229
1230        sub     $VARIABLE_OFFSET, %rsp
1231        and     $~63, %rsp                  # align rsp to 64 bytes
1232
1233
1234        vmovdqu  HashKey(arg1), %xmm13      # xmm13 = HashKey
1235
1236        mov     arg4, %r13                  # save the number of bytes of plaintext/ciphertext
1237        and     $-16, %r13                  # r13 = r13 - (r13 mod 16)
1238
1239        mov     %r13, %r12
1240        shr     $4, %r12
1241        and     $7, %r12
1242        jz      _initial_num_blocks_is_0\@
1243
1244        cmp     $7, %r12
1245        je      _initial_num_blocks_is_7\@
1246        cmp     $6, %r12
1247        je      _initial_num_blocks_is_6\@
1248        cmp     $5, %r12
1249        je      _initial_num_blocks_is_5\@
1250        cmp     $4, %r12
1251        je      _initial_num_blocks_is_4\@
1252        cmp     $3, %r12
1253        je      _initial_num_blocks_is_3\@
1254        cmp     $2, %r12
1255        je      _initial_num_blocks_is_2\@
1256
1257        jmp     _initial_num_blocks_is_1\@
1258
1259_initial_num_blocks_is_7\@:
1260        INITIAL_BLOCKS_AVX  7, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
1261        sub     $16*7, %r13
1262        jmp     _initial_blocks_encrypted\@
1263
1264_initial_num_blocks_is_6\@:
1265        INITIAL_BLOCKS_AVX  6, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
1266        sub     $16*6, %r13
1267        jmp     _initial_blocks_encrypted\@
1268
1269_initial_num_blocks_is_5\@:
1270        INITIAL_BLOCKS_AVX  5, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
1271        sub     $16*5, %r13
1272        jmp     _initial_blocks_encrypted\@
1273
1274_initial_num_blocks_is_4\@:
1275        INITIAL_BLOCKS_AVX  4, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
1276        sub     $16*4, %r13
1277        jmp     _initial_blocks_encrypted\@
1278
1279_initial_num_blocks_is_3\@:
1280        INITIAL_BLOCKS_AVX  3, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
1281        sub     $16*3, %r13
1282        jmp     _initial_blocks_encrypted\@
1283
1284_initial_num_blocks_is_2\@:
1285        INITIAL_BLOCKS_AVX  2, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
1286        sub     $16*2, %r13
1287        jmp     _initial_blocks_encrypted\@
1288
1289_initial_num_blocks_is_1\@:
1290        INITIAL_BLOCKS_AVX  1, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
1291        sub     $16*1, %r13
1292        jmp     _initial_blocks_encrypted\@
1293
1294_initial_num_blocks_is_0\@:
1295        INITIAL_BLOCKS_AVX  0, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
1296
1297
1298_initial_blocks_encrypted\@:
1299        cmp     $0, %r13
1300        je      _zero_cipher_left\@
1301
1302        sub     $128, %r13
1303        je      _eight_cipher_left\@
1304
1305
1306
1307
1308        vmovd   %xmm9, %r15d
1309        and     $255, %r15d
1310        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
1311
1312
1313_encrypt_by_8_new\@:
1314        cmp     $(255-8), %r15d
1315        jg      _encrypt_by_8\@
1316
1317
1318
1319        add     $8, %r15b
1320        GHASH_8_ENCRYPT_8_PARALLEL_AVX      %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm15, out_order, \ENC_DEC
1321        add     $128, %r11
1322        sub     $128, %r13
1323        jne     _encrypt_by_8_new\@
1324
1325        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
1326        jmp     _eight_cipher_left\@
1327
1328_encrypt_by_8\@:
1329        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
1330        add     $8, %r15b
1331        GHASH_8_ENCRYPT_8_PARALLEL_AVX      %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm15, in_order, \ENC_DEC
1332        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
1333        add     $128, %r11
1334        sub     $128, %r13
1335        jne     _encrypt_by_8_new\@
1336
1337        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
1338
1339
1340
1341
1342_eight_cipher_left\@:
1343        GHASH_LAST_8_AVX    %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8
1344
1345
1346_zero_cipher_left\@:
1347        cmp     $16, arg4
1348        jl      _only_less_than_16\@
1349
1350        mov     arg4, %r13
1351        and     $15, %r13                            # r13 = (arg4 mod 16)
1352
1353        je      _multiple_of_16_bytes\@
1354
1355        # handle the last <16 Byte block seperately
1356
1357
1358        vpaddd   ONE(%rip), %xmm9, %xmm9             # INCR CNT to get Yn
1359        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
1360        ENCRYPT_SINGLE_BLOCK    %xmm9                # E(K, Yn)
1361
1362        sub     $16, %r11
1363        add     %r13, %r11
1364        vmovdqu (arg3, %r11), %xmm1                  # receive the last <16 Byte block
1365
1366        lea     SHIFT_MASK+16(%rip), %r12
1367        sub     %r13, %r12                           # adjust the shuffle mask pointer to be
1368                                                     # able to shift 16-r13 bytes (r13 is the
1369                                                     # number of bytes in plaintext mod 16)
1370        vmovdqu (%r12), %xmm2                        # get the appropriate shuffle mask
1371        vpshufb %xmm2, %xmm1, %xmm1                  # shift right 16-r13 bytes
1372        jmp     _final_ghash_mul\@
1373
1374_only_less_than_16\@:
1375        # check for 0 length
1376        mov     arg4, %r13
1377        and     $15, %r13                            # r13 = (arg4 mod 16)
1378
1379        je      _multiple_of_16_bytes\@
1380
1381        # handle the last <16 Byte block seperately
1382
1383
1384        vpaddd  ONE(%rip), %xmm9, %xmm9              # INCR CNT to get Yn
1385        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
1386        ENCRYPT_SINGLE_BLOCK    %xmm9                # E(K, Yn)
1387
1388
1389        lea     SHIFT_MASK+16(%rip), %r12
1390        sub     %r13, %r12                           # adjust the shuffle mask pointer to be
1391                                                     # able to shift 16-r13 bytes (r13 is the
1392                                                     # number of bytes in plaintext mod 16)
1393
1394_get_last_16_byte_loop\@:
1395        movb    (arg3, %r11),  %al
1396        movb    %al,  TMP1 (%rsp , %r11)
1397        add     $1, %r11
1398        cmp     %r13,  %r11
1399        jne     _get_last_16_byte_loop\@
1400
1401        vmovdqu  TMP1(%rsp), %xmm1
1402
1403        sub     $16, %r11
1404
1405_final_ghash_mul\@:
1406        .if  \ENC_DEC ==  DEC
1407        vmovdqa %xmm1, %xmm2
1408        vpxor   %xmm1, %xmm9, %xmm9                  # Plaintext XOR E(K, Yn)
1409        vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1        # get the appropriate mask to
1410                                                     # mask out top 16-r13 bytes of xmm9
1411        vpand   %xmm1, %xmm9, %xmm9                  # mask out top 16-r13 bytes of xmm9
1412        vpand   %xmm1, %xmm2, %xmm2
1413        vpshufb SHUF_MASK(%rip), %xmm2, %xmm2
1414        vpxor   %xmm2, %xmm14, %xmm14
1415        #GHASH computation for the last <16 Byte block
1416        GHASH_MUL_AVX       %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
1417        sub     %r13, %r11
1418        add     $16, %r11
1419        .else
1420        vpxor   %xmm1, %xmm9, %xmm9                  # Plaintext XOR E(K, Yn)
1421        vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1        # get the appropriate mask to
1422                                                     # mask out top 16-r13 bytes of xmm9
1423        vpand   %xmm1, %xmm9, %xmm9                  # mask out top 16-r13 bytes of xmm9
1424        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
1425        vpxor   %xmm9, %xmm14, %xmm14
1426        #GHASH computation for the last <16 Byte block
1427        GHASH_MUL_AVX       %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
1428        sub     %r13, %r11
1429        add     $16, %r11
1430        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9        # shuffle xmm9 back to output as ciphertext
1431        .endif
1432
1433
1434        #############################
1435        # output r13 Bytes
1436        vmovq   %xmm9, %rax
1437        cmp     $8, %r13
1438        jle     _less_than_8_bytes_left\@
1439
1440        mov     %rax, (arg2 , %r11)
1441        add     $8, %r11
1442        vpsrldq $8, %xmm9, %xmm9
1443        vmovq   %xmm9, %rax
1444        sub     $8, %r13
1445
1446_less_than_8_bytes_left\@:
1447        movb    %al, (arg2 , %r11)
1448        add     $1, %r11
1449        shr     $8, %rax
1450        sub     $1, %r13
1451        jne     _less_than_8_bytes_left\@
1452        #############################
1453
1454_multiple_of_16_bytes\@:
1455        mov     arg7, %r12                           # r12 = aadLen (number of bytes)
1456        shl     $3, %r12                             # convert into number of bits
1457        vmovd   %r12d, %xmm15                        # len(A) in xmm15
1458
1459        shl     $3, arg4                             # len(C) in bits  (*128)
1460        vmovq   arg4, %xmm1
1461        vpslldq $8, %xmm15, %xmm15                   # xmm15 = len(A)|| 0x0000000000000000
1462        vpxor   %xmm1, %xmm15, %xmm15                # xmm15 = len(A)||len(C)
1463
1464        vpxor   %xmm15, %xmm14, %xmm14
1465        GHASH_MUL_AVX       %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6    # final GHASH computation
1466        vpshufb SHUF_MASK(%rip), %xmm14, %xmm14      # perform a 16Byte swap
1467
1468        mov     arg5, %rax                           # rax = *Y0
1469        vmovdqu (%rax), %xmm9                        # xmm9 = Y0
1470
1471        ENCRYPT_SINGLE_BLOCK    %xmm9                # E(K, Y0)
1472
1473        vpxor   %xmm14, %xmm9, %xmm9
1474
1475
1476
1477_return_T\@:
1478        mov     arg8, %r10              # r10 = authTag
1479        mov     arg9, %r11              # r11 = auth_tag_len
1480
1481        cmp     $16, %r11
1482        je      _T_16\@
1483
1484        cmp     $8, %r11
1485        jl      _T_4\@
1486
1487_T_8\@:
1488        vmovq   %xmm9, %rax
1489        mov     %rax, (%r10)
1490        add     $8, %r10
1491        sub     $8, %r11
1492        vpsrldq $8, %xmm9, %xmm9
1493        cmp     $0, %r11
1494        je     _return_T_done\@
1495_T_4\@:
1496        vmovd   %xmm9, %eax
1497        mov     %eax, (%r10)
1498        add     $4, %r10
1499        sub     $4, %r11
1500        vpsrldq     $4, %xmm9, %xmm9
1501        cmp     $0, %r11
1502        je     _return_T_done\@
1503_T_123\@:
1504        vmovd     %xmm9, %eax
1505        cmp     $2, %r11
1506        jl     _T_1\@
1507        mov     %ax, (%r10)
1508        cmp     $2, %r11
1509        je     _return_T_done\@
1510        add     $2, %r10
1511        sar     $16, %eax
1512_T_1\@:
1513        mov     %al, (%r10)
1514        jmp     _return_T_done\@
1515
1516_T_16\@:
1517        vmovdqu %xmm9, (%r10)
1518
1519_return_T_done\@:
1520        mov     %r14, %rsp
1521
1522        pop     %r15
1523        pop     %r14
1524        pop     %r13
1525        pop     %r12
1526.endm
1527
1528
1529#############################################################
1530#void   aesni_gcm_precomp_avx_gen2
1531#        (gcm_data     *my_ctx_data,
1532#        u8     *hash_subkey)# /* H, the Hash sub key input. Data starts on a 16-byte boundary. */
1533#############################################################
1534ENTRY(aesni_gcm_precomp_avx_gen2)
1535        #the number of pushes must equal STACK_OFFSET
1536        push    %r12
1537        push    %r13
1538        push    %r14
1539        push    %r15
1540
1541        mov     %rsp, %r14
1542
1543
1544
1545        sub     $VARIABLE_OFFSET, %rsp
1546        and     $~63, %rsp                  # align rsp to 64 bytes
1547
1548        vmovdqu  (arg2), %xmm6              # xmm6 = HashKey
1549
1550        vpshufb  SHUF_MASK(%rip), %xmm6, %xmm6
1551        ###############  PRECOMPUTATION of HashKey<<1 mod poly from the HashKey
1552        vmovdqa  %xmm6, %xmm2
1553        vpsllq   $1, %xmm6, %xmm6
1554        vpsrlq   $63, %xmm2, %xmm2
1555        vmovdqa  %xmm2, %xmm1
1556        vpslldq  $8, %xmm2, %xmm2
1557        vpsrldq  $8, %xmm1, %xmm1
1558        vpor     %xmm2, %xmm6, %xmm6
1559        #reduction
1560        vpshufd  $0b00100100, %xmm1, %xmm2
1561        vpcmpeqd TWOONE(%rip), %xmm2, %xmm2
1562        vpand    POLY(%rip), %xmm2, %xmm2
1563        vpxor    %xmm2, %xmm6, %xmm6        # xmm6 holds the HashKey<<1 mod poly
1564        #######################################################################
1565        vmovdqa  %xmm6, HashKey(arg1)       # store HashKey<<1 mod poly
1566
1567
1568        PRECOMPUTE_AVX  %xmm6, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5
1569
1570        mov     %r14, %rsp
1571
1572        pop     %r15
1573        pop     %r14
1574        pop     %r13
1575        pop     %r12
1576        ret
1577ENDPROC(aesni_gcm_precomp_avx_gen2)
1578
1579###############################################################################
1580#void   aesni_gcm_enc_avx_gen2(
1581#        gcm_data        *my_ctx_data,     /* aligned to 16 Bytes */
1582#        u8      *out, /* Ciphertext output. Encrypt in-place is allowed.  */
1583#        const   u8 *in, /* Plaintext input */
1584#        u64     plaintext_len, /* Length of data in Bytes for encryption. */
1585#        u8      *iv, /* Pre-counter block j0: 4 byte salt
1586#                       (from Security Association) concatenated with 8 byte
1587#                       Initialisation Vector (from IPSec ESP Payload)
1588#                       concatenated with 0x00000001. 16-byte aligned pointer. */
1589#        const   u8 *aad, /* Additional Authentication Data (AAD)*/
1590#        u64     aad_len, /* Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 Bytes */
1591#        u8      *auth_tag, /* Authenticated Tag output. */
1592#        u64     auth_tag_len)# /* Authenticated Tag Length in bytes.
1593#                               Valid values are 16 (most likely), 12 or 8. */
1594###############################################################################
1595ENTRY(aesni_gcm_enc_avx_gen2)
1596        GCM_ENC_DEC_AVX     ENC
1597        ret
1598ENDPROC(aesni_gcm_enc_avx_gen2)
1599
1600###############################################################################
1601#void   aesni_gcm_dec_avx_gen2(
1602#        gcm_data        *my_ctx_data,     /* aligned to 16 Bytes */
1603#        u8      *out, /* Plaintext output. Decrypt in-place is allowed.  */
1604#        const   u8 *in, /* Ciphertext input */
1605#        u64     plaintext_len, /* Length of data in Bytes for encryption. */
1606#        u8      *iv, /* Pre-counter block j0: 4 byte salt
1607#                       (from Security Association) concatenated with 8 byte
1608#                       Initialisation Vector (from IPSec ESP Payload)
1609#                       concatenated with 0x00000001. 16-byte aligned pointer. */
1610#        const   u8 *aad, /* Additional Authentication Data (AAD)*/
1611#        u64     aad_len, /* Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 Bytes */
1612#        u8      *auth_tag, /* Authenticated Tag output. */
1613#        u64     auth_tag_len)# /* Authenticated Tag Length in bytes.
1614#                               Valid values are 16 (most likely), 12 or 8. */
1615###############################################################################
1616ENTRY(aesni_gcm_dec_avx_gen2)
1617        GCM_ENC_DEC_AVX     DEC
1618        ret
1619ENDPROC(aesni_gcm_dec_avx_gen2)
1620#endif /* CONFIG_AS_AVX */
1621
1622#ifdef CONFIG_AS_AVX2
1623###############################################################################
1624# GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
1625# Input: A and B (128-bits each, bit-reflected)
1626# Output: C = A*B*x mod poly, (i.e. >>1 )
1627# To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input
1628# GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly.
1629###############################################################################
1630.macro  GHASH_MUL_AVX2 GH HK T1 T2 T3 T4 T5
1631
1632        vpclmulqdq      $0x11,\HK,\GH,\T1      # T1 = a1*b1
1633        vpclmulqdq      $0x00,\HK,\GH,\T2      # T2 = a0*b0
1634        vpclmulqdq      $0x01,\HK,\GH,\T3      # T3 = a1*b0
1635        vpclmulqdq      $0x10,\HK,\GH,\GH      # GH = a0*b1
1636        vpxor           \T3, \GH, \GH
1637
1638
1639        vpsrldq         $8 , \GH, \T3          # shift-R GH 2 DWs
1640        vpslldq         $8 , \GH, \GH          # shift-L GH 2 DWs
1641
1642        vpxor           \T3, \T1, \T1
1643        vpxor           \T2, \GH, \GH
1644
1645        #######################################################################
1646        #first phase of the reduction
1647        vmovdqa         POLY2(%rip), \T3
1648
1649        vpclmulqdq      $0x01, \GH, \T3, \T2
1650        vpslldq         $8, \T2, \T2           # shift-L T2 2 DWs
1651
1652        vpxor           \T2, \GH, \GH          # first phase of the reduction complete
1653        #######################################################################
1654        #second phase of the reduction
1655        vpclmulqdq      $0x00, \GH, \T3, \T2
1656        vpsrldq         $4, \T2, \T2           # shift-R T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R)
1657
1658        vpclmulqdq      $0x10, \GH, \T3, \GH
1659        vpslldq         $4, \GH, \GH           # shift-L GH 1 DW (Shift-L 1-DW to obtain result with no shifts)
1660
1661        vpxor           \T2, \GH, \GH          # second phase of the reduction complete
1662        #######################################################################
1663        vpxor           \T1, \GH, \GH          # the result is in GH
1664
1665
1666.endm
1667
1668.macro PRECOMPUTE_AVX2 HK T1 T2 T3 T4 T5 T6
1669
1670        # Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
1671        vmovdqa  \HK, \T5
1672        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^2<<1 mod poly
1673        vmovdqa  \T5, HashKey_2(arg1)                       #  [HashKey_2] = HashKey^2<<1 mod poly
1674
1675        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^3<<1 mod poly
1676        vmovdqa  \T5, HashKey_3(arg1)
1677
1678        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^4<<1 mod poly
1679        vmovdqa  \T5, HashKey_4(arg1)
1680
1681        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^5<<1 mod poly
1682        vmovdqa  \T5, HashKey_5(arg1)
1683
1684        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^6<<1 mod poly
1685        vmovdqa  \T5, HashKey_6(arg1)
1686
1687        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^7<<1 mod poly
1688        vmovdqa  \T5, HashKey_7(arg1)
1689
1690        GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2    #  T5 = HashKey^8<<1 mod poly
1691        vmovdqa  \T5, HashKey_8(arg1)
1692
1693.endm
1694
1695
1696## if a = number of total plaintext bytes
1697## b = floor(a/16)
1698## num_initial_blocks = b mod 4#
1699## encrypt the initial num_initial_blocks blocks and apply ghash on the ciphertext
1700## r10, r11, r12, rax are clobbered
1701## arg1, arg2, arg3, r14 are used as a pointer only, not modified
1702
1703.macro INITIAL_BLOCKS_AVX2 num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T6 T_key ENC_DEC VER
1704        i = (8-\num_initial_blocks)
1705        j = 0
1706        setreg
1707
1708        mov     arg6, %r10                       # r10 = AAD
1709        mov     arg7, %r12                       # r12 = aadLen
1710
1711
1712        mov     %r12, %r11
1713
1714        vpxor   reg_j, reg_j, reg_j
1715        vpxor   reg_i, reg_i, reg_i
1716
1717        cmp     $16, %r11
1718        jl      _get_AAD_rest8\@
1719_get_AAD_blocks\@:
1720        vmovdqu (%r10), reg_i
1721        vpshufb SHUF_MASK(%rip), reg_i, reg_i
1722        vpxor   reg_i, reg_j, reg_j
1723        GHASH_MUL_AVX2      reg_j, \T2, \T1, \T3, \T4, \T5, \T6
1724        add     $16, %r10
1725        sub     $16, %r12
1726        sub     $16, %r11
1727        cmp     $16, %r11
1728        jge     _get_AAD_blocks\@
1729        vmovdqu reg_j, reg_i
1730        cmp     $0, %r11
1731        je      _get_AAD_done\@
1732
1733        vpxor   reg_i, reg_i, reg_i
1734
1735        /* read the last <16B of AAD. since we have at least 4B of
1736        data right after the AAD (the ICV, and maybe some CT), we can
1737        read 4B/8B blocks safely, and then get rid of the extra stuff */
1738_get_AAD_rest8\@:
1739        cmp     $4, %r11
1740        jle     _get_AAD_rest4\@
1741        movq    (%r10), \T1
1742        add     $8, %r10
1743        sub     $8, %r11
1744        vpslldq $8, \T1, \T1
1745        vpsrldq $8, reg_i, reg_i
1746        vpxor   \T1, reg_i, reg_i
1747        jmp     _get_AAD_rest8\@
1748_get_AAD_rest4\@:
1749        cmp     $0, %r11
1750        jle     _get_AAD_rest0\@
1751        mov     (%r10), %eax
1752        movq    %rax, \T1
1753        add     $4, %r10
1754        sub     $4, %r11
1755        vpslldq $12, \T1, \T1
1756        vpsrldq $4, reg_i, reg_i
1757        vpxor   \T1, reg_i, reg_i
1758_get_AAD_rest0\@:
1759        /* finalize: shift out the extra bytes we read, and align
1760        left. since pslldq can only shift by an immediate, we use
1761        vpshufb and an array of shuffle masks */
1762        movq    %r12, %r11
1763        salq    $4, %r11
1764        movdqu  aad_shift_arr(%r11), \T1
1765        vpshufb \T1, reg_i, reg_i
1766_get_AAD_rest_final\@:
1767        vpshufb SHUF_MASK(%rip), reg_i, reg_i
1768        vpxor   reg_j, reg_i, reg_i
1769        GHASH_MUL_AVX2      reg_i, \T2, \T1, \T3, \T4, \T5, \T6
1770
1771_get_AAD_done\@:
1772        # initialize the data pointer offset as zero
1773        xor     %r11, %r11
1774
1775        # start AES for num_initial_blocks blocks
1776        mov     arg5, %rax                     # rax = *Y0
1777        vmovdqu (%rax), \CTR                   # CTR = Y0
1778        vpshufb SHUF_MASK(%rip), \CTR, \CTR
1779
1780
1781        i = (9-\num_initial_blocks)
1782        setreg
1783.rep \num_initial_blocks
1784                vpaddd  ONE(%rip), \CTR, \CTR   # INCR Y0
1785                vmovdqa \CTR, reg_i
1786                vpshufb SHUF_MASK(%rip), reg_i, reg_i     # perform a 16Byte swap
1787        i = (i+1)
1788        setreg
1789.endr
1790
1791        vmovdqa  (arg1), \T_key
1792        i = (9-\num_initial_blocks)
1793        setreg
1794.rep \num_initial_blocks
1795                vpxor   \T_key, reg_i, reg_i
1796        i = (i+1)
1797        setreg
1798.endr
1799
1800        j = 1
1801        setreg
1802.rep 9
1803        vmovdqa  16*j(arg1), \T_key
1804        i = (9-\num_initial_blocks)
1805        setreg
1806.rep \num_initial_blocks
1807        vaesenc \T_key, reg_i, reg_i
1808        i = (i+1)
1809        setreg
1810.endr
1811
1812        j = (j+1)
1813        setreg
1814.endr
1815
1816
1817        vmovdqa  16*10(arg1), \T_key
1818        i = (9-\num_initial_blocks)
1819        setreg
1820.rep \num_initial_blocks
1821        vaesenclast      \T_key, reg_i, reg_i
1822        i = (i+1)
1823        setreg
1824.endr
1825
1826        i = (9-\num_initial_blocks)
1827        setreg
1828.rep \num_initial_blocks
1829                vmovdqu (arg3, %r11), \T1
1830                vpxor   \T1, reg_i, reg_i
1831                vmovdqu reg_i, (arg2 , %r11)           # write back ciphertext for
1832                                                       # num_initial_blocks blocks
1833                add     $16, %r11
1834.if  \ENC_DEC == DEC
1835                vmovdqa \T1, reg_i
1836.endif
1837                vpshufb SHUF_MASK(%rip), reg_i, reg_i  # prepare ciphertext for GHASH computations
1838        i = (i+1)
1839        setreg
1840.endr
1841
1842
1843        i = (8-\num_initial_blocks)
1844        j = (9-\num_initial_blocks)
1845        setreg
1846
1847.rep \num_initial_blocks
1848        vpxor    reg_i, reg_j, reg_j
1849        GHASH_MUL_AVX2       reg_j, \T2, \T1, \T3, \T4, \T5, \T6  # apply GHASH on num_initial_blocks blocks
1850        i = (i+1)
1851        j = (j+1)
1852        setreg
1853.endr
1854        # XMM8 has the combined result here
1855
1856        vmovdqa  \XMM8, TMP1(%rsp)
1857        vmovdqa  \XMM8, \T3
1858
1859        cmp     $128, %r13
1860        jl      _initial_blocks_done\@                  # no need for precomputed constants
1861
1862###############################################################################
1863# Haskey_i_k holds XORed values of the low and high parts of the Haskey_i
1864                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1865                vmovdqa  \CTR, \XMM1
1866                vpshufb  SHUF_MASK(%rip), \XMM1, \XMM1  # perform a 16Byte swap
1867
1868                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1869                vmovdqa  \CTR, \XMM2
1870                vpshufb  SHUF_MASK(%rip), \XMM2, \XMM2  # perform a 16Byte swap
1871
1872                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1873                vmovdqa  \CTR, \XMM3
1874                vpshufb  SHUF_MASK(%rip), \XMM3, \XMM3  # perform a 16Byte swap
1875
1876                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1877                vmovdqa  \CTR, \XMM4
1878                vpshufb  SHUF_MASK(%rip), \XMM4, \XMM4  # perform a 16Byte swap
1879
1880                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1881                vmovdqa  \CTR, \XMM5
1882                vpshufb  SHUF_MASK(%rip), \XMM5, \XMM5  # perform a 16Byte swap
1883
1884                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1885                vmovdqa  \CTR, \XMM6
1886                vpshufb  SHUF_MASK(%rip), \XMM6, \XMM6  # perform a 16Byte swap
1887
1888                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1889                vmovdqa  \CTR, \XMM7
1890                vpshufb  SHUF_MASK(%rip), \XMM7, \XMM7  # perform a 16Byte swap
1891
1892                vpaddd   ONE(%rip), \CTR, \CTR          # INCR Y0
1893                vmovdqa  \CTR, \XMM8
1894                vpshufb  SHUF_MASK(%rip), \XMM8, \XMM8  # perform a 16Byte swap
1895
1896                vmovdqa  (arg1), \T_key
1897                vpxor    \T_key, \XMM1, \XMM1
1898                vpxor    \T_key, \XMM2, \XMM2
1899                vpxor    \T_key, \XMM3, \XMM3
1900                vpxor    \T_key, \XMM4, \XMM4
1901                vpxor    \T_key, \XMM5, \XMM5
1902                vpxor    \T_key, \XMM6, \XMM6
1903                vpxor    \T_key, \XMM7, \XMM7
1904                vpxor    \T_key, \XMM8, \XMM8
1905
1906                i = 1
1907                setreg
1908.rep    9       # do 9 rounds
1909                vmovdqa  16*i(arg1), \T_key
1910                vaesenc  \T_key, \XMM1, \XMM1
1911                vaesenc  \T_key, \XMM2, \XMM2
1912                vaesenc  \T_key, \XMM3, \XMM3
1913                vaesenc  \T_key, \XMM4, \XMM4
1914                vaesenc  \T_key, \XMM5, \XMM5
1915                vaesenc  \T_key, \XMM6, \XMM6
1916                vaesenc  \T_key, \XMM7, \XMM7
1917                vaesenc  \T_key, \XMM8, \XMM8
1918                i = (i+1)
1919                setreg
1920.endr
1921
1922
1923                vmovdqa  16*i(arg1), \T_key
1924                vaesenclast  \T_key, \XMM1, \XMM1
1925                vaesenclast  \T_key, \XMM2, \XMM2
1926                vaesenclast  \T_key, \XMM3, \XMM3
1927                vaesenclast  \T_key, \XMM4, \XMM4
1928                vaesenclast  \T_key, \XMM5, \XMM5
1929                vaesenclast  \T_key, \XMM6, \XMM6
1930                vaesenclast  \T_key, \XMM7, \XMM7
1931                vaesenclast  \T_key, \XMM8, \XMM8
1932
1933                vmovdqu  (arg3, %r11), \T1
1934                vpxor    \T1, \XMM1, \XMM1
1935                vmovdqu  \XMM1, (arg2 , %r11)
1936                .if   \ENC_DEC == DEC
1937                vmovdqa  \T1, \XMM1
1938                .endif
1939
1940                vmovdqu  16*1(arg3, %r11), \T1
1941                vpxor    \T1, \XMM2, \XMM2
1942                vmovdqu  \XMM2, 16*1(arg2 , %r11)
1943                .if   \ENC_DEC == DEC
1944                vmovdqa  \T1, \XMM2
1945                .endif
1946
1947                vmovdqu  16*2(arg3, %r11), \T1
1948                vpxor    \T1, \XMM3, \XMM3
1949                vmovdqu  \XMM3, 16*2(arg2 , %r11)
1950                .if   \ENC_DEC == DEC
1951                vmovdqa  \T1, \XMM3
1952                .endif
1953
1954                vmovdqu  16*3(arg3, %r11), \T1
1955                vpxor    \T1, \XMM4, \XMM4
1956                vmovdqu  \XMM4, 16*3(arg2 , %r11)
1957                .if   \ENC_DEC == DEC
1958                vmovdqa  \T1, \XMM4
1959                .endif
1960
1961                vmovdqu  16*4(arg3, %r11), \T1
1962                vpxor    \T1, \XMM5, \XMM5
1963                vmovdqu  \XMM5, 16*4(arg2 , %r11)
1964                .if   \ENC_DEC == DEC
1965                vmovdqa  \T1, \XMM5
1966                .endif
1967
1968                vmovdqu  16*5(arg3, %r11), \T1
1969                vpxor    \T1, \XMM6, \XMM6
1970                vmovdqu  \XMM6, 16*5(arg2 , %r11)
1971                .if   \ENC_DEC == DEC
1972                vmovdqa  \T1, \XMM6
1973                .endif
1974
1975                vmovdqu  16*6(arg3, %r11), \T1
1976                vpxor    \T1, \XMM7, \XMM7
1977                vmovdqu  \XMM7, 16*6(arg2 , %r11)
1978                .if   \ENC_DEC == DEC
1979                vmovdqa  \T1, \XMM7
1980                .endif
1981
1982                vmovdqu  16*7(arg3, %r11), \T1
1983                vpxor    \T1, \XMM8, \XMM8
1984                vmovdqu  \XMM8, 16*7(arg2 , %r11)
1985                .if   \ENC_DEC == DEC
1986                vmovdqa  \T1, \XMM8
1987                .endif
1988
1989                add     $128, %r11
1990
1991                vpshufb  SHUF_MASK(%rip), \XMM1, \XMM1     # perform a 16Byte swap
1992                vpxor    TMP1(%rsp), \XMM1, \XMM1          # combine GHASHed value with
1993                                                           # the corresponding ciphertext
1994                vpshufb  SHUF_MASK(%rip), \XMM2, \XMM2     # perform a 16Byte swap
1995                vpshufb  SHUF_MASK(%rip), \XMM3, \XMM3     # perform a 16Byte swap
1996                vpshufb  SHUF_MASK(%rip), \XMM4, \XMM4     # perform a 16Byte swap
1997                vpshufb  SHUF_MASK(%rip), \XMM5, \XMM5     # perform a 16Byte swap
1998                vpshufb  SHUF_MASK(%rip), \XMM6, \XMM6     # perform a 16Byte swap
1999                vpshufb  SHUF_MASK(%rip), \XMM7, \XMM7     # perform a 16Byte swap
2000                vpshufb  SHUF_MASK(%rip), \XMM8, \XMM8     # perform a 16Byte swap

2001
2002###############################################################################
2003
2004_initial_blocks_done\@:
2005
2006
2007.endm
2008
2009
2010
2011# encrypt 8 blocks at a time
2012# ghash the 8 previously encrypted ciphertext blocks
2013# arg1, arg2, arg3 are used as pointers only, not modified
2014# r11 is the data offset value
2015.macro GHASH_8_ENCRYPT_8_PARALLEL_AVX2 T1 T2 T3 T4 T5 T6 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 T7 loop_idx ENC_DEC
2016
2017        vmovdqa \XMM1, \T2
2018        vmovdqa \XMM2, TMP2(%rsp)
2019        vmovdqa \XMM3, TMP3(%rsp)
2020        vmovdqa \XMM4, TMP4(%rsp)
2021        vmovdqa \XMM5, TMP5(%rsp)
2022        vmovdqa \XMM6, TMP6(%rsp)
2023        vmovdqa \XMM7, TMP7(%rsp)
2024        vmovdqa \XMM8, TMP8(%rsp)
2025
2026.if \loop_idx == in_order
2027                vpaddd  ONE(%rip), \CTR, \XMM1            # INCR CNT
2028                vpaddd  ONE(%rip), \XMM1, \XMM2
2029                vpaddd  ONE(%rip), \XMM2, \XMM3
2030                vpaddd  ONE(%rip), \XMM3, \XMM4
2031                vpaddd  ONE(%rip), \XMM4, \XMM5
2032                vpaddd  ONE(%rip), \XMM5, \XMM6
2033                vpaddd  ONE(%rip), \XMM6, \XMM7
2034                vpaddd  ONE(%rip), \XMM7, \XMM8
2035                vmovdqa \XMM8, \CTR
2036
2037                vpshufb SHUF_MASK(%rip), \XMM1, \XMM1     # perform a 16Byte swap
2038                vpshufb SHUF_MASK(%rip), \XMM2, \XMM2     # perform a 16Byte swap
2039                vpshufb SHUF_MASK(%rip), \XMM3, \XMM3     # perform a 16Byte swap
2040                vpshufb SHUF_MASK(%rip), \XMM4, \XMM4     # perform a 16Byte swap
2041                vpshufb SHUF_MASK(%rip), \XMM5, \XMM5     # perform a 16Byte swap
2042                vpshufb SHUF_MASK(%rip), \XMM6, \XMM6     # perform a 16Byte swap
2043                vpshufb SHUF_MASK(%rip), \XMM7, \XMM7     # perform a 16Byte swap
2044                vpshufb SHUF_MASK(%rip), \XMM8, \XMM8     # perform a 16Byte swap
2045.else
2046                vpaddd  ONEf(%rip), \CTR, \XMM1            # INCR CNT
2047                vpaddd  ONEf(%rip), \XMM1, \XMM2
2048                vpaddd  ONEf(%rip), \XMM2, \XMM3
2049                vpaddd  ONEf(%rip), \XMM3, \XMM4
2050                vpaddd  ONEf(%rip), \XMM4, \XMM5
2051                vpaddd  ONEf(%rip), \XMM5, \XMM6
2052                vpaddd  ONEf(%rip), \XMM6, \XMM7
2053                vpaddd  ONEf(%rip), \XMM7, \XMM8
2054                vmovdqa \XMM8, \CTR
2055.endif
2056
2057
2058        #######################################################################
2059
2060                vmovdqu (arg1), \T1
2061                vpxor   \T1, \XMM1, \XMM1
2062                vpxor   \T1, \XMM2, \XMM2
2063                vpxor   \T1, \XMM3, \XMM3
2064                vpxor   \T1, \XMM4, \XMM4
2065                vpxor   \T1, \XMM5, \XMM5
2066                vpxor   \T1, \XMM6, \XMM6
2067                vpxor   \T1, \XMM7, \XMM7
2068                vpxor   \T1, \XMM8, \XMM8
2069
2070        #######################################################################
2071
2072
2073
2074
2075
2076                vmovdqu 16*1(arg1), \T1
2077                vaesenc \T1, \XMM1, \XMM1
2078                vaesenc \T1, \XMM2, \XMM2
2079                vaesenc \T1, \XMM3, \XMM3
2080                vaesenc \T1, \XMM4, \XMM4
2081                vaesenc \T1, \XMM5, \XMM5
2082                vaesenc \T1, \XMM6, \XMM6
2083                vaesenc \T1, \XMM7, \XMM7
2084                vaesenc \T1, \XMM8, \XMM8
2085
2086                vmovdqu 16*2(arg1), \T1
2087                vaesenc \T1, \XMM1, \XMM1
2088                vaesenc \T1, \XMM2, \XMM2
2089                vaesenc \T1, \XMM3, \XMM3
2090                vaesenc \T1, \XMM4, \XMM4
2091                vaesenc \T1, \XMM5, \XMM5
2092                vaesenc \T1, \XMM6, \XMM6
2093                vaesenc \T1, \XMM7, \XMM7
2094                vaesenc \T1, \XMM8, \XMM8
2095
2096
2097        #######################################################################
2098
2099        vmovdqa         HashKey_8(arg1), \T5
2100        vpclmulqdq      $0x11, \T5, \T2, \T4              # T4 = a1*b1
2101        vpclmulqdq      $0x00, \T5, \T2, \T7              # T7 = a0*b0
2102        vpclmulqdq      $0x01, \T5, \T2, \T6              # T6 = a1*b0
2103        vpclmulqdq      $0x10, \T5, \T2, \T5              # T5 = a0*b1
2104        vpxor           \T5, \T6, \T6
2105
2106                vmovdqu 16*3(arg1), \T1
2107                vaesenc \T1, \XMM1, \XMM1
2108                vaesenc \T1, \XMM2, \XMM2
2109                vaesenc \T1, \XMM3, \XMM3
2110                vaesenc \T1, \XMM4, \XMM4
2111                vaesenc \T1, \XMM5, \XMM5
2112                vaesenc \T1, \XMM6, \XMM6
2113                vaesenc \T1, \XMM7, \XMM7
2114                vaesenc \T1, \XMM8, \XMM8
2115
2116        vmovdqa         TMP2(%rsp), \T1
2117        vmovdqa         HashKey_7(arg1), \T5
2118        vpclmulqdq      $0x11, \T5, \T1, \T3
2119        vpxor           \T3, \T4, \T4
2120
2121        vpclmulqdq      $0x00, \T5, \T1, \T3
2122        vpxor           \T3, \T7, \T7
2123
2124        vpclmulqdq      $0x01, \T5, \T1, \T3
2125        vpxor           \T3, \T6, \T6
2126
2127        vpclmulqdq      $0x10, \T5, \T1, \T3
2128        vpxor           \T3, \T6, \T6
2129
2130                vmovdqu 16*4(arg1), \T1
2131                vaesenc \T1, \XMM1, \XMM1
2132                vaesenc \T1, \XMM2, \XMM2
2133                vaesenc \T1, \XMM3, \XMM3
2134                vaesenc \T1, \XMM4, \XMM4
2135                vaesenc \T1, \XMM5, \XMM5
2136                vaesenc \T1, \XMM6, \XMM6
2137                vaesenc \T1, \XMM7, \XMM7
2138                vaesenc \T1, \XMM8, \XMM8
2139
2140        #######################################################################
2141
2142        vmovdqa         TMP3(%rsp), \T1
2143        vmovdqa         HashKey_6(arg1), \T5
2144        vpclmulqdq      $0x11, \T5, \T1, \T3
2145        vpxor           \T3, \T4, \T4
2146
2147        vpclmulqdq      $0x00, \T5, \T1, \T3
2148        vpxor           \T3, \T7, \T7
2149
2150        vpclmulqdq      $0x01, \T5, \T1, \T3
2151        vpxor           \T3, \T6, \T6
2152
2153        vpclmulqdq      $0x10, \T5, \T1, \T3
2154        vpxor           \T3, \T6, \T6
2155
2156                vmovdqu 16*5(arg1), \T1
2157                vaesenc \T1, \XMM1, \XMM1
2158                vaesenc \T1, \XMM2, \XMM2
2159                vaesenc \T1, \XMM3, \XMM3
2160                vaesenc \T1, \XMM4, \XMM4
2161                vaesenc \T1, \XMM5, \XMM5
2162                vaesenc \T1, \XMM6, \XMM6
2163                vaesenc \T1, \XMM7, \XMM7
2164                vaesenc \T1, \XMM8, \XMM8
2165
2166        vmovdqa         TMP4(%rsp), \T1
2167        vmovdqa         HashKey_5(arg1), \T5
2168        vpclmulqdq      $0x11, \T5, \T1, \T3
2169        vpxor           \T3, \T4, \T4
2170
2171        vpclmulqdq      $0x00, \T5, \T1, \T3
2172        vpxor           \T3, \T7, \T7
2173
2174        vpclmulqdq      $0x01, \T5, \T1, \T3
2175        vpxor           \T3, \T6, \T6
2176
2177        vpclmulqdq      $0x10, \T5, \T1, \T3
2178        vpxor           \T3, \T6, \T6
2179
2180                vmovdqu 16*6(arg1), \T1
2181                vaesenc \T1, \XMM1, \XMM1
2182                vaesenc \T1, \XMM2, \XMM2
2183                vaesenc \T1, \XMM3, \XMM3
2184                vaesenc \T1, \XMM4, \XMM4
2185                vaesenc \T1, \XMM5, \XMM5
2186                vaesenc \T1, \XMM6, \XMM6
2187                vaesenc \T1, \XMM7, \XMM7
2188                vaesenc \T1, \XMM8, \XMM8
2189
2190
2191        vmovdqa         TMP5(%rsp), \T1
2192        vmovdqa         HashKey_4(arg1), \T5
2193        vpclmulqdq      $0x11, \T5, \T1, \T3
2194        vpxor           \T3, \T4, \T4
2195
2196        vpclmulqdq      $0x00, \T5, \T1, \T3
2197        vpxor           \T3, \T7, \T7
2198
2199        vpclmulqdq      $0x01, \T5, \T1, \T3
2200        vpxor           \T3, \T6, \T6
2201
2202        vpclmulqdq      $0x10, \T5, \T1, \T3
2203        vpxor           \T3, \T6, \T6
2204
2205                vmovdqu 16*7(arg1), \T1
2206                vaesenc \T1, \XMM1, \XMM1
2207                vaesenc \T1, \XMM2, \XMM2
2208                vaesenc \T1, \XMM3, \XMM3
2209                vaesenc \T1, \XMM4, \XMM4
2210                vaesenc \T1, \XMM5, \XMM5
2211                vaesenc \T1, \XMM6, \XMM6
2212                vaesenc \T1, \XMM7, \XMM7
2213                vaesenc \T1, \XMM8, \XMM8
2214
2215        vmovdqa         TMP6(%rsp), \T1
2216        vmovdqa         HashKey_3(arg1), \T5
2217        vpclmulqdq      $0x11, \T5, \T1, \T3
2218        vpxor           \T3, \T4, \T4
2219
2220        vpclmulqdq      $0x00, \T5, \T1, \T3
2221        vpxor           \T3, \T7, \T7
2222
2223        vpclmulqdq      $0x01, \T5, \T1, \T3
2224        vpxor           \T3, \T6, \T6
2225
2226        vpclmulqdq      $0x10, \T5, \T1, \T3
2227        vpxor           \T3, \T6, \T6
2228
2229                vmovdqu 16*8(arg1), \T1
2230                vaesenc \T1, \XMM1, \XMM1
2231                vaesenc \T1, \XMM2, \XMM2
2232                vaesenc \T1, \XMM3, \XMM3
2233                vaesenc \T1, \XMM4, \XMM4
2234                vaesenc \T1, \XMM5, \XMM5
2235                vaesenc \T1, \XMM6, \XMM6
2236                vaesenc \T1, \XMM7, \XMM7
2237                vaesenc \T1, \XMM8, \XMM8
2238
2239        vmovdqa         TMP7(%rsp), \T1
2240        vmovdqa         HashKey_2(arg1), \T5
2241        vpclmulqdq      $0x11, \T5, \T1, \T3
2242        vpxor           \T3, \T4, \T4
2243
2244        vpclmulqdq      $0x00, \T5, \T1, \T3
2245        vpxor           \T3, \T7, \T7
2246
2247        vpclmulqdq      $0x01, \T5, \T1, \T3
2248        vpxor           \T3, \T6, \T6
2249
2250        vpclmulqdq      $0x10, \T5, \T1, \T3
2251        vpxor           \T3, \T6, \T6
2252
2253
2254        #######################################################################
2255
2256                vmovdqu 16*9(arg1), \T5
2257                vaesenc \T5, \XMM1, \XMM1
2258                vaesenc \T5, \XMM2, \XMM2
2259                vaesenc \T5, \XMM3, \XMM3
2260                vaesenc \T5, \XMM4, \XMM4
2261                vaesenc \T5, \XMM5, \XMM5
2262                vaesenc \T5, \XMM6, \XMM6
2263                vaesenc \T5, \XMM7, \XMM7
2264                vaesenc \T5, \XMM8, \XMM8
2265
2266        vmovdqa         TMP8(%rsp), \T1
2267        vmovdqa         HashKey(arg1), \T5
2268
2269        vpclmulqdq      $0x00, \T5, \T1, \T3
2270        vpxor           \T3, \T7, \T7
2271
2272        vpclmulqdq      $0x01, \T5, \T1, \T3
2273        vpxor           \T3, \T6, \T6
2274
2275        vpclmulqdq      $0x10, \T5, \T1, \T3
2276        vpxor           \T3, \T6, \T6
2277
2278        vpclmulqdq      $0x11, \T5, \T1, \T3
2279        vpxor           \T3, \T4, \T1
2280
2281
2282                vmovdqu 16*10(arg1), \T5
2283
2284        i = 0
2285        j = 1
2286        setreg
2287.rep 8
2288                vpxor   16*i(arg3, %r11), \T5, \T2
2289                .if \ENC_DEC == ENC
2290                vaesenclast     \T2, reg_j, reg_j
2291                .else
2292                vaesenclast     \T2, reg_j, \T3
2293                vmovdqu 16*i(arg3, %r11), reg_j
2294                vmovdqu \T3, 16*i(arg2, %r11)
2295                .endif
2296        i = (i+1)
2297        j = (j+1)
2298        setreg
2299.endr
2300        #######################################################################
2301
2302
2303        vpslldq $8, \T6, \T3                            # shift-L T3 2 DWs
2304        vpsrldq $8, \T6, \T6                            # shift-R T2 2 DWs
2305        vpxor   \T3, \T7, \T7
2306        vpxor   \T6, \T1, \T1                           # accumulate the results in T1:T7
2307
2308
2309
2310        #######################################################################
2311        #first phase of the reduction
2312        vmovdqa         POLY2(%rip), \T3
2313
2314        vpclmulqdq      $0x01, \T7, \T3, \T2
2315        vpslldq         $8, \T2, \T2                    # shift-L xmm2 2 DWs
2316
2317        vpxor           \T2, \T7, \T7                   # first phase of the reduction complete
2318        #######################################################################
2319                .if \ENC_DEC == ENC
2320                vmovdqu  \XMM1, 16*0(arg2,%r11)         # Write to the Ciphertext buffer
2321                vmovdqu  \XMM2, 16*1(arg2,%r11)         # Write to the Ciphertext buffer
2322                vmovdqu  \XMM3, 16*2(arg2,%r11)         # Write to the Ciphertext buffer
2323                vmovdqu  \XMM4, 16*3(arg2,%r11)         # Write to the Ciphertext buffer
2324                vmovdqu  \XMM5, 16*4(arg2,%r11)         # Write to the Ciphertext buffer
2325                vmovdqu  \XMM6, 16*5(arg2,%r11)         # Write to the Ciphertext buffer
2326                vmovdqu  \XMM7, 16*6(arg2,%r11)         # Write to the Ciphertext buffer
2327                vmovdqu  \XMM8, 16*7(arg2,%r11)         # Write to the Ciphertext buffer
2328                .endif
2329
2330        #######################################################################
2331        #second phase of the reduction
2332        vpclmulqdq      $0x00, \T7, \T3, \T2
2333        vpsrldq         $4, \T2, \T2                    # shift-R xmm2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R)
2334
2335        vpclmulqdq      $0x10, \T7, \T3, \T4
2336        vpslldq         $4, \T4, \T4                    # shift-L xmm0 1 DW (Shift-L 1-DW to obtain result with no shifts)
2337
2338        vpxor           \T2, \T4, \T4                   # second phase of the reduction complete
2339        #######################################################################
2340        vpxor           \T4, \T1, \T1                   # the result is in T1
2341
2342                vpshufb SHUF_MASK(%rip), \XMM1, \XMM1   # perform a 16Byte swap
2343                vpshufb SHUF_MASK(%rip), \XMM2, \XMM2   # perform a 16Byte swap
2344                vpshufb SHUF_MASK(%rip), \XMM3, \XMM3   # perform a 16Byte swap
2345                vpshufb SHUF_MASK(%rip), \XMM4, \XMM4   # perform a 16Byte swap
2346                vpshufb SHUF_MASK(%rip), \XMM5, \XMM5   # perform a 16Byte swap
2347                vpshufb SHUF_MASK(%rip), \XMM6, \XMM6   # perform a 16Byte swap
2348                vpshufb SHUF_MASK(%rip), \XMM7, \XMM7   # perform a 16Byte swap
2349                vpshufb SHUF_MASK(%rip), \XMM8, \XMM8   # perform a 16Byte swap
2350
2351
2352        vpxor   \T1, \XMM1, \XMM1
2353
2354
2355
2356.endm
2357
2358
2359# GHASH the last 4 ciphertext blocks.
2360.macro  GHASH_LAST_8_AVX2 T1 T2 T3 T4 T5 T6 T7 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8
2361
2362        ## Karatsuba Method
2363
2364        vmovdqa         HashKey_8(arg1), \T5
2365
2366        vpshufd         $0b01001110, \XMM1, \T2
2367        vpshufd         $0b01001110, \T5, \T3
2368        vpxor           \XMM1, \T2, \T2
2369        vpxor           \T5, \T3, \T3
2370
2371        vpclmulqdq      $0x11, \T5, \XMM1, \T6
2372        vpclmulqdq      $0x00, \T5, \XMM1, \T7
2373
2374        vpclmulqdq      $0x00, \T3, \T2, \XMM1
2375
2376        ######################
2377
2378        vmovdqa         HashKey_7(arg1), \T5
2379        vpshufd         $0b01001110, \XMM2, \T2
2380        vpshufd         $0b01001110, \T5, \T3
2381        vpxor           \XMM2, \T2, \T2
2382        vpxor           \T5, \T3, \T3
2383
2384        vpclmulqdq      $0x11, \T5, \XMM2, \T4
2385        vpxor           \T4, \T6, \T6
2386
2387        vpclmulqdq      $0x00, \T5, \XMM2, \T4
2388        vpxor           \T4, \T7, \T7
2389
2390        vpclmulqdq      $0x00, \T3, \T2, \T2
2391
2392        vpxor           \T2, \XMM1, \XMM1
2393
2394        ######################
2395
2396        vmovdqa         HashKey_6(arg1), \T5
2397        vpshufd         $0b01001110, \XMM3, \T2
2398        vpshufd         $0b01001110, \T5, \T3
2399        vpxor           \XMM3, \T2, \T2
2400        vpxor           \T5, \T3, \T3
2401
2402        vpclmulqdq      $0x11, \T5, \XMM3, \T4
2403        vpxor           \T4, \T6, \T6
2404
2405        vpclmulqdq      $0x00, \T5, \XMM3, \T4
2406        vpxor           \T4, \T7, \T7
2407
2408        vpclmulqdq      $0x00, \T3, \T2, \T2
2409
2410        vpxor           \T2, \XMM1, \XMM1
2411
2412        ######################
2413
2414        vmovdqa         HashKey_5(arg1), \T5
2415        vpshufd         $0b01001110, \XMM4, \T2
2416        vpshufd         $0b01001110, \T5, \T3
2417        vpxor           \XMM4, \T2, \T2
2418        vpxor           \T5, \T3, \T3
2419
2420        vpclmulqdq      $0x11, \T5, \XMM4, \T4
2421        vpxor           \T4, \T6, \T6
2422
2423        vpclmulqdq      $0x00, \T5, \XMM4, \T4
2424        vpxor           \T4, \T7, \T7
2425
2426        vpclmulqdq      $0x00, \T3, \T2, \T2
2427
2428        vpxor           \T2, \XMM1, \XMM1
2429
2430        ######################
2431
2432        vmovdqa         HashKey_4(arg1), \T5
2433        vpshufd         $0b01001110, \XMM5, \T2
2434        vpshufd         $0b01001110, \T5, \T3
2435        vpxor           \XMM5, \T2, \T2
2436        vpxor           \T5, \T3, \T3
2437
2438        vpclmulqdq      $0x11, \T5, \XMM5, \T4
2439        vpxor           \T4, \T6, \T6
2440
2441        vpclmulqdq      $0x00, \T5, \XMM5, \T4
2442        vpxor           \T4, \T7, \T7
2443
2444        vpclmulqdq      $0x00, \T3, \T2, \T2
2445
2446        vpxor           \T2, \XMM1, \XMM1
2447
2448        ######################
2449
2450        vmovdqa         HashKey_3(arg1), \T5
2451        vpshufd         $0b01001110, \XMM6, \T2
2452        vpshufd         $0b01001110, \T5, \T3
2453        vpxor           \XMM6, \T2, \T2
2454        vpxor           \T5, \T3, \T3
2455
2456        vpclmulqdq      $0x11, \T5, \XMM6, \T4
2457        vpxor           \T4, \T6, \T6
2458
2459        vpclmulqdq      $0x00, \T5, \XMM6, \T4
2460        vpxor           \T4, \T7, \T7
2461
2462        vpclmulqdq      $0x00, \T3, \T2, \T2
2463
2464        vpxor           \T2, \XMM1, \XMM1
2465
2466        ######################
2467
2468        vmovdqa         HashKey_2(arg1), \T5
2469        vpshufd         $0b01001110, \XMM7, \T2
2470        vpshufd         $0b01001110, \T5, \T3
2471        vpxor           \XMM7, \T2, \T2
2472        vpxor           \T5, \T3, \T3
2473
2474        vpclmulqdq      $0x11, \T5, \XMM7, \T4
2475        vpxor           \T4, \T6, \T6
2476
2477        vpclmulqdq      $0x00, \T5, \XMM7, \T4
2478        vpxor           \T4, \T7, \T7
2479
2480        vpclmulqdq      $0x00, \T3, \T2, \T2
2481
2482        vpxor           \T2, \XMM1, \XMM1
2483
2484        ######################
2485
2486        vmovdqa         HashKey(arg1), \T5
2487        vpshufd         $0b01001110, \XMM8, \T2
2488        vpshufd         $0b01001110, \T5, \T3
2489        vpxor           \XMM8, \T2, \T2
2490        vpxor           \T5, \T3, \T3
2491
2492        vpclmulqdq      $0x11, \T5, \XMM8, \T4
2493        vpxor           \T4, \T6, \T6
2494
2495        vpclmulqdq      $0x00, \T5, \XMM8, \T4
2496        vpxor           \T4, \T7, \T7
2497
2498        vpclmulqdq      $0x00, \T3, \T2, \T2
2499
2500        vpxor           \T2, \XMM1, \XMM1
2501        vpxor           \T6, \XMM1, \XMM1
2502        vpxor           \T7, \XMM1, \T2
2503
2504
2505
2506
2507        vpslldq $8, \T2, \T4
2508        vpsrldq $8, \T2, \T2
2509
2510        vpxor   \T4, \T7, \T7
2511        vpxor   \T2, \T6, \T6                      # <T6:T7> holds the result of the
2512                                                   # accumulated carry-less multiplications
2513
2514        #######################################################################
2515        #first phase of the reduction
2516        vmovdqa         POLY2(%rip), \T3
2517
2518        vpclmulqdq      $0x01, \T7, \T3, \T2
2519        vpslldq         $8, \T2, \T2               # shift-L xmm2 2 DWs
2520
2521        vpxor           \T2, \T7, \T7              # first phase of the reduction complete
2522        #######################################################################
2523
2524
2525        #second phase of the reduction
2526        vpclmulqdq      $0x00, \T7, \T3, \T2
2527        vpsrldq         $4, \T2, \T2               # shift-R T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R)
2528
2529        vpclmulqdq      $0x10, \T7, \T3, \T4
2530        vpslldq         $4, \T4, \T4               # shift-L T4 1 DW (Shift-L 1-DW to obtain result with no shifts)
2531
2532        vpxor           \T2, \T4, \T4              # second phase of the reduction complete
2533        #######################################################################
2534        vpxor           \T4, \T6, \T6              # the result is in T6
2535.endm
2536
2537
2538
2539# combined for GCM encrypt and decrypt functions
2540# clobbering all xmm registers
2541# clobbering r10, r11, r12, r13, r14, r15
2542.macro  GCM_ENC_DEC_AVX2     ENC_DEC
2543
2544        #the number of pushes must equal STACK_OFFSET
2545        push    %r12
2546        push    %r13
2547        push    %r14
2548        push    %r15
2549
2550        mov     %rsp, %r14
2551
2552
2553
2554
2555        sub     $VARIABLE_OFFSET, %rsp
2556        and     $~63, %rsp                         # align rsp to 64 bytes
2557
2558
2559        vmovdqu  HashKey(arg1), %xmm13             # xmm13 = HashKey
2560
2561        mov     arg4, %r13                         # save the number of bytes of plaintext/ciphertext
2562        and     $-16, %r13                         # r13 = r13 - (r13 mod 16)
2563
2564        mov     %r13, %r12
2565        shr     $4, %r12
2566        and     $7, %r12
2567        jz      _initial_num_blocks_is_0\@
2568
2569        cmp     $7, %r12
2570        je      _initial_num_blocks_is_7\@
2571        cmp     $6, %r12
2572        je      _initial_num_blocks_is_6\@
2573        cmp     $5, %r12
2574        je      _initial_num_blocks_is_5\@
2575        cmp     $4, %r12
2576        je      _initial_num_blocks_is_4\@
2577        cmp     $3, %r12
2578        je      _initial_num_blocks_is_3\@
2579        cmp     $2, %r12
2580        je      _initial_num_blocks_is_2\@
2581
2582        jmp     _initial_num_blocks_is_1\@
2583
2584_initial_num_blocks_is_7\@:
2585        INITIAL_BLOCKS_AVX2  7, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
2586        sub     $16*7, %r13
2587        jmp     _initial_blocks_encrypted\@
2588
2589_initial_num_blocks_is_6\@:
2590        INITIAL_BLOCKS_AVX2  6, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
2591        sub     $16*6, %r13
2592        jmp     _initial_blocks_encrypted\@
2593
2594_initial_num_blocks_is_5\@:
2595        INITIAL_BLOCKS_AVX2  5, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
2596        sub     $16*5, %r13
2597        jmp     _initial_blocks_encrypted\@
2598
2599_initial_num_blocks_is_4\@:
2600        INITIAL_BLOCKS_AVX2  4, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
2601        sub     $16*4, %r13
2602        jmp     _initial_blocks_encrypted\@
2603
2604_initial_num_blocks_is_3\@:
2605        INITIAL_BLOCKS_AVX2  3, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
2606        sub     $16*3, %r13
2607        jmp     _initial_blocks_encrypted\@
2608
2609_initial_num_blocks_is_2\@:
2610        INITIAL_BLOCKS_AVX2  2, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
2611        sub     $16*2, %r13
2612        jmp     _initial_blocks_encrypted\@
2613
2614_initial_num_blocks_is_1\@:
2615        INITIAL_BLOCKS_AVX2  1, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
2616        sub     $16*1, %r13
2617        jmp     _initial_blocks_encrypted\@
2618
2619_initial_num_blocks_is_0\@:
2620        INITIAL_BLOCKS_AVX2  0, %xmm12, %xmm13, %xmm14, %xmm15, %xmm11, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm10, %xmm0, \ENC_DEC
2621
2622
2623_initial_blocks_encrypted\@:
2624        cmp     $0, %r13
2625        je      _zero_cipher_left\@
2626
2627        sub     $128, %r13
2628        je      _eight_cipher_left\@
2629
2630
2631
2632
2633        vmovd   %xmm9, %r15d
2634        and     $255, %r15d
2635        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
2636
2637
2638_encrypt_by_8_new\@:
2639        cmp     $(255-8), %r15d
2640        jg      _encrypt_by_8\@
2641
2642
2643
2644        add     $8, %r15b
2645        GHASH_8_ENCRYPT_8_PARALLEL_AVX2      %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm15, out_order, \ENC_DEC
2646        add     $128, %r11
2647        sub     $128, %r13
2648        jne     _encrypt_by_8_new\@
2649
2650        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
2651        jmp     _eight_cipher_left\@
2652
2653_encrypt_by_8\@:
2654        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
2655        add     $8, %r15b
2656        GHASH_8_ENCRYPT_8_PARALLEL_AVX2      %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm9, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8, %xmm15, in_order, \ENC_DEC
2657        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
2658        add     $128, %r11
2659        sub     $128, %r13
2660        jne     _encrypt_by_8_new\@
2661
2662        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
2663
2664
2665
2666
2667_eight_cipher_left\@:
2668        GHASH_LAST_8_AVX2    %xmm0, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14, %xmm15, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7, %xmm8
2669
2670
2671_zero_cipher_left\@:
2672        cmp     $16, arg4
2673        jl      _only_less_than_16\@
2674
2675        mov     arg4, %r13
2676        and     $15, %r13                            # r13 = (arg4 mod 16)
2677
2678        je      _multiple_of_16_bytes\@
2679
2680        # handle the last <16 Byte block seperately
2681
2682
2683        vpaddd   ONE(%rip), %xmm9, %xmm9             # INCR CNT to get Yn
2684        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
2685        ENCRYPT_SINGLE_BLOCK    %xmm9                # E(K, Yn)
2686
2687        sub     $16, %r11
2688        add     %r13, %r11
2689        vmovdqu (arg3, %r11), %xmm1                  # receive the last <16 Byte block
2690
2691        lea     SHIFT_MASK+16(%rip), %r12
2692        sub     %r13, %r12                           # adjust the shuffle mask pointer
2693                                                     # to be able to shift 16-r13 bytes
2694                                                     # (r13 is the number of bytes in plaintext mod 16)
2695        vmovdqu (%r12), %xmm2                        # get the appropriate shuffle mask
2696        vpshufb %xmm2, %xmm1, %xmm1                  # shift right 16-r13 bytes
2697        jmp     _final_ghash_mul\@
2698
2699_only_less_than_16\@:
2700        # check for 0 length
2701        mov     arg4, %r13
2702        and     $15, %r13                            # r13 = (arg4 mod 16)
2703
2704        je      _multiple_of_16_bytes\@
2705
2706        # handle the last <16 Byte block seperately
2707
2708
2709        vpaddd  ONE(%rip), %xmm9, %xmm9              # INCR CNT to get Yn
2710        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
2711        ENCRYPT_SINGLE_BLOCK    %xmm9                # E(K, Yn)
2712
2713
2714        lea     SHIFT_MASK+16(%rip), %r12
2715        sub     %r13, %r12                           # adjust the shuffle mask pointer to be
2716                                                     # able to shift 16-r13 bytes (r13 is the
2717                                                     # number of bytes in plaintext mod 16)
2718
2719_get_last_16_byte_loop\@:
2720        movb    (arg3, %r11),  %al
2721        movb    %al,  TMP1 (%rsp , %r11)
2722        add     $1, %r11
2723        cmp     %r13,  %r11
2724        jne     _get_last_16_byte_loop\@
2725
2726        vmovdqu  TMP1(%rsp), %xmm1
2727
2728        sub     $16, %r11
2729
2730_final_ghash_mul\@:
2731        .if  \ENC_DEC ==  DEC
2732        vmovdqa %xmm1, %xmm2
2733        vpxor   %xmm1, %xmm9, %xmm9                  # Plaintext XOR E(K, Yn)
2734        vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1        # get the appropriate mask to mask out top 16-r13 bytes of xmm9
2735        vpand   %xmm1, %xmm9, %xmm9                  # mask out top 16-r13 bytes of xmm9
2736        vpand   %xmm1, %xmm2, %xmm2
2737        vpshufb SHUF_MASK(%rip), %xmm2, %xmm2
2738        vpxor   %xmm2, %xmm14, %xmm14
2739        #GHASH computation for the last <16 Byte block
2740        GHASH_MUL_AVX2       %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
2741        sub     %r13, %r11
2742        add     $16, %r11
2743        .else
2744        vpxor   %xmm1, %xmm9, %xmm9                  # Plaintext XOR E(K, Yn)
2745        vmovdqu ALL_F-SHIFT_MASK(%r12), %xmm1        # get the appropriate mask to mask out top 16-r13 bytes of xmm9
2746        vpand   %xmm1, %xmm9, %xmm9                  # mask out top 16-r13 bytes of xmm9
2747        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9
2748        vpxor   %xmm9, %xmm14, %xmm14
2749        #GHASH computation for the last <16 Byte block
2750        GHASH_MUL_AVX2       %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6
2751        sub     %r13, %r11
2752        add     $16, %r11
2753        vpshufb SHUF_MASK(%rip), %xmm9, %xmm9        # shuffle xmm9 back to output as ciphertext
2754        .endif
2755
2756
2757        #############################
2758        # output r13 Bytes
2759        vmovq   %xmm9, %rax
2760        cmp     $8, %r13
2761        jle     _less_than_8_bytes_left\@
2762
2763        mov     %rax, (arg2 , %r11)
2764        add     $8, %r11
2765        vpsrldq $8, %xmm9, %xmm9
2766        vmovq   %xmm9, %rax
2767        sub     $8, %r13
2768
2769_less_than_8_bytes_left\@:
2770        movb    %al, (arg2 , %r11)
2771        add     $1, %r11
2772        shr     $8, %rax
2773        sub     $1, %r13
2774        jne     _less_than_8_bytes_left\@
2775        #############################
2776
2777_multiple_of_16_bytes\@:
2778        mov     arg7, %r12                           # r12 = aadLen (number of bytes)
2779        shl     $3, %r12                             # convert into number of bits
2780        vmovd   %r12d, %xmm15                        # len(A) in xmm15
2781
2782        shl     $3, arg4                             # len(C) in bits  (*128)
2783        vmovq   arg4, %xmm1
2784        vpslldq $8, %xmm15, %xmm15                   # xmm15 = len(A)|| 0x0000000000000000
2785        vpxor   %xmm1, %xmm15, %xmm15                # xmm15 = len(A)||len(C)
2786
2787        vpxor   %xmm15, %xmm14, %xmm14
2788        GHASH_MUL_AVX2       %xmm14, %xmm13, %xmm0, %xmm10, %xmm11, %xmm5, %xmm6    # final GHASH computation
2789        vpshufb SHUF_MASK(%rip), %xmm14, %xmm14              # perform a 16Byte swap
2790
2791        mov     arg5, %rax                           # rax = *Y0
2792        vmovdqu (%rax), %xmm9                        # xmm9 = Y0
2793
2794        ENCRYPT_SINGLE_BLOCK    %xmm9                # E(K, Y0)
2795
2796        vpxor   %xmm14, %xmm9, %xmm9
2797
2798
2799
2800_return_T\@:
2801        mov     arg8, %r10              # r10 = authTag
2802        mov     arg9, %r11              # r11 = auth_tag_len
2803
2804        cmp     $16, %r11
2805        je      _T_16\@
2806
2807        cmp     $8, %r11
2808        jl      _T_4\@
2809
2810_T_8\@:
2811        vmovq   %xmm9, %rax
2812        mov     %rax, (%r10)
2813        add     $8, %r10
2814        sub     $8, %r11
2815        vpsrldq $8, %xmm9, %xmm9
2816        cmp     $0, %r11
2817        je     _return_T_done\@
2818_T_4\@:
2819        vmovd   %xmm9, %eax
2820        mov     %eax, (%r10)
2821        add     $4, %r10
2822        sub     $4, %r11
2823        vpsrldq     $4, %xmm9, %xmm9
2824        cmp     $0, %r11
2825        je     _return_T_done\@
2826_T_123\@:
2827        vmovd     %xmm9, %eax
2828        cmp     $2, %r11
2829        jl     _T_1\@
2830        mov     %ax, (%r10)
2831        cmp     $2, %r11
2832        je     _return_T_done\@
2833        add     $2, %r10
2834        sar     $16, %eax
2835_T_1\@:
2836        mov     %al, (%r10)
2837        jmp     _return_T_done\@
2838
2839_T_16\@:
2840        vmovdqu %xmm9, (%r10)
2841
2842_return_T_done\@:
2843        mov     %r14, %rsp
2844
2845        pop     %r15
2846        pop     %r14
2847        pop     %r13
2848        pop     %r12
2849.endm
2850
2851
2852#############################################################
2853#void   aesni_gcm_precomp_avx_gen4
2854#        (gcm_data     *my_ctx_data,
2855#        u8     *hash_subkey)# /* H, the Hash sub key input.
2856#                               Data starts on a 16-byte boundary. */
2857#############################################################
2858ENTRY(aesni_gcm_precomp_avx_gen4)
2859        #the number of pushes must equal STACK_OFFSET
2860        push    %r12
2861        push    %r13
2862        push    %r14
2863        push    %r15
2864
2865        mov     %rsp, %r14
2866
2867
2868
2869        sub     $VARIABLE_OFFSET, %rsp
2870        and     $~63, %rsp                    # align rsp to 64 bytes
2871
2872        vmovdqu  (arg2), %xmm6                # xmm6 = HashKey
2873
2874        vpshufb  SHUF_MASK(%rip), %xmm6, %xmm6
2875        ###############  PRECOMPUTATION of HashKey<<1 mod poly from the HashKey
2876        vmovdqa  %xmm6, %xmm2
2877        vpsllq   $1, %xmm6, %xmm6
2878        vpsrlq   $63, %xmm2, %xmm2
2879        vmovdqa  %xmm2, %xmm1
2880        vpslldq  $8, %xmm2, %xmm2
2881        vpsrldq  $8, %xmm1, %xmm1
2882        vpor     %xmm2, %xmm6, %xmm6
2883        #reduction
2884        vpshufd  $0b00100100, %xmm1, %xmm2
2885        vpcmpeqd TWOONE(%rip), %xmm2, %xmm2
2886        vpand    POLY(%rip), %xmm2, %xmm2
2887        vpxor    %xmm2, %xmm6, %xmm6          # xmm6 holds the HashKey<<1 mod poly
2888        #######################################################################
2889        vmovdqa  %xmm6, HashKey(arg1)         # store HashKey<<1 mod poly
2890
2891
2892        PRECOMPUTE_AVX2  %xmm6, %xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5
2893
2894        mov     %r14, %rsp
2895
2896        pop     %r15
2897        pop     %r14
2898        pop     %r13
2899        pop     %r12
2900        ret
2901ENDPROC(aesni_gcm_precomp_avx_gen4)
2902
2903
2904###############################################################################
2905#void   aesni_gcm_enc_avx_gen4(
2906#        gcm_data        *my_ctx_data,     /* aligned to 16 Bytes */
2907#        u8      *out, /* Ciphertext output. Encrypt in-place is allowed.  */
2908#        const   u8 *in, /* Plaintext input */
2909#        u64     plaintext_len, /* Length of data in Bytes for encryption. */
2910#        u8      *iv, /* Pre-counter block j0: 4 byte salt
2911#                       (from Security Association) concatenated with 8 byte
2912#                        Initialisation Vector (from IPSec ESP Payload)
2913#                        concatenated with 0x00000001. 16-byte aligned pointer. */
2914#        const   u8 *aad, /* Additional Authentication Data (AAD)*/
2915#        u64     aad_len, /* Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 Bytes */
2916#        u8      *auth_tag, /* Authenticated Tag output. */
2917#        u64     auth_tag_len)# /* Authenticated Tag Length in bytes.
2918#                               Valid values are 16 (most likely), 12 or 8. */
2919###############################################################################
2920ENTRY(aesni_gcm_enc_avx_gen4)
2921        GCM_ENC_DEC_AVX2     ENC
2922        ret
2923ENDPROC(aesni_gcm_enc_avx_gen4)
2924
2925###############################################################################
2926#void   aesni_gcm_dec_avx_gen4(
2927#        gcm_data        *my_ctx_data,     /* aligned to 16 Bytes */
2928#        u8      *out, /* Plaintext output. Decrypt in-place is allowed.  */
2929#        const   u8 *in, /* Ciphertext input */
2930#        u64     plaintext_len, /* Length of data in Bytes for encryption. */
2931#        u8      *iv, /* Pre-counter block j0: 4 byte salt
2932#                       (from Security Association) concatenated with 8 byte
2933#                       Initialisation Vector (from IPSec ESP Payload)
2934#                       concatenated with 0x00000001. 16-byte aligned pointer. */
2935#        const   u8 *aad, /* Additional Authentication Data (AAD)*/
2936#        u64     aad_len, /* Length of AAD in bytes. With RFC4106 this is going to be 8 or 12 Bytes */
2937#        u8      *auth_tag, /* Authenticated Tag output. */
2938#        u64     auth_tag_len)# /* Authenticated Tag Length in bytes.
2939#                               Valid values are 16 (most likely), 12 or 8. */
2940###############################################################################
2941ENTRY(aesni_gcm_dec_avx_gen4)
2942        GCM_ENC_DEC_AVX2     DEC
2943        ret
2944ENDPROC(aesni_gcm_dec_avx_gen4)
2945
2946#endif /* CONFIG_AS_AVX2 */
2947