LXR linux/arch/x86/crypto/sha256-avx2-asm.S

   1########################################################################
   2# Implement fast SHA-256 with AVX2 instructions. (x86_64)
   3#
   4# Copyright (C) 2013 Intel Corporation.
   5#
   6# Authors:
   7#     James Guilford <james.guilford@intel.com>
   8#     Kirk Yap <kirk.s.yap@intel.com>
   9#     Tim Chen <tim.c.chen@linux.intel.com>
  10#
  11# This software is available to you under a choice of one of two
  12# licenses.  You may choose to be licensed under the terms of the GNU
  13# General Public License (GPL) Version 2, available from the file
  14# COPYING in the main directory of this source tree, or the
  15# OpenIB.org BSD license below:
  16#
  17#     Redistribution and use in source and binary forms, with or
  18#     without modification, are permitted provided that the following
  19#     conditions are met:
  20#
  21#      - Redistributions of source code must retain the above
  22#        copyright notice, this list of conditions and the following
  23#        disclaimer.
  24#
  25#      - Redistributions in binary form must reproduce the above
  26#        copyright notice, this list of conditions and the following
  27#        disclaimer in the documentation and/or other materials
  28#        provided with the distribution.
  29#
  30# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  31# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  32# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  33# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  34# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  35# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  36# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  37# SOFTWARE.
  38#
  39########################################################################
  40#
  41# This code is described in an Intel White-Paper:
  42# "Fast SHA-256 Implementations on Intel Architecture Processors"
  43#
  44# To find it, surf to http://www.intel.com/p/en_US/embedded
  45# and search for that title.
  46#
  47########################################################################
  48# This code schedules 2 blocks at a time, with 4 lanes per block
  49########################################################################
  50
  51#include <linux/linkage.h>
  52
  53## assume buffers not aligned
  54#define VMOVDQ vmovdqu
  55
  56################################ Define Macros
  57
  58# addm [mem], reg
  59# Add reg to mem using reg-mem add and store
  60.macro addm p1 p2
  61        add     \p1, \p2
  62        mov     \p2, \p1
  63.endm
  64
  65################################
  66
  67X0 = %ymm4
  68X1 = %ymm5
  69X2 = %ymm6
  70X3 = %ymm7
  71
  72# XMM versions of above
  73XWORD0 = %xmm4
  74XWORD1 = %xmm5
  75XWORD2 = %xmm6
  76XWORD3 = %xmm7
  77
  78XTMP0 = %ymm0
  79XTMP1 = %ymm1
  80XTMP2 = %ymm2
  81XTMP3 = %ymm3
  82XTMP4 = %ymm8
  83XFER  = %ymm9
  84XTMP5 = %ymm11
  85
  86SHUF_00BA =     %ymm10 # shuffle xBxA -> 00BA
  87SHUF_DC00 =     %ymm12 # shuffle xDxC -> DC00
  88BYTE_FLIP_MASK = %ymm13
  89
  90X_BYTE_FLIP_MASK = %xmm13 # XMM version of BYTE_FLIP_MASK
  91
  92NUM_BLKS = %rdx # 3rd arg
  93INP     = %rsi  # 2nd arg
  94CTX     = %rdi  # 1st arg
  95c       = %ecx
  96d       = %r8d
  97e       = %edx  # clobbers NUM_BLKS
  98y3      = %esi  # clobbers INP
  99
 100SRND    = CTX   # SRND is same register as CTX
 101
 102a = %eax
 103b = %ebx
 104f = %r9d
 105g = %r10d
 106h = %r11d
 107old_h = %r11d
 108
 109T1 = %r12d
 110y0 = %r13d
 111y1 = %r14d
 112y2 = %r15d
 113
 114
 115_XFER_SIZE      = 2*64*4        # 2 blocks, 64 rounds, 4 bytes/round
 116_XMM_SAVE_SIZE  = 0
 117_INP_END_SIZE   = 8
 118_INP_SIZE       = 8
 119_CTX_SIZE       = 8
 120
 121_XFER           = 0
 122_XMM_SAVE       = _XFER     + _XFER_SIZE
 123_INP_END        = _XMM_SAVE + _XMM_SAVE_SIZE
 124_INP            = _INP_END  + _INP_END_SIZE
 125_CTX            = _INP      + _INP_SIZE
 126STACK_SIZE      = _CTX      + _CTX_SIZE
 127
 128# rotate_Xs
 129# Rotate values of symbols X0...X3
 130.macro rotate_Xs
 131        X_ = X0
 132        X0 = X1
 133        X1 = X2
 134        X2 = X3
 135        X3 = X_
 136.endm
 137
 138# ROTATE_ARGS
 139# Rotate values of symbols a...h
 140.macro ROTATE_ARGS
 141        old_h = h
 142        TMP_ = h
 143        h = g
 144        g = f
 145        f = e
 146        e = d
 147        d = c
 148        c = b
 149        b = a
 150        a = TMP_
 151.endm
 152
 153.macro FOUR_ROUNDS_AND_SCHED disp
 154################################### RND N + 0 ############################
 155
 156        mov     a, y3           # y3 = a                                # MAJA
 157        rorx    $25, e, y0      # y0 = e >> 25                          # S1A
 158        rorx    $11, e, y1      # y1 = e >> 11                          # S1B
 159
 160        addl    \disp(%rsp, SRND), h            # h = k + w + h         # --
 161        or      c, y3           # y3 = a|c                              # MAJA
 162        vpalignr $4, X2, X3, XTMP0 # XTMP0 = W[-7]
 163        mov     f, y2           # y2 = f                                # CH
 164        rorx    $13, a, T1      # T1 = a >> 13                          # S0B
 165
 166        xor     y1, y0          # y0 = (e>>25) ^ (e>>11)                # S1
 167        xor     g, y2           # y2 = f^g                              # CH
 168        vpaddd  X0, XTMP0, XTMP0 # XTMP0 = W[-7] + W[-16]# y1 = (e >> 6)# S1
 169        rorx    $6, e, y1       # y1 = (e >> 6)                         # S1
 170
 171        and     e, y2           # y2 = (f^g)&e                          # CH
 172        xor     y1, y0          # y0 = (e>>25) ^ (e>>11) ^ (e>>6)       # S1
 173        rorx    $22, a, y1      # y1 = a >> 22                          # S0A
 174        add     h, d            # d = k + w + h + d                     # --
 175
 176        and     b, y3           # y3 = (a|c)&b                          # MAJA
 177        vpalignr $4, X0, X1, XTMP1      # XTMP1 = W[-15]
 178        xor     T1, y1          # y1 = (a>>22) ^ (a>>13)                # S0
 179        rorx    $2, a, T1       # T1 = (a >> 2)                         # S0
 180
 181        xor     g, y2           # y2 = CH = ((f^g)&e)^g                 # CH
 182        vpsrld  $7, XTMP1, XTMP2
 183        xor     T1, y1          # y1 = (a>>22) ^ (a>>13) ^ (a>>2)       # S0
 184        mov     a, T1           # T1 = a                                # MAJB
 185        and     c, T1           # T1 = a&c                              # MAJB
 186
 187        add     y0, y2          # y2 = S1 + CH                          # --
 188        vpslld  $(32-7), XTMP1, XTMP3
 189        or      T1, y3          # y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
 190        add     y1, h           # h = k + w + h + S0                    # --
 191
 192        add     y2, d           # d = k + w + h + d + S1 + CH = d + t1  # --
 193        vpor    XTMP2, XTMP3, XTMP3     # XTMP3 = W[-15] ror 7
 194
 195        vpsrld  $18, XTMP1, XTMP2
 196        add     y2, h           # h = k + w + h + S0 + S1 + CH = t1 + S0# --
 197        add     y3, h           # h = t1 + S0 + MAJ                     # --
 198
 199
 200        ROTATE_ARGS
 201
 202################################### RND N + 1 ############################
 203
 204        mov     a, y3           # y3 = a                                # MAJA
 205        rorx    $25, e, y0      # y0 = e >> 25                          # S1A
 206        rorx    $11, e, y1      # y1 = e >> 11                          # S1B
 207        offset = \disp + 1*4
 208        addl    offset(%rsp, SRND), h   # h = k + w + h         # --
 209        or      c, y3           # y3 = a|c                              # MAJA
 210
 211
 212        vpsrld  $3, XTMP1, XTMP4 # XTMP4 = W[-15] >> 3
 213        mov     f, y2           # y2 = f                                # CH
 214        rorx    $13, a, T1      # T1 = a >> 13                          # S0B
 215        xor     y1, y0          # y0 = (e>>25) ^ (e>>11)                # S1
 216        xor     g, y2           # y2 = f^g                              # CH
 217
 218
 219        rorx    $6, e, y1       # y1 = (e >> 6)                         # S1
 220        xor     y1, y0          # y0 = (e>>25) ^ (e>>11) ^ (e>>6)       # S1
 221        rorx    $22, a, y1      # y1 = a >> 22                          # S0A
 222        and     e, y2           # y2 = (f^g)&e                          # CH
 223        add     h, d            # d = k + w + h + d                     # --
 224
 225        vpslld  $(32-18), XTMP1, XTMP1
 226        and     b, y3           # y3 = (a|c)&b                          # MAJA
 227        xor     T1, y1          # y1 = (a>>22) ^ (a>>13)                # S0
 228
 229        vpxor   XTMP1, XTMP3, XTMP3
 230        rorx    $2, a, T1       # T1 = (a >> 2)                         # S0
 231        xor     g, y2           # y2 = CH = ((f^g)&e)^g                 # CH
 232
 233        vpxor   XTMP2, XTMP3, XTMP3     # XTMP3 = W[-15] ror 7 ^ W[-15] ror 18
 234        xor     T1, y1          # y1 = (a>>22) ^ (a>>13) ^ (a>>2)       # S0
 235        mov     a, T1           # T1 = a                                # MAJB
 236        and     c, T1           # T1 = a&c                              # MAJB
 237        add     y0, y2          # y2 = S1 + CH                          # --
 238
 239        vpxor   XTMP4, XTMP3, XTMP1     # XTMP1 = s0
 240        vpshufd $0b11111010, X3, XTMP2  # XTMP2 = W[-2] {BBAA}
 241        or      T1, y3          # y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
 242        add     y1, h           # h = k + w + h + S0                    # --
 243
 244        vpaddd  XTMP1, XTMP0, XTMP0     # XTMP0 = W[-16] + W[-7] + s0
 245        add     y2, d           # d = k + w + h + d + S1 + CH = d + t1  # --
 246        add     y2, h           # h = k + w + h + S0 + S1 + CH = t1 + S0# --
 247        add     y3, h           # h = t1 + S0 + MAJ                     # --
 248
 249        vpsrld  $10, XTMP2, XTMP4 # XTMP4 = W[-2] >> 10 {BBAA}
 250
 251
 252        ROTATE_ARGS
 253
 254################################### RND N + 2 ############################
 255
 256        mov     a, y3           # y3 = a                                # MAJA
 257        rorx    $25, e, y0      # y0 = e >> 25                          # S1A
 258        offset = \disp + 2*4
 259        addl    offset(%rsp, SRND), h   # h = k + w + h         # --
 260
 261        vpsrlq  $19, XTMP2, XTMP3 # XTMP3 = W[-2] ror 19 {xBxA}
 262        rorx    $11, e, y1      # y1 = e >> 11                          # S1B
 263        or      c, y3           # y3 = a|c                              # MAJA
 264        mov     f, y2           # y2 = f                                # CH
 265        xor     g, y2           # y2 = f^g                              # CH
 266
 267        rorx    $13, a, T1      # T1 = a >> 13                          # S0B
 268        xor     y1, y0          # y0 = (e>>25) ^ (e>>11)                # S1
 269        vpsrlq  $17, XTMP2, XTMP2       # XTMP2 = W[-2] ror 17 {xBxA}
 270        and     e, y2           # y2 = (f^g)&e                          # CH
 271
 272        rorx    $6, e, y1       # y1 = (e >> 6)                         # S1
 273        vpxor   XTMP3, XTMP2, XTMP2
 274        add     h, d            # d = k + w + h + d                     # --
 275        and     b, y3           # y3 = (a|c)&b                          # MAJA
 276
 277        xor     y1, y0          # y0 = (e>>25) ^ (e>>11) ^ (e>>6)       # S1
 278        rorx    $22, a, y1      # y1 = a >> 22                          # S0A
 279        vpxor   XTMP2, XTMP4, XTMP4     # XTMP4 = s1 {xBxA}
 280        xor     g, y2           # y2 = CH = ((f^g)&e)^g                 # CH
 281
 282        vpshufb SHUF_00BA, XTMP4, XTMP4 # XTMP4 = s1 {00BA}
 283        xor     T1, y1          # y1 = (a>>22) ^ (a>>13)                # S0
 284        rorx    $2, a ,T1       # T1 = (a >> 2)                         # S0
 285        vpaddd  XTMP4, XTMP0, XTMP0     # XTMP0 = {..., ..., W[1], W[0]}
 286
 287        xor     T1, y1          # y1 = (a>>22) ^ (a>>13) ^ (a>>2)       # S0
 288        mov     a, T1           # T1 = a                                # MAJB
 289        and     c, T1           # T1 = a&c                              # MAJB
 290        add     y0, y2          # y2 = S1 + CH                          # --
 291        vpshufd $0b01010000, XTMP0, XTMP2       # XTMP2 = W[-2] {DDCC}
 292
 293        or      T1, y3          # y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
 294        add     y1,h            # h = k + w + h + S0                    # --
 295        add     y2,d            # d = k + w + h + d + S1 + CH = d + t1  # --
 296        add     y2,h            # h = k + w + h + S0 + S1 + CH = t1 + S0# --
 297
 298        add     y3,h            # h = t1 + S0 + MAJ                     # --
 299
 300
 301        ROTATE_ARGS
 302
 303################################### RND N + 3 ############################
 304
 305        mov     a, y3           # y3 = a                                # MAJA
 306        rorx    $25, e, y0      # y0 = e >> 25                          # S1A
 307        rorx    $11, e, y1      # y1 = e >> 11                          # S1B
 308        offset = \disp + 3*4
 309        addl    offset(%rsp, SRND), h   # h = k + w + h         # --
 310        or      c, y3           # y3 = a|c                              # MAJA
 311
 312
 313        vpsrld  $10, XTMP2, XTMP5       # XTMP5 = W[-2] >> 10 {DDCC}
 314        mov     f, y2           # y2 = f                                # CH
 315        rorx    $13, a, T1      # T1 = a >> 13                          # S0B
 316        xor     y1, y0          # y0 = (e>>25) ^ (e>>11)                # S1
 317        xor     g, y2           # y2 = f^g                              # CH
 318
 319
 320        vpsrlq  $19, XTMP2, XTMP3       # XTMP3 = W[-2] ror 19 {xDxC}
 321        rorx    $6, e, y1       # y1 = (e >> 6)                         # S1
 322        and     e, y2           # y2 = (f^g)&e                          # CH
 323        add     h, d            # d = k + w + h + d                     # --
 324        and     b, y3           # y3 = (a|c)&b                          # MAJA
 325
 326        vpsrlq  $17, XTMP2, XTMP2       # XTMP2 = W[-2] ror 17 {xDxC}
 327        xor     y1, y0          # y0 = (e>>25) ^ (e>>11) ^ (e>>6)       # S1
 328        xor     g, y2           # y2 = CH = ((f^g)&e)^g                 # CH
 329
 330        vpxor   XTMP3, XTMP2, XTMP2
 331        rorx    $22, a, y1      # y1 = a >> 22                          # S0A
 332        add     y0, y2          # y2 = S1 + CH                          # --
 333
 334        vpxor   XTMP2, XTMP5, XTMP5     # XTMP5 = s1 {xDxC}
 335        xor     T1, y1          # y1 = (a>>22) ^ (a>>13)                # S0
 336        add     y2, d           # d = k + w + h + d + S1 + CH = d + t1  # --
 337
 338        rorx    $2, a, T1       # T1 = (a >> 2)                         # S0
 339        vpshufb SHUF_DC00, XTMP5, XTMP5 # XTMP5 = s1 {DC00}
 340
 341        vpaddd  XTMP0, XTMP5, X0        # X0 = {W[3], W[2], W[1], W[0]}
 342        xor     T1, y1          # y1 = (a>>22) ^ (a>>13) ^ (a>>2)       # S0
 343        mov     a, T1           # T1 = a                                # MAJB
 344        and     c, T1           # T1 = a&c                              # MAJB
 345        or      T1, y3          # y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
 346
 347        add     y1, h           # h = k + w + h + S0                    # --
 348        add     y2, h           # h = k + w + h + S0 + S1 + CH = t1 + S0# --
 349        add     y3, h           # h = t1 + S0 + MAJ                     # --
 350
 351        ROTATE_ARGS
 352        rotate_Xs
 353.endm
 354
 355.macro DO_4ROUNDS disp
 356################################### RND N + 0 ###########################
 357
 358        mov     f, y2           # y2 = f                                # CH
 359        rorx    $25, e, y0      # y0 = e >> 25                          # S1A
 360        rorx    $11, e, y1      # y1 = e >> 11                          # S1B
 361        xor     g, y2           # y2 = f^g                              # CH
 362
 363        xor     y1, y0          # y0 = (e>>25) ^ (e>>11)                # S1
 364        rorx    $6, e, y1       # y1 = (e >> 6)                         # S1
 365        and     e, y2           # y2 = (f^g)&e                          # CH
 366
 367        xor     y1, y0          # y0 = (e>>25) ^ (e>>11) ^ (e>>6)       # S1
 368        rorx    $13, a, T1      # T1 = a >> 13                          # S0B
 369        xor     g, y2           # y2 = CH = ((f^g)&e)^g                 # CH
 370        rorx    $22, a, y1      # y1 = a >> 22                          # S0A
 371        mov     a, y3           # y3 = a                                # MAJA
 372
 373        xor     T1, y1          # y1 = (a>>22) ^ (a>>13)                # S0
 374        rorx    $2, a, T1       # T1 = (a >> 2)                         # S0
 375        addl    \disp(%rsp, SRND), h            # h = k + w + h # --
 376        or      c, y3           # y3 = a|c                              # MAJA
 377
 378        xor     T1, y1          # y1 = (a>>22) ^ (a>>13) ^ (a>>2)       # S0
 379        mov     a, T1           # T1 = a                                # MAJB
 380        and     b, y3           # y3 = (a|c)&b                          # MAJA
 381        and     c, T1           # T1 = a&c                              # MAJB
 382        add     y0, y2          # y2 = S1 + CH                          # --
 383
 384
 385        add     h, d            # d = k + w + h + d                     # --
 386        or      T1, y3          # y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
 387        add     y1, h           # h = k + w + h + S0                    # --
 388        add     y2, d           # d = k + w + h + d + S1 + CH = d + t1  # --
 389
 390        ROTATE_ARGS
 391
 392################################### RND N + 1 ###########################
 393
 394        add     y2, old_h       # h = k + w + h + S0 + S1 + CH = t1 + S0# --
 395        mov     f, y2           # y2 = f                                # CH
 396        rorx    $25, e, y0      # y0 = e >> 25                          # S1A
 397        rorx    $11, e, y1      # y1 = e >> 11                          # S1B
 398        xor     g, y2           # y2 = f^g                              # CH
 399
 400        xor     y1, y0          # y0 = (e>>25) ^ (e>>11)                # S1
 401        rorx    $6, e, y1       # y1 = (e >> 6)                         # S1
 402        and     e, y2           # y2 = (f^g)&e                          # CH
 403        add     y3, old_h       # h = t1 + S0 + MAJ                     # --
 404
 405        xor     y1, y0          # y0 = (e>>25) ^ (e>>11) ^ (e>>6)       # S1
 406        rorx    $13, a, T1      # T1 = a >> 13                          # S0B
 407        xor     g, y2           # y2 = CH = ((f^g)&e)^g                 # CH
 408        rorx    $22, a, y1      # y1 = a >> 22                          # S0A
 409        mov     a, y3           # y3 = a                                # MAJA
 410
 411        xor     T1, y1          # y1 = (a>>22) ^ (a>>13)                # S0
 412        rorx    $2, a, T1       # T1 = (a >> 2)                         # S0
 413        offset = 4*1 + \disp
 414        addl    offset(%rsp, SRND), h           # h = k + w + h # --
 415        or      c, y3           # y3 = a|c                              # MAJA
 416
 417        xor     T1, y1          # y1 = (a>>22) ^ (a>>13) ^ (a>>2)       # S0
 418        mov     a, T1           # T1 = a                                # MAJB
 419        and     b, y3           # y3 = (a|c)&b                          # MAJA
 420        and     c, T1           # T1 = a&c                              # MAJB
 421        add     y0, y2          # y2 = S1 + CH                          # --
 422
 423
 424        add     h, d            # d = k + w + h + d                     # --
 425        or      T1, y3          # y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
 426        add     y1, h           # h = k + w + h + S0                    # --
 427
 428        add     y2, d           # d = k + w + h + d + S1 + CH = d + t1  # --
 429
 430        ROTATE_ARGS
 431
 432################################### RND N + 2 ##############################
 433
 434        add     y2, old_h       # h = k + w + h + S0 + S1 + CH = t1 + S0# --
 435        mov     f, y2           # y2 = f                                # CH
 436        rorx    $25, e, y0      # y0 = e >> 25                          # S1A
 437        rorx    $11, e, y1      # y1 = e >> 11                          # S1B
 438        xor     g, y2           # y2 = f^g                              # CH
 439
 440        xor     y1, y0          # y0 = (e>>25) ^ (e>>11)                # S1
 441        rorx    $6, e, y1       # y1 = (e >> 6)                         # S1
 442        and     e, y2           # y2 = (f^g)&e                          # CH
 443        add     y3, old_h       # h = t1 + S0 + MAJ                     # --
 444
 445        xor     y1, y0          # y0 = (e>>25) ^ (e>>11) ^ (e>>6)       # S1
 446        rorx    $13, a, T1      # T1 = a >> 13                          # S0B
 447        xor     g, y2           # y2 = CH = ((f^g)&e)^g                 # CH
 448        rorx    $22, a, y1      # y1 = a >> 22                          # S0A
 449        mov     a, y3           # y3 = a                                # MAJA
 450
 451        xor     T1, y1          # y1 = (a>>22) ^ (a>>13)                # S0
 452        rorx    $2, a, T1       # T1 = (a >> 2)                         # S0
 453        offset = 4*2 + \disp
 454        addl    offset(%rsp, SRND), h           # h = k + w + h # --
 455        or      c, y3           # y3 = a|c                              # MAJA
 456
 457        xor     T1, y1          # y1 = (a>>22) ^ (a>>13) ^ (a>>2)       # S0
 458        mov     a, T1           # T1 = a                                # MAJB
 459        and     b, y3           # y3 = (a|c)&b                          # MAJA
 460        and     c, T1           # T1 = a&c                              # MAJB
 461        add     y0, y2          # y2 = S1 + CH                          # --
 462
 463
 464        add     h, d            # d = k + w + h + d                     # --
 465        or      T1, y3          # y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
 466        add     y1, h           # h = k + w + h + S0                    # --
 467
 468        add     y2, d           # d = k + w + h + d + S1 + CH = d + t1  # --
 469
 470        ROTATE_ARGS
 471
 472################################### RND N + 3 ###########################
 473
 474        add     y2, old_h       # h = k + w + h + S0 + S1 + CH = t1 + S0# --
 475        mov     f, y2           # y2 = f                                # CH
 476        rorx    $25, e, y0      # y0 = e >> 25                          # S1A
 477        rorx    $11, e, y1      # y1 = e >> 11                          # S1B
 478        xor     g, y2           # y2 = f^g                              # CH
 479
 480        xor     y1, y0          # y0 = (e>>25) ^ (e>>11)                # S1
 481        rorx    $6, e, y1       # y1 = (e >> 6)                         # S1
 482        and     e, y2           # y2 = (f^g)&e                          # CH
 483        add     y3, old_h       # h = t1 + S0 + MAJ                     # --
 484
 485        xor     y1, y0          # y0 = (e>>25) ^ (e>>11) ^ (e>>6)       # S1
 486        rorx    $13, a, T1      # T1 = a >> 13                          # S0B
 487        xor     g, y2           # y2 = CH = ((f^g)&e)^g                 # CH
 488        rorx    $22, a, y1      # y1 = a >> 22                          # S0A
 489        mov     a, y3           # y3 = a                                # MAJA
 490
 491        xor     T1, y1          # y1 = (a>>22) ^ (a>>13)                # S0
 492        rorx    $2, a, T1       # T1 = (a >> 2)                         # S0
 493        offset = 4*3 + \disp
 494        addl    offset(%rsp, SRND), h           # h = k + w + h # --
 495        or      c, y3           # y3 = a|c                              # MAJA
 496
 497        xor     T1, y1          # y1 = (a>>22) ^ (a>>13) ^ (a>>2)       # S0
 498        mov     a, T1           # T1 = a                                # MAJB
 499        and     b, y3           # y3 = (a|c)&b                          # MAJA
 500        and     c, T1           # T1 = a&c                              # MAJB
 501        add     y0, y2          # y2 = S1 + CH                          # --
 502
 503
 504        add     h, d            # d = k + w + h + d                     # --
 505        or      T1, y3          # y3 = MAJ = (a|c)&b)|(a&c)             # MAJ
 506        add     y1, h           # h = k + w + h + S0                    # --
 507
 508        add     y2, d           # d = k + w + h + d + S1 + CH = d + t1  # --
 509
 510
 511        add     y2, h           # h = k + w + h + S0 + S1 + CH = t1 + S0# --
 512
 513        add     y3, h           # h = t1 + S0 + MAJ                     # --
 514
 515        ROTATE_ARGS
 516
 517.endm
 518
 519########################################################################
 520## void sha256_transform_rorx(struct sha256_state *state, const u8 *data, int blocks)
 521## arg 1 : pointer to state
 522## arg 2 : pointer to input data
 523## arg 3 : Num blocks
 524########################################################################
 525.text
 526SYM_FUNC_START(sha256_transform_rorx)
 527.align 32
 528        pushq   %rbx
 529        pushq   %r12
 530        pushq   %r13
 531        pushq   %r14
 532        pushq   %r15
 533
 534        push    %rbp
 535        mov     %rsp, %rbp
 536
 537        subq    $STACK_SIZE, %rsp
 538        and     $-32, %rsp      # align rsp to 32 byte boundary
 539
 540        shl     $6, NUM_BLKS    # convert to bytes
 541        jz      done_hash
 542        lea     -64(INP, NUM_BLKS), NUM_BLKS # pointer to last block
 543        mov     NUM_BLKS, _INP_END(%rsp)
 544
 545        cmp     NUM_BLKS, INP
 546        je      only_one_block
 547
 548        ## load initial digest
 549        mov     (CTX), a
 550        mov     4*1(CTX), b
 551        mov     4*2(CTX), c
 552        mov     4*3(CTX), d
 553        mov     4*4(CTX), e
 554        mov     4*5(CTX), f
 555        mov     4*6(CTX), g
 556        mov     4*7(CTX), h
 557
 558        vmovdqa  PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
 559        vmovdqa  _SHUF_00BA(%rip), SHUF_00BA
 560        vmovdqa  _SHUF_DC00(%rip), SHUF_DC00
 561
 562        mov     CTX, _CTX(%rsp)
 563
 564loop0:
 565        ## Load first 16 dwords from two blocks
 566        VMOVDQ  0*32(INP),XTMP0
 567        VMOVDQ  1*32(INP),XTMP1
 568        VMOVDQ  2*32(INP),XTMP2
 569        VMOVDQ  3*32(INP),XTMP3
 570
 571        ## byte swap data
 572        vpshufb BYTE_FLIP_MASK, XTMP0, XTMP0
 573        vpshufb BYTE_FLIP_MASK, XTMP1, XTMP1
 574        vpshufb BYTE_FLIP_MASK, XTMP2, XTMP2
 575        vpshufb BYTE_FLIP_MASK, XTMP3, XTMP3
 576
 577        ## transpose data into high/low halves
 578        vperm2i128      $0x20, XTMP2, XTMP0, X0
 579        vperm2i128      $0x31, XTMP2, XTMP0, X1
 580        vperm2i128      $0x20, XTMP3, XTMP1, X2
 581        vperm2i128      $0x31, XTMP3, XTMP1, X3
 582
 583last_block_enter:
 584        add     $64, INP
 585        mov     INP, _INP(%rsp)
 586
 587        ## schedule 48 input dwords, by doing 3 rounds of 12 each
 588        xor     SRND, SRND
 589
 590.align 16
 591loop1:
 592        vpaddd  K256+0*32(SRND), X0, XFER
 593        vmovdqa XFER, 0*32+_XFER(%rsp, SRND)
 594        FOUR_ROUNDS_AND_SCHED   _XFER + 0*32
 595
 596        vpaddd  K256+1*32(SRND), X0, XFER
 597        vmovdqa XFER, 1*32+_XFER(%rsp, SRND)
 598        FOUR_ROUNDS_AND_SCHED   _XFER + 1*32
 599
 600        vpaddd  K256+2*32(SRND), X0, XFER
 601        vmovdqa XFER, 2*32+_XFER(%rsp, SRND)
 602        FOUR_ROUNDS_AND_SCHED   _XFER + 2*32
 603
 604        vpaddd  K256+3*32(SRND), X0, XFER
 605        vmovdqa XFER, 3*32+_XFER(%rsp, SRND)
 606        FOUR_ROUNDS_AND_SCHED   _XFER + 3*32
 607
 608        add     $4*32, SRND
 609        cmp     $3*4*32, SRND
 610        jb      loop1
 611
 612loop2:
 613        ## Do last 16 rounds with no scheduling
 614        vpaddd  K256+0*32(SRND), X0, XFER
 615        vmovdqa XFER, 0*32+_XFER(%rsp, SRND)
 616        DO_4ROUNDS      _XFER + 0*32
 617
 618        vpaddd  K256+1*32(SRND), X1, XFER
 619        vmovdqa XFER, 1*32+_XFER(%rsp, SRND)
 620        DO_4ROUNDS      _XFER + 1*32
 621        add     $2*32, SRND
 622
 623        vmovdqa X2, X0
 624        vmovdqa X3, X1
 625
 626        cmp     $4*4*32, SRND
 627        jb      loop2
 628
 629        mov     _CTX(%rsp), CTX
 630        mov     _INP(%rsp), INP
 631
 632        addm    (4*0)(CTX),a
 633        addm    (4*1)(CTX),b
 634        addm    (4*2)(CTX),c
 635        addm    (4*3)(CTX),d
 636        addm    (4*4)(CTX),e
 637        addm    (4*5)(CTX),f
 638        addm    (4*6)(CTX),g
 639        addm    (4*7)(CTX),h
 640
 641        cmp     _INP_END(%rsp), INP
 642        ja      done_hash
 643
 644        #### Do second block using previously scheduled results
 645        xor     SRND, SRND
 646.align 16
 647loop3:
 648        DO_4ROUNDS       _XFER + 0*32 + 16
 649        DO_4ROUNDS       _XFER + 1*32 + 16
 650        add     $2*32, SRND
 651        cmp     $4*4*32, SRND
 652        jb      loop3
 653
 654        mov     _CTX(%rsp), CTX
 655        mov     _INP(%rsp), INP
 656        add     $64, INP
 657
 658        addm    (4*0)(CTX),a
 659        addm    (4*1)(CTX),b
 660        addm    (4*2)(CTX),c
 661        addm    (4*3)(CTX),d
 662        addm    (4*4)(CTX),e
 663        addm    (4*5)(CTX),f
 664        addm    (4*6)(CTX),g
 665        addm    (4*7)(CTX),h
 666
 667        cmp     _INP_END(%rsp), INP
 668        jb      loop0
 669        ja      done_hash
 670
 671do_last_block:
 672        VMOVDQ  0*16(INP),XWORD0
 673        VMOVDQ  1*16(INP),XWORD1
 674        VMOVDQ  2*16(INP),XWORD2
 675        VMOVDQ  3*16(INP),XWORD3
 676
 677        vpshufb X_BYTE_FLIP_MASK, XWORD0, XWORD0
 678        vpshufb X_BYTE_FLIP_MASK, XWORD1, XWORD1
 679        vpshufb X_BYTE_FLIP_MASK, XWORD2, XWORD2
 680        vpshufb X_BYTE_FLIP_MASK, XWORD3, XWORD3
 681
 682        jmp     last_block_enter
 683
 684only_one_block:
 685
 686        ## load initial digest
 687        mov     (4*0)(CTX),a
 688        mov     (4*1)(CTX),b
 689        mov     (4*2)(CTX),c
 690        mov     (4*3)(CTX),d
 691        mov     (4*4)(CTX),e
 692        mov     (4*5)(CTX),f
 693        mov     (4*6)(CTX),g
 694        mov     (4*7)(CTX),h
 695
 696        vmovdqa PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
 697        vmovdqa _SHUF_00BA(%rip), SHUF_00BA
 698        vmovdqa _SHUF_DC00(%rip), SHUF_DC00
 699
 700        mov     CTX, _CTX(%rsp)
 701        jmp     do_last_block
 702
 703done_hash:
 704
 705        mov     %rbp, %rsp
 706        pop     %rbp
 707
 708        popq    %r15
 709        popq    %r14
 710        popq    %r13
 711        popq    %r12
 712        popq    %rbx
 713        ret
 714SYM_FUNC_END(sha256_transform_rorx)
 715
 716.section        .rodata.cst512.K256, "aM", @progbits, 512
 717.align 64
 718K256:
 719        .long   0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
 720        .long   0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
 721        .long   0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
 722        .long   0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
 723        .long   0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
 724        .long   0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
 725        .long   0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
 726        .long   0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
 727        .long   0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
 728        .long   0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
 729        .long   0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
 730        .long   0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
 731        .long   0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
 732        .long   0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
 733        .long   0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
 734        .long   0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
 735        .long   0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
 736        .long   0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
 737        .long   0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
 738        .long   0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
 739        .long   0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
 740        .long   0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
 741        .long   0xd192e819,0xd6990624,0xf40e3585,0x106aa070
 742        .long   0xd192e819,0xd6990624,0xf40e3585,0x106aa070
 743        .long   0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
 744        .long   0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
 745        .long   0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
 746        .long   0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
 747        .long   0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
 748        .long   0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
 749        .long   0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
 750        .long   0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
 751
 752.section        .rodata.cst32.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 32
 753.align 32
 754PSHUFFLE_BYTE_FLIP_MASK:
 755        .octa 0x0c0d0e0f08090a0b0405060700010203,0x0c0d0e0f08090a0b0405060700010203
 756
 757# shuffle xBxA -> 00BA
 758.section        .rodata.cst32._SHUF_00BA, "aM", @progbits, 32
 759.align 32
 760_SHUF_00BA:
 761        .octa 0xFFFFFFFFFFFFFFFF0b0a090803020100,0xFFFFFFFFFFFFFFFF0b0a090803020100
 762
 763# shuffle xDxC -> DC00
 764.section        .rodata.cst32._SHUF_DC00, "aM", @progbits, 32
 765.align 32
 766_SHUF_DC00:
 767        .octa 0x0b0a090803020100FFFFFFFFFFFFFFFF,0x0b0a090803020100FFFFFFFFFFFFFFFF
 768