linux/arch/x86/crypto/sha1_ni_asm.S
<<
>>
Prefs 
   1/*
   2 * Intel SHA Extensions optimized implementation of a SHA-1 update function
   3 *
   4 * This file is provided under a dual BSD/GPLv2 license.  When using or
   5 * redistributing this file, you may do so under either license.
   6 *
   7 * GPL LICENSE SUMMARY
   8 *
   9 * Copyright(c) 2015 Intel Corporation.
  10 *
  11 * This program is free software; you can redistribute it and/or modify
  12 * it under the terms of version 2 of the GNU General Public License as
  13 * published by the Free Software Foundation.
  14 *
  15 * This program is distributed in the hope that it will be useful, but
  16 * WITHOUT ANY WARRANTY; without even the implied warranty of
  17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  18 * General Public License for more details.
  19 *
  20 * Contact Information:
  21 *      Sean Gulley <sean.m.gulley@intel.com>
  22 *      Tim Chen <tim.c.chen@linux.intel.com>
  23 *
  24 * BSD LICENSE
  25 *
  26 * Copyright(c) 2015 Intel Corporation.
  27 *
  28 * Redistribution and use in source and binary forms, with or without
  29 * modification, are permitted provided that the following conditions
  30 * are met:
  31 *
  32 *      * Redistributions of source code must retain the above copyright
  33 *        notice, this list of conditions and the following disclaimer.
  34 *      * Redistributions in binary form must reproduce the above copyright
  35 *        notice, this list of conditions and the following disclaimer in
  36 *        the documentation and/or other materials provided with the
  37 *        distribution.
  38 *      * Neither the name of Intel Corporation nor the names of its
  39 *        contributors may be used to endorse or promote products derived
  40 *        from this software without specific prior written permission.
  41 *
  42 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  43 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  44 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  45 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  46 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  47 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  48 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  49 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  50 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  51 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  52 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  53 *
  54 */
  55
  56#include <linux/linkage.h>
  57
  58#define DIGEST_PTR      %rdi    /* 1st arg */
  59#define DATA_PTR        %rsi    /* 2nd arg */
  60#define NUM_BLKS        %rdx    /* 3rd arg */
  61
  62/* gcc conversion */
  63#define FRAME_SIZE      32      /* space for 2x16 bytes */
  64
  65#define ABCD            %xmm0
  66#define E0              %xmm1   /* Need two E's b/c they ping pong */
  67#define E1              %xmm2
  68#define MSG0            %xmm3
  69#define MSG1            %xmm4
  70#define MSG2            %xmm5
  71#define MSG3            %xmm6
  72#define SHUF_MASK       %xmm7
  73
  74
  75/*
  76 * Intel SHA Extensions optimized implementation of a SHA-1 update function
  77 *
  78 * The function takes a pointer to the current hash values, a pointer to the
  79 * input data, and a number of 64 byte blocks to process.  Once all blocks have
  80 * been processed, the digest pointer is  updated with the resulting hash value.
  81 * The function only processes complete blocks, there is no functionality to
  82 * store partial blocks. All message padding and hash value initialization must
  83 * be done outside the update function.
  84 *
  85 * The indented lines in the loop are instructions related to rounds processing.
  86 * The non-indented lines are instructions related to the message schedule.
  87 *
  88 * void sha1_ni_transform(uint32_t *digest, const void *data,
  89                uint32_t numBlocks)
  90 * digest : pointer to digest
  91 * data: pointer to input data
  92 * numBlocks: Number of blocks to process
  93 */
  94.text
  95.align 32
  96SYM_FUNC_START(sha1_ni_transform)
  97        push            %rbp
  98        mov             %rsp, %rbp
  99        sub             $FRAME_SIZE, %rsp
 100        and             $~0xF, %rsp
 101
 102        shl             $6, NUM_BLKS            /* convert to bytes */
 103        jz              .Ldone_hash
 104        add             DATA_PTR, NUM_BLKS      /* pointer to end of data */
 105
 106        /* load initial hash values */
 107        pinsrd          $3, 1*16(DIGEST_PTR), E0
 108        movdqu          0*16(DIGEST_PTR), ABCD
 109        pand            UPPER_WORD_MASK(%rip), E0
 110        pshufd          $0x1B, ABCD, ABCD
 111
 112        movdqa          PSHUFFLE_BYTE_FLIP_MASK(%rip), SHUF_MASK
 113
 114.Lloop0:
 115        /* Save hash values for addition after rounds */
 116        movdqa          E0, (0*16)(%rsp)
 117        movdqa          ABCD, (1*16)(%rsp)
 118
 119        /* Rounds 0-3 */
 120        movdqu          0*16(DATA_PTR), MSG0
 121        pshufb          SHUF_MASK, MSG0
 122                paddd           MSG0, E0
 123                movdqa          ABCD, E1
 124                sha1rnds4       $0, E0, ABCD
 125
 126        /* Rounds 4-7 */
 127        movdqu          1*16(DATA_PTR), MSG1
 128        pshufb          SHUF_MASK, MSG1
 129                sha1nexte       MSG1, E1
 130                movdqa          ABCD, E0
 131                sha1rnds4       $0, E1, ABCD
 132        sha1msg1        MSG1, MSG0
 133
 134        /* Rounds 8-11 */
 135        movdqu          2*16(DATA_PTR), MSG2
 136        pshufb          SHUF_MASK, MSG2
 137                sha1nexte       MSG2, E0
 138                movdqa          ABCD, E1
 139                sha1rnds4       $0, E0, ABCD
 140        sha1msg1        MSG2, MSG1
 141        pxor            MSG2, MSG0
 142
 143        /* Rounds 12-15 */
 144        movdqu          3*16(DATA_PTR), MSG3
 145        pshufb          SHUF_MASK, MSG3
 146                sha1nexte       MSG3, E1
 147                movdqa          ABCD, E0
 148        sha1msg2        MSG3, MSG0
 149                sha1rnds4       $0, E1, ABCD
 150        sha1msg1        MSG3, MSG2
 151        pxor            MSG3, MSG1
 152
 153        /* Rounds 16-19 */
 154                sha1nexte       MSG0, E0
 155                movdqa          ABCD, E1
 156        sha1msg2        MSG0, MSG1
 157                sha1rnds4       $0, E0, ABCD
 158        sha1msg1        MSG0, MSG3
 159        pxor            MSG0, MSG2
 160
 161        /* Rounds 20-23 */
 162                sha1nexte       MSG1, E1
 163                movdqa          ABCD, E0
 164        sha1msg2        MSG1, MSG2
 165                sha1rnds4       $1, E1, ABCD
 166        sha1msg1        MSG1, MSG0
 167        pxor            MSG1, MSG3
 168
 169        /* Rounds 24-27 */
 170                sha1nexte       MSG2, E0
 171                movdqa          ABCD, E1
 172        sha1msg2        MSG2, MSG3
 173                sha1rnds4       $1, E0, ABCD
 174        sha1msg1        MSG2, MSG1
 175        pxor            MSG2, MSG0
 176
 177        /* Rounds 28-31 */
 178                sha1nexte       MSG3, E1
 179                movdqa          ABCD, E0
 180        sha1msg2        MSG3, MSG0
 181                sha1rnds4       $1, E1, ABCD
 182        sha1msg1        MSG3, MSG2
 183        pxor            MSG3, MSG1
 184
 185        /* Rounds 32-35 */
 186                sha1nexte       MSG0, E0
 187                movdqa          ABCD, E1
 188        sha1msg2        MSG0, MSG1
 189                sha1rnds4       $1, E0, ABCD
 190        sha1msg1        MSG0, MSG3
 191        pxor            MSG0, MSG2
 192
 193        /* Rounds 36-39 */
 194                sha1nexte       MSG1, E1
 195                movdqa          ABCD, E0
 196        sha1msg2        MSG1, MSG2
 197                sha1rnds4       $1, E1, ABCD
 198        sha1msg1        MSG1, MSG0
 199        pxor            MSG1, MSG3
 200
 201        /* Rounds 40-43 */
 202                sha1nexte       MSG2, E0
 203                movdqa          ABCD, E1
 204        sha1msg2        MSG2, MSG3
 205                sha1rnds4       $2, E0, ABCD
 206        sha1msg1        MSG2, MSG1
 207        pxor            MSG2, MSG0
 208
 209        /* Rounds 44-47 */
 210                sha1nexte       MSG3, E1
 211                movdqa          ABCD, E0
 212        sha1msg2        MSG3, MSG0
 213                sha1rnds4       $2, E1, ABCD
 214        sha1msg1        MSG3, MSG2
 215        pxor            MSG3, MSG1
 216
 217        /* Rounds 48-51 */
 218                sha1nexte       MSG0, E0
 219                movdqa          ABCD, E1
 220        sha1msg2        MSG0, MSG1
 221                sha1rnds4       $2, E0, ABCD
 222        sha1msg1        MSG0, MSG3
 223        pxor            MSG0, MSG2
 224
 225        /* Rounds 52-55 */
 226                sha1nexte       MSG1, E1
 227                movdqa          ABCD, E0
 228        sha1msg2        MSG1, MSG2
 229                sha1rnds4       $2, E1, ABCD
 230        sha1msg1        MSG1, MSG0
 231        pxor            MSG1, MSG3
 232
 233        /* Rounds 56-59 */
 234                sha1nexte       MSG2, E0
 235                movdqa          ABCD, E1
 236        sha1msg2        MSG2, MSG3
 237                sha1rnds4       $2, E0, ABCD
 238        sha1msg1        MSG2, MSG1
 239        pxor            MSG2, MSG0
 240
 241        /* Rounds 60-63 */
 242                sha1nexte       MSG3, E1
 243                movdqa          ABCD, E0
 244        sha1msg2        MSG3, MSG0
 245                sha1rnds4       $3, E1, ABCD
 246        sha1msg1        MSG3, MSG2
 247        pxor            MSG3, MSG1
 248
 249        /* Rounds 64-67 */
 250                sha1nexte       MSG0, E0
 251                movdqa          ABCD, E1
 252        sha1msg2        MSG0, MSG1
 253                sha1rnds4       $3, E0, ABCD
 254        sha1msg1        MSG0, MSG3
 255        pxor            MSG0, MSG2
 256
 257        /* Rounds 68-71 */
 258                sha1nexte       MSG1, E1
 259                movdqa          ABCD, E0
 260        sha1msg2        MSG1, MSG2
 261                sha1rnds4       $3, E1, ABCD
 262        pxor            MSG1, MSG3
 263
 264        /* Rounds 72-75 */
 265                sha1nexte       MSG2, E0
 266                movdqa          ABCD, E1
 267        sha1msg2        MSG2, MSG3
 268                sha1rnds4       $3, E0, ABCD
 269
 270        /* Rounds 76-79 */
 271                sha1nexte       MSG3, E1
 272                movdqa          ABCD, E0
 273                sha1rnds4       $3, E1, ABCD
 274
 275        /* Add current hash values with previously saved */
 276        sha1nexte       (0*16)(%rsp), E0
 277        paddd           (1*16)(%rsp), ABCD
 278
 279        /* Increment data pointer and loop if more to process */
 280        add             $64, DATA_PTR
 281        cmp             NUM_BLKS, DATA_PTR
 282        jne             .Lloop0
 283
 284        /* Write hash values back in the correct order */
 285        pshufd          $0x1B, ABCD, ABCD
 286        movdqu          ABCD, 0*16(DIGEST_PTR)
 287        pextrd          $3, E0, 1*16(DIGEST_PTR)
 288
 289.Ldone_hash:
 290        mov             %rbp, %rsp
 291        pop             %rbp
 292
 293        ret
 294SYM_FUNC_END(sha1_ni_transform)
 295
 296.section        .rodata.cst16.PSHUFFLE_BYTE_FLIP_MASK, "aM", @progbits, 16
 297.align 16
 298PSHUFFLE_BYTE_FLIP_MASK:
 299        .octa 0x000102030405060708090a0b0c0d0e0f
 300
 301.section        .rodata.cst16.UPPER_WORD_MASK, "aM", @progbits, 16
 302.align 16
 303UPPER_WORD_MASK:
 304        .octa 0xFFFFFFFF000000000000000000000000
 305
Hosted by Missing Link Electronics. The original LXR software by the LXR community, this experimental version by lxr@linux.no.