1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/linkage.h>
17#include <asm/frame.h>
18
19#define rRIP (%rip)
20
21#define RX0 %xmm0
22#define RX1 %xmm1
23#define MASK_4BIT %xmm2
24#define RTMP0 %xmm3
25#define RTMP1 %xmm4
26#define RTMP2 %xmm5
27#define RTMP3 %xmm6
28#define RTMP4 %xmm7
29
30#define RA0 %xmm8
31#define RA1 %xmm9
32#define RA2 %xmm10
33#define RA3 %xmm11
34
35#define RB0 %xmm12
36#define RB1 %xmm13
37#define RB2 %xmm14
38#define RB3 %xmm15
39
40#define RNOT %xmm0
41#define RBSWAP %xmm1
42
43
44
45#define transpose_4x4(x0, x1, x2, x3, t1, t2) \
46 vpunpckhdq x1, x0, t2; \
47 vpunpckldq x1, x0, x0; \
48 \
49 vpunpckldq x3, x2, t1; \
50 vpunpckhdq x3, x2, x2; \
51 \
52 vpunpckhqdq t1, x0, x1; \
53 vpunpcklqdq t1, x0, x0; \
54 \
55 vpunpckhqdq x2, t2, x3; \
56 vpunpcklqdq x2, t2, x2;
57
58
59#define transform_pre(x, lo_t, hi_t, mask4bit, tmp0) \
60 vpand x, mask4bit, tmp0; \
61 vpandn x, mask4bit, x; \
62 vpsrld $4, x, x; \
63 \
64 vpshufb tmp0, lo_t, tmp0; \
65 vpshufb x, hi_t, x; \
66 vpxor tmp0, x, x;
67
68
69
70
71#define transform_post(x, lo_t, hi_t, mask4bit, tmp0) \
72 vpandn mask4bit, x, tmp0; \
73 vpsrld $4, x, x; \
74 vpand x, mask4bit, x; \
75 \
76 vpshufb tmp0, lo_t, tmp0; \
77 vpshufb x, hi_t, x; \
78 vpxor tmp0, x, x;
79
80
81.section .rodata.cst16, "aM", @progbits, 16
82.align 16
83
84
85
86
87
88
89
90
91
92.Lpre_tf_lo_s:
93 .quad 0x9197E2E474720701, 0xC7C1B4B222245157
94.Lpre_tf_hi_s:
95 .quad 0xE240AB09EB49A200, 0xF052B91BF95BB012
96
97
98.Lpost_tf_lo_s:
99 .quad 0x5B67F2CEA19D0834, 0xEDD14478172BBE82
100.Lpost_tf_hi_s:
101 .quad 0xAE7201DD73AFDC00, 0x11CDBE62CC1063BF
102
103
104.Linv_shift_row:
105 .byte 0x00, 0x0d, 0x0a, 0x07, 0x04, 0x01, 0x0e, 0x0b
106 .byte 0x08, 0x05, 0x02, 0x0f, 0x0c, 0x09, 0x06, 0x03
107
108
109.Linv_shift_row_rol_8:
110 .byte 0x07, 0x00, 0x0d, 0x0a, 0x0b, 0x04, 0x01, 0x0e
111 .byte 0x0f, 0x08, 0x05, 0x02, 0x03, 0x0c, 0x09, 0x06
112
113
114.Linv_shift_row_rol_16:
115 .byte 0x0a, 0x07, 0x00, 0x0d, 0x0e, 0x0b, 0x04, 0x01
116 .byte 0x02, 0x0f, 0x08, 0x05, 0x06, 0x03, 0x0c, 0x09
117
118
119.Linv_shift_row_rol_24:
120 .byte 0x0d, 0x0a, 0x07, 0x00, 0x01, 0x0e, 0x0b, 0x04
121 .byte 0x05, 0x02, 0x0f, 0x08, 0x09, 0x06, 0x03, 0x0c
122
123
124.Lbswap128_mask:
125 .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0
126
127
128.Lbswap32_mask:
129 .byte 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12
130
131.align 4
132
133.L0f0f0f0f:
134 .long 0x0f0f0f0f
135
136
137.Lpadding_deadbeef:
138 .long 0xdeadbeef, 0xdeadbeef, 0xdeadbeef
139
140
141.text
142.align 16
143
144
145
146
147
148.align 8
149SYM_FUNC_START(sm4_aesni_avx_crypt4)
150
151
152
153
154
155
156 FRAME_BEGIN
157
158 vmovdqu 0*16(%rdx), RA0;
159 vmovdqa RA0, RA1;
160 vmovdqa RA0, RA2;
161 vmovdqa RA0, RA3;
162 cmpq $2, %rcx;
163 jb .Lblk4_load_input_done;
164 vmovdqu 1*16(%rdx), RA1;
165 je .Lblk4_load_input_done;
166 vmovdqu 2*16(%rdx), RA2;
167 cmpq $3, %rcx;
168 je .Lblk4_load_input_done;
169 vmovdqu 3*16(%rdx), RA3;
170
171.Lblk4_load_input_done:
172
173 vmovdqa .Lbswap32_mask rRIP, RTMP2;
174 vpshufb RTMP2, RA0, RA0;
175 vpshufb RTMP2, RA1, RA1;
176 vpshufb RTMP2, RA2, RA2;
177 vpshufb RTMP2, RA3, RA3;
178
179 vbroadcastss .L0f0f0f0f rRIP, MASK_4BIT;
180 vmovdqa .Lpre_tf_lo_s rRIP, RTMP4;
181 vmovdqa .Lpre_tf_hi_s rRIP, RB0;
182 vmovdqa .Lpost_tf_lo_s rRIP, RB1;
183 vmovdqa .Lpost_tf_hi_s rRIP, RB2;
184 vmovdqa .Linv_shift_row rRIP, RB3;
185 vmovdqa .Linv_shift_row_rol_8 rRIP, RTMP2;
186 vmovdqa .Linv_shift_row_rol_16 rRIP, RTMP3;
187 transpose_4x4(RA0, RA1, RA2, RA3, RTMP0, RTMP1);
188
189#define ROUND(round, s0, s1, s2, s3) \
190 vbroadcastss (4*(round))(%rdi), RX0; \
191 vpxor s1, RX0, RX0; \
192 vpxor s2, RX0, RX0; \
193 vpxor s3, RX0, RX0; \
194 \
195 \
196 transform_pre(RX0, RTMP4, RB0, MASK_4BIT, RTMP0); \
197 vaesenclast MASK_4BIT, RX0, RX0; \
198 transform_post(RX0, RB1, RB2, MASK_4BIT, RTMP0); \
199 \
200 \
201 vpshufb RB3, RX0, RTMP0; \
202 vpxor RTMP0, s0, s0; \
203 vpshufb RTMP2, RX0, RTMP1; \
204 vpxor RTMP1, RTMP0, RTMP0; \
205 vpshufb RTMP3, RX0, RTMP1; \
206 vpxor RTMP1, RTMP0, RTMP0; \
207 vpshufb .Linv_shift_row_rol_24 rRIP, RX0, RTMP1; \
208 vpxor RTMP1, s0, s0; \
209 vpslld $2, RTMP0, RTMP1; \
210 vpsrld $30, RTMP0, RTMP0; \
211 vpxor RTMP0, s0, s0; \
212 \
213 vpxor RTMP1, s0, s0;
214
215 leaq (32*4)(%rdi), %rax;
216.align 16
217.Lroundloop_blk4:
218 ROUND(0, RA0, RA1, RA2, RA3);
219 ROUND(1, RA1, RA2, RA3, RA0);
220 ROUND(2, RA2, RA3, RA0, RA1);
221 ROUND(3, RA3, RA0, RA1, RA2);
222 leaq (4*4)(%rdi), %rdi;
223 cmpq %rax, %rdi;
224 jne .Lroundloop_blk4;
225
226#undef ROUND
227
228 vmovdqa .Lbswap128_mask rRIP, RTMP2;
229
230 transpose_4x4(RA0, RA1, RA2, RA3, RTMP0, RTMP1);
231 vpshufb RTMP2, RA0, RA0;
232 vpshufb RTMP2, RA1, RA1;
233 vpshufb RTMP2, RA2, RA2;
234 vpshufb RTMP2, RA3, RA3;
235
236 vmovdqu RA0, 0*16(%rsi);
237 cmpq $2, %rcx;
238 jb .Lblk4_store_output_done;
239 vmovdqu RA1, 1*16(%rsi);
240 je .Lblk4_store_output_done;
241 vmovdqu RA2, 2*16(%rsi);
242 cmpq $3, %rcx;
243 je .Lblk4_store_output_done;
244 vmovdqu RA3, 3*16(%rsi);
245
246.Lblk4_store_output_done:
247 vzeroall;
248 FRAME_END
249 ret;
250SYM_FUNC_END(sm4_aesni_avx_crypt4)
251
252.align 8
253SYM_FUNC_START_LOCAL(__sm4_crypt_blk8)
254
255
256
257
258
259
260
261
262 FRAME_BEGIN
263
264 vmovdqa .Lbswap32_mask rRIP, RTMP2;
265 vpshufb RTMP2, RA0, RA0;
266 vpshufb RTMP2, RA1, RA1;
267 vpshufb RTMP2, RA2, RA2;
268 vpshufb RTMP2, RA3, RA3;
269 vpshufb RTMP2, RB0, RB0;
270 vpshufb RTMP2, RB1, RB1;
271 vpshufb RTMP2, RB2, RB2;
272 vpshufb RTMP2, RB3, RB3;
273
274 vbroadcastss .L0f0f0f0f rRIP, MASK_4BIT;
275 transpose_4x4(RA0, RA1, RA2, RA3, RTMP0, RTMP1);
276 transpose_4x4(RB0, RB1, RB2, RB3, RTMP0, RTMP1);
277
278#define ROUND(round, s0, s1, s2, s3, r0, r1, r2, r3) \
279 vbroadcastss (4*(round))(%rdi), RX0; \
280 vmovdqa .Lpre_tf_lo_s rRIP, RTMP4; \
281 vmovdqa .Lpre_tf_hi_s rRIP, RTMP1; \
282 vmovdqa RX0, RX1; \
283 vpxor s1, RX0, RX0; \
284 vpxor s2, RX0, RX0; \
285 vpxor s3, RX0, RX0; \
286 vmovdqa .Lpost_tf_lo_s rRIP, RTMP2; \
287 vmovdqa .Lpost_tf_hi_s rRIP, RTMP3; \
288 vpxor r1, RX1, RX1; \
289 vpxor r2, RX1, RX1; \
290 vpxor r3, RX1, RX1; \
291 \
292 \
293 transform_pre(RX0, RTMP4, RTMP1, MASK_4BIT, RTMP0); \
294 transform_pre(RX1, RTMP4, RTMP1, MASK_4BIT, RTMP0); \
295 vmovdqa .Linv_shift_row rRIP, RTMP4; \
296 vaesenclast MASK_4BIT, RX0, RX0; \
297 vaesenclast MASK_4BIT, RX1, RX1; \
298 transform_post(RX0, RTMP2, RTMP3, MASK_4BIT, RTMP0); \
299 transform_post(RX1, RTMP2, RTMP3, MASK_4BIT, RTMP0); \
300 \
301 \
302 vpshufb RTMP4, RX0, RTMP0; \
303 vpxor RTMP0, s0, s0; \
304 vpshufb RTMP4, RX1, RTMP2; \
305 vmovdqa .Linv_shift_row_rol_8 rRIP, RTMP4; \
306 vpxor RTMP2, r0, r0; \
307 vpshufb RTMP4, RX0, RTMP1; \
308 vpxor RTMP1, RTMP0, RTMP0; \
309 vpshufb RTMP4, RX1, RTMP3; \
310 vmovdqa .Linv_shift_row_rol_16 rRIP, RTMP4; \
311 vpxor RTMP3, RTMP2, RTMP2; \
312 vpshufb RTMP4, RX0, RTMP1; \
313 vpxor RTMP1, RTMP0, RTMP0; \
314 vpshufb RTMP4, RX1, RTMP3; \
315 vmovdqa .Linv_shift_row_rol_24 rRIP, RTMP4; \
316 vpxor RTMP3, RTMP2, RTMP2; \
317 vpshufb RTMP4, RX0, RTMP1; \
318 vpxor RTMP1, s0, s0; \
319 \
320 vpslld $2, RTMP0, RTMP1; \
321 vpsrld $30, RTMP0, RTMP0; \
322 vpxor RTMP0, s0, s0; \
323 vpxor RTMP1, s0, s0; \
324 vpshufb RTMP4, RX1, RTMP3; \
325 vpxor RTMP3, r0, r0; \
326 \
327 vpslld $2, RTMP2, RTMP3; \
328 vpsrld $30, RTMP2, RTMP2; \
329 vpxor RTMP2, r0, r0; \
330 vpxor RTMP3, r0, r0;
331
332 leaq (32*4)(%rdi), %rax;
333.align 16
334.Lroundloop_blk8:
335 ROUND(0, RA0, RA1, RA2, RA3, RB0, RB1, RB2, RB3);
336 ROUND(1, RA1, RA2, RA3, RA0, RB1, RB2, RB3, RB0);
337 ROUND(2, RA2, RA3, RA0, RA1, RB2, RB3, RB0, RB1);
338 ROUND(3, RA3, RA0, RA1, RA2, RB3, RB0, RB1, RB2);
339 leaq (4*4)(%rdi), %rdi;
340 cmpq %rax, %rdi;
341 jne .Lroundloop_blk8;
342
343#undef ROUND
344
345 vmovdqa .Lbswap128_mask rRIP, RTMP2;
346
347 transpose_4x4(RA0, RA1, RA2, RA3, RTMP0, RTMP1);
348 transpose_4x4(RB0, RB1, RB2, RB3, RTMP0, RTMP1);
349 vpshufb RTMP2, RA0, RA0;
350 vpshufb RTMP2, RA1, RA1;
351 vpshufb RTMP2, RA2, RA2;
352 vpshufb RTMP2, RA3, RA3;
353 vpshufb RTMP2, RB0, RB0;
354 vpshufb RTMP2, RB1, RB1;
355 vpshufb RTMP2, RB2, RB2;
356 vpshufb RTMP2, RB3, RB3;
357
358 FRAME_END
359 ret;
360SYM_FUNC_END(__sm4_crypt_blk8)
361
362
363
364
365
366.align 8
367SYM_FUNC_START(sm4_aesni_avx_crypt8)
368
369
370
371
372
373
374 cmpq $5, %rcx;
375 jb sm4_aesni_avx_crypt4;
376
377 FRAME_BEGIN
378
379 vmovdqu (0 * 16)(%rdx), RA0;
380 vmovdqu (1 * 16)(%rdx), RA1;
381 vmovdqu (2 * 16)(%rdx), RA2;
382 vmovdqu (3 * 16)(%rdx), RA3;
383 vmovdqu (4 * 16)(%rdx), RB0;
384 vmovdqa RB0, RB1;
385 vmovdqa RB0, RB2;
386 vmovdqa RB0, RB3;
387 je .Lblk8_load_input_done;
388 vmovdqu (5 * 16)(%rdx), RB1;
389 cmpq $7, %rcx;
390 jb .Lblk8_load_input_done;
391 vmovdqu (6 * 16)(%rdx), RB2;
392 je .Lblk8_load_input_done;
393 vmovdqu (7 * 16)(%rdx), RB3;
394
395.Lblk8_load_input_done:
396 call __sm4_crypt_blk8;
397
398 cmpq $6, %rcx;
399 vmovdqu RA0, (0 * 16)(%rsi);
400 vmovdqu RA1, (1 * 16)(%rsi);
401 vmovdqu RA2, (2 * 16)(%rsi);
402 vmovdqu RA3, (3 * 16)(%rsi);
403 vmovdqu RB0, (4 * 16)(%rsi);
404 jb .Lblk8_store_output_done;
405 vmovdqu RB1, (5 * 16)(%rsi);
406 je .Lblk8_store_output_done;
407 vmovdqu RB2, (6 * 16)(%rsi);
408 cmpq $7, %rcx;
409 je .Lblk8_store_output_done;
410 vmovdqu RB3, (7 * 16)(%rsi);
411
412.Lblk8_store_output_done:
413 vzeroall;
414 FRAME_END
415 ret;
416SYM_FUNC_END(sm4_aesni_avx_crypt8)
417
418
419
420
421
422.align 8
423SYM_FUNC_START(sm4_aesni_avx_ctr_enc_blk8)
424
425
426
427
428
429
430 FRAME_BEGIN
431
432
433 vmovdqu (%rcx), RA0;
434
435 vmovdqa .Lbswap128_mask rRIP, RBSWAP;
436 vpshufb RBSWAP, RA0, RTMP0;
437
438 vpcmpeqd RNOT, RNOT, RNOT;
439 vpsrldq $8, RNOT, RNOT;
440
441#define inc_le128(x, minus_one, tmp) \
442 vpcmpeqq minus_one, x, tmp; \
443 vpsubq minus_one, x, x; \
444 vpslldq $8, tmp, tmp; \
445 vpsubq tmp, x, x;
446
447
448 inc_le128(RTMP0, RNOT, RTMP2);
449 vpshufb RBSWAP, RTMP0, RA1;
450 inc_le128(RTMP0, RNOT, RTMP2);
451 vpshufb RBSWAP, RTMP0, RA2;
452 inc_le128(RTMP0, RNOT, RTMP2);
453 vpshufb RBSWAP, RTMP0, RA3;
454 inc_le128(RTMP0, RNOT, RTMP2);
455 vpshufb RBSWAP, RTMP0, RB0;
456 inc_le128(RTMP0, RNOT, RTMP2);
457 vpshufb RBSWAP, RTMP0, RB1;
458 inc_le128(RTMP0, RNOT, RTMP2);
459 vpshufb RBSWAP, RTMP0, RB2;
460 inc_le128(RTMP0, RNOT, RTMP2);
461 vpshufb RBSWAP, RTMP0, RB3;
462 inc_le128(RTMP0, RNOT, RTMP2);
463 vpshufb RBSWAP, RTMP0, RTMP1;
464
465
466 vmovdqu RTMP1, (%rcx);
467
468 call __sm4_crypt_blk8;
469
470 vpxor (0 * 16)(%rdx), RA0, RA0;
471 vpxor (1 * 16)(%rdx), RA1, RA1;
472 vpxor (2 * 16)(%rdx), RA2, RA2;
473 vpxor (3 * 16)(%rdx), RA3, RA3;
474 vpxor (4 * 16)(%rdx), RB0, RB0;
475 vpxor (5 * 16)(%rdx), RB1, RB1;
476 vpxor (6 * 16)(%rdx), RB2, RB2;
477 vpxor (7 * 16)(%rdx), RB3, RB3;
478
479 vmovdqu RA0, (0 * 16)(%rsi);
480 vmovdqu RA1, (1 * 16)(%rsi);
481 vmovdqu RA2, (2 * 16)(%rsi);
482 vmovdqu RA3, (3 * 16)(%rsi);
483 vmovdqu RB0, (4 * 16)(%rsi);
484 vmovdqu RB1, (5 * 16)(%rsi);
485 vmovdqu RB2, (6 * 16)(%rsi);
486 vmovdqu RB3, (7 * 16)(%rsi);
487
488 vzeroall;
489 FRAME_END
490 ret;
491SYM_FUNC_END(sm4_aesni_avx_ctr_enc_blk8)
492
493
494
495
496
497.align 8
498SYM_FUNC_START(sm4_aesni_avx_cbc_dec_blk8)
499
500
501
502
503
504
505 FRAME_BEGIN
506
507 vmovdqu (0 * 16)(%rdx), RA0;
508 vmovdqu (1 * 16)(%rdx), RA1;
509 vmovdqu (2 * 16)(%rdx), RA2;
510 vmovdqu (3 * 16)(%rdx), RA3;
511 vmovdqu (4 * 16)(%rdx), RB0;
512 vmovdqu (5 * 16)(%rdx), RB1;
513 vmovdqu (6 * 16)(%rdx), RB2;
514 vmovdqu (7 * 16)(%rdx), RB3;
515
516 call __sm4_crypt_blk8;
517
518 vmovdqu (7 * 16)(%rdx), RNOT;
519 vpxor (%rcx), RA0, RA0;
520 vpxor (0 * 16)(%rdx), RA1, RA1;
521 vpxor (1 * 16)(%rdx), RA2, RA2;
522 vpxor (2 * 16)(%rdx), RA3, RA3;
523 vpxor (3 * 16)(%rdx), RB0, RB0;
524 vpxor (4 * 16)(%rdx), RB1, RB1;
525 vpxor (5 * 16)(%rdx), RB2, RB2;
526 vpxor (6 * 16)(%rdx), RB3, RB3;
527 vmovdqu RNOT, (%rcx);
528
529 vmovdqu RA0, (0 * 16)(%rsi);
530 vmovdqu RA1, (1 * 16)(%rsi);
531 vmovdqu RA2, (2 * 16)(%rsi);
532 vmovdqu RA3, (3 * 16)(%rsi);
533 vmovdqu RB0, (4 * 16)(%rsi);
534 vmovdqu RB1, (5 * 16)(%rsi);
535 vmovdqu RB2, (6 * 16)(%rsi);
536 vmovdqu RB3, (7 * 16)(%rsi);
537
538 vzeroall;
539 FRAME_END
540 ret;
541SYM_FUNC_END(sm4_aesni_avx_cbc_dec_blk8)
542
543
544
545
546
547.align 8
548SYM_FUNC_START(sm4_aesni_avx_cfb_dec_blk8)
549
550
551
552
553
554
555 FRAME_BEGIN
556
557
558 vmovdqu (%rcx), RA0;
559 vmovdqu 0 * 16(%rdx), RA1;
560 vmovdqu 1 * 16(%rdx), RA2;
561 vmovdqu 2 * 16(%rdx), RA3;
562 vmovdqu 3 * 16(%rdx), RB0;
563 vmovdqu 4 * 16(%rdx), RB1;
564 vmovdqu 5 * 16(%rdx), RB2;
565 vmovdqu 6 * 16(%rdx), RB3;
566
567
568 vmovdqu 7 * 16(%rdx), RNOT;
569 vmovdqu RNOT, (%rcx);
570
571 call __sm4_crypt_blk8;
572
573 vpxor (0 * 16)(%rdx), RA0, RA0;
574 vpxor (1 * 16)(%rdx), RA1, RA1;
575 vpxor (2 * 16)(%rdx), RA2, RA2;
576 vpxor (3 * 16)(%rdx), RA3, RA3;
577 vpxor (4 * 16)(%rdx), RB0, RB0;
578 vpxor (5 * 16)(%rdx), RB1, RB1;
579 vpxor (6 * 16)(%rdx), RB2, RB2;
580 vpxor (7 * 16)(%rdx), RB3, RB3;
581
582 vmovdqu RA0, (0 * 16)(%rsi);
583 vmovdqu RA1, (1 * 16)(%rsi);
584 vmovdqu RA2, (2 * 16)(%rsi);
585 vmovdqu RA3, (3 * 16)(%rsi);
586 vmovdqu RB0, (4 * 16)(%rsi);
587 vmovdqu RB1, (5 * 16)(%rsi);
588 vmovdqu RB2, (6 * 16)(%rsi);
589 vmovdqu RB3, (7 * 16)(%rsi);
590
591 vzeroall;
592 FRAME_END
593 ret;
594SYM_FUNC_END(sm4_aesni_avx_cfb_dec_blk8)
595