1
2
3
4
5
6
7
8#include "compat.h"
9#include "desc_constr.h"
10#include "caamalg_desc.h"
11
12
13
14
15
16static inline void aead_append_src_dst(u32 *desc, u32 msg_type)
17{
18 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
19 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH |
20 KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH);
21}
22
23
24static inline void append_dec_op1(u32 *desc, u32 type)
25{
26 u32 *jump_cmd, *uncond_jump_cmd;
27
28
29 if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) {
30 append_operation(desc, type | OP_ALG_AS_INITFINAL |
31 OP_ALG_DECRYPT);
32 return;
33 }
34
35 jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD);
36 append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT);
37 uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL);
38 set_jump_tgt_here(desc, jump_cmd);
39 append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT |
40 OP_ALG_AAI_DK);
41 set_jump_tgt_here(desc, uncond_jump_cmd);
42}
43
44
45
46
47
48
49
50
51
52
53
54
55
56void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata,
57 unsigned int icvsize, int era)
58{
59 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
60
61 init_sh_desc(desc, HDR_SHARE_SERIAL);
62
63
64 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
65 JUMP_COND_SHRD);
66 if (era < 6) {
67 if (adata->key_inline)
68 append_key_as_imm(desc, adata->key_virt,
69 adata->keylen_pad, adata->keylen,
70 CLASS_2 | KEY_DEST_MDHA_SPLIT |
71 KEY_ENC);
72 else
73 append_key(desc, adata->key_dma, adata->keylen,
74 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
75 } else {
76 append_proto_dkp(desc, adata);
77 }
78 set_jump_tgt_here(desc, key_jump_cmd);
79
80
81 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
82
83
84 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
85 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
86
87
88
89
90
91
92 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
93 MOVE_DEST_MATH3 |
94 (0x6 << MOVE_LEN_SHIFT));
95 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 |
96 MOVE_DEST_DESCBUF |
97 MOVE_WAITCOMP |
98 (0x8 << MOVE_LEN_SHIFT));
99
100
101 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
102 OP_ALG_ENCRYPT);
103
104
105 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
106
107 set_move_tgt_here(desc, read_move_cmd);
108 set_move_tgt_here(desc, write_move_cmd);
109 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
110 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
111 MOVE_AUX_LS);
112
113
114 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
115 LDST_SRCDST_BYTE_CONTEXT);
116
117 print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ",
118 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
119 1);
120}
121EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap);
122
123
124
125
126
127
128
129
130
131
132
133
134
135void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata,
136 unsigned int icvsize, int era)
137{
138 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd;
139
140 init_sh_desc(desc, HDR_SHARE_SERIAL);
141
142
143 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
144 JUMP_COND_SHRD);
145 if (era < 6) {
146 if (adata->key_inline)
147 append_key_as_imm(desc, adata->key_virt,
148 adata->keylen_pad, adata->keylen,
149 CLASS_2 | KEY_DEST_MDHA_SPLIT |
150 KEY_ENC);
151 else
152 append_key(desc, adata->key_dma, adata->keylen,
153 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
154 } else {
155 append_proto_dkp(desc, adata);
156 }
157 set_jump_tgt_here(desc, key_jump_cmd);
158
159
160 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
161 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
162
163
164 append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ);
165
166
167 append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ);
168 append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ);
169
170
171
172
173
174
175 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF |
176 MOVE_DEST_MATH2 |
177 (0x6 << MOVE_LEN_SHIFT));
178 write_move_cmd = append_move(desc, MOVE_SRC_MATH2 |
179 MOVE_DEST_DESCBUF |
180 MOVE_WAITCOMP |
181 (0x8 << MOVE_LEN_SHIFT));
182
183
184 aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
185
186
187
188
189
190 jump_cmd = append_jump(desc, JUMP_TEST_ALL);
191 set_jump_tgt_here(desc, jump_cmd);
192
193 set_move_tgt_here(desc, read_move_cmd);
194 set_move_tgt_here(desc, write_move_cmd);
195 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
196 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO |
197 MOVE_AUX_LS);
198 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
199
200
201 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
202 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
203
204 print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ",
205 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
206 1);
207}
208EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap);
209
210static void init_sh_desc_key_aead(u32 * const desc,
211 struct alginfo * const cdata,
212 struct alginfo * const adata,
213 const bool is_rfc3686, u32 *nonce, int era)
214{
215 u32 *key_jump_cmd;
216 unsigned int enckeylen = cdata->keylen;
217
218
219 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
220
221
222 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
223 JUMP_COND_SHRD);
224
225
226
227
228
229
230 if (is_rfc3686)
231 enckeylen -= CTR_RFC3686_NONCE_SIZE;
232
233 if (era < 6) {
234 if (adata->key_inline)
235 append_key_as_imm(desc, adata->key_virt,
236 adata->keylen_pad, adata->keylen,
237 CLASS_2 | KEY_DEST_MDHA_SPLIT |
238 KEY_ENC);
239 else
240 append_key(desc, adata->key_dma, adata->keylen,
241 CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC);
242 } else {
243 append_proto_dkp(desc, adata);
244 }
245
246 if (cdata->key_inline)
247 append_key_as_imm(desc, cdata->key_virt, enckeylen,
248 enckeylen, CLASS_1 | KEY_DEST_CLASS_REG);
249 else
250 append_key(desc, cdata->key_dma, enckeylen, CLASS_1 |
251 KEY_DEST_CLASS_REG);
252
253
254 if (is_rfc3686) {
255 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
256 LDST_CLASS_IND_CCB |
257 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
258 append_move(desc,
259 MOVE_SRC_OUTFIFO |
260 MOVE_DEST_CLASS1CTX |
261 (16 << MOVE_OFFSET_SHIFT) |
262 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
263 }
264
265 set_jump_tgt_here(desc, key_jump_cmd);
266}
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata,
289 struct alginfo *adata, unsigned int ivsize,
290 unsigned int icvsize, const bool is_rfc3686,
291 u32 *nonce, const u32 ctx1_iv_off, const bool is_qi,
292 int era)
293{
294
295 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
296
297
298 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
299 OP_ALG_ENCRYPT);
300
301 if (is_qi) {
302 u32 *wait_load_cmd;
303
304
305 append_seq_load(desc, 4, LDST_CLASS_DECO |
306 LDST_SRCDST_WORD_DECO_MATH3 |
307 (4 << LDST_OFFSET_SHIFT));
308
309 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
310 JUMP_COND_CALM | JUMP_COND_NCP |
311 JUMP_COND_NOP | JUMP_COND_NIP |
312 JUMP_COND_NIFP);
313 set_jump_tgt_here(desc, wait_load_cmd);
314
315 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
316 LDST_SRCDST_BYTE_CONTEXT |
317 (ctx1_iv_off << LDST_OFFSET_SHIFT));
318 }
319
320
321 if (is_qi || era < 3) {
322 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
323 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
324 } else {
325 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
326 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
327 }
328
329
330 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
331
332
333 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
334 FIFOLDST_VLF);
335
336
337 if (is_rfc3686)
338 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
339 LDST_SRCDST_BYTE_CONTEXT |
340 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
341 LDST_OFFSET_SHIFT));
342
343
344 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
345 OP_ALG_ENCRYPT);
346
347
348 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
349 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
350 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
351
352
353 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
354 LDST_SRCDST_BYTE_CONTEXT);
355
356 print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ",
357 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
358 1);
359}
360EXPORT_SYMBOL(cnstr_shdsc_aead_encap);
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata,
384 struct alginfo *adata, unsigned int ivsize,
385 unsigned int icvsize, const bool geniv,
386 const bool is_rfc3686, u32 *nonce,
387 const u32 ctx1_iv_off, const bool is_qi, int era)
388{
389
390 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
391
392
393 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
394 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
395
396 if (is_qi) {
397 u32 *wait_load_cmd;
398
399
400 append_seq_load(desc, 4, LDST_CLASS_DECO |
401 LDST_SRCDST_WORD_DECO_MATH3 |
402 (4 << LDST_OFFSET_SHIFT));
403
404 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
405 JUMP_COND_CALM | JUMP_COND_NCP |
406 JUMP_COND_NOP | JUMP_COND_NIP |
407 JUMP_COND_NIFP);
408 set_jump_tgt_here(desc, wait_load_cmd);
409
410 if (!geniv)
411 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
412 LDST_SRCDST_BYTE_CONTEXT |
413 (ctx1_iv_off << LDST_OFFSET_SHIFT));
414 }
415
416
417 if (is_qi || era < 3) {
418 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
419 if (geniv)
420 append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM,
421 ivsize);
422 else
423 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3,
424 CAAM_CMD_SZ);
425 } else {
426 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
427 if (geniv)
428 append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM,
429 ivsize);
430 else
431 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD,
432 CAAM_CMD_SZ);
433 }
434
435
436 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
437
438
439 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
440 KEY_VLF);
441
442 if (geniv) {
443 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
444 LDST_SRCDST_BYTE_CONTEXT |
445 (ctx1_iv_off << LDST_OFFSET_SHIFT));
446 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO |
447 (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize);
448 }
449
450
451 if (is_rfc3686)
452 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
453 LDST_SRCDST_BYTE_CONTEXT |
454 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
455 LDST_OFFSET_SHIFT));
456
457
458 if (ctx1_iv_off)
459 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
460 OP_ALG_DECRYPT);
461 else
462 append_dec_op1(desc, cdata->algtype);
463
464
465 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
466 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
467 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
468
469
470 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
471 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
472
473 print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ",
474 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
475 1);
476}
477EXPORT_SYMBOL(cnstr_shdsc_aead_decap);
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata,
501 struct alginfo *adata, unsigned int ivsize,
502 unsigned int icvsize, const bool is_rfc3686,
503 u32 *nonce, const u32 ctx1_iv_off,
504 const bool is_qi, int era)
505{
506 u32 geniv, moveiv;
507 u32 *wait_cmd;
508
509
510 init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era);
511
512 if (is_qi) {
513 u32 *wait_load_cmd;
514
515
516 append_seq_load(desc, 4, LDST_CLASS_DECO |
517 LDST_SRCDST_WORD_DECO_MATH3 |
518 (4 << LDST_OFFSET_SHIFT));
519
520 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
521 JUMP_COND_CALM | JUMP_COND_NCP |
522 JUMP_COND_NOP | JUMP_COND_NIP |
523 JUMP_COND_NIFP);
524 set_jump_tgt_here(desc, wait_load_cmd);
525 }
526
527 if (is_rfc3686) {
528 if (is_qi)
529 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
530 LDST_SRCDST_BYTE_CONTEXT |
531 (ctx1_iv_off << LDST_OFFSET_SHIFT));
532
533 goto copy_iv;
534 }
535
536
537 geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO |
538 NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 |
539 NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT);
540 append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB |
541 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
542 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
543 append_move(desc, MOVE_WAITCOMP |
544 MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX |
545 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
546 (ivsize << MOVE_LEN_SHIFT));
547 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
548
549copy_iv:
550
551 append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO |
552 (ctx1_iv_off << MOVE_OFFSET_SHIFT) |
553 (ivsize << MOVE_LEN_SHIFT));
554
555
556 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
557 OP_ALG_ENCRYPT);
558
559
560 if (is_qi || era < 3) {
561 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
562 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
563 } else {
564 append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
565 append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ);
566 }
567
568
569 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
570
571
572 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG |
573 KEY_VLF);
574
575
576 moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 |
577 NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT);
578 append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB |
579 LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM);
580 append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB |
581 LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM);
582
583
584 if (is_rfc3686)
585 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
586 LDST_SRCDST_BYTE_CONTEXT |
587 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
588 LDST_OFFSET_SHIFT));
589
590
591 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
592 OP_ALG_ENCRYPT);
593
594
595 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
596
597
598 append_seq_fifo_load(desc, ivsize,
599 FIFOLD_CLASS_SKIP);
600
601
602 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
603
604
605
606
607
608 wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP);
609 set_jump_tgt_here(desc, wait_cmd);
610
611 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF |
612 FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH);
613 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
614
615
616 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
617 LDST_SRCDST_BYTE_CONTEXT);
618
619 print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ",
620 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
621 1);
622}
623EXPORT_SYMBOL(cnstr_shdsc_aead_givencap);
624
625
626
627
628
629
630
631
632
633
634void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata,
635 unsigned int ivsize, unsigned int icvsize,
636 const bool is_qi)
637{
638 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1,
639 *zero_assoc_jump_cmd2;
640
641 init_sh_desc(desc, HDR_SHARE_SERIAL);
642
643
644 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
645 JUMP_COND_SHRD);
646 if (cdata->key_inline)
647 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
648 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
649 else
650 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
651 KEY_DEST_CLASS_REG);
652 set_jump_tgt_here(desc, key_jump_cmd);
653
654
655 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
656 OP_ALG_ENCRYPT);
657
658 if (is_qi) {
659 u32 *wait_load_cmd;
660
661
662 append_seq_load(desc, 4, LDST_CLASS_DECO |
663 LDST_SRCDST_WORD_DECO_MATH3 |
664 (4 << LDST_OFFSET_SHIFT));
665
666 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
667 JUMP_COND_CALM | JUMP_COND_NCP |
668 JUMP_COND_NOP | JUMP_COND_NIP |
669 JUMP_COND_NIFP);
670 set_jump_tgt_here(desc, wait_load_cmd);
671
672 append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM,
673 ivsize);
674 } else {
675 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0,
676 CAAM_CMD_SZ);
677 }
678
679
680 zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL |
681 JUMP_COND_MATH_Z);
682
683 if (is_qi)
684 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
685 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
686
687
688 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
689 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
690 JUMP_COND_MATH_Z);
691
692 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
693
694
695 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
696
697
698 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
699
700
701 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
702 JUMP_COND_MATH_Z);
703
704
705 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
706 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
707 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
708
709 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
710
711
712 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
713
714
715 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
716 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
717
718
719 if (is_qi)
720 append_jump(desc, JUMP_TEST_ALL | 4);
721 else
722 append_jump(desc, JUMP_TEST_ALL | 2);
723
724
725 set_jump_tgt_here(desc, zero_payload_jump_cmd);
726
727
728 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
729 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
730 if (is_qi)
731
732 append_jump(desc, JUMP_TEST_ALL | 2);
733
734
735 set_jump_tgt_here(desc, zero_assoc_jump_cmd2);
736
737 if (is_qi)
738 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
739 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 |
740 FIFOLD_TYPE_LAST1);
741
742
743 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
744 LDST_SRCDST_BYTE_CONTEXT);
745
746 print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ",
747 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
748 1);
749}
750EXPORT_SYMBOL(cnstr_shdsc_gcm_encap);
751
752
753
754
755
756
757
758
759
760
761void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata,
762 unsigned int ivsize, unsigned int icvsize,
763 const bool is_qi)
764{
765 u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1;
766
767 init_sh_desc(desc, HDR_SHARE_SERIAL);
768
769
770 key_jump_cmd = append_jump(desc, JUMP_JSL |
771 JUMP_TEST_ALL | JUMP_COND_SHRD);
772 if (cdata->key_inline)
773 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
774 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
775 else
776 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
777 KEY_DEST_CLASS_REG);
778 set_jump_tgt_here(desc, key_jump_cmd);
779
780
781 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
782 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
783
784 if (is_qi) {
785 u32 *wait_load_cmd;
786
787
788 append_seq_load(desc, 4, LDST_CLASS_DECO |
789 LDST_SRCDST_WORD_DECO_MATH3 |
790 (4 << LDST_OFFSET_SHIFT));
791
792 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
793 JUMP_COND_CALM | JUMP_COND_NCP |
794 JUMP_COND_NOP | JUMP_COND_NIP |
795 JUMP_COND_NIFP);
796 set_jump_tgt_here(desc, wait_load_cmd);
797
798 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
799 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
800 }
801
802
803 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
804 zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL |
805 JUMP_COND_MATH_Z);
806
807 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
808
809
810 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
811
812
813 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
814 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
815
816 set_jump_tgt_here(desc, zero_assoc_jump_cmd1);
817
818
819 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
820
821
822 zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
823 JUMP_COND_MATH_Z);
824
825 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
826
827
828 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
829
830
831 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
832 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
833
834
835 set_jump_tgt_here(desc, zero_payload_jump_cmd);
836
837
838 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
839 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
840
841 print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ",
842 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
843 1);
844}
845EXPORT_SYMBOL(cnstr_shdsc_gcm_decap);
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata,
862 unsigned int ivsize, unsigned int icvsize,
863 const bool is_qi)
864{
865 u32 *key_jump_cmd, *zero_cryptlen_jump_cmd, *skip_instructions;
866 init_sh_desc(desc, HDR_SHARE_SERIAL);
867
868
869 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
870 JUMP_COND_SHRD);
871 if (cdata->key_inline)
872 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
873 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
874 else
875 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
876 KEY_DEST_CLASS_REG);
877 set_jump_tgt_here(desc, key_jump_cmd);
878
879
880 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
881 OP_ALG_ENCRYPT);
882
883 if (is_qi) {
884 u32 *wait_load_cmd;
885
886
887 append_seq_load(desc, 4, LDST_CLASS_DECO |
888 LDST_SRCDST_WORD_DECO_MATH3 |
889 (4 << LDST_OFFSET_SHIFT));
890
891 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
892 JUMP_COND_CALM | JUMP_COND_NCP |
893 JUMP_COND_NOP | JUMP_COND_NIP |
894 JUMP_COND_NIFP);
895 set_jump_tgt_here(desc, wait_load_cmd);
896
897
898 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
899 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
900 FIFOLD_TYPE_IV);
901 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
902 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
903 }
904
905 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
906 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
907
908
909 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
910
911
912 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ);
913
914
915 zero_cryptlen_jump_cmd = append_jump(desc, JUMP_TEST_ALL |
916 JUMP_COND_MATH_Z);
917
918
919 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
920 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
921
922
923 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA);
924
925
926 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
927 append_math_add(desc, VARSEQINLEN, VARSEQOUTLEN, REG0, CAAM_CMD_SZ);
928
929
930 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
931
932
933 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
934 FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
935
936
937 skip_instructions = append_jump(desc, JUMP_TEST_ALL);
938
939
940 set_jump_tgt_here(desc, zero_cryptlen_jump_cmd);
941
942
943 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
944 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1);
945
946 set_jump_tgt_here(desc, skip_instructions);
947
948
949 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
950 LDST_SRCDST_BYTE_CONTEXT);
951
952 print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ",
953 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
954 1);
955}
956EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap);
957
958
959
960
961
962
963
964
965
966
967
968void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata,
969 unsigned int ivsize, unsigned int icvsize,
970 const bool is_qi)
971{
972 u32 *key_jump_cmd;
973
974 init_sh_desc(desc, HDR_SHARE_SERIAL);
975
976
977 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
978 JUMP_COND_SHRD);
979 if (cdata->key_inline)
980 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
981 cdata->keylen, CLASS_1 |
982 KEY_DEST_CLASS_REG);
983 else
984 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
985 KEY_DEST_CLASS_REG);
986 set_jump_tgt_here(desc, key_jump_cmd);
987
988
989 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
990 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
991
992 if (is_qi) {
993 u32 *wait_load_cmd;
994
995
996 append_seq_load(desc, 4, LDST_CLASS_DECO |
997 LDST_SRCDST_WORD_DECO_MATH3 |
998 (4 << LDST_OFFSET_SHIFT));
999
1000 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1001 JUMP_COND_CALM | JUMP_COND_NCP |
1002 JUMP_COND_NOP | JUMP_COND_NIP |
1003 JUMP_COND_NIFP);
1004 set_jump_tgt_here(desc, wait_load_cmd);
1005
1006
1007 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1008 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1009 FIFOLD_TYPE_IV);
1010 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1011 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1012 }
1013
1014 append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize);
1015 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1016
1017
1018 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1019 FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1);
1020
1021
1022 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP);
1023
1024
1025 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ);
1026
1027
1028 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG);
1029
1030
1031 append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF);
1032
1033
1034 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1035
1036
1037 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1038
1039
1040 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF |
1041 FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1);
1042
1043
1044 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1045 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1046
1047 print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ",
1048 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1049 1);
1050}
1051EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap);
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata,
1064 unsigned int ivsize, unsigned int icvsize,
1065 const bool is_qi)
1066{
1067 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1068
1069 init_sh_desc(desc, HDR_SHARE_SERIAL);
1070
1071
1072 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1073 JUMP_COND_SHRD);
1074 if (cdata->key_inline)
1075 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1076 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1077 else
1078 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1079 KEY_DEST_CLASS_REG);
1080 set_jump_tgt_here(desc, key_jump_cmd);
1081
1082
1083 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1084 OP_ALG_ENCRYPT);
1085
1086 if (is_qi) {
1087
1088 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1089
1090
1091 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1092 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1093 FIFOLD_TYPE_IV);
1094 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1095 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1096 }
1097
1098
1099 append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ);
1100
1101
1102
1103
1104
1105
1106 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1107 (0x6 << MOVE_LEN_SHIFT));
1108 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1109 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1110
1111
1112 append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1113
1114
1115 append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1116
1117
1118 aead_append_src_dst(desc, FIFOLD_TYPE_AAD);
1119
1120 set_move_tgt_here(desc, read_move_cmd);
1121 set_move_tgt_here(desc, write_move_cmd);
1122 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1123
1124 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1125
1126
1127 append_seq_store(desc, icvsize, LDST_CLASS_1_CCB |
1128 LDST_SRCDST_BYTE_CONTEXT);
1129
1130 print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ",
1131 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1132 1);
1133}
1134EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap);
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata,
1147 unsigned int ivsize, unsigned int icvsize,
1148 const bool is_qi)
1149{
1150 u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd;
1151
1152 init_sh_desc(desc, HDR_SHARE_SERIAL);
1153
1154
1155 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1156 JUMP_COND_SHRD);
1157 if (cdata->key_inline)
1158 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1159 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1160 else
1161 append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 |
1162 KEY_DEST_CLASS_REG);
1163 set_jump_tgt_here(desc, key_jump_cmd);
1164
1165
1166 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1167 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1168
1169 if (is_qi) {
1170
1171 append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP);
1172
1173
1174 append_fifo_load_as_imm(desc, (void *)(cdata->key_virt +
1175 cdata->keylen), 4, FIFOLD_CLASS_CLASS1 |
1176 FIFOLD_TYPE_IV);
1177 append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 |
1178 FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1);
1179 }
1180
1181
1182 append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1183
1184
1185
1186
1187
1188
1189 read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 |
1190 (0x6 << MOVE_LEN_SHIFT));
1191 write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF |
1192 (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP);
1193
1194
1195 append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1196
1197
1198 append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ);
1199
1200
1201 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF);
1202
1203
1204 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF |
1205 FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1);
1206
1207 set_move_tgt_here(desc, read_move_cmd);
1208 set_move_tgt_here(desc, write_move_cmd);
1209 append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO);
1210
1211 append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO);
1212 append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO);
1213
1214
1215 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 |
1216 FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1);
1217
1218 print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ",
1219 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1220 1);
1221}
1222EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap);
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata,
1241 struct alginfo *adata, unsigned int ivsize,
1242 unsigned int icvsize, const bool encap,
1243 const bool is_qi)
1244{
1245 u32 *key_jump_cmd, *wait_cmd;
1246 u32 nfifo;
1247 const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE);
1248
1249
1250 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1251
1252
1253 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1254 JUMP_COND_SHRD);
1255
1256 append_key_as_imm(desc, cdata->key_virt, cdata->keylen, cdata->keylen,
1257 CLASS_1 | KEY_DEST_CLASS_REG);
1258
1259
1260 if (is_ipsec)
1261 append_load_as_imm(desc, cdata->key_virt + cdata->keylen, 4,
1262 LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT |
1263 4 << LDST_OFFSET_SHIFT);
1264
1265 set_jump_tgt_here(desc, key_jump_cmd);
1266
1267
1268 if (encap) {
1269 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1270 OP_ALG_ENCRYPT);
1271 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1272 OP_ALG_ENCRYPT);
1273 } else {
1274 append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL |
1275 OP_ALG_DECRYPT | OP_ALG_ICV_ON);
1276 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1277 OP_ALG_DECRYPT);
1278 }
1279
1280 if (is_qi) {
1281 u32 *wait_load_cmd;
1282 u32 ctx1_iv_off = is_ipsec ? 8 : 4;
1283
1284
1285 append_seq_load(desc, 4, LDST_CLASS_DECO |
1286 LDST_SRCDST_WORD_DECO_MATH3 |
1287 4 << LDST_OFFSET_SHIFT);
1288
1289 wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1290 JUMP_COND_CALM | JUMP_COND_NCP |
1291 JUMP_COND_NOP | JUMP_COND_NIP |
1292 JUMP_COND_NIFP);
1293 set_jump_tgt_here(desc, wait_load_cmd);
1294
1295 append_seq_load(desc, ivsize, LDST_CLASS_1_CCB |
1296 LDST_SRCDST_BYTE_CONTEXT |
1297 ctx1_iv_off << LDST_OFFSET_SHIFT);
1298 }
1299
1300
1301
1302
1303
1304
1305
1306 nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 |
1307 NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND;
1308 append_load_imm_u32(desc, nfifo, LDST_CLASS_IND_CCB |
1309 LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3);
1310
1311 append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ);
1312 append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ);
1313 append_seq_fifo_load(desc, 0, FIFOLD_TYPE_NOINFOFIFO |
1314 FIFOLD_CLASS_CLASS1 | LDST_VLF);
1315 append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK |
1316 MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3);
1317 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF);
1318
1319
1320 if (is_ipsec)
1321 append_seq_fifo_store(desc, ivsize, FIFOST_TYPE_METADATA |
1322 0x2 << 25);
1323
1324 wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL |
1325 JUMP_COND_NOP | JUMP_TEST_ALL);
1326 set_jump_tgt_here(desc, wait_cmd);
1327
1328 if (encap) {
1329
1330 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1331 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0,
1332 CAAM_CMD_SZ);
1333 aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2);
1334
1335
1336 append_seq_store(desc, icvsize, LDST_CLASS_2_CCB |
1337 LDST_SRCDST_BYTE_CONTEXT);
1338 } else {
1339
1340 append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0,
1341 CAAM_CMD_SZ);
1342 append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0,
1343 CAAM_CMD_SZ);
1344 aead_append_src_dst(desc, FIFOLD_TYPE_MSG);
1345
1346
1347 append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 |
1348 FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV);
1349 }
1350
1351 print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": ",
1352 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1353 1);
1354}
1355EXPORT_SYMBOL(cnstr_shdsc_chachapoly);
1356
1357
1358static inline void skcipher_append_src_dst(u32 *desc)
1359{
1360 append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1361 append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ);
1362 append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 |
1363 KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1);
1364 append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF);
1365}
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata,
1379 unsigned int ivsize, const bool is_rfc3686,
1380 const u32 ctx1_iv_off)
1381{
1382 u32 *key_jump_cmd;
1383 u32 options = cdata->algtype | OP_ALG_AS_INIT | OP_ALG_ENCRYPT;
1384 bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1385 OP_ALG_ALGSEL_CHACHA20);
1386
1387 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1388
1389 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1390 JUMP_COND_SHRD);
1391
1392
1393 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1394 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1395
1396
1397 if (is_rfc3686) {
1398 const u8 *nonce = cdata->key_virt + cdata->keylen;
1399
1400 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1401 LDST_CLASS_IND_CCB |
1402 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1403 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1404 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1405 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1406 }
1407
1408 set_jump_tgt_here(desc, key_jump_cmd);
1409
1410
1411 if (ivsize)
1412 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1413 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1414 LDST_OFFSET_SHIFT));
1415
1416
1417 if (is_rfc3686)
1418 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1419 LDST_SRCDST_BYTE_CONTEXT |
1420 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1421 LDST_OFFSET_SHIFT));
1422
1423
1424 if (is_chacha20)
1425 options |= OP_ALG_AS_FINALIZE;
1426 append_operation(desc, options);
1427
1428
1429 skcipher_append_src_dst(desc);
1430
1431
1432 if (!is_chacha20 && ivsize)
1433 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1434 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1435 LDST_OFFSET_SHIFT));
1436
1437 print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ",
1438 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1439 1);
1440}
1441EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap);
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata,
1455 unsigned int ivsize, const bool is_rfc3686,
1456 const u32 ctx1_iv_off)
1457{
1458 u32 *key_jump_cmd;
1459 bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) ==
1460 OP_ALG_ALGSEL_CHACHA20);
1461
1462 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1463
1464 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1465 JUMP_COND_SHRD);
1466
1467
1468 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1469 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1470
1471
1472 if (is_rfc3686) {
1473 const u8 *nonce = cdata->key_virt + cdata->keylen;
1474
1475 append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE,
1476 LDST_CLASS_IND_CCB |
1477 LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM);
1478 append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO |
1479 MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) |
1480 (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT));
1481 }
1482
1483 set_jump_tgt_here(desc, key_jump_cmd);
1484
1485
1486 if (ivsize)
1487 append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1488 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1489 LDST_OFFSET_SHIFT));
1490
1491
1492 if (is_rfc3686)
1493 append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB |
1494 LDST_SRCDST_BYTE_CONTEXT |
1495 ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) <<
1496 LDST_OFFSET_SHIFT));
1497
1498
1499 if (ctx1_iv_off)
1500 append_operation(desc, cdata->algtype | OP_ALG_AS_INIT |
1501 OP_ALG_DECRYPT);
1502 else
1503 append_dec_op1(desc, cdata->algtype);
1504
1505
1506 skcipher_append_src_dst(desc);
1507
1508
1509 if (!is_chacha20 && ivsize)
1510 append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT |
1511 LDST_CLASS_1_CCB | (ctx1_iv_off <<
1512 LDST_OFFSET_SHIFT));
1513
1514 print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ",
1515 DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc),
1516 1);
1517}
1518EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap);
1519
1520
1521
1522
1523
1524
1525
1526void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata)
1527{
1528
1529
1530
1531
1532
1533
1534 __be64 sector_size = cpu_to_be64(BIT(15));
1535 u32 *key_jump_cmd;
1536
1537 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1538
1539 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1540 JUMP_COND_SHRD);
1541
1542
1543 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1544 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1545
1546
1547 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB |
1548 LDST_SRCDST_BYTE_CONTEXT |
1549 (0x28 << LDST_OFFSET_SHIFT));
1550
1551 set_jump_tgt_here(desc, key_jump_cmd);
1552
1553
1554
1555
1556
1557
1558 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1559 (0x20 << LDST_OFFSET_SHIFT));
1560 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1561 (0x30 << LDST_OFFSET_SHIFT));
1562
1563
1564 append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL |
1565 OP_ALG_ENCRYPT);
1566
1567
1568 skcipher_append_src_dst(desc);
1569
1570
1571 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1572 (0x20 << LDST_OFFSET_SHIFT));
1573 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1574 (0x30 << LDST_OFFSET_SHIFT));
1575
1576 print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__)
1577 ": ", DUMP_PREFIX_ADDRESS, 16, 4,
1578 desc, desc_bytes(desc), 1);
1579}
1580EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap);
1581
1582
1583
1584
1585
1586
1587
1588void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata)
1589{
1590
1591
1592
1593
1594
1595
1596 __be64 sector_size = cpu_to_be64(BIT(15));
1597 u32 *key_jump_cmd;
1598
1599 init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX);
1600
1601 key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL |
1602 JUMP_COND_SHRD);
1603
1604
1605 append_key_as_imm(desc, cdata->key_virt, cdata->keylen,
1606 cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG);
1607
1608
1609 append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB |
1610 LDST_SRCDST_BYTE_CONTEXT |
1611 (0x28 << LDST_OFFSET_SHIFT));
1612
1613 set_jump_tgt_here(desc, key_jump_cmd);
1614
1615
1616
1617
1618
1619
1620 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1621 (0x20 << LDST_OFFSET_SHIFT));
1622 append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1623 (0x30 << LDST_OFFSET_SHIFT));
1624
1625 append_dec_op1(desc, cdata->algtype);
1626
1627
1628 skcipher_append_src_dst(desc);
1629
1630
1631 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1632 (0x20 << LDST_OFFSET_SHIFT));
1633 append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB |
1634 (0x30 << LDST_OFFSET_SHIFT));
1635
1636 print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__)
1637 ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc,
1638 desc_bytes(desc), 1);
1639}
1640EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap);
1641
1642MODULE_LICENSE("GPL");
1643MODULE_DESCRIPTION("FSL CAAM descriptor support");
1644MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
1645