1
2
3
4
5
6
7
8#include <linux/dma-mapping.h>
9#include <linux/interrupt.h>
10#include <linux/platform_device.h>
11#include <linux/slab.h>
12#include <linux/swab.h>
13
14#include <asm/byteorder.h>
15#include <asm/errno.h>
16
17#include <crypto/aes.h>
18#include <crypto/gcm.h>
19
20#include "ocs-aes.h"
21
22#define AES_COMMAND_OFFSET 0x0000
23#define AES_KEY_0_OFFSET 0x0004
24#define AES_KEY_1_OFFSET 0x0008
25#define AES_KEY_2_OFFSET 0x000C
26#define AES_KEY_3_OFFSET 0x0010
27#define AES_KEY_4_OFFSET 0x0014
28#define AES_KEY_5_OFFSET 0x0018
29#define AES_KEY_6_OFFSET 0x001C
30#define AES_KEY_7_OFFSET 0x0020
31#define AES_IV_0_OFFSET 0x0024
32#define AES_IV_1_OFFSET 0x0028
33#define AES_IV_2_OFFSET 0x002C
34#define AES_IV_3_OFFSET 0x0030
35#define AES_ACTIVE_OFFSET 0x0034
36#define AES_STATUS_OFFSET 0x0038
37#define AES_KEY_SIZE_OFFSET 0x0044
38#define AES_IER_OFFSET 0x0048
39#define AES_ISR_OFFSET 0x005C
40#define AES_MULTIPURPOSE1_0_OFFSET 0x0200
41#define AES_MULTIPURPOSE1_1_OFFSET 0x0204
42#define AES_MULTIPURPOSE1_2_OFFSET 0x0208
43#define AES_MULTIPURPOSE1_3_OFFSET 0x020C
44#define AES_MULTIPURPOSE2_0_OFFSET 0x0220
45#define AES_MULTIPURPOSE2_1_OFFSET 0x0224
46#define AES_MULTIPURPOSE2_2_OFFSET 0x0228
47#define AES_MULTIPURPOSE2_3_OFFSET 0x022C
48#define AES_BYTE_ORDER_CFG_OFFSET 0x02C0
49#define AES_TLEN_OFFSET 0x0300
50#define AES_T_MAC_0_OFFSET 0x0304
51#define AES_T_MAC_1_OFFSET 0x0308
52#define AES_T_MAC_2_OFFSET 0x030C
53#define AES_T_MAC_3_OFFSET 0x0310
54#define AES_PLEN_OFFSET 0x0314
55#define AES_A_DMA_SRC_ADDR_OFFSET 0x0400
56#define AES_A_DMA_DST_ADDR_OFFSET 0x0404
57#define AES_A_DMA_SRC_SIZE_OFFSET 0x0408
58#define AES_A_DMA_DST_SIZE_OFFSET 0x040C
59#define AES_A_DMA_DMA_MODE_OFFSET 0x0410
60#define AES_A_DMA_NEXT_SRC_DESCR_OFFSET 0x0418
61#define AES_A_DMA_NEXT_DST_DESCR_OFFSET 0x041C
62#define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET 0x0420
63#define AES_A_DMA_LOG_OFFSET 0x0424
64#define AES_A_DMA_STATUS_OFFSET 0x0428
65#define AES_A_DMA_PERF_CNTR_OFFSET 0x042C
66#define AES_A_DMA_MSI_ISR_OFFSET 0x0480
67#define AES_A_DMA_MSI_IER_OFFSET 0x0484
68#define AES_A_DMA_MSI_MASK_OFFSET 0x0488
69#define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET 0x0600
70#define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET 0x0700
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87#define AES_A_DMA_DMA_MODE_ACTIVE BIT(31)
88#define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN BIT(25)
89#define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN BIT(24)
90
91
92
93
94
95
96
97
98
99
100
101#define AES_ACTIVE_LAST_ADATA BIT(9)
102#define AES_ACTIVE_LAST_CCM_GCM BIT(8)
103#define AES_ACTIVE_TERMINATION BIT(1)
104#define AES_ACTIVE_TRIGGER BIT(0)
105
106#define AES_DISABLE_INT 0x00000000
107#define AES_DMA_CPD_ERR_INT BIT(8)
108#define AES_DMA_OUTBUF_RD_ERR_INT BIT(7)
109#define AES_DMA_OUTBUF_WR_ERR_INT BIT(6)
110#define AES_DMA_INBUF_RD_ERR_INT BIT(5)
111#define AES_DMA_INBUF_WR_ERR_INT BIT(4)
112#define AES_DMA_BAD_COMP_INT BIT(3)
113#define AES_DMA_SAI_INT BIT(2)
114#define AES_DMA_SRC_DONE_INT BIT(0)
115#define AES_COMPLETE_INT BIT(1)
116
117#define AES_DMA_MSI_MASK_CLEAR BIT(0)
118
119#define AES_128_BIT_KEY 0x00000000
120#define AES_256_BIT_KEY BIT(0)
121
122#define AES_DEACTIVATE_PERF_CNTR 0x00000000
123#define AES_ACTIVATE_PERF_CNTR BIT(0)
124
125#define AES_MAX_TAG_SIZE_U32 4
126
127#define OCS_LL_DMA_FLAG_TERMINATE BIT(31)
128
129
130
131
132
133#define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK 0x3FF
134
135
136
137
138
139
140#define CCM_DECRYPT_DELAY_TAG_CLK_COUNT 36UL
141
142
143
144
145
146
147#define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 42UL
148
149
150#define L_PRIME_MIN (1)
151#define L_PRIME_MAX (7)
152
153
154
155
156
157
158
159
160
161
162
163#define L_PRIME_IDX 0
164#define COUNTER_START(lprime) (16 - ((lprime) + 1))
165#define COUNTER_LEN(lprime) ((lprime) + 1)
166
167enum aes_counter_mode {
168 AES_CTR_M_NO_INC = 0,
169 AES_CTR_M_32_INC = 1,
170 AES_CTR_M_64_INC = 2,
171 AES_CTR_M_128_INC = 3,
172};
173
174
175
176
177
178
179
180
181struct ocs_dma_linked_list {
182 u32 src_addr;
183 u32 src_len;
184 u32 next;
185 u32 ll_flags;
186} __packed;
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev)
205{
206 iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET);
207}
208
209
210static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev)
211{
212 iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET);
213}
214
215
216static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev)
217{
218 iowrite32(AES_ACTIVE_TERMINATION,
219 aes_dev->base_reg + AES_ACTIVE_OFFSET);
220}
221
222
223
224
225
226
227
228
229
230
231static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev)
232{
233 iowrite32(AES_ACTIVE_LAST_CCM_GCM,
234 aes_dev->base_reg + AES_ACTIVE_OFFSET);
235}
236
237
238static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev)
239{
240 u32 aes_active_reg;
241
242 do {
243 aes_active_reg = ioread32(aes_dev->base_reg +
244 AES_ACTIVE_OFFSET);
245 } while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM);
246}
247
248
249static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev)
250{
251 u32 reg;
252
253 do {
254 reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET);
255 } while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK);
256}
257
258
259
260
261
262
263
264
265static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev)
266{
267 iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM,
268 aes_dev->base_reg + AES_ACTIVE_OFFSET);
269}
270
271
272static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev)
273{
274 iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
275 iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
276}
277
278
279static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev)
280{
281 iowrite32(AES_A_DMA_DMA_MODE_ACTIVE,
282 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
283}
284
285
286static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev)
287{
288 iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
289 AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN,
290 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
291}
292
293
294static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev)
295{
296 iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
297 AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN,
298 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
299}
300
301
302static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev)
303{
304 iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
305 AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN |
306 AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN,
307 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
308}
309
310
311static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev)
312{
313 iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET);
314 iowrite32(AES_ACTIVATE_PERF_CNTR,
315 aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
316}
317
318
319static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev,
320 int delay)
321{
322 while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay)
323 ;
324 iowrite32(AES_DEACTIVATE_PERF_CNTR,
325 aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
326}
327
328
329static void aes_irq_disable(struct ocs_aes_dev *aes_dev)
330{
331 u32 isr_val = 0;
332
333
334 iowrite32(AES_DISABLE_INT,
335 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
336 iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
337
338
339 isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
340 if (isr_val)
341 iowrite32(isr_val,
342 aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
343
344 isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
345 if (isr_val)
346 iowrite32(isr_val,
347 aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
348
349 isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET);
350 if (isr_val)
351 iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET);
352}
353
354
355static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq)
356{
357 if (irq == AES_COMPLETE_INT) {
358
359 iowrite32(AES_DMA_CPD_ERR_INT |
360 AES_DMA_OUTBUF_RD_ERR_INT |
361 AES_DMA_OUTBUF_WR_ERR_INT |
362 AES_DMA_INBUF_RD_ERR_INT |
363 AES_DMA_INBUF_WR_ERR_INT |
364 AES_DMA_BAD_COMP_INT |
365 AES_DMA_SAI_INT,
366 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
367
368
369
370
371
372
373
374
375 iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET);
376 return;
377 }
378 if (irq == AES_DMA_SRC_DONE_INT) {
379
380 iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395 iowrite32(AES_DMA_CPD_ERR_INT |
396 AES_DMA_OUTBUF_RD_ERR_INT |
397 AES_DMA_OUTBUF_WR_ERR_INT |
398 AES_DMA_INBUF_RD_ERR_INT |
399 AES_DMA_INBUF_WR_ERR_INT |
400 AES_DMA_BAD_COMP_INT |
401 AES_DMA_SAI_INT |
402 AES_DMA_SRC_DONE_INT,
403 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
404 }
405}
406
407
408static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq)
409{
410 int rc;
411
412 reinit_completion(&aes_dev->irq_completion);
413 aes_irq_enable(aes_dev, irq);
414 rc = wait_for_completion_interruptible(&aes_dev->irq_completion);
415 if (rc)
416 return rc;
417
418 return aes_dev->dma_err_mask ? -EIO : 0;
419}
420
421
422static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev,
423 dma_addr_t dma_list)
424{
425 iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
426 iowrite32(dma_list,
427 aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET);
428}
429
430
431static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev,
432 dma_addr_t dma_list)
433{
434 iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
435 iowrite32(dma_list,
436 aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET);
437}
438
439irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id)
440{
441 struct ocs_aes_dev *aes_dev = dev_id;
442 u32 aes_dma_isr;
443
444
445 aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
446
447
448 aes_irq_disable(aes_dev);
449
450
451 aes_dev->dma_err_mask = aes_dma_isr &
452 (AES_DMA_CPD_ERR_INT |
453 AES_DMA_OUTBUF_RD_ERR_INT |
454 AES_DMA_OUTBUF_WR_ERR_INT |
455 AES_DMA_INBUF_RD_ERR_INT |
456 AES_DMA_INBUF_WR_ERR_INT |
457 AES_DMA_BAD_COMP_INT |
458 AES_DMA_SAI_INT);
459
460
461 complete(&aes_dev->irq_completion);
462
463 return IRQ_HANDLED;
464}
465
466
467
468
469
470
471
472
473
474
475
476
477int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key,
478 enum ocs_cipher cipher)
479{
480 const u32 *key_u32;
481 u32 val;
482 int i;
483
484
485 if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) {
486 dev_err(aes_dev->dev,
487 "%d-bit keys not supported by AES cipher\n",
488 key_size * 8);
489 return -EINVAL;
490 }
491
492 if (cipher == OCS_SM4 && key_size != 16) {
493 dev_err(aes_dev->dev,
494 "%d-bit keys not supported for SM4 cipher\n",
495 key_size * 8);
496 return -EINVAL;
497 }
498
499 if (!key)
500 return -EINVAL;
501
502 key_u32 = (const u32 *)key;
503
504
505 for (i = 0; i < (key_size / sizeof(u32)); i++) {
506 iowrite32(key_u32[i],
507 aes_dev->base_reg + AES_KEY_0_OFFSET +
508 (i * sizeof(u32)));
509 }
510
511
512
513
514
515
516
517 val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY;
518 iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET);
519
520 return 0;
521}
522
523
524static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev,
525 enum ocs_cipher cipher,
526 enum ocs_mode mode,
527 enum ocs_instruction instruction)
528{
529 u32 val;
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554 val = (cipher << 14) | (mode << 8) | (instruction << 6) |
555 (AES_CTR_M_128_INC << 2);
556 iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET);
557}
558
559static void ocs_aes_init(struct ocs_aes_dev *aes_dev,
560 enum ocs_mode mode,
561 enum ocs_cipher cipher,
562 enum ocs_instruction instruction)
563{
564
565 aes_irq_disable(aes_dev);
566
567
568 aes_a_set_endianness(aes_dev);
569
570
571 set_ocs_aes_command(aes_dev, cipher, mode, instruction);
572}
573
574
575
576
577
578static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev,
579 u32 size)
580{
581 u32 val;
582
583 if (size == 0) {
584 val = 0;
585 goto exit;
586 }
587
588 val = size % AES_BLOCK_SIZE;
589 if (val == 0)
590 val = AES_BLOCK_SIZE;
591
592exit:
593 iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET);
594}
595
596
597
598
599
600static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size,
601 const u8 *iv, u32 iv_size,
602 dma_addr_t aad_dma_list, u32 aad_size,
603 const u8 *tag, u32 tag_size,
604 enum ocs_cipher cipher, enum ocs_mode mode,
605 enum ocs_instruction instruction,
606 dma_addr_t dst_dma_list)
607{
608
609 if (!(cipher == OCS_AES || cipher == OCS_SM4))
610 return -EINVAL;
611
612 if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC &&
613 mode != OCS_MODE_CTR && mode != OCS_MODE_CCM &&
614 mode != OCS_MODE_GCM && mode != OCS_MODE_CTS)
615 return -EINVAL;
616
617 if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT &&
618 instruction != OCS_EXPAND && instruction != OCS_BYPASS)
619 return -EINVAL;
620
621
622
623
624
625
626
627
628 if (instruction == OCS_BYPASS) {
629 if (src_dma_list == DMA_MAPPING_ERROR ||
630 dst_dma_list == DMA_MAPPING_ERROR)
631 return -EINVAL;
632
633 return 0;
634 }
635
636
637
638
639
640 switch (mode) {
641 case OCS_MODE_ECB:
642
643 if (src_size % AES_BLOCK_SIZE != 0)
644 return -EINVAL;
645
646
647 if (src_dma_list == DMA_MAPPING_ERROR ||
648 dst_dma_list == DMA_MAPPING_ERROR)
649 return -EINVAL;
650
651 return 0;
652
653 case OCS_MODE_CBC:
654
655 if (src_size % AES_BLOCK_SIZE != 0)
656 return -EINVAL;
657
658
659 if (src_dma_list == DMA_MAPPING_ERROR ||
660 dst_dma_list == DMA_MAPPING_ERROR)
661 return -EINVAL;
662
663
664 if (!iv || iv_size != AES_BLOCK_SIZE)
665 return -EINVAL;
666
667 return 0;
668
669 case OCS_MODE_CTR:
670
671 if (src_size == 0)
672 return -EINVAL;
673
674
675 if (src_dma_list == DMA_MAPPING_ERROR ||
676 dst_dma_list == DMA_MAPPING_ERROR)
677 return -EINVAL;
678
679
680 if (!iv || iv_size != AES_BLOCK_SIZE)
681 return -EINVAL;
682
683 return 0;
684
685 case OCS_MODE_CTS:
686
687 if (src_size < AES_BLOCK_SIZE)
688 return -EINVAL;
689
690
691 if (src_dma_list == DMA_MAPPING_ERROR ||
692 dst_dma_list == DMA_MAPPING_ERROR)
693 return -EINVAL;
694
695
696 if (!iv || iv_size != AES_BLOCK_SIZE)
697 return -EINVAL;
698
699 return 0;
700
701 case OCS_MODE_GCM:
702
703 if (!iv || iv_size != GCM_AES_IV_SIZE)
704 return -EINVAL;
705
706
707
708
709
710 if (src_size && (src_dma_list == DMA_MAPPING_ERROR ||
711 dst_dma_list == DMA_MAPPING_ERROR))
712 return -EINVAL;
713
714
715 if (aad_size && aad_dma_list == DMA_MAPPING_ERROR)
716 return -EINVAL;
717
718
719 if (!tag)
720 return -EINVAL;
721
722
723 if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32)))
724 return -EINVAL;
725
726 return 0;
727
728 case OCS_MODE_CCM:
729
730 if (!iv || iv_size != AES_BLOCK_SIZE)
731 return -EINVAL;
732
733
734 if (iv[L_PRIME_IDX] < L_PRIME_MIN ||
735 iv[L_PRIME_IDX] > L_PRIME_MAX)
736 return -EINVAL;
737
738
739 if (aad_size && aad_dma_list == DMA_MAPPING_ERROR)
740 return -EINVAL;
741
742
743 if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32)))
744 return -EINVAL;
745
746 if (instruction == OCS_DECRYPT) {
747
748
749
750
751 if (src_size && (src_dma_list == DMA_MAPPING_ERROR ||
752 dst_dma_list == DMA_MAPPING_ERROR))
753 return -EINVAL;
754
755
756 if (!tag)
757 return -EINVAL;
758
759 return 0;
760 }
761
762
763
764
765
766
767
768 if (dst_dma_list == DMA_MAPPING_ERROR)
769 return -EINVAL;
770
771
772 if (src_size && src_dma_list == DMA_MAPPING_ERROR)
773 return -EINVAL;
774
775 return 0;
776
777 default:
778 return -EINVAL;
779 }
780}
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796int ocs_aes_op(struct ocs_aes_dev *aes_dev,
797 enum ocs_mode mode,
798 enum ocs_cipher cipher,
799 enum ocs_instruction instruction,
800 dma_addr_t dst_dma_list,
801 dma_addr_t src_dma_list,
802 u32 src_size,
803 u8 *iv,
804 u32 iv_size)
805{
806 u32 *iv32;
807 int rc;
808
809 rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0,
810 NULL, 0, cipher, mode, instruction,
811 dst_dma_list);
812 if (rc)
813 return rc;
814
815
816
817
818 if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM)
819 return -EINVAL;
820
821
822 iv32 = (u32 *)iv;
823
824 ocs_aes_init(aes_dev, mode, cipher, instruction);
825
826 if (mode == OCS_MODE_CTS) {
827
828 ocs_aes_write_last_data_blk_len(aes_dev, src_size);
829 }
830
831
832 if (mode != OCS_MODE_ECB) {
833 iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET);
834 iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET);
835 iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET);
836 iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET);
837 }
838
839
840 aes_a_op_trigger(aes_dev);
841
842
843 dma_to_ocs_aes_ll(aes_dev, src_dma_list);
844 dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
845 aes_a_dma_active_src_dst_ll_en(aes_dev);
846
847 if (mode == OCS_MODE_CTS) {
848
849
850
851
852 aes_a_set_last_gcx(aes_dev);
853 } else {
854
855 aes_a_op_termination(aes_dev);
856 }
857
858
859 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
860 if (rc)
861 return rc;
862
863 if (mode == OCS_MODE_CTR) {
864
865 iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET);
866 iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET);
867 iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET);
868 iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET);
869 }
870
871 return 0;
872}
873
874
875static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev,
876 const u8 *iv)
877{
878 const u32 *j0 = (u32 *)iv;
879
880
881
882
883
884 iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET);
885 iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET);
886 iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET);
887 iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET);
888}
889
890
891static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev,
892 u8 *tag, u32 tag_size)
893{
894 u32 tag_u32[AES_MAX_TAG_SIZE_U32];
895
896
897
898
899
900
901 tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET));
902 tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET));
903 tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET));
904 tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET));
905
906 memcpy(tag, tag_u32, tag_size);
907}
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev,
926 enum ocs_cipher cipher,
927 enum ocs_instruction instruction,
928 dma_addr_t dst_dma_list,
929 dma_addr_t src_dma_list,
930 u32 src_size,
931 const u8 *iv,
932 dma_addr_t aad_dma_list,
933 u32 aad_size,
934 u8 *out_tag,
935 u32 tag_size)
936{
937 u64 bit_len;
938 u32 val;
939 int rc;
940
941 rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv,
942 GCM_AES_IV_SIZE, aad_dma_list,
943 aad_size, out_tag, tag_size, cipher,
944 OCS_MODE_GCM, instruction,
945 dst_dma_list);
946 if (rc)
947 return rc;
948
949 ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction);
950
951
952 ocs_aes_gcm_write_j0(aes_dev, iv);
953
954
955 iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
956
957
958 ocs_aes_write_last_data_blk_len(aes_dev, src_size);
959
960
961 bit_len = (u64)src_size * 8;
962 val = bit_len & 0xFFFFFFFF;
963 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET);
964 val = bit_len >> 32;
965 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET);
966
967
968 bit_len = (u64)aad_size * 8;
969 val = bit_len & 0xFFFFFFFF;
970 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET);
971 val = bit_len >> 32;
972 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET);
973
974
975 aes_a_op_trigger(aes_dev);
976
977
978 if (aad_size) {
979
980 dma_to_ocs_aes_ll(aes_dev, aad_dma_list);
981 aes_a_dma_active_src_ll_en(aes_dev);
982
983
984 aes_a_set_last_gcx_and_adata(aes_dev);
985
986
987 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
988 if (rc)
989 return rc;
990 } else {
991 aes_a_set_last_gcx_and_adata(aes_dev);
992 }
993
994
995 aes_a_wait_last_gcx(aes_dev);
996 aes_a_dma_wait_input_buffer_occupancy(aes_dev);
997
998
999 if (src_size) {
1000
1001 dma_to_ocs_aes_ll(aes_dev, src_dma_list);
1002 dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
1003 aes_a_dma_active_src_dst_ll_en(aes_dev);
1004 } else {
1005 aes_a_dma_set_xfer_size_zero(aes_dev);
1006 aes_a_dma_active(aes_dev);
1007 }
1008
1009
1010 aes_a_set_last_gcx(aes_dev);
1011
1012
1013 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
1014 if (rc)
1015 return rc;
1016
1017 ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size);
1018
1019 return 0;
1020}
1021
1022
1023static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev,
1024 const u8 *in_tag, u32 tag_size)
1025{
1026 int i;
1027
1028
1029 aes_a_dma_wait_input_buffer_occupancy(aes_dev);
1030
1031
1032
1033
1034
1035
1036 aes_a_dma_reset_and_activate_perf_cntr(aes_dev);
1037 aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev,
1038 CCM_DECRYPT_DELAY_TAG_CLK_COUNT);
1039
1040
1041 for (i = 0; i < tag_size; i++) {
1042 iowrite8(in_tag[i], aes_dev->base_reg +
1043 AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
1044 }
1045}
1046
1047
1048
1049
1050
1051
1052
1053
1054static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev,
1055 const u8 *iv, u32 adata_size, u32 tag_size,
1056 u32 cryptlen)
1057{
1058 u8 b0[16];
1059 int i, q;
1060
1061
1062 memset(b0, 0, sizeof(b0));
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072 if (adata_size)
1073 b0[0] |= BIT(6);
1074
1075
1076
1077
1078
1079 b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3;
1080
1081
1082
1083
1084
1085 b0[0] |= iv[0] & 0x7;
1086
1087
1088
1089
1090
1091 q = (iv[0] & 0x7) + 1;
1092 for (i = 1; i <= 15 - q; i++)
1093 b0[i] = iv[i];
1094
1095
1096
1097
1098
1099 i = sizeof(b0) - 1;
1100 while (q) {
1101 b0[i] = cryptlen & 0xff;
1102 cryptlen >>= 8;
1103 i--;
1104 q--;
1105 }
1106
1107
1108
1109
1110 if (cryptlen)
1111 return -EOVERFLOW;
1112
1113 for (i = 0; i < sizeof(b0); i++)
1114 iowrite8(b0[i], aes_dev->base_reg +
1115 AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
1116 return 0;
1117}
1118
1119
1120
1121
1122
1123
1124
1125
1126static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev,
1127 u64 adata_len)
1128{
1129 u8 enc_a[10];
1130 int i, len;
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141 if (adata_len < 65280) {
1142 len = 2;
1143 *(__be16 *)enc_a = cpu_to_be16(adata_len);
1144 } else if (adata_len <= 0xFFFFFFFF) {
1145 len = 6;
1146 *(__be16 *)enc_a = cpu_to_be16(0xfffe);
1147 *(__be32 *)&enc_a[2] = cpu_to_be32(adata_len);
1148 } else {
1149 len = 10;
1150 *(__be16 *)enc_a = cpu_to_be16(0xffff);
1151 *(__be64 *)&enc_a[2] = cpu_to_be64(adata_len);
1152 }
1153 for (i = 0; i < len; i++)
1154 iowrite8(enc_a[i],
1155 aes_dev->base_reg +
1156 AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
1157}
1158
1159static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev,
1160 dma_addr_t adata_dma_list, u32 adata_size)
1161{
1162 int rc;
1163
1164 if (!adata_size) {
1165
1166 aes_a_set_last_gcx_and_adata(aes_dev);
1167 goto exit;
1168 }
1169
1170
1171
1172
1173
1174
1175
1176 ocs_aes_ccm_write_adata_len(aes_dev, adata_size);
1177
1178
1179 dma_to_ocs_aes_ll(aes_dev, adata_dma_list);
1180
1181
1182 aes_a_dma_active_src_ll_en(aes_dev);
1183
1184
1185 aes_a_set_last_gcx_and_adata(aes_dev);
1186
1187
1188 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
1189 if (rc)
1190 return rc;
1191
1192exit:
1193
1194 aes_a_wait_last_gcx(aes_dev);
1195 aes_a_dma_wait_input_buffer_occupancy(aes_dev);
1196
1197 return 0;
1198}
1199
1200static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev,
1201 dma_addr_t dst_dma_list,
1202 dma_addr_t src_dma_list,
1203 u32 src_size)
1204{
1205 if (src_size) {
1206
1207
1208
1209
1210 dma_to_ocs_aes_ll(aes_dev, src_dma_list);
1211 dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
1212 aes_a_dma_active_src_dst_ll_en(aes_dev);
1213 } else {
1214
1215 dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
1216 aes_a_dma_active_dst_ll_en(aes_dev);
1217 }
1218
1219
1220
1221
1222
1223 aes_a_set_last_gcx(aes_dev);
1224
1225
1226 return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
1227}
1228
1229static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev,
1230 dma_addr_t dst_dma_list,
1231 dma_addr_t src_dma_list,
1232 u32 src_size)
1233{
1234 if (!src_size) {
1235
1236 aes_a_dma_set_xfer_size_zero(aes_dev);
1237 aes_a_dma_active(aes_dev);
1238 aes_a_set_last_gcx(aes_dev);
1239
1240 return 0;
1241 }
1242
1243
1244
1245
1246
1247 dma_to_ocs_aes_ll(aes_dev, src_dma_list);
1248 dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
1249 aes_a_dma_active_src_dst_ll_en(aes_dev);
1250
1251
1252
1253
1254
1255 aes_a_set_last_gcx(aes_dev);
1256
1257
1258
1259
1260 return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
1261}
1262
1263
1264
1265
1266
1267
1268
1269static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev,
1270 u8 tag_size_bytes)
1271{
1272 u32 tag[AES_MAX_TAG_SIZE_U32];
1273 u32 yr[AES_MAX_TAG_SIZE_U32];
1274 u8 i;
1275
1276
1277 for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) {
1278 tag[i] = ioread32(aes_dev->base_reg +
1279 AES_T_MAC_0_OFFSET + (i * sizeof(u32)));
1280 yr[i] = ioread32(aes_dev->base_reg +
1281 AES_MULTIPURPOSE2_0_OFFSET +
1282 (i * sizeof(u32)));
1283 }
1284
1285 return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0;
1286}
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev,
1308 enum ocs_cipher cipher,
1309 enum ocs_instruction instruction,
1310 dma_addr_t dst_dma_list,
1311 dma_addr_t src_dma_list,
1312 u32 src_size,
1313 u8 *iv,
1314 dma_addr_t adata_dma_list,
1315 u32 adata_size,
1316 u8 *in_tag,
1317 u32 tag_size)
1318{
1319 u32 *iv_32;
1320 u8 lprime;
1321 int rc;
1322
1323 rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv,
1324 AES_BLOCK_SIZE, adata_dma_list, adata_size,
1325 in_tag, tag_size, cipher, OCS_MODE_CCM,
1326 instruction, dst_dma_list);
1327 if (rc)
1328 return rc;
1329
1330 ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction);
1331
1332
1333
1334
1335
1336 lprime = iv[L_PRIME_IDX];
1337 memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime));
1338
1339
1340
1341
1342
1343 iv_32 = (u32 *)iv;
1344 iowrite32(__swab32(iv_32[0]),
1345 aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET);
1346 iowrite32(__swab32(iv_32[1]),
1347 aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET);
1348 iowrite32(__swab32(iv_32[2]),
1349 aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET);
1350 iowrite32(__swab32(iv_32[3]),
1351 aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET);
1352
1353
1354 iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
1355
1356
1357
1358
1359
1360 ocs_aes_write_last_data_blk_len(aes_dev, src_size);
1361
1362
1363 aes_a_op_trigger(aes_dev);
1364
1365 aes_a_dma_reset_and_activate_perf_cntr(aes_dev);
1366
1367
1368 rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size);
1369 if (rc)
1370 return rc;
1371
1372
1373
1374
1375 aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev,
1376 CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT);
1377
1378
1379 ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size);
1380
1381
1382 if (instruction == OCS_ENCRYPT) {
1383 return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list,
1384 src_dma_list, src_size);
1385 }
1386
1387 rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list,
1388 src_dma_list, src_size);
1389 if (rc)
1390 return rc;
1391
1392
1393 ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size);
1394 rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
1395 if (rc)
1396 return rc;
1397
1398 return ccm_compare_tag_to_yr(aes_dev, tag_size);
1399}
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev,
1418 struct scatterlist *sg,
1419 int sg_dma_count,
1420 struct ocs_dll_desc *dll_desc,
1421 size_t data_size, size_t data_offset)
1422{
1423 struct ocs_dma_linked_list *ll = NULL;
1424 struct scatterlist *sg_tmp;
1425 unsigned int tmp;
1426 int dma_nents;
1427 int i;
1428
1429 if (!dll_desc || !sg || !aes_dev)
1430 return -EINVAL;
1431
1432
1433 dll_desc->vaddr = NULL;
1434 dll_desc->dma_addr = DMA_MAPPING_ERROR;
1435 dll_desc->size = 0;
1436
1437 if (data_size == 0)
1438 return 0;
1439
1440
1441 while (data_offset >= sg_dma_len(sg)) {
1442 data_offset -= sg_dma_len(sg);
1443 sg_dma_count--;
1444 sg = sg_next(sg);
1445
1446 if (!sg || sg_dma_count == 0)
1447 return -EINVAL;
1448 }
1449
1450
1451 dma_nents = 0;
1452 tmp = 0;
1453 sg_tmp = sg;
1454 while (tmp < data_offset + data_size) {
1455
1456 if (!sg_tmp)
1457 return -EINVAL;
1458 tmp += sg_dma_len(sg_tmp);
1459 dma_nents++;
1460 sg_tmp = sg_next(sg_tmp);
1461 }
1462 if (dma_nents > sg_dma_count)
1463 return -EINVAL;
1464
1465
1466 dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents;
1467 dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size,
1468 &dll_desc->dma_addr, GFP_KERNEL);
1469 if (!dll_desc->vaddr)
1470 return -ENOMEM;
1471
1472
1473 ll = dll_desc->vaddr;
1474 for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) {
1475 ll[i].src_addr = sg_dma_address(sg) + data_offset;
1476 ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ?
1477 (sg_dma_len(sg) - data_offset) : data_size;
1478 data_offset = 0;
1479 data_size -= ll[i].src_len;
1480
1481 ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1));
1482 ll[i].ll_flags = 0;
1483 }
1484
1485 ll[i - 1].next = 0;
1486 ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE;
1487
1488 return 0;
1489}
1490