1
2
3
4
5
6
7
8
9
10#include <linux/module.h>
11#include <linux/kernel.h>
12#include <linux/pci.h>
13#include <linux/kthread.h>
14#include <linux/debugfs.h>
15#include <linux/dma-mapping.h>
16#include <linux/interrupt.h>
17#include <linux/compiler.h>
18#include <linux/ccp.h>
19
20#include "ccp-dev.h"
21
22
23
24
25
26
27
28static u32 ccp_lsb_alloc(struct ccp_cmd_queue *cmd_q, unsigned int count)
29{
30 struct ccp_device *ccp;
31 int start;
32
33
34 if (cmd_q->lsb >= 0) {
35 start = (u32)bitmap_find_next_zero_area(cmd_q->lsbmap,
36 LSB_SIZE,
37 0, count, 0);
38 if (start < LSB_SIZE) {
39 bitmap_set(cmd_q->lsbmap, start, count);
40 return start + cmd_q->lsb * LSB_SIZE;
41 }
42 }
43
44
45 ccp = cmd_q->ccp;
46 for (;;) {
47 mutex_lock(&ccp->sb_mutex);
48
49 start = (u32)bitmap_find_next_zero_area(ccp->lsbmap,
50 MAX_LSB_CNT * LSB_SIZE,
51 0,
52 count, 0);
53 if (start <= MAX_LSB_CNT * LSB_SIZE) {
54 bitmap_set(ccp->lsbmap, start, count);
55
56 mutex_unlock(&ccp->sb_mutex);
57 return start;
58 }
59
60 ccp->sb_avail = 0;
61
62 mutex_unlock(&ccp->sb_mutex);
63
64
65 if (wait_event_interruptible(ccp->sb_queue, ccp->sb_avail))
66 return 0;
67 }
68}
69
70
71
72
73static void ccp_lsb_free(struct ccp_cmd_queue *cmd_q, unsigned int start,
74 unsigned int count)
75{
76 if (!start)
77 return;
78
79 if (cmd_q->lsb == start) {
80
81 bitmap_clear(cmd_q->lsbmap, start, count);
82 } else {
83
84 struct ccp_device *ccp = cmd_q->ccp;
85
86 mutex_lock(&ccp->sb_mutex);
87 bitmap_clear(ccp->lsbmap, start, count);
88 ccp->sb_avail = 1;
89 mutex_unlock(&ccp->sb_mutex);
90 wake_up_interruptible_all(&ccp->sb_queue);
91 }
92}
93
94
95union ccp_function {
96 struct {
97 u16 size:7;
98 u16 encrypt:1;
99 u16 mode:5;
100 u16 type:2;
101 } aes;
102 struct {
103 u16 size:7;
104 u16 encrypt:1;
105 u16 rsvd:5;
106 u16 type:2;
107 } aes_xts;
108 struct {
109 u16 size:7;
110 u16 encrypt:1;
111 u16 mode:5;
112 u16 type:2;
113 } des3;
114 struct {
115 u16 rsvd1:10;
116 u16 type:4;
117 u16 rsvd2:1;
118 } sha;
119 struct {
120 u16 mode:3;
121 u16 size:12;
122 } rsa;
123 struct {
124 u16 byteswap:2;
125 u16 bitwise:3;
126 u16 reflect:2;
127 u16 rsvd:8;
128 } pt;
129 struct {
130 u16 rsvd:13;
131 } zlib;
132 struct {
133 u16 size:10;
134 u16 type:2;
135 u16 mode:3;
136 } ecc;
137 u16 raw;
138};
139
140#define CCP_AES_SIZE(p) ((p)->aes.size)
141#define CCP_AES_ENCRYPT(p) ((p)->aes.encrypt)
142#define CCP_AES_MODE(p) ((p)->aes.mode)
143#define CCP_AES_TYPE(p) ((p)->aes.type)
144#define CCP_XTS_SIZE(p) ((p)->aes_xts.size)
145#define CCP_XTS_TYPE(p) ((p)->aes_xts.type)
146#define CCP_XTS_ENCRYPT(p) ((p)->aes_xts.encrypt)
147#define CCP_DES3_SIZE(p) ((p)->des3.size)
148#define CCP_DES3_ENCRYPT(p) ((p)->des3.encrypt)
149#define CCP_DES3_MODE(p) ((p)->des3.mode)
150#define CCP_DES3_TYPE(p) ((p)->des3.type)
151#define CCP_SHA_TYPE(p) ((p)->sha.type)
152#define CCP_RSA_SIZE(p) ((p)->rsa.size)
153#define CCP_PT_BYTESWAP(p) ((p)->pt.byteswap)
154#define CCP_PT_BITWISE(p) ((p)->pt.bitwise)
155#define CCP_ECC_MODE(p) ((p)->ecc.mode)
156#define CCP_ECC_AFFINE(p) ((p)->ecc.one)
157
158
159#define CCP5_CMD_DW0(p) ((p)->dw0)
160#define CCP5_CMD_SOC(p) (CCP5_CMD_DW0(p).soc)
161#define CCP5_CMD_IOC(p) (CCP5_CMD_DW0(p).ioc)
162#define CCP5_CMD_INIT(p) (CCP5_CMD_DW0(p).init)
163#define CCP5_CMD_EOM(p) (CCP5_CMD_DW0(p).eom)
164#define CCP5_CMD_FUNCTION(p) (CCP5_CMD_DW0(p).function)
165#define CCP5_CMD_ENGINE(p) (CCP5_CMD_DW0(p).engine)
166#define CCP5_CMD_PROT(p) (CCP5_CMD_DW0(p).prot)
167
168
169#define CCP5_CMD_DW1(p) ((p)->length)
170#define CCP5_CMD_LEN(p) (CCP5_CMD_DW1(p))
171
172
173#define CCP5_CMD_DW2(p) ((p)->src_lo)
174#define CCP5_CMD_SRC_LO(p) (CCP5_CMD_DW2(p))
175
176
177#define CCP5_CMD_DW3(p) ((p)->dw3)
178#define CCP5_CMD_SRC_MEM(p) ((p)->dw3.src_mem)
179#define CCP5_CMD_SRC_HI(p) ((p)->dw3.src_hi)
180#define CCP5_CMD_LSB_ID(p) ((p)->dw3.lsb_cxt_id)
181#define CCP5_CMD_FIX_SRC(p) ((p)->dw3.fixed)
182
183
184#define CCP5_CMD_DW4(p) ((p)->dw4)
185#define CCP5_CMD_DST_LO(p) (CCP5_CMD_DW4(p).dst_lo)
186#define CCP5_CMD_DW5(p) ((p)->dw5.fields.dst_hi)
187#define CCP5_CMD_DST_HI(p) (CCP5_CMD_DW5(p))
188#define CCP5_CMD_DST_MEM(p) ((p)->dw5.fields.dst_mem)
189#define CCP5_CMD_FIX_DST(p) ((p)->dw5.fields.fixed)
190#define CCP5_CMD_SHA_LO(p) ((p)->dw4.sha_len_lo)
191#define CCP5_CMD_SHA_HI(p) ((p)->dw5.sha_len_hi)
192
193
194#define CCP5_CMD_DW6(p) ((p)->key_lo)
195#define CCP5_CMD_KEY_LO(p) (CCP5_CMD_DW6(p))
196#define CCP5_CMD_DW7(p) ((p)->dw7)
197#define CCP5_CMD_KEY_HI(p) ((p)->dw7.key_hi)
198#define CCP5_CMD_KEY_MEM(p) ((p)->dw7.key_mem)
199
200static inline u32 low_address(unsigned long addr)
201{
202 return (u64)addr & 0x0ffffffff;
203}
204
205static inline u32 high_address(unsigned long addr)
206{
207 return ((u64)addr >> 32) & 0x00000ffff;
208}
209
210static unsigned int ccp5_get_free_slots(struct ccp_cmd_queue *cmd_q)
211{
212 unsigned int head_idx, n;
213 u32 head_lo, queue_start;
214
215 queue_start = low_address(cmd_q->qdma_tail);
216 head_lo = ioread32(cmd_q->reg_head_lo);
217 head_idx = (head_lo - queue_start) / sizeof(struct ccp5_desc);
218
219 n = head_idx + COMMANDS_PER_QUEUE - cmd_q->qidx - 1;
220
221 return n % COMMANDS_PER_QUEUE;
222}
223
224static int ccp5_do_cmd(struct ccp5_desc *desc,
225 struct ccp_cmd_queue *cmd_q)
226{
227 u32 *mP;
228 __le32 *dP;
229 u32 tail;
230 int i;
231 int ret = 0;
232
233 cmd_q->total_ops++;
234
235 if (CCP5_CMD_SOC(desc)) {
236 CCP5_CMD_IOC(desc) = 1;
237 CCP5_CMD_SOC(desc) = 0;
238 }
239 mutex_lock(&cmd_q->q_mutex);
240
241 mP = (u32 *) &cmd_q->qbase[cmd_q->qidx];
242 dP = (__le32 *) desc;
243 for (i = 0; i < 8; i++)
244 mP[i] = cpu_to_le32(dP[i]);
245
246 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
247
248
249 wmb();
250
251
252 tail = low_address(cmd_q->qdma_tail + cmd_q->qidx * Q_DESC_SIZE);
253 iowrite32(tail, cmd_q->reg_tail_lo);
254
255
256 iowrite32(cmd_q->qcontrol | CMD5_Q_RUN, cmd_q->reg_control);
257 mutex_unlock(&cmd_q->q_mutex);
258
259 if (CCP5_CMD_IOC(desc)) {
260
261 ret = wait_event_interruptible(cmd_q->int_queue,
262 cmd_q->int_rcvd);
263 if (ret || cmd_q->cmd_error) {
264
265
266
267 if (cmd_q->cmd_error)
268 ccp_log_error(cmd_q->ccp,
269 cmd_q->cmd_error);
270 iowrite32(tail, cmd_q->reg_head_lo);
271 if (!ret)
272 ret = -EIO;
273 }
274 cmd_q->int_rcvd = 0;
275 }
276
277 return ret;
278}
279
280static int ccp5_perform_aes(struct ccp_op *op)
281{
282 struct ccp5_desc desc;
283 union ccp_function function;
284 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
285
286 op->cmd_q->total_aes_ops++;
287
288
289 memset(&desc, 0, Q_DESC_SIZE);
290
291 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_AES;
292
293 CCP5_CMD_SOC(&desc) = op->soc;
294 CCP5_CMD_IOC(&desc) = 1;
295 CCP5_CMD_INIT(&desc) = op->init;
296 CCP5_CMD_EOM(&desc) = op->eom;
297 CCP5_CMD_PROT(&desc) = 0;
298
299 function.raw = 0;
300 CCP_AES_ENCRYPT(&function) = op->u.aes.action;
301 CCP_AES_MODE(&function) = op->u.aes.mode;
302 CCP_AES_TYPE(&function) = op->u.aes.type;
303 CCP_AES_SIZE(&function) = op->u.aes.size;
304
305 CCP5_CMD_FUNCTION(&desc) = function.raw;
306
307 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
308
309 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
310 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
311 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
312
313 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
314 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
315 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
316
317 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
318 CCP5_CMD_KEY_HI(&desc) = 0;
319 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
320 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
321
322 return ccp5_do_cmd(&desc, op->cmd_q);
323}
324
325static int ccp5_perform_xts_aes(struct ccp_op *op)
326{
327 struct ccp5_desc desc;
328 union ccp_function function;
329 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
330
331 op->cmd_q->total_xts_aes_ops++;
332
333
334 memset(&desc, 0, Q_DESC_SIZE);
335
336 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_XTS_AES_128;
337
338 CCP5_CMD_SOC(&desc) = op->soc;
339 CCP5_CMD_IOC(&desc) = 1;
340 CCP5_CMD_INIT(&desc) = op->init;
341 CCP5_CMD_EOM(&desc) = op->eom;
342 CCP5_CMD_PROT(&desc) = 0;
343
344 function.raw = 0;
345 CCP_XTS_TYPE(&function) = op->u.xts.type;
346 CCP_XTS_ENCRYPT(&function) = op->u.xts.action;
347 CCP_XTS_SIZE(&function) = op->u.xts.unit_size;
348 CCP5_CMD_FUNCTION(&desc) = function.raw;
349
350 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
351
352 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
353 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
354 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
355
356 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
357 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
358 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
359
360 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
361 CCP5_CMD_KEY_HI(&desc) = 0;
362 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
363 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
364
365 return ccp5_do_cmd(&desc, op->cmd_q);
366}
367
368static int ccp5_perform_sha(struct ccp_op *op)
369{
370 struct ccp5_desc desc;
371 union ccp_function function;
372
373 op->cmd_q->total_sha_ops++;
374
375
376 memset(&desc, 0, Q_DESC_SIZE);
377
378 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_SHA;
379
380 CCP5_CMD_SOC(&desc) = op->soc;
381 CCP5_CMD_IOC(&desc) = 1;
382 CCP5_CMD_INIT(&desc) = 1;
383 CCP5_CMD_EOM(&desc) = op->eom;
384 CCP5_CMD_PROT(&desc) = 0;
385
386 function.raw = 0;
387 CCP_SHA_TYPE(&function) = op->u.sha.type;
388 CCP5_CMD_FUNCTION(&desc) = function.raw;
389
390 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
391
392 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
393 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
394 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
395
396 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
397
398 if (op->eom) {
399 CCP5_CMD_SHA_LO(&desc) = lower_32_bits(op->u.sha.msg_bits);
400 CCP5_CMD_SHA_HI(&desc) = upper_32_bits(op->u.sha.msg_bits);
401 } else {
402 CCP5_CMD_SHA_LO(&desc) = 0;
403 CCP5_CMD_SHA_HI(&desc) = 0;
404 }
405
406 return ccp5_do_cmd(&desc, op->cmd_q);
407}
408
409static int ccp5_perform_des3(struct ccp_op *op)
410{
411 struct ccp5_desc desc;
412 union ccp_function function;
413 u32 key_addr = op->sb_key * LSB_ITEM_SIZE;
414
415 op->cmd_q->total_3des_ops++;
416
417
418 memset(&desc, 0, sizeof(struct ccp5_desc));
419
420 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_DES3;
421
422 CCP5_CMD_SOC(&desc) = op->soc;
423 CCP5_CMD_IOC(&desc) = 1;
424 CCP5_CMD_INIT(&desc) = op->init;
425 CCP5_CMD_EOM(&desc) = op->eom;
426 CCP5_CMD_PROT(&desc) = 0;
427
428 function.raw = 0;
429 CCP_DES3_ENCRYPT(&function) = op->u.des3.action;
430 CCP_DES3_MODE(&function) = op->u.des3.mode;
431 CCP_DES3_TYPE(&function) = op->u.des3.type;
432 CCP5_CMD_FUNCTION(&desc) = function.raw;
433
434 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
435
436 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
437 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
438 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
439
440 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
441 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
442 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
443
444 CCP5_CMD_KEY_LO(&desc) = lower_32_bits(key_addr);
445 CCP5_CMD_KEY_HI(&desc) = 0;
446 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SB;
447 CCP5_CMD_LSB_ID(&desc) = op->sb_ctx;
448
449 return ccp5_do_cmd(&desc, op->cmd_q);
450}
451
452static int ccp5_perform_rsa(struct ccp_op *op)
453{
454 struct ccp5_desc desc;
455 union ccp_function function;
456
457 op->cmd_q->total_rsa_ops++;
458
459
460 memset(&desc, 0, Q_DESC_SIZE);
461
462 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_RSA;
463
464 CCP5_CMD_SOC(&desc) = op->soc;
465 CCP5_CMD_IOC(&desc) = 1;
466 CCP5_CMD_INIT(&desc) = 0;
467 CCP5_CMD_EOM(&desc) = 1;
468 CCP5_CMD_PROT(&desc) = 0;
469
470 function.raw = 0;
471 CCP_RSA_SIZE(&function) = (op->u.rsa.mod_size + 7) >> 3;
472 CCP5_CMD_FUNCTION(&desc) = function.raw;
473
474 CCP5_CMD_LEN(&desc) = op->u.rsa.input_len;
475
476
477 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
478 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
479 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
480
481
482 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
483 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
484 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
485
486
487 CCP5_CMD_KEY_LO(&desc) = ccp_addr_lo(&op->exp.u.dma);
488 CCP5_CMD_KEY_HI(&desc) = ccp_addr_hi(&op->exp.u.dma);
489 CCP5_CMD_KEY_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
490
491 return ccp5_do_cmd(&desc, op->cmd_q);
492}
493
494static int ccp5_perform_passthru(struct ccp_op *op)
495{
496 struct ccp5_desc desc;
497 union ccp_function function;
498 struct ccp_dma_info *saddr = &op->src.u.dma;
499 struct ccp_dma_info *daddr = &op->dst.u.dma;
500
501
502 op->cmd_q->total_pt_ops++;
503
504 memset(&desc, 0, Q_DESC_SIZE);
505
506 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_PASSTHRU;
507
508 CCP5_CMD_SOC(&desc) = 0;
509 CCP5_CMD_IOC(&desc) = 1;
510 CCP5_CMD_INIT(&desc) = 0;
511 CCP5_CMD_EOM(&desc) = op->eom;
512 CCP5_CMD_PROT(&desc) = 0;
513
514 function.raw = 0;
515 CCP_PT_BYTESWAP(&function) = op->u.passthru.byte_swap;
516 CCP_PT_BITWISE(&function) = op->u.passthru.bit_mod;
517 CCP5_CMD_FUNCTION(&desc) = function.raw;
518
519
520 if (op->src.type == CCP_MEMTYPE_SYSTEM)
521 CCP5_CMD_LEN(&desc) = saddr->length;
522 else
523 CCP5_CMD_LEN(&desc) = daddr->length;
524
525 if (op->src.type == CCP_MEMTYPE_SYSTEM) {
526 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
527 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
528 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
529
530 if (op->u.passthru.bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
531 CCP5_CMD_LSB_ID(&desc) = op->sb_key;
532 } else {
533 u32 key_addr = op->src.u.sb * CCP_SB_BYTES;
534
535 CCP5_CMD_SRC_LO(&desc) = lower_32_bits(key_addr);
536 CCP5_CMD_SRC_HI(&desc) = 0;
537 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SB;
538 }
539
540 if (op->dst.type == CCP_MEMTYPE_SYSTEM) {
541 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
542 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
543 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
544 } else {
545 u32 key_addr = op->dst.u.sb * CCP_SB_BYTES;
546
547 CCP5_CMD_DST_LO(&desc) = lower_32_bits(key_addr);
548 CCP5_CMD_DST_HI(&desc) = 0;
549 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SB;
550 }
551
552 return ccp5_do_cmd(&desc, op->cmd_q);
553}
554
555static int ccp5_perform_ecc(struct ccp_op *op)
556{
557 struct ccp5_desc desc;
558 union ccp_function function;
559
560 op->cmd_q->total_ecc_ops++;
561
562
563 memset(&desc, 0, Q_DESC_SIZE);
564
565 CCP5_CMD_ENGINE(&desc) = CCP_ENGINE_ECC;
566
567 CCP5_CMD_SOC(&desc) = 0;
568 CCP5_CMD_IOC(&desc) = 1;
569 CCP5_CMD_INIT(&desc) = 0;
570 CCP5_CMD_EOM(&desc) = 1;
571 CCP5_CMD_PROT(&desc) = 0;
572
573 function.raw = 0;
574 function.ecc.mode = op->u.ecc.function;
575 CCP5_CMD_FUNCTION(&desc) = function.raw;
576
577 CCP5_CMD_LEN(&desc) = op->src.u.dma.length;
578
579 CCP5_CMD_SRC_LO(&desc) = ccp_addr_lo(&op->src.u.dma);
580 CCP5_CMD_SRC_HI(&desc) = ccp_addr_hi(&op->src.u.dma);
581 CCP5_CMD_SRC_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
582
583 CCP5_CMD_DST_LO(&desc) = ccp_addr_lo(&op->dst.u.dma);
584 CCP5_CMD_DST_HI(&desc) = ccp_addr_hi(&op->dst.u.dma);
585 CCP5_CMD_DST_MEM(&desc) = CCP_MEMTYPE_SYSTEM;
586
587 return ccp5_do_cmd(&desc, op->cmd_q);
588}
589
590static int ccp_find_lsb_regions(struct ccp_cmd_queue *cmd_q, u64 status)
591{
592 int q_mask = 1 << cmd_q->id;
593 int queues = 0;
594 int j;
595
596
597
598
599 for (j = 1; j < MAX_LSB_CNT; j++) {
600 if (status & q_mask)
601 bitmap_set(cmd_q->lsbmask, j, 1);
602 status >>= LSB_REGION_WIDTH;
603 }
604 queues = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
605 dev_dbg(cmd_q->ccp->dev, "Queue %d can access %d LSB regions\n",
606 cmd_q->id, queues);
607
608 return queues ? 0 : -EINVAL;
609}
610
611static int ccp_find_and_assign_lsb_to_q(struct ccp_device *ccp,
612 int lsb_cnt, int n_lsbs,
613 unsigned long *lsb_pub)
614{
615 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
616 int bitno;
617 int qlsb_wgt;
618 int i;
619
620
621
622
623
624
625
626
627
628
629
630 for (i = 0; i < ccp->cmd_q_count; i++) {
631 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
632
633 qlsb_wgt = bitmap_weight(cmd_q->lsbmask, MAX_LSB_CNT);
634
635 if (qlsb_wgt == lsb_cnt) {
636 bitmap_copy(qlsb, cmd_q->lsbmask, MAX_LSB_CNT);
637
638 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
639 while (bitno < MAX_LSB_CNT) {
640 if (test_bit(bitno, lsb_pub)) {
641
642
643
644 cmd_q->lsb = bitno;
645 bitmap_clear(lsb_pub, bitno, 1);
646 dev_dbg(ccp->dev,
647 "Queue %d gets LSB %d\n",
648 i, bitno);
649 break;
650 }
651 bitmap_clear(qlsb, bitno, 1);
652 bitno = find_first_bit(qlsb, MAX_LSB_CNT);
653 }
654 if (bitno >= MAX_LSB_CNT)
655 return -EINVAL;
656 n_lsbs--;
657 }
658 }
659 return n_lsbs;
660}
661
662
663
664
665
666
667
668static int ccp_assign_lsbs(struct ccp_device *ccp)
669{
670 DECLARE_BITMAP(lsb_pub, MAX_LSB_CNT);
671 DECLARE_BITMAP(qlsb, MAX_LSB_CNT);
672 int n_lsbs = 0;
673 int bitno;
674 int i, lsb_cnt;
675 int rc = 0;
676
677 bitmap_zero(lsb_pub, MAX_LSB_CNT);
678
679
680 for (i = 0; i < ccp->cmd_q_count; i++)
681 bitmap_or(lsb_pub,
682 lsb_pub, ccp->cmd_q[i].lsbmask,
683 MAX_LSB_CNT);
684
685 n_lsbs = bitmap_weight(lsb_pub, MAX_LSB_CNT);
686
687 if (n_lsbs >= ccp->cmd_q_count) {
688
689
690
691
692
693
694 for (lsb_cnt = 1;
695 n_lsbs && (lsb_cnt <= MAX_LSB_CNT);
696 lsb_cnt++) {
697 rc = ccp_find_and_assign_lsb_to_q(ccp, lsb_cnt, n_lsbs,
698 lsb_pub);
699 if (rc < 0)
700 return -EINVAL;
701 n_lsbs = rc;
702 }
703 }
704
705 rc = 0;
706
707
708
709
710
711 bitmap_copy(qlsb, lsb_pub, MAX_LSB_CNT);
712
713 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
714 while (bitno < MAX_LSB_CNT) {
715 bitmap_set(ccp->lsbmap, bitno * LSB_SIZE, LSB_SIZE);
716 bitmap_set(qlsb, bitno, 1);
717 bitno = find_first_zero_bit(qlsb, MAX_LSB_CNT);
718 }
719
720 return rc;
721}
722
723static void ccp5_disable_queue_interrupts(struct ccp_device *ccp)
724{
725 unsigned int i;
726
727 for (i = 0; i < ccp->cmd_q_count; i++)
728 iowrite32(0x0, ccp->cmd_q[i].reg_int_enable);
729}
730
731static void ccp5_enable_queue_interrupts(struct ccp_device *ccp)
732{
733 unsigned int i;
734
735 for (i = 0; i < ccp->cmd_q_count; i++)
736 iowrite32(SUPPORTED_INTERRUPTS, ccp->cmd_q[i].reg_int_enable);
737}
738
739static void ccp5_irq_bh(unsigned long data)
740{
741 struct ccp_device *ccp = (struct ccp_device *)data;
742 u32 status;
743 unsigned int i;
744
745 for (i = 0; i < ccp->cmd_q_count; i++) {
746 struct ccp_cmd_queue *cmd_q = &ccp->cmd_q[i];
747
748 status = ioread32(cmd_q->reg_interrupt_status);
749
750 if (status) {
751 cmd_q->int_status = status;
752 cmd_q->q_status = ioread32(cmd_q->reg_status);
753 cmd_q->q_int_status = ioread32(cmd_q->reg_int_status);
754
755
756 if ((status & INT_ERROR) && !cmd_q->cmd_error)
757 cmd_q->cmd_error = CMD_Q_ERROR(cmd_q->q_status);
758
759 cmd_q->int_rcvd = 1;
760
761
762 iowrite32(status, cmd_q->reg_interrupt_status);
763 wake_up_interruptible(&cmd_q->int_queue);
764 }
765 }
766 ccp5_enable_queue_interrupts(ccp);
767}
768
769static irqreturn_t ccp5_irq_handler(int irq, void *data)
770{
771 struct ccp_device *ccp = (struct ccp_device *)data;
772
773 ccp5_disable_queue_interrupts(ccp);
774 ccp->total_interrupts++;
775 if (ccp->use_tasklet)
776 tasklet_schedule(&ccp->irq_tasklet);
777 else
778 ccp5_irq_bh((unsigned long)ccp);
779 return IRQ_HANDLED;
780}
781
782static int ccp5_init(struct ccp_device *ccp)
783{
784 struct device *dev = ccp->dev;
785 struct ccp_cmd_queue *cmd_q;
786 struct dma_pool *dma_pool;
787 char dma_pool_name[MAX_DMAPOOL_NAME_LEN];
788 unsigned int qmr, i;
789 u64 status;
790 u32 status_lo, status_hi;
791 int ret;
792
793
794 qmr = ioread32(ccp->io_regs + Q_MASK_REG);
795 for (i = 0; i < MAX_HW_QUEUES; i++) {
796
797 if (!(qmr & (1 << i)))
798 continue;
799
800
801 snprintf(dma_pool_name, sizeof(dma_pool_name), "%s_q%d",
802 ccp->name, i);
803 dma_pool = dma_pool_create(dma_pool_name, dev,
804 CCP_DMAPOOL_MAX_SIZE,
805 CCP_DMAPOOL_ALIGN, 0);
806 if (!dma_pool) {
807 dev_err(dev, "unable to allocate dma pool\n");
808 ret = -ENOMEM;
809 }
810
811 cmd_q = &ccp->cmd_q[ccp->cmd_q_count];
812 ccp->cmd_q_count++;
813
814 cmd_q->ccp = ccp;
815 cmd_q->id = i;
816 cmd_q->dma_pool = dma_pool;
817 mutex_init(&cmd_q->q_mutex);
818
819
820 BUILD_BUG_ON(COMMANDS_PER_QUEUE > 128);
821 cmd_q->qsize = Q_SIZE(Q_DESC_SIZE);
822 cmd_q->qbase = dma_alloc_coherent(dev, cmd_q->qsize,
823 &cmd_q->qbase_dma,
824 GFP_KERNEL);
825 if (!cmd_q->qbase) {
826 dev_err(dev, "unable to allocate command queue\n");
827 ret = -ENOMEM;
828 goto e_pool;
829 }
830
831 cmd_q->qidx = 0;
832
833
834
835 cmd_q->reg_control = ccp->io_regs +
836 CMD5_Q_STATUS_INCR * (i + 1);
837 cmd_q->reg_tail_lo = cmd_q->reg_control + CMD5_Q_TAIL_LO_BASE;
838 cmd_q->reg_head_lo = cmd_q->reg_control + CMD5_Q_HEAD_LO_BASE;
839 cmd_q->reg_int_enable = cmd_q->reg_control +
840 CMD5_Q_INT_ENABLE_BASE;
841 cmd_q->reg_interrupt_status = cmd_q->reg_control +
842 CMD5_Q_INTERRUPT_STATUS_BASE;
843 cmd_q->reg_status = cmd_q->reg_control + CMD5_Q_STATUS_BASE;
844 cmd_q->reg_int_status = cmd_q->reg_control +
845 CMD5_Q_INT_STATUS_BASE;
846 cmd_q->reg_dma_status = cmd_q->reg_control +
847 CMD5_Q_DMA_STATUS_BASE;
848 cmd_q->reg_dma_read_status = cmd_q->reg_control +
849 CMD5_Q_DMA_READ_STATUS_BASE;
850 cmd_q->reg_dma_write_status = cmd_q->reg_control +
851 CMD5_Q_DMA_WRITE_STATUS_BASE;
852
853 init_waitqueue_head(&cmd_q->int_queue);
854
855 dev_dbg(dev, "queue #%u available\n", i);
856 }
857
858 if (ccp->cmd_q_count == 0) {
859 dev_notice(dev, "no command queues available\n");
860 ret = -EIO;
861 goto e_pool;
862 }
863
864
865 ccp5_disable_queue_interrupts(ccp);
866 for (i = 0; i < ccp->cmd_q_count; i++) {
867 cmd_q = &ccp->cmd_q[i];
868
869 cmd_q->qcontrol = 0;
870 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
871
872 ioread32(cmd_q->reg_int_status);
873 ioread32(cmd_q->reg_status);
874
875
876 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
877 }
878
879 dev_dbg(dev, "Requesting an IRQ...\n");
880
881 ret = sp_request_ccp_irq(ccp->sp, ccp5_irq_handler, ccp->name, ccp);
882 if (ret) {
883 dev_err(dev, "unable to allocate an IRQ\n");
884 goto e_pool;
885 }
886
887 if (ccp->use_tasklet)
888 tasklet_init(&ccp->irq_tasklet, ccp5_irq_bh,
889 (unsigned long)ccp);
890
891 dev_dbg(dev, "Loading LSB map...\n");
892
893 status_lo = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
894 status_hi = ioread32(ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
895 iowrite32(status_lo, ccp->io_regs + LSB_PUBLIC_MASK_LO_OFFSET);
896 iowrite32(status_hi, ccp->io_regs + LSB_PUBLIC_MASK_HI_OFFSET);
897 status = ((u64)status_hi<<30) | (u64)status_lo;
898
899 dev_dbg(dev, "Configuring virtual queues...\n");
900
901 for (i = 0; i < ccp->cmd_q_count; i++) {
902 u32 dma_addr_lo;
903 u32 dma_addr_hi;
904
905 cmd_q = &ccp->cmd_q[i];
906
907 cmd_q->qcontrol &= ~(CMD5_Q_SIZE << CMD5_Q_SHIFT);
908 cmd_q->qcontrol |= QUEUE_SIZE_VAL << CMD5_Q_SHIFT;
909
910 cmd_q->qdma_tail = cmd_q->qbase_dma;
911 dma_addr_lo = low_address(cmd_q->qdma_tail);
912 iowrite32((u32)dma_addr_lo, cmd_q->reg_tail_lo);
913 iowrite32((u32)dma_addr_lo, cmd_q->reg_head_lo);
914
915 dma_addr_hi = high_address(cmd_q->qdma_tail);
916 cmd_q->qcontrol |= (dma_addr_hi << 16);
917 iowrite32(cmd_q->qcontrol, cmd_q->reg_control);
918
919
920 ccp_find_lsb_regions(cmd_q, status);
921 cmd_q->lsb = -1;
922 }
923
924 dev_dbg(dev, "Assigning LSBs...\n");
925 ret = ccp_assign_lsbs(ccp);
926 if (ret) {
927 dev_err(dev, "Unable to assign LSBs (%d)\n", ret);
928 goto e_irq;
929 }
930
931
932 for (i = 0; i < ccp->cmd_q_count; i++) {
933 ccp->cmd_q[i].sb_key = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
934 ccp->cmd_q[i].sb_ctx = ccp_lsb_alloc(&ccp->cmd_q[i], 2);
935 }
936
937 dev_dbg(dev, "Starting threads...\n");
938
939 for (i = 0; i < ccp->cmd_q_count; i++) {
940 struct task_struct *kthread;
941
942 cmd_q = &ccp->cmd_q[i];
943
944 kthread = kthread_create(ccp_cmd_queue_thread, cmd_q,
945 "%s-q%u", ccp->name, cmd_q->id);
946 if (IS_ERR(kthread)) {
947 dev_err(dev, "error creating queue thread (%ld)\n",
948 PTR_ERR(kthread));
949 ret = PTR_ERR(kthread);
950 goto e_kthread;
951 }
952
953 cmd_q->kthread = kthread;
954 wake_up_process(kthread);
955 }
956
957 dev_dbg(dev, "Enabling interrupts...\n");
958 ccp5_enable_queue_interrupts(ccp);
959
960 dev_dbg(dev, "Registering device...\n");
961
962 ccp_add_device(ccp);
963
964 ret = ccp_register_rng(ccp);
965 if (ret)
966 goto e_kthread;
967
968
969 ret = ccp_dmaengine_register(ccp);
970 if (ret)
971 goto e_hwrng;
972
973
974 ccp5_debugfs_setup(ccp);
975
976 return 0;
977
978e_hwrng:
979 ccp_unregister_rng(ccp);
980
981e_kthread:
982 for (i = 0; i < ccp->cmd_q_count; i++)
983 if (ccp->cmd_q[i].kthread)
984 kthread_stop(ccp->cmd_q[i].kthread);
985
986e_irq:
987 sp_free_ccp_irq(ccp->sp, ccp);
988
989e_pool:
990 for (i = 0; i < ccp->cmd_q_count; i++)
991 dma_pool_destroy(ccp->cmd_q[i].dma_pool);
992
993 return ret;
994}
995
996static void ccp5_destroy(struct ccp_device *ccp)
997{
998 struct device *dev = ccp->dev;
999 struct ccp_cmd_queue *cmd_q;
1000 struct ccp_cmd *cmd;
1001 unsigned int i;
1002
1003
1004 ccp_dmaengine_unregister(ccp);
1005
1006
1007 ccp_unregister_rng(ccp);
1008
1009
1010 ccp_del_device(ccp);
1011
1012
1013
1014
1015 if (ccp_present())
1016 ccp5_debugfs_destroy();
1017
1018
1019 ccp5_disable_queue_interrupts(ccp);
1020 for (i = 0; i < ccp->cmd_q_count; i++) {
1021 cmd_q = &ccp->cmd_q[i];
1022
1023
1024 iowrite32(cmd_q->qcontrol & ~CMD5_Q_RUN, cmd_q->reg_control);
1025
1026
1027 iowrite32(SUPPORTED_INTERRUPTS, cmd_q->reg_interrupt_status);
1028 ioread32(cmd_q->reg_int_status);
1029 ioread32(cmd_q->reg_status);
1030 }
1031
1032
1033 for (i = 0; i < ccp->cmd_q_count; i++)
1034 if (ccp->cmd_q[i].kthread)
1035 kthread_stop(ccp->cmd_q[i].kthread);
1036
1037 sp_free_ccp_irq(ccp->sp, ccp);
1038
1039 for (i = 0; i < ccp->cmd_q_count; i++) {
1040 cmd_q = &ccp->cmd_q[i];
1041 dma_free_coherent(dev, cmd_q->qsize, cmd_q->qbase,
1042 cmd_q->qbase_dma);
1043 }
1044
1045
1046 while (!list_empty(&ccp->cmd)) {
1047
1048 cmd = list_first_entry(&ccp->cmd, struct ccp_cmd, entry);
1049 list_del(&cmd->entry);
1050 cmd->callback(cmd->data, -ENODEV);
1051 }
1052 while (!list_empty(&ccp->backlog)) {
1053
1054 cmd = list_first_entry(&ccp->backlog, struct ccp_cmd, entry);
1055 list_del(&cmd->entry);
1056 cmd->callback(cmd->data, -ENODEV);
1057 }
1058}
1059
1060static void ccp5_config(struct ccp_device *ccp)
1061{
1062
1063 iowrite32(0x0, ccp->io_regs + CMD5_REQID_CONFIG_OFFSET);
1064}
1065
1066static void ccp5other_config(struct ccp_device *ccp)
1067{
1068 int i;
1069 u32 rnd;
1070
1071
1072
1073 iowrite32(0x00012D57, ccp->io_regs + CMD5_TRNG_CTL_OFFSET);
1074 iowrite32(0x00000003, ccp->io_regs + CMD5_CONFIG_0_OFFSET);
1075 for (i = 0; i < 12; i++) {
1076 rnd = ioread32(ccp->io_regs + TRNG_OUT_REG);
1077 iowrite32(rnd, ccp->io_regs + CMD5_AES_MASK_OFFSET);
1078 }
1079
1080 iowrite32(0x0000001F, ccp->io_regs + CMD5_QUEUE_MASK_OFFSET);
1081 iowrite32(0x00005B6D, ccp->io_regs + CMD5_QUEUE_PRIO_OFFSET);
1082 iowrite32(0x00000000, ccp->io_regs + CMD5_CMD_TIMEOUT_OFFSET);
1083
1084 iowrite32(0x3FFFFFFF, ccp->io_regs + LSB_PRIVATE_MASK_LO_OFFSET);
1085 iowrite32(0x000003FF, ccp->io_regs + LSB_PRIVATE_MASK_HI_OFFSET);
1086
1087 iowrite32(0x00108823, ccp->io_regs + CMD5_CLK_GATE_CTL_OFFSET);
1088
1089 ccp5_config(ccp);
1090}
1091
1092
1093static const struct ccp_actions ccp5_actions = {
1094 .aes = ccp5_perform_aes,
1095 .xts_aes = ccp5_perform_xts_aes,
1096 .sha = ccp5_perform_sha,
1097 .des3 = ccp5_perform_des3,
1098 .rsa = ccp5_perform_rsa,
1099 .passthru = ccp5_perform_passthru,
1100 .ecc = ccp5_perform_ecc,
1101 .sballoc = ccp_lsb_alloc,
1102 .sbfree = ccp_lsb_free,
1103 .init = ccp5_init,
1104 .destroy = ccp5_destroy,
1105 .get_free_slots = ccp5_get_free_slots,
1106};
1107
1108const struct ccp_vdata ccpv5a = {
1109 .version = CCP_VERSION(5, 0),
1110 .setup = ccp5_config,
1111 .perform = &ccp5_actions,
1112 .offset = 0x0,
1113 .rsamax = CCP5_RSA_MAX_WIDTH,
1114};
1115
1116const struct ccp_vdata ccpv5b = {
1117 .version = CCP_VERSION(5, 0),
1118 .dma_chan_attr = DMA_PRIVATE,
1119 .setup = ccp5other_config,
1120 .perform = &ccp5_actions,
1121 .offset = 0x0,
1122 .rsamax = CCP5_RSA_MAX_WIDTH,
1123};
1124