1
2
3
4
5
6
7
8#include <linux/clk.h>
9#include <linux/device.h>
10#include <linux/dma-mapping.h>
11#include <linux/dmapool.h>
12#include <linux/firmware.h>
13#include <linux/interrupt.h>
14#include <linux/module.h>
15#include <linux/of_platform.h>
16#include <linux/of_irq.h>
17#include <linux/platform_device.h>
18#include <linux/workqueue.h>
19
20#include <crypto/internal/aead.h>
21#include <crypto/internal/hash.h>
22#include <crypto/internal/skcipher.h>
23
24#include "safexcel.h"
25
26static u32 max_rings = EIP197_MAX_RINGS;
27module_param(max_rings, uint, 0644);
28MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");
29
30static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
31{
32 u32 val, htable_offset;
33 int i, cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc;
34
35 if (priv->version == EIP197B) {
36 cs_rc_max = EIP197B_CS_RC_MAX;
37 cs_ht_wc = EIP197B_CS_HT_WC;
38 cs_trc_rec_wc = EIP197B_CS_TRC_REC_WC;
39 cs_trc_lg_rec_wc = EIP197B_CS_TRC_LG_REC_WC;
40 } else {
41 cs_rc_max = EIP197D_CS_RC_MAX;
42 cs_ht_wc = EIP197D_CS_HT_WC;
43 cs_trc_rec_wc = EIP197D_CS_TRC_REC_WC;
44 cs_trc_lg_rec_wc = EIP197D_CS_TRC_LG_REC_WC;
45 }
46
47
48 val = readl(priv->base + EIP197_CS_RAM_CTRL);
49 val &= ~EIP197_TRC_ENABLE_MASK;
50 val |= EIP197_TRC_ENABLE_0;
51 writel(val, priv->base + EIP197_CS_RAM_CTRL);
52
53
54 writel(0, priv->base + EIP197_TRC_ECCCTRL);
55
56
57
58
59
60 val = readl(priv->base + EIP197_TRC_PARAMS);
61 val |= EIP197_TRC_PARAMS_SW_RESET;
62 val &= ~EIP197_TRC_PARAMS_DATA_ACCESS;
63 writel(val, priv->base + EIP197_TRC_PARAMS);
64
65
66 for (i = 0; i < cs_rc_max; i++) {
67 u32 val, offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
68
69 writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
70 EIP197_CS_RC_PREV(EIP197_RC_NULL),
71 priv->base + offset);
72
73 val = EIP197_CS_RC_NEXT(i+1) | EIP197_CS_RC_PREV(i-1);
74 if (i == 0)
75 val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
76 else if (i == cs_rc_max - 1)
77 val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
78 writel(val, priv->base + offset + sizeof(u32));
79 }
80
81
82 htable_offset = cs_rc_max * EIP197_CS_RC_SIZE;
83 for (i = 0; i < cs_ht_wc; i++)
84 writel(GENMASK(29, 0),
85 priv->base + EIP197_CLASSIFICATION_RAMS + htable_offset + i * sizeof(u32));
86
87
88 val = readl(priv->base + EIP197_CS_RAM_CTRL);
89 val &= ~EIP197_TRC_ENABLE_MASK;
90 writel(val, priv->base + EIP197_CS_RAM_CTRL);
91
92
93 val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |
94 EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1);
95 writel(val, priv->base + EIP197_TRC_FREECHAIN);
96
97
98 val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) |
99 EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max);
100 writel(val, priv->base + EIP197_TRC_PARAMS2);
101
102
103 val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) |
104 EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
105 EIP197_TRC_PARAMS_HTABLE_SZ(2);
106 writel(val, priv->base + EIP197_TRC_PARAMS);
107}
108
109static void eip197_write_firmware(struct safexcel_crypto_priv *priv,
110 const struct firmware *fw, int pe, u32 ctrl,
111 u32 prog_en)
112{
113 const u32 *data = (const u32 *)fw->data;
114 u32 val;
115 int i;
116
117
118 writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
119 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
120 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
121 EIP197_PE(priv) + ctrl);
122
123
124 writel(prog_en, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
125
126
127 for (i = 0; i < fw->size / sizeof(u32); i++)
128 writel(be32_to_cpu(data[i]),
129 priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32));
130
131
132 writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
133
134
135 val = readl(EIP197_PE(priv) + ctrl);
136 val &= ~EIP197_PE_ICE_x_CTRL_SW_RESET;
137 writel(val, EIP197_PE(priv) + ctrl);
138}
139
140static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
141{
142 const char *fw_name[] = {"ifpp.bin", "ipue.bin"};
143 const struct firmware *fw[FW_NB];
144 char fw_path[31], *dir = NULL;
145 int i, j, ret = 0, pe;
146 u32 val;
147
148 switch (priv->version) {
149 case EIP197B:
150 dir = "eip197b";
151 break;
152 case EIP197D:
153 dir = "eip197d";
154 break;
155 default:
156
157 return 0;
158 }
159
160 for (i = 0; i < FW_NB; i++) {
161 snprintf(fw_path, 31, "inside-secure/%s/%s", dir, fw_name[i]);
162 ret = request_firmware(&fw[i], fw_path, priv->dev);
163 if (ret) {
164 if (priv->version != EIP197B)
165 goto release_fw;
166
167
168
169
170 ret = request_firmware(&fw[i], fw_name[i], priv->dev);
171 if (ret) {
172 dev_err(priv->dev,
173 "Failed to request firmware %s (%d)\n",
174 fw_name[i], ret);
175 goto release_fw;
176 }
177 }
178 }
179
180 for (pe = 0; pe < priv->config.pes; pe++) {
181
182 val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
183 val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
184 EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
185 EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
186 EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
187 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
188
189 memset_io(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_RAM(pe), 0,
190 EIP197_NUM_OF_SCRATCH_BLOCKS * sizeof(u32));
191
192 eip197_write_firmware(priv, fw[FW_IFPP], pe,
193 EIP197_PE_ICE_FPP_CTRL(pe),
194 EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN);
195
196 eip197_write_firmware(priv, fw[FW_IPUE], pe,
197 EIP197_PE_ICE_PUE_CTRL(pe),
198 EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN);
199 }
200
201release_fw:
202 for (j = 0; j < i; j++)
203 release_firmware(fw[j]);
204
205 return ret;
206}
207
208static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
209{
210 u32 hdw, cd_size_rnd, val;
211 int i;
212
213 hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
214 hdw &= GENMASK(27, 25);
215 hdw >>= 25;
216
217 cd_size_rnd = (priv->config.cd_size + (BIT(hdw) - 1)) >> hdw;
218
219 for (i = 0; i < priv->config.rings; i++) {
220
221 writel(lower_32_bits(priv->ring[i].cdr.base_dma),
222 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
223 writel(upper_32_bits(priv->ring[i].cdr.base_dma),
224 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
225
226 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) |
227 priv->config.cd_size,
228 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
229 writel(((EIP197_FETCH_COUNT * (cd_size_rnd << hdw)) << 16) |
230 (EIP197_FETCH_COUNT * priv->config.cd_offset),
231 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
232
233
234 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
235 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
236 writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
237
238
239 writel(GENMASK(5, 0),
240 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
241 }
242
243 return 0;
244}
245
246static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
247{
248 u32 hdw, rd_size_rnd, val;
249 int i;
250
251 hdw = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
252 hdw &= GENMASK(27, 25);
253 hdw >>= 25;
254
255 rd_size_rnd = (priv->config.rd_size + (BIT(hdw) - 1)) >> hdw;
256
257 for (i = 0; i < priv->config.rings; i++) {
258
259 writel(lower_32_bits(priv->ring[i].rdr.base_dma),
260 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
261 writel(upper_32_bits(priv->ring[i].rdr.base_dma),
262 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
263
264 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) |
265 priv->config.rd_size,
266 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
267
268 writel(((EIP197_FETCH_COUNT * (rd_size_rnd << hdw)) << 16) |
269 (EIP197_FETCH_COUNT * priv->config.rd_offset),
270 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
271
272
273 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
274 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
275 val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF;
276 writel(val,
277 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
278
279
280 writel(GENMASK(7, 0),
281 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
282
283
284 val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
285 val |= EIP197_RDR_IRQ(i);
286 writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
287 }
288
289 return 0;
290}
291
292static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
293{
294 u32 version, val;
295 int i, ret, pe;
296
297
298 version = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_VERSION);
299 val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
300
301 if ((version & 0xffff) == EIP197_HIA_VERSION_BE)
302 val |= EIP197_MST_CTRL_BYTE_SWAP;
303 else if (((version >> 16) & 0xffff) == EIP197_HIA_VERSION_LE)
304 val |= (EIP197_MST_CTRL_NO_BYTE_SWAP >> 24);
305
306
307 if (priv->version == EIP197B || priv->version == EIP197D)
308 val |= EIP197_MST_CTRL_TX_MAX_CMD(5);
309
310 writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
311
312
313 writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
314 EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),
315 EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL);
316
317
318
319
320 writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL);
321
322
323 writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
324
325
326 for (pe = 0; pe < priv->config.pes; pe++) {
327
328
329
330 writel(EIP197_DxE_THR_CTRL_RESET_PE,
331 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
332
333 if (priv->version == EIP197B || priv->version == EIP197D) {
334
335 writel(EIP197_HIA_RA_PE_CTRL_RESET,
336 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
337 }
338
339
340 val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
341 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) |
342 EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
343 val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) |
344 EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
345 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
346 val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
347 writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe));
348
349
350 writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
351
352
353 writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
354 EIP197_PE_IN_xBUF_THRES_MAX(9),
355 EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe));
356 writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
357 EIP197_PE_IN_xBUF_THRES_MAX(7),
358 EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe));
359
360 if (priv->version == EIP197B || priv->version == EIP197D) {
361
362 writel(EIP197_HIA_RA_PE_CTRL_EN |
363 GENMASK(priv->config.rings - 1, 0),
364 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
365 }
366
367
368
369
370 writel(EIP197_DxE_THR_CTRL_RESET_PE,
371 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
372
373
374 while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) &
375 GENMASK(15, 12)) != GENMASK(15, 12))
376 ;
377
378
379 val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
380 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) |
381 EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8);
382 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
383 val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;
384
385
386
387 if (priv->version == EIP197B || priv->version == EIP197D)
388 val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
389 writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe));
390
391
392 writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
393
394
395 writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) |
396 EIP197_PE_OUT_DBUF_THRES_MAX(8),
397 EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe));
398
399
400
401
402 val = EIP197_FUNCTION_RSVD;
403 val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY;
404 val |= EIP197_PROTOCOL_ENCRYPT_HASH | EIP197_PROTOCOL_HASH_DECRYPT;
405 val |= EIP197_ALG_DES_ECB | EIP197_ALG_DES_CBC;
406 val |= EIP197_ALG_3DES_ECB | EIP197_ALG_3DES_CBC;
407 val |= EIP197_ALG_AES_ECB | EIP197_ALG_AES_CBC;
408 val |= EIP197_ALG_MD5 | EIP197_ALG_HMAC_MD5;
409 val |= EIP197_ALG_SHA1 | EIP197_ALG_HMAC_SHA1;
410 val |= EIP197_ALG_SHA2 | EIP197_ALG_HMAC_SHA2;
411 writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe));
412 }
413
414
415 for (i = 0; i < priv->config.rings; i++) {
416
417 writel(GENMASK(31, 0),
418 EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i));
419
420
421 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
422
423
424 writel(EIP197_xDR_PREP_CLR_COUNT,
425 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
426
427
428 writel(EIP197_xDR_PROC_CLR_COUNT,
429 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
430
431 writel(0,
432 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
433 writel(0,
434 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
435
436 writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2,
437 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
438 }
439
440
441 for (i = 0; i < priv->config.rings; i++) {
442
443 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
444
445
446 writel(EIP197_xDR_PREP_CLR_COUNT,
447 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
448
449
450 writel(EIP197_xDR_PROC_CLR_COUNT,
451 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
452
453 writel(0,
454 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
455 writel(0,
456 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
457
458
459 writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2,
460 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
461 }
462
463 for (pe = 0; pe < priv->config.pes; pe++) {
464
465 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
466 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
467
468
469 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
470 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
471 }
472
473
474 writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
475
476 if (priv->version == EIP197B || priv->version == EIP197D) {
477 eip197_trc_cache_init(priv);
478
479 ret = eip197_load_firmwares(priv);
480 if (ret)
481 return ret;
482 }
483
484 safexcel_hw_setup_cdesc_rings(priv);
485 safexcel_hw_setup_rdesc_rings(priv);
486
487 return 0;
488}
489
490
491static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv,
492 int ring)
493{
494 int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ);
495
496 if (!coal)
497 return;
498
499
500 writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
501 EIP197_HIA_RDR_THRESH_PROC_PKT(coal),
502 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH);
503}
504
505void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
506{
507 struct crypto_async_request *req, *backlog;
508 struct safexcel_context *ctx;
509 int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;
510
511
512
513
514 req = priv->ring[ring].req;
515 backlog = priv->ring[ring].backlog;
516 if (req)
517 goto handle_req;
518
519 while (true) {
520 spin_lock_bh(&priv->ring[ring].queue_lock);
521 backlog = crypto_get_backlog(&priv->ring[ring].queue);
522 req = crypto_dequeue_request(&priv->ring[ring].queue);
523 spin_unlock_bh(&priv->ring[ring].queue_lock);
524
525 if (!req) {
526 priv->ring[ring].req = NULL;
527 priv->ring[ring].backlog = NULL;
528 goto finalize;
529 }
530
531handle_req:
532 ctx = crypto_tfm_ctx(req->tfm);
533 ret = ctx->send(req, ring, &commands, &results);
534 if (ret)
535 goto request_failed;
536
537 if (backlog)
538 backlog->complete(backlog, -EINPROGRESS);
539
540
541
542
543
544 if (!commands && !results)
545 continue;
546
547 cdesc += commands;
548 rdesc += results;
549 nreq++;
550 }
551
552request_failed:
553
554
555
556 priv->ring[ring].req = req;
557 priv->ring[ring].backlog = backlog;
558
559finalize:
560 if (!nreq)
561 return;
562
563 spin_lock_bh(&priv->ring[ring].lock);
564
565 priv->ring[ring].requests += nreq;
566
567 if (!priv->ring[ring].busy) {
568 safexcel_try_push_requests(priv, ring);
569 priv->ring[ring].busy = true;
570 }
571
572 spin_unlock_bh(&priv->ring[ring].lock);
573
574
575 writel((rdesc * priv->config.rd_offset) << 2,
576 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
577
578
579 writel((cdesc * priv->config.cd_offset) << 2,
580 EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
581}
582
583inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
584 struct safexcel_result_desc *rdesc)
585{
586 if (likely(!rdesc->result_data.error_code))
587 return 0;
588
589 if (rdesc->result_data.error_code & 0x407f) {
590
591 dev_err(priv->dev,
592 "cipher: result: result descriptor error (%d)\n",
593 rdesc->result_data.error_code);
594 return -EIO;
595 } else if (rdesc->result_data.error_code == BIT(9)) {
596
597 return -EBADMSG;
598 }
599
600
601 return -EINVAL;
602}
603
604inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv,
605 int ring,
606 struct safexcel_result_desc *rdesc,
607 struct crypto_async_request *req)
608{
609 int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc);
610
611 priv->ring[ring].rdr_req[i] = req;
612}
613
614inline struct crypto_async_request *
615safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring)
616{
617 int i = safexcel_ring_first_rdr_index(priv, ring);
618
619 return priv->ring[ring].rdr_req[i];
620}
621
622void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
623{
624 struct safexcel_command_desc *cdesc;
625
626
627 do {
628 cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr);
629 if (IS_ERR(cdesc)) {
630 dev_err(priv->dev,
631 "Could not retrieve the command descriptor\n");
632 return;
633 }
634 } while (!cdesc->last_seg);
635}
636
637void safexcel_inv_complete(struct crypto_async_request *req, int error)
638{
639 struct safexcel_inv_result *result = req->data;
640
641 if (error == -EINPROGRESS)
642 return;
643
644 result->error = error;
645 complete(&result->completion);
646}
647
648int safexcel_invalidate_cache(struct crypto_async_request *async,
649 struct safexcel_crypto_priv *priv,
650 dma_addr_t ctxr_dma, int ring)
651{
652 struct safexcel_command_desc *cdesc;
653 struct safexcel_result_desc *rdesc;
654 int ret = 0;
655
656
657 cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma);
658 if (IS_ERR(cdesc))
659 return PTR_ERR(cdesc);
660
661 cdesc->control_data.type = EIP197_TYPE_EXTENDED;
662 cdesc->control_data.options = 0;
663 cdesc->control_data.refresh = 0;
664 cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR;
665
666
667 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
668
669 if (IS_ERR(rdesc)) {
670 ret = PTR_ERR(rdesc);
671 goto cdesc_rollback;
672 }
673
674 safexcel_rdr_req_set(priv, ring, rdesc, async);
675
676 return ret;
677
678cdesc_rollback:
679 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
680
681 return ret;
682}
683
684static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
685 int ring)
686{
687 struct crypto_async_request *req;
688 struct safexcel_context *ctx;
689 int ret, i, nreq, ndesc, tot_descs, handled = 0;
690 bool should_complete;
691
692handle_results:
693 tot_descs = 0;
694
695 nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
696 nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET;
697 nreq &= EIP197_xDR_PROC_xD_PKT_MASK;
698 if (!nreq)
699 goto requests_left;
700
701 for (i = 0; i < nreq; i++) {
702 req = safexcel_rdr_req_get(priv, ring);
703
704 ctx = crypto_tfm_ctx(req->tfm);
705 ndesc = ctx->handle_result(priv, ring, req,
706 &should_complete, &ret);
707 if (ndesc < 0) {
708 dev_err(priv->dev, "failed to handle result (%d)", ndesc);
709 goto acknowledge;
710 }
711
712 if (should_complete) {
713 local_bh_disable();
714 req->complete(req, ret);
715 local_bh_enable();
716 }
717
718 tot_descs += ndesc;
719 handled++;
720 }
721
722acknowledge:
723 if (i) {
724 writel(EIP197_xDR_PROC_xD_PKT(i) |
725 EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset),
726 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
727 }
728
729
730
731
732 if (nreq == EIP197_xDR_PROC_xD_PKT_MASK)
733 goto handle_results;
734
735requests_left:
736 spin_lock_bh(&priv->ring[ring].lock);
737
738 priv->ring[ring].requests -= handled;
739 safexcel_try_push_requests(priv, ring);
740
741 if (!priv->ring[ring].requests)
742 priv->ring[ring].busy = false;
743
744 spin_unlock_bh(&priv->ring[ring].lock);
745}
746
747static void safexcel_dequeue_work(struct work_struct *work)
748{
749 struct safexcel_work_data *data =
750 container_of(work, struct safexcel_work_data, work);
751
752 safexcel_dequeue(data->priv, data->ring);
753}
754
755struct safexcel_ring_irq_data {
756 struct safexcel_crypto_priv *priv;
757 int ring;
758};
759
760static irqreturn_t safexcel_irq_ring(int irq, void *data)
761{
762 struct safexcel_ring_irq_data *irq_data = data;
763 struct safexcel_crypto_priv *priv = irq_data->priv;
764 int ring = irq_data->ring, rc = IRQ_NONE;
765 u32 status, stat;
766
767 status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
768 if (!status)
769 return rc;
770
771
772 if (status & EIP197_RDR_IRQ(ring)) {
773 stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
774
775 if (unlikely(stat & EIP197_xDR_ERR)) {
776
777
778
779
780
781 dev_err(priv->dev, "RDR: fatal error.");
782 } else if (likely(stat & EIP197_xDR_THRESH)) {
783 rc = IRQ_WAKE_THREAD;
784 }
785
786
787 writel(stat & 0xff,
788 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
789 }
790
791
792 writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring));
793
794 return rc;
795}
796
797static irqreturn_t safexcel_irq_ring_thread(int irq, void *data)
798{
799 struct safexcel_ring_irq_data *irq_data = data;
800 struct safexcel_crypto_priv *priv = irq_data->priv;
801 int ring = irq_data->ring;
802
803 safexcel_handle_result_descriptor(priv, ring);
804
805 queue_work(priv->ring[ring].workqueue,
806 &priv->ring[ring].work_data.work);
807
808 return IRQ_HANDLED;
809}
810
811static int safexcel_request_ring_irq(struct platform_device *pdev, const char *name,
812 irq_handler_t handler,
813 irq_handler_t threaded_handler,
814 struct safexcel_ring_irq_data *ring_irq_priv)
815{
816 int ret, irq = platform_get_irq_byname(pdev, name);
817
818 if (irq < 0) {
819 dev_err(&pdev->dev, "unable to get IRQ '%s'\n", name);
820 return irq;
821 }
822
823 ret = devm_request_threaded_irq(&pdev->dev, irq, handler,
824 threaded_handler, IRQF_ONESHOT,
825 dev_name(&pdev->dev), ring_irq_priv);
826 if (ret) {
827 dev_err(&pdev->dev, "unable to request IRQ %d\n", irq);
828 return ret;
829 }
830
831 return irq;
832}
833
834static struct safexcel_alg_template *safexcel_algs[] = {
835 &safexcel_alg_ecb_des,
836 &safexcel_alg_cbc_des,
837 &safexcel_alg_ecb_des3_ede,
838 &safexcel_alg_cbc_des3_ede,
839 &safexcel_alg_ecb_aes,
840 &safexcel_alg_cbc_aes,
841 &safexcel_alg_md5,
842 &safexcel_alg_sha1,
843 &safexcel_alg_sha224,
844 &safexcel_alg_sha256,
845 &safexcel_alg_sha384,
846 &safexcel_alg_sha512,
847 &safexcel_alg_hmac_md5,
848 &safexcel_alg_hmac_sha1,
849 &safexcel_alg_hmac_sha224,
850 &safexcel_alg_hmac_sha256,
851 &safexcel_alg_hmac_sha384,
852 &safexcel_alg_hmac_sha512,
853 &safexcel_alg_authenc_hmac_sha1_cbc_aes,
854 &safexcel_alg_authenc_hmac_sha224_cbc_aes,
855 &safexcel_alg_authenc_hmac_sha256_cbc_aes,
856 &safexcel_alg_authenc_hmac_sha384_cbc_aes,
857 &safexcel_alg_authenc_hmac_sha512_cbc_aes,
858};
859
860static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
861{
862 int i, j, ret = 0;
863
864 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
865 safexcel_algs[i]->priv = priv;
866
867 if (!(safexcel_algs[i]->engines & priv->version))
868 continue;
869
870 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
871 ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
872 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
873 ret = crypto_register_aead(&safexcel_algs[i]->alg.aead);
874 else
875 ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);
876
877 if (ret)
878 goto fail;
879 }
880
881 return 0;
882
883fail:
884 for (j = 0; j < i; j++) {
885 if (!(safexcel_algs[j]->engines & priv->version))
886 continue;
887
888 if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
889 crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
890 else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD)
891 crypto_unregister_aead(&safexcel_algs[j]->alg.aead);
892 else
893 crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);
894 }
895
896 return ret;
897}
898
899static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
900{
901 int i;
902
903 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
904 if (!(safexcel_algs[i]->engines & priv->version))
905 continue;
906
907 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
908 crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
909 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
910 crypto_unregister_aead(&safexcel_algs[i]->alg.aead);
911 else
912 crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);
913 }
914}
915
916static void safexcel_configure(struct safexcel_crypto_priv *priv)
917{
918 u32 val, mask = 0;
919
920 val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
921
922
923 switch (priv->version) {
924 case EIP197B:
925 case EIP197D:
926 mask = EIP197_N_PES_MASK;
927 break;
928 default:
929 mask = EIP97_N_PES_MASK;
930 }
931 priv->config.pes = (val >> EIP197_N_PES_OFFSET) & mask;
932
933 val = (val & GENMASK(27, 25)) >> 25;
934 mask = BIT(val) - 1;
935
936 val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
937 priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings);
938
939 priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32));
940 priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;
941
942 priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32));
943 priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;
944}
945
946static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
947{
948 struct safexcel_register_offsets *offsets = &priv->offsets;
949
950 switch (priv->version) {
951 case EIP197B:
952 case EIP197D:
953 offsets->hia_aic = EIP197_HIA_AIC_BASE;
954 offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE;
955 offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE;
956 offsets->hia_aic_xdr = EIP197_HIA_AIC_xDR_BASE;
957 offsets->hia_dfe = EIP197_HIA_DFE_BASE;
958 offsets->hia_dfe_thr = EIP197_HIA_DFE_THR_BASE;
959 offsets->hia_dse = EIP197_HIA_DSE_BASE;
960 offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE;
961 offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE;
962 offsets->pe = EIP197_PE_BASE;
963 break;
964 case EIP97IES:
965 offsets->hia_aic = EIP97_HIA_AIC_BASE;
966 offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE;
967 offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE;
968 offsets->hia_aic_xdr = EIP97_HIA_AIC_xDR_BASE;
969 offsets->hia_dfe = EIP97_HIA_DFE_BASE;
970 offsets->hia_dfe_thr = EIP97_HIA_DFE_THR_BASE;
971 offsets->hia_dse = EIP97_HIA_DSE_BASE;
972 offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE;
973 offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE;
974 offsets->pe = EIP97_PE_BASE;
975 break;
976 }
977}
978
979static int safexcel_probe(struct platform_device *pdev)
980{
981 struct device *dev = &pdev->dev;
982 struct resource *res;
983 struct safexcel_crypto_priv *priv;
984 int i, ret;
985
986 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
987 if (!priv)
988 return -ENOMEM;
989
990 priv->dev = dev;
991 priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev);
992
993 if (priv->version == EIP197B || priv->version == EIP197D)
994 priv->flags |= EIP197_TRC_CACHE;
995
996 safexcel_init_register_offsets(priv);
997
998 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
999 priv->base = devm_ioremap_resource(dev, res);
1000 if (IS_ERR(priv->base)) {
1001 dev_err(dev, "failed to get resource\n");
1002 return PTR_ERR(priv->base);
1003 }
1004
1005 priv->clk = devm_clk_get(&pdev->dev, NULL);
1006 ret = PTR_ERR_OR_ZERO(priv->clk);
1007
1008 if (ret != -ENOENT) {
1009 if (ret)
1010 return ret;
1011
1012 ret = clk_prepare_enable(priv->clk);
1013 if (ret) {
1014 dev_err(dev, "unable to enable clk (%d)\n", ret);
1015 return ret;
1016 }
1017 }
1018
1019 priv->reg_clk = devm_clk_get(&pdev->dev, "reg");
1020 ret = PTR_ERR_OR_ZERO(priv->reg_clk);
1021
1022 if (ret != -ENOENT) {
1023 if (ret)
1024 goto err_core_clk;
1025
1026 ret = clk_prepare_enable(priv->reg_clk);
1027 if (ret) {
1028 dev_err(dev, "unable to enable reg clk (%d)\n", ret);
1029 goto err_core_clk;
1030 }
1031 }
1032
1033 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
1034 if (ret)
1035 goto err_reg_clk;
1036
1037 priv->context_pool = dmam_pool_create("safexcel-context", dev,
1038 sizeof(struct safexcel_context_record),
1039 1, 0);
1040 if (!priv->context_pool) {
1041 ret = -ENOMEM;
1042 goto err_reg_clk;
1043 }
1044
1045 safexcel_configure(priv);
1046
1047 priv->ring = devm_kcalloc(dev, priv->config.rings,
1048 sizeof(*priv->ring),
1049 GFP_KERNEL);
1050 if (!priv->ring) {
1051 ret = -ENOMEM;
1052 goto err_reg_clk;
1053 }
1054
1055 for (i = 0; i < priv->config.rings; i++) {
1056 char irq_name[6] = {0};
1057 char wq_name[9] = {0};
1058 int irq;
1059 struct safexcel_ring_irq_data *ring_irq;
1060
1061 ret = safexcel_init_ring_descriptors(priv,
1062 &priv->ring[i].cdr,
1063 &priv->ring[i].rdr);
1064 if (ret)
1065 goto err_reg_clk;
1066
1067 priv->ring[i].rdr_req = devm_kcalloc(dev,
1068 EIP197_DEFAULT_RING_SIZE,
1069 sizeof(priv->ring[i].rdr_req),
1070 GFP_KERNEL);
1071 if (!priv->ring[i].rdr_req) {
1072 ret = -ENOMEM;
1073 goto err_reg_clk;
1074 }
1075
1076 ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
1077 if (!ring_irq) {
1078 ret = -ENOMEM;
1079 goto err_reg_clk;
1080 }
1081
1082 ring_irq->priv = priv;
1083 ring_irq->ring = i;
1084
1085 snprintf(irq_name, 6, "ring%d", i);
1086 irq = safexcel_request_ring_irq(pdev, irq_name, safexcel_irq_ring,
1087 safexcel_irq_ring_thread,
1088 ring_irq);
1089 if (irq < 0) {
1090 ret = irq;
1091 goto err_reg_clk;
1092 }
1093
1094 priv->ring[i].work_data.priv = priv;
1095 priv->ring[i].work_data.ring = i;
1096 INIT_WORK(&priv->ring[i].work_data.work, safexcel_dequeue_work);
1097
1098 snprintf(wq_name, 9, "wq_ring%d", i);
1099 priv->ring[i].workqueue = create_singlethread_workqueue(wq_name);
1100 if (!priv->ring[i].workqueue) {
1101 ret = -ENOMEM;
1102 goto err_reg_clk;
1103 }
1104
1105 priv->ring[i].requests = 0;
1106 priv->ring[i].busy = false;
1107
1108 crypto_init_queue(&priv->ring[i].queue,
1109 EIP197_DEFAULT_RING_SIZE);
1110
1111 spin_lock_init(&priv->ring[i].lock);
1112 spin_lock_init(&priv->ring[i].queue_lock);
1113 }
1114
1115 platform_set_drvdata(pdev, priv);
1116 atomic_set(&priv->ring_used, 0);
1117
1118 ret = safexcel_hw_init(priv);
1119 if (ret) {
1120 dev_err(dev, "EIP h/w init failed (%d)\n", ret);
1121 goto err_reg_clk;
1122 }
1123
1124 ret = safexcel_register_algorithms(priv);
1125 if (ret) {
1126 dev_err(dev, "Failed to register algorithms (%d)\n", ret);
1127 goto err_reg_clk;
1128 }
1129
1130 return 0;
1131
1132err_reg_clk:
1133 clk_disable_unprepare(priv->reg_clk);
1134err_core_clk:
1135 clk_disable_unprepare(priv->clk);
1136 return ret;
1137}
1138
1139static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv)
1140{
1141 int i;
1142
1143 for (i = 0; i < priv->config.rings; i++) {
1144
1145 writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
1146 writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
1147
1148
1149 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
1150 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
1151
1152
1153 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
1154 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
1155 }
1156}
1157
1158static int safexcel_remove(struct platform_device *pdev)
1159{
1160 struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);
1161 int i;
1162
1163 safexcel_unregister_algorithms(priv);
1164 safexcel_hw_reset_rings(priv);
1165
1166 clk_disable_unprepare(priv->clk);
1167
1168 for (i = 0; i < priv->config.rings; i++)
1169 destroy_workqueue(priv->ring[i].workqueue);
1170
1171 return 0;
1172}
1173
1174static const struct of_device_id safexcel_of_match_table[] = {
1175 {
1176 .compatible = "inside-secure,safexcel-eip97ies",
1177 .data = (void *)EIP97IES,
1178 },
1179 {
1180 .compatible = "inside-secure,safexcel-eip197b",
1181 .data = (void *)EIP197B,
1182 },
1183 {
1184 .compatible = "inside-secure,safexcel-eip197d",
1185 .data = (void *)EIP197D,
1186 },
1187 {
1188
1189 .compatible = "inside-secure,safexcel-eip97",
1190 .data = (void *)EIP97IES,
1191 },
1192 {
1193
1194 .compatible = "inside-secure,safexcel-eip197",
1195 .data = (void *)EIP197B,
1196 },
1197 {},
1198};
1199
1200
1201static struct platform_driver crypto_safexcel = {
1202 .probe = safexcel_probe,
1203 .remove = safexcel_remove,
1204 .driver = {
1205 .name = "crypto-safexcel",
1206 .of_match_table = safexcel_of_match_table,
1207 },
1208};
1209module_platform_driver(crypto_safexcel);
1210
1211MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
1212MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");
1213MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");
1214MODULE_DESCRIPTION("Support for SafeXcel cryptographic engine EIP197");
1215MODULE_LICENSE("GPL v2");
1216
