1
2
3
4
5
6
7
8#include <linux/clk.h>
9#include <linux/device.h>
10#include <linux/dma-mapping.h>
11#include <linux/dmapool.h>
12#include <linux/firmware.h>
13#include <linux/interrupt.h>
14#include <linux/module.h>
15#include <linux/of_platform.h>
16#include <linux/of_irq.h>
17#include <linux/pci.h>
18#include <linux/platform_device.h>
19#include <linux/workqueue.h>
20
21#include <crypto/internal/aead.h>
22#include <crypto/internal/hash.h>
23#include <crypto/internal/skcipher.h>
24
25#include "safexcel.h"
26
27static u32 max_rings = EIP197_MAX_RINGS;
28module_param(max_rings, uint, 0644);
29MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");
30
31static void eip197_trc_cache_setupvirt(struct safexcel_crypto_priv *priv)
32{
33 int i;
34
35
36
37
38
39
40
41
42
43
44 for (i = 0; i < 4; i++)
45 writel(0, priv->base + EIP197_FLUE_IFC_LUT(i));
46
47
48
49
50
51 for (i = 0; i < priv->config.rings; i++) {
52 writel(0, priv->base + EIP197_FLUE_CACHEBASE_LO(i));
53 writel(0, priv->base + EIP197_FLUE_CACHEBASE_HI(i));
54 writel(EIP197_FLUE_CONFIG_MAGIC,
55 priv->base + EIP197_FLUE_CONFIG(i));
56 }
57 writel(0, priv->base + EIP197_FLUE_OFFSETS);
58 writel(0, priv->base + EIP197_FLUE_ARC4_OFFSET);
59}
60
61static void eip197_trc_cache_banksel(struct safexcel_crypto_priv *priv,
62 u32 addrmid, int *actbank)
63{
64 u32 val;
65 int curbank;
66
67 curbank = addrmid >> 16;
68 if (curbank != *actbank) {
69 val = readl(priv->base + EIP197_CS_RAM_CTRL);
70 val = (val & ~EIP197_CS_BANKSEL_MASK) |
71 (curbank << EIP197_CS_BANKSEL_OFS);
72 writel(val, priv->base + EIP197_CS_RAM_CTRL);
73 *actbank = curbank;
74 }
75}
76
77static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv,
78 int maxbanks, u32 probemask, u32 stride)
79{
80 u32 val, addrhi, addrlo, addrmid, addralias, delta, marker;
81 int actbank;
82
83
84
85
86
87 addrhi = 1 << (16 + maxbanks);
88 addrlo = 0;
89 actbank = min(maxbanks - 1, 0);
90 while ((addrhi - addrlo) > stride) {
91
92 addrmid = (addrhi + addrlo) >> 1;
93 marker = (addrmid ^ 0xabadbabe) & probemask;
94 eip197_trc_cache_banksel(priv, addrmid, &actbank);
95 writel(marker,
96 priv->base + EIP197_CLASSIFICATION_RAMS +
97 (addrmid & 0xffff));
98
99
100 delta = 1 << __fls(addrmid);
101 while (delta >= stride) {
102 addralias = addrmid - delta;
103 eip197_trc_cache_banksel(priv, addralias, &actbank);
104 writel(~marker,
105 priv->base + EIP197_CLASSIFICATION_RAMS +
106 (addralias & 0xffff));
107 delta >>= 1;
108 }
109
110
111 eip197_trc_cache_banksel(priv, addrmid, &actbank);
112 val = readl(priv->base + EIP197_CLASSIFICATION_RAMS +
113 (addrmid & 0xffff));
114
115 if ((val & probemask) == marker)
116
117 addrlo = addrmid;
118 else
119
120 addrhi = addrmid;
121 }
122 return addrhi;
123}
124
125static void eip197_trc_cache_clear(struct safexcel_crypto_priv *priv,
126 int cs_rc_max, int cs_ht_wc)
127{
128 int i;
129 u32 htable_offset, val, offset;
130
131
132 for (i = 0; i < cs_rc_max; i++) {
133 offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
134
135 writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
136 EIP197_CS_RC_PREV(EIP197_RC_NULL),
137 priv->base + offset);
138
139 val = EIP197_CS_RC_NEXT(i + 1) | EIP197_CS_RC_PREV(i - 1);
140 if (i == 0)
141 val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
142 else if (i == cs_rc_max - 1)
143 val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
144 writel(val, priv->base + offset + 4);
145
146 writel(0, priv->base + offset + 8);
147 writel(0, priv->base + offset + 12);
148 }
149
150
151 htable_offset = cs_rc_max * EIP197_CS_RC_SIZE;
152 for (i = 0; i < cs_ht_wc; i++)
153 writel(GENMASK(29, 0),
154 priv->base + EIP197_CLASSIFICATION_RAMS +
155 htable_offset + i * sizeof(u32));
156}
157
158static int eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
159{
160 u32 val, dsize, asize;
161 int cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc;
162 int cs_rc_abs_max, cs_ht_sz;
163 int maxbanks;
164
165
166 eip197_trc_cache_setupvirt(priv);
167
168
169
170
171
172 val = readl(priv->base + EIP197_CS_RAM_CTRL);
173 val &= ~EIP197_TRC_ENABLE_MASK;
174 val |= EIP197_TRC_ENABLE_0 | EIP197_CS_BANKSEL_MASK;
175 writel(val, priv->base + EIP197_CS_RAM_CTRL);
176 val = readl(priv->base + EIP197_CS_RAM_CTRL);
177 maxbanks = ((val&EIP197_CS_BANKSEL_MASK)>>EIP197_CS_BANKSEL_OFS) + 1;
178
179
180 writel(0, priv->base + EIP197_TRC_ECCCTRL);
181
182
183
184
185
186 val = readl(priv->base + EIP197_TRC_PARAMS);
187 val |= EIP197_TRC_PARAMS_SW_RESET | EIP197_TRC_PARAMS_DATA_ACCESS;
188 writel(val, priv->base + EIP197_TRC_PARAMS);
189
190
191 dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff, 32);
192
193
194
195
196
197
198 val = readl(priv->base + EIP197_TRC_PARAMS);
199
200 val &= ~(EIP197_TRC_PARAMS_DATA_ACCESS | EIP197_CS_BANKSEL_MASK);
201 writel(val, priv->base + EIP197_TRC_PARAMS);
202
203
204 asize = eip197_trc_cache_probe(priv, 0, 0x3fffffff, 16) >> 4;
205
206
207 writel(0, priv->base + EIP197_TRC_ECCCTRL);
208
209
210 if (dsize < EIP197_MIN_DSIZE || asize < EIP197_MIN_ASIZE) {
211 dev_err(priv->dev, "Record cache probing failed (%d,%d).",
212 dsize, asize);
213 return -ENODEV;
214 }
215
216
217
218
219
220
221
222
223 cs_trc_rec_wc = EIP197_CS_TRC_REC_WC;
224 cs_trc_lg_rec_wc = EIP197_CS_TRC_REC_WC;
225
226
227
228
229
230 cs_rc_abs_max = min_t(uint, ((dsize >> 2) / cs_trc_lg_rec_wc), 1023);
231
232 cs_rc_max = min_t(uint, cs_rc_abs_max, (asize >> 1));
233
234 cs_ht_sz = __fls(asize - cs_rc_max) - 2;
235
236 cs_ht_wc = 16 << cs_ht_sz;
237
238 cs_rc_max = min_t(uint, cs_rc_abs_max, asize - (cs_ht_wc >> 2));
239
240
241 eip197_trc_cache_clear(priv, cs_rc_max, cs_ht_wc);
242
243
244 val = readl(priv->base + EIP197_CS_RAM_CTRL);
245 val &= ~EIP197_TRC_ENABLE_MASK;
246 writel(val, priv->base + EIP197_CS_RAM_CTRL);
247
248
249 val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |
250 EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1);
251 writel(val, priv->base + EIP197_TRC_FREECHAIN);
252
253
254 val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) |
255 EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max);
256 writel(val, priv->base + EIP197_TRC_PARAMS2);
257
258
259 val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) |
260 EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
261 EIP197_TRC_PARAMS_HTABLE_SZ(cs_ht_sz);
262 writel(val, priv->base + EIP197_TRC_PARAMS);
263
264 dev_info(priv->dev, "TRC init: %dd,%da (%dr,%dh)\n",
265 dsize, asize, cs_rc_max, cs_ht_wc + cs_ht_wc);
266 return 0;
267}
268
269static void eip197_init_firmware(struct safexcel_crypto_priv *priv)
270{
271 int pe, i;
272 u32 val;
273
274 for (pe = 0; pe < priv->config.pes; pe++) {
275
276 writel(3, EIP197_PE(priv) + EIP197_PE_ICE_PUTF_CTRL(pe));
277 writel(0, EIP197_PE(priv) + EIP197_PE_ICE_PPTF_CTRL(pe));
278
279
280 val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
281 val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
282 EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
283 EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
284 EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
285 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
286
287
288 for (i = 0; i < EIP197_NUM_OF_SCRATCH_BLOCKS; i++)
289 writel(0, EIP197_PE(priv) +
290 EIP197_PE_ICE_SCRATCH_RAM(pe) + (i << 2));
291
292
293 writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
294 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
295 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
296 EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
297
298
299 writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
300 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
301 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
302 EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
303
304
305 writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN,
306 EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
307
308
309 if (priv->flags & EIP197_OCE)
310 writel(EIP197_DEBUG_OCE_BYPASS, EIP197_PE(priv) +
311 EIP197_PE_DEBUG(pe));
312 }
313
314}
315
316static int eip197_write_firmware(struct safexcel_crypto_priv *priv,
317 const struct firmware *fw)
318{
319 const __be32 *data = (const __be32 *)fw->data;
320 int i;
321
322
323 for (i = 0; i < fw->size / sizeof(u32); i++)
324 writel(be32_to_cpu(data[i]),
325 priv->base + EIP197_CLASSIFICATION_RAMS +
326 i * sizeof(__be32));
327
328
329 return i - EIP197_FW_TERMINAL_NOPS;
330}
331
332
333
334
335
336static bool poll_fw_ready(struct safexcel_crypto_priv *priv, int fpp)
337{
338 int pe, pollcnt;
339 u32 base, pollofs;
340
341 if (fpp)
342 pollofs = EIP197_FW_FPP_READY;
343 else
344 pollofs = EIP197_FW_PUE_READY;
345
346 for (pe = 0; pe < priv->config.pes; pe++) {
347 base = EIP197_PE_ICE_SCRATCH_RAM(pe);
348 pollcnt = EIP197_FW_START_POLLCNT;
349 while (pollcnt &&
350 (readl_relaxed(EIP197_PE(priv) + base +
351 pollofs) != 1)) {
352 pollcnt--;
353 }
354 if (!pollcnt) {
355 dev_err(priv->dev, "FW(%d) for PE %d failed to start\n",
356 fpp, pe);
357 return false;
358 }
359 }
360 return true;
361}
362
363static bool eip197_start_firmware(struct safexcel_crypto_priv *priv,
364 int ipuesz, int ifppsz, int minifw)
365{
366 int pe;
367 u32 val;
368
369 for (pe = 0; pe < priv->config.pes; pe++) {
370
371 writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
372
373
374 if (minifw)
375 val = 0;
376 else
377 val = EIP197_PE_ICE_UENG_START_OFFSET((ifppsz - 1) &
378 EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
379 EIP197_PE_ICE_UENG_DEBUG_RESET;
380 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
381
382
383 if (minifw)
384 val = 0;
385 else
386 val = EIP197_PE_ICE_UENG_START_OFFSET((ipuesz - 1) &
387 EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
388 EIP197_PE_ICE_UENG_DEBUG_RESET;
389 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
390 }
391
392
393 if (minifw)
394 return true;
395
396
397 if (!poll_fw_ready(priv, 1))
398 return false;
399 if (!poll_fw_ready(priv, 0))
400 return false;
401
402 return true;
403}
404
405static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
406{
407 const char *fw_name[] = {"ifpp.bin", "ipue.bin"};
408 const struct firmware *fw[FW_NB];
409 char fw_path[37], *dir = NULL;
410 int i, j, ret = 0, pe;
411 int ipuesz, ifppsz, minifw = 0;
412
413 if (priv->version == EIP197D_MRVL)
414 dir = "eip197d";
415 else if (priv->version == EIP197B_MRVL ||
416 priv->version == EIP197_DEVBRD)
417 dir = "eip197b";
418 else
419 return -ENODEV;
420
421retry_fw:
422 for (i = 0; i < FW_NB; i++) {
423 snprintf(fw_path, 37, "inside-secure/%s/%s", dir, fw_name[i]);
424 ret = firmware_request_nowarn(&fw[i], fw_path, priv->dev);
425 if (ret) {
426 if (minifw || priv->version != EIP197B_MRVL)
427 goto release_fw;
428
429
430
431
432 ret = firmware_request_nowarn(&fw[i], fw_name[i],
433 priv->dev);
434 if (ret)
435 goto release_fw;
436 }
437 }
438
439 eip197_init_firmware(priv);
440
441 ifppsz = eip197_write_firmware(priv, fw[FW_IFPP]);
442
443
444 for (pe = 0; pe < priv->config.pes; pe++)
445 writel(EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN,
446 EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
447
448 ipuesz = eip197_write_firmware(priv, fw[FW_IPUE]);
449
450 if (eip197_start_firmware(priv, ipuesz, ifppsz, minifw)) {
451 dev_dbg(priv->dev, "Firmware loaded successfully\n");
452 return 0;
453 }
454
455 ret = -ENODEV;
456
457release_fw:
458 for (j = 0; j < i; j++)
459 release_firmware(fw[j]);
460
461 if (!minifw) {
462
463 dev_dbg(priv->dev, "Firmware set not (fully) present or init failed, falling back to BCLA mode\n");
464 dir = "eip197_minifw";
465 minifw = 1;
466 goto retry_fw;
467 }
468
469 dev_dbg(priv->dev, "Firmware load failed.\n");
470
471 return ret;
472}
473
474static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
475{
476 u32 cd_size_rnd, val;
477 int i, cd_fetch_cnt;
478
479 cd_size_rnd = (priv->config.cd_size +
480 (BIT(priv->hwconfig.hwdataw) - 1)) >>
481 priv->hwconfig.hwdataw;
482
483 if (priv->flags & SAFEXCEL_HW_EIP197) {
484
485 cd_fetch_cnt = (1 << priv->hwconfig.hwcfsize) / cd_size_rnd;
486 cd_fetch_cnt = min_t(uint, cd_fetch_cnt,
487 (priv->config.pes * EIP197_FETCH_DEPTH));
488 } else {
489
490 cd_fetch_cnt = ((1 << priv->hwconfig.hwcfsize) /
491 cd_size_rnd) - 1;
492 }
493
494
495
496
497 if (!cd_fetch_cnt) {
498 dev_err(priv->dev, "Unable to fit even 1 command desc!\n");
499 return -ENODEV;
500 }
501
502 for (i = 0; i < priv->config.rings; i++) {
503
504 writel(lower_32_bits(priv->ring[i].cdr.base_dma),
505 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
506 writel(upper_32_bits(priv->ring[i].cdr.base_dma),
507 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
508
509 writel(EIP197_xDR_DESC_MODE_64BIT | EIP197_CDR_DESC_MODE_ADCP |
510 (priv->config.cd_offset << 14) | priv->config.cd_size,
511 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
512 writel(((cd_fetch_cnt *
513 (cd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
514 (cd_fetch_cnt * (priv->config.cd_offset / sizeof(u32))),
515 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
516
517
518 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
519 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
520 writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
521
522
523 writel(GENMASK(5, 0),
524 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
525 }
526
527 return 0;
528}
529
530static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
531{
532 u32 rd_size_rnd, val;
533 int i, rd_fetch_cnt;
534
535
536 rd_size_rnd = (EIP197_RD64_FETCH_SIZE +
537 (BIT(priv->hwconfig.hwdataw) - 1)) >>
538 priv->hwconfig.hwdataw;
539 if (priv->flags & SAFEXCEL_HW_EIP197) {
540
541 rd_fetch_cnt = (1 << priv->hwconfig.hwrfsize) / rd_size_rnd;
542 rd_fetch_cnt = min_t(uint, rd_fetch_cnt,
543 (priv->config.pes * EIP197_FETCH_DEPTH));
544 } else {
545
546 rd_fetch_cnt = ((1 << priv->hwconfig.hwrfsize) /
547 rd_size_rnd) - 1;
548 }
549
550 for (i = 0; i < priv->config.rings; i++) {
551
552 writel(lower_32_bits(priv->ring[i].rdr.base_dma),
553 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
554 writel(upper_32_bits(priv->ring[i].rdr.base_dma),
555 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
556
557 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 14) |
558 priv->config.rd_size,
559 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
560
561 writel(((rd_fetch_cnt *
562 (rd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
563 (rd_fetch_cnt * (priv->config.rd_offset / sizeof(u32))),
564 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
565
566
567 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
568 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
569 val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF;
570 writel(val,
571 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
572
573
574 writel(GENMASK(7, 0),
575 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
576
577
578 val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
579 val |= EIP197_RDR_IRQ(i);
580 writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
581 }
582
583 return 0;
584}
585
586static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
587{
588 u32 val;
589 int i, ret, pe, opbuflo, opbufhi;
590
591 dev_dbg(priv->dev, "HW init: using %d pipe(s) and %d ring(s)\n",
592 priv->config.pes, priv->config.rings);
593
594
595
596
597
598 if (priv->flags & SAFEXCEL_HW_EIP197) {
599 val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
600 val |= EIP197_MST_CTRL_TX_MAX_CMD(5);
601 writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
602 }
603
604
605 writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
606 EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),
607 EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL);
608
609
610
611
612 writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL);
613
614
615 writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
616
617
618 for (pe = 0; pe < priv->config.pes; pe++) {
619
620
621
622 writel(EIP197_DxE_THR_CTRL_RESET_PE,
623 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
624
625 if (priv->flags & EIP197_PE_ARB)
626
627 writel(EIP197_HIA_RA_PE_CTRL_RESET,
628 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
629
630
631 val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
632 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) |
633 EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
634 val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) |
635 EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
636 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
637 val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
638 writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe));
639
640
641 writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
642
643
644 writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
645 EIP197_PE_IN_xBUF_THRES_MAX(9),
646 EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe));
647 writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
648 EIP197_PE_IN_xBUF_THRES_MAX(7),
649 EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe));
650
651 if (priv->flags & SAFEXCEL_HW_EIP197)
652
653 writel(EIP197_HIA_RA_PE_CTRL_EN |
654 GENMASK(priv->config.rings - 1, 0),
655 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
656
657
658
659
660 writel(EIP197_DxE_THR_CTRL_RESET_PE,
661 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
662
663
664 while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) &
665 GENMASK(15, 12)) != GENMASK(15, 12))
666 ;
667
668
669 if (priv->hwconfig.hwnumpes > 4) {
670 opbuflo = 9;
671 opbufhi = 10;
672 } else {
673 opbuflo = 7;
674 opbufhi = 8;
675 }
676 val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
677 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(opbuflo) |
678 EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(opbufhi);
679 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
680 val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;
681
682
683
684 if (priv->flags & SAFEXCEL_HW_EIP197)
685 val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
686 writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe));
687
688
689 writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
690
691
692 writel(EIP197_PE_OUT_DBUF_THRES_MIN(opbuflo) |
693 EIP197_PE_OUT_DBUF_THRES_MAX(opbufhi),
694 EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe));
695
696
697
698
699 val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES |
700 EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT |
701 EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT;
702 writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe));
703
704
705 writel(EIP197_FUNCTION_ALL,
706 EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe));
707 writel(EIP197_FUNCTION_ALL,
708 EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION2_EN(pe));
709 }
710
711
712 for (i = 0; i < priv->config.rings; i++) {
713
714 writel(GENMASK(31, 0),
715 EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i));
716
717
718 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
719
720
721 writel(EIP197_xDR_PREP_CLR_COUNT,
722 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
723
724
725 writel(EIP197_xDR_PROC_CLR_COUNT,
726 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
727
728 writel(0,
729 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
730 writel(0,
731 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
732
733 writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset),
734 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
735 }
736
737
738 for (i = 0; i < priv->config.rings; i++) {
739
740 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
741
742
743 writel(EIP197_xDR_PREP_CLR_COUNT,
744 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
745
746
747 writel(EIP197_xDR_PROC_CLR_COUNT,
748 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
749
750 writel(0,
751 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
752 writel(0,
753 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
754
755
756 writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset),
757 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
758 }
759
760 for (pe = 0; pe < priv->config.pes; pe++) {
761
762 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
763 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
764
765
766 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
767 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
768 }
769
770
771 writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
772
773 if (priv->flags & EIP197_SIMPLE_TRC) {
774 writel(EIP197_STRC_CONFIG_INIT |
775 EIP197_STRC_CONFIG_LARGE_REC(EIP197_CS_TRC_REC_WC) |
776 EIP197_STRC_CONFIG_SMALL_REC(EIP197_CS_TRC_REC_WC),
777 priv->base + EIP197_STRC_CONFIG);
778 writel(EIP197_PE_EIP96_TOKEN_CTRL2_CTX_DONE,
779 EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL2(0));
780 } else if (priv->flags & SAFEXCEL_HW_EIP197) {
781 ret = eip197_trc_cache_init(priv);
782 if (ret)
783 return ret;
784 }
785
786 if (priv->flags & EIP197_ICE) {
787 ret = eip197_load_firmwares(priv);
788 if (ret)
789 return ret;
790 }
791
792 return safexcel_hw_setup_cdesc_rings(priv) ?:
793 safexcel_hw_setup_rdesc_rings(priv) ?:
794 0;
795}
796
797
798static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv,
799 int ring)
800{
801 int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ);
802
803 if (!coal)
804 return;
805
806
807 writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
808 EIP197_HIA_RDR_THRESH_PROC_PKT(coal),
809 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH);
810}
811
812void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
813{
814 struct crypto_async_request *req, *backlog;
815 struct safexcel_context *ctx;
816 int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;
817
818
819
820
821 req = priv->ring[ring].req;
822 backlog = priv->ring[ring].backlog;
823 if (req)
824 goto handle_req;
825
826 while (true) {
827 spin_lock_bh(&priv->ring[ring].queue_lock);
828 backlog = crypto_get_backlog(&priv->ring[ring].queue);
829 req = crypto_dequeue_request(&priv->ring[ring].queue);
830 spin_unlock_bh(&priv->ring[ring].queue_lock);
831
832 if (!req) {
833 priv->ring[ring].req = NULL;
834 priv->ring[ring].backlog = NULL;
835 goto finalize;
836 }
837
838handle_req:
839 ctx = crypto_tfm_ctx(req->tfm);
840 ret = ctx->send(req, ring, &commands, &results);
841 if (ret)
842 goto request_failed;
843
844 if (backlog)
845 backlog->complete(backlog, -EINPROGRESS);
846
847
848
849
850
851 if (!commands && !results)
852 continue;
853
854 cdesc += commands;
855 rdesc += results;
856 nreq++;
857 }
858
859request_failed:
860
861
862
863 priv->ring[ring].req = req;
864 priv->ring[ring].backlog = backlog;
865
866finalize:
867 if (!nreq)
868 return;
869
870 spin_lock_bh(&priv->ring[ring].lock);
871
872 priv->ring[ring].requests += nreq;
873
874 if (!priv->ring[ring].busy) {
875 safexcel_try_push_requests(priv, ring);
876 priv->ring[ring].busy = true;
877 }
878
879 spin_unlock_bh(&priv->ring[ring].lock);
880
881
882 writel((rdesc * priv->config.rd_offset),
883 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
884
885
886 writel((cdesc * priv->config.cd_offset),
887 EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
888}
889
890inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
891 void *rdp)
892{
893 struct safexcel_result_desc *rdesc = rdp;
894 struct result_data_desc *result_data = rdp + priv->config.res_offset;
895
896 if (likely((!rdesc->last_seg) ||
897 ((!rdesc->descriptor_overflow) &&
898 (!rdesc->buffer_overflow) &&
899 (!result_data->error_code))))
900 return 0;
901
902 if (rdesc->descriptor_overflow)
903 dev_err(priv->dev, "Descriptor overflow detected");
904
905 if (rdesc->buffer_overflow)
906 dev_err(priv->dev, "Buffer overflow detected");
907
908 if (result_data->error_code & 0x4066) {
909
910 dev_err(priv->dev,
911 "result descriptor error (%x)",
912 result_data->error_code);
913
914 return -EIO;
915 } else if (result_data->error_code &
916 (BIT(7) | BIT(4) | BIT(3) | BIT(0))) {
917
918
919
920
921
922 return -EINVAL;
923 } else if (result_data->error_code & BIT(9)) {
924
925 return -EBADMSG;
926 }
927
928
929 return -EINVAL;
930}
931
932inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv,
933 int ring,
934 struct safexcel_result_desc *rdesc,
935 struct crypto_async_request *req)
936{
937 int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc);
938
939 priv->ring[ring].rdr_req[i] = req;
940}
941
942inline struct crypto_async_request *
943safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring)
944{
945 int i = safexcel_ring_first_rdr_index(priv, ring);
946
947 return priv->ring[ring].rdr_req[i];
948}
949
950void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
951{
952 struct safexcel_command_desc *cdesc;
953
954
955 do {
956 cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr);
957 if (IS_ERR(cdesc)) {
958 dev_err(priv->dev,
959 "Could not retrieve the command descriptor\n");
960 return;
961 }
962 } while (!cdesc->last_seg);
963}
964
965void safexcel_inv_complete(struct crypto_async_request *req, int error)
966{
967 struct safexcel_inv_result *result = req->data;
968
969 if (error == -EINPROGRESS)
970 return;
971
972 result->error = error;
973 complete(&result->completion);
974}
975
976int safexcel_invalidate_cache(struct crypto_async_request *async,
977 struct safexcel_crypto_priv *priv,
978 dma_addr_t ctxr_dma, int ring)
979{
980 struct safexcel_command_desc *cdesc;
981 struct safexcel_result_desc *rdesc;
982 struct safexcel_token *dmmy;
983 int ret = 0;
984
985
986 cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma,
987 &dmmy);
988 if (IS_ERR(cdesc))
989 return PTR_ERR(cdesc);
990
991 cdesc->control_data.type = EIP197_TYPE_EXTENDED;
992 cdesc->control_data.options = 0;
993 cdesc->control_data.context_lo &= ~EIP197_CONTEXT_SIZE_MASK;
994 cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR;
995
996
997 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
998
999 if (IS_ERR(rdesc)) {
1000 ret = PTR_ERR(rdesc);
1001 goto cdesc_rollback;
1002 }
1003
1004 safexcel_rdr_req_set(priv, ring, rdesc, async);
1005
1006 return ret;
1007
1008cdesc_rollback:
1009 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
1010
1011 return ret;
1012}
1013
1014static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
1015 int ring)
1016{
1017 struct crypto_async_request *req;
1018 struct safexcel_context *ctx;
1019 int ret, i, nreq, ndesc, tot_descs, handled = 0;
1020 bool should_complete;
1021
1022handle_results:
1023 tot_descs = 0;
1024
1025 nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
1026 nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET;
1027 nreq &= EIP197_xDR_PROC_xD_PKT_MASK;
1028 if (!nreq)
1029 goto requests_left;
1030
1031 for (i = 0; i < nreq; i++) {
1032 req = safexcel_rdr_req_get(priv, ring);
1033
1034 ctx = crypto_tfm_ctx(req->tfm);
1035 ndesc = ctx->handle_result(priv, ring, req,
1036 &should_complete, &ret);
1037 if (ndesc < 0) {
1038 dev_err(priv->dev, "failed to handle result (%d)\n",
1039 ndesc);
1040 goto acknowledge;
1041 }
1042
1043 if (should_complete) {
1044 local_bh_disable();
1045 req->complete(req, ret);
1046 local_bh_enable();
1047 }
1048
1049 tot_descs += ndesc;
1050 handled++;
1051 }
1052
1053acknowledge:
1054 if (i)
1055 writel(EIP197_xDR_PROC_xD_PKT(i) |
1056 (tot_descs * priv->config.rd_offset),
1057 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
1058
1059
1060
1061
1062 if (nreq == EIP197_xDR_PROC_xD_PKT_MASK)
1063 goto handle_results;
1064
1065requests_left:
1066 spin_lock_bh(&priv->ring[ring].lock);
1067
1068 priv->ring[ring].requests -= handled;
1069 safexcel_try_push_requests(priv, ring);
1070
1071 if (!priv->ring[ring].requests)
1072 priv->ring[ring].busy = false;
1073
1074 spin_unlock_bh(&priv->ring[ring].lock);
1075}
1076
1077static void safexcel_dequeue_work(struct work_struct *work)
1078{
1079 struct safexcel_work_data *data =
1080 container_of(work, struct safexcel_work_data, work);
1081
1082 safexcel_dequeue(data->priv, data->ring);
1083}
1084
1085struct safexcel_ring_irq_data {
1086 struct safexcel_crypto_priv *priv;
1087 int ring;
1088};
1089
1090static irqreturn_t safexcel_irq_ring(int irq, void *data)
1091{
1092 struct safexcel_ring_irq_data *irq_data = data;
1093 struct safexcel_crypto_priv *priv = irq_data->priv;
1094 int ring = irq_data->ring, rc = IRQ_NONE;
1095 u32 status, stat;
1096
1097 status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
1098 if (!status)
1099 return rc;
1100
1101
1102 if (status & EIP197_RDR_IRQ(ring)) {
1103 stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
1104
1105 if (unlikely(stat & EIP197_xDR_ERR)) {
1106
1107
1108
1109
1110
1111 dev_err(priv->dev, "RDR: fatal error.\n");
1112 } else if (likely(stat & EIP197_xDR_THRESH)) {
1113 rc = IRQ_WAKE_THREAD;
1114 }
1115
1116
1117 writel(stat & 0xff,
1118 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
1119 }
1120
1121
1122 writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring));
1123
1124 return rc;
1125}
1126
1127static irqreturn_t safexcel_irq_ring_thread(int irq, void *data)
1128{
1129 struct safexcel_ring_irq_data *irq_data = data;
1130 struct safexcel_crypto_priv *priv = irq_data->priv;
1131 int ring = irq_data->ring;
1132
1133 safexcel_handle_result_descriptor(priv, ring);
1134
1135 queue_work(priv->ring[ring].workqueue,
1136 &priv->ring[ring].work_data.work);
1137
1138 return IRQ_HANDLED;
1139}
1140
1141static int safexcel_request_ring_irq(void *pdev, int irqid,
1142 int is_pci_dev,
1143 int ring_id,
1144 irq_handler_t handler,
1145 irq_handler_t threaded_handler,
1146 struct safexcel_ring_irq_data *ring_irq_priv)
1147{
1148 int ret, irq, cpu;
1149 struct device *dev;
1150
1151 if (IS_ENABLED(CONFIG_PCI) && is_pci_dev) {
1152 struct pci_dev *pci_pdev = pdev;
1153
1154 dev = &pci_pdev->dev;
1155 irq = pci_irq_vector(pci_pdev, irqid);
1156 if (irq < 0) {
1157 dev_err(dev, "unable to get device MSI IRQ %d (err %d)\n",
1158 irqid, irq);
1159 return irq;
1160 }
1161 } else if (IS_ENABLED(CONFIG_OF)) {
1162 struct platform_device *plf_pdev = pdev;
1163 char irq_name[6] = {0};
1164
1165 snprintf(irq_name, 6, "ring%d", irqid);
1166 dev = &plf_pdev->dev;
1167 irq = platform_get_irq_byname(plf_pdev, irq_name);
1168
1169 if (irq < 0)
1170 return irq;
1171 } else {
1172 return -ENXIO;
1173 }
1174
1175 ret = devm_request_threaded_irq(dev, irq, handler,
1176 threaded_handler, IRQF_ONESHOT,
1177 dev_name(dev), ring_irq_priv);
1178 if (ret) {
1179 dev_err(dev, "unable to request IRQ %d\n", irq);
1180 return ret;
1181 }
1182
1183
1184 cpu = cpumask_local_spread(ring_id, NUMA_NO_NODE);
1185 irq_set_affinity_hint(irq, get_cpu_mask(cpu));
1186
1187 return irq;
1188}
1189
1190static struct safexcel_alg_template *safexcel_algs[] = {
1191 &safexcel_alg_ecb_des,
1192 &safexcel_alg_cbc_des,
1193 &safexcel_alg_ecb_des3_ede,
1194 &safexcel_alg_cbc_des3_ede,
1195 &safexcel_alg_ecb_aes,
1196 &safexcel_alg_cbc_aes,
1197 &safexcel_alg_cfb_aes,
1198 &safexcel_alg_ofb_aes,
1199 &safexcel_alg_ctr_aes,
1200 &safexcel_alg_md5,
1201 &safexcel_alg_sha1,
1202 &safexcel_alg_sha224,
1203 &safexcel_alg_sha256,
1204 &safexcel_alg_sha384,
1205 &safexcel_alg_sha512,
1206 &safexcel_alg_hmac_md5,
1207 &safexcel_alg_hmac_sha1,
1208 &safexcel_alg_hmac_sha224,
1209 &safexcel_alg_hmac_sha256,
1210 &safexcel_alg_hmac_sha384,
1211 &safexcel_alg_hmac_sha512,
1212 &safexcel_alg_authenc_hmac_sha1_cbc_aes,
1213 &safexcel_alg_authenc_hmac_sha224_cbc_aes,
1214 &safexcel_alg_authenc_hmac_sha256_cbc_aes,
1215 &safexcel_alg_authenc_hmac_sha384_cbc_aes,
1216 &safexcel_alg_authenc_hmac_sha512_cbc_aes,
1217 &safexcel_alg_authenc_hmac_sha1_cbc_des3_ede,
1218 &safexcel_alg_authenc_hmac_sha1_ctr_aes,
1219 &safexcel_alg_authenc_hmac_sha224_ctr_aes,
1220 &safexcel_alg_authenc_hmac_sha256_ctr_aes,
1221 &safexcel_alg_authenc_hmac_sha384_ctr_aes,
1222 &safexcel_alg_authenc_hmac_sha512_ctr_aes,
1223 &safexcel_alg_xts_aes,
1224 &safexcel_alg_gcm,
1225 &safexcel_alg_ccm,
1226 &safexcel_alg_crc32,
1227 &safexcel_alg_cbcmac,
1228 &safexcel_alg_xcbcmac,
1229 &safexcel_alg_cmac,
1230 &safexcel_alg_chacha20,
1231 &safexcel_alg_chachapoly,
1232 &safexcel_alg_chachapoly_esp,
1233 &safexcel_alg_sm3,
1234 &safexcel_alg_hmac_sm3,
1235 &safexcel_alg_ecb_sm4,
1236 &safexcel_alg_cbc_sm4,
1237 &safexcel_alg_ofb_sm4,
1238 &safexcel_alg_cfb_sm4,
1239 &safexcel_alg_ctr_sm4,
1240 &safexcel_alg_authenc_hmac_sha1_cbc_sm4,
1241 &safexcel_alg_authenc_hmac_sm3_cbc_sm4,
1242 &safexcel_alg_authenc_hmac_sha1_ctr_sm4,
1243 &safexcel_alg_authenc_hmac_sm3_ctr_sm4,
1244 &safexcel_alg_sha3_224,
1245 &safexcel_alg_sha3_256,
1246 &safexcel_alg_sha3_384,
1247 &safexcel_alg_sha3_512,
1248 &safexcel_alg_hmac_sha3_224,
1249 &safexcel_alg_hmac_sha3_256,
1250 &safexcel_alg_hmac_sha3_384,
1251 &safexcel_alg_hmac_sha3_512,
1252 &safexcel_alg_authenc_hmac_sha1_cbc_des,
1253 &safexcel_alg_authenc_hmac_sha256_cbc_des3_ede,
1254 &safexcel_alg_authenc_hmac_sha224_cbc_des3_ede,
1255 &safexcel_alg_authenc_hmac_sha512_cbc_des3_ede,
1256 &safexcel_alg_authenc_hmac_sha384_cbc_des3_ede,
1257 &safexcel_alg_authenc_hmac_sha256_cbc_des,
1258 &safexcel_alg_authenc_hmac_sha224_cbc_des,
1259 &safexcel_alg_authenc_hmac_sha512_cbc_des,
1260 &safexcel_alg_authenc_hmac_sha384_cbc_des,
1261 &safexcel_alg_rfc4106_gcm,
1262 &safexcel_alg_rfc4543_gcm,
1263 &safexcel_alg_rfc4309_ccm,
1264};
1265
1266static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
1267{
1268 int i, j, ret = 0;
1269
1270 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
1271 safexcel_algs[i]->priv = priv;
1272
1273
1274 if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
1275 safexcel_algs[i]->algo_mask)
1276
1277 continue;
1278
1279 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1280 ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
1281 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
1282 ret = crypto_register_aead(&safexcel_algs[i]->alg.aead);
1283 else
1284 ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);
1285
1286 if (ret)
1287 goto fail;
1288 }
1289
1290 return 0;
1291
1292fail:
1293 for (j = 0; j < i; j++) {
1294
1295 if ((safexcel_algs[j]->algo_mask & priv->hwconfig.algo_flags) !=
1296 safexcel_algs[j]->algo_mask)
1297
1298 continue;
1299
1300 if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1301 crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
1302 else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD)
1303 crypto_unregister_aead(&safexcel_algs[j]->alg.aead);
1304 else
1305 crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);
1306 }
1307
1308 return ret;
1309}
1310
1311static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
1312{
1313 int i;
1314
1315 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
1316
1317 if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
1318 safexcel_algs[i]->algo_mask)
1319
1320 continue;
1321
1322 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1323 crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
1324 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
1325 crypto_unregister_aead(&safexcel_algs[i]->alg.aead);
1326 else
1327 crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);
1328 }
1329}
1330
1331static void safexcel_configure(struct safexcel_crypto_priv *priv)
1332{
1333 u32 mask = BIT(priv->hwconfig.hwdataw) - 1;
1334
1335 priv->config.pes = priv->hwconfig.hwnumpes;
1336 priv->config.rings = min_t(u32, priv->hwconfig.hwnumrings, max_rings);
1337
1338 priv->config.rings = min_t(u32, priv->config.rings,
1339 priv->hwconfig.hwnumraic);
1340
1341 priv->config.cd_size = EIP197_CD64_FETCH_SIZE;
1342 priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;
1343 priv->config.cdsh_offset = (EIP197_MAX_TOKENS + mask) & ~mask;
1344
1345
1346 priv->config.res_offset = (EIP197_RD64_FETCH_SIZE + mask) & ~mask;
1347
1348 priv->config.rd_size = priv->config.res_offset +
1349 EIP197_RD64_RESULT_SIZE;
1350 priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;
1351
1352
1353 priv->config.cd_offset *= sizeof(u32);
1354 priv->config.cdsh_offset *= sizeof(u32);
1355 priv->config.rd_offset *= sizeof(u32);
1356 priv->config.res_offset *= sizeof(u32);
1357}
1358
1359static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
1360{
1361 struct safexcel_register_offsets *offsets = &priv->offsets;
1362
1363 if (priv->flags & SAFEXCEL_HW_EIP197) {
1364 offsets->hia_aic = EIP197_HIA_AIC_BASE;
1365 offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE;
1366 offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE;
1367 offsets->hia_aic_xdr = EIP197_HIA_AIC_xDR_BASE;
1368 offsets->hia_dfe = EIP197_HIA_DFE_BASE;
1369 offsets->hia_dfe_thr = EIP197_HIA_DFE_THR_BASE;
1370 offsets->hia_dse = EIP197_HIA_DSE_BASE;
1371 offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE;
1372 offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE;
1373 offsets->pe = EIP197_PE_BASE;
1374 offsets->global = EIP197_GLOBAL_BASE;
1375 } else {
1376 offsets->hia_aic = EIP97_HIA_AIC_BASE;
1377 offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE;
1378 offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE;
1379 offsets->hia_aic_xdr = EIP97_HIA_AIC_xDR_BASE;
1380 offsets->hia_dfe = EIP97_HIA_DFE_BASE;
1381 offsets->hia_dfe_thr = EIP97_HIA_DFE_THR_BASE;
1382 offsets->hia_dse = EIP97_HIA_DSE_BASE;
1383 offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE;
1384 offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE;
1385 offsets->pe = EIP97_PE_BASE;
1386 offsets->global = EIP97_GLOBAL_BASE;
1387 }
1388}
1389
1390
1391
1392
1393
1394
1395
1396
1397static int safexcel_probe_generic(void *pdev,
1398 struct safexcel_crypto_priv *priv,
1399 int is_pci_dev)
1400{
1401 struct device *dev = priv->dev;
1402 u32 peid, version, mask, val, hiaopt, hwopt, peopt;
1403 int i, ret, hwctg;
1404
1405 priv->context_pool = dmam_pool_create("safexcel-context", dev,
1406 sizeof(struct safexcel_context_record),
1407 1, 0);
1408 if (!priv->context_pool)
1409 return -ENOMEM;
1410
1411
1412
1413
1414
1415
1416 version = readl(priv->base + EIP97_HIA_AIC_BASE + EIP197_HIA_VERSION);
1417
1418 mask = 0;
1419 if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
1420 priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
1421 } else if (EIP197_REG_HI16(version) == EIP197_HIA_VERSION_BE) {
1422
1423 mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
1424 priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
1425 } else {
1426
1427 version = readl(priv->base + EIP197_HIA_AIC_BASE +
1428 EIP197_HIA_VERSION);
1429 if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
1430 priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
1431 priv->flags |= SAFEXCEL_HW_EIP197;
1432 } else if (EIP197_REG_HI16(version) ==
1433 EIP197_HIA_VERSION_BE) {
1434
1435 mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
1436 priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
1437 priv->flags |= SAFEXCEL_HW_EIP197;
1438 } else {
1439 return -ENODEV;
1440 }
1441 }
1442
1443
1444 safexcel_init_register_offsets(priv);
1445
1446
1447
1448
1449
1450
1451 if (mask) {
1452 val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
1453 val = val ^ (mask >> 24);
1454 writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
1455 }
1456
1457
1458
1459
1460
1461
1462 version = readl(EIP197_GLOBAL(priv) + EIP197_VERSION);
1463 if (((priv->flags & SAFEXCEL_HW_EIP197) &&
1464 (EIP197_REG_LO16(version) != EIP197_VERSION_LE) &&
1465 (EIP197_REG_LO16(version) != EIP196_VERSION_LE)) ||
1466 ((!(priv->flags & SAFEXCEL_HW_EIP197) &&
1467 (EIP197_REG_LO16(version) != EIP97_VERSION_LE)))) {
1468
1469
1470
1471
1472 dev_err(priv->dev, "Probing for EIP97/EIP19x failed - no such device (read %08x)\n",
1473 version);
1474 return -ENODEV;
1475 }
1476
1477 priv->hwconfig.hwver = EIP197_VERSION_MASK(version);
1478 hwctg = version >> 28;
1479 peid = version & 255;
1480
1481
1482 version = readl(EIP197_PE(priv) + + EIP197_PE_VERSION(0));
1483 if (EIP197_REG_LO16(version) != EIP206_VERSION_LE) {
1484 dev_err(priv->dev, "EIP%d: EIP206 not detected\n", peid);
1485 return -ENODEV;
1486 }
1487 priv->hwconfig.ppver = EIP197_VERSION_MASK(version);
1488
1489
1490 version = readl(EIP197_PE(priv) + EIP197_PE_EIP96_VERSION(0));
1491 if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {
1492 dev_err(dev, "EIP%d: EIP96 not detected.\n", peid);
1493 return -ENODEV;
1494 }
1495 priv->hwconfig.pever = EIP197_VERSION_MASK(version);
1496
1497 hwopt = readl(EIP197_GLOBAL(priv) + EIP197_OPTIONS);
1498 hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS);
1499
1500 priv->hwconfig.icever = 0;
1501 priv->hwconfig.ocever = 0;
1502 priv->hwconfig.psever = 0;
1503 if (priv->flags & SAFEXCEL_HW_EIP197) {
1504
1505 peopt = readl(EIP197_PE(priv) + EIP197_PE_OPTIONS(0));
1506
1507 priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) &
1508 EIP197_HWDATAW_MASK;
1509 priv->hwconfig.hwcfsize = ((hiaopt >> EIP197_CFSIZE_OFFSET) &
1510 EIP197_CFSIZE_MASK) +
1511 EIP197_CFSIZE_ADJUST;
1512 priv->hwconfig.hwrfsize = ((hiaopt >> EIP197_RFSIZE_OFFSET) &
1513 EIP197_RFSIZE_MASK) +
1514 EIP197_RFSIZE_ADJUST;
1515 priv->hwconfig.hwnumpes = (hiaopt >> EIP197_N_PES_OFFSET) &
1516 EIP197_N_PES_MASK;
1517 priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) &
1518 EIP197_N_RINGS_MASK;
1519 if (hiaopt & EIP197_HIA_OPT_HAS_PE_ARB)
1520 priv->flags |= EIP197_PE_ARB;
1521 if (EIP206_OPT_ICE_TYPE(peopt) == 1) {
1522 priv->flags |= EIP197_ICE;
1523
1524 version = readl(EIP197_PE(priv) +
1525 EIP197_PE_ICE_VERSION(0));
1526 if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) {
1527 dev_err(dev, "EIP%d: ICE EIP207 not detected.\n",
1528 peid);
1529 return -ENODEV;
1530 }
1531 priv->hwconfig.icever = EIP197_VERSION_MASK(version);
1532 }
1533 if (EIP206_OPT_OCE_TYPE(peopt) == 1) {
1534 priv->flags |= EIP197_OCE;
1535
1536 version = readl(EIP197_PE(priv) + EIP197_PE_PSE_VERSION(0));
1537 if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {
1538 dev_err(dev, "EIP%d: EIP96PP not detected.\n", peid);
1539 return -ENODEV;
1540 }
1541 priv->hwconfig.psever = EIP197_VERSION_MASK(version);
1542
1543 version = readl(EIP197_PE(priv) +
1544 EIP197_PE_ICE_VERSION(0));
1545 if (EIP197_REG_LO16(version) != EIP207_VERSION_LE) {
1546 dev_err(dev, "EIP%d: OCE EIP207 not detected.\n",
1547 peid);
1548 return -ENODEV;
1549 }
1550 priv->hwconfig.ocever = EIP197_VERSION_MASK(version);
1551 }
1552
1553 if (!(hwopt & EIP197_OPT_HAS_TRC))
1554 priv->flags |= EIP197_SIMPLE_TRC;
1555
1556 priv->flags |= EIP197_TRC_CACHE;
1557 } else {
1558
1559 priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) &
1560 EIP97_HWDATAW_MASK;
1561 priv->hwconfig.hwcfsize = (hiaopt >> EIP97_CFSIZE_OFFSET) &
1562 EIP97_CFSIZE_MASK;
1563 priv->hwconfig.hwrfsize = (hiaopt >> EIP97_RFSIZE_OFFSET) &
1564 EIP97_RFSIZE_MASK;
1565 priv->hwconfig.hwnumpes = 1;
1566 priv->hwconfig.hwnumrings = (hiaopt >> EIP197_N_RINGS_OFFSET) &
1567 EIP197_N_RINGS_MASK;
1568 }
1569
1570
1571 for (i = 0; i < EIP197_MAX_RING_AIC; i++) {
1572 version = readl(EIP197_HIA_AIC_R(priv) +
1573 EIP197_HIA_AIC_R_VERSION(i));
1574 if (EIP197_REG_LO16(version) != EIP201_VERSION_LE)
1575 break;
1576 }
1577 priv->hwconfig.hwnumraic = i;
1578
1579 if (!priv->hwconfig.hwnumraic) {
1580 dev_err(priv->dev, "No ring interrupt controller present!\n");
1581 return -ENODEV;
1582 }
1583
1584
1585 priv->hwconfig.algo_flags = readl(EIP197_PE(priv) +
1586 EIP197_PE_EIP96_OPTIONS(0));
1587
1588
1589 dev_info(priv->dev, "EIP%d:%x(%d,%d,%d,%d)-HIA:%x(%d,%d,%d),PE:%x/%x(alg:%08x)/%x/%x/%x\n",
1590 peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hwnumpes,
1591 priv->hwconfig.hwnumrings, priv->hwconfig.hwnumraic,
1592 priv->hwconfig.hiaver, priv->hwconfig.hwdataw,
1593 priv->hwconfig.hwcfsize, priv->hwconfig.hwrfsize,
1594 priv->hwconfig.ppver, priv->hwconfig.pever,
1595 priv->hwconfig.algo_flags, priv->hwconfig.icever,
1596 priv->hwconfig.ocever, priv->hwconfig.psever);
1597
1598 safexcel_configure(priv);
1599
1600 if (IS_ENABLED(CONFIG_PCI) && priv->version == EIP197_DEVBRD) {
1601
1602
1603
1604
1605 struct pci_dev *pci_pdev = pdev;
1606
1607 ret = pci_alloc_irq_vectors(pci_pdev,
1608 priv->config.rings + 1,
1609 priv->config.rings + 1,
1610 PCI_IRQ_MSI | PCI_IRQ_MSIX);
1611 if (ret < 0) {
1612 dev_err(dev, "Failed to allocate PCI MSI interrupts\n");
1613 return ret;
1614 }
1615 }
1616
1617
1618 priv->ring = devm_kcalloc(dev, priv->config.rings,
1619 sizeof(*priv->ring),
1620 GFP_KERNEL);
1621 if (!priv->ring)
1622 return -ENOMEM;
1623
1624 for (i = 0; i < priv->config.rings; i++) {
1625 char wq_name[9] = {0};
1626 int irq;
1627 struct safexcel_ring_irq_data *ring_irq;
1628
1629 ret = safexcel_init_ring_descriptors(priv,
1630 &priv->ring[i].cdr,
1631 &priv->ring[i].rdr);
1632 if (ret) {
1633 dev_err(dev, "Failed to initialize rings\n");
1634 return ret;
1635 }
1636
1637 priv->ring[i].rdr_req = devm_kcalloc(dev,
1638 EIP197_DEFAULT_RING_SIZE,
1639 sizeof(*priv->ring[i].rdr_req),
1640 GFP_KERNEL);
1641 if (!priv->ring[i].rdr_req)
1642 return -ENOMEM;
1643
1644 ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
1645 if (!ring_irq)
1646 return -ENOMEM;
1647
1648 ring_irq->priv = priv;
1649 ring_irq->ring = i;
1650
1651 irq = safexcel_request_ring_irq(pdev,
1652 EIP197_IRQ_NUMBER(i, is_pci_dev),
1653 is_pci_dev,
1654 i,
1655 safexcel_irq_ring,
1656 safexcel_irq_ring_thread,
1657 ring_irq);
1658 if (irq < 0) {
1659 dev_err(dev, "Failed to get IRQ ID for ring %d\n", i);
1660 return irq;
1661 }
1662
1663 priv->ring[i].irq = irq;
1664 priv->ring[i].work_data.priv = priv;
1665 priv->ring[i].work_data.ring = i;
1666 INIT_WORK(&priv->ring[i].work_data.work,
1667 safexcel_dequeue_work);
1668
1669 snprintf(wq_name, 9, "wq_ring%d", i);
1670 priv->ring[i].workqueue =
1671 create_singlethread_workqueue(wq_name);
1672 if (!priv->ring[i].workqueue)
1673 return -ENOMEM;
1674
1675 priv->ring[i].requests = 0;
1676 priv->ring[i].busy = false;
1677
1678 crypto_init_queue(&priv->ring[i].queue,
1679 EIP197_DEFAULT_RING_SIZE);
1680
1681 spin_lock_init(&priv->ring[i].lock);
1682 spin_lock_init(&priv->ring[i].queue_lock);
1683 }
1684
1685 atomic_set(&priv->ring_used, 0);
1686
1687 ret = safexcel_hw_init(priv);
1688 if (ret) {
1689 dev_err(dev, "HW init failed (%d)\n", ret);
1690 return ret;
1691 }
1692
1693 ret = safexcel_register_algorithms(priv);
1694 if (ret) {
1695 dev_err(dev, "Failed to register algorithms (%d)\n", ret);
1696 return ret;
1697 }
1698
1699 return 0;
1700}
1701
1702static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv)
1703{
1704 int i;
1705
1706 for (i = 0; i < priv->config.rings; i++) {
1707
1708 writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
1709 writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
1710
1711
1712 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
1713 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
1714
1715
1716 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
1717 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
1718 }
1719}
1720
1721
1722
1723static int safexcel_probe(struct platform_device *pdev)
1724{
1725 struct device *dev = &pdev->dev;
1726 struct safexcel_crypto_priv *priv;
1727 int ret;
1728
1729 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
1730 if (!priv)
1731 return -ENOMEM;
1732
1733 priv->dev = dev;
1734 priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev);
1735
1736 platform_set_drvdata(pdev, priv);
1737
1738 priv->base = devm_platform_ioremap_resource(pdev, 0);
1739 if (IS_ERR(priv->base)) {
1740 dev_err(dev, "failed to get resource\n");
1741 return PTR_ERR(priv->base);
1742 }
1743
1744 priv->clk = devm_clk_get(&pdev->dev, NULL);
1745 ret = PTR_ERR_OR_ZERO(priv->clk);
1746
1747 if (ret != -ENOENT) {
1748 if (ret)
1749 return ret;
1750
1751 ret = clk_prepare_enable(priv->clk);
1752 if (ret) {
1753 dev_err(dev, "unable to enable clk (%d)\n", ret);
1754 return ret;
1755 }
1756 }
1757
1758 priv->reg_clk = devm_clk_get(&pdev->dev, "reg");
1759 ret = PTR_ERR_OR_ZERO(priv->reg_clk);
1760
1761 if (ret != -ENOENT) {
1762 if (ret)
1763 goto err_core_clk;
1764
1765 ret = clk_prepare_enable(priv->reg_clk);
1766 if (ret) {
1767 dev_err(dev, "unable to enable reg clk (%d)\n", ret);
1768 goto err_core_clk;
1769 }
1770 }
1771
1772 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
1773 if (ret)
1774 goto err_reg_clk;
1775
1776
1777 ret = safexcel_probe_generic(pdev, priv, 0);
1778 if (ret)
1779 goto err_reg_clk;
1780
1781 return 0;
1782
1783err_reg_clk:
1784 clk_disable_unprepare(priv->reg_clk);
1785err_core_clk:
1786 clk_disable_unprepare(priv->clk);
1787 return ret;
1788}
1789
1790static int safexcel_remove(struct platform_device *pdev)
1791{
1792 struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);
1793 int i;
1794
1795 safexcel_unregister_algorithms(priv);
1796 safexcel_hw_reset_rings(priv);
1797
1798 clk_disable_unprepare(priv->reg_clk);
1799 clk_disable_unprepare(priv->clk);
1800
1801 for (i = 0; i < priv->config.rings; i++) {
1802 irq_set_affinity_hint(priv->ring[i].irq, NULL);
1803 destroy_workqueue(priv->ring[i].workqueue);
1804 }
1805
1806 return 0;
1807}
1808
1809static const struct of_device_id safexcel_of_match_table[] = {
1810 {
1811 .compatible = "inside-secure,safexcel-eip97ies",
1812 .data = (void *)EIP97IES_MRVL,
1813 },
1814 {
1815 .compatible = "inside-secure,safexcel-eip197b",
1816 .data = (void *)EIP197B_MRVL,
1817 },
1818 {
1819 .compatible = "inside-secure,safexcel-eip197d",
1820 .data = (void *)EIP197D_MRVL,
1821 },
1822
1823 {
1824 .compatible = "inside-secure,safexcel-eip97",
1825 .data = (void *)EIP97IES_MRVL,
1826 },
1827 {
1828 .compatible = "inside-secure,safexcel-eip197",
1829 .data = (void *)EIP197B_MRVL,
1830 },
1831 {},
1832};
1833
1834static struct platform_driver crypto_safexcel = {
1835 .probe = safexcel_probe,
1836 .remove = safexcel_remove,
1837 .driver = {
1838 .name = "crypto-safexcel",
1839 .of_match_table = safexcel_of_match_table,
1840 },
1841};
1842
1843
1844
1845static int safexcel_pci_probe(struct pci_dev *pdev,
1846 const struct pci_device_id *ent)
1847{
1848 struct device *dev = &pdev->dev;
1849 struct safexcel_crypto_priv *priv;
1850 void __iomem *pciebase;
1851 int rc;
1852 u32 val;
1853
1854 dev_dbg(dev, "Probing PCIE device: vendor %04x, device %04x, subv %04x, subdev %04x, ctxt %lx\n",
1855 ent->vendor, ent->device, ent->subvendor,
1856 ent->subdevice, ent->driver_data);
1857
1858 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1859 if (!priv)
1860 return -ENOMEM;
1861
1862 priv->dev = dev;
1863 priv->version = (enum safexcel_eip_version)ent->driver_data;
1864
1865 pci_set_drvdata(pdev, priv);
1866
1867
1868 rc = pcim_enable_device(pdev);
1869 if (rc) {
1870 dev_err(dev, "Failed to enable PCI device\n");
1871 return rc;
1872 }
1873
1874
1875 rc = pcim_iomap_regions(pdev, 1, "crypto_safexcel");
1876 if (rc) {
1877 dev_err(dev, "Failed to map IO region for BAR0\n");
1878 return rc;
1879 }
1880 priv->base = pcim_iomap_table(pdev)[0];
1881
1882 if (priv->version == EIP197_DEVBRD) {
1883 dev_dbg(dev, "Device identified as FPGA based development board - applying HW reset\n");
1884
1885 rc = pcim_iomap_regions(pdev, 4, "crypto_safexcel");
1886 if (rc) {
1887 dev_err(dev, "Failed to map IO region for BAR4\n");
1888 return rc;
1889 }
1890
1891 pciebase = pcim_iomap_table(pdev)[2];
1892 val = readl(pciebase + EIP197_XLX_IRQ_BLOCK_ID_ADDR);
1893 if ((val >> 16) == EIP197_XLX_IRQ_BLOCK_ID_VALUE) {
1894 dev_dbg(dev, "Detected Xilinx PCIE IRQ block version %d, multiple MSI support enabled\n",
1895 (val & 0xff));
1896
1897
1898 writel(EIP197_XLX_USER_VECT_LUT0_IDENT,
1899 pciebase + EIP197_XLX_USER_VECT_LUT0_ADDR);
1900 writel(EIP197_XLX_USER_VECT_LUT1_IDENT,
1901 pciebase + EIP197_XLX_USER_VECT_LUT1_ADDR);
1902 writel(EIP197_XLX_USER_VECT_LUT2_IDENT,
1903 pciebase + EIP197_XLX_USER_VECT_LUT2_ADDR);
1904 writel(EIP197_XLX_USER_VECT_LUT3_IDENT,
1905 pciebase + EIP197_XLX_USER_VECT_LUT3_ADDR);
1906
1907
1908 writel(GENMASK(31, 0),
1909 pciebase + EIP197_XLX_USER_INT_ENB_MSK);
1910 } else {
1911 dev_err(dev, "Unrecognised IRQ block identifier %x\n",
1912 val);
1913 return -ENODEV;
1914 }
1915
1916
1917
1918 writel(1, priv->base + EIP197_XLX_GPIO_BASE);
1919 wmb();
1920
1921 writel(0, priv->base + EIP197_XLX_GPIO_BASE);
1922 wmb();
1923 }
1924
1925
1926 pci_set_master(pdev);
1927
1928
1929 rc = safexcel_probe_generic(pdev, priv, 1);
1930 return rc;
1931}
1932
1933static void safexcel_pci_remove(struct pci_dev *pdev)
1934{
1935 struct safexcel_crypto_priv *priv = pci_get_drvdata(pdev);
1936 int i;
1937
1938 safexcel_unregister_algorithms(priv);
1939
1940 for (i = 0; i < priv->config.rings; i++)
1941 destroy_workqueue(priv->ring[i].workqueue);
1942
1943 safexcel_hw_reset_rings(priv);
1944}
1945
1946static const struct pci_device_id safexcel_pci_ids[] = {
1947 {
1948 PCI_DEVICE_SUB(PCI_VENDOR_ID_XILINX, 0x9038,
1949 0x16ae, 0xc522),
1950 .driver_data = EIP197_DEVBRD,
1951 },
1952 {},
1953};
1954
1955MODULE_DEVICE_TABLE(pci, safexcel_pci_ids);
1956
1957static struct pci_driver safexcel_pci_driver = {
1958 .name = "crypto-safexcel",
1959 .id_table = safexcel_pci_ids,
1960 .probe = safexcel_pci_probe,
1961 .remove = safexcel_pci_remove,
1962};
1963
1964static int __init safexcel_init(void)
1965{
1966 int ret;
1967
1968
1969 ret = pci_register_driver(&safexcel_pci_driver);
1970
1971
1972 if (IS_ENABLED(CONFIG_OF) && !ret) {
1973 ret = platform_driver_register(&crypto_safexcel);
1974 if (ret)
1975 pci_unregister_driver(&safexcel_pci_driver);
1976 }
1977
1978 return ret;
1979}
1980
1981static void __exit safexcel_exit(void)
1982{
1983
1984 if (IS_ENABLED(CONFIG_OF))
1985 platform_driver_unregister(&crypto_safexcel);
1986
1987
1988 pci_unregister_driver(&safexcel_pci_driver);
1989}
1990
1991module_init(safexcel_init);
1992module_exit(safexcel_exit);
1993
1994MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
1995MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");
1996MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");
1997MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197");
1998MODULE_LICENSE("GPL v2");
1999MODULE_IMPORT_NS(CRYPTO_INTERNAL);
2000