1
2
3
4
5
6
7
8#include <linux/of_irq.h>
9#include <linux/of_address.h>
10
11#include "compat.h"
12#include "regs.h"
13#include "jr.h"
14#include "desc.h"
15#include "intern.h"
16
17struct jr_driver_data {
18
19 struct list_head jr_list;
20 spinlock_t jr_alloc_lock;
21} ____cacheline_aligned;
22
23static struct jr_driver_data driver_data;
24
25static int caam_reset_hw_jr(struct device *dev)
26{
27 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
28 unsigned int timeout = 100000;
29
30
31
32
33
34 clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK);
35
36
37 wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
38 while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
39 JRINT_ERR_HALT_INPROGRESS) && --timeout)
40 cpu_relax();
41
42 if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) !=
43 JRINT_ERR_HALT_COMPLETE || timeout == 0) {
44 dev_err(dev, "failed to flush job ring %d\n", jrp->ridx);
45 return -EIO;
46 }
47
48
49 timeout = 100000;
50 wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
51 while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
52 cpu_relax();
53
54 if (timeout == 0) {
55 dev_err(dev, "failed to reset job ring %d\n", jrp->ridx);
56 return -EIO;
57 }
58
59
60 clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
61
62 return 0;
63}
64
65
66
67
68static int caam_jr_shutdown(struct device *dev)
69{
70 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
71 dma_addr_t inpbusaddr, outbusaddr;
72 int ret;
73
74 ret = caam_reset_hw_jr(dev);
75
76 tasklet_kill(&jrp->irqtask);
77
78
79 free_irq(jrp->irq, dev);
80
81
82 inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
83 outbusaddr = rd_reg64(&jrp->rregs->outring_base);
84 dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
85 jrp->inpring, inpbusaddr);
86 dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
87 jrp->outring, outbusaddr);
88 kfree(jrp->entinfo);
89
90 return ret;
91}
92
93static int caam_jr_remove(struct platform_device *pdev)
94{
95 int ret;
96 struct device *jrdev;
97 struct caam_drv_private_jr *jrpriv;
98
99 jrdev = &pdev->dev;
100 jrpriv = dev_get_drvdata(jrdev);
101
102
103
104
105 if (atomic_read(&jrpriv->tfm_count)) {
106 dev_err(jrdev, "Device is busy\n");
107 return -EBUSY;
108 }
109
110
111 spin_lock(&driver_data.jr_alloc_lock);
112 list_del(&jrpriv->list_node);
113 spin_unlock(&driver_data.jr_alloc_lock);
114
115
116 ret = caam_jr_shutdown(jrdev);
117 if (ret)
118 dev_err(jrdev, "Failed to shut down job ring\n");
119 irq_dispose_mapping(jrpriv->irq);
120
121 return ret;
122}
123
124
125static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
126{
127 struct device *dev = st_dev;
128 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
129 u32 irqstate;
130
131
132
133
134
135 irqstate = rd_reg32(&jrp->rregs->jrintstatus);
136 if (!irqstate)
137 return IRQ_NONE;
138
139
140
141
142
143
144 if (irqstate & JRINT_JR_ERROR) {
145 dev_err(dev, "job ring error: irqstate: %08x\n", irqstate);
146 BUG();
147 }
148
149
150 clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JRCFG_IMSK);
151
152
153 wr_reg32(&jrp->rregs->jrintstatus, irqstate);
154
155 preempt_disable();
156 tasklet_schedule(&jrp->irqtask);
157 preempt_enable();
158
159 return IRQ_HANDLED;
160}
161
162
163static void caam_jr_dequeue(unsigned long devarg)
164{
165 int hw_idx, sw_idx, i, head, tail;
166 struct device *dev = (struct device *)devarg;
167 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
168 void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
169 u32 *userdesc, userstatus;
170 void *userarg;
171
172 while (rd_reg32(&jrp->rregs->outring_used)) {
173
174 head = ACCESS_ONCE(jrp->head);
175
176 spin_lock(&jrp->outlock);
177
178 sw_idx = tail = jrp->tail;
179 hw_idx = jrp->out_ring_read_index;
180
181 for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
182 sw_idx = (tail + i) & (JOBR_DEPTH - 1);
183
184 if (jrp->outring[hw_idx].desc ==
185 caam_dma_to_cpu(jrp->entinfo[sw_idx].desc_addr_dma))
186 break;
187 }
188
189 BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0);
190
191
192 dma_unmap_single(dev, jrp->outring[hw_idx].desc,
193 jrp->entinfo[sw_idx].desc_size,
194 DMA_TO_DEVICE);
195
196
197 jrp->entinfo[sw_idx].desc_addr_dma = 0;
198
199
200 usercall = jrp->entinfo[sw_idx].callbk;
201 userarg = jrp->entinfo[sw_idx].cbkarg;
202 userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
203 userstatus = caam32_to_cpu(jrp->outring[hw_idx].jrstatus);
204
205
206
207
208
209
210 mb();
211
212
213 wr_reg32(&jrp->rregs->outring_rmvd, 1);
214
215 jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) &
216 (JOBR_DEPTH - 1);
217
218
219
220
221
222
223 if (sw_idx == tail) {
224 do {
225 tail = (tail + 1) & (JOBR_DEPTH - 1);
226 } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
227 jrp->entinfo[tail].desc_addr_dma == 0);
228
229 jrp->tail = tail;
230 }
231
232 spin_unlock(&jrp->outlock);
233
234
235 usercall(dev, userdesc, userstatus, userarg);
236 }
237
238
239 clrsetbits_32(&jrp->rregs->rconfig_lo, JRCFG_IMSK, 0);
240}
241
242
243
244
245
246
247
248struct device *caam_jr_alloc(void)
249{
250 struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL;
251 struct device *dev = ERR_PTR(-ENODEV);
252 int min_tfm_cnt = INT_MAX;
253 int tfm_cnt;
254
255 spin_lock(&driver_data.jr_alloc_lock);
256
257 if (list_empty(&driver_data.jr_list)) {
258 spin_unlock(&driver_data.jr_alloc_lock);
259 return ERR_PTR(-ENODEV);
260 }
261
262 list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) {
263 tfm_cnt = atomic_read(&jrpriv->tfm_count);
264 if (tfm_cnt < min_tfm_cnt) {
265 min_tfm_cnt = tfm_cnt;
266 min_jrpriv = jrpriv;
267 }
268 if (!min_tfm_cnt)
269 break;
270 }
271
272 if (min_jrpriv) {
273 atomic_inc(&min_jrpriv->tfm_count);
274 dev = min_jrpriv->dev;
275 }
276 spin_unlock(&driver_data.jr_alloc_lock);
277
278 return dev;
279}
280EXPORT_SYMBOL(caam_jr_alloc);
281
282
283
284
285
286
287void caam_jr_free(struct device *rdev)
288{
289 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
290
291 atomic_dec(&jrpriv->tfm_count);
292}
293EXPORT_SYMBOL(caam_jr_free);
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323int caam_jr_enqueue(struct device *dev, u32 *desc,
324 void (*cbk)(struct device *dev, u32 *desc,
325 u32 status, void *areq),
326 void *areq)
327{
328 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
329 struct caam_jrentry_info *head_entry;
330 int head, tail, desc_size;
331 dma_addr_t desc_dma;
332
333 desc_size = (caam32_to_cpu(*desc) & HDR_JD_LENGTH_MASK) * sizeof(u32);
334 desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE);
335 if (dma_mapping_error(dev, desc_dma)) {
336 dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n");
337 return -EIO;
338 }
339
340 spin_lock_bh(&jrp->inplock);
341
342 head = jrp->head;
343 tail = ACCESS_ONCE(jrp->tail);
344
345 if (!rd_reg32(&jrp->rregs->inpring_avail) ||
346 CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) {
347 spin_unlock_bh(&jrp->inplock);
348 dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE);
349 return -EBUSY;
350 }
351
352 head_entry = &jrp->entinfo[head];
353 head_entry->desc_addr_virt = desc;
354 head_entry->desc_size = desc_size;
355 head_entry->callbk = (void *)cbk;
356 head_entry->cbkarg = areq;
357 head_entry->desc_addr_dma = desc_dma;
358
359 jrp->inpring[jrp->inp_ring_write_index] = cpu_to_caam_dma(desc_dma);
360
361
362
363
364
365
366 smp_wmb();
367
368 jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) &
369 (JOBR_DEPTH - 1);
370 jrp->head = (head + 1) & (JOBR_DEPTH - 1);
371
372
373
374
375
376 wmb();
377
378 wr_reg32(&jrp->rregs->inpring_jobadd, 1);
379
380 spin_unlock_bh(&jrp->inplock);
381
382 return 0;
383}
384EXPORT_SYMBOL(caam_jr_enqueue);
385
386
387
388
389static int caam_jr_init(struct device *dev)
390{
391 struct caam_drv_private_jr *jrp;
392 dma_addr_t inpbusaddr, outbusaddr;
393 int i, error;
394
395 jrp = dev_get_drvdata(dev);
396
397 tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
398
399
400 error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
401 dev_name(dev), dev);
402 if (error) {
403 dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
404 jrp->ridx, jrp->irq);
405 goto out_kill_deq;
406 }
407
408 error = caam_reset_hw_jr(dev);
409 if (error)
410 goto out_free_irq;
411
412 error = -ENOMEM;
413 jrp->inpring = dma_alloc_coherent(dev, sizeof(*jrp->inpring) *
414 JOBR_DEPTH, &inpbusaddr, GFP_KERNEL);
415 if (!jrp->inpring)
416 goto out_free_irq;
417
418 jrp->outring = dma_alloc_coherent(dev, sizeof(*jrp->outring) *
419 JOBR_DEPTH, &outbusaddr, GFP_KERNEL);
420 if (!jrp->outring)
421 goto out_free_inpring;
422
423 jrp->entinfo = kcalloc(JOBR_DEPTH, sizeof(*jrp->entinfo), GFP_KERNEL);
424 if (!jrp->entinfo)
425 goto out_free_outring;
426
427 for (i = 0; i < JOBR_DEPTH; i++)
428 jrp->entinfo[i].desc_addr_dma = !0;
429
430
431 jrp->inp_ring_write_index = 0;
432 jrp->out_ring_read_index = 0;
433 jrp->head = 0;
434 jrp->tail = 0;
435
436 wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
437 wr_reg64(&jrp->rregs->outring_base, outbusaddr);
438 wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
439 wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
440
441 jrp->ringsize = JOBR_DEPTH;
442
443 spin_lock_init(&jrp->inplock);
444 spin_lock_init(&jrp->outlock);
445
446
447 clrsetbits_32(&jrp->rregs->rconfig_lo, 0, JOBR_INTC |
448 (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
449 (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
450
451 return 0;
452
453out_free_outring:
454 dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
455 jrp->outring, outbusaddr);
456out_free_inpring:
457 dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
458 jrp->inpring, inpbusaddr);
459 dev_err(dev, "can't allocate job rings for %d\n", jrp->ridx);
460out_free_irq:
461 free_irq(jrp->irq, dev);
462out_kill_deq:
463 tasklet_kill(&jrp->irqtask);
464 return error;
465}
466
467
468
469
470
471static int caam_jr_probe(struct platform_device *pdev)
472{
473 struct device *jrdev;
474 struct device_node *nprop;
475 struct caam_job_ring __iomem *ctrl;
476 struct caam_drv_private_jr *jrpriv;
477 static int total_jobrs;
478 int error;
479
480 jrdev = &pdev->dev;
481 jrpriv = devm_kmalloc(jrdev, sizeof(*jrpriv), GFP_KERNEL);
482 if (!jrpriv)
483 return -ENOMEM;
484
485 dev_set_drvdata(jrdev, jrpriv);
486
487
488 jrpriv->ridx = total_jobrs++;
489
490 nprop = pdev->dev.of_node;
491
492
493 ctrl = of_iomap(nprop, 0);
494 if (!ctrl) {
495 dev_err(jrdev, "of_iomap() failed\n");
496 return -ENOMEM;
497 }
498
499 jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl;
500
501 if (sizeof(dma_addr_t) == sizeof(u64)) {
502 if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring"))
503 error = dma_set_mask_and_coherent(jrdev,
504 DMA_BIT_MASK(40));
505 else
506 error = dma_set_mask_and_coherent(jrdev,
507 DMA_BIT_MASK(36));
508 } else {
509 error = dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32));
510 }
511 if (error) {
512 dev_err(jrdev, "dma_set_mask_and_coherent failed (%d)\n",
513 error);
514 iounmap(ctrl);
515 return error;
516 }
517
518
519 jrpriv->irq = irq_of_parse_and_map(nprop, 0);
520
521
522 error = caam_jr_init(jrdev);
523 if (error) {
524 irq_dispose_mapping(jrpriv->irq);
525 iounmap(ctrl);
526 return error;
527 }
528
529 jrpriv->dev = jrdev;
530 spin_lock(&driver_data.jr_alloc_lock);
531 list_add_tail(&jrpriv->list_node, &driver_data.jr_list);
532 spin_unlock(&driver_data.jr_alloc_lock);
533
534 atomic_set(&jrpriv->tfm_count, 0);
535
536 return 0;
537}
538
539static struct of_device_id caam_jr_match[] = {
540 {
541 .compatible = "fsl,sec-v4.0-job-ring",
542 },
543 {
544 .compatible = "fsl,sec4.0-job-ring",
545 },
546 {},
547};
548MODULE_DEVICE_TABLE(of, caam_jr_match);
549
550static struct platform_driver caam_jr_driver = {
551 .driver = {
552 .name = "caam_jr",
553 .of_match_table = caam_jr_match,
554 },
555 .probe = caam_jr_probe,
556 .remove = caam_jr_remove,
557};
558
559static int __init jr_driver_init(void)
560{
561 spin_lock_init(&driver_data.jr_alloc_lock);
562 INIT_LIST_HEAD(&driver_data.jr_list);
563 return platform_driver_register(&caam_jr_driver);
564}
565
566static void __exit jr_driver_exit(void)
567{
568 platform_driver_unregister(&caam_jr_driver);
569}
570
571module_init(jr_driver_init);
572module_exit(jr_driver_exit);
573
574MODULE_LICENSE("GPL");
575MODULE_DESCRIPTION("FSL CAAM JR request backend");
576MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
577