1
2
3
4
5
6
7
8#include <linux/of_irq.h>
9#include <linux/of_address.h>
10
11#include "compat.h"
12#include "regs.h"
13#include "jr.h"
14#include "desc.h"
15#include "intern.h"
16
17struct jr_driver_data {
18
19 struct list_head jr_list;
20 spinlock_t jr_alloc_lock;
21} ____cacheline_aligned;
22
23static struct jr_driver_data driver_data;
24
25static int caam_reset_hw_jr(struct device *dev)
26{
27 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
28 unsigned int timeout = 100000;
29
30
31
32
33
34 setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
35
36
37 wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
38 while (((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) ==
39 JRINT_ERR_HALT_INPROGRESS) && --timeout)
40 cpu_relax();
41
42 if ((rd_reg32(&jrp->rregs->jrintstatus) & JRINT_ERR_HALT_MASK) !=
43 JRINT_ERR_HALT_COMPLETE || timeout == 0) {
44 dev_err(dev, "failed to flush job ring %d\n", jrp->ridx);
45 return -EIO;
46 }
47
48
49 timeout = 100000;
50 wr_reg32(&jrp->rregs->jrcommand, JRCR_RESET);
51 while ((rd_reg32(&jrp->rregs->jrcommand) & JRCR_RESET) && --timeout)
52 cpu_relax();
53
54 if (timeout == 0) {
55 dev_err(dev, "failed to reset job ring %d\n", jrp->ridx);
56 return -EIO;
57 }
58
59
60 clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
61
62 return 0;
63}
64
65
66
67
68int caam_jr_shutdown(struct device *dev)
69{
70 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
71 dma_addr_t inpbusaddr, outbusaddr;
72 int ret;
73
74 ret = caam_reset_hw_jr(dev);
75
76 tasklet_kill(&jrp->irqtask);
77
78
79 free_irq(jrp->irq, dev);
80
81
82 inpbusaddr = rd_reg64(&jrp->rregs->inpring_base);
83 outbusaddr = rd_reg64(&jrp->rregs->outring_base);
84 dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
85 jrp->inpring, inpbusaddr);
86 dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
87 jrp->outring, outbusaddr);
88 kfree(jrp->entinfo);
89
90 return ret;
91}
92
93static int caam_jr_remove(struct platform_device *pdev)
94{
95 int ret;
96 struct device *jrdev;
97 struct caam_drv_private_jr *jrpriv;
98
99 jrdev = &pdev->dev;
100 jrpriv = dev_get_drvdata(jrdev);
101
102
103
104
105 if (atomic_read(&jrpriv->tfm_count)) {
106 dev_err(jrdev, "Device is busy\n");
107 return -EBUSY;
108 }
109
110
111 spin_lock(&driver_data.jr_alloc_lock);
112 list_del(&jrpriv->list_node);
113 spin_unlock(&driver_data.jr_alloc_lock);
114
115
116 ret = caam_jr_shutdown(jrdev);
117 if (ret)
118 dev_err(jrdev, "Failed to shut down job ring\n");
119 irq_dispose_mapping(jrpriv->irq);
120
121 return ret;
122}
123
124
125static irqreturn_t caam_jr_interrupt(int irq, void *st_dev)
126{
127 struct device *dev = st_dev;
128 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
129 u32 irqstate;
130
131
132
133
134
135 irqstate = rd_reg32(&jrp->rregs->jrintstatus);
136 if (!irqstate)
137 return IRQ_NONE;
138
139
140
141
142
143
144 if (irqstate & JRINT_JR_ERROR) {
145 dev_err(dev, "job ring error: irqstate: %08x\n", irqstate);
146 BUG();
147 }
148
149
150 setbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
151
152
153 wr_reg32(&jrp->rregs->jrintstatus, irqstate);
154
155 preempt_disable();
156 tasklet_schedule(&jrp->irqtask);
157 preempt_enable();
158
159 return IRQ_HANDLED;
160}
161
162
163static void caam_jr_dequeue(unsigned long devarg)
164{
165 int hw_idx, sw_idx, i, head, tail;
166 struct device *dev = (struct device *)devarg;
167 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
168 void (*usercall)(struct device *dev, u32 *desc, u32 status, void *arg);
169 u32 *userdesc, userstatus;
170 void *userarg;
171
172 while (rd_reg32(&jrp->rregs->outring_used)) {
173
174 head = ACCESS_ONCE(jrp->head);
175
176 spin_lock(&jrp->outlock);
177
178 sw_idx = tail = jrp->tail;
179 hw_idx = jrp->out_ring_read_index;
180
181 for (i = 0; CIRC_CNT(head, tail + i, JOBR_DEPTH) >= 1; i++) {
182 sw_idx = (tail + i) & (JOBR_DEPTH - 1);
183
184 if (jrp->outring[hw_idx].desc ==
185 jrp->entinfo[sw_idx].desc_addr_dma)
186 break;
187 }
188
189 BUG_ON(CIRC_CNT(head, tail + i, JOBR_DEPTH) <= 0);
190
191
192 dma_unmap_single(dev, jrp->outring[hw_idx].desc,
193 jrp->entinfo[sw_idx].desc_size,
194 DMA_TO_DEVICE);
195
196
197 jrp->entinfo[sw_idx].desc_addr_dma = 0;
198
199
200 usercall = jrp->entinfo[sw_idx].callbk;
201 userarg = jrp->entinfo[sw_idx].cbkarg;
202 userdesc = jrp->entinfo[sw_idx].desc_addr_virt;
203 userstatus = jrp->outring[hw_idx].jrstatus;
204
205
206 wr_reg32(&jrp->rregs->outring_rmvd, 1);
207
208 jrp->out_ring_read_index = (jrp->out_ring_read_index + 1) &
209 (JOBR_DEPTH - 1);
210
211
212
213
214
215
216 if (sw_idx == tail) {
217 do {
218 tail = (tail + 1) & (JOBR_DEPTH - 1);
219 } while (CIRC_CNT(head, tail, JOBR_DEPTH) >= 1 &&
220 jrp->entinfo[tail].desc_addr_dma == 0);
221
222 jrp->tail = tail;
223 }
224
225 spin_unlock(&jrp->outlock);
226
227
228 usercall(dev, userdesc, userstatus, userarg);
229 }
230
231
232 clrbits32(&jrp->rregs->rconfig_lo, JRCFG_IMSK);
233}
234
235
236
237
238
239
240
241struct device *caam_jr_alloc(void)
242{
243 struct caam_drv_private_jr *jrpriv, *min_jrpriv = NULL;
244 struct device *dev = NULL;
245 int min_tfm_cnt = INT_MAX;
246 int tfm_cnt;
247
248 spin_lock(&driver_data.jr_alloc_lock);
249
250 if (list_empty(&driver_data.jr_list)) {
251 spin_unlock(&driver_data.jr_alloc_lock);
252 return ERR_PTR(-ENODEV);
253 }
254
255 list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) {
256 tfm_cnt = atomic_read(&jrpriv->tfm_count);
257 if (tfm_cnt < min_tfm_cnt) {
258 min_tfm_cnt = tfm_cnt;
259 min_jrpriv = jrpriv;
260 }
261 if (!min_tfm_cnt)
262 break;
263 }
264
265 if (min_jrpriv) {
266 atomic_inc(&min_jrpriv->tfm_count);
267 dev = min_jrpriv->dev;
268 }
269 spin_unlock(&driver_data.jr_alloc_lock);
270
271 return dev;
272}
273EXPORT_SYMBOL(caam_jr_alloc);
274
275
276
277
278
279
280void caam_jr_free(struct device *rdev)
281{
282 struct caam_drv_private_jr *jrpriv = dev_get_drvdata(rdev);
283
284 atomic_dec(&jrpriv->tfm_count);
285}
286EXPORT_SYMBOL(caam_jr_free);
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316int caam_jr_enqueue(struct device *dev, u32 *desc,
317 void (*cbk)(struct device *dev, u32 *desc,
318 u32 status, void *areq),
319 void *areq)
320{
321 struct caam_drv_private_jr *jrp = dev_get_drvdata(dev);
322 struct caam_jrentry_info *head_entry;
323 int head, tail, desc_size;
324 dma_addr_t desc_dma;
325
326 desc_size = (*desc & HDR_JD_LENGTH_MASK) * sizeof(u32);
327 desc_dma = dma_map_single(dev, desc, desc_size, DMA_TO_DEVICE);
328 if (dma_mapping_error(dev, desc_dma)) {
329 dev_err(dev, "caam_jr_enqueue(): can't map jobdesc\n");
330 return -EIO;
331 }
332
333 spin_lock_bh(&jrp->inplock);
334
335 head = jrp->head;
336 tail = ACCESS_ONCE(jrp->tail);
337
338 if (!rd_reg32(&jrp->rregs->inpring_avail) ||
339 CIRC_SPACE(head, tail, JOBR_DEPTH) <= 0) {
340 spin_unlock_bh(&jrp->inplock);
341 dma_unmap_single(dev, desc_dma, desc_size, DMA_TO_DEVICE);
342 return -EBUSY;
343 }
344
345 head_entry = &jrp->entinfo[head];
346 head_entry->desc_addr_virt = desc;
347 head_entry->desc_size = desc_size;
348 head_entry->callbk = (void *)cbk;
349 head_entry->cbkarg = areq;
350 head_entry->desc_addr_dma = desc_dma;
351
352 jrp->inpring[jrp->inp_ring_write_index] = desc_dma;
353
354 smp_wmb();
355
356 jrp->inp_ring_write_index = (jrp->inp_ring_write_index + 1) &
357 (JOBR_DEPTH - 1);
358 jrp->head = (head + 1) & (JOBR_DEPTH - 1);
359
360 wr_reg32(&jrp->rregs->inpring_jobadd, 1);
361
362 spin_unlock_bh(&jrp->inplock);
363
364 return 0;
365}
366EXPORT_SYMBOL(caam_jr_enqueue);
367
368
369
370
371static int caam_jr_init(struct device *dev)
372{
373 struct caam_drv_private_jr *jrp;
374 dma_addr_t inpbusaddr, outbusaddr;
375 int i, error;
376
377 jrp = dev_get_drvdata(dev);
378
379 tasklet_init(&jrp->irqtask, caam_jr_dequeue, (unsigned long)dev);
380
381
382 error = request_irq(jrp->irq, caam_jr_interrupt, IRQF_SHARED,
383 dev_name(dev), dev);
384 if (error) {
385 dev_err(dev, "can't connect JobR %d interrupt (%d)\n",
386 jrp->ridx, jrp->irq);
387 goto out_kill_deq;
388 }
389
390 error = caam_reset_hw_jr(dev);
391 if (error)
392 goto out_free_irq;
393
394 error = -ENOMEM;
395 jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
396 &inpbusaddr, GFP_KERNEL);
397 if (!jrp->inpring)
398 goto out_free_irq;
399
400 jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) *
401 JOBR_DEPTH, &outbusaddr, GFP_KERNEL);
402 if (!jrp->outring)
403 goto out_free_inpring;
404
405 jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH,
406 GFP_KERNEL);
407 if (!jrp->entinfo)
408 goto out_free_outring;
409
410 for (i = 0; i < JOBR_DEPTH; i++)
411 jrp->entinfo[i].desc_addr_dma = !0;
412
413
414 jrp->inp_ring_write_index = 0;
415 jrp->out_ring_read_index = 0;
416 jrp->head = 0;
417 jrp->tail = 0;
418
419 wr_reg64(&jrp->rregs->inpring_base, inpbusaddr);
420 wr_reg64(&jrp->rregs->outring_base, outbusaddr);
421 wr_reg32(&jrp->rregs->inpring_size, JOBR_DEPTH);
422 wr_reg32(&jrp->rregs->outring_size, JOBR_DEPTH);
423
424 jrp->ringsize = JOBR_DEPTH;
425
426 spin_lock_init(&jrp->inplock);
427 spin_lock_init(&jrp->outlock);
428
429
430 setbits32(&jrp->rregs->rconfig_lo, JOBR_INTC |
431 (JOBR_INTC_COUNT_THLD << JRCFG_ICDCT_SHIFT) |
432 (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT));
433
434 return 0;
435
436out_free_outring:
437 dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH,
438 jrp->outring, outbusaddr);
439out_free_inpring:
440 dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH,
441 jrp->inpring, inpbusaddr);
442 dev_err(dev, "can't allocate job rings for %d\n", jrp->ridx);
443out_free_irq:
444 free_irq(jrp->irq, dev);
445out_kill_deq:
446 tasklet_kill(&jrp->irqtask);
447 return error;
448}
449
450
451
452
453
454static int caam_jr_probe(struct platform_device *pdev)
455{
456 struct device *jrdev;
457 struct device_node *nprop;
458 struct caam_job_ring __iomem *ctrl;
459 struct caam_drv_private_jr *jrpriv;
460 static int total_jobrs;
461 int error;
462
463 jrdev = &pdev->dev;
464 jrpriv = devm_kmalloc(jrdev, sizeof(struct caam_drv_private_jr),
465 GFP_KERNEL);
466 if (!jrpriv)
467 return -ENOMEM;
468
469 dev_set_drvdata(jrdev, jrpriv);
470
471
472 jrpriv->ridx = total_jobrs++;
473
474 nprop = pdev->dev.of_node;
475
476
477 ctrl = of_iomap(nprop, 0);
478 if (!ctrl) {
479 dev_err(jrdev, "of_iomap() failed\n");
480 return -ENOMEM;
481 }
482
483 jrpriv->rregs = (struct caam_job_ring __force *)ctrl;
484
485 if (sizeof(dma_addr_t) == sizeof(u64))
486 if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring"))
487 dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(40));
488 else
489 dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(36));
490 else
491 dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32));
492
493
494 jrpriv->irq = irq_of_parse_and_map(nprop, 0);
495
496
497 error = caam_jr_init(jrdev);
498 if (error) {
499 irq_dispose_mapping(jrpriv->irq);
500 return error;
501 }
502
503 jrpriv->dev = jrdev;
504 spin_lock(&driver_data.jr_alloc_lock);
505 list_add_tail(&jrpriv->list_node, &driver_data.jr_list);
506 spin_unlock(&driver_data.jr_alloc_lock);
507
508 atomic_set(&jrpriv->tfm_count, 0);
509
510 return 0;
511}
512
513static struct of_device_id caam_jr_match[] = {
514 {
515 .compatible = "fsl,sec-v4.0-job-ring",
516 },
517 {
518 .compatible = "fsl,sec4.0-job-ring",
519 },
520 {},
521};
522MODULE_DEVICE_TABLE(of, caam_jr_match);
523
524static struct platform_driver caam_jr_driver = {
525 .driver = {
526 .name = "caam_jr",
527 .of_match_table = caam_jr_match,
528 },
529 .probe = caam_jr_probe,
530 .remove = caam_jr_remove,
531};
532
533static int __init jr_driver_init(void)
534{
535 spin_lock_init(&driver_data.jr_alloc_lock);
536 INIT_LIST_HEAD(&driver_data.jr_list);
537 return platform_driver_register(&caam_jr_driver);
538}
539
540static void __exit jr_driver_exit(void)
541{
542 platform_driver_unregister(&caam_jr_driver);
543}
544
545module_init(jr_driver_init);
546module_exit(jr_driver_exit);
547
548MODULE_LICENSE("GPL");
549MODULE_DESCRIPTION("FSL CAAM JR request backend");
550MODULE_AUTHOR("Freescale Semiconductor - NMG/STC");
551