1
2
3
4
5
6
7
8
9#include <linux/bitmap.h>
10#include <linux/delay.h>
11#include <linux/dma-mapping.h>
12#include <linux/err.h>
13#include <linux/export.h>
14#include <linux/init.h>
15#include <linux/interrupt.h>
16#include <linux/io.h>
17#include <linux/io-pgtable.h>
18#include <linux/iommu.h>
19#include <linux/of.h>
20#include <linux/of_device.h>
21#include <linux/of_platform.h>
22#include <linux/platform_device.h>
23#include <linux/sizes.h>
24#include <linux/slab.h>
25#include <linux/sys_soc.h>
26
27#if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA)
28#include <asm/dma-iommu.h>
29#else
30#define arm_iommu_create_mapping(...) NULL
31#define arm_iommu_attach_device(...) -ENODEV
32#define arm_iommu_release_mapping(...) do {} while (0)
33#define arm_iommu_detach_device(...) do {} while (0)
34#endif
35
36#define IPMMU_CTX_MAX 8U
37#define IPMMU_CTX_INVALID -1
38
39#define IPMMU_UTLB_MAX 48U
40
41struct ipmmu_features {
42 bool use_ns_alias_offset;
43 bool has_cache_leaf_nodes;
44 unsigned int number_of_contexts;
45 unsigned int num_utlbs;
46 bool setup_imbuscr;
47 bool twobit_imttbcr_sl0;
48 bool reserved_context;
49 bool cache_snoop;
50 unsigned int ctx_offset_base;
51 unsigned int ctx_offset_stride;
52 unsigned int utlb_offset_base;
53};
54
55struct ipmmu_vmsa_device {
56 struct device *dev;
57 void __iomem *base;
58 struct iommu_device iommu;
59 struct ipmmu_vmsa_device *root;
60 const struct ipmmu_features *features;
61 unsigned int num_ctx;
62 spinlock_t lock;
63 DECLARE_BITMAP(ctx, IPMMU_CTX_MAX);
64 struct ipmmu_vmsa_domain *domains[IPMMU_CTX_MAX];
65 s8 utlb_ctx[IPMMU_UTLB_MAX];
66
67 struct iommu_group *group;
68 struct dma_iommu_mapping *mapping;
69};
70
71struct ipmmu_vmsa_domain {
72 struct ipmmu_vmsa_device *mmu;
73 struct iommu_domain io_domain;
74
75 struct io_pgtable_cfg cfg;
76 struct io_pgtable_ops *iop;
77
78 unsigned int context_id;
79 struct mutex mutex;
80};
81
82static struct ipmmu_vmsa_domain *to_vmsa_domain(struct iommu_domain *dom)
83{
84 return container_of(dom, struct ipmmu_vmsa_domain, io_domain);
85}
86
87static struct ipmmu_vmsa_device *to_ipmmu(struct device *dev)
88{
89 return dev_iommu_priv_get(dev);
90}
91
92#define TLB_LOOP_TIMEOUT 100
93
94
95
96
97
98#define IM_NS_ALIAS_OFFSET 0x800
99
100
101#define IMCTR 0x0000
102#define IMCTR_INTEN (1 << 2)
103#define IMCTR_FLUSH (1 << 1)
104#define IMCTR_MMUEN (1 << 0)
105
106#define IMTTBCR 0x0008
107#define IMTTBCR_EAE (1 << 31)
108#define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12)
109#define IMTTBCR_ORGN0_WB_WA (1 << 10)
110#define IMTTBCR_IRGN0_WB_WA (1 << 8)
111#define IMTTBCR_SL0_TWOBIT_LVL_1 (2 << 6)
112#define IMTTBCR_SL0_LVL_1 (1 << 4)
113
114#define IMBUSCR 0x000c
115#define IMBUSCR_DVM (1 << 2)
116#define IMBUSCR_BUSSEL_MASK (3 << 0)
117
118#define IMTTLBR0 0x0010
119#define IMTTUBR0 0x0014
120
121#define IMSTR 0x0020
122#define IMSTR_MHIT (1 << 4)
123#define IMSTR_ABORT (1 << 2)
124#define IMSTR_PF (1 << 1)
125#define IMSTR_TF (1 << 0)
126
127#define IMMAIR0 0x0028
128
129#define IMELAR 0x0030
130#define IMEUAR 0x0034
131
132
133#define IMUCTR(n) ((n) < 32 ? IMUCTR0(n) : IMUCTR32(n))
134#define IMUCTR0(n) (0x0300 + ((n) * 16))
135#define IMUCTR32(n) (0x0600 + (((n) - 32) * 16))
136#define IMUCTR_TTSEL_MMU(n) ((n) << 4)
137#define IMUCTR_FLUSH (1 << 1)
138#define IMUCTR_MMUEN (1 << 0)
139
140#define IMUASID(n) ((n) < 32 ? IMUASID0(n) : IMUASID32(n))
141#define IMUASID0(n) (0x0308 + ((n) * 16))
142#define IMUASID32(n) (0x0608 + (((n) - 32) * 16))
143
144
145
146
147
148static struct platform_driver ipmmu_driver;
149
150static bool ipmmu_is_root(struct ipmmu_vmsa_device *mmu)
151{
152 return mmu->root == mmu;
153}
154
155static int __ipmmu_check_device(struct device *dev, void *data)
156{
157 struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev);
158 struct ipmmu_vmsa_device **rootp = data;
159
160 if (ipmmu_is_root(mmu))
161 *rootp = mmu;
162
163 return 0;
164}
165
166static struct ipmmu_vmsa_device *ipmmu_find_root(void)
167{
168 struct ipmmu_vmsa_device *root = NULL;
169
170 return driver_for_each_device(&ipmmu_driver.driver, NULL, &root,
171 __ipmmu_check_device) == 0 ? root : NULL;
172}
173
174
175
176
177
178static u32 ipmmu_read(struct ipmmu_vmsa_device *mmu, unsigned int offset)
179{
180 return ioread32(mmu->base + offset);
181}
182
183static void ipmmu_write(struct ipmmu_vmsa_device *mmu, unsigned int offset,
184 u32 data)
185{
186 iowrite32(data, mmu->base + offset);
187}
188
189static unsigned int ipmmu_ctx_reg(struct ipmmu_vmsa_device *mmu,
190 unsigned int context_id, unsigned int reg)
191{
192 return mmu->features->ctx_offset_base +
193 context_id * mmu->features->ctx_offset_stride + reg;
194}
195
196static u32 ipmmu_ctx_read(struct ipmmu_vmsa_device *mmu,
197 unsigned int context_id, unsigned int reg)
198{
199 return ipmmu_read(mmu, ipmmu_ctx_reg(mmu, context_id, reg));
200}
201
202static void ipmmu_ctx_write(struct ipmmu_vmsa_device *mmu,
203 unsigned int context_id, unsigned int reg, u32 data)
204{
205 ipmmu_write(mmu, ipmmu_ctx_reg(mmu, context_id, reg), data);
206}
207
208static u32 ipmmu_ctx_read_root(struct ipmmu_vmsa_domain *domain,
209 unsigned int reg)
210{
211 return ipmmu_ctx_read(domain->mmu->root, domain->context_id, reg);
212}
213
214static void ipmmu_ctx_write_root(struct ipmmu_vmsa_domain *domain,
215 unsigned int reg, u32 data)
216{
217 ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);
218}
219
220static void ipmmu_ctx_write_all(struct ipmmu_vmsa_domain *domain,
221 unsigned int reg, u32 data)
222{
223 if (domain->mmu != domain->mmu->root)
224 ipmmu_ctx_write(domain->mmu, domain->context_id, reg, data);
225
226 ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data);
227}
228
229static u32 ipmmu_utlb_reg(struct ipmmu_vmsa_device *mmu, unsigned int reg)
230{
231 return mmu->features->utlb_offset_base + reg;
232}
233
234static void ipmmu_imuasid_write(struct ipmmu_vmsa_device *mmu,
235 unsigned int utlb, u32 data)
236{
237 ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUASID(utlb)), data);
238}
239
240static void ipmmu_imuctr_write(struct ipmmu_vmsa_device *mmu,
241 unsigned int utlb, u32 data)
242{
243 ipmmu_write(mmu, ipmmu_utlb_reg(mmu, IMUCTR(utlb)), data);
244}
245
246
247
248
249
250
251static void ipmmu_tlb_sync(struct ipmmu_vmsa_domain *domain)
252{
253 unsigned int count = 0;
254
255 while (ipmmu_ctx_read_root(domain, IMCTR) & IMCTR_FLUSH) {
256 cpu_relax();
257 if (++count == TLB_LOOP_TIMEOUT) {
258 dev_err_ratelimited(domain->mmu->dev,
259 "TLB sync timed out -- MMU may be deadlocked\n");
260 return;
261 }
262 udelay(1);
263 }
264}
265
266static void ipmmu_tlb_invalidate(struct ipmmu_vmsa_domain *domain)
267{
268 u32 reg;
269
270 reg = ipmmu_ctx_read_root(domain, IMCTR);
271 reg |= IMCTR_FLUSH;
272 ipmmu_ctx_write_all(domain, IMCTR, reg);
273
274 ipmmu_tlb_sync(domain);
275}
276
277
278
279
280static void ipmmu_utlb_enable(struct ipmmu_vmsa_domain *domain,
281 unsigned int utlb)
282{
283 struct ipmmu_vmsa_device *mmu = domain->mmu;
284
285
286
287
288
289
290
291 ipmmu_imuasid_write(mmu, utlb, 0);
292
293 ipmmu_imuctr_write(mmu, utlb, IMUCTR_TTSEL_MMU(domain->context_id) |
294 IMUCTR_FLUSH | IMUCTR_MMUEN);
295 mmu->utlb_ctx[utlb] = domain->context_id;
296}
297
298
299
300
301static void ipmmu_utlb_disable(struct ipmmu_vmsa_domain *domain,
302 unsigned int utlb)
303{
304 struct ipmmu_vmsa_device *mmu = domain->mmu;
305
306 ipmmu_imuctr_write(mmu, utlb, 0);
307 mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID;
308}
309
310static void ipmmu_tlb_flush_all(void *cookie)
311{
312 struct ipmmu_vmsa_domain *domain = cookie;
313
314 ipmmu_tlb_invalidate(domain);
315}
316
317static void ipmmu_tlb_flush(unsigned long iova, size_t size,
318 size_t granule, void *cookie)
319{
320 ipmmu_tlb_flush_all(cookie);
321}
322
323static const struct iommu_flush_ops ipmmu_flush_ops = {
324 .tlb_flush_all = ipmmu_tlb_flush_all,
325 .tlb_flush_walk = ipmmu_tlb_flush,
326};
327
328
329
330
331
332static int ipmmu_domain_allocate_context(struct ipmmu_vmsa_device *mmu,
333 struct ipmmu_vmsa_domain *domain)
334{
335 unsigned long flags;
336 int ret;
337
338 spin_lock_irqsave(&mmu->lock, flags);
339
340 ret = find_first_zero_bit(mmu->ctx, mmu->num_ctx);
341 if (ret != mmu->num_ctx) {
342 mmu->domains[ret] = domain;
343 set_bit(ret, mmu->ctx);
344 } else
345 ret = -EBUSY;
346
347 spin_unlock_irqrestore(&mmu->lock, flags);
348
349 return ret;
350}
351
352static void ipmmu_domain_free_context(struct ipmmu_vmsa_device *mmu,
353 unsigned int context_id)
354{
355 unsigned long flags;
356
357 spin_lock_irqsave(&mmu->lock, flags);
358
359 clear_bit(context_id, mmu->ctx);
360 mmu->domains[context_id] = NULL;
361
362 spin_unlock_irqrestore(&mmu->lock, flags);
363}
364
365static void ipmmu_domain_setup_context(struct ipmmu_vmsa_domain *domain)
366{
367 u64 ttbr;
368 u32 tmp;
369
370
371 ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr;
372 ipmmu_ctx_write_root(domain, IMTTLBR0, ttbr);
373 ipmmu_ctx_write_root(domain, IMTTUBR0, ttbr >> 32);
374
375
376
377
378
379
380 if (domain->mmu->features->twobit_imttbcr_sl0)
381 tmp = IMTTBCR_SL0_TWOBIT_LVL_1;
382 else
383 tmp = IMTTBCR_SL0_LVL_1;
384
385 if (domain->mmu->features->cache_snoop)
386 tmp |= IMTTBCR_SH0_INNER_SHAREABLE | IMTTBCR_ORGN0_WB_WA |
387 IMTTBCR_IRGN0_WB_WA;
388
389 ipmmu_ctx_write_root(domain, IMTTBCR, IMTTBCR_EAE | tmp);
390
391
392 ipmmu_ctx_write_root(domain, IMMAIR0,
393 domain->cfg.arm_lpae_s1_cfg.mair);
394
395
396 if (domain->mmu->features->setup_imbuscr)
397 ipmmu_ctx_write_root(domain, IMBUSCR,
398 ipmmu_ctx_read_root(domain, IMBUSCR) &
399 ~(IMBUSCR_DVM | IMBUSCR_BUSSEL_MASK));
400
401
402
403
404
405 ipmmu_ctx_write_root(domain, IMSTR, ipmmu_ctx_read_root(domain, IMSTR));
406
407
408
409
410
411
412
413
414 ipmmu_ctx_write_all(domain, IMCTR,
415 IMCTR_INTEN | IMCTR_FLUSH | IMCTR_MMUEN);
416}
417
418static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
419{
420 int ret;
421
422
423
424
425
426
427
428
429
430
431
432
433 domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS;
434 domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K;
435 domain->cfg.ias = 32;
436 domain->cfg.oas = 40;
437 domain->cfg.tlb = &ipmmu_flush_ops;
438 domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32);
439 domain->io_domain.geometry.force_aperture = true;
440
441
442
443
444 domain->cfg.coherent_walk = false;
445 domain->cfg.iommu_dev = domain->mmu->root->dev;
446
447
448
449
450 ret = ipmmu_domain_allocate_context(domain->mmu->root, domain);
451 if (ret < 0)
452 return ret;
453
454 domain->context_id = ret;
455
456 domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg,
457 domain);
458 if (!domain->iop) {
459 ipmmu_domain_free_context(domain->mmu->root,
460 domain->context_id);
461 return -EINVAL;
462 }
463
464 ipmmu_domain_setup_context(domain);
465 return 0;
466}
467
468static void ipmmu_domain_destroy_context(struct ipmmu_vmsa_domain *domain)
469{
470 if (!domain->mmu)
471 return;
472
473
474
475
476
477
478
479 ipmmu_ctx_write_all(domain, IMCTR, IMCTR_FLUSH);
480 ipmmu_tlb_sync(domain);
481 ipmmu_domain_free_context(domain->mmu->root, domain->context_id);
482}
483
484
485
486
487
488static irqreturn_t ipmmu_domain_irq(struct ipmmu_vmsa_domain *domain)
489{
490 const u32 err_mask = IMSTR_MHIT | IMSTR_ABORT | IMSTR_PF | IMSTR_TF;
491 struct ipmmu_vmsa_device *mmu = domain->mmu;
492 unsigned long iova;
493 u32 status;
494
495 status = ipmmu_ctx_read_root(domain, IMSTR);
496 if (!(status & err_mask))
497 return IRQ_NONE;
498
499 iova = ipmmu_ctx_read_root(domain, IMELAR);
500 if (IS_ENABLED(CONFIG_64BIT))
501 iova |= (u64)ipmmu_ctx_read_root(domain, IMEUAR) << 32;
502
503
504
505
506
507
508
509 ipmmu_ctx_write_root(domain, IMSTR, 0);
510
511
512 if (status & IMSTR_MHIT)
513 dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%lx\n",
514 iova);
515 if (status & IMSTR_ABORT)
516 dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%lx\n",
517 iova);
518
519 if (!(status & (IMSTR_PF | IMSTR_TF)))
520 return IRQ_NONE;
521
522
523
524
525
526
527
528 if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0))
529 return IRQ_HANDLED;
530
531 dev_err_ratelimited(mmu->dev,
532 "Unhandled fault: status 0x%08x iova 0x%lx\n",
533 status, iova);
534
535 return IRQ_HANDLED;
536}
537
538static irqreturn_t ipmmu_irq(int irq, void *dev)
539{
540 struct ipmmu_vmsa_device *mmu = dev;
541 irqreturn_t status = IRQ_NONE;
542 unsigned int i;
543 unsigned long flags;
544
545 spin_lock_irqsave(&mmu->lock, flags);
546
547
548
549
550 for (i = 0; i < mmu->num_ctx; i++) {
551 if (!mmu->domains[i])
552 continue;
553 if (ipmmu_domain_irq(mmu->domains[i]) == IRQ_HANDLED)
554 status = IRQ_HANDLED;
555 }
556
557 spin_unlock_irqrestore(&mmu->lock, flags);
558
559 return status;
560}
561
562
563
564
565
566static struct iommu_domain *ipmmu_domain_alloc(unsigned type)
567{
568 struct ipmmu_vmsa_domain *domain;
569
570 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA)
571 return NULL;
572
573 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
574 if (!domain)
575 return NULL;
576
577 mutex_init(&domain->mutex);
578
579 return &domain->io_domain;
580}
581
582static void ipmmu_domain_free(struct iommu_domain *io_domain)
583{
584 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
585
586
587
588
589
590 ipmmu_domain_destroy_context(domain);
591 free_io_pgtable_ops(domain->iop);
592 kfree(domain);
593}
594
595static int ipmmu_attach_device(struct iommu_domain *io_domain,
596 struct device *dev)
597{
598 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
599 struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
600 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
601 unsigned int i;
602 int ret = 0;
603
604 if (!mmu) {
605 dev_err(dev, "Cannot attach to IPMMU\n");
606 return -ENXIO;
607 }
608
609 mutex_lock(&domain->mutex);
610
611 if (!domain->mmu) {
612
613 domain->mmu = mmu;
614 ret = ipmmu_domain_init_context(domain);
615 if (ret < 0) {
616 dev_err(dev, "Unable to initialize IPMMU context\n");
617 domain->mmu = NULL;
618 } else {
619 dev_info(dev, "Using IPMMU context %u\n",
620 domain->context_id);
621 }
622 } else if (domain->mmu != mmu) {
623
624
625
626
627 dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n",
628 dev_name(mmu->dev), dev_name(domain->mmu->dev));
629 ret = -EINVAL;
630 } else
631 dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id);
632
633 mutex_unlock(&domain->mutex);
634
635 if (ret < 0)
636 return ret;
637
638 for (i = 0; i < fwspec->num_ids; ++i)
639 ipmmu_utlb_enable(domain, fwspec->ids[i]);
640
641 return 0;
642}
643
644static void ipmmu_detach_device(struct iommu_domain *io_domain,
645 struct device *dev)
646{
647 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
648 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
649 unsigned int i;
650
651 for (i = 0; i < fwspec->num_ids; ++i)
652 ipmmu_utlb_disable(domain, fwspec->ids[i]);
653
654
655
656
657}
658
659static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
660 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
661{
662 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
663
664 if (!domain)
665 return -ENODEV;
666
667 return domain->iop->map(domain->iop, iova, paddr, size, prot, gfp);
668}
669
670static size_t ipmmu_unmap(struct iommu_domain *io_domain, unsigned long iova,
671 size_t size, struct iommu_iotlb_gather *gather)
672{
673 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
674
675 return domain->iop->unmap(domain->iop, iova, size, gather);
676}
677
678static void ipmmu_flush_iotlb_all(struct iommu_domain *io_domain)
679{
680 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
681
682 if (domain->mmu)
683 ipmmu_tlb_flush_all(domain);
684}
685
686static void ipmmu_iotlb_sync(struct iommu_domain *io_domain,
687 struct iommu_iotlb_gather *gather)
688{
689 ipmmu_flush_iotlb_all(io_domain);
690}
691
692static phys_addr_t ipmmu_iova_to_phys(struct iommu_domain *io_domain,
693 dma_addr_t iova)
694{
695 struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
696
697
698
699 return domain->iop->iova_to_phys(domain->iop, iova);
700}
701
702static int ipmmu_init_platform_device(struct device *dev,
703 struct of_phandle_args *args)
704{
705 struct platform_device *ipmmu_pdev;
706
707 ipmmu_pdev = of_find_device_by_node(args->np);
708 if (!ipmmu_pdev)
709 return -ENODEV;
710
711 dev_iommu_priv_set(dev, platform_get_drvdata(ipmmu_pdev));
712
713 return 0;
714}
715
716static const struct soc_device_attribute soc_needs_opt_in[] = {
717 { .family = "R-Car Gen3", },
718 { .family = "RZ/G2", },
719 { }
720};
721
722static const struct soc_device_attribute soc_denylist[] = {
723 { .soc_id = "r8a774a1", },
724 { .soc_id = "r8a7795", .revision = "ES1.*" },
725 { .soc_id = "r8a7795", .revision = "ES2.*" },
726 { .soc_id = "r8a7796", },
727 { }
728};
729
730static const char * const devices_allowlist[] = {
731 "ee100000.mmc",
732 "ee120000.mmc",
733 "ee140000.mmc",
734 "ee160000.mmc"
735};
736
737static bool ipmmu_device_is_allowed(struct device *dev)
738{
739 unsigned int i;
740
741
742
743
744
745 if (!soc_device_match(soc_needs_opt_in))
746 return true;
747
748
749 if (soc_device_match(soc_denylist))
750 return false;
751
752
753 for (i = 0; i < ARRAY_SIZE(devices_allowlist); i++) {
754 if (!strcmp(dev_name(dev), devices_allowlist[i]))
755 return true;
756 }
757
758
759 return false;
760}
761
762static int ipmmu_of_xlate(struct device *dev,
763 struct of_phandle_args *spec)
764{
765 if (!ipmmu_device_is_allowed(dev))
766 return -ENODEV;
767
768 iommu_fwspec_add_ids(dev, spec->args, 1);
769
770
771 if (to_ipmmu(dev))
772 return 0;
773
774 return ipmmu_init_platform_device(dev, spec);
775}
776
777static int ipmmu_init_arm_mapping(struct device *dev)
778{
779 struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
780 int ret;
781
782
783
784
785
786
787
788
789
790
791 if (!mmu->mapping) {
792 struct dma_iommu_mapping *mapping;
793
794 mapping = arm_iommu_create_mapping(&platform_bus_type,
795 SZ_1G, SZ_2G);
796 if (IS_ERR(mapping)) {
797 dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n");
798 ret = PTR_ERR(mapping);
799 goto error;
800 }
801
802 mmu->mapping = mapping;
803 }
804
805
806 ret = arm_iommu_attach_device(dev, mmu->mapping);
807 if (ret < 0) {
808 dev_err(dev, "Failed to attach device to VA mapping\n");
809 goto error;
810 }
811
812 return 0;
813
814error:
815 if (mmu->mapping)
816 arm_iommu_release_mapping(mmu->mapping);
817
818 return ret;
819}
820
821static struct iommu_device *ipmmu_probe_device(struct device *dev)
822{
823 struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
824
825
826
827
828 if (!mmu)
829 return ERR_PTR(-ENODEV);
830
831 return &mmu->iommu;
832}
833
834static void ipmmu_probe_finalize(struct device *dev)
835{
836 int ret = 0;
837
838 if (IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA))
839 ret = ipmmu_init_arm_mapping(dev);
840
841 if (ret)
842 dev_err(dev, "Can't create IOMMU mapping - DMA-OPS will not work\n");
843}
844
845static void ipmmu_release_device(struct device *dev)
846{
847 arm_iommu_detach_device(dev);
848}
849
850static struct iommu_group *ipmmu_find_group(struct device *dev)
851{
852 struct ipmmu_vmsa_device *mmu = to_ipmmu(dev);
853 struct iommu_group *group;
854
855 if (mmu->group)
856 return iommu_group_ref_get(mmu->group);
857
858 group = iommu_group_alloc();
859 if (!IS_ERR(group))
860 mmu->group = group;
861
862 return group;
863}
864
865static const struct iommu_ops ipmmu_ops = {
866 .domain_alloc = ipmmu_domain_alloc,
867 .domain_free = ipmmu_domain_free,
868 .attach_dev = ipmmu_attach_device,
869 .detach_dev = ipmmu_detach_device,
870 .map = ipmmu_map,
871 .unmap = ipmmu_unmap,
872 .flush_iotlb_all = ipmmu_flush_iotlb_all,
873 .iotlb_sync = ipmmu_iotlb_sync,
874 .iova_to_phys = ipmmu_iova_to_phys,
875 .probe_device = ipmmu_probe_device,
876 .release_device = ipmmu_release_device,
877 .probe_finalize = ipmmu_probe_finalize,
878 .device_group = IS_ENABLED(CONFIG_ARM) && !IS_ENABLED(CONFIG_IOMMU_DMA)
879 ? generic_device_group : ipmmu_find_group,
880 .pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K,
881 .of_xlate = ipmmu_of_xlate,
882};
883
884
885
886
887
888static void ipmmu_device_reset(struct ipmmu_vmsa_device *mmu)
889{
890 unsigned int i;
891
892
893 for (i = 0; i < mmu->num_ctx; ++i)
894 ipmmu_ctx_write(mmu, i, IMCTR, 0);
895}
896
897static const struct ipmmu_features ipmmu_features_default = {
898 .use_ns_alias_offset = true,
899 .has_cache_leaf_nodes = false,
900 .number_of_contexts = 1,
901 .num_utlbs = 32,
902 .setup_imbuscr = true,
903 .twobit_imttbcr_sl0 = false,
904 .reserved_context = false,
905 .cache_snoop = true,
906 .ctx_offset_base = 0,
907 .ctx_offset_stride = 0x40,
908 .utlb_offset_base = 0,
909};
910
911static const struct ipmmu_features ipmmu_features_rcar_gen3 = {
912 .use_ns_alias_offset = false,
913 .has_cache_leaf_nodes = true,
914 .number_of_contexts = 8,
915 .num_utlbs = 48,
916 .setup_imbuscr = false,
917 .twobit_imttbcr_sl0 = true,
918 .reserved_context = true,
919 .cache_snoop = false,
920 .ctx_offset_base = 0,
921 .ctx_offset_stride = 0x40,
922 .utlb_offset_base = 0,
923};
924
925static const struct of_device_id ipmmu_of_ids[] = {
926 {
927 .compatible = "renesas,ipmmu-vmsa",
928 .data = &ipmmu_features_default,
929 }, {
930 .compatible = "renesas,ipmmu-r8a774a1",
931 .data = &ipmmu_features_rcar_gen3,
932 }, {
933 .compatible = "renesas,ipmmu-r8a774b1",
934 .data = &ipmmu_features_rcar_gen3,
935 }, {
936 .compatible = "renesas,ipmmu-r8a774c0",
937 .data = &ipmmu_features_rcar_gen3,
938 }, {
939 .compatible = "renesas,ipmmu-r8a774e1",
940 .data = &ipmmu_features_rcar_gen3,
941 }, {
942 .compatible = "renesas,ipmmu-r8a7795",
943 .data = &ipmmu_features_rcar_gen3,
944 }, {
945 .compatible = "renesas,ipmmu-r8a7796",
946 .data = &ipmmu_features_rcar_gen3,
947 }, {
948 .compatible = "renesas,ipmmu-r8a77961",
949 .data = &ipmmu_features_rcar_gen3,
950 }, {
951 .compatible = "renesas,ipmmu-r8a77965",
952 .data = &ipmmu_features_rcar_gen3,
953 }, {
954 .compatible = "renesas,ipmmu-r8a77970",
955 .data = &ipmmu_features_rcar_gen3,
956 }, {
957 .compatible = "renesas,ipmmu-r8a77990",
958 .data = &ipmmu_features_rcar_gen3,
959 }, {
960 .compatible = "renesas,ipmmu-r8a77995",
961 .data = &ipmmu_features_rcar_gen3,
962 }, {
963
964 },
965};
966
967static int ipmmu_probe(struct platform_device *pdev)
968{
969 struct ipmmu_vmsa_device *mmu;
970 struct resource *res;
971 int irq;
972 int ret;
973
974 mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL);
975 if (!mmu) {
976 dev_err(&pdev->dev, "cannot allocate device data\n");
977 return -ENOMEM;
978 }
979
980 mmu->dev = &pdev->dev;
981 spin_lock_init(&mmu->lock);
982 bitmap_zero(mmu->ctx, IPMMU_CTX_MAX);
983 mmu->features = of_device_get_match_data(&pdev->dev);
984 memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs);
985 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40));
986
987
988 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
989 mmu->base = devm_ioremap_resource(&pdev->dev, res);
990 if (IS_ERR(mmu->base))
991 return PTR_ERR(mmu->base);
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005 if (mmu->features->use_ns_alias_offset)
1006 mmu->base += IM_NS_ALIAS_OFFSET;
1007
1008 mmu->num_ctx = min(IPMMU_CTX_MAX, mmu->features->number_of_contexts);
1009
1010
1011
1012
1013
1014 if (!mmu->features->has_cache_leaf_nodes ||
1015 !of_find_property(pdev->dev.of_node, "renesas,ipmmu-main", NULL))
1016 mmu->root = mmu;
1017 else
1018 mmu->root = ipmmu_find_root();
1019
1020
1021
1022
1023 if (!mmu->root)
1024 return -EPROBE_DEFER;
1025
1026
1027 if (ipmmu_is_root(mmu)) {
1028 irq = platform_get_irq(pdev, 0);
1029 if (irq < 0)
1030 return irq;
1031
1032 ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0,
1033 dev_name(&pdev->dev), mmu);
1034 if (ret < 0) {
1035 dev_err(&pdev->dev, "failed to request IRQ %d\n", irq);
1036 return ret;
1037 }
1038
1039 ipmmu_device_reset(mmu);
1040
1041 if (mmu->features->reserved_context) {
1042 dev_info(&pdev->dev, "IPMMU context 0 is reserved\n");
1043 set_bit(0, mmu->ctx);
1044 }
1045 }
1046
1047
1048
1049
1050
1051
1052 if (!mmu->features->has_cache_leaf_nodes || !ipmmu_is_root(mmu)) {
1053 ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL,
1054 dev_name(&pdev->dev));
1055 if (ret)
1056 return ret;
1057
1058 ret = iommu_device_register(&mmu->iommu, &ipmmu_ops, &pdev->dev);
1059 if (ret)
1060 return ret;
1061
1062#if defined(CONFIG_IOMMU_DMA)
1063 if (!iommu_present(&platform_bus_type))
1064 bus_set_iommu(&platform_bus_type, &ipmmu_ops);
1065#endif
1066 }
1067
1068
1069
1070
1071
1072
1073
1074 platform_set_drvdata(pdev, mmu);
1075
1076 return 0;
1077}
1078
1079static int ipmmu_remove(struct platform_device *pdev)
1080{
1081 struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev);
1082
1083 iommu_device_sysfs_remove(&mmu->iommu);
1084 iommu_device_unregister(&mmu->iommu);
1085
1086 arm_iommu_release_mapping(mmu->mapping);
1087
1088 ipmmu_device_reset(mmu);
1089
1090 return 0;
1091}
1092
1093#ifdef CONFIG_PM_SLEEP
1094static int ipmmu_resume_noirq(struct device *dev)
1095{
1096 struct ipmmu_vmsa_device *mmu = dev_get_drvdata(dev);
1097 unsigned int i;
1098
1099
1100 if (ipmmu_is_root(mmu)) {
1101 ipmmu_device_reset(mmu);
1102
1103 for (i = 0; i < mmu->num_ctx; i++) {
1104 if (!mmu->domains[i])
1105 continue;
1106
1107 ipmmu_domain_setup_context(mmu->domains[i]);
1108 }
1109 }
1110
1111
1112 for (i = 0; i < mmu->features->num_utlbs; i++) {
1113 if (mmu->utlb_ctx[i] == IPMMU_CTX_INVALID)
1114 continue;
1115
1116 ipmmu_utlb_enable(mmu->root->domains[mmu->utlb_ctx[i]], i);
1117 }
1118
1119 return 0;
1120}
1121
1122static const struct dev_pm_ops ipmmu_pm = {
1123 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(NULL, ipmmu_resume_noirq)
1124};
1125#define DEV_PM_OPS &ipmmu_pm
1126#else
1127#define DEV_PM_OPS NULL
1128#endif
1129
1130static struct platform_driver ipmmu_driver = {
1131 .driver = {
1132 .name = "ipmmu-vmsa",
1133 .of_match_table = of_match_ptr(ipmmu_of_ids),
1134 .pm = DEV_PM_OPS,
1135 },
1136 .probe = ipmmu_probe,
1137 .remove = ipmmu_remove,
1138};
1139
1140static int __init ipmmu_init(void)
1141{
1142 struct device_node *np;
1143 static bool setup_done;
1144 int ret;
1145
1146 if (setup_done)
1147 return 0;
1148
1149 np = of_find_matching_node(NULL, ipmmu_of_ids);
1150 if (!np)
1151 return 0;
1152
1153 of_node_put(np);
1154
1155 ret = platform_driver_register(&ipmmu_driver);
1156 if (ret < 0)
1157 return ret;
1158
1159#if defined(CONFIG_ARM) && !defined(CONFIG_IOMMU_DMA)
1160 if (!iommu_present(&platform_bus_type))
1161 bus_set_iommu(&platform_bus_type, &ipmmu_ops);
1162#endif
1163
1164 setup_done = true;
1165 return 0;
1166}
1167subsys_initcall(ipmmu_init);
1168