1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/atomic.h>
15#include <linux/bitfield.h>
16#include <linux/clk.h>
17#include <linux/dev_printk.h>
18#include <linux/dma-iommu.h>
19#include <linux/dma-mapping.h>
20#include <linux/err.h>
21#include <linux/interrupt.h>
22#include <linux/io-pgtable.h>
23#include <linux/iommu.h>
24#include <linux/iopoll.h>
25#include <linux/module.h>
26#include <linux/of.h>
27#include <linux/of_address.h>
28#include <linux/of_iommu.h>
29#include <linux/of_platform.h>
30#include <linux/pci.h>
31#include <linux/platform_device.h>
32#include <linux/slab.h>
33#include <linux/swab.h>
34#include <linux/types.h>
35
36#define DART_MAX_STREAMS 16
37#define DART_MAX_TTBR 4
38#define MAX_DARTS_PER_DEVICE 2
39
40#define DART_STREAM_ALL 0xffff
41
42#define DART_PARAMS1 0x00
43#define DART_PARAMS_PAGE_SHIFT GENMASK(27, 24)
44
45#define DART_PARAMS2 0x04
46#define DART_PARAMS_BYPASS_SUPPORT BIT(0)
47
48#define DART_STREAM_COMMAND 0x20
49#define DART_STREAM_COMMAND_BUSY BIT(2)
50#define DART_STREAM_COMMAND_INVALIDATE BIT(20)
51
52#define DART_STREAM_SELECT 0x34
53
54#define DART_ERROR 0x40
55#define DART_ERROR_STREAM GENMASK(27, 24)
56#define DART_ERROR_CODE GENMASK(11, 0)
57#define DART_ERROR_FLAG BIT(31)
58
59#define DART_ERROR_READ_FAULT BIT(4)
60#define DART_ERROR_WRITE_FAULT BIT(3)
61#define DART_ERROR_NO_PTE BIT(2)
62#define DART_ERROR_NO_PMD BIT(1)
63#define DART_ERROR_NO_TTBR BIT(0)
64
65#define DART_CONFIG 0x60
66#define DART_CONFIG_LOCK BIT(15)
67
68#define DART_STREAM_COMMAND_BUSY_TIMEOUT 100
69
70#define DART_ERROR_ADDR_HI 0x54
71#define DART_ERROR_ADDR_LO 0x50
72
73#define DART_TCR(sid) (0x100 + 4 * (sid))
74#define DART_TCR_TRANSLATE_ENABLE BIT(7)
75#define DART_TCR_BYPASS0_ENABLE BIT(8)
76#define DART_TCR_BYPASS1_ENABLE BIT(12)
77
78#define DART_TTBR(sid, idx) (0x200 + 16 * (sid) + 4 * (idx))
79#define DART_TTBR_VALID BIT(31)
80#define DART_TTBR_SHIFT 12
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97struct apple_dart {
98 struct device *dev;
99
100 void __iomem *regs;
101
102 int irq;
103 struct clk_bulk_data *clks;
104 int num_clks;
105
106 spinlock_t lock;
107
108 u32 pgsize;
109 u32 supports_bypass : 1;
110 u32 force_bypass : 1;
111
112 struct iommu_group *sid2group[DART_MAX_STREAMS];
113 struct iommu_device iommu;
114};
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131struct apple_dart_stream_map {
132 struct apple_dart *dart;
133 unsigned long sidmap;
134};
135struct apple_dart_atomic_stream_map {
136 struct apple_dart *dart;
137 atomic64_t sidmap;
138};
139
140
141
142
143
144
145
146
147
148
149struct apple_dart_domain {
150 struct io_pgtable_ops *pgtbl_ops;
151
152 bool finalized;
153 struct mutex init_lock;
154 struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
155
156 struct iommu_domain domain;
157};
158
159
160
161
162
163
164
165
166
167struct apple_dart_master_cfg {
168 struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
169};
170
171
172
173
174
175
176
177
178
179#define for_each_stream_map(i, base, stream_map) \
180 for (i = 0, stream_map = &(base)->stream_maps[0]; \
181 i < MAX_DARTS_PER_DEVICE && stream_map->dart; \
182 stream_map = &(base)->stream_maps[++i])
183
184static struct platform_driver apple_dart_driver;
185static const struct iommu_ops apple_dart_iommu_ops;
186
187static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom)
188{
189 return container_of(dom, struct apple_dart_domain, domain);
190}
191
192static void
193apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map)
194{
195 int sid;
196
197 for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
198 writel(DART_TCR_TRANSLATE_ENABLE,
199 stream_map->dart->regs + DART_TCR(sid));
200}
201
202static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map)
203{
204 int sid;
205
206 for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
207 writel(0, stream_map->dart->regs + DART_TCR(sid));
208}
209
210static void
211apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map)
212{
213 int sid;
214
215 WARN_ON(!stream_map->dart->supports_bypass);
216 for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
217 writel(DART_TCR_BYPASS0_ENABLE | DART_TCR_BYPASS1_ENABLE,
218 stream_map->dart->regs + DART_TCR(sid));
219}
220
221static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map,
222 u8 idx, phys_addr_t paddr)
223{
224 int sid;
225
226 WARN_ON(paddr & ((1 << DART_TTBR_SHIFT) - 1));
227 for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
228 writel(DART_TTBR_VALID | (paddr >> DART_TTBR_SHIFT),
229 stream_map->dart->regs + DART_TTBR(sid, idx));
230}
231
232static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map,
233 u8 idx)
234{
235 int sid;
236
237 for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
238 writel(0, stream_map->dart->regs + DART_TTBR(sid, idx));
239}
240
241static void
242apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map)
243{
244 int i;
245
246 for (i = 0; i < DART_MAX_TTBR; ++i)
247 apple_dart_hw_clear_ttbr(stream_map, i);
248}
249
250static int
251apple_dart_hw_stream_command(struct apple_dart_stream_map *stream_map,
252 u32 command)
253{
254 unsigned long flags;
255 int ret;
256 u32 command_reg;
257
258 spin_lock_irqsave(&stream_map->dart->lock, flags);
259
260 writel(stream_map->sidmap, stream_map->dart->regs + DART_STREAM_SELECT);
261 writel(command, stream_map->dart->regs + DART_STREAM_COMMAND);
262
263 ret = readl_poll_timeout_atomic(
264 stream_map->dart->regs + DART_STREAM_COMMAND, command_reg,
265 !(command_reg & DART_STREAM_COMMAND_BUSY), 1,
266 DART_STREAM_COMMAND_BUSY_TIMEOUT);
267
268 spin_unlock_irqrestore(&stream_map->dart->lock, flags);
269
270 if (ret) {
271 dev_err(stream_map->dart->dev,
272 "busy bit did not clear after command %x for streams %lx\n",
273 command, stream_map->sidmap);
274 return ret;
275 }
276
277 return 0;
278}
279
280static int
281apple_dart_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
282{
283 return apple_dart_hw_stream_command(stream_map,
284 DART_STREAM_COMMAND_INVALIDATE);
285}
286
287static int apple_dart_hw_reset(struct apple_dart *dart)
288{
289 u32 config;
290 struct apple_dart_stream_map stream_map;
291
292 config = readl(dart->regs + DART_CONFIG);
293 if (config & DART_CONFIG_LOCK) {
294 dev_err(dart->dev, "DART is locked down until reboot: %08x\n",
295 config);
296 return -EINVAL;
297 }
298
299 stream_map.dart = dart;
300 stream_map.sidmap = DART_STREAM_ALL;
301 apple_dart_hw_disable_dma(&stream_map);
302 apple_dart_hw_clear_all_ttbrs(&stream_map);
303
304
305 writel(readl(dart->regs + DART_ERROR), dart->regs + DART_ERROR);
306
307 return apple_dart_hw_invalidate_tlb(&stream_map);
308}
309
310static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain)
311{
312 int i;
313 struct apple_dart_atomic_stream_map *domain_stream_map;
314 struct apple_dart_stream_map stream_map;
315
316 for_each_stream_map(i, domain, domain_stream_map) {
317 stream_map.dart = domain_stream_map->dart;
318 stream_map.sidmap = atomic64_read(&domain_stream_map->sidmap);
319 apple_dart_hw_invalidate_tlb(&stream_map);
320 }
321}
322
323static void apple_dart_flush_iotlb_all(struct iommu_domain *domain)
324{
325 apple_dart_domain_flush_tlb(to_dart_domain(domain));
326}
327
328static void apple_dart_iotlb_sync(struct iommu_domain *domain,
329 struct iommu_iotlb_gather *gather)
330{
331 apple_dart_domain_flush_tlb(to_dart_domain(domain));
332}
333
334static void apple_dart_iotlb_sync_map(struct iommu_domain *domain,
335 unsigned long iova, size_t size)
336{
337 apple_dart_domain_flush_tlb(to_dart_domain(domain));
338}
339
340static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain,
341 dma_addr_t iova)
342{
343 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
344 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
345
346 if (!ops)
347 return 0;
348
349 return ops->iova_to_phys(ops, iova);
350}
351
352static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova,
353 phys_addr_t paddr, size_t pgsize,
354 size_t pgcount, int prot, gfp_t gfp,
355 size_t *mapped)
356{
357 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
358 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
359
360 if (!ops)
361 return -ENODEV;
362
363 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp,
364 mapped);
365}
366
367static size_t apple_dart_unmap_pages(struct iommu_domain *domain,
368 unsigned long iova, size_t pgsize,
369 size_t pgcount,
370 struct iommu_iotlb_gather *gather)
371{
372 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
373 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
374
375 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
376}
377
378static void
379apple_dart_setup_translation(struct apple_dart_domain *domain,
380 struct apple_dart_stream_map *stream_map)
381{
382 int i;
383 struct io_pgtable_cfg *pgtbl_cfg =
384 &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;
385
386 for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)
387 apple_dart_hw_set_ttbr(stream_map, i,
388 pgtbl_cfg->apple_dart_cfg.ttbr[i]);
389 for (; i < DART_MAX_TTBR; ++i)
390 apple_dart_hw_clear_ttbr(stream_map, i);
391
392 apple_dart_hw_enable_translation(stream_map);
393 apple_dart_hw_invalidate_tlb(stream_map);
394}
395
396static int apple_dart_finalize_domain(struct iommu_domain *domain,
397 struct apple_dart_master_cfg *cfg)
398{
399 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
400 struct apple_dart *dart = cfg->stream_maps[0].dart;
401 struct io_pgtable_cfg pgtbl_cfg;
402 int ret = 0;
403 int i;
404
405 mutex_lock(&dart_domain->init_lock);
406
407 if (dart_domain->finalized)
408 goto done;
409
410 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
411 dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart;
412 atomic64_set(&dart_domain->stream_maps[i].sidmap,
413 cfg->stream_maps[i].sidmap);
414 }
415
416 pgtbl_cfg = (struct io_pgtable_cfg){
417 .pgsize_bitmap = dart->pgsize,
418 .ias = 32,
419 .oas = 36,
420 .coherent_walk = 1,
421 .iommu_dev = dart->dev,
422 };
423
424 dart_domain->pgtbl_ops =
425 alloc_io_pgtable_ops(APPLE_DART, &pgtbl_cfg, domain);
426 if (!dart_domain->pgtbl_ops) {
427 ret = -ENOMEM;
428 goto done;
429 }
430
431 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
432 domain->geometry.aperture_start = 0;
433 domain->geometry.aperture_end = DMA_BIT_MASK(32);
434 domain->geometry.force_aperture = true;
435
436 dart_domain->finalized = true;
437
438done:
439 mutex_unlock(&dart_domain->init_lock);
440 return ret;
441}
442
443static int
444apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps,
445 struct apple_dart_stream_map *master_maps,
446 bool add_streams)
447{
448 int i;
449
450 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
451 if (domain_maps[i].dart != master_maps[i].dart)
452 return -EINVAL;
453 }
454
455 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
456 if (!domain_maps[i].dart)
457 break;
458 if (add_streams)
459 atomic64_or(master_maps[i].sidmap,
460 &domain_maps[i].sidmap);
461 else
462 atomic64_and(~master_maps[i].sidmap,
463 &domain_maps[i].sidmap);
464 }
465
466 return 0;
467}
468
469static int apple_dart_domain_add_streams(struct apple_dart_domain *domain,
470 struct apple_dart_master_cfg *cfg)
471{
472 return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,
473 true);
474}
475
476static int apple_dart_domain_remove_streams(struct apple_dart_domain *domain,
477 struct apple_dart_master_cfg *cfg)
478{
479 return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,
480 false);
481}
482
483static int apple_dart_attach_dev(struct iommu_domain *domain,
484 struct device *dev)
485{
486 int ret, i;
487 struct apple_dart_stream_map *stream_map;
488 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
489 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
490
491 if (cfg->stream_maps[0].dart->force_bypass &&
492 domain->type != IOMMU_DOMAIN_IDENTITY)
493 return -EINVAL;
494 if (!cfg->stream_maps[0].dart->supports_bypass &&
495 domain->type == IOMMU_DOMAIN_IDENTITY)
496 return -EINVAL;
497
498 ret = apple_dart_finalize_domain(domain, cfg);
499 if (ret)
500 return ret;
501
502 switch (domain->type) {
503 case IOMMU_DOMAIN_DMA:
504 case IOMMU_DOMAIN_UNMANAGED:
505 ret = apple_dart_domain_add_streams(dart_domain, cfg);
506 if (ret)
507 return ret;
508
509 for_each_stream_map(i, cfg, stream_map)
510 apple_dart_setup_translation(dart_domain, stream_map);
511 break;
512 case IOMMU_DOMAIN_BLOCKED:
513 for_each_stream_map(i, cfg, stream_map)
514 apple_dart_hw_disable_dma(stream_map);
515 break;
516 case IOMMU_DOMAIN_IDENTITY:
517 for_each_stream_map(i, cfg, stream_map)
518 apple_dart_hw_enable_bypass(stream_map);
519 break;
520 }
521
522 return ret;
523}
524
525static void apple_dart_detach_dev(struct iommu_domain *domain,
526 struct device *dev)
527{
528 int i;
529 struct apple_dart_stream_map *stream_map;
530 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
531 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
532
533 for_each_stream_map(i, cfg, stream_map)
534 apple_dart_hw_disable_dma(stream_map);
535
536 if (domain->type == IOMMU_DOMAIN_DMA ||
537 domain->type == IOMMU_DOMAIN_UNMANAGED)
538 apple_dart_domain_remove_streams(dart_domain, cfg);
539}
540
541static struct iommu_device *apple_dart_probe_device(struct device *dev)
542{
543 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
544 struct apple_dart_stream_map *stream_map;
545 int i;
546
547 if (!cfg)
548 return ERR_PTR(-ENODEV);
549
550 for_each_stream_map(i, cfg, stream_map)
551 device_link_add(
552 dev, stream_map->dart->dev,
553 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
554
555 return &cfg->stream_maps[0].dart->iommu;
556}
557
558static void apple_dart_release_device(struct device *dev)
559{
560 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
561
562 if (!cfg)
563 return;
564
565 dev_iommu_priv_set(dev, NULL);
566 kfree(cfg);
567}
568
569static struct iommu_domain *apple_dart_domain_alloc(unsigned int type)
570{
571 struct apple_dart_domain *dart_domain;
572
573 if (type != IOMMU_DOMAIN_DMA && type != IOMMU_DOMAIN_UNMANAGED &&
574 type != IOMMU_DOMAIN_IDENTITY && type != IOMMU_DOMAIN_BLOCKED)
575 return NULL;
576
577 dart_domain = kzalloc(sizeof(*dart_domain), GFP_KERNEL);
578 if (!dart_domain)
579 return NULL;
580
581 iommu_get_dma_cookie(&dart_domain->domain);
582 mutex_init(&dart_domain->init_lock);
583
584
585 if (type == IOMMU_DOMAIN_IDENTITY || type == IOMMU_DOMAIN_BLOCKED)
586 dart_domain->finalized = true;
587
588 return &dart_domain->domain;
589}
590
591static void apple_dart_domain_free(struct iommu_domain *domain)
592{
593 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
594
595 if (dart_domain->pgtbl_ops)
596 free_io_pgtable_ops(dart_domain->pgtbl_ops);
597
598 kfree(dart_domain);
599}
600
601static int apple_dart_of_xlate(struct device *dev, struct of_phandle_args *args)
602{
603 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
604 struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
605 struct apple_dart *dart = platform_get_drvdata(iommu_pdev);
606 struct apple_dart *cfg_dart;
607 int i, sid;
608
609 if (args->args_count != 1)
610 return -EINVAL;
611 sid = args->args[0];
612
613 if (!cfg)
614 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
615 if (!cfg)
616 return -ENOMEM;
617 dev_iommu_priv_set(dev, cfg);
618
619 cfg_dart = cfg->stream_maps[0].dart;
620 if (cfg_dart) {
621 if (cfg_dart->supports_bypass != dart->supports_bypass)
622 return -EINVAL;
623 if (cfg_dart->force_bypass != dart->force_bypass)
624 return -EINVAL;
625 if (cfg_dart->pgsize != dart->pgsize)
626 return -EINVAL;
627 }
628
629 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
630 if (cfg->stream_maps[i].dart == dart) {
631 cfg->stream_maps[i].sidmap |= 1 << sid;
632 return 0;
633 }
634 }
635 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
636 if (!cfg->stream_maps[i].dart) {
637 cfg->stream_maps[i].dart = dart;
638 cfg->stream_maps[i].sidmap = 1 << sid;
639 return 0;
640 }
641 }
642
643 return -EINVAL;
644}
645
646static DEFINE_MUTEX(apple_dart_groups_lock);
647
648static void apple_dart_release_group(void *iommu_data)
649{
650 int i, sid;
651 struct apple_dart_stream_map *stream_map;
652 struct apple_dart_master_cfg *group_master_cfg = iommu_data;
653
654 mutex_lock(&apple_dart_groups_lock);
655
656 for_each_stream_map(i, group_master_cfg, stream_map)
657 for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
658 stream_map->dart->sid2group[sid] = NULL;
659
660 kfree(iommu_data);
661 mutex_unlock(&apple_dart_groups_lock);
662}
663
664static struct iommu_group *apple_dart_device_group(struct device *dev)
665{
666 int i, sid;
667 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
668 struct apple_dart_stream_map *stream_map;
669 struct apple_dart_master_cfg *group_master_cfg;
670 struct iommu_group *group = NULL;
671 struct iommu_group *res = ERR_PTR(-EINVAL);
672
673 mutex_lock(&apple_dart_groups_lock);
674
675 for_each_stream_map(i, cfg, stream_map) {
676 for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) {
677 struct iommu_group *stream_group =
678 stream_map->dart->sid2group[sid];
679
680 if (group && group != stream_group) {
681 res = ERR_PTR(-EINVAL);
682 goto out;
683 }
684
685 group = stream_group;
686 }
687 }
688
689 if (group) {
690 res = iommu_group_ref_get(group);
691 goto out;
692 }
693
694#ifdef CONFIG_PCI
695 if (dev_is_pci(dev))
696 group = pci_device_group(dev);
697 else
698#endif
699 group = generic_device_group(dev);
700
701 res = ERR_PTR(-ENOMEM);
702 if (!group)
703 goto out;
704
705 group_master_cfg = kzalloc(sizeof(*group_master_cfg), GFP_KERNEL);
706 if (!group_master_cfg) {
707 iommu_group_put(group);
708 goto out;
709 }
710
711 memcpy(group_master_cfg, cfg, sizeof(*group_master_cfg));
712 iommu_group_set_iommudata(group, group_master_cfg,
713 apple_dart_release_group);
714
715 for_each_stream_map(i, cfg, stream_map)
716 for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)
717 stream_map->dart->sid2group[sid] = group;
718
719 res = group;
720
721out:
722 mutex_unlock(&apple_dart_groups_lock);
723 return res;
724}
725
726static int apple_dart_def_domain_type(struct device *dev)
727{
728 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
729
730 if (cfg->stream_maps[0].dart->force_bypass)
731 return IOMMU_DOMAIN_IDENTITY;
732 if (!cfg->stream_maps[0].dart->supports_bypass)
733 return IOMMU_DOMAIN_DMA;
734
735 return 0;
736}
737
738static const struct iommu_ops apple_dart_iommu_ops = {
739 .domain_alloc = apple_dart_domain_alloc,
740 .domain_free = apple_dart_domain_free,
741 .attach_dev = apple_dart_attach_dev,
742 .detach_dev = apple_dart_detach_dev,
743 .map_pages = apple_dart_map_pages,
744 .unmap_pages = apple_dart_unmap_pages,
745 .flush_iotlb_all = apple_dart_flush_iotlb_all,
746 .iotlb_sync = apple_dart_iotlb_sync,
747 .iotlb_sync_map = apple_dart_iotlb_sync_map,
748 .iova_to_phys = apple_dart_iova_to_phys,
749 .probe_device = apple_dart_probe_device,
750 .release_device = apple_dart_release_device,
751 .device_group = apple_dart_device_group,
752 .of_xlate = apple_dart_of_xlate,
753 .def_domain_type = apple_dart_def_domain_type,
754 .pgsize_bitmap = -1UL,
755};
756
757static irqreturn_t apple_dart_irq(int irq, void *dev)
758{
759 struct apple_dart *dart = dev;
760 const char *fault_name = NULL;
761 u32 error = readl(dart->regs + DART_ERROR);
762 u32 error_code = FIELD_GET(DART_ERROR_CODE, error);
763 u32 addr_lo = readl(dart->regs + DART_ERROR_ADDR_LO);
764 u32 addr_hi = readl(dart->regs + DART_ERROR_ADDR_HI);
765 u64 addr = addr_lo | (((u64)addr_hi) << 32);
766 u8 stream_idx = FIELD_GET(DART_ERROR_STREAM, error);
767
768 if (!(error & DART_ERROR_FLAG))
769 return IRQ_NONE;
770
771
772 if (error_code == DART_ERROR_READ_FAULT)
773 fault_name = "READ FAULT";
774 else if (error_code == DART_ERROR_WRITE_FAULT)
775 fault_name = "WRITE FAULT";
776 else if (error_code == DART_ERROR_NO_PTE)
777 fault_name = "NO PTE FOR IOVA";
778 else if (error_code == DART_ERROR_NO_PMD)
779 fault_name = "NO PMD FOR IOVA";
780 else if (error_code == DART_ERROR_NO_TTBR)
781 fault_name = "NO TTBR FOR IOVA";
782 else
783 fault_name = "unknown";
784
785 dev_err_ratelimited(
786 dart->dev,
787 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
788 error, stream_idx, error_code, fault_name, addr);
789
790 writel(error, dart->regs + DART_ERROR);
791 return IRQ_HANDLED;
792}
793
794static int apple_dart_set_bus_ops(const struct iommu_ops *ops)
795{
796 int ret;
797
798 if (!iommu_present(&platform_bus_type)) {
799 ret = bus_set_iommu(&platform_bus_type, ops);
800 if (ret)
801 return ret;
802 }
803#ifdef CONFIG_PCI
804 if (!iommu_present(&pci_bus_type)) {
805 ret = bus_set_iommu(&pci_bus_type, ops);
806 if (ret) {
807 bus_set_iommu(&platform_bus_type, NULL);
808 return ret;
809 }
810 }
811#endif
812 return 0;
813}
814
815static int apple_dart_probe(struct platform_device *pdev)
816{
817 int ret;
818 u32 dart_params[2];
819 struct resource *res;
820 struct apple_dart *dart;
821 struct device *dev = &pdev->dev;
822
823 dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL);
824 if (!dart)
825 return -ENOMEM;
826
827 dart->dev = dev;
828 spin_lock_init(&dart->lock);
829
830 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
831 if (resource_size(res) < 0x4000) {
832 dev_err(dev, "MMIO region too small (%pr)\n", res);
833 return -EINVAL;
834 }
835
836 dart->regs = devm_ioremap_resource(dev, res);
837 if (IS_ERR(dart->regs))
838 return PTR_ERR(dart->regs);
839
840 dart->irq = platform_get_irq(pdev, 0);
841 if (dart->irq < 0)
842 return -ENODEV;
843
844 ret = devm_clk_bulk_get_all(dev, &dart->clks);
845 if (ret < 0)
846 return ret;
847 dart->num_clks = ret;
848
849 ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks);
850 if (ret)
851 return ret;
852
853 ret = apple_dart_hw_reset(dart);
854 if (ret)
855 goto err_clk_disable;
856
857 dart_params[0] = readl(dart->regs + DART_PARAMS1);
858 dart_params[1] = readl(dart->regs + DART_PARAMS2);
859 dart->pgsize = 1 << FIELD_GET(DART_PARAMS_PAGE_SHIFT, dart_params[0]);
860 dart->supports_bypass = dart_params[1] & DART_PARAMS_BYPASS_SUPPORT;
861 dart->force_bypass = dart->pgsize > PAGE_SIZE;
862
863 ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED,
864 "apple-dart fault handler", dart);
865 if (ret)
866 goto err_clk_disable;
867
868 platform_set_drvdata(pdev, dart);
869
870 ret = apple_dart_set_bus_ops(&apple_dart_iommu_ops);
871 if (ret)
872 goto err_free_irq;
873
874 ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s",
875 dev_name(&pdev->dev));
876 if (ret)
877 goto err_remove_bus_ops;
878
879 ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev);
880 if (ret)
881 goto err_sysfs_remove;
882
883 dev_info(
884 &pdev->dev,
885 "DART [pagesize %x, bypass support: %d, bypass forced: %d] initialized\n",
886 dart->pgsize, dart->supports_bypass, dart->force_bypass);
887 return 0;
888
889err_sysfs_remove:
890 iommu_device_sysfs_remove(&dart->iommu);
891err_remove_bus_ops:
892 apple_dart_set_bus_ops(NULL);
893err_free_irq:
894 free_irq(dart->irq, dart);
895err_clk_disable:
896 clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
897
898 return ret;
899}
900
901static int apple_dart_remove(struct platform_device *pdev)
902{
903 struct apple_dart *dart = platform_get_drvdata(pdev);
904
905 apple_dart_hw_reset(dart);
906 free_irq(dart->irq, dart);
907 apple_dart_set_bus_ops(NULL);
908
909 iommu_device_unregister(&dart->iommu);
910 iommu_device_sysfs_remove(&dart->iommu);
911
912 clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
913
914 return 0;
915}
916
917static const struct of_device_id apple_dart_of_match[] = {
918 { .compatible = "apple,t8103-dart", .data = NULL },
919 {},
920};
921MODULE_DEVICE_TABLE(of, apple_dart_of_match);
922
923static struct platform_driver apple_dart_driver = {
924 .driver = {
925 .name = "apple-dart",
926 .of_match_table = apple_dart_of_match,
927 .suppress_bind_attrs = true,
928 },
929 .probe = apple_dart_probe,
930 .remove = apple_dart_remove,
931};
932
933module_platform_driver(apple_dart_driver);
934
935MODULE_DESCRIPTION("IOMMU API for Apple's DART");
936MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
937MODULE_LICENSE("GPL v2");
938
