1
2
3
4
5#include <linux/bitfield.h>
6#include <linux/bug.h>
7#include <linux/clk.h>
8#include <linux/device.h>
9#include <linux/dma-direction.h>
10#include <linux/dma-iommu.h>
11#include <linux/dma-mapping.h>
12#include <linux/err.h>
13#include <linux/errno.h>
14#include <linux/interrupt.h>
15#include <linux/iommu.h>
16#include <linux/iopoll.h>
17#include <linux/ioport.h>
18#include <linux/log2.h>
19#include <linux/module.h>
20#include <linux/of_platform.h>
21#include <linux/platform_device.h>
22#include <linux/pm.h>
23#include <linux/pm_runtime.h>
24#include <linux/reset.h>
25#include <linux/sizes.h>
26#include <linux/slab.h>
27#include <linux/spinlock.h>
28#include <linux/types.h>
29
30#define IOMMU_RESET_REG 0x010
31#define IOMMU_ENABLE_REG 0x020
32#define IOMMU_ENABLE_ENABLE BIT(0)
33
34#define IOMMU_BYPASS_REG 0x030
35#define IOMMU_AUTO_GATING_REG 0x040
36#define IOMMU_AUTO_GATING_ENABLE BIT(0)
37
38#define IOMMU_WBUF_CTRL_REG 0x044
39#define IOMMU_OOO_CTRL_REG 0x048
40#define IOMMU_4KB_BDY_PRT_CTRL_REG 0x04c
41#define IOMMU_TTB_REG 0x050
42#define IOMMU_TLB_ENABLE_REG 0x060
43#define IOMMU_TLB_PREFETCH_REG 0x070
44#define IOMMU_TLB_PREFETCH_MASTER_ENABLE(m) BIT(m)
45
46#define IOMMU_TLB_FLUSH_REG 0x080
47#define IOMMU_TLB_FLUSH_PTW_CACHE BIT(17)
48#define IOMMU_TLB_FLUSH_MACRO_TLB BIT(16)
49#define IOMMU_TLB_FLUSH_MICRO_TLB(i) (BIT(i) & GENMASK(5, 0))
50
51#define IOMMU_TLB_IVLD_ADDR_REG 0x090
52#define IOMMU_TLB_IVLD_ADDR_MASK_REG 0x094
53#define IOMMU_TLB_IVLD_ENABLE_REG 0x098
54#define IOMMU_TLB_IVLD_ENABLE_ENABLE BIT(0)
55
56#define IOMMU_PC_IVLD_ADDR_REG 0x0a0
57#define IOMMU_PC_IVLD_ENABLE_REG 0x0a8
58#define IOMMU_PC_IVLD_ENABLE_ENABLE BIT(0)
59
60#define IOMMU_DM_AUT_CTRL_REG(d) (0x0b0 + ((d) / 2) * 4)
61#define IOMMU_DM_AUT_CTRL_RD_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2)))
62#define IOMMU_DM_AUT_CTRL_WR_UNAVAIL(d, m) (1 << (((d & 1) * 16) + ((m) * 2) + 1))
63
64#define IOMMU_DM_AUT_OVWT_REG 0x0d0
65#define IOMMU_INT_ENABLE_REG 0x100
66#define IOMMU_INT_CLR_REG 0x104
67#define IOMMU_INT_STA_REG 0x108
68#define IOMMU_INT_ERR_ADDR_REG(i) (0x110 + (i) * 4)
69#define IOMMU_INT_ERR_ADDR_L1_REG 0x130
70#define IOMMU_INT_ERR_ADDR_L2_REG 0x134
71#define IOMMU_INT_ERR_DATA_REG(i) (0x150 + (i) * 4)
72#define IOMMU_L1PG_INT_REG 0x0180
73#define IOMMU_L2PG_INT_REG 0x0184
74
75#define IOMMU_INT_INVALID_L2PG BIT(17)
76#define IOMMU_INT_INVALID_L1PG BIT(16)
77#define IOMMU_INT_MASTER_PERMISSION(m) BIT(m)
78#define IOMMU_INT_MASTER_MASK (IOMMU_INT_MASTER_PERMISSION(0) | \
79 IOMMU_INT_MASTER_PERMISSION(1) | \
80 IOMMU_INT_MASTER_PERMISSION(2) | \
81 IOMMU_INT_MASTER_PERMISSION(3) | \
82 IOMMU_INT_MASTER_PERMISSION(4) | \
83 IOMMU_INT_MASTER_PERMISSION(5))
84#define IOMMU_INT_MASK (IOMMU_INT_INVALID_L1PG | \
85 IOMMU_INT_INVALID_L2PG | \
86 IOMMU_INT_MASTER_MASK)
87
88#define PT_ENTRY_SIZE sizeof(u32)
89
90#define NUM_DT_ENTRIES 4096
91#define DT_SIZE (NUM_DT_ENTRIES * PT_ENTRY_SIZE)
92
93#define NUM_PT_ENTRIES 256
94#define PT_SIZE (NUM_PT_ENTRIES * PT_ENTRY_SIZE)
95
96struct sun50i_iommu {
97 struct iommu_device iommu;
98
99
100 spinlock_t iommu_lock;
101
102 struct device *dev;
103 void __iomem *base;
104 struct reset_control *reset;
105 struct clk *clk;
106
107 struct iommu_domain *domain;
108 struct iommu_group *group;
109 struct kmem_cache *pt_pool;
110};
111
112struct sun50i_iommu_domain {
113 struct iommu_domain domain;
114
115
116 refcount_t refcnt;
117
118
119 u32 *dt;
120 dma_addr_t dt_dma;
121
122 struct sun50i_iommu *iommu;
123};
124
125static struct sun50i_iommu_domain *to_sun50i_domain(struct iommu_domain *domain)
126{
127 return container_of(domain, struct sun50i_iommu_domain, domain);
128}
129
130static struct sun50i_iommu *sun50i_iommu_from_dev(struct device *dev)
131{
132 return dev_iommu_priv_get(dev);
133}
134
135static u32 iommu_read(struct sun50i_iommu *iommu, u32 offset)
136{
137 return readl(iommu->base + offset);
138}
139
140static void iommu_write(struct sun50i_iommu *iommu, u32 offset, u32 value)
141{
142 writel(value, iommu->base + offset);
143}
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159#define SUN50I_IOVA_DTE_MASK GENMASK(31, 20)
160#define SUN50I_IOVA_PTE_MASK GENMASK(19, 12)
161#define SUN50I_IOVA_PAGE_MASK GENMASK(11, 0)
162
163static u32 sun50i_iova_get_dte_index(dma_addr_t iova)
164{
165 return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova);
166}
167
168static u32 sun50i_iova_get_pte_index(dma_addr_t iova)
169{
170 return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova);
171}
172
173static u32 sun50i_iova_get_page_offset(dma_addr_t iova)
174{
175 return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova);
176}
177
178
179
180
181
182
183
184
185
186
187
188
189
190#define SUN50I_DTE_PT_ADDRESS_MASK GENMASK(31, 10)
191#define SUN50I_DTE_PT_ATTRS GENMASK(1, 0)
192#define SUN50I_DTE_PT_VALID 1
193
194static phys_addr_t sun50i_dte_get_pt_address(u32 dte)
195{
196 return (phys_addr_t)dte & SUN50I_DTE_PT_ADDRESS_MASK;
197}
198
199static bool sun50i_dte_is_pt_valid(u32 dte)
200{
201 return (dte & SUN50I_DTE_PT_ATTRS) == SUN50I_DTE_PT_VALID;
202}
203
204static u32 sun50i_mk_dte(dma_addr_t pt_dma)
205{
206 return (pt_dma & SUN50I_DTE_PT_ADDRESS_MASK) | SUN50I_DTE_PT_VALID;
207}
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242enum sun50i_iommu_aci {
243 SUN50I_IOMMU_ACI_DO_NOT_USE = 0,
244 SUN50I_IOMMU_ACI_NONE,
245 SUN50I_IOMMU_ACI_RD,
246 SUN50I_IOMMU_ACI_WR,
247 SUN50I_IOMMU_ACI_RD_WR,
248};
249
250#define SUN50I_PTE_PAGE_ADDRESS_MASK GENMASK(31, 12)
251#define SUN50I_PTE_ACI_MASK GENMASK(7, 4)
252#define SUN50I_PTE_PAGE_VALID BIT(1)
253
254static phys_addr_t sun50i_pte_get_page_address(u32 pte)
255{
256 return (phys_addr_t)pte & SUN50I_PTE_PAGE_ADDRESS_MASK;
257}
258
259static enum sun50i_iommu_aci sun50i_get_pte_aci(u32 pte)
260{
261 return FIELD_GET(SUN50I_PTE_ACI_MASK, pte);
262}
263
264static bool sun50i_pte_is_page_valid(u32 pte)
265{
266 return pte & SUN50I_PTE_PAGE_VALID;
267}
268
269static u32 sun50i_mk_pte(phys_addr_t page, int prot)
270{
271 enum sun50i_iommu_aci aci;
272 u32 flags = 0;
273
274 if (prot & (IOMMU_READ | IOMMU_WRITE))
275 aci = SUN50I_IOMMU_ACI_RD_WR;
276 else if (prot & IOMMU_READ)
277 aci = SUN50I_IOMMU_ACI_RD;
278 else if (prot & IOMMU_WRITE)
279 aci = SUN50I_IOMMU_ACI_WR;
280 else
281 aci = SUN50I_IOMMU_ACI_NONE;
282
283 flags |= FIELD_PREP(SUN50I_PTE_ACI_MASK, aci);
284 page &= SUN50I_PTE_PAGE_ADDRESS_MASK;
285 return page | flags | SUN50I_PTE_PAGE_VALID;
286}
287
288static void sun50i_table_flush(struct sun50i_iommu_domain *sun50i_domain,
289 void *vaddr, unsigned int count)
290{
291 struct sun50i_iommu *iommu = sun50i_domain->iommu;
292 dma_addr_t dma = virt_to_phys(vaddr);
293 size_t size = count * PT_ENTRY_SIZE;
294
295 dma_sync_single_for_device(iommu->dev, dma, size, DMA_TO_DEVICE);
296}
297
298static int sun50i_iommu_flush_all_tlb(struct sun50i_iommu *iommu)
299{
300 u32 reg;
301 int ret;
302
303 assert_spin_locked(&iommu->iommu_lock);
304
305 iommu_write(iommu,
306 IOMMU_TLB_FLUSH_REG,
307 IOMMU_TLB_FLUSH_PTW_CACHE |
308 IOMMU_TLB_FLUSH_MACRO_TLB |
309 IOMMU_TLB_FLUSH_MICRO_TLB(5) |
310 IOMMU_TLB_FLUSH_MICRO_TLB(4) |
311 IOMMU_TLB_FLUSH_MICRO_TLB(3) |
312 IOMMU_TLB_FLUSH_MICRO_TLB(2) |
313 IOMMU_TLB_FLUSH_MICRO_TLB(1) |
314 IOMMU_TLB_FLUSH_MICRO_TLB(0));
315
316 ret = readl_poll_timeout_atomic(iommu->base + IOMMU_TLB_FLUSH_REG,
317 reg, !reg,
318 1, 2000);
319 if (ret)
320 dev_warn(iommu->dev, "TLB Flush timed out!\n");
321
322 return ret;
323}
324
325static void sun50i_iommu_flush_iotlb_all(struct iommu_domain *domain)
326{
327 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
328 struct sun50i_iommu *iommu = sun50i_domain->iommu;
329 unsigned long flags;
330
331
332
333
334
335
336
337
338
339 if (!iommu)
340 return;
341
342 spin_lock_irqsave(&iommu->iommu_lock, flags);
343 sun50i_iommu_flush_all_tlb(iommu);
344 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
345}
346
347static void sun50i_iommu_iotlb_sync(struct iommu_domain *domain,
348 struct iommu_iotlb_gather *gather)
349{
350 sun50i_iommu_flush_iotlb_all(domain);
351}
352
353static int sun50i_iommu_enable(struct sun50i_iommu *iommu)
354{
355 struct sun50i_iommu_domain *sun50i_domain;
356 unsigned long flags;
357 int ret;
358
359 if (!iommu->domain)
360 return 0;
361
362 sun50i_domain = to_sun50i_domain(iommu->domain);
363
364 ret = reset_control_deassert(iommu->reset);
365 if (ret)
366 return ret;
367
368 ret = clk_prepare_enable(iommu->clk);
369 if (ret)
370 goto err_reset_assert;
371
372 spin_lock_irqsave(&iommu->iommu_lock, flags);
373
374 iommu_write(iommu, IOMMU_TTB_REG, sun50i_domain->dt_dma);
375 iommu_write(iommu, IOMMU_TLB_PREFETCH_REG,
376 IOMMU_TLB_PREFETCH_MASTER_ENABLE(0) |
377 IOMMU_TLB_PREFETCH_MASTER_ENABLE(1) |
378 IOMMU_TLB_PREFETCH_MASTER_ENABLE(2) |
379 IOMMU_TLB_PREFETCH_MASTER_ENABLE(3) |
380 IOMMU_TLB_PREFETCH_MASTER_ENABLE(4) |
381 IOMMU_TLB_PREFETCH_MASTER_ENABLE(5));
382 iommu_write(iommu, IOMMU_INT_ENABLE_REG, IOMMU_INT_MASK);
383 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_NONE),
384 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
385 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 0) |
386 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
387 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 1) |
388 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
389 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 2) |
390 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
391 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 3) |
392 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
393 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 4) |
394 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5) |
395 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_NONE, 5));
396
397 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_RD),
398 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 0) |
399 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 1) |
400 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 2) |
401 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 3) |
402 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 4) |
403 IOMMU_DM_AUT_CTRL_WR_UNAVAIL(SUN50I_IOMMU_ACI_RD, 5));
404
405 iommu_write(iommu, IOMMU_DM_AUT_CTRL_REG(SUN50I_IOMMU_ACI_WR),
406 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 0) |
407 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 1) |
408 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 2) |
409 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 3) |
410 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 4) |
411 IOMMU_DM_AUT_CTRL_RD_UNAVAIL(SUN50I_IOMMU_ACI_WR, 5));
412
413 ret = sun50i_iommu_flush_all_tlb(iommu);
414 if (ret) {
415 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
416 goto err_clk_disable;
417 }
418
419 iommu_write(iommu, IOMMU_AUTO_GATING_REG, IOMMU_AUTO_GATING_ENABLE);
420 iommu_write(iommu, IOMMU_ENABLE_REG, IOMMU_ENABLE_ENABLE);
421
422 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
423
424 return 0;
425
426err_clk_disable:
427 clk_disable_unprepare(iommu->clk);
428
429err_reset_assert:
430 reset_control_assert(iommu->reset);
431
432 return ret;
433}
434
435static void sun50i_iommu_disable(struct sun50i_iommu *iommu)
436{
437 unsigned long flags;
438
439 spin_lock_irqsave(&iommu->iommu_lock, flags);
440
441 iommu_write(iommu, IOMMU_ENABLE_REG, 0);
442 iommu_write(iommu, IOMMU_TTB_REG, 0);
443
444 spin_unlock_irqrestore(&iommu->iommu_lock, flags);
445
446 clk_disable_unprepare(iommu->clk);
447 reset_control_assert(iommu->reset);
448}
449
450static void *sun50i_iommu_alloc_page_table(struct sun50i_iommu *iommu,
451 gfp_t gfp)
452{
453 dma_addr_t pt_dma;
454 u32 *page_table;
455
456 page_table = kmem_cache_zalloc(iommu->pt_pool, gfp);
457 if (!page_table)
458 return ERR_PTR(-ENOMEM);
459
460 pt_dma = dma_map_single(iommu->dev, page_table, PT_SIZE, DMA_TO_DEVICE);
461 if (dma_mapping_error(iommu->dev, pt_dma)) {
462 dev_err(iommu->dev, "Couldn't map L2 Page Table\n");
463 kmem_cache_free(iommu->pt_pool, page_table);
464 return ERR_PTR(-ENOMEM);
465 }
466
467
468 WARN_ON(pt_dma != virt_to_phys(page_table));
469
470 return page_table;
471}
472
473static void sun50i_iommu_free_page_table(struct sun50i_iommu *iommu,
474 u32 *page_table)
475{
476 phys_addr_t pt_phys = virt_to_phys(page_table);
477
478 dma_unmap_single(iommu->dev, pt_phys, PT_SIZE, DMA_TO_DEVICE);
479 kmem_cache_free(iommu->pt_pool, page_table);
480}
481
482static u32 *sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain,
483 dma_addr_t iova, gfp_t gfp)
484{
485 struct sun50i_iommu *iommu = sun50i_domain->iommu;
486 u32 *page_table;
487 u32 *dte_addr;
488 u32 old_dte;
489 u32 dte;
490
491 dte_addr = &sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
492 dte = *dte_addr;
493 if (sun50i_dte_is_pt_valid(dte)) {
494 phys_addr_t pt_phys = sun50i_dte_get_pt_address(dte);
495 return (u32 *)phys_to_virt(pt_phys);
496 }
497
498 page_table = sun50i_iommu_alloc_page_table(iommu, gfp);
499 if (IS_ERR(page_table))
500 return page_table;
501
502 dte = sun50i_mk_dte(virt_to_phys(page_table));
503 old_dte = cmpxchg(dte_addr, 0, dte);
504 if (old_dte) {
505 phys_addr_t installed_pt_phys =
506 sun50i_dte_get_pt_address(old_dte);
507 u32 *installed_pt = phys_to_virt(installed_pt_phys);
508 u32 *drop_pt = page_table;
509
510 page_table = installed_pt;
511 dte = old_dte;
512 sun50i_iommu_free_page_table(iommu, drop_pt);
513 }
514
515 sun50i_table_flush(sun50i_domain, page_table, PT_SIZE);
516 sun50i_table_flush(sun50i_domain, dte_addr, 1);
517
518 return page_table;
519}
520
521static int sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova,
522 phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
523{
524 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
525 struct sun50i_iommu *iommu = sun50i_domain->iommu;
526 u32 pte_index;
527 u32 *page_table, *pte_addr;
528 int ret = 0;
529
530 page_table = sun50i_dte_get_page_table(sun50i_domain, iova, gfp);
531 if (IS_ERR(page_table)) {
532 ret = PTR_ERR(page_table);
533 goto out;
534 }
535
536 pte_index = sun50i_iova_get_pte_index(iova);
537 pte_addr = &page_table[pte_index];
538 if (unlikely(sun50i_pte_is_page_valid(*pte_addr))) {
539 phys_addr_t page_phys = sun50i_pte_get_page_address(*pte_addr);
540 dev_err(iommu->dev,
541 "iova %pad already mapped to %pa cannot remap to %pa prot: %#x\n",
542 &iova, &page_phys, &paddr, prot);
543 ret = -EBUSY;
544 goto out;
545 }
546
547 *pte_addr = sun50i_mk_pte(paddr, prot);
548 sun50i_table_flush(sun50i_domain, pte_addr, 1);
549
550out:
551 return ret;
552}
553
554static size_t sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova,
555 size_t size, struct iommu_iotlb_gather *gather)
556{
557 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
558 phys_addr_t pt_phys;
559 u32 *pte_addr;
560 u32 dte;
561
562 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
563 if (!sun50i_dte_is_pt_valid(dte))
564 return 0;
565
566 pt_phys = sun50i_dte_get_pt_address(dte);
567 pte_addr = (u32 *)phys_to_virt(pt_phys) + sun50i_iova_get_pte_index(iova);
568
569 if (!sun50i_pte_is_page_valid(*pte_addr))
570 return 0;
571
572 memset(pte_addr, 0, sizeof(*pte_addr));
573 sun50i_table_flush(sun50i_domain, pte_addr, 1);
574
575 return SZ_4K;
576}
577
578static phys_addr_t sun50i_iommu_iova_to_phys(struct iommu_domain *domain,
579 dma_addr_t iova)
580{
581 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
582 phys_addr_t pt_phys;
583 u32 *page_table;
584 u32 dte, pte;
585
586 dte = sun50i_domain->dt[sun50i_iova_get_dte_index(iova)];
587 if (!sun50i_dte_is_pt_valid(dte))
588 return 0;
589
590 pt_phys = sun50i_dte_get_pt_address(dte);
591 page_table = (u32 *)phys_to_virt(pt_phys);
592 pte = page_table[sun50i_iova_get_pte_index(iova)];
593 if (!sun50i_pte_is_page_valid(pte))
594 return 0;
595
596 return sun50i_pte_get_page_address(pte) +
597 sun50i_iova_get_page_offset(iova);
598}
599
600static struct iommu_domain *sun50i_iommu_domain_alloc(unsigned type)
601{
602 struct sun50i_iommu_domain *sun50i_domain;
603
604 if (type != IOMMU_DOMAIN_DMA &&
605 type != IOMMU_DOMAIN_IDENTITY &&
606 type != IOMMU_DOMAIN_UNMANAGED)
607 return NULL;
608
609 sun50i_domain = kzalloc(sizeof(*sun50i_domain), GFP_KERNEL);
610 if (!sun50i_domain)
611 return NULL;
612
613 if (type == IOMMU_DOMAIN_DMA &&
614 iommu_get_dma_cookie(&sun50i_domain->domain))
615 goto err_free_domain;
616
617 sun50i_domain->dt = (u32 *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
618 get_order(DT_SIZE));
619 if (!sun50i_domain->dt)
620 goto err_put_cookie;
621
622 refcount_set(&sun50i_domain->refcnt, 1);
623
624 sun50i_domain->domain.geometry.aperture_start = 0;
625 sun50i_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32);
626 sun50i_domain->domain.geometry.force_aperture = true;
627
628 return &sun50i_domain->domain;
629
630err_put_cookie:
631 if (type == IOMMU_DOMAIN_DMA)
632 iommu_put_dma_cookie(&sun50i_domain->domain);
633
634err_free_domain:
635 kfree(sun50i_domain);
636
637 return NULL;
638}
639
640static void sun50i_iommu_domain_free(struct iommu_domain *domain)
641{
642 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
643
644 free_pages((unsigned long)sun50i_domain->dt, get_order(DT_SIZE));
645 sun50i_domain->dt = NULL;
646
647 iommu_put_dma_cookie(domain);
648
649 kfree(sun50i_domain);
650}
651
652static int sun50i_iommu_attach_domain(struct sun50i_iommu *iommu,
653 struct sun50i_iommu_domain *sun50i_domain)
654{
655 iommu->domain = &sun50i_domain->domain;
656 sun50i_domain->iommu = iommu;
657
658 sun50i_domain->dt_dma = dma_map_single(iommu->dev, sun50i_domain->dt,
659 DT_SIZE, DMA_TO_DEVICE);
660 if (dma_mapping_error(iommu->dev, sun50i_domain->dt_dma)) {
661 dev_err(iommu->dev, "Couldn't map L1 Page Table\n");
662 return -ENOMEM;
663 }
664
665 return sun50i_iommu_enable(iommu);
666}
667
668static void sun50i_iommu_detach_domain(struct sun50i_iommu *iommu,
669 struct sun50i_iommu_domain *sun50i_domain)
670{
671 unsigned int i;
672
673 for (i = 0; i < NUM_DT_ENTRIES; i++) {
674 phys_addr_t pt_phys;
675 u32 *page_table;
676 u32 *dte_addr;
677 u32 dte;
678
679 dte_addr = &sun50i_domain->dt[i];
680 dte = *dte_addr;
681 if (!sun50i_dte_is_pt_valid(dte))
682 continue;
683
684 memset(dte_addr, 0, sizeof(*dte_addr));
685 sun50i_table_flush(sun50i_domain, dte_addr, 1);
686
687 pt_phys = sun50i_dte_get_pt_address(dte);
688 page_table = phys_to_virt(pt_phys);
689 sun50i_iommu_free_page_table(iommu, page_table);
690 }
691
692
693 sun50i_iommu_disable(iommu);
694
695 dma_unmap_single(iommu->dev, virt_to_phys(sun50i_domain->dt),
696 DT_SIZE, DMA_TO_DEVICE);
697
698 iommu->domain = NULL;
699}
700
701static void sun50i_iommu_detach_device(struct iommu_domain *domain,
702 struct device *dev)
703{
704 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
705 struct sun50i_iommu *iommu = dev_iommu_priv_get(dev);
706
707 dev_dbg(dev, "Detaching from IOMMU domain\n");
708
709 if (iommu->domain != domain)
710 return;
711
712 if (refcount_dec_and_test(&sun50i_domain->refcnt))
713 sun50i_iommu_detach_domain(iommu, sun50i_domain);
714}
715
716static int sun50i_iommu_attach_device(struct iommu_domain *domain,
717 struct device *dev)
718{
719 struct sun50i_iommu_domain *sun50i_domain = to_sun50i_domain(domain);
720 struct sun50i_iommu *iommu;
721
722 iommu = sun50i_iommu_from_dev(dev);
723 if (!iommu)
724 return -ENODEV;
725
726 dev_dbg(dev, "Attaching to IOMMU domain\n");
727
728 refcount_inc(&sun50i_domain->refcnt);
729
730 if (iommu->domain == domain)
731 return 0;
732
733 if (iommu->domain)
734 sun50i_iommu_detach_device(iommu->domain, dev);
735
736 sun50i_iommu_attach_domain(iommu, sun50i_domain);
737
738 return 0;
739}
740
741static struct iommu_device *sun50i_iommu_probe_device(struct device *dev)
742{
743 struct sun50i_iommu *iommu;
744
745 iommu = sun50i_iommu_from_dev(dev);
746 if (!iommu)
747 return ERR_PTR(-ENODEV);
748
749 return &iommu->iommu;
750}
751
752static void sun50i_iommu_release_device(struct device *dev) {}
753
754static struct iommu_group *sun50i_iommu_device_group(struct device *dev)
755{
756 struct sun50i_iommu *iommu = sun50i_iommu_from_dev(dev);
757
758 return iommu_group_ref_get(iommu->group);
759}
760
761static int sun50i_iommu_of_xlate(struct device *dev,
762 struct of_phandle_args *args)
763{
764 struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
765 unsigned id = args->args[0];
766
767 dev_iommu_priv_set(dev, platform_get_drvdata(iommu_pdev));
768
769 return iommu_fwspec_add_ids(dev, &id, 1);
770}
771
772static const struct iommu_ops sun50i_iommu_ops = {
773 .pgsize_bitmap = SZ_4K,
774 .attach_dev = sun50i_iommu_attach_device,
775 .detach_dev = sun50i_iommu_detach_device,
776 .device_group = sun50i_iommu_device_group,
777 .domain_alloc = sun50i_iommu_domain_alloc,
778 .domain_free = sun50i_iommu_domain_free,
779 .flush_iotlb_all = sun50i_iommu_flush_iotlb_all,
780 .iotlb_sync = sun50i_iommu_iotlb_sync,
781 .iova_to_phys = sun50i_iommu_iova_to_phys,
782 .map = sun50i_iommu_map,
783 .of_xlate = sun50i_iommu_of_xlate,
784 .probe_device = sun50i_iommu_probe_device,
785 .release_device = sun50i_iommu_release_device,
786 .unmap = sun50i_iommu_unmap,
787};
788
789static void sun50i_iommu_report_fault(struct sun50i_iommu *iommu,
790 unsigned master, phys_addr_t iova,
791 unsigned prot)
792{
793 dev_err(iommu->dev, "Page fault for %pad (master %d, dir %s)\n",
794 &iova, master, (prot == IOMMU_FAULT_WRITE) ? "wr" : "rd");
795
796 if (iommu->domain)
797 report_iommu_fault(iommu->domain, iommu->dev, iova, prot);
798 else
799 dev_err(iommu->dev, "Page fault while iommu not attached to any domain?\n");
800}
801
802static phys_addr_t sun50i_iommu_handle_pt_irq(struct sun50i_iommu *iommu,
803 unsigned addr_reg,
804 unsigned blame_reg)
805{
806 phys_addr_t iova;
807 unsigned master;
808 u32 blame;
809
810 assert_spin_locked(&iommu->iommu_lock);
811
812 iova = iommu_read(iommu, addr_reg);
813 blame = iommu_read(iommu, blame_reg);
814 master = ilog2(blame & IOMMU_INT_MASTER_MASK);
815
816
817
818
819
820
821 sun50i_iommu_report_fault(iommu, master, iova, IOMMU_FAULT_READ);
822
823 return iova;
824}
825
826static phys_addr_t sun50i_iommu_handle_perm_irq(struct sun50i_iommu *iommu)
827{
828 enum sun50i_iommu_aci aci;
829 phys_addr_t iova;
830 unsigned master;
831 unsigned dir;
832 u32 blame;
833
834 assert_spin_locked(&iommu->iommu_lock);
835
836 blame = iommu_read(iommu, IOMMU_INT_STA_REG);
837 master = ilog2(blame & IOMMU_INT_MASTER_MASK);
838 iova = iommu_read(iommu, IOMMU_INT_ERR_ADDR_REG(master));
839 aci = sun50i_get_pte_aci(iommu_read(iommu,
840 IOMMU_INT_ERR_DATA_REG(master)));
841
842 switch (aci) {
843
844
845
846
847 case SUN50I_IOMMU_ACI_RD:
848 dir = IOMMU_FAULT_WRITE;
849 break;
850
851
852
853
854
855 case SUN50I_IOMMU_ACI_WR:
856
857
858
859
860
861
862 case SUN50I_IOMMU_ACI_NONE:
863
864
865 case SUN50I_IOMMU_ACI_RD_WR:
866 default:
867 dir = IOMMU_FAULT_READ;
868 break;
869 }
870
871
872
873
874
875
876 sun50i_iommu_report_fault(iommu, master, iova, dir);
877
878 return iova;
879}
880
881static irqreturn_t sun50i_iommu_irq(int irq, void *dev_id)
882{
883 struct sun50i_iommu *iommu = dev_id;
884 phys_addr_t iova;
885 u32 status;
886
887 spin_lock(&iommu->iommu_lock);
888
889 status = iommu_read(iommu, IOMMU_INT_STA_REG);
890 if (!(status & IOMMU_INT_MASK)) {
891 spin_unlock(&iommu->iommu_lock);
892 return IRQ_NONE;
893 }
894
895 if (status & IOMMU_INT_INVALID_L2PG)
896 iova = sun50i_iommu_handle_pt_irq(iommu,
897 IOMMU_INT_ERR_ADDR_L2_REG,
898 IOMMU_L2PG_INT_REG);
899 else if (status & IOMMU_INT_INVALID_L1PG)
900 iova = sun50i_iommu_handle_pt_irq(iommu,
901 IOMMU_INT_ERR_ADDR_L1_REG,
902 IOMMU_L1PG_INT_REG);
903 else
904 iova = sun50i_iommu_handle_perm_irq(iommu);
905
906 iommu_write(iommu, IOMMU_INT_CLR_REG, status);
907
908 iommu_write(iommu, IOMMU_RESET_REG, ~status);
909 iommu_write(iommu, IOMMU_RESET_REG, status);
910
911 spin_unlock(&iommu->iommu_lock);
912
913 return IRQ_HANDLED;
914}
915
916static int sun50i_iommu_probe(struct platform_device *pdev)
917{
918 struct sun50i_iommu *iommu;
919 int ret, irq;
920
921 iommu = devm_kzalloc(&pdev->dev, sizeof(*iommu), GFP_KERNEL);
922 if (!iommu)
923 return -ENOMEM;
924 spin_lock_init(&iommu->iommu_lock);
925 platform_set_drvdata(pdev, iommu);
926 iommu->dev = &pdev->dev;
927
928 iommu->pt_pool = kmem_cache_create(dev_name(&pdev->dev),
929 PT_SIZE, PT_SIZE,
930 SLAB_HWCACHE_ALIGN,
931 NULL);
932 if (!iommu->pt_pool)
933 return -ENOMEM;
934
935 iommu->group = iommu_group_alloc();
936 if (IS_ERR(iommu->group)) {
937 ret = PTR_ERR(iommu->group);
938 goto err_free_cache;
939 }
940
941 iommu->base = devm_platform_ioremap_resource(pdev, 0);
942 if (IS_ERR(iommu->base)) {
943 ret = PTR_ERR(iommu->base);
944 goto err_free_group;
945 }
946
947 irq = platform_get_irq(pdev, 0);
948 if (irq < 0) {
949 ret = irq;
950 goto err_free_group;
951 }
952
953 iommu->clk = devm_clk_get(&pdev->dev, NULL);
954 if (IS_ERR(iommu->clk)) {
955 dev_err(&pdev->dev, "Couldn't get our clock.\n");
956 ret = PTR_ERR(iommu->clk);
957 goto err_free_group;
958 }
959
960 iommu->reset = devm_reset_control_get(&pdev->dev, NULL);
961 if (IS_ERR(iommu->reset)) {
962 dev_err(&pdev->dev, "Couldn't get our reset line.\n");
963 ret = PTR_ERR(iommu->reset);
964 goto err_free_group;
965 }
966
967 ret = iommu_device_sysfs_add(&iommu->iommu, &pdev->dev,
968 NULL, dev_name(&pdev->dev));
969 if (ret)
970 goto err_free_group;
971
972 iommu_device_set_ops(&iommu->iommu, &sun50i_iommu_ops);
973 iommu_device_set_fwnode(&iommu->iommu, &pdev->dev.of_node->fwnode);
974
975 ret = iommu_device_register(&iommu->iommu);
976 if (ret)
977 goto err_remove_sysfs;
978
979 ret = devm_request_irq(&pdev->dev, irq, sun50i_iommu_irq, 0,
980 dev_name(&pdev->dev), iommu);
981 if (ret < 0)
982 goto err_unregister;
983
984 bus_set_iommu(&platform_bus_type, &sun50i_iommu_ops);
985
986 return 0;
987
988err_unregister:
989 iommu_device_unregister(&iommu->iommu);
990
991err_remove_sysfs:
992 iommu_device_sysfs_remove(&iommu->iommu);
993
994err_free_group:
995 iommu_group_put(iommu->group);
996
997err_free_cache:
998 kmem_cache_destroy(iommu->pt_pool);
999
1000 return ret;
1001}
1002
1003static const struct of_device_id sun50i_iommu_dt[] = {
1004 { .compatible = "allwinner,sun50i-h6-iommu", },
1005 { },
1006};
1007MODULE_DEVICE_TABLE(of, sun50i_iommu_dt);
1008
1009static struct platform_driver sun50i_iommu_driver = {
1010 .driver = {
1011 .name = "sun50i-iommu",
1012 .of_match_table = sun50i_iommu_dt,
1013 .suppress_bind_attrs = true,
1014 }
1015};
1016builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe);
1017
1018MODULE_DESCRIPTION("Allwinner H6 IOMMU driver");
1019MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>");
1020MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>");
1021MODULE_LICENSE("Dual BSD/GPL");
1022
