1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/dma-mapping.h>
14#include <linux/iopoll.h>
15#include <linux/pm_runtime.h>
16#include <linux/slab.h>
17#include <linux/vmalloc.h>
18
19#include <asm/set_memory.h>
20
21#include "ipu3-mmu.h"
22
23#define IPU3_PT_BITS 10
24#define IPU3_PT_PTES (1UL << IPU3_PT_BITS)
25#define IPU3_PT_SIZE (IPU3_PT_PTES << 2)
26#define IPU3_PT_ORDER (IPU3_PT_SIZE >> PAGE_SHIFT)
27
28#define IPU3_ADDR2PTE(addr) ((addr) >> IPU3_PAGE_SHIFT)
29#define IPU3_PTE2ADDR(pte) ((phys_addr_t)(pte) << IPU3_PAGE_SHIFT)
30
31#define IPU3_L2PT_SHIFT IPU3_PT_BITS
32#define IPU3_L2PT_MASK ((1UL << IPU3_L2PT_SHIFT) - 1)
33
34#define IPU3_L1PT_SHIFT IPU3_PT_BITS
35#define IPU3_L1PT_MASK ((1UL << IPU3_L1PT_SHIFT) - 1)
36
37#define IPU3_MMU_ADDRESS_BITS (IPU3_PAGE_SHIFT + \
38 IPU3_L2PT_SHIFT + \
39 IPU3_L1PT_SHIFT)
40
41#define IMGU_REG_BASE 0x4000
42#define REG_TLB_INVALIDATE (IMGU_REG_BASE + 0x300)
43#define TLB_INVALIDATE 1
44#define REG_L1_PHYS (IMGU_REG_BASE + 0x304)
45#define REG_GP_HALT (IMGU_REG_BASE + 0x5dc)
46#define REG_GP_HALTED (IMGU_REG_BASE + 0x5e0)
47
48struct imgu_mmu {
49 struct device *dev;
50 void __iomem *base;
51
52 spinlock_t lock;
53
54 void *dummy_page;
55 u32 dummy_page_pteval;
56
57 u32 *dummy_l2pt;
58 u32 dummy_l2pt_pteval;
59
60 u32 **l2pts;
61 u32 *l1pt;
62
63 struct imgu_mmu_info geometry;
64};
65
66static inline struct imgu_mmu *to_imgu_mmu(struct imgu_mmu_info *info)
67{
68 return container_of(info, struct imgu_mmu, geometry);
69}
70
71
72
73
74
75
76
77
78static void imgu_mmu_tlb_invalidate(struct imgu_mmu *mmu)
79{
80 writel(TLB_INVALIDATE, mmu->base + REG_TLB_INVALIDATE);
81}
82
83static void call_if_imgu_is_powered(struct imgu_mmu *mmu,
84 void (*func)(struct imgu_mmu *mmu))
85{
86 if (!pm_runtime_get_if_in_use(mmu->dev))
87 return;
88
89 func(mmu);
90 pm_runtime_put(mmu->dev);
91}
92
93
94
95
96
97
98
99
100
101static void imgu_mmu_set_halt(struct imgu_mmu *mmu, bool halt)
102{
103 int ret;
104 u32 val;
105
106 writel(halt, mmu->base + REG_GP_HALT);
107 ret = readl_poll_timeout(mmu->base + REG_GP_HALTED,
108 val, (val & 1) == halt, 1000, 100000);
109
110 if (ret)
111 dev_err(mmu->dev, "failed to %s CIO gate halt\n",
112 halt ? "set" : "clear");
113}
114
115
116
117
118
119
120
121static u32 *imgu_mmu_alloc_page_table(u32 pteval)
122{
123 u32 *pt;
124 int pte;
125
126 pt = (u32 *)__get_free_page(GFP_KERNEL);
127 if (!pt)
128 return NULL;
129
130 for (pte = 0; pte < IPU3_PT_PTES; pte++)
131 pt[pte] = pteval;
132
133 set_memory_uc((unsigned long)pt, IPU3_PT_ORDER);
134
135 return pt;
136}
137
138
139
140
141
142static void imgu_mmu_free_page_table(u32 *pt)
143{
144 set_memory_wb((unsigned long)pt, IPU3_PT_ORDER);
145 free_page((unsigned long)pt);
146}
147
148
149
150
151
152
153
154static inline void address_to_pte_idx(unsigned long iova, u32 *l1pt_idx,
155 u32 *l2pt_idx)
156{
157 iova >>= IPU3_PAGE_SHIFT;
158
159 if (l2pt_idx)
160 *l2pt_idx = iova & IPU3_L2PT_MASK;
161
162 iova >>= IPU3_L2PT_SHIFT;
163
164 if (l1pt_idx)
165 *l1pt_idx = iova & IPU3_L1PT_MASK;
166}
167
168static u32 *imgu_mmu_get_l2pt(struct imgu_mmu *mmu, u32 l1pt_idx)
169{
170 unsigned long flags;
171 u32 *l2pt, *new_l2pt;
172 u32 pteval;
173
174 spin_lock_irqsave(&mmu->lock, flags);
175
176 l2pt = mmu->l2pts[l1pt_idx];
177 if (l2pt) {
178 spin_unlock_irqrestore(&mmu->lock, flags);
179 return l2pt;
180 }
181
182 spin_unlock_irqrestore(&mmu->lock, flags);
183
184 new_l2pt = imgu_mmu_alloc_page_table(mmu->dummy_page_pteval);
185 if (!new_l2pt)
186 return NULL;
187
188 spin_lock_irqsave(&mmu->lock, flags);
189
190 dev_dbg(mmu->dev, "allocated page table %p for l1pt_idx %u\n",
191 new_l2pt, l1pt_idx);
192
193 l2pt = mmu->l2pts[l1pt_idx];
194 if (l2pt) {
195 spin_unlock_irqrestore(&mmu->lock, flags);
196 imgu_mmu_free_page_table(new_l2pt);
197 return l2pt;
198 }
199
200 l2pt = new_l2pt;
201 mmu->l2pts[l1pt_idx] = new_l2pt;
202
203 pteval = IPU3_ADDR2PTE(virt_to_phys(new_l2pt));
204 mmu->l1pt[l1pt_idx] = pteval;
205
206 spin_unlock_irqrestore(&mmu->lock, flags);
207 return l2pt;
208}
209
210static int __imgu_mmu_map(struct imgu_mmu *mmu, unsigned long iova,
211 phys_addr_t paddr)
212{
213 u32 l1pt_idx, l2pt_idx;
214 unsigned long flags;
215 u32 *l2pt;
216
217 if (!mmu)
218 return -ENODEV;
219
220 address_to_pte_idx(iova, &l1pt_idx, &l2pt_idx);
221
222 l2pt = imgu_mmu_get_l2pt(mmu, l1pt_idx);
223 if (!l2pt)
224 return -ENOMEM;
225
226 spin_lock_irqsave(&mmu->lock, flags);
227
228 if (l2pt[l2pt_idx] != mmu->dummy_page_pteval) {
229 spin_unlock_irqrestore(&mmu->lock, flags);
230 return -EBUSY;
231 }
232
233 l2pt[l2pt_idx] = IPU3_ADDR2PTE(paddr);
234
235 spin_unlock_irqrestore(&mmu->lock, flags);
236
237 return 0;
238}
239
240
241
242
243
244
245
246
247
248
249
250
251int imgu_mmu_map(struct imgu_mmu_info *info, unsigned long iova,
252 phys_addr_t paddr, size_t size)
253{
254 struct imgu_mmu *mmu = to_imgu_mmu(info);
255 int ret = 0;
256
257
258
259
260
261
262 if (!IS_ALIGNED(iova | paddr | size, IPU3_PAGE_SIZE)) {
263 dev_err(mmu->dev, "unaligned: iova 0x%lx pa %pa size 0x%zx\n",
264 iova, &paddr, size);
265 return -EINVAL;
266 }
267
268 dev_dbg(mmu->dev, "map: iova 0x%lx pa %pa size 0x%zx\n",
269 iova, &paddr, size);
270
271 while (size) {
272 dev_dbg(mmu->dev, "mapping: iova 0x%lx pa %pa\n", iova, &paddr);
273
274 ret = __imgu_mmu_map(mmu, iova, paddr);
275 if (ret)
276 break;
277
278 iova += IPU3_PAGE_SIZE;
279 paddr += IPU3_PAGE_SIZE;
280 size -= IPU3_PAGE_SIZE;
281 }
282
283 call_if_imgu_is_powered(mmu, imgu_mmu_tlb_invalidate);
284
285 return ret;
286}
287
288
289
290
291
292
293
294
295
296
297
298
299size_t imgu_mmu_map_sg(struct imgu_mmu_info *info, unsigned long iova,
300 struct scatterlist *sg, unsigned int nents)
301{
302 struct imgu_mmu *mmu = to_imgu_mmu(info);
303 struct scatterlist *s;
304 size_t s_length, mapped = 0;
305 unsigned int i;
306 int ret;
307
308 for_each_sg(sg, s, nents, i) {
309 phys_addr_t phys = page_to_phys(sg_page(s)) + s->offset;
310
311 s_length = s->length;
312
313 if (!IS_ALIGNED(s->offset, IPU3_PAGE_SIZE))
314 goto out_err;
315
316
317 if (i == nents - 1 && !IS_ALIGNED(s->length, IPU3_PAGE_SIZE))
318 s_length = PAGE_ALIGN(s->length);
319
320 ret = imgu_mmu_map(info, iova + mapped, phys, s_length);
321 if (ret)
322 goto out_err;
323
324 mapped += s_length;
325 }
326
327 call_if_imgu_is_powered(mmu, imgu_mmu_tlb_invalidate);
328
329 return mapped;
330
331out_err:
332
333 imgu_mmu_unmap(info, iova, mapped);
334
335 return 0;
336}
337
338static size_t __imgu_mmu_unmap(struct imgu_mmu *mmu,
339 unsigned long iova, size_t size)
340{
341 u32 l1pt_idx, l2pt_idx;
342 unsigned long flags;
343 size_t unmap = size;
344 u32 *l2pt;
345
346 if (!mmu)
347 return 0;
348
349 address_to_pte_idx(iova, &l1pt_idx, &l2pt_idx);
350
351 spin_lock_irqsave(&mmu->lock, flags);
352
353 l2pt = mmu->l2pts[l1pt_idx];
354 if (!l2pt) {
355 spin_unlock_irqrestore(&mmu->lock, flags);
356 return 0;
357 }
358
359 if (l2pt[l2pt_idx] == mmu->dummy_page_pteval)
360 unmap = 0;
361
362 l2pt[l2pt_idx] = mmu->dummy_page_pteval;
363
364 spin_unlock_irqrestore(&mmu->lock, flags);
365
366 return unmap;
367}
368
369
370
371
372
373
374
375
376
377
378
379size_t imgu_mmu_unmap(struct imgu_mmu_info *info, unsigned long iova,
380 size_t size)
381{
382 struct imgu_mmu *mmu = to_imgu_mmu(info);
383 size_t unmapped_page, unmapped = 0;
384
385
386
387
388
389
390 if (!IS_ALIGNED(iova | size, IPU3_PAGE_SIZE)) {
391 dev_err(mmu->dev, "unaligned: iova 0x%lx size 0x%zx\n",
392 iova, size);
393 return -EINVAL;
394 }
395
396 dev_dbg(mmu->dev, "unmap this: iova 0x%lx size 0x%zx\n", iova, size);
397
398
399
400
401
402 while (unmapped < size) {
403 unmapped_page = __imgu_mmu_unmap(mmu, iova, IPU3_PAGE_SIZE);
404 if (!unmapped_page)
405 break;
406
407 dev_dbg(mmu->dev, "unmapped: iova 0x%lx size 0x%zx\n",
408 iova, unmapped_page);
409
410 iova += unmapped_page;
411 unmapped += unmapped_page;
412 }
413
414 call_if_imgu_is_powered(mmu, imgu_mmu_tlb_invalidate);
415
416 return unmapped;
417}
418
419
420
421
422
423
424
425
426
427struct imgu_mmu_info *imgu_mmu_init(struct device *parent, void __iomem *base)
428{
429 struct imgu_mmu *mmu;
430 u32 pteval;
431
432 mmu = kzalloc(sizeof(*mmu), GFP_KERNEL);
433 if (!mmu)
434 return ERR_PTR(-ENOMEM);
435
436 mmu->dev = parent;
437 mmu->base = base;
438 spin_lock_init(&mmu->lock);
439
440
441 imgu_mmu_set_halt(mmu, true);
442
443
444
445
446
447 mmu->dummy_page = (void *)__get_free_page(GFP_KERNEL);
448 if (!mmu->dummy_page)
449 goto fail_group;
450 pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->dummy_page));
451 mmu->dummy_page_pteval = pteval;
452
453
454
455
456
457 mmu->dummy_l2pt = imgu_mmu_alloc_page_table(pteval);
458 if (!mmu->dummy_l2pt)
459 goto fail_dummy_page;
460 pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->dummy_l2pt));
461 mmu->dummy_l2pt_pteval = pteval;
462
463
464
465
466
467 mmu->l2pts = vzalloc(IPU3_PT_PTES * sizeof(*mmu->l2pts));
468 if (!mmu->l2pts)
469 goto fail_l2pt;
470
471
472 mmu->l1pt = imgu_mmu_alloc_page_table(mmu->dummy_l2pt_pteval);
473 if (!mmu->l1pt)
474 goto fail_l2pts;
475
476 pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->l1pt));
477 writel(pteval, mmu->base + REG_L1_PHYS);
478 imgu_mmu_tlb_invalidate(mmu);
479 imgu_mmu_set_halt(mmu, false);
480
481 mmu->geometry.aperture_start = 0;
482 mmu->geometry.aperture_end = DMA_BIT_MASK(IPU3_MMU_ADDRESS_BITS);
483
484 return &mmu->geometry;
485
486fail_l2pts:
487 vfree(mmu->l2pts);
488fail_l2pt:
489 imgu_mmu_free_page_table(mmu->dummy_l2pt);
490fail_dummy_page:
491 free_page((unsigned long)mmu->dummy_page);
492fail_group:
493 kfree(mmu);
494
495 return ERR_PTR(-ENOMEM);
496}
497
498
499
500
501
502
503void imgu_mmu_exit(struct imgu_mmu_info *info)
504{
505 struct imgu_mmu *mmu = to_imgu_mmu(info);
506
507
508 imgu_mmu_set_halt(mmu, true);
509 imgu_mmu_tlb_invalidate(mmu);
510
511 imgu_mmu_free_page_table(mmu->l1pt);
512 vfree(mmu->l2pts);
513 imgu_mmu_free_page_table(mmu->dummy_l2pt);
514 free_page((unsigned long)mmu->dummy_page);
515 kfree(mmu);
516}
517
518void imgu_mmu_suspend(struct imgu_mmu_info *info)
519{
520 struct imgu_mmu *mmu = to_imgu_mmu(info);
521
522 imgu_mmu_set_halt(mmu, true);
523}
524
525void imgu_mmu_resume(struct imgu_mmu_info *info)
526{
527 struct imgu_mmu *mmu = to_imgu_mmu(info);
528 u32 pteval;
529
530 imgu_mmu_set_halt(mmu, true);
531
532 pteval = IPU3_ADDR2PTE(virt_to_phys(mmu->l1pt));
533 writel(pteval, mmu->base + REG_L1_PHYS);
534
535 imgu_mmu_tlb_invalidate(mmu);
536 imgu_mmu_set_halt(mmu, false);
537}
538