1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/iommu.h>
18#include <linux/platform_device.h>
19#include <linux/sizes.h>
20#include <linux/slab.h>
21#include <linux/dma-mapping.h>
22#include <linux/bitops.h>
23
24#include "etnaviv_gpu.h"
25#include "etnaviv_mmu.h"
26#include "etnaviv_iommu.h"
27#include "state.xml.h"
28#include "state_hi.xml.h"
29
30#define MMUv2_PTE_PRESENT BIT(0)
31#define MMUv2_PTE_EXCEPTION BIT(1)
32#define MMUv2_PTE_WRITEABLE BIT(2)
33
34#define MMUv2_MTLB_MASK 0xffc00000
35#define MMUv2_MTLB_SHIFT 22
36#define MMUv2_STLB_MASK 0x003ff000
37#define MMUv2_STLB_SHIFT 12
38
39#define MMUv2_MAX_STLB_ENTRIES 1024
40
41struct etnaviv_iommuv2_domain {
42 struct iommu_domain domain;
43 struct device *dev;
44 void *bad_page_cpu;
45 dma_addr_t bad_page_dma;
46
47 u32 *mtlb_cpu;
48 dma_addr_t mtlb_dma;
49
50 u32 *stlb_cpu[1024];
51 dma_addr_t stlb_dma[1024];
52};
53
54static struct etnaviv_iommuv2_domain *to_etnaviv_domain(struct iommu_domain *domain)
55{
56 return container_of(domain, struct etnaviv_iommuv2_domain, domain);
57}
58
59static int etnaviv_iommuv2_map(struct iommu_domain *domain, unsigned long iova,
60 phys_addr_t paddr, size_t size, int prot)
61{
62 struct etnaviv_iommuv2_domain *etnaviv_domain =
63 to_etnaviv_domain(domain);
64 int mtlb_entry, stlb_entry;
65 u32 entry = (u32)paddr | MMUv2_PTE_PRESENT;
66
67 if (size != SZ_4K)
68 return -EINVAL;
69
70 if (prot & IOMMU_WRITE)
71 entry |= MMUv2_PTE_WRITEABLE;
72
73 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
74 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
75
76 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = entry;
77
78 return 0;
79}
80
81static size_t etnaviv_iommuv2_unmap(struct iommu_domain *domain,
82 unsigned long iova, size_t size)
83{
84 struct etnaviv_iommuv2_domain *etnaviv_domain =
85 to_etnaviv_domain(domain);
86 int mtlb_entry, stlb_entry;
87
88 if (size != SZ_4K)
89 return -EINVAL;
90
91 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
92 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
93
94 etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] = MMUv2_PTE_EXCEPTION;
95
96 return SZ_4K;
97}
98
99static phys_addr_t etnaviv_iommuv2_iova_to_phys(struct iommu_domain *domain,
100 dma_addr_t iova)
101{
102 struct etnaviv_iommuv2_domain *etnaviv_domain =
103 to_etnaviv_domain(domain);
104 int mtlb_entry, stlb_entry;
105
106 mtlb_entry = (iova & MMUv2_MTLB_MASK) >> MMUv2_MTLB_SHIFT;
107 stlb_entry = (iova & MMUv2_STLB_MASK) >> MMUv2_STLB_SHIFT;
108
109 return etnaviv_domain->stlb_cpu[mtlb_entry][stlb_entry] & ~(SZ_4K - 1);
110}
111
112static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain)
113{
114 u32 *p;
115 int ret, i, j;
116
117
118 etnaviv_domain->bad_page_cpu = dma_alloc_coherent(etnaviv_domain->dev,
119 SZ_4K,
120 &etnaviv_domain->bad_page_dma,
121 GFP_KERNEL);
122 if (!etnaviv_domain->bad_page_cpu) {
123 ret = -ENOMEM;
124 goto fail_mem;
125 }
126 p = etnaviv_domain->bad_page_cpu;
127 for (i = 0; i < SZ_4K / 4; i++)
128 *p++ = 0xdead55aa;
129
130 etnaviv_domain->mtlb_cpu = dma_alloc_coherent(etnaviv_domain->dev,
131 SZ_4K,
132 &etnaviv_domain->mtlb_dma,
133 GFP_KERNEL);
134 if (!etnaviv_domain->mtlb_cpu) {
135 ret = -ENOMEM;
136 goto fail_mem;
137 }
138
139
140 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
141 etnaviv_domain->stlb_cpu[i] =
142 dma_alloc_coherent(etnaviv_domain->dev,
143 SZ_4K,
144 &etnaviv_domain->stlb_dma[i],
145 GFP_KERNEL);
146 if (!etnaviv_domain->stlb_cpu[i]) {
147 ret = -ENOMEM;
148 goto fail_mem;
149 }
150 p = etnaviv_domain->stlb_cpu[i];
151 for (j = 0; j < SZ_4K / 4; j++)
152 *p++ = MMUv2_PTE_EXCEPTION;
153
154 etnaviv_domain->mtlb_cpu[i] = etnaviv_domain->stlb_dma[i] |
155 MMUv2_PTE_PRESENT;
156 }
157
158 return 0;
159
160fail_mem:
161 if (etnaviv_domain->bad_page_cpu)
162 dma_free_coherent(etnaviv_domain->dev, SZ_4K,
163 etnaviv_domain->bad_page_cpu,
164 etnaviv_domain->bad_page_dma);
165
166 if (etnaviv_domain->mtlb_cpu)
167 dma_free_coherent(etnaviv_domain->dev, SZ_4K,
168 etnaviv_domain->mtlb_cpu,
169 etnaviv_domain->mtlb_dma);
170
171 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
172 if (etnaviv_domain->stlb_cpu[i])
173 dma_free_coherent(etnaviv_domain->dev, SZ_4K,
174 etnaviv_domain->stlb_cpu[i],
175 etnaviv_domain->stlb_dma[i]);
176 }
177
178 return ret;
179}
180
181static void etnaviv_iommuv2_domain_free(struct iommu_domain *domain)
182{
183 struct etnaviv_iommuv2_domain *etnaviv_domain =
184 to_etnaviv_domain(domain);
185 int i;
186
187 dma_free_coherent(etnaviv_domain->dev, SZ_4K,
188 etnaviv_domain->bad_page_cpu,
189 etnaviv_domain->bad_page_dma);
190
191 dma_free_coherent(etnaviv_domain->dev, SZ_4K,
192 etnaviv_domain->mtlb_cpu,
193 etnaviv_domain->mtlb_dma);
194
195 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++) {
196 if (etnaviv_domain->stlb_cpu[i])
197 dma_free_coherent(etnaviv_domain->dev, SZ_4K,
198 etnaviv_domain->stlb_cpu[i],
199 etnaviv_domain->stlb_dma[i]);
200 }
201
202 vfree(etnaviv_domain);
203}
204
205static size_t etnaviv_iommuv2_dump_size(struct iommu_domain *domain)
206{
207 struct etnaviv_iommuv2_domain *etnaviv_domain =
208 to_etnaviv_domain(domain);
209 size_t dump_size = SZ_4K;
210 int i;
211
212 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++)
213 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
214 dump_size += SZ_4K;
215
216 return dump_size;
217}
218
219static void etnaviv_iommuv2_dump(struct iommu_domain *domain, void *buf)
220{
221 struct etnaviv_iommuv2_domain *etnaviv_domain =
222 to_etnaviv_domain(domain);
223 int i;
224
225 memcpy(buf, etnaviv_domain->mtlb_cpu, SZ_4K);
226 buf += SZ_4K;
227 for (i = 0; i < MMUv2_MAX_STLB_ENTRIES; i++, buf += SZ_4K)
228 if (etnaviv_domain->mtlb_cpu[i] & MMUv2_PTE_PRESENT)
229 memcpy(buf, etnaviv_domain->stlb_cpu[i], SZ_4K);
230}
231
232static struct etnaviv_iommu_ops etnaviv_iommu_ops = {
233 .ops = {
234 .domain_free = etnaviv_iommuv2_domain_free,
235 .map = etnaviv_iommuv2_map,
236 .unmap = etnaviv_iommuv2_unmap,
237 .iova_to_phys = etnaviv_iommuv2_iova_to_phys,
238 .pgsize_bitmap = SZ_4K,
239 },
240 .dump_size = etnaviv_iommuv2_dump_size,
241 .dump = etnaviv_iommuv2_dump,
242};
243
244void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu)
245{
246 struct etnaviv_iommuv2_domain *etnaviv_domain =
247 to_etnaviv_domain(gpu->mmu->domain);
248 u16 prefetch;
249
250
251 if (gpu_read(gpu, VIVS_MMUv2_CONTROL) & VIVS_MMUv2_CONTROL_ENABLE)
252 return;
253
254 prefetch = etnaviv_buffer_config_mmuv2(gpu,
255 (u32)etnaviv_domain->mtlb_dma,
256 (u32)etnaviv_domain->bad_page_dma);
257 etnaviv_gpu_start_fe(gpu, gpu->buffer->paddr, prefetch);
258 etnaviv_gpu_wait_idle(gpu, 100);
259
260 gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE);
261}
262struct iommu_domain *etnaviv_iommuv2_domain_alloc(struct etnaviv_gpu *gpu)
263{
264 struct etnaviv_iommuv2_domain *etnaviv_domain;
265 int ret;
266
267 etnaviv_domain = vzalloc(sizeof(*etnaviv_domain));
268 if (!etnaviv_domain)
269 return NULL;
270
271 etnaviv_domain->dev = gpu->dev;
272
273 etnaviv_domain->domain.type = __IOMMU_DOMAIN_PAGING;
274 etnaviv_domain->domain.ops = &etnaviv_iommu_ops.ops;
275 etnaviv_domain->domain.pgsize_bitmap = SZ_4K;
276 etnaviv_domain->domain.geometry.aperture_start = 0;
277 etnaviv_domain->domain.geometry.aperture_end = ~0UL & ~(SZ_4K - 1);
278
279 ret = etnaviv_iommuv2_init(etnaviv_domain);
280 if (ret)
281 goto out_free;
282
283 return &etnaviv_domain->domain;
284
285out_free:
286 vfree(etnaviv_domain);
287 return NULL;
288}
289
