1
2
3
4
5
6
7
8
9
10
11#include <linux/slab.h>
12#include <linux/file.h>
13#include <linux/anon_inodes.h>
14#include <linux/highmem.h>
15#include <linux/vmalloc.h>
16#include <linux/vdpa.h>
17
18#include "iova_domain.h"
19
20static int vduse_iotlb_add_range(struct vduse_iova_domain *domain,
21 u64 start, u64 last,
22 u64 addr, unsigned int perm,
23 struct file *file, u64 offset)
24{
25 struct vdpa_map_file *map_file;
26 int ret;
27
28 map_file = kmalloc(sizeof(*map_file), GFP_ATOMIC);
29 if (!map_file)
30 return -ENOMEM;
31
32 map_file->file = get_file(file);
33 map_file->offset = offset;
34
35 ret = vhost_iotlb_add_range_ctx(domain->iotlb, start, last,
36 addr, perm, map_file);
37 if (ret) {
38 fput(map_file->file);
39 kfree(map_file);
40 return ret;
41 }
42 return 0;
43}
44
45static void vduse_iotlb_del_range(struct vduse_iova_domain *domain,
46 u64 start, u64 last)
47{
48 struct vdpa_map_file *map_file;
49 struct vhost_iotlb_map *map;
50
51 while ((map = vhost_iotlb_itree_first(domain->iotlb, start, last))) {
52 map_file = (struct vdpa_map_file *)map->opaque;
53 fput(map_file->file);
54 kfree(map_file);
55 vhost_iotlb_map_free(domain->iotlb, map);
56 }
57}
58
59int vduse_domain_set_map(struct vduse_iova_domain *domain,
60 struct vhost_iotlb *iotlb)
61{
62 struct vdpa_map_file *map_file;
63 struct vhost_iotlb_map *map;
64 u64 start = 0ULL, last = ULLONG_MAX;
65 int ret;
66
67 spin_lock(&domain->iotlb_lock);
68 vduse_iotlb_del_range(domain, start, last);
69
70 for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
71 map = vhost_iotlb_itree_next(map, start, last)) {
72 map_file = (struct vdpa_map_file *)map->opaque;
73 ret = vduse_iotlb_add_range(domain, map->start, map->last,
74 map->addr, map->perm,
75 map_file->file,
76 map_file->offset);
77 if (ret)
78 goto err;
79 }
80 spin_unlock(&domain->iotlb_lock);
81
82 return 0;
83err:
84 vduse_iotlb_del_range(domain, start, last);
85 spin_unlock(&domain->iotlb_lock);
86 return ret;
87}
88
89void vduse_domain_clear_map(struct vduse_iova_domain *domain,
90 struct vhost_iotlb *iotlb)
91{
92 struct vhost_iotlb_map *map;
93 u64 start = 0ULL, last = ULLONG_MAX;
94
95 spin_lock(&domain->iotlb_lock);
96 for (map = vhost_iotlb_itree_first(iotlb, start, last); map;
97 map = vhost_iotlb_itree_next(map, start, last)) {
98 vduse_iotlb_del_range(domain, map->start, map->last);
99 }
100 spin_unlock(&domain->iotlb_lock);
101}
102
103static int vduse_domain_map_bounce_page(struct vduse_iova_domain *domain,
104 u64 iova, u64 size, u64 paddr)
105{
106 struct vduse_bounce_map *map;
107 u64 last = iova + size - 1;
108
109 while (iova <= last) {
110 map = &domain->bounce_maps[iova >> PAGE_SHIFT];
111 if (!map->bounce_page) {
112 map->bounce_page = alloc_page(GFP_ATOMIC);
113 if (!map->bounce_page)
114 return -ENOMEM;
115 }
116 map->orig_phys = paddr;
117 paddr += PAGE_SIZE;
118 iova += PAGE_SIZE;
119 }
120 return 0;
121}
122
123static void vduse_domain_unmap_bounce_page(struct vduse_iova_domain *domain,
124 u64 iova, u64 size)
125{
126 struct vduse_bounce_map *map;
127 u64 last = iova + size - 1;
128
129 while (iova <= last) {
130 map = &domain->bounce_maps[iova >> PAGE_SHIFT];
131 map->orig_phys = INVALID_PHYS_ADDR;
132 iova += PAGE_SIZE;
133 }
134}
135
136static void do_bounce(phys_addr_t orig, void *addr, size_t size,
137 enum dma_data_direction dir)
138{
139 unsigned long pfn = PFN_DOWN(orig);
140 unsigned int offset = offset_in_page(orig);
141 char *buffer;
142 unsigned int sz = 0;
143
144 while (size) {
145 sz = min_t(size_t, PAGE_SIZE - offset, size);
146
147 buffer = kmap_atomic(pfn_to_page(pfn));
148 if (dir == DMA_TO_DEVICE)
149 memcpy(addr, buffer + offset, sz);
150 else
151 memcpy(buffer + offset, addr, sz);
152 kunmap_atomic(buffer);
153
154 size -= sz;
155 pfn++;
156 addr += sz;
157 offset = 0;
158 }
159}
160
161static void vduse_domain_bounce(struct vduse_iova_domain *domain,
162 dma_addr_t iova, size_t size,
163 enum dma_data_direction dir)
164{
165 struct vduse_bounce_map *map;
166 unsigned int offset;
167 void *addr;
168 size_t sz;
169
170 if (iova >= domain->bounce_size)
171 return;
172
173 while (size) {
174 map = &domain->bounce_maps[iova >> PAGE_SHIFT];
175 offset = offset_in_page(iova);
176 sz = min_t(size_t, PAGE_SIZE - offset, size);
177
178 if (WARN_ON(!map->bounce_page ||
179 map->orig_phys == INVALID_PHYS_ADDR))
180 return;
181
182 addr = page_address(map->bounce_page) + offset;
183 do_bounce(map->orig_phys + offset, addr, sz, dir);
184 size -= sz;
185 iova += sz;
186 }
187}
188
189static struct page *
190vduse_domain_get_coherent_page(struct vduse_iova_domain *domain, u64 iova)
191{
192 u64 start = iova & PAGE_MASK;
193 u64 last = start + PAGE_SIZE - 1;
194 struct vhost_iotlb_map *map;
195 struct page *page = NULL;
196
197 spin_lock(&domain->iotlb_lock);
198 map = vhost_iotlb_itree_first(domain->iotlb, start, last);
199 if (!map)
200 goto out;
201
202 page = pfn_to_page((map->addr + iova - map->start) >> PAGE_SHIFT);
203 get_page(page);
204out:
205 spin_unlock(&domain->iotlb_lock);
206
207 return page;
208}
209
210static struct page *
211vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova)
212{
213 struct vduse_bounce_map *map;
214 struct page *page = NULL;
215
216 spin_lock(&domain->iotlb_lock);
217 map = &domain->bounce_maps[iova >> PAGE_SHIFT];
218 if (!map->bounce_page)
219 goto out;
220
221 page = map->bounce_page;
222 get_page(page);
223out:
224 spin_unlock(&domain->iotlb_lock);
225
226 return page;
227}
228
229static void
230vduse_domain_free_bounce_pages(struct vduse_iova_domain *domain)
231{
232 struct vduse_bounce_map *map;
233 unsigned long pfn, bounce_pfns;
234
235 bounce_pfns = domain->bounce_size >> PAGE_SHIFT;
236
237 for (pfn = 0; pfn < bounce_pfns; pfn++) {
238 map = &domain->bounce_maps[pfn];
239 if (WARN_ON(map->orig_phys != INVALID_PHYS_ADDR))
240 continue;
241
242 if (!map->bounce_page)
243 continue;
244
245 __free_page(map->bounce_page);
246 map->bounce_page = NULL;
247 }
248}
249
250void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain)
251{
252 if (!domain->bounce_map)
253 return;
254
255 spin_lock(&domain->iotlb_lock);
256 if (!domain->bounce_map)
257 goto unlock;
258
259 vduse_iotlb_del_range(domain, 0, domain->bounce_size - 1);
260 domain->bounce_map = 0;
261unlock:
262 spin_unlock(&domain->iotlb_lock);
263}
264
265static int vduse_domain_init_bounce_map(struct vduse_iova_domain *domain)
266{
267 int ret = 0;
268
269 if (domain->bounce_map)
270 return 0;
271
272 spin_lock(&domain->iotlb_lock);
273 if (domain->bounce_map)
274 goto unlock;
275
276 ret = vduse_iotlb_add_range(domain, 0, domain->bounce_size - 1,
277 0, VHOST_MAP_RW, domain->file, 0);
278 if (ret)
279 goto unlock;
280
281 domain->bounce_map = 1;
282unlock:
283 spin_unlock(&domain->iotlb_lock);
284 return ret;
285}
286
287static dma_addr_t
288vduse_domain_alloc_iova(struct iova_domain *iovad,
289 unsigned long size, unsigned long limit)
290{
291 unsigned long shift = iova_shift(iovad);
292 unsigned long iova_len = iova_align(iovad, size) >> shift;
293 unsigned long iova_pfn;
294
295 iova_pfn = alloc_iova_fast(iovad, iova_len, limit >> shift, true);
296
297 return (dma_addr_t)iova_pfn << shift;
298}
299
300static void vduse_domain_free_iova(struct iova_domain *iovad,
301 dma_addr_t iova, size_t size)
302{
303 unsigned long shift = iova_shift(iovad);
304 unsigned long iova_len = iova_align(iovad, size) >> shift;
305
306 free_iova_fast(iovad, iova >> shift, iova_len);
307}
308
309dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain,
310 struct page *page, unsigned long offset,
311 size_t size, enum dma_data_direction dir,
312 unsigned long attrs)
313{
314 struct iova_domain *iovad = &domain->stream_iovad;
315 unsigned long limit = domain->bounce_size - 1;
316 phys_addr_t pa = page_to_phys(page) + offset;
317 dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit);
318
319 if (!iova)
320 return DMA_MAPPING_ERROR;
321
322 if (vduse_domain_init_bounce_map(domain))
323 goto err;
324
325 if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa))
326 goto err;
327
328 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
329 vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE);
330
331 return iova;
332err:
333 vduse_domain_free_iova(iovad, iova, size);
334 return DMA_MAPPING_ERROR;
335}
336
337void vduse_domain_unmap_page(struct vduse_iova_domain *domain,
338 dma_addr_t dma_addr, size_t size,
339 enum dma_data_direction dir, unsigned long attrs)
340{
341 struct iova_domain *iovad = &domain->stream_iovad;
342
343 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
344 vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE);
345
346 vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size);
347 vduse_domain_free_iova(iovad, dma_addr, size);
348}
349
350void *vduse_domain_alloc_coherent(struct vduse_iova_domain *domain,
351 size_t size, dma_addr_t *dma_addr,
352 gfp_t flag, unsigned long attrs)
353{
354 struct iova_domain *iovad = &domain->consistent_iovad;
355 unsigned long limit = domain->iova_limit;
356 dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit);
357 void *orig = alloc_pages_exact(size, flag);
358
359 if (!iova || !orig)
360 goto err;
361
362 spin_lock(&domain->iotlb_lock);
363 if (vduse_iotlb_add_range(domain, (u64)iova, (u64)iova + size - 1,
364 virt_to_phys(orig), VHOST_MAP_RW,
365 domain->file, (u64)iova)) {
366 spin_unlock(&domain->iotlb_lock);
367 goto err;
368 }
369 spin_unlock(&domain->iotlb_lock);
370
371 *dma_addr = iova;
372
373 return orig;
374err:
375 *dma_addr = DMA_MAPPING_ERROR;
376 if (orig)
377 free_pages_exact(orig, size);
378 if (iova)
379 vduse_domain_free_iova(iovad, iova, size);
380
381 return NULL;
382}
383
384void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size,
385 void *vaddr, dma_addr_t dma_addr,
386 unsigned long attrs)
387{
388 struct iova_domain *iovad = &domain->consistent_iovad;
389 struct vhost_iotlb_map *map;
390 struct vdpa_map_file *map_file;
391 phys_addr_t pa;
392
393 spin_lock(&domain->iotlb_lock);
394 map = vhost_iotlb_itree_first(domain->iotlb, (u64)dma_addr,
395 (u64)dma_addr + size - 1);
396 if (WARN_ON(!map)) {
397 spin_unlock(&domain->iotlb_lock);
398 return;
399 }
400 map_file = (struct vdpa_map_file *)map->opaque;
401 fput(map_file->file);
402 kfree(map_file);
403 pa = map->addr;
404 vhost_iotlb_map_free(domain->iotlb, map);
405 spin_unlock(&domain->iotlb_lock);
406
407 vduse_domain_free_iova(iovad, dma_addr, size);
408 free_pages_exact(phys_to_virt(pa), size);
409}
410
411static vm_fault_t vduse_domain_mmap_fault(struct vm_fault *vmf)
412{
413 struct vduse_iova_domain *domain = vmf->vma->vm_private_data;
414 unsigned long iova = vmf->pgoff << PAGE_SHIFT;
415 struct page *page;
416
417 if (!domain)
418 return VM_FAULT_SIGBUS;
419
420 if (iova < domain->bounce_size)
421 page = vduse_domain_get_bounce_page(domain, iova);
422 else
423 page = vduse_domain_get_coherent_page(domain, iova);
424
425 if (!page)
426 return VM_FAULT_SIGBUS;
427
428 vmf->page = page;
429
430 return 0;
431}
432
433static const struct vm_operations_struct vduse_domain_mmap_ops = {
434 .fault = vduse_domain_mmap_fault,
435};
436
437static int vduse_domain_mmap(struct file *file, struct vm_area_struct *vma)
438{
439 struct vduse_iova_domain *domain = file->private_data;
440
441 vma->vm_flags |= VM_DONTDUMP | VM_DONTEXPAND;
442 vma->vm_private_data = domain;
443 vma->vm_ops = &vduse_domain_mmap_ops;
444
445 return 0;
446}
447
448static int vduse_domain_release(struct inode *inode, struct file *file)
449{
450 struct vduse_iova_domain *domain = file->private_data;
451
452 spin_lock(&domain->iotlb_lock);
453 vduse_iotlb_del_range(domain, 0, ULLONG_MAX);
454 vduse_domain_free_bounce_pages(domain);
455 spin_unlock(&domain->iotlb_lock);
456 put_iova_domain(&domain->stream_iovad);
457 put_iova_domain(&domain->consistent_iovad);
458 vhost_iotlb_free(domain->iotlb);
459 vfree(domain->bounce_maps);
460 kfree(domain);
461
462 return 0;
463}
464
465static const struct file_operations vduse_domain_fops = {
466 .owner = THIS_MODULE,
467 .mmap = vduse_domain_mmap,
468 .release = vduse_domain_release,
469};
470
471void vduse_domain_destroy(struct vduse_iova_domain *domain)
472{
473 fput(domain->file);
474}
475
476struct vduse_iova_domain *
477vduse_domain_create(unsigned long iova_limit, size_t bounce_size)
478{
479 struct vduse_iova_domain *domain;
480 struct file *file;
481 struct vduse_bounce_map *map;
482 unsigned long pfn, bounce_pfns;
483 int ret;
484
485 bounce_pfns = PAGE_ALIGN(bounce_size) >> PAGE_SHIFT;
486 if (iova_limit <= bounce_size)
487 return NULL;
488
489 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
490 if (!domain)
491 return NULL;
492
493 domain->iotlb = vhost_iotlb_alloc(0, 0);
494 if (!domain->iotlb)
495 goto err_iotlb;
496
497 domain->iova_limit = iova_limit;
498 domain->bounce_size = PAGE_ALIGN(bounce_size);
499 domain->bounce_maps = vzalloc(bounce_pfns *
500 sizeof(struct vduse_bounce_map));
501 if (!domain->bounce_maps)
502 goto err_map;
503
504 for (pfn = 0; pfn < bounce_pfns; pfn++) {
505 map = &domain->bounce_maps[pfn];
506 map->orig_phys = INVALID_PHYS_ADDR;
507 }
508 file = anon_inode_getfile("[vduse-domain]", &vduse_domain_fops,
509 domain, O_RDWR);
510 if (IS_ERR(file))
511 goto err_file;
512
513 domain->file = file;
514 spin_lock_init(&domain->iotlb_lock);
515 init_iova_domain(&domain->stream_iovad,
516 PAGE_SIZE, IOVA_START_PFN);
517 ret = iova_domain_init_rcaches(&domain->stream_iovad);
518 if (ret)
519 goto err_iovad_stream;
520 init_iova_domain(&domain->consistent_iovad,
521 PAGE_SIZE, bounce_pfns);
522 ret = iova_domain_init_rcaches(&domain->consistent_iovad);
523 if (ret)
524 goto err_iovad_consistent;
525
526 return domain;
527err_iovad_consistent:
528 put_iova_domain(&domain->stream_iovad);
529err_iovad_stream:
530 fput(file);
531err_file:
532 vfree(domain->bounce_maps);
533err_map:
534 vhost_iotlb_free(domain->iotlb);
535err_iotlb:
536 kfree(domain);
537 return NULL;
538}
539
540int vduse_domain_init(void)
541{
542 return iova_cache_get();
543}
544
545void vduse_domain_exit(void)
546{
547 iova_cache_put();
548}
549
