1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include <linux/export.h>
30#include <linux/dma-buf.h>
31#include <drm/drmP.h>
32#include <drm/drm_gem.h>
33
34#include "drm_internal.h"
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63struct drm_prime_member {
64 struct list_head entry;
65 struct dma_buf *dma_buf;
66 uint32_t handle;
67};
68
69struct drm_prime_attachment {
70 struct sg_table *sgt;
71 enum dma_data_direction dir;
72};
73
74static int drm_prime_add_buf_handle(struct drm_prime_file_private *prime_fpriv,
75 struct dma_buf *dma_buf, uint32_t handle)
76{
77 struct drm_prime_member *member;
78
79 member = kmalloc(sizeof(*member), GFP_KERNEL);
80 if (!member)
81 return -ENOMEM;
82
83 get_dma_buf(dma_buf);
84 member->dma_buf = dma_buf;
85 member->handle = handle;
86 list_add(&member->entry, &prime_fpriv->head);
87 return 0;
88}
89
90static struct dma_buf *drm_prime_lookup_buf_by_handle(struct drm_prime_file_private *prime_fpriv,
91 uint32_t handle)
92{
93 struct drm_prime_member *member;
94
95 list_for_each_entry(member, &prime_fpriv->head, entry) {
96 if (member->handle == handle)
97 return member->dma_buf;
98 }
99
100 return NULL;
101}
102
103static int drm_prime_lookup_buf_handle(struct drm_prime_file_private *prime_fpriv,
104 struct dma_buf *dma_buf,
105 uint32_t *handle)
106{
107 struct drm_prime_member *member;
108
109 list_for_each_entry(member, &prime_fpriv->head, entry) {
110 if (member->dma_buf == dma_buf) {
111 *handle = member->handle;
112 return 0;
113 }
114 }
115 return -ENOENT;
116}
117
118static int drm_gem_map_attach(struct dma_buf *dma_buf,
119 struct device *target_dev,
120 struct dma_buf_attachment *attach)
121{
122 struct drm_prime_attachment *prime_attach;
123 struct drm_gem_object *obj = dma_buf->priv;
124 struct drm_device *dev = obj->dev;
125
126 prime_attach = kzalloc(sizeof(*prime_attach), GFP_KERNEL);
127 if (!prime_attach)
128 return -ENOMEM;
129
130 prime_attach->dir = DMA_NONE;
131 attach->priv = prime_attach;
132
133 if (!dev->driver->gem_prime_pin)
134 return 0;
135
136 return dev->driver->gem_prime_pin(obj);
137}
138
139static void drm_gem_map_detach(struct dma_buf *dma_buf,
140 struct dma_buf_attachment *attach)
141{
142 struct drm_prime_attachment *prime_attach = attach->priv;
143 struct drm_gem_object *obj = dma_buf->priv;
144 struct drm_device *dev = obj->dev;
145 struct sg_table *sgt;
146
147 if (dev->driver->gem_prime_unpin)
148 dev->driver->gem_prime_unpin(obj);
149
150 if (!prime_attach)
151 return;
152
153 sgt = prime_attach->sgt;
154 if (sgt) {
155 if (prime_attach->dir != DMA_NONE)
156 dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents,
157 prime_attach->dir);
158 sg_free_table(sgt);
159 }
160
161 kfree(sgt);
162 kfree(prime_attach);
163 attach->priv = NULL;
164}
165
166void drm_prime_remove_buf_handle_locked(struct drm_prime_file_private *prime_fpriv,
167 struct dma_buf *dma_buf)
168{
169 struct drm_prime_member *member, *safe;
170
171 list_for_each_entry_safe(member, safe, &prime_fpriv->head, entry) {
172 if (member->dma_buf == dma_buf) {
173 dma_buf_put(dma_buf);
174 list_del(&member->entry);
175 kfree(member);
176 }
177 }
178}
179
180static struct sg_table *drm_gem_map_dma_buf(struct dma_buf_attachment *attach,
181 enum dma_data_direction dir)
182{
183 struct drm_prime_attachment *prime_attach = attach->priv;
184 struct drm_gem_object *obj = attach->dmabuf->priv;
185 struct sg_table *sgt;
186
187 if (WARN_ON(dir == DMA_NONE || !prime_attach))
188 return ERR_PTR(-EINVAL);
189
190
191 if (prime_attach->dir == dir)
192 return prime_attach->sgt;
193
194
195
196
197
198 if (WARN_ON(prime_attach->dir != DMA_NONE))
199 return ERR_PTR(-EBUSY);
200
201 sgt = obj->dev->driver->gem_prime_get_sg_table(obj);
202
203 if (!IS_ERR(sgt)) {
204 if (!dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir)) {
205 sg_free_table(sgt);
206 kfree(sgt);
207 sgt = ERR_PTR(-ENOMEM);
208 } else {
209 prime_attach->sgt = sgt;
210 prime_attach->dir = dir;
211 }
212 }
213
214 return sgt;
215}
216
217static void drm_gem_unmap_dma_buf(struct dma_buf_attachment *attach,
218 struct sg_table *sgt,
219 enum dma_data_direction dir)
220{
221
222}
223
224
225
226
227
228
229
230
231void drm_gem_dmabuf_release(struct dma_buf *dma_buf)
232{
233 struct drm_gem_object *obj = dma_buf->priv;
234
235
236 drm_gem_object_unreference_unlocked(obj);
237}
238EXPORT_SYMBOL(drm_gem_dmabuf_release);
239
240static void *drm_gem_dmabuf_vmap(struct dma_buf *dma_buf)
241{
242 struct drm_gem_object *obj = dma_buf->priv;
243 struct drm_device *dev = obj->dev;
244
245 return dev->driver->gem_prime_vmap(obj);
246}
247
248static void drm_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr)
249{
250 struct drm_gem_object *obj = dma_buf->priv;
251 struct drm_device *dev = obj->dev;
252
253 dev->driver->gem_prime_vunmap(obj, vaddr);
254}
255
256static void *drm_gem_dmabuf_kmap_atomic(struct dma_buf *dma_buf,
257 unsigned long page_num)
258{
259 return NULL;
260}
261
262static void drm_gem_dmabuf_kunmap_atomic(struct dma_buf *dma_buf,
263 unsigned long page_num, void *addr)
264{
265
266}
267static void *drm_gem_dmabuf_kmap(struct dma_buf *dma_buf,
268 unsigned long page_num)
269{
270 return NULL;
271}
272
273static void drm_gem_dmabuf_kunmap(struct dma_buf *dma_buf,
274 unsigned long page_num, void *addr)
275{
276
277}
278
279static int drm_gem_dmabuf_mmap(struct dma_buf *dma_buf,
280 struct vm_area_struct *vma)
281{
282 struct drm_gem_object *obj = dma_buf->priv;
283 struct drm_device *dev = obj->dev;
284
285 if (!dev->driver->gem_prime_mmap)
286 return -ENOSYS;
287
288 return dev->driver->gem_prime_mmap(obj, vma);
289}
290
291static const struct dma_buf_ops drm_gem_prime_dmabuf_ops = {
292 .attach = drm_gem_map_attach,
293 .detach = drm_gem_map_detach,
294 .map_dma_buf = drm_gem_map_dma_buf,
295 .unmap_dma_buf = drm_gem_unmap_dma_buf,
296 .release = drm_gem_dmabuf_release,
297 .kmap = drm_gem_dmabuf_kmap,
298 .kmap_atomic = drm_gem_dmabuf_kmap_atomic,
299 .kunmap = drm_gem_dmabuf_kunmap,
300 .kunmap_atomic = drm_gem_dmabuf_kunmap_atomic,
301 .mmap = drm_gem_dmabuf_mmap,
302 .vmap = drm_gem_dmabuf_vmap,
303 .vunmap = drm_gem_dmabuf_vunmap,
304};
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339struct dma_buf *drm_gem_prime_export(struct drm_device *dev,
340 struct drm_gem_object *obj, int flags)
341{
342 struct reservation_object *robj = NULL;
343
344 if (dev->driver->gem_prime_res_obj)
345 robj = dev->driver->gem_prime_res_obj(obj);
346
347 return dma_buf_export(obj, &drm_gem_prime_dmabuf_ops, obj->size,
348 flags, robj);
349}
350EXPORT_SYMBOL(drm_gem_prime_export);
351
352static struct dma_buf *export_and_register_object(struct drm_device *dev,
353 struct drm_gem_object *obj,
354 uint32_t flags)
355{
356 struct dma_buf *dmabuf;
357
358
359 if (obj->handle_count == 0) {
360 dmabuf = ERR_PTR(-ENOENT);
361 return dmabuf;
362 }
363
364 dmabuf = dev->driver->gem_prime_export(dev, obj, flags);
365 if (IS_ERR(dmabuf)) {
366
367
368
369 return dmabuf;
370 }
371
372
373
374
375
376
377 obj->dma_buf = dmabuf;
378 get_dma_buf(obj->dma_buf);
379
380 drm_gem_object_reference(obj);
381
382 return dmabuf;
383}
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398int drm_gem_prime_handle_to_fd(struct drm_device *dev,
399 struct drm_file *file_priv, uint32_t handle,
400 uint32_t flags,
401 int *prime_fd)
402{
403 struct drm_gem_object *obj;
404 int ret = 0;
405 struct dma_buf *dmabuf;
406
407 mutex_lock(&file_priv->prime.lock);
408 obj = drm_gem_object_lookup(dev, file_priv, handle);
409 if (!obj) {
410 ret = -ENOENT;
411 goto out_unlock;
412 }
413
414 dmabuf = drm_prime_lookup_buf_by_handle(&file_priv->prime, handle);
415 if (dmabuf) {
416 get_dma_buf(dmabuf);
417 goto out_have_handle;
418 }
419
420 mutex_lock(&dev->object_name_lock);
421
422 if (obj->import_attach) {
423 dmabuf = obj->import_attach->dmabuf;
424 get_dma_buf(dmabuf);
425 goto out_have_obj;
426 }
427
428 if (obj->dma_buf) {
429 get_dma_buf(obj->dma_buf);
430 dmabuf = obj->dma_buf;
431 goto out_have_obj;
432 }
433
434 dmabuf = export_and_register_object(dev, obj, flags);
435 if (IS_ERR(dmabuf)) {
436
437
438
439 ret = PTR_ERR(dmabuf);
440 mutex_unlock(&dev->object_name_lock);
441 goto out;
442 }
443
444out_have_obj:
445
446
447
448
449
450
451 ret = drm_prime_add_buf_handle(&file_priv->prime,
452 dmabuf, handle);
453 mutex_unlock(&dev->object_name_lock);
454 if (ret)
455 goto fail_put_dmabuf;
456
457out_have_handle:
458 ret = dma_buf_fd(dmabuf, flags);
459
460
461
462
463
464
465 if (ret < 0) {
466 goto fail_put_dmabuf;
467 } else {
468 *prime_fd = ret;
469 ret = 0;
470 }
471
472 goto out;
473
474fail_put_dmabuf:
475 dma_buf_put(dmabuf);
476out:
477 drm_gem_object_unreference_unlocked(obj);
478out_unlock:
479 mutex_unlock(&file_priv->prime.lock);
480
481 return ret;
482}
483EXPORT_SYMBOL(drm_gem_prime_handle_to_fd);
484
485
486
487
488
489
490
491
492
493struct drm_gem_object *drm_gem_prime_import(struct drm_device *dev,
494 struct dma_buf *dma_buf)
495{
496 struct dma_buf_attachment *attach;
497 struct sg_table *sgt;
498 struct drm_gem_object *obj;
499 int ret;
500
501 if (!dev->driver->gem_prime_import_sg_table)
502 return ERR_PTR(-EINVAL);
503
504 if (dma_buf->ops == &drm_gem_prime_dmabuf_ops) {
505 obj = dma_buf->priv;
506 if (obj->dev == dev) {
507
508
509
510
511 drm_gem_object_reference(obj);
512 return obj;
513 }
514 }
515
516 attach = dma_buf_attach(dma_buf, dev->dev);
517 if (IS_ERR(attach))
518 return ERR_CAST(attach);
519
520 get_dma_buf(dma_buf);
521
522 sgt = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL);
523 if (IS_ERR(sgt)) {
524 ret = PTR_ERR(sgt);
525 goto fail_detach;
526 }
527
528 obj = dev->driver->gem_prime_import_sg_table(dev, attach, sgt);
529 if (IS_ERR(obj)) {
530 ret = PTR_ERR(obj);
531 goto fail_unmap;
532 }
533
534 obj->import_attach = attach;
535
536 return obj;
537
538fail_unmap:
539 dma_buf_unmap_attachment(attach, sgt, DMA_BIDIRECTIONAL);
540fail_detach:
541 dma_buf_detach(dma_buf, attach);
542 dma_buf_put(dma_buf);
543
544 return ERR_PTR(ret);
545}
546EXPORT_SYMBOL(drm_gem_prime_import);
547
548
549
550
551
552
553
554
555
556
557
558
559
560int drm_gem_prime_fd_to_handle(struct drm_device *dev,
561 struct drm_file *file_priv, int prime_fd,
562 uint32_t *handle)
563{
564 struct dma_buf *dma_buf;
565 struct drm_gem_object *obj;
566 int ret;
567
568 dma_buf = dma_buf_get(prime_fd);
569 if (IS_ERR(dma_buf))
570 return PTR_ERR(dma_buf);
571
572 mutex_lock(&file_priv->prime.lock);
573
574 ret = drm_prime_lookup_buf_handle(&file_priv->prime,
575 dma_buf, handle);
576 if (ret == 0)
577 goto out_put;
578
579
580 mutex_lock(&dev->object_name_lock);
581 obj = dev->driver->gem_prime_import(dev, dma_buf);
582 if (IS_ERR(obj)) {
583 ret = PTR_ERR(obj);
584 goto out_unlock;
585 }
586
587 if (obj->dma_buf) {
588 WARN_ON(obj->dma_buf != dma_buf);
589 } else {
590 obj->dma_buf = dma_buf;
591 get_dma_buf(dma_buf);
592 }
593
594
595 ret = drm_gem_handle_create_tail(file_priv, obj, handle);
596 drm_gem_object_unreference_unlocked(obj);
597 if (ret)
598 goto out_put;
599
600 ret = drm_prime_add_buf_handle(&file_priv->prime,
601 dma_buf, *handle);
602 if (ret)
603 goto fail;
604
605 mutex_unlock(&file_priv->prime.lock);
606
607 dma_buf_put(dma_buf);
608
609 return 0;
610
611fail:
612
613
614
615 drm_gem_handle_delete(file_priv, *handle);
616out_unlock:
617 mutex_unlock(&dev->object_name_lock);
618out_put:
619 dma_buf_put(dma_buf);
620 mutex_unlock(&file_priv->prime.lock);
621 return ret;
622}
623EXPORT_SYMBOL(drm_gem_prime_fd_to_handle);
624
625int drm_prime_handle_to_fd_ioctl(struct drm_device *dev, void *data,
626 struct drm_file *file_priv)
627{
628 struct drm_prime_handle *args = data;
629 uint32_t flags;
630
631 if (!drm_core_check_feature(dev, DRIVER_PRIME))
632 return -EINVAL;
633
634 if (!dev->driver->prime_handle_to_fd)
635 return -ENOSYS;
636
637
638 if (args->flags & ~DRM_CLOEXEC)
639 return -EINVAL;
640
641
642 flags = args->flags & DRM_CLOEXEC;
643
644 return dev->driver->prime_handle_to_fd(dev, file_priv,
645 args->handle, flags, &args->fd);
646}
647
648int drm_prime_fd_to_handle_ioctl(struct drm_device *dev, void *data,
649 struct drm_file *file_priv)
650{
651 struct drm_prime_handle *args = data;
652
653 if (!drm_core_check_feature(dev, DRIVER_PRIME))
654 return -EINVAL;
655
656 if (!dev->driver->prime_fd_to_handle)
657 return -ENOSYS;
658
659 return dev->driver->prime_fd_to_handle(dev, file_priv,
660 args->fd, &args->handle);
661}
662
663
664
665
666
667
668
669
670
671
672struct sg_table *drm_prime_pages_to_sg(struct page **pages, unsigned int nr_pages)
673{
674 struct sg_table *sg = NULL;
675 int ret;
676
677 sg = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
678 if (!sg) {
679 ret = -ENOMEM;
680 goto out;
681 }
682
683 ret = sg_alloc_table_from_pages(sg, pages, nr_pages, 0,
684 nr_pages << PAGE_SHIFT, GFP_KERNEL);
685 if (ret)
686 goto out;
687
688 return sg;
689out:
690 kfree(sg);
691 return ERR_PTR(ret);
692}
693EXPORT_SYMBOL(drm_prime_pages_to_sg);
694
695
696
697
698
699
700
701
702
703
704
705int drm_prime_sg_to_page_addr_arrays(struct sg_table *sgt, struct page **pages,
706 dma_addr_t *addrs, int max_pages)
707{
708 unsigned count;
709 struct scatterlist *sg;
710 struct page *page;
711 u32 len;
712 int pg_index;
713 dma_addr_t addr;
714
715 pg_index = 0;
716 for_each_sg(sgt->sgl, sg, sgt->nents, count) {
717 len = sg->length;
718 page = sg_page(sg);
719 addr = sg_dma_address(sg);
720
721 while (len > 0) {
722 if (WARN_ON(pg_index >= max_pages))
723 return -1;
724 pages[pg_index] = page;
725 if (addrs)
726 addrs[pg_index] = addr;
727
728 page++;
729 addr += PAGE_SIZE;
730 len -= PAGE_SIZE;
731 pg_index++;
732 }
733 }
734 return 0;
735}
736EXPORT_SYMBOL(drm_prime_sg_to_page_addr_arrays);
737
738
739
740
741
742
743
744
745
746void drm_prime_gem_destroy(struct drm_gem_object *obj, struct sg_table *sg)
747{
748 struct dma_buf_attachment *attach;
749 struct dma_buf *dma_buf;
750 attach = obj->import_attach;
751 if (sg)
752 dma_buf_unmap_attachment(attach, sg, DMA_BIDIRECTIONAL);
753 dma_buf = attach->dmabuf;
754 dma_buf_detach(attach->dmabuf, attach);
755
756 dma_buf_put(dma_buf);
757}
758EXPORT_SYMBOL(drm_prime_gem_destroy);
759
760void drm_prime_init_file_private(struct drm_prime_file_private *prime_fpriv)
761{
762 INIT_LIST_HEAD(&prime_fpriv->head);
763 mutex_init(&prime_fpriv->lock);
764}
765
766void drm_prime_destroy_file_private(struct drm_prime_file_private *prime_fpriv)
767{
768
769 WARN_ON(!list_empty(&prime_fpriv->head));
770}
771