1
2
3
4
5
6
7
8
9
10
11
12#include <drm/drmP.h>
13
14#include <linux/shmem_fs.h>
15#include <drm/exynos_drm.h>
16
17#include "exynos_drm_drv.h"
18#include "exynos_drm_gem.h"
19#include "exynos_drm_buf.h"
20
21static unsigned int convert_to_vm_err_msg(int msg)
22{
23 unsigned int out_msg;
24
25 switch (msg) {
26 case 0:
27 case -ERESTARTSYS:
28 case -EINTR:
29 out_msg = VM_FAULT_NOPAGE;
30 break;
31
32 case -ENOMEM:
33 out_msg = VM_FAULT_OOM;
34 break;
35
36 default:
37 out_msg = VM_FAULT_SIGBUS;
38 break;
39 }
40
41 return out_msg;
42}
43
44static int check_gem_flags(unsigned int flags)
45{
46 if (flags & ~(EXYNOS_BO_MASK)) {
47 DRM_ERROR("invalid flags.\n");
48 return -EINVAL;
49 }
50
51 return 0;
52}
53
54static void update_vm_cache_attr(struct exynos_drm_gem_obj *obj,
55 struct vm_area_struct *vma)
56{
57 DRM_DEBUG_KMS("flags = 0x%x\n", obj->flags);
58
59
60 if (obj->flags & EXYNOS_BO_CACHABLE)
61 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
62 else if (obj->flags & EXYNOS_BO_WC)
63 vma->vm_page_prot =
64 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
65 else
66 vma->vm_page_prot =
67 pgprot_noncached(vm_get_page_prot(vma->vm_flags));
68}
69
70static unsigned long roundup_gem_size(unsigned long size, unsigned int flags)
71{
72
73
74 return roundup(size, PAGE_SIZE);
75}
76
77static int exynos_drm_gem_map_buf(struct drm_gem_object *obj,
78 struct vm_area_struct *vma,
79 unsigned long f_vaddr,
80 pgoff_t page_offset)
81{
82 struct exynos_drm_gem_obj *exynos_gem_obj = to_exynos_gem_obj(obj);
83 struct exynos_drm_gem_buf *buf = exynos_gem_obj->buffer;
84 struct scatterlist *sgl;
85 unsigned long pfn;
86 int i;
87
88 if (!buf->sgt)
89 return -EINTR;
90
91 if (page_offset >= (buf->size >> PAGE_SHIFT)) {
92 DRM_ERROR("invalid page offset\n");
93 return -EINVAL;
94 }
95
96 sgl = buf->sgt->sgl;
97 for_each_sg(buf->sgt->sgl, sgl, buf->sgt->nents, i) {
98 if (page_offset < (sgl->length >> PAGE_SHIFT))
99 break;
100 page_offset -= (sgl->length >> PAGE_SHIFT);
101 }
102
103 pfn = __phys_to_pfn(sg_phys(sgl)) + page_offset;
104
105 return vm_insert_mixed(vma, f_vaddr, pfn);
106}
107
108static int exynos_drm_gem_handle_create(struct drm_gem_object *obj,
109 struct drm_file *file_priv,
110 unsigned int *handle)
111{
112 int ret;
113
114
115
116
117
118 ret = drm_gem_handle_create(file_priv, obj, handle);
119 if (ret)
120 return ret;
121
122 DRM_DEBUG_KMS("gem handle = 0x%x\n", *handle);
123
124
125 drm_gem_object_unreference_unlocked(obj);
126
127 return 0;
128}
129
130void exynos_drm_gem_destroy(struct exynos_drm_gem_obj *exynos_gem_obj)
131{
132 struct drm_gem_object *obj;
133 struct exynos_drm_gem_buf *buf;
134
135 obj = &exynos_gem_obj->base;
136 buf = exynos_gem_obj->buffer;
137
138 DRM_DEBUG_KMS("handle count = %d\n", atomic_read(&obj->handle_count));
139
140
141
142
143
144
145
146 if (obj->import_attach)
147 goto out;
148
149 exynos_drm_free_buf(obj->dev, exynos_gem_obj->flags, buf);
150
151out:
152 exynos_drm_fini_buf(obj->dev, buf);
153 exynos_gem_obj->buffer = NULL;
154
155 if (obj->map_list.map)
156 drm_gem_free_mmap_offset(obj);
157
158
159 drm_gem_object_release(obj);
160
161 kfree(exynos_gem_obj);
162 exynos_gem_obj = NULL;
163}
164
165unsigned long exynos_drm_gem_get_size(struct drm_device *dev,
166 unsigned int gem_handle,
167 struct drm_file *file_priv)
168{
169 struct exynos_drm_gem_obj *exynos_gem_obj;
170 struct drm_gem_object *obj;
171
172 obj = drm_gem_object_lookup(dev, file_priv, gem_handle);
173 if (!obj) {
174 DRM_ERROR("failed to lookup gem object.\n");
175 return 0;
176 }
177
178 exynos_gem_obj = to_exynos_gem_obj(obj);
179
180 drm_gem_object_unreference_unlocked(obj);
181
182 return exynos_gem_obj->buffer->size;
183}
184
185
186struct exynos_drm_gem_obj *exynos_drm_gem_init(struct drm_device *dev,
187 unsigned long size)
188{
189 struct exynos_drm_gem_obj *exynos_gem_obj;
190 struct drm_gem_object *obj;
191 int ret;
192
193 exynos_gem_obj = kzalloc(sizeof(*exynos_gem_obj), GFP_KERNEL);
194 if (!exynos_gem_obj) {
195 DRM_ERROR("failed to allocate exynos gem object\n");
196 return NULL;
197 }
198
199 exynos_gem_obj->size = size;
200 obj = &exynos_gem_obj->base;
201
202 ret = drm_gem_object_init(dev, obj, size);
203 if (ret < 0) {
204 DRM_ERROR("failed to initialize gem object\n");
205 kfree(exynos_gem_obj);
206 return NULL;
207 }
208
209 DRM_DEBUG_KMS("created file object = 0x%x\n", (unsigned int)obj->filp);
210
211 return exynos_gem_obj;
212}
213
214struct exynos_drm_gem_obj *exynos_drm_gem_create(struct drm_device *dev,
215 unsigned int flags,
216 unsigned long size)
217{
218 struct exynos_drm_gem_obj *exynos_gem_obj;
219 struct exynos_drm_gem_buf *buf;
220 int ret;
221
222 if (!size) {
223 DRM_ERROR("invalid size.\n");
224 return ERR_PTR(-EINVAL);
225 }
226
227 size = roundup_gem_size(size, flags);
228
229 ret = check_gem_flags(flags);
230 if (ret)
231 return ERR_PTR(ret);
232
233 buf = exynos_drm_init_buf(dev, size);
234 if (!buf)
235 return ERR_PTR(-ENOMEM);
236
237 exynos_gem_obj = exynos_drm_gem_init(dev, size);
238 if (!exynos_gem_obj) {
239 ret = -ENOMEM;
240 goto err_fini_buf;
241 }
242
243 exynos_gem_obj->buffer = buf;
244
245
246 exynos_gem_obj->flags = flags;
247
248 ret = exynos_drm_alloc_buf(dev, buf, flags);
249 if (ret < 0)
250 goto err_gem_fini;
251
252 return exynos_gem_obj;
253
254err_gem_fini:
255 drm_gem_object_release(&exynos_gem_obj->base);
256 kfree(exynos_gem_obj);
257err_fini_buf:
258 exynos_drm_fini_buf(dev, buf);
259 return ERR_PTR(ret);
260}
261
262int exynos_drm_gem_create_ioctl(struct drm_device *dev, void *data,
263 struct drm_file *file_priv)
264{
265 struct drm_exynos_gem_create *args = data;
266 struct exynos_drm_gem_obj *exynos_gem_obj;
267 int ret;
268
269 exynos_gem_obj = exynos_drm_gem_create(dev, args->flags, args->size);
270 if (IS_ERR(exynos_gem_obj))
271 return PTR_ERR(exynos_gem_obj);
272
273 ret = exynos_drm_gem_handle_create(&exynos_gem_obj->base, file_priv,
274 &args->handle);
275 if (ret) {
276 exynos_drm_gem_destroy(exynos_gem_obj);
277 return ret;
278 }
279
280 return 0;
281}
282
283dma_addr_t *exynos_drm_gem_get_dma_addr(struct drm_device *dev,
284 unsigned int gem_handle,
285 struct drm_file *filp)
286{
287 struct exynos_drm_gem_obj *exynos_gem_obj;
288 struct drm_gem_object *obj;
289
290 obj = drm_gem_object_lookup(dev, filp, gem_handle);
291 if (!obj) {
292 DRM_ERROR("failed to lookup gem object.\n");
293 return ERR_PTR(-EINVAL);
294 }
295
296 exynos_gem_obj = to_exynos_gem_obj(obj);
297
298 return &exynos_gem_obj->buffer->dma_addr;
299}
300
301void exynos_drm_gem_put_dma_addr(struct drm_device *dev,
302 unsigned int gem_handle,
303 struct drm_file *filp)
304{
305 struct exynos_drm_gem_obj *exynos_gem_obj;
306 struct drm_gem_object *obj;
307
308 obj = drm_gem_object_lookup(dev, filp, gem_handle);
309 if (!obj) {
310 DRM_ERROR("failed to lookup gem object.\n");
311 return;
312 }
313
314 exynos_gem_obj = to_exynos_gem_obj(obj);
315
316 drm_gem_object_unreference_unlocked(obj);
317
318
319
320
321
322 drm_gem_object_unreference_unlocked(obj);
323}
324
325int exynos_drm_gem_map_offset_ioctl(struct drm_device *dev, void *data,
326 struct drm_file *file_priv)
327{
328 struct drm_exynos_gem_map_off *args = data;
329
330 DRM_DEBUG_KMS("handle = 0x%x, offset = 0x%lx\n",
331 args->handle, (unsigned long)args->offset);
332
333 if (!(dev->driver->driver_features & DRIVER_GEM)) {
334 DRM_ERROR("does not support GEM.\n");
335 return -ENODEV;
336 }
337
338 return exynos_drm_gem_dumb_map_offset(file_priv, dev, args->handle,
339 &args->offset);
340}
341
342static struct drm_file *exynos_drm_find_drm_file(struct drm_device *drm_dev,
343 struct file *filp)
344{
345 struct drm_file *file_priv;
346
347
348 list_for_each_entry(file_priv, &drm_dev->filelist, lhead)
349 if (file_priv->filp == filp)
350 return file_priv;
351
352 WARN_ON(1);
353
354 return ERR_PTR(-EFAULT);
355}
356
357static int exynos_drm_gem_mmap_buffer(struct file *filp,
358 struct vm_area_struct *vma)
359{
360 struct drm_gem_object *obj = filp->private_data;
361 struct exynos_drm_gem_obj *exynos_gem_obj = to_exynos_gem_obj(obj);
362 struct drm_device *drm_dev = obj->dev;
363 struct exynos_drm_gem_buf *buffer;
364 struct drm_file *file_priv;
365 unsigned long vm_size;
366 int ret;
367
368 vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP;
369 vma->vm_private_data = obj;
370 vma->vm_ops = drm_dev->driver->gem_vm_ops;
371
372
373 filp->f_op = fops_get(drm_dev->driver->fops);
374
375 file_priv = exynos_drm_find_drm_file(drm_dev, filp);
376 if (IS_ERR(file_priv))
377 return PTR_ERR(file_priv);
378
379
380 filp->private_data = file_priv;
381
382 update_vm_cache_attr(exynos_gem_obj, vma);
383
384 vm_size = vma->vm_end - vma->vm_start;
385
386
387
388
389
390 buffer = exynos_gem_obj->buffer;
391
392
393 if (vm_size > buffer->size)
394 return -EINVAL;
395
396 ret = dma_mmap_attrs(drm_dev->dev, vma, buffer->pages,
397 buffer->dma_addr, buffer->size,
398 &buffer->dma_attrs);
399 if (ret < 0) {
400 DRM_ERROR("failed to mmap.\n");
401 return ret;
402 }
403
404
405
406
407
408 drm_gem_object_reference(obj);
409
410 drm_vm_open_locked(drm_dev, vma);
411
412 return 0;
413}
414
415static const struct file_operations exynos_drm_gem_fops = {
416 .mmap = exynos_drm_gem_mmap_buffer,
417};
418
419int exynos_drm_gem_mmap_ioctl(struct drm_device *dev, void *data,
420 struct drm_file *file_priv)
421{
422 struct drm_exynos_gem_mmap *args = data;
423 struct drm_gem_object *obj;
424 unsigned long addr;
425
426 if (!(dev->driver->driver_features & DRIVER_GEM)) {
427 DRM_ERROR("does not support GEM.\n");
428 return -ENODEV;
429 }
430
431 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
432 if (!obj) {
433 DRM_ERROR("failed to lookup gem object.\n");
434 return -EINVAL;
435 }
436
437
438
439
440
441
442
443
444
445 mutex_lock(&dev->struct_mutex);
446
447
448
449
450
451
452 file_priv->filp->f_op = &exynos_drm_gem_fops;
453
454
455
456
457
458
459 file_priv->filp->private_data = obj;
460
461 addr = vm_mmap(file_priv->filp, 0, args->size,
462 PROT_READ | PROT_WRITE, MAP_SHARED, 0);
463
464 drm_gem_object_unreference(obj);
465
466 if (IS_ERR_VALUE(addr)) {
467
468 if (file_priv->filp->f_op == &exynos_drm_gem_fops) {
469 file_priv->filp->f_op = fops_get(dev->driver->fops);
470 file_priv->filp->private_data = file_priv;
471 }
472 mutex_unlock(&dev->struct_mutex);
473 return (int)addr;
474 }
475
476 mutex_unlock(&dev->struct_mutex);
477
478 args->mapped = addr;
479
480 DRM_DEBUG_KMS("mapped = 0x%lx\n", (unsigned long)args->mapped);
481
482 return 0;
483}
484
485int exynos_drm_gem_get_ioctl(struct drm_device *dev, void *data,
486 struct drm_file *file_priv)
487{ struct exynos_drm_gem_obj *exynos_gem_obj;
488 struct drm_exynos_gem_info *args = data;
489 struct drm_gem_object *obj;
490
491 mutex_lock(&dev->struct_mutex);
492
493 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
494 if (!obj) {
495 DRM_ERROR("failed to lookup gem object.\n");
496 mutex_unlock(&dev->struct_mutex);
497 return -EINVAL;
498 }
499
500 exynos_gem_obj = to_exynos_gem_obj(obj);
501
502 args->flags = exynos_gem_obj->flags;
503 args->size = exynos_gem_obj->size;
504
505 drm_gem_object_unreference(obj);
506 mutex_unlock(&dev->struct_mutex);
507
508 return 0;
509}
510
511struct vm_area_struct *exynos_gem_get_vma(struct vm_area_struct *vma)
512{
513 struct vm_area_struct *vma_copy;
514
515 vma_copy = kmalloc(sizeof(*vma_copy), GFP_KERNEL);
516 if (!vma_copy)
517 return NULL;
518
519 if (vma->vm_ops && vma->vm_ops->open)
520 vma->vm_ops->open(vma);
521
522 if (vma->vm_file)
523 get_file(vma->vm_file);
524
525 memcpy(vma_copy, vma, sizeof(*vma));
526
527 vma_copy->vm_mm = NULL;
528 vma_copy->vm_next = NULL;
529 vma_copy->vm_prev = NULL;
530
531 return vma_copy;
532}
533
534void exynos_gem_put_vma(struct vm_area_struct *vma)
535{
536 if (!vma)
537 return;
538
539 if (vma->vm_ops && vma->vm_ops->close)
540 vma->vm_ops->close(vma);
541
542 if (vma->vm_file)
543 fput(vma->vm_file);
544
545 kfree(vma);
546}
547
548int exynos_gem_get_pages_from_userptr(unsigned long start,
549 unsigned int npages,
550 struct page **pages,
551 struct vm_area_struct *vma)
552{
553 int get_npages;
554
555
556 if (vma_is_io(vma)) {
557 unsigned int i;
558
559 for (i = 0; i < npages; ++i, start += PAGE_SIZE) {
560 unsigned long pfn;
561 int ret = follow_pfn(vma, start, &pfn);
562 if (ret)
563 return ret;
564
565 pages[i] = pfn_to_page(pfn);
566 }
567
568 if (i != npages) {
569 DRM_ERROR("failed to get user_pages.\n");
570 return -EINVAL;
571 }
572
573 return 0;
574 }
575
576 get_npages = get_user_pages(current, current->mm, start,
577 npages, 1, 1, pages, NULL);
578 get_npages = max(get_npages, 0);
579 if (get_npages != npages) {
580 DRM_ERROR("failed to get user_pages.\n");
581 while (get_npages)
582 put_page(pages[--get_npages]);
583 return -EFAULT;
584 }
585
586 return 0;
587}
588
589void exynos_gem_put_pages_to_userptr(struct page **pages,
590 unsigned int npages,
591 struct vm_area_struct *vma)
592{
593 if (!vma_is_io(vma)) {
594 unsigned int i;
595
596 for (i = 0; i < npages; i++) {
597 set_page_dirty_lock(pages[i]);
598
599
600
601
602
603 put_page(pages[i]);
604 }
605 }
606}
607
608int exynos_gem_map_sgt_with_dma(struct drm_device *drm_dev,
609 struct sg_table *sgt,
610 enum dma_data_direction dir)
611{
612 int nents;
613
614 mutex_lock(&drm_dev->struct_mutex);
615
616 nents = dma_map_sg(drm_dev->dev, sgt->sgl, sgt->nents, dir);
617 if (!nents) {
618 DRM_ERROR("failed to map sgl with dma.\n");
619 mutex_unlock(&drm_dev->struct_mutex);
620 return nents;
621 }
622
623 mutex_unlock(&drm_dev->struct_mutex);
624 return 0;
625}
626
627void exynos_gem_unmap_sgt_from_dma(struct drm_device *drm_dev,
628 struct sg_table *sgt,
629 enum dma_data_direction dir)
630{
631 dma_unmap_sg(drm_dev->dev, sgt->sgl, sgt->nents, dir);
632}
633
634int exynos_drm_gem_init_object(struct drm_gem_object *obj)
635{
636 return 0;
637}
638
639void exynos_drm_gem_free_object(struct drm_gem_object *obj)
640{
641 struct exynos_drm_gem_obj *exynos_gem_obj;
642 struct exynos_drm_gem_buf *buf;
643
644 exynos_gem_obj = to_exynos_gem_obj(obj);
645 buf = exynos_gem_obj->buffer;
646
647 if (obj->import_attach)
648 drm_prime_gem_destroy(obj, buf->sgt);
649
650 exynos_drm_gem_destroy(to_exynos_gem_obj(obj));
651}
652
653int exynos_drm_gem_dumb_create(struct drm_file *file_priv,
654 struct drm_device *dev,
655 struct drm_mode_create_dumb *args)
656{
657 struct exynos_drm_gem_obj *exynos_gem_obj;
658 int ret;
659
660
661
662
663
664
665
666 args->pitch = args->width * ((args->bpp + 7) / 8);
667 args->size = args->pitch * args->height;
668
669 exynos_gem_obj = exynos_drm_gem_create(dev, EXYNOS_BO_CONTIG |
670 EXYNOS_BO_WC, args->size);
671 if (IS_ERR(exynos_gem_obj))
672 return PTR_ERR(exynos_gem_obj);
673
674 ret = exynos_drm_gem_handle_create(&exynos_gem_obj->base, file_priv,
675 &args->handle);
676 if (ret) {
677 exynos_drm_gem_destroy(exynos_gem_obj);
678 return ret;
679 }
680
681 return 0;
682}
683
684int exynos_drm_gem_dumb_map_offset(struct drm_file *file_priv,
685 struct drm_device *dev, uint32_t handle,
686 uint64_t *offset)
687{
688 struct drm_gem_object *obj;
689 int ret = 0;
690
691 mutex_lock(&dev->struct_mutex);
692
693
694
695
696
697
698
699 obj = drm_gem_object_lookup(dev, file_priv, handle);
700 if (!obj) {
701 DRM_ERROR("failed to lookup gem object.\n");
702 ret = -EINVAL;
703 goto unlock;
704 }
705
706 if (!obj->map_list.map) {
707 ret = drm_gem_create_mmap_offset(obj);
708 if (ret)
709 goto out;
710 }
711
712 *offset = (u64)obj->map_list.hash.key << PAGE_SHIFT;
713 DRM_DEBUG_KMS("offset = 0x%lx\n", (unsigned long)*offset);
714
715out:
716 drm_gem_object_unreference(obj);
717unlock:
718 mutex_unlock(&dev->struct_mutex);
719 return ret;
720}
721
722int exynos_drm_gem_dumb_destroy(struct drm_file *file_priv,
723 struct drm_device *dev,
724 unsigned int handle)
725{
726 int ret;
727
728
729
730
731
732
733 ret = drm_gem_handle_delete(file_priv, handle);
734 if (ret < 0) {
735 DRM_ERROR("failed to delete drm_gem_handle.\n");
736 return ret;
737 }
738
739 return 0;
740}
741
742int exynos_drm_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
743{
744 struct drm_gem_object *obj = vma->vm_private_data;
745 struct drm_device *dev = obj->dev;
746 unsigned long f_vaddr;
747 pgoff_t page_offset;
748 int ret;
749
750 page_offset = ((unsigned long)vmf->virtual_address -
751 vma->vm_start) >> PAGE_SHIFT;
752 f_vaddr = (unsigned long)vmf->virtual_address;
753
754 mutex_lock(&dev->struct_mutex);
755
756 ret = exynos_drm_gem_map_buf(obj, vma, f_vaddr, page_offset);
757 if (ret < 0)
758 DRM_ERROR("failed to map a buffer with user.\n");
759
760 mutex_unlock(&dev->struct_mutex);
761
762 return convert_to_vm_err_msg(ret);
763}
764
765int exynos_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
766{
767 struct exynos_drm_gem_obj *exynos_gem_obj;
768 struct drm_gem_object *obj;
769 int ret;
770
771
772 ret = drm_gem_mmap(filp, vma);
773 if (ret < 0) {
774 DRM_ERROR("failed to mmap.\n");
775 return ret;
776 }
777
778 obj = vma->vm_private_data;
779 exynos_gem_obj = to_exynos_gem_obj(obj);
780
781 ret = check_gem_flags(exynos_gem_obj->flags);
782 if (ret) {
783 drm_gem_vm_close(vma);
784 drm_gem_free_mmap_offset(obj);
785 return ret;
786 }
787
788 vma->vm_flags &= ~VM_PFNMAP;
789 vma->vm_flags |= VM_MIXEDMAP;
790
791 update_vm_cache_attr(exynos_gem_obj, vma);
792
793 return ret;
794}
795