1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28#include <linux/types.h>
29#include <linux/slab.h>
30#include <linux/mm.h>
31#include <linux/uaccess.h>
32#include <linux/fs.h>
33#include <linux/file.h>
34#include <linux/module.h>
35#include <linux/mman.h>
36#include <linux/pagemap.h>
37#include <linux/shmem_fs.h>
38#include <linux/dma-buf.h>
39#include <linux/mem_encrypt.h>
40#include <drm/drmP.h>
41#include <drm/drm_vma_manager.h>
42#include <drm/drm_gem.h>
43#include <drm/drm_print.h>
44#include "drm_internal.h"
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83#if BITS_PER_LONG == 64
84#define DRM_FILE_PAGE_OFFSET_START ((0xFFFFFFFFUL >> PAGE_SHIFT) + 1)
85#define DRM_FILE_PAGE_OFFSET_SIZE ((0xFFFFFFFFUL >> PAGE_SHIFT) * 16)
86#else
87#define DRM_FILE_PAGE_OFFSET_START ((0xFFFFFFFUL >> PAGE_SHIFT) + 1)
88#define DRM_FILE_PAGE_OFFSET_SIZE ((0xFFFFFFFUL >> PAGE_SHIFT) * 16)
89#endif
90
91
92
93
94
95int
96drm_gem_init(struct drm_device *dev)
97{
98 struct drm_vma_offset_manager *vma_offset_manager;
99
100 mutex_init(&dev->object_name_lock);
101 idr_init(&dev->object_name_idr);
102
103 vma_offset_manager = kzalloc(sizeof(*vma_offset_manager), GFP_KERNEL);
104 if (!vma_offset_manager) {
105 DRM_ERROR("out of memory\n");
106 return -ENOMEM;
107 }
108
109 dev->vma_offset_manager = vma_offset_manager;
110 drm_vma_offset_manager_init(vma_offset_manager,
111 DRM_FILE_PAGE_OFFSET_START,
112 DRM_FILE_PAGE_OFFSET_SIZE);
113
114 return 0;
115}
116
117void
118drm_gem_destroy(struct drm_device *dev)
119{
120
121 drm_vma_offset_manager_destroy(dev->vma_offset_manager);
122 kfree(dev->vma_offset_manager);
123 dev->vma_offset_manager = NULL;
124}
125
126
127
128
129
130
131
132
133
134
135int drm_gem_object_init(struct drm_device *dev,
136 struct drm_gem_object *obj, size_t size)
137{
138 struct file *filp;
139
140 drm_gem_private_object_init(dev, obj, size);
141
142 filp = shmem_file_setup("drm mm object", size, VM_NORESERVE);
143 if (IS_ERR(filp))
144 return PTR_ERR(filp);
145
146 obj->filp = filp;
147
148 return 0;
149}
150EXPORT_SYMBOL(drm_gem_object_init);
151
152
153
154
155
156
157
158
159
160
161
162void drm_gem_private_object_init(struct drm_device *dev,
163 struct drm_gem_object *obj, size_t size)
164{
165 BUG_ON((size & (PAGE_SIZE - 1)) != 0);
166
167 obj->dev = dev;
168 obj->filp = NULL;
169
170 kref_init(&obj->refcount);
171 obj->handle_count = 0;
172 obj->size = size;
173 drm_vma_node_reset(&obj->vma_node);
174}
175EXPORT_SYMBOL(drm_gem_private_object_init);
176
177static void
178drm_gem_remove_prime_handles(struct drm_gem_object *obj, struct drm_file *filp)
179{
180
181
182
183
184 mutex_lock(&filp->prime.lock);
185 if (obj->dma_buf) {
186 drm_prime_remove_buf_handle_locked(&filp->prime,
187 obj->dma_buf);
188 }
189 mutex_unlock(&filp->prime.lock);
190}
191
192
193
194
195
196
197
198
199
200
201
202static void drm_gem_object_handle_free(struct drm_gem_object *obj)
203{
204 struct drm_device *dev = obj->dev;
205
206
207 if (obj->name) {
208 idr_remove(&dev->object_name_idr, obj->name);
209 obj->name = 0;
210 }
211}
212
213static void drm_gem_object_exported_dma_buf_free(struct drm_gem_object *obj)
214{
215
216 if (obj->dma_buf) {
217 dma_buf_put(obj->dma_buf);
218 obj->dma_buf = NULL;
219 }
220}
221
222static void
223drm_gem_object_handle_put_unlocked(struct drm_gem_object *obj)
224{
225 struct drm_device *dev = obj->dev;
226 bool final = false;
227
228 if (WARN_ON(obj->handle_count == 0))
229 return;
230
231
232
233
234
235
236
237 mutex_lock(&dev->object_name_lock);
238 if (--obj->handle_count == 0) {
239 drm_gem_object_handle_free(obj);
240 drm_gem_object_exported_dma_buf_free(obj);
241 final = true;
242 }
243 mutex_unlock(&dev->object_name_lock);
244
245 if (final)
246 drm_gem_object_put_unlocked(obj);
247}
248
249
250
251
252
253static int
254drm_gem_object_release_handle(int id, void *ptr, void *data)
255{
256 struct drm_file *file_priv = data;
257 struct drm_gem_object *obj = ptr;
258 struct drm_device *dev = obj->dev;
259
260 if (dev->driver->gem_close_object)
261 dev->driver->gem_close_object(obj, file_priv);
262
263 if (drm_core_check_feature(dev, DRIVER_PRIME))
264 drm_gem_remove_prime_handles(obj, file_priv);
265 drm_vma_node_revoke(&obj->vma_node, file_priv);
266
267 drm_gem_object_handle_put_unlocked(obj);
268
269 return 0;
270}
271
272
273
274
275
276
277
278
279
280
281int
282drm_gem_handle_delete(struct drm_file *filp, u32 handle)
283{
284 struct drm_gem_object *obj;
285
286 spin_lock(&filp->table_lock);
287
288
289 obj = idr_replace(&filp->object_idr, NULL, handle);
290 spin_unlock(&filp->table_lock);
291 if (IS_ERR_OR_NULL(obj))
292 return -EINVAL;
293
294
295 drm_gem_object_release_handle(handle, obj, filp);
296
297
298 spin_lock(&filp->table_lock);
299 idr_remove(&filp->object_idr, handle);
300 spin_unlock(&filp->table_lock);
301
302 return 0;
303}
304EXPORT_SYMBOL(drm_gem_handle_delete);
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319int drm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
320 u32 handle, u64 *offset)
321{
322 struct drm_gem_object *obj;
323 int ret;
324
325 obj = drm_gem_object_lookup(file, handle);
326 if (!obj)
327 return -ENOENT;
328
329
330 if (obj->import_attach) {
331 ret = -EINVAL;
332 goto out;
333 }
334
335 ret = drm_gem_create_mmap_offset(obj);
336 if (ret)
337 goto out;
338
339 *offset = drm_vma_node_offset_addr(&obj->vma_node);
340out:
341 drm_gem_object_put_unlocked(obj);
342
343 return ret;
344}
345EXPORT_SYMBOL_GPL(drm_gem_dumb_map_offset);
346
347
348
349
350
351
352
353
354
355
356int drm_gem_dumb_destroy(struct drm_file *file,
357 struct drm_device *dev,
358 uint32_t handle)
359{
360 return drm_gem_handle_delete(file, handle);
361}
362EXPORT_SYMBOL(drm_gem_dumb_destroy);
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378int
379drm_gem_handle_create_tail(struct drm_file *file_priv,
380 struct drm_gem_object *obj,
381 u32 *handlep)
382{
383 struct drm_device *dev = obj->dev;
384 u32 handle;
385 int ret;
386
387 WARN_ON(!mutex_is_locked(&dev->object_name_lock));
388 if (obj->handle_count++ == 0)
389 drm_gem_object_get(obj);
390
391
392
393
394
395 idr_preload(GFP_KERNEL);
396 spin_lock(&file_priv->table_lock);
397
398 ret = idr_alloc(&file_priv->object_idr, obj, 1, 0, GFP_NOWAIT);
399
400 spin_unlock(&file_priv->table_lock);
401 idr_preload_end();
402
403 mutex_unlock(&dev->object_name_lock);
404 if (ret < 0)
405 goto err_unref;
406
407 handle = ret;
408
409 ret = drm_vma_node_allow(&obj->vma_node, file_priv);
410 if (ret)
411 goto err_remove;
412
413 if (dev->driver->gem_open_object) {
414 ret = dev->driver->gem_open_object(obj, file_priv);
415 if (ret)
416 goto err_revoke;
417 }
418
419 *handlep = handle;
420 return 0;
421
422err_revoke:
423 drm_vma_node_revoke(&obj->vma_node, file_priv);
424err_remove:
425 spin_lock(&file_priv->table_lock);
426 idr_remove(&file_priv->object_idr, handle);
427 spin_unlock(&file_priv->table_lock);
428err_unref:
429 drm_gem_object_handle_put_unlocked(obj);
430 return ret;
431}
432
433
434
435
436
437
438
439
440
441
442
443int drm_gem_handle_create(struct drm_file *file_priv,
444 struct drm_gem_object *obj,
445 u32 *handlep)
446{
447 mutex_lock(&obj->dev->object_name_lock);
448
449 return drm_gem_handle_create_tail(file_priv, obj, handlep);
450}
451EXPORT_SYMBOL(drm_gem_handle_create);
452
453
454
455
456
457
458
459
460
461
462
463
464void
465drm_gem_free_mmap_offset(struct drm_gem_object *obj)
466{
467 struct drm_device *dev = obj->dev;
468
469 drm_vma_offset_remove(dev->vma_offset_manager, &obj->vma_node);
470}
471EXPORT_SYMBOL(drm_gem_free_mmap_offset);
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490int
491drm_gem_create_mmap_offset_size(struct drm_gem_object *obj, size_t size)
492{
493 struct drm_device *dev = obj->dev;
494
495 return drm_vma_offset_add(dev->vma_offset_manager, &obj->vma_node,
496 size / PAGE_SIZE);
497}
498EXPORT_SYMBOL(drm_gem_create_mmap_offset_size);
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514int drm_gem_create_mmap_offset(struct drm_gem_object *obj)
515{
516 return drm_gem_create_mmap_offset_size(obj, obj->size);
517}
518EXPORT_SYMBOL(drm_gem_create_mmap_offset);
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541struct page **drm_gem_get_pages(struct drm_gem_object *obj)
542{
543 struct address_space *mapping;
544 struct page *p, **pages;
545 int i, npages;
546
547
548 mapping = obj->filp->f_mapping;
549
550
551
552
553
554 WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);
555
556 npages = obj->size >> PAGE_SHIFT;
557
558 pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
559 if (pages == NULL)
560 return ERR_PTR(-ENOMEM);
561
562 for (i = 0; i < npages; i++) {
563 p = shmem_read_mapping_page(mapping, i);
564 if (IS_ERR(p))
565 goto fail;
566 pages[i] = p;
567
568
569
570
571
572
573 BUG_ON(mapping_gfp_constraint(mapping, __GFP_DMA32) &&
574 (page_to_pfn(p) >= 0x00100000UL));
575 }
576
577 return pages;
578
579fail:
580 while (i--)
581 put_page(pages[i]);
582
583 kvfree(pages);
584 return ERR_CAST(p);
585}
586EXPORT_SYMBOL(drm_gem_get_pages);
587
588
589
590
591
592
593
594
595void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages,
596 bool dirty, bool accessed)
597{
598 int i, npages;
599
600
601
602
603
604 WARN_ON((obj->size & (PAGE_SIZE - 1)) != 0);
605
606 npages = obj->size >> PAGE_SHIFT;
607
608 for (i = 0; i < npages; i++) {
609 if (dirty)
610 set_page_dirty(pages[i]);
611
612 if (accessed)
613 mark_page_accessed(pages[i]);
614
615
616 put_page(pages[i]);
617 }
618
619 kvfree(pages);
620}
621EXPORT_SYMBOL(drm_gem_put_pages);
622
623
624
625
626
627
628
629
630
631
632
633struct drm_gem_object *
634drm_gem_object_lookup(struct drm_file *filp, u32 handle)
635{
636 struct drm_gem_object *obj;
637
638 spin_lock(&filp->table_lock);
639
640
641 obj = idr_find(&filp->object_idr, handle);
642 if (obj)
643 drm_gem_object_get(obj);
644
645 spin_unlock(&filp->table_lock);
646
647 return obj;
648}
649EXPORT_SYMBOL(drm_gem_object_lookup);
650
651
652
653
654
655
656
657
658
659int
660drm_gem_close_ioctl(struct drm_device *dev, void *data,
661 struct drm_file *file_priv)
662{
663 struct drm_gem_close *args = data;
664 int ret;
665
666 if (!drm_core_check_feature(dev, DRIVER_GEM))
667 return -ENODEV;
668
669 ret = drm_gem_handle_delete(file_priv, args->handle);
670
671 return ret;
672}
673
674
675
676
677
678
679
680
681
682
683
684
685int
686drm_gem_flink_ioctl(struct drm_device *dev, void *data,
687 struct drm_file *file_priv)
688{
689 struct drm_gem_flink *args = data;
690 struct drm_gem_object *obj;
691 int ret;
692
693 if (!drm_core_check_feature(dev, DRIVER_GEM))
694 return -ENODEV;
695
696 obj = drm_gem_object_lookup(file_priv, args->handle);
697 if (obj == NULL)
698 return -ENOENT;
699
700 mutex_lock(&dev->object_name_lock);
701
702 if (obj->handle_count == 0) {
703 ret = -ENOENT;
704 goto err;
705 }
706
707 if (!obj->name) {
708 ret = idr_alloc(&dev->object_name_idr, obj, 1, 0, GFP_KERNEL);
709 if (ret < 0)
710 goto err;
711
712 obj->name = ret;
713 }
714
715 args->name = (uint64_t) obj->name;
716 ret = 0;
717
718err:
719 mutex_unlock(&dev->object_name_lock);
720 drm_gem_object_put_unlocked(obj);
721 return ret;
722}
723
724
725
726
727
728
729
730
731
732
733
734
735int
736drm_gem_open_ioctl(struct drm_device *dev, void *data,
737 struct drm_file *file_priv)
738{
739 struct drm_gem_open *args = data;
740 struct drm_gem_object *obj;
741 int ret;
742 u32 handle;
743
744 if (!drm_core_check_feature(dev, DRIVER_GEM))
745 return -ENODEV;
746
747 mutex_lock(&dev->object_name_lock);
748 obj = idr_find(&dev->object_name_idr, (int) args->name);
749 if (obj) {
750 drm_gem_object_get(obj);
751 } else {
752 mutex_unlock(&dev->object_name_lock);
753 return -ENOENT;
754 }
755
756
757 ret = drm_gem_handle_create_tail(file_priv, obj, &handle);
758 drm_gem_object_put_unlocked(obj);
759 if (ret)
760 return ret;
761
762 args->handle = handle;
763 args->size = obj->size;
764
765 return 0;
766}
767
768
769
770
771
772
773
774
775
776void
777drm_gem_open(struct drm_device *dev, struct drm_file *file_private)
778{
779 idr_init(&file_private->object_idr);
780 spin_lock_init(&file_private->table_lock);
781}
782
783
784
785
786
787
788
789
790
791
792void
793drm_gem_release(struct drm_device *dev, struct drm_file *file_private)
794{
795 idr_for_each(&file_private->object_idr,
796 &drm_gem_object_release_handle, file_private);
797 idr_destroy(&file_private->object_idr);
798}
799
800
801
802
803
804
805
806
807void
808drm_gem_object_release(struct drm_gem_object *obj)
809{
810 WARN_ON(obj->dma_buf);
811
812 if (obj->filp)
813 fput(obj->filp);
814
815 drm_gem_free_mmap_offset(obj);
816}
817EXPORT_SYMBOL(drm_gem_object_release);
818
819
820
821
822
823
824
825
826
827
828void
829drm_gem_object_free(struct kref *kref)
830{
831 struct drm_gem_object *obj =
832 container_of(kref, struct drm_gem_object, refcount);
833 struct drm_device *dev = obj->dev;
834
835 if (dev->driver->gem_free_object_unlocked) {
836 dev->driver->gem_free_object_unlocked(obj);
837 } else if (dev->driver->gem_free_object) {
838 WARN_ON(!mutex_is_locked(&dev->struct_mutex));
839
840 dev->driver->gem_free_object(obj);
841 }
842}
843EXPORT_SYMBOL(drm_gem_object_free);
844
845
846
847
848
849
850
851
852
853
854void
855drm_gem_object_put_unlocked(struct drm_gem_object *obj)
856{
857 struct drm_device *dev;
858
859 if (!obj)
860 return;
861
862 dev = obj->dev;
863
864 if (dev->driver->gem_free_object_unlocked) {
865 kref_put(&obj->refcount, drm_gem_object_free);
866 } else {
867 might_lock(&dev->struct_mutex);
868 if (kref_put_mutex(&obj->refcount, drm_gem_object_free,
869 &dev->struct_mutex))
870 mutex_unlock(&dev->struct_mutex);
871 }
872}
873EXPORT_SYMBOL(drm_gem_object_put_unlocked);
874
875
876
877
878
879
880
881
882
883
884
885
886void
887drm_gem_object_put(struct drm_gem_object *obj)
888{
889 if (obj) {
890 WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
891
892 kref_put(&obj->refcount, drm_gem_object_free);
893 }
894}
895EXPORT_SYMBOL(drm_gem_object_put);
896
897
898
899
900
901
902
903
904void drm_gem_vm_open(struct vm_area_struct *vma)
905{
906 struct drm_gem_object *obj = vma->vm_private_data;
907
908 drm_gem_object_get(obj);
909}
910EXPORT_SYMBOL(drm_gem_vm_open);
911
912
913
914
915
916
917
918
919void drm_gem_vm_close(struct vm_area_struct *vma)
920{
921 struct drm_gem_object *obj = vma->vm_private_data;
922
923 drm_gem_object_put_unlocked(obj);
924}
925EXPORT_SYMBOL(drm_gem_vm_close);
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,
952 struct vm_area_struct *vma)
953{
954 struct drm_device *dev = obj->dev;
955
956
957 if (obj_size < vma->vm_end - vma->vm_start)
958 return -EINVAL;
959
960 if (!dev->driver->gem_vm_ops)
961 return -EINVAL;
962
963 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
964 vma->vm_ops = dev->driver->gem_vm_ops;
965 vma->vm_private_data = obj;
966 vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
967 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
968
969
970
971
972
973
974
975 drm_gem_object_get(obj);
976
977 return 0;
978}
979EXPORT_SYMBOL(drm_gem_mmap_obj);
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
997{
998 struct drm_file *priv = filp->private_data;
999 struct drm_device *dev = priv->minor->dev;
1000 struct drm_gem_object *obj = NULL;
1001 struct drm_vma_offset_node *node;
1002 int ret;
1003
1004 if (drm_dev_is_unplugged(dev))
1005 return -ENODEV;
1006
1007 drm_vma_offset_lock_lookup(dev->vma_offset_manager);
1008 node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager,
1009 vma->vm_pgoff,
1010 vma_pages(vma));
1011 if (likely(node)) {
1012 obj = container_of(node, struct drm_gem_object, vma_node);
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023 if (!kref_get_unless_zero(&obj->refcount))
1024 obj = NULL;
1025 }
1026 drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
1027
1028 if (!obj)
1029 return -EINVAL;
1030
1031 if (!drm_vma_node_is_allowed(node, priv)) {
1032 drm_gem_object_put_unlocked(obj);
1033 return -EACCES;
1034 }
1035
1036 ret = drm_gem_mmap_obj(obj, drm_vma_node_size(node) << PAGE_SHIFT,
1037 vma);
1038
1039 drm_gem_object_put_unlocked(obj);
1040
1041 return ret;
1042}
1043EXPORT_SYMBOL(drm_gem_mmap);
1044
1045void drm_gem_print_info(struct drm_printer *p, unsigned int indent,
1046 const struct drm_gem_object *obj)
1047{
1048 drm_printf_indent(p, indent, "name=%d\n", obj->name);
1049 drm_printf_indent(p, indent, "refcount=%u\n",
1050 kref_read(&obj->refcount));
1051 drm_printf_indent(p, indent, "start=%08lx\n",
1052 drm_vma_node_start(&obj->vma_node));
1053 drm_printf_indent(p, indent, "size=%zu\n", obj->size);
1054 drm_printf_indent(p, indent, "imported=%s\n",
1055 obj->import_attach ? "yes" : "no");
1056
1057 if (obj->dev->driver->gem_print_info)
1058 obj->dev->driver->gem_print_info(p, indent, obj);
1059}
1060