1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/device.h>
19#include <linux/err.h>
20#include <linux/file.h>
21#include <linux/freezer.h>
22#include <linux/fs.h>
23#include <linux/anon_inodes.h>
24#include <linux/kthread.h>
25#include <linux/list.h>
26#include <linux/memblock.h>
27#include <linux/miscdevice.h>
28#include <linux/export.h>
29#include <linux/mm.h>
30#include <linux/mm_types.h>
31#include <linux/rbtree.h>
32#include <linux/slab.h>
33#include <linux/seq_file.h>
34#include <linux/uaccess.h>
35#include <linux/vmalloc.h>
36#include <linux/debugfs.h>
37#include <linux/dma-buf.h>
38#include <linux/idr.h>
39
40#include "ion.h"
41#include "ion_priv.h"
42#include "compat_ion.h"
43
44
45
46
47
48
49
50
51
52
53struct ion_device {
54 struct miscdevice dev;
55 struct rb_root buffers;
56 struct mutex buffer_lock;
57 struct rw_semaphore lock;
58 struct plist_head heaps;
59 long (*custom_ioctl)(struct ion_client *client, unsigned int cmd,
60 unsigned long arg);
61 struct rb_root clients;
62 struct dentry *debug_root;
63 struct dentry *heaps_debug_root;
64 struct dentry *clients_debug_root;
65};
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83struct ion_client {
84 struct rb_node node;
85 struct ion_device *dev;
86 struct rb_root handles;
87 struct idr idr;
88 struct mutex lock;
89 const char *name;
90 char *display_name;
91 int display_serial;
92 struct task_struct *task;
93 pid_t pid;
94 struct dentry *debug_root;
95};
96
97
98
99
100
101
102
103
104
105
106
107
108
109struct ion_handle {
110 struct kref ref;
111 struct ion_client *client;
112 struct ion_buffer *buffer;
113 struct rb_node node;
114 unsigned int kmap_cnt;
115 int id;
116};
117
118bool ion_buffer_fault_user_mappings(struct ion_buffer *buffer)
119{
120 return (buffer->flags & ION_FLAG_CACHED) &&
121 !(buffer->flags & ION_FLAG_CACHED_NEEDS_SYNC);
122}
123
124bool ion_buffer_cached(struct ion_buffer *buffer)
125{
126 return !!(buffer->flags & ION_FLAG_CACHED);
127}
128
129static inline struct page *ion_buffer_page(struct page *page)
130{
131 return (struct page *)((unsigned long)page & ~(1UL));
132}
133
134static inline bool ion_buffer_page_is_dirty(struct page *page)
135{
136 return !!((unsigned long)page & 1UL);
137}
138
139static inline void ion_buffer_page_dirty(struct page **page)
140{
141 *page = (struct page *)((unsigned long)(*page) | 1UL);
142}
143
144static inline void ion_buffer_page_clean(struct page **page)
145{
146 *page = (struct page *)((unsigned long)(*page) & ~(1UL));
147}
148
149
150static void ion_buffer_add(struct ion_device *dev,
151 struct ion_buffer *buffer)
152{
153 struct rb_node **p = &dev->buffers.rb_node;
154 struct rb_node *parent = NULL;
155 struct ion_buffer *entry;
156
157 while (*p) {
158 parent = *p;
159 entry = rb_entry(parent, struct ion_buffer, node);
160
161 if (buffer < entry) {
162 p = &(*p)->rb_left;
163 } else if (buffer > entry) {
164 p = &(*p)->rb_right;
165 } else {
166 pr_err("%s: buffer already found.", __func__);
167 BUG();
168 }
169 }
170
171 rb_link_node(&buffer->node, parent, p);
172 rb_insert_color(&buffer->node, &dev->buffers);
173}
174
175
176static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
177 struct ion_device *dev,
178 unsigned long len,
179 unsigned long align,
180 unsigned long flags)
181{
182 struct ion_buffer *buffer;
183 struct sg_table *table;
184 struct scatterlist *sg;
185 int i, ret;
186
187 buffer = kzalloc(sizeof(struct ion_buffer), GFP_KERNEL);
188 if (!buffer)
189 return ERR_PTR(-ENOMEM);
190
191 buffer->heap = heap;
192 buffer->flags = flags;
193 kref_init(&buffer->ref);
194
195 ret = heap->ops->allocate(heap, buffer, len, align, flags);
196
197 if (ret) {
198 if (!(heap->flags & ION_HEAP_FLAG_DEFER_FREE))
199 goto err2;
200
201 ion_heap_freelist_drain(heap, 0);
202 ret = heap->ops->allocate(heap, buffer, len, align,
203 flags);
204 if (ret)
205 goto err2;
206 }
207
208 buffer->dev = dev;
209 buffer->size = len;
210
211 table = heap->ops->map_dma(heap, buffer);
212 if (WARN_ONCE(table == NULL,
213 "heap->ops->map_dma should return ERR_PTR on error"))
214 table = ERR_PTR(-EINVAL);
215 if (IS_ERR(table)) {
216 heap->ops->free(buffer);
217 kfree(buffer);
218 return ERR_CAST(table);
219 }
220 buffer->sg_table = table;
221 if (ion_buffer_fault_user_mappings(buffer)) {
222 int num_pages = PAGE_ALIGN(buffer->size) / PAGE_SIZE;
223 struct scatterlist *sg;
224 int i, j, k = 0;
225
226 buffer->pages = vmalloc(sizeof(struct page *) * num_pages);
227 if (!buffer->pages) {
228 ret = -ENOMEM;
229 goto err1;
230 }
231
232 for_each_sg(table->sgl, sg, table->nents, i) {
233 struct page *page = sg_page(sg);
234
235 for (j = 0; j < sg->length / PAGE_SIZE; j++)
236 buffer->pages[k++] = page++;
237 }
238
239 if (ret)
240 goto err;
241 }
242
243 buffer->dev = dev;
244 buffer->size = len;
245 INIT_LIST_HEAD(&buffer->vmas);
246 mutex_init(&buffer->lock);
247
248
249
250
251
252
253
254
255 for_each_sg(buffer->sg_table->sgl, sg, buffer->sg_table->nents, i)
256 sg_dma_address(sg) = sg_phys(sg);
257 mutex_lock(&dev->buffer_lock);
258 ion_buffer_add(dev, buffer);
259 mutex_unlock(&dev->buffer_lock);
260 return buffer;
261
262err:
263 heap->ops->unmap_dma(heap, buffer);
264 heap->ops->free(buffer);
265err1:
266 vfree(buffer->pages);
267err2:
268 kfree(buffer);
269 return ERR_PTR(ret);
270}
271
272void ion_buffer_destroy(struct ion_buffer *buffer)
273{
274 if (WARN_ON(buffer->kmap_cnt > 0))
275 buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
276 buffer->heap->ops->unmap_dma(buffer->heap, buffer);
277 buffer->heap->ops->free(buffer);
278 vfree(buffer->pages);
279 kfree(buffer);
280}
281
282static void _ion_buffer_destroy(struct kref *kref)
283{
284 struct ion_buffer *buffer = container_of(kref, struct ion_buffer, ref);
285 struct ion_heap *heap = buffer->heap;
286 struct ion_device *dev = buffer->dev;
287
288 mutex_lock(&dev->buffer_lock);
289 rb_erase(&buffer->node, &dev->buffers);
290 mutex_unlock(&dev->buffer_lock);
291
292 if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
293 ion_heap_freelist_add(heap, buffer);
294 else
295 ion_buffer_destroy(buffer);
296}
297
298static void ion_buffer_get(struct ion_buffer *buffer)
299{
300 kref_get(&buffer->ref);
301}
302
303static int ion_buffer_put(struct ion_buffer *buffer)
304{
305 return kref_put(&buffer->ref, _ion_buffer_destroy);
306}
307
308static void ion_buffer_add_to_handle(struct ion_buffer *buffer)
309{
310 mutex_lock(&buffer->lock);
311 buffer->handle_count++;
312 mutex_unlock(&buffer->lock);
313}
314
315static void ion_buffer_remove_from_handle(struct ion_buffer *buffer)
316{
317
318
319
320
321
322
323
324
325
326 mutex_lock(&buffer->lock);
327 buffer->handle_count--;
328 BUG_ON(buffer->handle_count < 0);
329 if (!buffer->handle_count) {
330 struct task_struct *task;
331
332 task = current->group_leader;
333 get_task_comm(buffer->task_comm, task);
334 buffer->pid = task_pid_nr(task);
335 }
336 mutex_unlock(&buffer->lock);
337}
338
339static struct ion_handle *ion_handle_create(struct ion_client *client,
340 struct ion_buffer *buffer)
341{
342 struct ion_handle *handle;
343
344 handle = kzalloc(sizeof(struct ion_handle), GFP_KERNEL);
345 if (!handle)
346 return ERR_PTR(-ENOMEM);
347 kref_init(&handle->ref);
348 RB_CLEAR_NODE(&handle->node);
349 handle->client = client;
350 ion_buffer_get(buffer);
351 ion_buffer_add_to_handle(buffer);
352 handle->buffer = buffer;
353
354 return handle;
355}
356
357static void ion_handle_kmap_put(struct ion_handle *);
358
359static void ion_handle_destroy(struct kref *kref)
360{
361 struct ion_handle *handle = container_of(kref, struct ion_handle, ref);
362 struct ion_client *client = handle->client;
363 struct ion_buffer *buffer = handle->buffer;
364
365 mutex_lock(&buffer->lock);
366 while (handle->kmap_cnt)
367 ion_handle_kmap_put(handle);
368 mutex_unlock(&buffer->lock);
369
370 idr_remove(&client->idr, handle->id);
371 if (!RB_EMPTY_NODE(&handle->node))
372 rb_erase(&handle->node, &client->handles);
373
374 ion_buffer_remove_from_handle(buffer);
375 ion_buffer_put(buffer);
376
377 kfree(handle);
378}
379
380struct ion_buffer *ion_handle_buffer(struct ion_handle *handle)
381{
382 return handle->buffer;
383}
384
385static void ion_handle_get(struct ion_handle *handle)
386{
387 kref_get(&handle->ref);
388}
389
390static int ion_handle_put(struct ion_handle *handle)
391{
392 struct ion_client *client = handle->client;
393 int ret;
394
395 mutex_lock(&client->lock);
396 ret = kref_put(&handle->ref, ion_handle_destroy);
397 mutex_unlock(&client->lock);
398
399 return ret;
400}
401
402static struct ion_handle *ion_handle_lookup(struct ion_client *client,
403 struct ion_buffer *buffer)
404{
405 struct rb_node *n = client->handles.rb_node;
406
407 while (n) {
408 struct ion_handle *entry = rb_entry(n, struct ion_handle, node);
409
410 if (buffer < entry->buffer)
411 n = n->rb_left;
412 else if (buffer > entry->buffer)
413 n = n->rb_right;
414 else
415 return entry;
416 }
417 return ERR_PTR(-EINVAL);
418}
419
420static struct ion_handle *ion_handle_get_by_id(struct ion_client *client,
421 int id)
422{
423 struct ion_handle *handle;
424
425 mutex_lock(&client->lock);
426 handle = idr_find(&client->idr, id);
427 if (handle)
428 ion_handle_get(handle);
429 mutex_unlock(&client->lock);
430
431 return handle ? handle : ERR_PTR(-EINVAL);
432}
433
434static bool ion_handle_validate(struct ion_client *client,
435 struct ion_handle *handle)
436{
437 WARN_ON(!mutex_is_locked(&client->lock));
438 return idr_find(&client->idr, handle->id) == handle;
439}
440
441static int ion_handle_add(struct ion_client *client, struct ion_handle *handle)
442{
443 int id;
444 struct rb_node **p = &client->handles.rb_node;
445 struct rb_node *parent = NULL;
446 struct ion_handle *entry;
447
448 id = idr_alloc(&client->idr, handle, 1, 0, GFP_KERNEL);
449 if (id < 0)
450 return id;
451
452 handle->id = id;
453
454 while (*p) {
455 parent = *p;
456 entry = rb_entry(parent, struct ion_handle, node);
457
458 if (handle->buffer < entry->buffer)
459 p = &(*p)->rb_left;
460 else if (handle->buffer > entry->buffer)
461 p = &(*p)->rb_right;
462 else
463 WARN(1, "%s: buffer already found.", __func__);
464 }
465
466 rb_link_node(&handle->node, parent, p);
467 rb_insert_color(&handle->node, &client->handles);
468
469 return 0;
470}
471
472struct ion_handle *ion_alloc(struct ion_client *client, size_t len,
473 size_t align, unsigned int heap_id_mask,
474 unsigned int flags)
475{
476 struct ion_handle *handle;
477 struct ion_device *dev = client->dev;
478 struct ion_buffer *buffer = NULL;
479 struct ion_heap *heap;
480 int ret;
481
482 pr_debug("%s: len %zu align %zu heap_id_mask %u flags %x\n", __func__,
483 len, align, heap_id_mask, flags);
484
485
486
487
488
489
490 len = PAGE_ALIGN(len);
491
492 if (!len)
493 return ERR_PTR(-EINVAL);
494
495 down_read(&dev->lock);
496 plist_for_each_entry(heap, &dev->heaps, node) {
497
498 if (!((1 << heap->id) & heap_id_mask))
499 continue;
500 buffer = ion_buffer_create(heap, dev, len, align, flags);
501 if (!IS_ERR(buffer))
502 break;
503 }
504 up_read(&dev->lock);
505
506 if (buffer == NULL)
507 return ERR_PTR(-ENODEV);
508
509 if (IS_ERR(buffer))
510 return ERR_CAST(buffer);
511
512 handle = ion_handle_create(client, buffer);
513
514
515
516
517
518 ion_buffer_put(buffer);
519
520 if (IS_ERR(handle))
521 return handle;
522
523 mutex_lock(&client->lock);
524 ret = ion_handle_add(client, handle);
525 mutex_unlock(&client->lock);
526 if (ret) {
527 ion_handle_put(handle);
528 handle = ERR_PTR(ret);
529 }
530
531 return handle;
532}
533EXPORT_SYMBOL(ion_alloc);
534
535void ion_free(struct ion_client *client, struct ion_handle *handle)
536{
537 bool valid_handle;
538
539 BUG_ON(client != handle->client);
540
541 mutex_lock(&client->lock);
542 valid_handle = ion_handle_validate(client, handle);
543
544 if (!valid_handle) {
545 WARN(1, "%s: invalid handle passed to free.\n", __func__);
546 mutex_unlock(&client->lock);
547 return;
548 }
549 mutex_unlock(&client->lock);
550 ion_handle_put(handle);
551}
552EXPORT_SYMBOL(ion_free);
553
554int ion_phys(struct ion_client *client, struct ion_handle *handle,
555 ion_phys_addr_t *addr, size_t *len)
556{
557 struct ion_buffer *buffer;
558 int ret;
559
560 mutex_lock(&client->lock);
561 if (!ion_handle_validate(client, handle)) {
562 mutex_unlock(&client->lock);
563 return -EINVAL;
564 }
565
566 buffer = handle->buffer;
567
568 if (!buffer->heap->ops->phys) {
569 pr_err("%s: ion_phys is not implemented by this heap (name=%s, type=%d).\n",
570 __func__, buffer->heap->name, buffer->heap->type);
571 mutex_unlock(&client->lock);
572 return -ENODEV;
573 }
574 mutex_unlock(&client->lock);
575 ret = buffer->heap->ops->phys(buffer->heap, buffer, addr, len);
576 return ret;
577}
578EXPORT_SYMBOL(ion_phys);
579
580static void *ion_buffer_kmap_get(struct ion_buffer *buffer)
581{
582 void *vaddr;
583
584 if (buffer->kmap_cnt) {
585 buffer->kmap_cnt++;
586 return buffer->vaddr;
587 }
588 vaddr = buffer->heap->ops->map_kernel(buffer->heap, buffer);
589 if (WARN_ONCE(vaddr == NULL,
590 "heap->ops->map_kernel should return ERR_PTR on error"))
591 return ERR_PTR(-EINVAL);
592 if (IS_ERR(vaddr))
593 return vaddr;
594 buffer->vaddr = vaddr;
595 buffer->kmap_cnt++;
596 return vaddr;
597}
598
599static void *ion_handle_kmap_get(struct ion_handle *handle)
600{
601 struct ion_buffer *buffer = handle->buffer;
602 void *vaddr;
603
604 if (handle->kmap_cnt) {
605 handle->kmap_cnt++;
606 return buffer->vaddr;
607 }
608 vaddr = ion_buffer_kmap_get(buffer);
609 if (IS_ERR(vaddr))
610 return vaddr;
611 handle->kmap_cnt++;
612 return vaddr;
613}
614
615static void ion_buffer_kmap_put(struct ion_buffer *buffer)
616{
617 buffer->kmap_cnt--;
618 if (!buffer->kmap_cnt) {
619 buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
620 buffer->vaddr = NULL;
621 }
622}
623
624static void ion_handle_kmap_put(struct ion_handle *handle)
625{
626 struct ion_buffer *buffer = handle->buffer;
627
628 if (!handle->kmap_cnt) {
629 WARN(1, "%s: Double unmap detected! bailing...\n", __func__);
630 return;
631 }
632 handle->kmap_cnt--;
633 if (!handle->kmap_cnt)
634 ion_buffer_kmap_put(buffer);
635}
636
637void *ion_map_kernel(struct ion_client *client, struct ion_handle *handle)
638{
639 struct ion_buffer *buffer;
640 void *vaddr;
641
642 mutex_lock(&client->lock);
643 if (!ion_handle_validate(client, handle)) {
644 pr_err("%s: invalid handle passed to map_kernel.\n",
645 __func__);
646 mutex_unlock(&client->lock);
647 return ERR_PTR(-EINVAL);
648 }
649
650 buffer = handle->buffer;
651
652 if (!handle->buffer->heap->ops->map_kernel) {
653 pr_err("%s: map_kernel is not implemented by this heap.\n",
654 __func__);
655 mutex_unlock(&client->lock);
656 return ERR_PTR(-ENODEV);
657 }
658
659 mutex_lock(&buffer->lock);
660 vaddr = ion_handle_kmap_get(handle);
661 mutex_unlock(&buffer->lock);
662 mutex_unlock(&client->lock);
663 return vaddr;
664}
665EXPORT_SYMBOL(ion_map_kernel);
666
667void ion_unmap_kernel(struct ion_client *client, struct ion_handle *handle)
668{
669 struct ion_buffer *buffer;
670
671 mutex_lock(&client->lock);
672 buffer = handle->buffer;
673 mutex_lock(&buffer->lock);
674 ion_handle_kmap_put(handle);
675 mutex_unlock(&buffer->lock);
676 mutex_unlock(&client->lock);
677}
678EXPORT_SYMBOL(ion_unmap_kernel);
679
680static int ion_debug_client_show(struct seq_file *s, void *unused)
681{
682 struct ion_client *client = s->private;
683 struct rb_node *n;
684 size_t sizes[ION_NUM_HEAP_IDS] = {0};
685 const char *names[ION_NUM_HEAP_IDS] = {NULL};
686 int i;
687
688 mutex_lock(&client->lock);
689 for (n = rb_first(&client->handles); n; n = rb_next(n)) {
690 struct ion_handle *handle = rb_entry(n, struct ion_handle,
691 node);
692 unsigned int id = handle->buffer->heap->id;
693
694 if (!names[id])
695 names[id] = handle->buffer->heap->name;
696 sizes[id] += handle->buffer->size;
697 }
698 mutex_unlock(&client->lock);
699
700 seq_printf(s, "%16.16s: %16.16s\n", "heap_name", "size_in_bytes");
701 for (i = 0; i < ION_NUM_HEAP_IDS; i++) {
702 if (!names[i])
703 continue;
704 seq_printf(s, "%16.16s: %16zu\n", names[i], sizes[i]);
705 }
706 return 0;
707}
708
709static int ion_debug_client_open(struct inode *inode, struct file *file)
710{
711 return single_open(file, ion_debug_client_show, inode->i_private);
712}
713
714static const struct file_operations debug_client_fops = {
715 .open = ion_debug_client_open,
716 .read = seq_read,
717 .llseek = seq_lseek,
718 .release = single_release,
719};
720
721static int ion_get_client_serial(const struct rb_root *root,
722 const unsigned char *name)
723{
724 int serial = -1;
725 struct rb_node *node;
726
727 for (node = rb_first(root); node; node = rb_next(node)) {
728 struct ion_client *client = rb_entry(node, struct ion_client,
729 node);
730
731 if (strcmp(client->name, name))
732 continue;
733 serial = max(serial, client->display_serial);
734 }
735 return serial + 1;
736}
737
738struct ion_client *ion_client_create(struct ion_device *dev,
739 const char *name)
740{
741 struct ion_client *client;
742 struct task_struct *task;
743 struct rb_node **p;
744 struct rb_node *parent = NULL;
745 struct ion_client *entry;
746 pid_t pid;
747
748 if (!name) {
749 pr_err("%s: Name cannot be null\n", __func__);
750 return ERR_PTR(-EINVAL);
751 }
752
753 get_task_struct(current->group_leader);
754 task_lock(current->group_leader);
755 pid = task_pid_nr(current->group_leader);
756
757
758 if (current->group_leader->flags & PF_KTHREAD) {
759 put_task_struct(current->group_leader);
760 task = NULL;
761 } else {
762 task = current->group_leader;
763 }
764 task_unlock(current->group_leader);
765
766 client = kzalloc(sizeof(struct ion_client), GFP_KERNEL);
767 if (!client)
768 goto err_put_task_struct;
769
770 client->dev = dev;
771 client->handles = RB_ROOT;
772 idr_init(&client->idr);
773 mutex_init(&client->lock);
774 client->task = task;
775 client->pid = pid;
776 client->name = kstrdup(name, GFP_KERNEL);
777 if (!client->name)
778 goto err_free_client;
779
780 down_write(&dev->lock);
781 client->display_serial = ion_get_client_serial(&dev->clients, name);
782 client->display_name = kasprintf(
783 GFP_KERNEL, "%s-%d", name, client->display_serial);
784 if (!client->display_name) {
785 up_write(&dev->lock);
786 goto err_free_client_name;
787 }
788 p = &dev->clients.rb_node;
789 while (*p) {
790 parent = *p;
791 entry = rb_entry(parent, struct ion_client, node);
792
793 if (client < entry)
794 p = &(*p)->rb_left;
795 else if (client > entry)
796 p = &(*p)->rb_right;
797 }
798 rb_link_node(&client->node, parent, p);
799 rb_insert_color(&client->node, &dev->clients);
800
801 client->debug_root = debugfs_create_file(client->display_name, 0664,
802 dev->clients_debug_root,
803 client, &debug_client_fops);
804 if (!client->debug_root) {
805 char buf[256], *path;
806
807 path = dentry_path(dev->clients_debug_root, buf, 256);
808 pr_err("Failed to create client debugfs at %s/%s\n",
809 path, client->display_name);
810 }
811
812 up_write(&dev->lock);
813
814 return client;
815
816err_free_client_name:
817 kfree(client->name);
818err_free_client:
819 kfree(client);
820err_put_task_struct:
821 if (task)
822 put_task_struct(current->group_leader);
823 return ERR_PTR(-ENOMEM);
824}
825EXPORT_SYMBOL(ion_client_create);
826
827void ion_client_destroy(struct ion_client *client)
828{
829 struct ion_device *dev = client->dev;
830 struct rb_node *n;
831
832 pr_debug("%s: %d\n", __func__, __LINE__);
833 while ((n = rb_first(&client->handles))) {
834 struct ion_handle *handle = rb_entry(n, struct ion_handle,
835 node);
836 ion_handle_destroy(&handle->ref);
837 }
838
839 idr_destroy(&client->idr);
840
841 down_write(&dev->lock);
842 if (client->task)
843 put_task_struct(client->task);
844 rb_erase(&client->node, &dev->clients);
845 debugfs_remove_recursive(client->debug_root);
846 up_write(&dev->lock);
847
848 kfree(client->display_name);
849 kfree(client->name);
850 kfree(client);
851}
852EXPORT_SYMBOL(ion_client_destroy);
853
854struct sg_table *ion_sg_table(struct ion_client *client,
855 struct ion_handle *handle)
856{
857 struct ion_buffer *buffer;
858 struct sg_table *table;
859
860 mutex_lock(&client->lock);
861 if (!ion_handle_validate(client, handle)) {
862 pr_err("%s: invalid handle passed to map_dma.\n",
863 __func__);
864 mutex_unlock(&client->lock);
865 return ERR_PTR(-EINVAL);
866 }
867 buffer = handle->buffer;
868 table = buffer->sg_table;
869 mutex_unlock(&client->lock);
870 return table;
871}
872EXPORT_SYMBOL(ion_sg_table);
873
874static void ion_buffer_sync_for_device(struct ion_buffer *buffer,
875 struct device *dev,
876 enum dma_data_direction direction);
877
878static struct sg_table *ion_map_dma_buf(struct dma_buf_attachment *attachment,
879 enum dma_data_direction direction)
880{
881 struct dma_buf *dmabuf = attachment->dmabuf;
882 struct ion_buffer *buffer = dmabuf->priv;
883
884 ion_buffer_sync_for_device(buffer, attachment->dev, direction);
885 return buffer->sg_table;
886}
887
888static void ion_unmap_dma_buf(struct dma_buf_attachment *attachment,
889 struct sg_table *table,
890 enum dma_data_direction direction)
891{
892}
893
894void ion_pages_sync_for_device(struct device *dev, struct page *page,
895 size_t size, enum dma_data_direction dir)
896{
897 struct scatterlist sg;
898
899 sg_init_table(&sg, 1);
900 sg_set_page(&sg, page, size, 0);
901
902
903
904
905
906 sg_dma_address(&sg) = page_to_phys(page);
907 dma_sync_sg_for_device(dev, &sg, 1, dir);
908}
909
910struct ion_vma_list {
911 struct list_head list;
912 struct vm_area_struct *vma;
913};
914
915static void ion_buffer_sync_for_device(struct ion_buffer *buffer,
916 struct device *dev,
917 enum dma_data_direction dir)
918{
919 struct ion_vma_list *vma_list;
920 int pages = PAGE_ALIGN(buffer->size) / PAGE_SIZE;
921 int i;
922
923 pr_debug("%s: syncing for device %s\n", __func__,
924 dev ? dev_name(dev) : "null");
925
926 if (!ion_buffer_fault_user_mappings(buffer))
927 return;
928
929 mutex_lock(&buffer->lock);
930 for (i = 0; i < pages; i++) {
931 struct page *page = buffer->pages[i];
932
933 if (ion_buffer_page_is_dirty(page))
934 ion_pages_sync_for_device(dev, ion_buffer_page(page),
935 PAGE_SIZE, dir);
936
937 ion_buffer_page_clean(buffer->pages + i);
938 }
939 list_for_each_entry(vma_list, &buffer->vmas, list) {
940 struct vm_area_struct *vma = vma_list->vma;
941
942 zap_page_range(vma, vma->vm_start, vma->vm_end - vma->vm_start,
943 NULL);
944 }
945 mutex_unlock(&buffer->lock);
946}
947
948static int ion_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
949{
950 struct ion_buffer *buffer = vma->vm_private_data;
951 unsigned long pfn;
952 int ret;
953
954 mutex_lock(&buffer->lock);
955 ion_buffer_page_dirty(buffer->pages + vmf->pgoff);
956 BUG_ON(!buffer->pages || !buffer->pages[vmf->pgoff]);
957
958 pfn = page_to_pfn(ion_buffer_page(buffer->pages[vmf->pgoff]));
959 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
960 mutex_unlock(&buffer->lock);
961 if (ret)
962 return VM_FAULT_ERROR;
963
964 return VM_FAULT_NOPAGE;
965}
966
967static void ion_vm_open(struct vm_area_struct *vma)
968{
969 struct ion_buffer *buffer = vma->vm_private_data;
970 struct ion_vma_list *vma_list;
971
972 vma_list = kmalloc(sizeof(struct ion_vma_list), GFP_KERNEL);
973 if (!vma_list)
974 return;
975 vma_list->vma = vma;
976 mutex_lock(&buffer->lock);
977 list_add(&vma_list->list, &buffer->vmas);
978 mutex_unlock(&buffer->lock);
979 pr_debug("%s: adding %p\n", __func__, vma);
980}
981
982static void ion_vm_close(struct vm_area_struct *vma)
983{
984 struct ion_buffer *buffer = vma->vm_private_data;
985 struct ion_vma_list *vma_list, *tmp;
986
987 pr_debug("%s\n", __func__);
988 mutex_lock(&buffer->lock);
989 list_for_each_entry_safe(vma_list, tmp, &buffer->vmas, list) {
990 if (vma_list->vma != vma)
991 continue;
992 list_del(&vma_list->list);
993 kfree(vma_list);
994 pr_debug("%s: deleting %p\n", __func__, vma);
995 break;
996 }
997 mutex_unlock(&buffer->lock);
998}
999
1000static struct vm_operations_struct ion_vma_ops = {
1001 .open = ion_vm_open,
1002 .close = ion_vm_close,
1003 .fault = ion_vm_fault,
1004};
1005
1006static int ion_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma)
1007{
1008 struct ion_buffer *buffer = dmabuf->priv;
1009 int ret = 0;
1010
1011 if (!buffer->heap->ops->map_user) {
1012 pr_err("%s: this heap does not define a method for mapping to userspace\n",
1013 __func__);
1014 return -EINVAL;
1015 }
1016
1017 if (ion_buffer_fault_user_mappings(buffer)) {
1018 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND |
1019 VM_DONTDUMP;
1020 vma->vm_private_data = buffer;
1021 vma->vm_ops = &ion_vma_ops;
1022 ion_vm_open(vma);
1023 return 0;
1024 }
1025
1026 if (!(buffer->flags & ION_FLAG_CACHED))
1027 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1028
1029 mutex_lock(&buffer->lock);
1030
1031 ret = buffer->heap->ops->map_user(buffer->heap, buffer, vma);
1032 mutex_unlock(&buffer->lock);
1033
1034 if (ret)
1035 pr_err("%s: failure mapping buffer to userspace\n",
1036 __func__);
1037
1038 return ret;
1039}
1040
1041static void ion_dma_buf_release(struct dma_buf *dmabuf)
1042{
1043 struct ion_buffer *buffer = dmabuf->priv;
1044
1045 ion_buffer_put(buffer);
1046}
1047
1048static void *ion_dma_buf_kmap(struct dma_buf *dmabuf, unsigned long offset)
1049{
1050 struct ion_buffer *buffer = dmabuf->priv;
1051
1052 return buffer->vaddr + offset * PAGE_SIZE;
1053}
1054
1055static void ion_dma_buf_kunmap(struct dma_buf *dmabuf, unsigned long offset,
1056 void *ptr)
1057{
1058}
1059
1060static int ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, size_t start,
1061 size_t len,
1062 enum dma_data_direction direction)
1063{
1064 struct ion_buffer *buffer = dmabuf->priv;
1065 void *vaddr;
1066
1067 if (!buffer->heap->ops->map_kernel) {
1068 pr_err("%s: map kernel is not implemented by this heap.\n",
1069 __func__);
1070 return -ENODEV;
1071 }
1072
1073 mutex_lock(&buffer->lock);
1074 vaddr = ion_buffer_kmap_get(buffer);
1075 mutex_unlock(&buffer->lock);
1076 return PTR_ERR_OR_ZERO(vaddr);
1077}
1078
1079static void ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, size_t start,
1080 size_t len,
1081 enum dma_data_direction direction)
1082{
1083 struct ion_buffer *buffer = dmabuf->priv;
1084
1085 mutex_lock(&buffer->lock);
1086 ion_buffer_kmap_put(buffer);
1087 mutex_unlock(&buffer->lock);
1088}
1089
1090static struct dma_buf_ops dma_buf_ops = {
1091 .map_dma_buf = ion_map_dma_buf,
1092 .unmap_dma_buf = ion_unmap_dma_buf,
1093 .mmap = ion_mmap,
1094 .release = ion_dma_buf_release,
1095 .begin_cpu_access = ion_dma_buf_begin_cpu_access,
1096 .end_cpu_access = ion_dma_buf_end_cpu_access,
1097 .kmap_atomic = ion_dma_buf_kmap,
1098 .kunmap_atomic = ion_dma_buf_kunmap,
1099 .kmap = ion_dma_buf_kmap,
1100 .kunmap = ion_dma_buf_kunmap,
1101};
1102
1103struct dma_buf *ion_share_dma_buf(struct ion_client *client,
1104 struct ion_handle *handle)
1105{
1106 struct ion_buffer *buffer;
1107 struct dma_buf *dmabuf;
1108 bool valid_handle;
1109 DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
1110
1111 mutex_lock(&client->lock);
1112 valid_handle = ion_handle_validate(client, handle);
1113 if (!valid_handle) {
1114 WARN(1, "%s: invalid handle passed to share.\n", __func__);
1115 mutex_unlock(&client->lock);
1116 return ERR_PTR(-EINVAL);
1117 }
1118 buffer = handle->buffer;
1119 ion_buffer_get(buffer);
1120 mutex_unlock(&client->lock);
1121
1122 exp_info.ops = &dma_buf_ops;
1123 exp_info.size = buffer->size;
1124 exp_info.flags = O_RDWR;
1125 exp_info.priv = buffer;
1126
1127 dmabuf = dma_buf_export(&exp_info);
1128 if (IS_ERR(dmabuf)) {
1129 ion_buffer_put(buffer);
1130 return dmabuf;
1131 }
1132
1133 return dmabuf;
1134}
1135EXPORT_SYMBOL(ion_share_dma_buf);
1136
1137int ion_share_dma_buf_fd(struct ion_client *client, struct ion_handle *handle)
1138{
1139 struct dma_buf *dmabuf;
1140 int fd;
1141
1142 dmabuf = ion_share_dma_buf(client, handle);
1143 if (IS_ERR(dmabuf))
1144 return PTR_ERR(dmabuf);
1145
1146 fd = dma_buf_fd(dmabuf, O_CLOEXEC);
1147 if (fd < 0)
1148 dma_buf_put(dmabuf);
1149
1150 return fd;
1151}
1152EXPORT_SYMBOL(ion_share_dma_buf_fd);
1153
1154struct ion_handle *ion_import_dma_buf(struct ion_client *client, int fd)
1155{
1156 struct dma_buf *dmabuf;
1157 struct ion_buffer *buffer;
1158 struct ion_handle *handle;
1159 int ret;
1160
1161 dmabuf = dma_buf_get(fd);
1162 if (IS_ERR(dmabuf))
1163 return ERR_CAST(dmabuf);
1164
1165
1166 if (dmabuf->ops != &dma_buf_ops) {
1167 pr_err("%s: can not import dmabuf from another exporter\n",
1168 __func__);
1169 dma_buf_put(dmabuf);
1170 return ERR_PTR(-EINVAL);
1171 }
1172 buffer = dmabuf->priv;
1173
1174 mutex_lock(&client->lock);
1175
1176 handle = ion_handle_lookup(client, buffer);
1177 if (!IS_ERR(handle)) {
1178 ion_handle_get(handle);
1179 mutex_unlock(&client->lock);
1180 goto end;
1181 }
1182 mutex_unlock(&client->lock);
1183
1184 handle = ion_handle_create(client, buffer);
1185 if (IS_ERR(handle))
1186 goto end;
1187
1188 mutex_lock(&client->lock);
1189 ret = ion_handle_add(client, handle);
1190 mutex_unlock(&client->lock);
1191 if (ret) {
1192 ion_handle_put(handle);
1193 handle = ERR_PTR(ret);
1194 }
1195
1196end:
1197 dma_buf_put(dmabuf);
1198 return handle;
1199}
1200EXPORT_SYMBOL(ion_import_dma_buf);
1201
1202static int ion_sync_for_device(struct ion_client *client, int fd)
1203{
1204 struct dma_buf *dmabuf;
1205 struct ion_buffer *buffer;
1206
1207 dmabuf = dma_buf_get(fd);
1208 if (IS_ERR(dmabuf))
1209 return PTR_ERR(dmabuf);
1210
1211
1212 if (dmabuf->ops != &dma_buf_ops) {
1213 pr_err("%s: can not sync dmabuf from another exporter\n",
1214 __func__);
1215 dma_buf_put(dmabuf);
1216 return -EINVAL;
1217 }
1218 buffer = dmabuf->priv;
1219
1220 dma_sync_sg_for_device(NULL, buffer->sg_table->sgl,
1221 buffer->sg_table->nents, DMA_BIDIRECTIONAL);
1222 dma_buf_put(dmabuf);
1223 return 0;
1224}
1225
1226
1227static unsigned int ion_ioctl_dir(unsigned int cmd)
1228{
1229 switch (cmd) {
1230 case ION_IOC_SYNC:
1231 case ION_IOC_FREE:
1232 case ION_IOC_CUSTOM:
1233 return _IOC_WRITE;
1234 default:
1235 return _IOC_DIR(cmd);
1236 }
1237}
1238
1239static long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
1240{
1241 struct ion_client *client = filp->private_data;
1242 struct ion_device *dev = client->dev;
1243 struct ion_handle *cleanup_handle = NULL;
1244 int ret = 0;
1245 unsigned int dir;
1246
1247 union {
1248 struct ion_fd_data fd;
1249 struct ion_allocation_data allocation;
1250 struct ion_handle_data handle;
1251 struct ion_custom_data custom;
1252 } data;
1253
1254 dir = ion_ioctl_dir(cmd);
1255
1256 if (_IOC_SIZE(cmd) > sizeof(data))
1257 return -EINVAL;
1258
1259 if (dir & _IOC_WRITE)
1260 if (copy_from_user(&data, (void __user *)arg, _IOC_SIZE(cmd)))
1261 return -EFAULT;
1262
1263 switch (cmd) {
1264 case ION_IOC_ALLOC:
1265 {
1266 struct ion_handle *handle;
1267
1268 handle = ion_alloc(client, data.allocation.len,
1269 data.allocation.align,
1270 data.allocation.heap_id_mask,
1271 data.allocation.flags);
1272 if (IS_ERR(handle))
1273 return PTR_ERR(handle);
1274
1275 data.allocation.handle = handle->id;
1276
1277 cleanup_handle = handle;
1278 break;
1279 }
1280 case ION_IOC_FREE:
1281 {
1282 struct ion_handle *handle;
1283
1284 handle = ion_handle_get_by_id(client, data.handle.handle);
1285 if (IS_ERR(handle))
1286 return PTR_ERR(handle);
1287 ion_free(client, handle);
1288 ion_handle_put(handle);
1289 break;
1290 }
1291 case ION_IOC_SHARE:
1292 case ION_IOC_MAP:
1293 {
1294 struct ion_handle *handle;
1295
1296 handle = ion_handle_get_by_id(client, data.handle.handle);
1297 if (IS_ERR(handle))
1298 return PTR_ERR(handle);
1299 data.fd.fd = ion_share_dma_buf_fd(client, handle);
1300 ion_handle_put(handle);
1301 if (data.fd.fd < 0)
1302 ret = data.fd.fd;
1303 break;
1304 }
1305 case ION_IOC_IMPORT:
1306 {
1307 struct ion_handle *handle;
1308
1309 handle = ion_import_dma_buf(client, data.fd.fd);
1310 if (IS_ERR(handle))
1311 ret = PTR_ERR(handle);
1312 else
1313 data.handle.handle = handle->id;
1314 break;
1315 }
1316 case ION_IOC_SYNC:
1317 {
1318 ret = ion_sync_for_device(client, data.fd.fd);
1319 break;
1320 }
1321 case ION_IOC_CUSTOM:
1322 {
1323 if (!dev->custom_ioctl)
1324 return -ENOTTY;
1325 ret = dev->custom_ioctl(client, data.custom.cmd,
1326 data.custom.arg);
1327 break;
1328 }
1329 default:
1330 return -ENOTTY;
1331 }
1332
1333 if (dir & _IOC_READ) {
1334 if (copy_to_user((void __user *)arg, &data, _IOC_SIZE(cmd))) {
1335 if (cleanup_handle)
1336 ion_free(client, cleanup_handle);
1337 return -EFAULT;
1338 }
1339 }
1340 return ret;
1341}
1342
1343static int ion_release(struct inode *inode, struct file *file)
1344{
1345 struct ion_client *client = file->private_data;
1346
1347 pr_debug("%s: %d\n", __func__, __LINE__);
1348 ion_client_destroy(client);
1349 return 0;
1350}
1351
1352static int ion_open(struct inode *inode, struct file *file)
1353{
1354 struct miscdevice *miscdev = file->private_data;
1355 struct ion_device *dev = container_of(miscdev, struct ion_device, dev);
1356 struct ion_client *client;
1357 char debug_name[64];
1358
1359 pr_debug("%s: %d\n", __func__, __LINE__);
1360 snprintf(debug_name, 64, "%u", task_pid_nr(current->group_leader));
1361 client = ion_client_create(dev, debug_name);
1362 if (IS_ERR(client))
1363 return PTR_ERR(client);
1364 file->private_data = client;
1365
1366 return 0;
1367}
1368
1369static const struct file_operations ion_fops = {
1370 .owner = THIS_MODULE,
1371 .open = ion_open,
1372 .release = ion_release,
1373 .unlocked_ioctl = ion_ioctl,
1374 .compat_ioctl = compat_ion_ioctl,
1375};
1376
1377static size_t ion_debug_heap_total(struct ion_client *client,
1378 unsigned int id)
1379{
1380 size_t size = 0;
1381 struct rb_node *n;
1382
1383 mutex_lock(&client->lock);
1384 for (n = rb_first(&client->handles); n; n = rb_next(n)) {
1385 struct ion_handle *handle = rb_entry(n,
1386 struct ion_handle,
1387 node);
1388 if (handle->buffer->heap->id == id)
1389 size += handle->buffer->size;
1390 }
1391 mutex_unlock(&client->lock);
1392 return size;
1393}
1394
1395static int ion_debug_heap_show(struct seq_file *s, void *unused)
1396{
1397 struct ion_heap *heap = s->private;
1398 struct ion_device *dev = heap->dev;
1399 struct rb_node *n;
1400 size_t total_size = 0;
1401 size_t total_orphaned_size = 0;
1402
1403 seq_printf(s, "%16s %16s %16s\n", "client", "pid", "size");
1404 seq_puts(s, "----------------------------------------------------\n");
1405
1406 for (n = rb_first(&dev->clients); n; n = rb_next(n)) {
1407 struct ion_client *client = rb_entry(n, struct ion_client,
1408 node);
1409 size_t size = ion_debug_heap_total(client, heap->id);
1410
1411 if (!size)
1412 continue;
1413 if (client->task) {
1414 char task_comm[TASK_COMM_LEN];
1415
1416 get_task_comm(task_comm, client->task);
1417 seq_printf(s, "%16s %16u %16zu\n", task_comm,
1418 client->pid, size);
1419 } else {
1420 seq_printf(s, "%16s %16u %16zu\n", client->name,
1421 client->pid, size);
1422 }
1423 }
1424 seq_puts(s, "----------------------------------------------------\n");
1425 seq_puts(s, "orphaned allocations (info is from last known client):\n");
1426 mutex_lock(&dev->buffer_lock);
1427 for (n = rb_first(&dev->buffers); n; n = rb_next(n)) {
1428 struct ion_buffer *buffer = rb_entry(n, struct ion_buffer,
1429 node);
1430 if (buffer->heap->id != heap->id)
1431 continue;
1432 total_size += buffer->size;
1433 if (!buffer->handle_count) {
1434 seq_printf(s, "%16s %16u %16zu %d %d\n",
1435 buffer->task_comm, buffer->pid,
1436 buffer->size, buffer->kmap_cnt,
1437 atomic_read(&buffer->ref.refcount));
1438 total_orphaned_size += buffer->size;
1439 }
1440 }
1441 mutex_unlock(&dev->buffer_lock);
1442 seq_puts(s, "----------------------------------------------------\n");
1443 seq_printf(s, "%16s %16zu\n", "total orphaned",
1444 total_orphaned_size);
1445 seq_printf(s, "%16s %16zu\n", "total ", total_size);
1446 if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
1447 seq_printf(s, "%16s %16zu\n", "deferred free",
1448 heap->free_list_size);
1449 seq_puts(s, "----------------------------------------------------\n");
1450
1451 if (heap->debug_show)
1452 heap->debug_show(heap, s, unused);
1453
1454 return 0;
1455}
1456
1457static int ion_debug_heap_open(struct inode *inode, struct file *file)
1458{
1459 return single_open(file, ion_debug_heap_show, inode->i_private);
1460}
1461
1462static const struct file_operations debug_heap_fops = {
1463 .open = ion_debug_heap_open,
1464 .read = seq_read,
1465 .llseek = seq_lseek,
1466 .release = single_release,
1467};
1468
1469#ifdef DEBUG_HEAP_SHRINKER
1470static int debug_shrink_set(void *data, u64 val)
1471{
1472 struct ion_heap *heap = data;
1473 struct shrink_control sc;
1474 int objs;
1475
1476 sc.gfp_mask = -1;
1477 sc.nr_to_scan = 0;
1478
1479 if (!val)
1480 return 0;
1481
1482 objs = heap->shrinker.shrink(&heap->shrinker, &sc);
1483 sc.nr_to_scan = objs;
1484
1485 heap->shrinker.shrink(&heap->shrinker, &sc);
1486 return 0;
1487}
1488
1489static int debug_shrink_get(void *data, u64 *val)
1490{
1491 struct ion_heap *heap = data;
1492 struct shrink_control sc;
1493 int objs;
1494
1495 sc.gfp_mask = -1;
1496 sc.nr_to_scan = 0;
1497
1498 objs = heap->shrinker.shrink(&heap->shrinker, &sc);
1499 *val = objs;
1500 return 0;
1501}
1502
1503DEFINE_SIMPLE_ATTRIBUTE(debug_shrink_fops, debug_shrink_get,
1504 debug_shrink_set, "%llu\n");
1505#endif
1506
1507void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap)
1508{
1509 struct dentry *debug_file;
1510
1511 if (!heap->ops->allocate || !heap->ops->free || !heap->ops->map_dma ||
1512 !heap->ops->unmap_dma)
1513 pr_err("%s: can not add heap with invalid ops struct.\n",
1514 __func__);
1515
1516 spin_lock_init(&heap->free_lock);
1517 heap->free_list_size = 0;
1518
1519 if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
1520 ion_heap_init_deferred_free(heap);
1521
1522 if ((heap->flags & ION_HEAP_FLAG_DEFER_FREE) || heap->ops->shrink)
1523 ion_heap_init_shrinker(heap);
1524
1525 heap->dev = dev;
1526 down_write(&dev->lock);
1527
1528
1529 plist_node_init(&heap->node, -heap->id);
1530 plist_add(&heap->node, &dev->heaps);
1531 debug_file = debugfs_create_file(heap->name, 0664,
1532 dev->heaps_debug_root, heap,
1533 &debug_heap_fops);
1534
1535 if (!debug_file) {
1536 char buf[256], *path;
1537
1538 path = dentry_path(dev->heaps_debug_root, buf, 256);
1539 pr_err("Failed to create heap debugfs at %s/%s\n",
1540 path, heap->name);
1541 }
1542
1543#ifdef DEBUG_HEAP_SHRINKER
1544 if (heap->shrinker.shrink) {
1545 char debug_name[64];
1546
1547 snprintf(debug_name, 64, "%s_shrink", heap->name);
1548 debug_file = debugfs_create_file(
1549 debug_name, 0644, dev->heaps_debug_root, heap,
1550 &debug_shrink_fops);
1551 if (!debug_file) {
1552 char buf[256], *path;
1553
1554 path = dentry_path(dev->heaps_debug_root, buf, 256);
1555 pr_err("Failed to create heap shrinker debugfs at %s/%s\n",
1556 path, debug_name);
1557 }
1558 }
1559#endif
1560 up_write(&dev->lock);
1561}
1562
1563struct ion_device *ion_device_create(long (*custom_ioctl)
1564 (struct ion_client *client,
1565 unsigned int cmd,
1566 unsigned long arg))
1567{
1568 struct ion_device *idev;
1569 int ret;
1570
1571 idev = kzalloc(sizeof(struct ion_device), GFP_KERNEL);
1572 if (!idev)
1573 return ERR_PTR(-ENOMEM);
1574
1575 idev->dev.minor = MISC_DYNAMIC_MINOR;
1576 idev->dev.name = "ion";
1577 idev->dev.fops = &ion_fops;
1578 idev->dev.parent = NULL;
1579 ret = misc_register(&idev->dev);
1580 if (ret) {
1581 pr_err("ion: failed to register misc device.\n");
1582 return ERR_PTR(ret);
1583 }
1584
1585 idev->debug_root = debugfs_create_dir("ion", NULL);
1586 if (!idev->debug_root) {
1587 pr_err("ion: failed to create debugfs root directory.\n");
1588 goto debugfs_done;
1589 }
1590 idev->heaps_debug_root = debugfs_create_dir("heaps", idev->debug_root);
1591 if (!idev->heaps_debug_root) {
1592 pr_err("ion: failed to create debugfs heaps directory.\n");
1593 goto debugfs_done;
1594 }
1595 idev->clients_debug_root = debugfs_create_dir("clients",
1596 idev->debug_root);
1597 if (!idev->clients_debug_root)
1598 pr_err("ion: failed to create debugfs clients directory.\n");
1599
1600debugfs_done:
1601
1602 idev->custom_ioctl = custom_ioctl;
1603 idev->buffers = RB_ROOT;
1604 mutex_init(&idev->buffer_lock);
1605 init_rwsem(&idev->lock);
1606 plist_head_init(&idev->heaps);
1607 idev->clients = RB_ROOT;
1608 return idev;
1609}
1610
1611void ion_device_destroy(struct ion_device *dev)
1612{
1613 misc_deregister(&dev->dev);
1614 debugfs_remove_recursive(dev->debug_root);
1615
1616 kfree(dev);
1617}
1618
1619void __init ion_reserve(struct ion_platform_data *data)
1620{
1621 int i;
1622
1623 for (i = 0; i < data->nr; i++) {
1624 if (data->heaps[i].size == 0)
1625 continue;
1626
1627 if (data->heaps[i].base == 0) {
1628 phys_addr_t paddr;
1629
1630 paddr = memblock_alloc_base(data->heaps[i].size,
1631 data->heaps[i].align,
1632 MEMBLOCK_ALLOC_ANYWHERE);
1633 if (!paddr) {
1634 pr_err("%s: error allocating memblock for heap %d\n",
1635 __func__, i);
1636 continue;
1637 }
1638 data->heaps[i].base = paddr;
1639 } else {
1640 int ret = memblock_reserve(data->heaps[i].base,
1641 data->heaps[i].size);
1642 if (ret)
1643 pr_err("memblock reserve of %zx@%lx failed\n",
1644 data->heaps[i].size,
1645 data->heaps[i].base);
1646 }
1647 pr_info("%s: %s reserved base %lx size %zu\n", __func__,
1648 data->heaps[i].name,
1649 data->heaps[i].base,
1650 data->heaps[i].size);
1651 }
1652}
1653