1
2
3
4
5
6
7
8
9#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
10
11#include <linux/list.h>
12#include <linux/sched/mm.h>
13#include <linux/module.h>
14#include <linux/rtmutex.h>
15#include <linux/rbtree.h>
16#include <linux/seq_file.h>
17#include <linux/vmalloc.h>
18#include <linux/slab.h>
19#include <linux/sched.h>
20#include <linux/list_lru.h>
21#include <linux/ratelimit.h>
22#include <asm/cacheflush.h>
23#include <linux/uaccess.h>
24#include <linux/highmem.h>
25#include "binder_alloc.h"
26#include "binder_trace.h"
27
28struct list_lru binder_alloc_lru;
29
30static DEFINE_MUTEX(binder_alloc_mmap_lock);
31
32enum {
33 BINDER_DEBUG_USER_ERROR = 1U << 0,
34 BINDER_DEBUG_OPEN_CLOSE = 1U << 1,
35 BINDER_DEBUG_BUFFER_ALLOC = 1U << 2,
36 BINDER_DEBUG_BUFFER_ALLOC_ASYNC = 1U << 3,
37};
38static uint32_t binder_alloc_debug_mask = BINDER_DEBUG_USER_ERROR;
39
40module_param_named(debug_mask, binder_alloc_debug_mask,
41 uint, 0644);
42
43#define binder_alloc_debug(mask, x...) \
44 do { \
45 if (binder_alloc_debug_mask & mask) \
46 pr_info_ratelimited(x); \
47 } while (0)
48
49static struct binder_buffer *binder_buffer_next(struct binder_buffer *buffer)
50{
51 return list_entry(buffer->entry.next, struct binder_buffer, entry);
52}
53
54static struct binder_buffer *binder_buffer_prev(struct binder_buffer *buffer)
55{
56 return list_entry(buffer->entry.prev, struct binder_buffer, entry);
57}
58
59static size_t binder_alloc_buffer_size(struct binder_alloc *alloc,
60 struct binder_buffer *buffer)
61{
62 if (list_is_last(&buffer->entry, &alloc->buffers))
63 return alloc->buffer + alloc->buffer_size - buffer->user_data;
64 return binder_buffer_next(buffer)->user_data - buffer->user_data;
65}
66
67static void binder_insert_free_buffer(struct binder_alloc *alloc,
68 struct binder_buffer *new_buffer)
69{
70 struct rb_node **p = &alloc->free_buffers.rb_node;
71 struct rb_node *parent = NULL;
72 struct binder_buffer *buffer;
73 size_t buffer_size;
74 size_t new_buffer_size;
75
76 BUG_ON(!new_buffer->free);
77
78 new_buffer_size = binder_alloc_buffer_size(alloc, new_buffer);
79
80 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
81 "%d: add free buffer, size %zd, at %pK\n",
82 alloc->pid, new_buffer_size, new_buffer);
83
84 while (*p) {
85 parent = *p;
86 buffer = rb_entry(parent, struct binder_buffer, rb_node);
87 BUG_ON(!buffer->free);
88
89 buffer_size = binder_alloc_buffer_size(alloc, buffer);
90
91 if (new_buffer_size < buffer_size)
92 p = &parent->rb_left;
93 else
94 p = &parent->rb_right;
95 }
96 rb_link_node(&new_buffer->rb_node, parent, p);
97 rb_insert_color(&new_buffer->rb_node, &alloc->free_buffers);
98}
99
100static void binder_insert_allocated_buffer_locked(
101 struct binder_alloc *alloc, struct binder_buffer *new_buffer)
102{
103 struct rb_node **p = &alloc->allocated_buffers.rb_node;
104 struct rb_node *parent = NULL;
105 struct binder_buffer *buffer;
106
107 BUG_ON(new_buffer->free);
108
109 while (*p) {
110 parent = *p;
111 buffer = rb_entry(parent, struct binder_buffer, rb_node);
112 BUG_ON(buffer->free);
113
114 if (new_buffer->user_data < buffer->user_data)
115 p = &parent->rb_left;
116 else if (new_buffer->user_data > buffer->user_data)
117 p = &parent->rb_right;
118 else
119 BUG();
120 }
121 rb_link_node(&new_buffer->rb_node, parent, p);
122 rb_insert_color(&new_buffer->rb_node, &alloc->allocated_buffers);
123}
124
125static struct binder_buffer *binder_alloc_prepare_to_free_locked(
126 struct binder_alloc *alloc,
127 uintptr_t user_ptr)
128{
129 struct rb_node *n = alloc->allocated_buffers.rb_node;
130 struct binder_buffer *buffer;
131 void __user *uptr;
132
133 uptr = (void __user *)user_ptr;
134
135 while (n) {
136 buffer = rb_entry(n, struct binder_buffer, rb_node);
137 BUG_ON(buffer->free);
138
139 if (uptr < buffer->user_data)
140 n = n->rb_left;
141 else if (uptr > buffer->user_data)
142 n = n->rb_right;
143 else {
144
145
146
147
148
149 if (!buffer->allow_user_free)
150 return ERR_PTR(-EPERM);
151 buffer->allow_user_free = 0;
152 return buffer;
153 }
154 }
155 return NULL;
156}
157
158
159
160
161
162
163
164
165
166
167
168
169struct binder_buffer *binder_alloc_prepare_to_free(struct binder_alloc *alloc,
170 uintptr_t user_ptr)
171{
172 struct binder_buffer *buffer;
173
174 mutex_lock(&alloc->mutex);
175 buffer = binder_alloc_prepare_to_free_locked(alloc, user_ptr);
176 mutex_unlock(&alloc->mutex);
177 return buffer;
178}
179
180static int binder_update_page_range(struct binder_alloc *alloc, int allocate,
181 void __user *start, void __user *end)
182{
183 void __user *page_addr;
184 unsigned long user_page_addr;
185 struct binder_lru_page *page;
186 struct vm_area_struct *vma = NULL;
187 struct mm_struct *mm = NULL;
188 bool need_mm = false;
189
190 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
191 "%d: %s pages %pK-%pK\n", alloc->pid,
192 allocate ? "allocate" : "free", start, end);
193
194 if (end <= start)
195 return 0;
196
197 trace_binder_update_page_range(alloc, allocate, start, end);
198
199 if (allocate == 0)
200 goto free_range;
201
202 for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) {
203 page = &alloc->pages[(page_addr - alloc->buffer) / PAGE_SIZE];
204 if (!page->page_ptr) {
205 need_mm = true;
206 break;
207 }
208 }
209
210 if (need_mm && mmget_not_zero(alloc->vma_vm_mm))
211 mm = alloc->vma_vm_mm;
212
213 if (mm) {
214 down_read(&mm->mmap_sem);
215 vma = alloc->vma;
216 }
217
218 if (!vma && need_mm) {
219 binder_alloc_debug(BINDER_DEBUG_USER_ERROR,
220 "%d: binder_alloc_buf failed to map pages in userspace, no vma\n",
221 alloc->pid);
222 goto err_no_vma;
223 }
224
225 for (page_addr = start; page_addr < end; page_addr += PAGE_SIZE) {
226 int ret;
227 bool on_lru;
228 size_t index;
229
230 index = (page_addr - alloc->buffer) / PAGE_SIZE;
231 page = &alloc->pages[index];
232
233 if (page->page_ptr) {
234 trace_binder_alloc_lru_start(alloc, index);
235
236 on_lru = list_lru_del(&binder_alloc_lru, &page->lru);
237 WARN_ON(!on_lru);
238
239 trace_binder_alloc_lru_end(alloc, index);
240 continue;
241 }
242
243 if (WARN_ON(!vma))
244 goto err_page_ptr_cleared;
245
246 trace_binder_alloc_page_start(alloc, index);
247 page->page_ptr = alloc_page(GFP_KERNEL |
248 __GFP_HIGHMEM |
249 __GFP_ZERO);
250 if (!page->page_ptr) {
251 pr_err("%d: binder_alloc_buf failed for page at %pK\n",
252 alloc->pid, page_addr);
253 goto err_alloc_page_failed;
254 }
255 page->alloc = alloc;
256 INIT_LIST_HEAD(&page->lru);
257
258 user_page_addr = (uintptr_t)page_addr;
259 ret = vm_insert_page(vma, user_page_addr, page[0].page_ptr);
260 if (ret) {
261 pr_err("%d: binder_alloc_buf failed to map page at %lx in userspace\n",
262 alloc->pid, user_page_addr);
263 goto err_vm_insert_page_failed;
264 }
265
266 if (index + 1 > alloc->pages_high)
267 alloc->pages_high = index + 1;
268
269 trace_binder_alloc_page_end(alloc, index);
270
271 }
272 if (mm) {
273 up_read(&mm->mmap_sem);
274 mmput(mm);
275 }
276 return 0;
277
278free_range:
279 for (page_addr = end - PAGE_SIZE; page_addr >= start;
280 page_addr -= PAGE_SIZE) {
281 bool ret;
282 size_t index;
283
284 index = (page_addr - alloc->buffer) / PAGE_SIZE;
285 page = &alloc->pages[index];
286
287 trace_binder_free_lru_start(alloc, index);
288
289 ret = list_lru_add(&binder_alloc_lru, &page->lru);
290 WARN_ON(!ret);
291
292 trace_binder_free_lru_end(alloc, index);
293 continue;
294
295err_vm_insert_page_failed:
296 __free_page(page->page_ptr);
297 page->page_ptr = NULL;
298err_alloc_page_failed:
299err_page_ptr_cleared:
300 ;
301 }
302err_no_vma:
303 if (mm) {
304 up_read(&mm->mmap_sem);
305 mmput(mm);
306 }
307 return vma ? -ENOMEM : -ESRCH;
308}
309
310
311static inline void binder_alloc_set_vma(struct binder_alloc *alloc,
312 struct vm_area_struct *vma)
313{
314 if (vma)
315 alloc->vma_vm_mm = vma->vm_mm;
316
317
318
319
320
321
322 smp_wmb();
323 alloc->vma = vma;
324}
325
326static inline struct vm_area_struct *binder_alloc_get_vma(
327 struct binder_alloc *alloc)
328{
329 struct vm_area_struct *vma = NULL;
330
331 if (alloc->vma) {
332
333 smp_rmb();
334 vma = alloc->vma;
335 }
336 return vma;
337}
338
339static struct binder_buffer *binder_alloc_new_buf_locked(
340 struct binder_alloc *alloc,
341 size_t data_size,
342 size_t offsets_size,
343 size_t extra_buffers_size,
344 int is_async)
345{
346 struct rb_node *n = alloc->free_buffers.rb_node;
347 struct binder_buffer *buffer;
348 size_t buffer_size;
349 struct rb_node *best_fit = NULL;
350 void __user *has_page_addr;
351 void __user *end_page_addr;
352 size_t size, data_offsets_size;
353 int ret;
354
355 if (!binder_alloc_get_vma(alloc)) {
356 binder_alloc_debug(BINDER_DEBUG_USER_ERROR,
357 "%d: binder_alloc_buf, no vma\n",
358 alloc->pid);
359 return ERR_PTR(-ESRCH);
360 }
361
362 data_offsets_size = ALIGN(data_size, sizeof(void *)) +
363 ALIGN(offsets_size, sizeof(void *));
364
365 if (data_offsets_size < data_size || data_offsets_size < offsets_size) {
366 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
367 "%d: got transaction with invalid size %zd-%zd\n",
368 alloc->pid, data_size, offsets_size);
369 return ERR_PTR(-EINVAL);
370 }
371 size = data_offsets_size + ALIGN(extra_buffers_size, sizeof(void *));
372 if (size < data_offsets_size || size < extra_buffers_size) {
373 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
374 "%d: got transaction with invalid extra_buffers_size %zd\n",
375 alloc->pid, extra_buffers_size);
376 return ERR_PTR(-EINVAL);
377 }
378 if (is_async &&
379 alloc->free_async_space < size + sizeof(struct binder_buffer)) {
380 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
381 "%d: binder_alloc_buf size %zd failed, no async space left\n",
382 alloc->pid, size);
383 return ERR_PTR(-ENOSPC);
384 }
385
386
387 size = max(size, sizeof(void *));
388
389 while (n) {
390 buffer = rb_entry(n, struct binder_buffer, rb_node);
391 BUG_ON(!buffer->free);
392 buffer_size = binder_alloc_buffer_size(alloc, buffer);
393
394 if (size < buffer_size) {
395 best_fit = n;
396 n = n->rb_left;
397 } else if (size > buffer_size)
398 n = n->rb_right;
399 else {
400 best_fit = n;
401 break;
402 }
403 }
404 if (best_fit == NULL) {
405 size_t allocated_buffers = 0;
406 size_t largest_alloc_size = 0;
407 size_t total_alloc_size = 0;
408 size_t free_buffers = 0;
409 size_t largest_free_size = 0;
410 size_t total_free_size = 0;
411
412 for (n = rb_first(&alloc->allocated_buffers); n != NULL;
413 n = rb_next(n)) {
414 buffer = rb_entry(n, struct binder_buffer, rb_node);
415 buffer_size = binder_alloc_buffer_size(alloc, buffer);
416 allocated_buffers++;
417 total_alloc_size += buffer_size;
418 if (buffer_size > largest_alloc_size)
419 largest_alloc_size = buffer_size;
420 }
421 for (n = rb_first(&alloc->free_buffers); n != NULL;
422 n = rb_next(n)) {
423 buffer = rb_entry(n, struct binder_buffer, rb_node);
424 buffer_size = binder_alloc_buffer_size(alloc, buffer);
425 free_buffers++;
426 total_free_size += buffer_size;
427 if (buffer_size > largest_free_size)
428 largest_free_size = buffer_size;
429 }
430 binder_alloc_debug(BINDER_DEBUG_USER_ERROR,
431 "%d: binder_alloc_buf size %zd failed, no address space\n",
432 alloc->pid, size);
433 binder_alloc_debug(BINDER_DEBUG_USER_ERROR,
434 "allocated: %zd (num: %zd largest: %zd), free: %zd (num: %zd largest: %zd)\n",
435 total_alloc_size, allocated_buffers,
436 largest_alloc_size, total_free_size,
437 free_buffers, largest_free_size);
438 return ERR_PTR(-ENOSPC);
439 }
440 if (n == NULL) {
441 buffer = rb_entry(best_fit, struct binder_buffer, rb_node);
442 buffer_size = binder_alloc_buffer_size(alloc, buffer);
443 }
444
445 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
446 "%d: binder_alloc_buf size %zd got buffer %pK size %zd\n",
447 alloc->pid, size, buffer, buffer_size);
448
449 has_page_addr = (void __user *)
450 (((uintptr_t)buffer->user_data + buffer_size) & PAGE_MASK);
451 WARN_ON(n && buffer_size != size);
452 end_page_addr =
453 (void __user *)PAGE_ALIGN((uintptr_t)buffer->user_data + size);
454 if (end_page_addr > has_page_addr)
455 end_page_addr = has_page_addr;
456 ret = binder_update_page_range(alloc, 1, (void __user *)
457 PAGE_ALIGN((uintptr_t)buffer->user_data), end_page_addr);
458 if (ret)
459 return ERR_PTR(ret);
460
461 if (buffer_size != size) {
462 struct binder_buffer *new_buffer;
463
464 new_buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
465 if (!new_buffer) {
466 pr_err("%s: %d failed to alloc new buffer struct\n",
467 __func__, alloc->pid);
468 goto err_alloc_buf_struct_failed;
469 }
470 new_buffer->user_data = (u8 __user *)buffer->user_data + size;
471 list_add(&new_buffer->entry, &buffer->entry);
472 new_buffer->free = 1;
473 binder_insert_free_buffer(alloc, new_buffer);
474 }
475
476 rb_erase(best_fit, &alloc->free_buffers);
477 buffer->free = 0;
478 buffer->allow_user_free = 0;
479 binder_insert_allocated_buffer_locked(alloc, buffer);
480 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
481 "%d: binder_alloc_buf size %zd got %pK\n",
482 alloc->pid, size, buffer);
483 buffer->data_size = data_size;
484 buffer->offsets_size = offsets_size;
485 buffer->async_transaction = is_async;
486 buffer->extra_buffers_size = extra_buffers_size;
487 if (is_async) {
488 alloc->free_async_space -= size + sizeof(struct binder_buffer);
489 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC,
490 "%d: binder_alloc_buf size %zd async free %zd\n",
491 alloc->pid, size, alloc->free_async_space);
492 }
493 return buffer;
494
495err_alloc_buf_struct_failed:
496 binder_update_page_range(alloc, 0, (void __user *)
497 PAGE_ALIGN((uintptr_t)buffer->user_data),
498 end_page_addr);
499 return ERR_PTR(-ENOMEM);
500}
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517struct binder_buffer *binder_alloc_new_buf(struct binder_alloc *alloc,
518 size_t data_size,
519 size_t offsets_size,
520 size_t extra_buffers_size,
521 int is_async)
522{
523 struct binder_buffer *buffer;
524
525 mutex_lock(&alloc->mutex);
526 buffer = binder_alloc_new_buf_locked(alloc, data_size, offsets_size,
527 extra_buffers_size, is_async);
528 mutex_unlock(&alloc->mutex);
529 return buffer;
530}
531
532static void __user *buffer_start_page(struct binder_buffer *buffer)
533{
534 return (void __user *)((uintptr_t)buffer->user_data & PAGE_MASK);
535}
536
537static void __user *prev_buffer_end_page(struct binder_buffer *buffer)
538{
539 return (void __user *)
540 (((uintptr_t)(buffer->user_data) - 1) & PAGE_MASK);
541}
542
543static void binder_delete_free_buffer(struct binder_alloc *alloc,
544 struct binder_buffer *buffer)
545{
546 struct binder_buffer *prev, *next = NULL;
547 bool to_free = true;
548 BUG_ON(alloc->buffers.next == &buffer->entry);
549 prev = binder_buffer_prev(buffer);
550 BUG_ON(!prev->free);
551 if (prev_buffer_end_page(prev) == buffer_start_page(buffer)) {
552 to_free = false;
553 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
554 "%d: merge free, buffer %pK share page with %pK\n",
555 alloc->pid, buffer->user_data,
556 prev->user_data);
557 }
558
559 if (!list_is_last(&buffer->entry, &alloc->buffers)) {
560 next = binder_buffer_next(buffer);
561 if (buffer_start_page(next) == buffer_start_page(buffer)) {
562 to_free = false;
563 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
564 "%d: merge free, buffer %pK share page with %pK\n",
565 alloc->pid,
566 buffer->user_data,
567 next->user_data);
568 }
569 }
570
571 if (PAGE_ALIGNED(buffer->user_data)) {
572 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
573 "%d: merge free, buffer start %pK is page aligned\n",
574 alloc->pid, buffer->user_data);
575 to_free = false;
576 }
577
578 if (to_free) {
579 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
580 "%d: merge free, buffer %pK do not share page with %pK or %pK\n",
581 alloc->pid, buffer->user_data,
582 prev->user_data,
583 next ? next->user_data : NULL);
584 binder_update_page_range(alloc, 0, buffer_start_page(buffer),
585 buffer_start_page(buffer) + PAGE_SIZE);
586 }
587 list_del(&buffer->entry);
588 kfree(buffer);
589}
590
591static void binder_free_buf_locked(struct binder_alloc *alloc,
592 struct binder_buffer *buffer)
593{
594 size_t size, buffer_size;
595
596 buffer_size = binder_alloc_buffer_size(alloc, buffer);
597
598 size = ALIGN(buffer->data_size, sizeof(void *)) +
599 ALIGN(buffer->offsets_size, sizeof(void *)) +
600 ALIGN(buffer->extra_buffers_size, sizeof(void *));
601
602 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
603 "%d: binder_free_buf %pK size %zd buffer_size %zd\n",
604 alloc->pid, buffer, size, buffer_size);
605
606 BUG_ON(buffer->free);
607 BUG_ON(size > buffer_size);
608 BUG_ON(buffer->transaction != NULL);
609 BUG_ON(buffer->user_data < alloc->buffer);
610 BUG_ON(buffer->user_data > alloc->buffer + alloc->buffer_size);
611
612 if (buffer->async_transaction) {
613 alloc->free_async_space += size + sizeof(struct binder_buffer);
614
615 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC_ASYNC,
616 "%d: binder_free_buf size %zd async free %zd\n",
617 alloc->pid, size, alloc->free_async_space);
618 }
619
620 binder_update_page_range(alloc, 0,
621 (void __user *)PAGE_ALIGN((uintptr_t)buffer->user_data),
622 (void __user *)(((uintptr_t)
623 buffer->user_data + buffer_size) & PAGE_MASK));
624
625 rb_erase(&buffer->rb_node, &alloc->allocated_buffers);
626 buffer->free = 1;
627 if (!list_is_last(&buffer->entry, &alloc->buffers)) {
628 struct binder_buffer *next = binder_buffer_next(buffer);
629
630 if (next->free) {
631 rb_erase(&next->rb_node, &alloc->free_buffers);
632 binder_delete_free_buffer(alloc, next);
633 }
634 }
635 if (alloc->buffers.next != &buffer->entry) {
636 struct binder_buffer *prev = binder_buffer_prev(buffer);
637
638 if (prev->free) {
639 binder_delete_free_buffer(alloc, buffer);
640 rb_erase(&prev->rb_node, &alloc->free_buffers);
641 buffer = prev;
642 }
643 }
644 binder_insert_free_buffer(alloc, buffer);
645}
646
647
648
649
650
651
652
653
654void binder_alloc_free_buf(struct binder_alloc *alloc,
655 struct binder_buffer *buffer)
656{
657 mutex_lock(&alloc->mutex);
658 binder_free_buf_locked(alloc, buffer);
659 mutex_unlock(&alloc->mutex);
660}
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675int binder_alloc_mmap_handler(struct binder_alloc *alloc,
676 struct vm_area_struct *vma)
677{
678 int ret;
679 const char *failure_string;
680 struct binder_buffer *buffer;
681
682 mutex_lock(&binder_alloc_mmap_lock);
683 if (alloc->buffer) {
684 ret = -EBUSY;
685 failure_string = "already mapped";
686 goto err_already_mapped;
687 }
688
689 alloc->buffer = (void __user *)vma->vm_start;
690 mutex_unlock(&binder_alloc_mmap_lock);
691
692 alloc->pages = kcalloc((vma->vm_end - vma->vm_start) / PAGE_SIZE,
693 sizeof(alloc->pages[0]),
694 GFP_KERNEL);
695 if (alloc->pages == NULL) {
696 ret = -ENOMEM;
697 failure_string = "alloc page array";
698 goto err_alloc_pages_failed;
699 }
700 alloc->buffer_size = vma->vm_end - vma->vm_start;
701
702 buffer = kzalloc(sizeof(*buffer), GFP_KERNEL);
703 if (!buffer) {
704 ret = -ENOMEM;
705 failure_string = "alloc buffer struct";
706 goto err_alloc_buf_struct_failed;
707 }
708
709 buffer->user_data = alloc->buffer;
710 list_add(&buffer->entry, &alloc->buffers);
711 buffer->free = 1;
712 binder_insert_free_buffer(alloc, buffer);
713 alloc->free_async_space = alloc->buffer_size / 2;
714 binder_alloc_set_vma(alloc, vma);
715 mmgrab(alloc->vma_vm_mm);
716
717 return 0;
718
719err_alloc_buf_struct_failed:
720 kfree(alloc->pages);
721 alloc->pages = NULL;
722err_alloc_pages_failed:
723 mutex_lock(&binder_alloc_mmap_lock);
724 alloc->buffer = NULL;
725err_already_mapped:
726 mutex_unlock(&binder_alloc_mmap_lock);
727 binder_alloc_debug(BINDER_DEBUG_USER_ERROR,
728 "%s: %d %lx-%lx %s failed %d\n", __func__,
729 alloc->pid, vma->vm_start, vma->vm_end,
730 failure_string, ret);
731 return ret;
732}
733
734
735void binder_alloc_deferred_release(struct binder_alloc *alloc)
736{
737 struct rb_node *n;
738 int buffers, page_count;
739 struct binder_buffer *buffer;
740
741 buffers = 0;
742 mutex_lock(&alloc->mutex);
743 BUG_ON(alloc->vma);
744
745 while ((n = rb_first(&alloc->allocated_buffers))) {
746 buffer = rb_entry(n, struct binder_buffer, rb_node);
747
748
749 BUG_ON(buffer->transaction);
750
751 binder_free_buf_locked(alloc, buffer);
752 buffers++;
753 }
754
755 while (!list_empty(&alloc->buffers)) {
756 buffer = list_first_entry(&alloc->buffers,
757 struct binder_buffer, entry);
758 WARN_ON(!buffer->free);
759
760 list_del(&buffer->entry);
761 WARN_ON_ONCE(!list_empty(&alloc->buffers));
762 kfree(buffer);
763 }
764
765 page_count = 0;
766 if (alloc->pages) {
767 int i;
768
769 for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) {
770 void __user *page_addr;
771 bool on_lru;
772
773 if (!alloc->pages[i].page_ptr)
774 continue;
775
776 on_lru = list_lru_del(&binder_alloc_lru,
777 &alloc->pages[i].lru);
778 page_addr = alloc->buffer + i * PAGE_SIZE;
779 binder_alloc_debug(BINDER_DEBUG_BUFFER_ALLOC,
780 "%s: %d: page %d at %pK %s\n",
781 __func__, alloc->pid, i, page_addr,
782 on_lru ? "on lru" : "active");
783 __free_page(alloc->pages[i].page_ptr);
784 page_count++;
785 }
786 kfree(alloc->pages);
787 }
788 mutex_unlock(&alloc->mutex);
789 if (alloc->vma_vm_mm)
790 mmdrop(alloc->vma_vm_mm);
791
792 binder_alloc_debug(BINDER_DEBUG_OPEN_CLOSE,
793 "%s: %d buffers %d, pages %d\n",
794 __func__, alloc->pid, buffers, page_count);
795}
796
797static void print_binder_buffer(struct seq_file *m, const char *prefix,
798 struct binder_buffer *buffer)
799{
800 seq_printf(m, "%s %d: %pK size %zd:%zd:%zd %s\n",
801 prefix, buffer->debug_id, buffer->user_data,
802 buffer->data_size, buffer->offsets_size,
803 buffer->extra_buffers_size,
804 buffer->transaction ? "active" : "delivered");
805}
806
807
808
809
810
811
812
813
814
815void binder_alloc_print_allocated(struct seq_file *m,
816 struct binder_alloc *alloc)
817{
818 struct rb_node *n;
819
820 mutex_lock(&alloc->mutex);
821 for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n))
822 print_binder_buffer(m, " buffer",
823 rb_entry(n, struct binder_buffer, rb_node));
824 mutex_unlock(&alloc->mutex);
825}
826
827
828
829
830
831
832void binder_alloc_print_pages(struct seq_file *m,
833 struct binder_alloc *alloc)
834{
835 struct binder_lru_page *page;
836 int i;
837 int active = 0;
838 int lru = 0;
839 int free = 0;
840
841 mutex_lock(&alloc->mutex);
842 for (i = 0; i < alloc->buffer_size / PAGE_SIZE; i++) {
843 page = &alloc->pages[i];
844 if (!page->page_ptr)
845 free++;
846 else if (list_empty(&page->lru))
847 active++;
848 else
849 lru++;
850 }
851 mutex_unlock(&alloc->mutex);
852 seq_printf(m, " pages: %d:%d:%d\n", active, lru, free);
853 seq_printf(m, " pages high watermark: %zu\n", alloc->pages_high);
854}
855
856
857
858
859
860
861
862int binder_alloc_get_allocated_count(struct binder_alloc *alloc)
863{
864 struct rb_node *n;
865 int count = 0;
866
867 mutex_lock(&alloc->mutex);
868 for (n = rb_first(&alloc->allocated_buffers); n != NULL; n = rb_next(n))
869 count++;
870 mutex_unlock(&alloc->mutex);
871 return count;
872}
873
874
875
876
877
878
879
880
881
882
883void binder_alloc_vma_close(struct binder_alloc *alloc)
884{
885 binder_alloc_set_vma(alloc, NULL);
886}
887
888
889
890
891
892
893
894
895
896
897enum lru_status binder_alloc_free_page(struct list_head *item,
898 struct list_lru_one *lru,
899 spinlock_t *lock,
900 void *cb_arg)
901 __must_hold(lock)
902{
903 struct mm_struct *mm = NULL;
904 struct binder_lru_page *page = container_of(item,
905 struct binder_lru_page,
906 lru);
907 struct binder_alloc *alloc;
908 uintptr_t page_addr;
909 size_t index;
910 struct vm_area_struct *vma;
911
912 alloc = page->alloc;
913 if (!mutex_trylock(&alloc->mutex))
914 goto err_get_alloc_mutex_failed;
915
916 if (!page->page_ptr)
917 goto err_page_already_freed;
918
919 index = page - alloc->pages;
920 page_addr = (uintptr_t)alloc->buffer + index * PAGE_SIZE;
921
922 mm = alloc->vma_vm_mm;
923 if (!mmget_not_zero(mm))
924 goto err_mmget;
925 if (!down_read_trylock(&mm->mmap_sem))
926 goto err_down_read_mmap_sem_failed;
927 vma = binder_alloc_get_vma(alloc);
928
929 list_lru_isolate(lru, item);
930 spin_unlock(lock);
931
932 if (vma) {
933 trace_binder_unmap_user_start(alloc, index);
934
935 zap_page_range(vma, page_addr, PAGE_SIZE);
936
937 trace_binder_unmap_user_end(alloc, index);
938 }
939 up_read(&mm->mmap_sem);
940 mmput(mm);
941
942 trace_binder_unmap_kernel_start(alloc, index);
943
944 __free_page(page->page_ptr);
945 page->page_ptr = NULL;
946
947 trace_binder_unmap_kernel_end(alloc, index);
948
949 spin_lock(lock);
950 mutex_unlock(&alloc->mutex);
951 return LRU_REMOVED_RETRY;
952
953err_down_read_mmap_sem_failed:
954 mmput_async(mm);
955err_mmget:
956err_page_already_freed:
957 mutex_unlock(&alloc->mutex);
958err_get_alloc_mutex_failed:
959 return LRU_SKIP;
960}
961
962static unsigned long
963binder_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
964{
965 unsigned long ret = list_lru_count(&binder_alloc_lru);
966 return ret;
967}
968
969static unsigned long
970binder_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
971{
972 unsigned long ret;
973
974 ret = list_lru_walk(&binder_alloc_lru, binder_alloc_free_page,
975 NULL, sc->nr_to_scan);
976 return ret;
977}
978
979static struct shrinker binder_shrinker = {
980 .count_objects = binder_shrink_count,
981 .scan_objects = binder_shrink_scan,
982 .seeks = DEFAULT_SEEKS,
983};
984
985
986
987
988
989
990
991
992void binder_alloc_init(struct binder_alloc *alloc)
993{
994 alloc->pid = current->group_leader->pid;
995 mutex_init(&alloc->mutex);
996 INIT_LIST_HEAD(&alloc->buffers);
997}
998
999int binder_alloc_shrinker_init(void)
1000{
1001 int ret = list_lru_init(&binder_alloc_lru);
1002
1003 if (ret == 0) {
1004 ret = register_shrinker(&binder_shrinker);
1005 if (ret)
1006 list_lru_destroy(&binder_alloc_lru);
1007 }
1008 return ret;
1009}
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030static inline bool check_buffer(struct binder_alloc *alloc,
1031 struct binder_buffer *buffer,
1032 binder_size_t offset, size_t bytes)
1033{
1034 size_t buffer_size = binder_alloc_buffer_size(alloc, buffer);
1035
1036 return buffer_size >= bytes &&
1037 offset <= buffer_size - bytes &&
1038 IS_ALIGNED(offset, sizeof(u32)) &&
1039 !buffer->free &&
1040 (!buffer->allow_user_free || !buffer->transaction);
1041}
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062static struct page *binder_alloc_get_page(struct binder_alloc *alloc,
1063 struct binder_buffer *buffer,
1064 binder_size_t buffer_offset,
1065 pgoff_t *pgoffp)
1066{
1067 binder_size_t buffer_space_offset = buffer_offset +
1068 (buffer->user_data - alloc->buffer);
1069 pgoff_t pgoff = buffer_space_offset & ~PAGE_MASK;
1070 size_t index = buffer_space_offset >> PAGE_SHIFT;
1071 struct binder_lru_page *lru_page;
1072
1073 lru_page = &alloc->pages[index];
1074 *pgoffp = pgoff;
1075 return lru_page->page_ptr;
1076}
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090unsigned long
1091binder_alloc_copy_user_to_buffer(struct binder_alloc *alloc,
1092 struct binder_buffer *buffer,
1093 binder_size_t buffer_offset,
1094 const void __user *from,
1095 size_t bytes)
1096{
1097 if (!check_buffer(alloc, buffer, buffer_offset, bytes))
1098 return bytes;
1099
1100 while (bytes) {
1101 unsigned long size;
1102 unsigned long ret;
1103 struct page *page;
1104 pgoff_t pgoff;
1105 void *kptr;
1106
1107 page = binder_alloc_get_page(alloc, buffer,
1108 buffer_offset, &pgoff);
1109 size = min_t(size_t, bytes, PAGE_SIZE - pgoff);
1110 kptr = kmap(page) + pgoff;
1111 ret = copy_from_user(kptr, from, size);
1112 kunmap(page);
1113 if (ret)
1114 return bytes - size + ret;
1115 bytes -= size;
1116 from += size;
1117 buffer_offset += size;
1118 }
1119 return 0;
1120}
1121
1122static int binder_alloc_do_buffer_copy(struct binder_alloc *alloc,
1123 bool to_buffer,
1124 struct binder_buffer *buffer,
1125 binder_size_t buffer_offset,
1126 void *ptr,
1127 size_t bytes)
1128{
1129
1130 if (!check_buffer(alloc, buffer, buffer_offset, bytes))
1131 return -EINVAL;
1132
1133 while (bytes) {
1134 unsigned long size;
1135 struct page *page;
1136 pgoff_t pgoff;
1137 void *tmpptr;
1138 void *base_ptr;
1139
1140 page = binder_alloc_get_page(alloc, buffer,
1141 buffer_offset, &pgoff);
1142 size = min_t(size_t, bytes, PAGE_SIZE - pgoff);
1143 base_ptr = kmap_atomic(page);
1144 tmpptr = base_ptr + pgoff;
1145 if (to_buffer)
1146 memcpy(tmpptr, ptr, size);
1147 else
1148 memcpy(ptr, tmpptr, size);
1149
1150
1151
1152
1153 kunmap_atomic(base_ptr);
1154 bytes -= size;
1155 pgoff = 0;
1156 ptr = ptr + size;
1157 buffer_offset += size;
1158 }
1159 return 0;
1160}
1161
1162int binder_alloc_copy_to_buffer(struct binder_alloc *alloc,
1163 struct binder_buffer *buffer,
1164 binder_size_t buffer_offset,
1165 void *src,
1166 size_t bytes)
1167{
1168 return binder_alloc_do_buffer_copy(alloc, true, buffer, buffer_offset,
1169 src, bytes);
1170}
1171
1172int binder_alloc_copy_from_buffer(struct binder_alloc *alloc,
1173 void *dest,
1174 struct binder_buffer *buffer,
1175 binder_size_t buffer_offset,
1176 size_t bytes)
1177{
1178 return binder_alloc_do_buffer_copy(alloc, false, buffer, buffer_offset,
1179 dest, bytes);
1180}
1181
1182