1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#include <linux/kernel.h>
24#include <linux/types.h>
25#include <linux/gfp.h>
26#include <linux/mm.h>
27#include <linux/mm_types.h>
28#include <linux/hugetlb.h>
29#include <linux/highmem.h>
30#include <linux/slab.h>
31#include <linux/module.h>
32#include <linux/moduleparam.h>
33#include <linux/string.h>
34#include <linux/list.h>
35#include <linux/errno.h>
36#include <linux/io.h>
37#include <asm/current.h>
38#include <linux/sched/signal.h>
39#include <linux/file.h>
40
41#include <asm/set_memory.h>
42
43#include "atomisp_internal.h"
44#include "hmm/hmm_common.h"
45#include "hmm/hmm_pool.h"
46#include "hmm/hmm_bo.h"
47
48static unsigned int order_to_nr(unsigned int order)
49{
50 return 1U << order;
51}
52
53static unsigned int nr_to_order_bottom(unsigned int nr)
54{
55 return fls(nr) - 1;
56}
57
58static int __bo_init(struct hmm_bo_device *bdev, struct hmm_buffer_object *bo,
59 unsigned int pgnr)
60{
61 check_bodev_null_return(bdev, -EINVAL);
62 var_equal_return(hmm_bo_device_inited(bdev), 0, -EINVAL,
63 "hmm_bo_device not inited yet.\n");
64
65 if (pgnr == 0) {
66 dev_err(atomisp_dev, "0 size buffer is not allowed.\n");
67 return -EINVAL;
68 }
69
70 memset(bo, 0, sizeof(*bo));
71 mutex_init(&bo->mutex);
72
73
74 INIT_LIST_HEAD(&bo->list);
75
76 bo->bdev = bdev;
77 bo->vmap_addr = NULL;
78 bo->status = HMM_BO_FREE;
79 bo->start = bdev->start;
80 bo->pgnr = pgnr;
81 bo->end = bo->start + pgnr_to_size(pgnr);
82 bo->prev = NULL;
83 bo->next = NULL;
84
85 return 0;
86}
87
88static struct hmm_buffer_object *__bo_search_and_remove_from_free_rbtree(
89 struct rb_node *node, unsigned int pgnr)
90{
91 struct hmm_buffer_object *this, *ret_bo, *temp_bo;
92
93 this = rb_entry(node, struct hmm_buffer_object, node);
94 if (this->pgnr == pgnr ||
95 (this->pgnr > pgnr && !this->node.rb_left)) {
96 goto remove_bo_and_return;
97 } else {
98 if (this->pgnr < pgnr) {
99 if (!this->node.rb_right)
100 return NULL;
101 ret_bo = __bo_search_and_remove_from_free_rbtree(
102 this->node.rb_right, pgnr);
103 } else {
104 ret_bo = __bo_search_and_remove_from_free_rbtree(
105 this->node.rb_left, pgnr);
106 }
107 if (!ret_bo) {
108 if (this->pgnr > pgnr)
109 goto remove_bo_and_return;
110 else
111 return NULL;
112 }
113 return ret_bo;
114 }
115
116remove_bo_and_return:
117
118
119
120
121 if (!this->next) {
122 rb_erase(&this->node, &this->bdev->free_rbtree);
123 return this;
124 }
125
126
127
128
129
130 temp_bo = this->next;
131 this->next = temp_bo->next;
132 if (temp_bo->next)
133 temp_bo->next->prev = this;
134 temp_bo->next = NULL;
135 temp_bo->prev = NULL;
136 return temp_bo;
137}
138
139static struct hmm_buffer_object *__bo_search_by_addr(struct rb_root *root,
140 ia_css_ptr start)
141{
142 struct rb_node *n = root->rb_node;
143 struct hmm_buffer_object *bo;
144
145 do {
146 bo = rb_entry(n, struct hmm_buffer_object, node);
147
148 if (bo->start > start) {
149 if (!n->rb_left)
150 return NULL;
151 n = n->rb_left;
152 } else if (bo->start < start) {
153 if (!n->rb_right)
154 return NULL;
155 n = n->rb_right;
156 } else {
157 return bo;
158 }
159 } while (n);
160
161 return NULL;
162}
163
164static struct hmm_buffer_object *__bo_search_by_addr_in_range(
165 struct rb_root *root, unsigned int start)
166{
167 struct rb_node *n = root->rb_node;
168 struct hmm_buffer_object *bo;
169
170 do {
171 bo = rb_entry(n, struct hmm_buffer_object, node);
172
173 if (bo->start > start) {
174 if (!n->rb_left)
175 return NULL;
176 n = n->rb_left;
177 } else {
178 if (bo->end > start)
179 return bo;
180 if (!n->rb_right)
181 return NULL;
182 n = n->rb_right;
183 }
184 } while (n);
185
186 return NULL;
187}
188
189static void __bo_insert_to_free_rbtree(struct rb_root *root,
190 struct hmm_buffer_object *bo)
191{
192 struct rb_node **new = &root->rb_node;
193 struct rb_node *parent = NULL;
194 struct hmm_buffer_object *this;
195 unsigned int pgnr = bo->pgnr;
196
197 while (*new) {
198 parent = *new;
199 this = container_of(*new, struct hmm_buffer_object, node);
200
201 if (pgnr < this->pgnr) {
202 new = &((*new)->rb_left);
203 } else if (pgnr > this->pgnr) {
204 new = &((*new)->rb_right);
205 } else {
206 bo->prev = this;
207 bo->next = this->next;
208 if (this->next)
209 this->next->prev = bo;
210 this->next = bo;
211 bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_FREE;
212 return;
213 }
214 }
215
216 bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_FREE;
217
218 rb_link_node(&bo->node, parent, new);
219 rb_insert_color(&bo->node, root);
220}
221
222static void __bo_insert_to_alloc_rbtree(struct rb_root *root,
223 struct hmm_buffer_object *bo)
224{
225 struct rb_node **new = &root->rb_node;
226 struct rb_node *parent = NULL;
227 struct hmm_buffer_object *this;
228 unsigned int start = bo->start;
229
230 while (*new) {
231 parent = *new;
232 this = container_of(*new, struct hmm_buffer_object, node);
233
234 if (start < this->start)
235 new = &((*new)->rb_left);
236 else
237 new = &((*new)->rb_right);
238 }
239
240 kref_init(&bo->kref);
241 bo->status = (bo->status & ~HMM_BO_MASK) | HMM_BO_ALLOCED;
242
243 rb_link_node(&bo->node, parent, new);
244 rb_insert_color(&bo->node, root);
245}
246
247static struct hmm_buffer_object *__bo_break_up(struct hmm_bo_device *bdev,
248 struct hmm_buffer_object *bo,
249 unsigned int pgnr)
250{
251 struct hmm_buffer_object *new_bo;
252 unsigned long flags;
253 int ret;
254
255 new_bo = kmem_cache_alloc(bdev->bo_cache, GFP_KERNEL);
256 if (!new_bo) {
257 dev_err(atomisp_dev, "%s: __bo_alloc failed!\n", __func__);
258 return NULL;
259 }
260 ret = __bo_init(bdev, new_bo, pgnr);
261 if (ret) {
262 dev_err(atomisp_dev, "%s: __bo_init failed!\n", __func__);
263 kmem_cache_free(bdev->bo_cache, new_bo);
264 return NULL;
265 }
266
267 new_bo->start = bo->start;
268 new_bo->end = new_bo->start + pgnr_to_size(pgnr);
269 bo->start = new_bo->end;
270 bo->pgnr = bo->pgnr - pgnr;
271
272 spin_lock_irqsave(&bdev->list_lock, flags);
273 list_add_tail(&new_bo->list, &bo->list);
274 spin_unlock_irqrestore(&bdev->list_lock, flags);
275
276 return new_bo;
277}
278
279static void __bo_take_off_handling(struct hmm_buffer_object *bo)
280{
281 struct hmm_bo_device *bdev = bo->bdev;
282
283
284
285
286
287 if (!bo->prev && !bo->next) {
288 rb_erase(&bo->node, &bdev->free_rbtree);
289
290
291
292
293
294 } else if (!bo->prev && bo->next) {
295 bo->next->prev = NULL;
296 rb_erase(&bo->node, &bdev->free_rbtree);
297 __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo->next);
298 bo->next = NULL;
299
300
301
302
303
304 } else if (bo->prev && !bo->next) {
305 bo->prev->next = NULL;
306 bo->prev = NULL;
307
308
309
310
311
312 } else if (bo->prev && bo->next) {
313 bo->next->prev = bo->prev;
314 bo->prev->next = bo->next;
315 bo->next = NULL;
316 bo->prev = NULL;
317 }
318}
319
320static struct hmm_buffer_object *__bo_merge(struct hmm_buffer_object *bo,
321 struct hmm_buffer_object *next_bo)
322{
323 struct hmm_bo_device *bdev;
324 unsigned long flags;
325
326 bdev = bo->bdev;
327 next_bo->start = bo->start;
328 next_bo->pgnr = next_bo->pgnr + bo->pgnr;
329
330 spin_lock_irqsave(&bdev->list_lock, flags);
331 list_del(&bo->list);
332 spin_unlock_irqrestore(&bdev->list_lock, flags);
333
334 kmem_cache_free(bo->bdev->bo_cache, bo);
335
336 return next_bo;
337}
338
339
340
341
342int hmm_bo_device_init(struct hmm_bo_device *bdev,
343 struct isp_mmu_client *mmu_driver,
344 unsigned int vaddr_start,
345 unsigned int size)
346{
347 struct hmm_buffer_object *bo;
348 unsigned long flags;
349 int ret;
350
351 check_bodev_null_return(bdev, -EINVAL);
352
353 ret = isp_mmu_init(&bdev->mmu, mmu_driver);
354 if (ret) {
355 dev_err(atomisp_dev, "isp_mmu_init failed.\n");
356 return ret;
357 }
358
359 bdev->start = vaddr_start;
360 bdev->pgnr = size_to_pgnr_ceil(size);
361 bdev->size = pgnr_to_size(bdev->pgnr);
362
363 spin_lock_init(&bdev->list_lock);
364 mutex_init(&bdev->rbtree_mutex);
365
366 bdev->flag = HMM_BO_DEVICE_INITED;
367
368 INIT_LIST_HEAD(&bdev->entire_bo_list);
369 bdev->allocated_rbtree = RB_ROOT;
370 bdev->free_rbtree = RB_ROOT;
371
372 bdev->bo_cache = kmem_cache_create("bo_cache",
373 sizeof(struct hmm_buffer_object), 0, 0, NULL);
374 if (!bdev->bo_cache) {
375 dev_err(atomisp_dev, "%s: create cache failed!\n", __func__);
376 isp_mmu_exit(&bdev->mmu);
377 return -ENOMEM;
378 }
379
380 bo = kmem_cache_alloc(bdev->bo_cache, GFP_KERNEL);
381 if (!bo) {
382 dev_err(atomisp_dev, "%s: __bo_alloc failed!\n", __func__);
383 isp_mmu_exit(&bdev->mmu);
384 return -ENOMEM;
385 }
386
387 ret = __bo_init(bdev, bo, bdev->pgnr);
388 if (ret) {
389 dev_err(atomisp_dev, "%s: __bo_init failed!\n", __func__);
390 kmem_cache_free(bdev->bo_cache, bo);
391 isp_mmu_exit(&bdev->mmu);
392 return -EINVAL;
393 }
394
395 spin_lock_irqsave(&bdev->list_lock, flags);
396 list_add_tail(&bo->list, &bdev->entire_bo_list);
397 spin_unlock_irqrestore(&bdev->list_lock, flags);
398
399 __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo);
400
401 return 0;
402}
403
404struct hmm_buffer_object *hmm_bo_alloc(struct hmm_bo_device *bdev,
405 unsigned int pgnr)
406{
407 struct hmm_buffer_object *bo, *new_bo;
408 struct rb_root *root = &bdev->free_rbtree;
409
410 check_bodev_null_return(bdev, NULL);
411 var_equal_return(hmm_bo_device_inited(bdev), 0, NULL,
412 "hmm_bo_device not inited yet.\n");
413
414 if (pgnr == 0) {
415 dev_err(atomisp_dev, "0 size buffer is not allowed.\n");
416 return NULL;
417 }
418
419 mutex_lock(&bdev->rbtree_mutex);
420 bo = __bo_search_and_remove_from_free_rbtree(root->rb_node, pgnr);
421 if (!bo) {
422 mutex_unlock(&bdev->rbtree_mutex);
423 dev_err(atomisp_dev, "%s: Out of Memory! hmm_bo_alloc failed",
424 __func__);
425 return NULL;
426 }
427
428 if (bo->pgnr > pgnr) {
429 new_bo = __bo_break_up(bdev, bo, pgnr);
430 if (!new_bo) {
431 mutex_unlock(&bdev->rbtree_mutex);
432 dev_err(atomisp_dev, "%s: __bo_break_up failed!\n",
433 __func__);
434 return NULL;
435 }
436
437 __bo_insert_to_alloc_rbtree(&bdev->allocated_rbtree, new_bo);
438 __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo);
439
440 mutex_unlock(&bdev->rbtree_mutex);
441 return new_bo;
442 }
443
444 __bo_insert_to_alloc_rbtree(&bdev->allocated_rbtree, bo);
445
446 mutex_unlock(&bdev->rbtree_mutex);
447 return bo;
448}
449
450void hmm_bo_release(struct hmm_buffer_object *bo)
451{
452 struct hmm_bo_device *bdev = bo->bdev;
453 struct hmm_buffer_object *next_bo, *prev_bo;
454
455 mutex_lock(&bdev->rbtree_mutex);
456
457
458
459
460
461
462
463
464
465
466
467 if (bo->status & HMM_BO_MMAPED) {
468 mutex_unlock(&bdev->rbtree_mutex);
469 dev_dbg(atomisp_dev, "destroy bo which is MMAPED, do nothing\n");
470 return;
471 }
472
473 if (bo->status & HMM_BO_BINDED) {
474 dev_warn(atomisp_dev, "the bo is still binded, unbind it first...\n");
475 hmm_bo_unbind(bo);
476 }
477
478 if (bo->status & HMM_BO_PAGE_ALLOCED) {
479 dev_warn(atomisp_dev, "the pages is not freed, free pages first\n");
480 hmm_bo_free_pages(bo);
481 }
482 if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
483 dev_warn(atomisp_dev, "the vunmap is not done, do it...\n");
484 hmm_bo_vunmap(bo);
485 }
486
487 rb_erase(&bo->node, &bdev->allocated_rbtree);
488
489 prev_bo = list_entry(bo->list.prev, struct hmm_buffer_object, list);
490 next_bo = list_entry(bo->list.next, struct hmm_buffer_object, list);
491
492 if (bo->list.prev != &bdev->entire_bo_list &&
493 prev_bo->end == bo->start &&
494 (prev_bo->status & HMM_BO_MASK) == HMM_BO_FREE) {
495 __bo_take_off_handling(prev_bo);
496 bo = __bo_merge(prev_bo, bo);
497 }
498
499 if (bo->list.next != &bdev->entire_bo_list &&
500 next_bo->start == bo->end &&
501 (next_bo->status & HMM_BO_MASK) == HMM_BO_FREE) {
502 __bo_take_off_handling(next_bo);
503 bo = __bo_merge(bo, next_bo);
504 }
505
506 __bo_insert_to_free_rbtree(&bdev->free_rbtree, bo);
507
508 mutex_unlock(&bdev->rbtree_mutex);
509 return;
510}
511
512void hmm_bo_device_exit(struct hmm_bo_device *bdev)
513{
514 struct hmm_buffer_object *bo;
515 unsigned long flags;
516
517 dev_dbg(atomisp_dev, "%s: entering!\n", __func__);
518
519 check_bodev_null_return_void(bdev);
520
521
522
523
524
525 while (!RB_EMPTY_ROOT(&bdev->allocated_rbtree))
526 hmm_bo_release(
527 rbtree_node_to_hmm_bo(bdev->allocated_rbtree.rb_node));
528
529 dev_dbg(atomisp_dev, "%s: finished releasing all allocated bos!\n",
530 __func__);
531
532
533 while (!list_empty(&bdev->entire_bo_list)) {
534 bo = list_to_hmm_bo(bdev->entire_bo_list.next);
535
536 spin_lock_irqsave(&bdev->list_lock, flags);
537 list_del(&bo->list);
538 spin_unlock_irqrestore(&bdev->list_lock, flags);
539
540 kmem_cache_free(bdev->bo_cache, bo);
541 }
542
543 dev_dbg(atomisp_dev, "%s: finished to free all bos!\n", __func__);
544
545 kmem_cache_destroy(bdev->bo_cache);
546
547 isp_mmu_exit(&bdev->mmu);
548}
549
550int hmm_bo_device_inited(struct hmm_bo_device *bdev)
551{
552 check_bodev_null_return(bdev, -EINVAL);
553
554 return bdev->flag == HMM_BO_DEVICE_INITED;
555}
556
557int hmm_bo_allocated(struct hmm_buffer_object *bo)
558{
559 check_bo_null_return(bo, 0);
560
561 return bo->status & HMM_BO_ALLOCED;
562}
563
564struct hmm_buffer_object *hmm_bo_device_search_start(
565 struct hmm_bo_device *bdev, ia_css_ptr vaddr)
566{
567 struct hmm_buffer_object *bo;
568
569 check_bodev_null_return(bdev, NULL);
570
571 mutex_lock(&bdev->rbtree_mutex);
572 bo = __bo_search_by_addr(&bdev->allocated_rbtree, vaddr);
573 if (!bo) {
574 mutex_unlock(&bdev->rbtree_mutex);
575 dev_err(atomisp_dev, "%s can not find bo with addr: 0x%x\n",
576 __func__, vaddr);
577 return NULL;
578 }
579 mutex_unlock(&bdev->rbtree_mutex);
580
581 return bo;
582}
583
584struct hmm_buffer_object *hmm_bo_device_search_in_range(
585 struct hmm_bo_device *bdev, unsigned int vaddr)
586{
587 struct hmm_buffer_object *bo;
588
589 check_bodev_null_return(bdev, NULL);
590
591 mutex_lock(&bdev->rbtree_mutex);
592 bo = __bo_search_by_addr_in_range(&bdev->allocated_rbtree, vaddr);
593 if (!bo) {
594 mutex_unlock(&bdev->rbtree_mutex);
595 dev_err(atomisp_dev, "%s can not find bo contain addr: 0x%x\n",
596 __func__, vaddr);
597 return NULL;
598 }
599 mutex_unlock(&bdev->rbtree_mutex);
600
601 return bo;
602}
603
604struct hmm_buffer_object *hmm_bo_device_search_vmap_start(
605 struct hmm_bo_device *bdev, const void *vaddr)
606{
607 struct list_head *pos;
608 struct hmm_buffer_object *bo;
609 unsigned long flags;
610
611 check_bodev_null_return(bdev, NULL);
612
613 spin_lock_irqsave(&bdev->list_lock, flags);
614 list_for_each(pos, &bdev->entire_bo_list) {
615 bo = list_to_hmm_bo(pos);
616
617 if ((bo->status & HMM_BO_MASK) == HMM_BO_FREE)
618 continue;
619 if (bo->vmap_addr == vaddr)
620 goto found;
621 }
622 spin_unlock_irqrestore(&bdev->list_lock, flags);
623 return NULL;
624found:
625 spin_unlock_irqrestore(&bdev->list_lock, flags);
626 return bo;
627}
628
629static void free_private_bo_pages(struct hmm_buffer_object *bo,
630 struct hmm_pool *dypool,
631 struct hmm_pool *repool,
632 int free_pgnr)
633{
634 int i, ret;
635
636 for (i = 0; i < free_pgnr; i++) {
637 switch (bo->page_obj[i].type) {
638 case HMM_PAGE_TYPE_RESERVED:
639 if (repool->pops
640 && repool->pops->pool_free_pages) {
641 repool->pops->pool_free_pages(repool->pool_info,
642 &bo->page_obj[i]);
643 hmm_mem_stat.res_cnt--;
644 }
645 break;
646
647
648
649
650
651 case HMM_PAGE_TYPE_DYNAMIC:
652 case HMM_PAGE_TYPE_GENERAL:
653 if (dypool->pops
654 && dypool->pops->pool_inited
655 && dypool->pops->pool_inited(dypool->pool_info)) {
656 if (dypool->pops->pool_free_pages)
657 dypool->pops->pool_free_pages(
658 dypool->pool_info,
659 &bo->page_obj[i]);
660 break;
661 }
662
663 fallthrough;
664
665
666
667
668
669 default:
670 ret = set_pages_wb(bo->page_obj[i].page, 1);
671 if (ret)
672 dev_err(atomisp_dev,
673 "set page to WB err ...ret = %d\n",
674 ret);
675
676
677
678
679
680
681
682 if (!ret) {
683 __free_pages(bo->page_obj[i].page, 0);
684 hmm_mem_stat.sys_size--;
685 }
686 break;
687 }
688 }
689
690 return;
691}
692
693
694static int alloc_private_pages(struct hmm_buffer_object *bo,
695 int from_highmem,
696 bool cached,
697 struct hmm_pool *dypool,
698 struct hmm_pool *repool)
699{
700 int ret;
701 unsigned int pgnr, order, blk_pgnr, alloc_pgnr;
702 struct page *pages;
703 gfp_t gfp = GFP_NOWAIT | __GFP_NOWARN;
704 int i, j;
705 int failure_number = 0;
706 bool reduce_order = false;
707 bool lack_mem = true;
708
709 if (from_highmem)
710 gfp |= __GFP_HIGHMEM;
711
712 pgnr = bo->pgnr;
713
714 bo->page_obj = kmalloc_array(pgnr, sizeof(struct hmm_page_object),
715 GFP_KERNEL);
716 if (unlikely(!bo->page_obj))
717 return -ENOMEM;
718
719 i = 0;
720 alloc_pgnr = 0;
721
722
723
724
725 if (dypool->pops && dypool->pops->pool_alloc_pages) {
726 alloc_pgnr = dypool->pops->pool_alloc_pages(dypool->pool_info,
727 bo->page_obj, pgnr,
728 cached);
729 hmm_mem_stat.dyc_size -= alloc_pgnr;
730
731 if (alloc_pgnr == pgnr)
732 return 0;
733 }
734
735 pgnr -= alloc_pgnr;
736 i += alloc_pgnr;
737
738
739
740
741 if (repool->pops && repool->pops->pool_alloc_pages) {
742 alloc_pgnr = repool->pops->pool_alloc_pages(repool->pool_info,
743 &bo->page_obj[i], pgnr,
744 cached);
745 hmm_mem_stat.res_cnt += alloc_pgnr;
746 if (alloc_pgnr == pgnr)
747 return 0;
748 }
749
750 pgnr -= alloc_pgnr;
751 i += alloc_pgnr;
752
753 while (pgnr) {
754 order = nr_to_order_bottom(pgnr);
755
756
757
758
759 if (lack_mem)
760 order = HMM_MIN_ORDER;
761 else if (order > HMM_MAX_ORDER)
762 order = HMM_MAX_ORDER;
763retry:
764
765
766
767
768
769
770
771
772
773 if (order == HMM_MIN_ORDER) {
774 gfp &= ~GFP_NOWAIT;
775 gfp |= __GFP_RECLAIM | __GFP_FS;
776 }
777
778 pages = alloc_pages(gfp, order);
779 if (unlikely(!pages)) {
780
781
782
783
784
785
786 if (order == HMM_MIN_ORDER) {
787 dev_err(atomisp_dev,
788 "%s: cannot allocate pages\n",
789 __func__);
790 goto cleanup;
791 }
792 order = HMM_MIN_ORDER;
793 failure_number++;
794 reduce_order = true;
795
796
797
798
799 if (failure_number == 2) {
800 lack_mem = true;
801 failure_number = 0;
802 }
803 goto retry;
804 } else {
805 blk_pgnr = order_to_nr(order);
806
807 if (!cached) {
808
809
810
811 ret = set_pages_uc(pages, blk_pgnr);
812 if (ret) {
813 dev_err(atomisp_dev,
814 "set page uncacheablefailed.\n");
815
816 __free_pages(pages, order);
817
818 goto cleanup;
819 }
820 }
821
822 for (j = 0; j < blk_pgnr; j++) {
823 bo->page_obj[i].page = pages + j;
824 bo->page_obj[i++].type = HMM_PAGE_TYPE_GENERAL;
825 }
826
827 pgnr -= blk_pgnr;
828 hmm_mem_stat.sys_size += blk_pgnr;
829
830
831
832
833
834 if (reduce_order)
835 reduce_order = false;
836 else
837 failure_number = 0;
838 }
839 }
840
841 return 0;
842cleanup:
843 alloc_pgnr = i;
844 free_private_bo_pages(bo, dypool, repool, alloc_pgnr);
845
846 kfree(bo->page_obj);
847
848 return -ENOMEM;
849}
850
851static void free_private_pages(struct hmm_buffer_object *bo,
852 struct hmm_pool *dypool,
853 struct hmm_pool *repool)
854{
855 free_private_bo_pages(bo, dypool, repool, bo->pgnr);
856
857 kfree(bo->page_obj);
858}
859
860static void free_user_pages(struct hmm_buffer_object *bo)
861{
862 int i;
863
864 hmm_mem_stat.usr_size -= bo->pgnr;
865
866 if (bo->mem_type == HMM_BO_MEM_TYPE_PFN) {
867 unpin_user_pages(bo->pages, bo->pgnr);
868 } else {
869 for (i = 0; i < bo->pgnr; i++)
870 put_page(bo->pages[i]);
871 }
872 kfree(bo->pages);
873 kfree(bo->page_obj);
874}
875
876
877
878
879static int alloc_user_pages(struct hmm_buffer_object *bo,
880 const void __user *userptr, bool cached)
881{
882 int page_nr;
883 int i;
884 struct vm_area_struct *vma;
885 struct page **pages;
886
887 pages = kmalloc_array(bo->pgnr, sizeof(struct page *), GFP_KERNEL);
888 if (unlikely(!pages))
889 return -ENOMEM;
890
891 bo->page_obj = kmalloc_array(bo->pgnr, sizeof(struct hmm_page_object),
892 GFP_KERNEL);
893 if (unlikely(!bo->page_obj)) {
894 kfree(pages);
895 return -ENOMEM;
896 }
897
898 mutex_unlock(&bo->mutex);
899 mmap_read_lock(current->mm);
900 vma = find_vma(current->mm, (unsigned long)userptr);
901 mmap_read_unlock(current->mm);
902 if (!vma) {
903 dev_err(atomisp_dev, "find_vma failed\n");
904 kfree(bo->page_obj);
905 kfree(pages);
906 mutex_lock(&bo->mutex);
907 return -EFAULT;
908 }
909 mutex_lock(&bo->mutex);
910
911
912
913
914
915 userptr = untagged_addr(userptr);
916
917 bo->pages = pages;
918
919 if (vma->vm_flags & (VM_IO | VM_PFNMAP)) {
920 page_nr = pin_user_pages((unsigned long)userptr, bo->pgnr,
921 FOLL_LONGTERM | FOLL_WRITE,
922 pages, NULL);
923 bo->mem_type = HMM_BO_MEM_TYPE_PFN;
924 } else {
925
926 mutex_unlock(&bo->mutex);
927 page_nr = get_user_pages_fast((unsigned long)userptr,
928 (int)(bo->pgnr), 1, pages);
929 mutex_lock(&bo->mutex);
930 bo->mem_type = HMM_BO_MEM_TYPE_USER;
931 }
932
933 dev_dbg(atomisp_dev, "%s: %d %s pages were allocated as 0x%08x\n",
934 __func__,
935 bo->pgnr,
936 bo->mem_type == HMM_BO_MEM_TYPE_USER ? "user" : "pfn", page_nr);
937
938 hmm_mem_stat.usr_size += bo->pgnr;
939
940
941 if (page_nr != bo->pgnr) {
942 dev_err(atomisp_dev,
943 "get_user_pages err: bo->pgnr = %d, pgnr actually pinned = %d.\n",
944 bo->pgnr, page_nr);
945 goto out_of_mem;
946 }
947
948 for (i = 0; i < bo->pgnr; i++) {
949 bo->page_obj[i].page = pages[i];
950 bo->page_obj[i].type = HMM_PAGE_TYPE_GENERAL;
951 }
952
953 return 0;
954
955out_of_mem:
956
957 free_user_pages(bo);
958
959 return -ENOMEM;
960}
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977int hmm_bo_alloc_pages(struct hmm_buffer_object *bo,
978 enum hmm_bo_type type, int from_highmem,
979 const void __user *userptr, bool cached)
980{
981 int ret = -EINVAL;
982
983 check_bo_null_return(bo, -EINVAL);
984
985 mutex_lock(&bo->mutex);
986 check_bo_status_no_goto(bo, HMM_BO_PAGE_ALLOCED, status_err);
987
988
989
990
991
992 if (type == HMM_BO_PRIVATE) {
993 ret = alloc_private_pages(bo, from_highmem,
994 cached, &dynamic_pool, &reserved_pool);
995 } else if (type == HMM_BO_USER) {
996 ret = alloc_user_pages(bo, userptr, cached);
997 } else {
998 dev_err(atomisp_dev, "invalid buffer type.\n");
999 ret = -EINVAL;
1000 }
1001 if (ret)
1002 goto alloc_err;
1003
1004 bo->type = type;
1005
1006 bo->status |= HMM_BO_PAGE_ALLOCED;
1007
1008 mutex_unlock(&bo->mutex);
1009
1010 return 0;
1011
1012alloc_err:
1013 mutex_unlock(&bo->mutex);
1014 dev_err(atomisp_dev, "alloc pages err...\n");
1015 return ret;
1016status_err:
1017 mutex_unlock(&bo->mutex);
1018 dev_err(atomisp_dev,
1019 "buffer object has already page allocated.\n");
1020 return -EINVAL;
1021}
1022
1023
1024
1025
1026void hmm_bo_free_pages(struct hmm_buffer_object *bo)
1027{
1028 check_bo_null_return_void(bo);
1029
1030 mutex_lock(&bo->mutex);
1031
1032 check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err2);
1033
1034
1035 bo->status &= (~HMM_BO_PAGE_ALLOCED);
1036
1037 if (bo->type == HMM_BO_PRIVATE)
1038 free_private_pages(bo, &dynamic_pool, &reserved_pool);
1039 else if (bo->type == HMM_BO_USER)
1040 free_user_pages(bo);
1041 else
1042 dev_err(atomisp_dev, "invalid buffer type.\n");
1043 mutex_unlock(&bo->mutex);
1044
1045 return;
1046
1047status_err2:
1048 mutex_unlock(&bo->mutex);
1049 dev_err(atomisp_dev,
1050 "buffer object not page allocated yet.\n");
1051}
1052
1053int hmm_bo_page_allocated(struct hmm_buffer_object *bo)
1054{
1055 check_bo_null_return(bo, 0);
1056
1057 return bo->status & HMM_BO_PAGE_ALLOCED;
1058}
1059
1060
1061
1062
1063int hmm_bo_get_page_info(struct hmm_buffer_object *bo,
1064 struct hmm_page_object **page_obj, int *pgnr)
1065{
1066 check_bo_null_return(bo, -EINVAL);
1067
1068 mutex_lock(&bo->mutex);
1069
1070 check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err);
1071
1072 *page_obj = bo->page_obj;
1073 *pgnr = bo->pgnr;
1074
1075 mutex_unlock(&bo->mutex);
1076
1077 return 0;
1078
1079status_err:
1080 dev_err(atomisp_dev,
1081 "buffer object not page allocated yet.\n");
1082 mutex_unlock(&bo->mutex);
1083 return -EINVAL;
1084}
1085
1086
1087
1088
1089int hmm_bo_bind(struct hmm_buffer_object *bo)
1090{
1091 int ret;
1092 unsigned int virt;
1093 struct hmm_bo_device *bdev;
1094 unsigned int i;
1095
1096 check_bo_null_return(bo, -EINVAL);
1097
1098 mutex_lock(&bo->mutex);
1099
1100 check_bo_status_yes_goto(bo,
1101 HMM_BO_PAGE_ALLOCED | HMM_BO_ALLOCED,
1102 status_err1);
1103
1104 check_bo_status_no_goto(bo, HMM_BO_BINDED, status_err2);
1105
1106 bdev = bo->bdev;
1107
1108 virt = bo->start;
1109
1110 for (i = 0; i < bo->pgnr; i++) {
1111 ret =
1112 isp_mmu_map(&bdev->mmu, virt,
1113 page_to_phys(bo->page_obj[i].page), 1);
1114 if (ret)
1115 goto map_err;
1116 virt += (1 << PAGE_SHIFT);
1117 }
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130 if (bo->start != 0x0)
1131 isp_mmu_flush_tlb_range(&bdev->mmu, bo->start,
1132 (bo->pgnr << PAGE_SHIFT));
1133
1134 bo->status |= HMM_BO_BINDED;
1135
1136 mutex_unlock(&bo->mutex);
1137
1138 return 0;
1139
1140map_err:
1141
1142 virt = bo->start;
1143 for ( ; i > 0; i--) {
1144 isp_mmu_unmap(&bdev->mmu, virt, 1);
1145 virt += pgnr_to_size(1);
1146 }
1147
1148 mutex_unlock(&bo->mutex);
1149 dev_err(atomisp_dev,
1150 "setup MMU address mapping failed.\n");
1151 return ret;
1152
1153status_err2:
1154 mutex_unlock(&bo->mutex);
1155 dev_err(atomisp_dev, "buffer object already binded.\n");
1156 return -EINVAL;
1157status_err1:
1158 mutex_unlock(&bo->mutex);
1159 dev_err(atomisp_dev,
1160 "buffer object vm_node or page not allocated.\n");
1161 return -EINVAL;
1162}
1163
1164
1165
1166
1167void hmm_bo_unbind(struct hmm_buffer_object *bo)
1168{
1169 unsigned int virt;
1170 struct hmm_bo_device *bdev;
1171 unsigned int i;
1172
1173 check_bo_null_return_void(bo);
1174
1175 mutex_lock(&bo->mutex);
1176
1177 check_bo_status_yes_goto(bo,
1178 HMM_BO_PAGE_ALLOCED |
1179 HMM_BO_ALLOCED |
1180 HMM_BO_BINDED, status_err);
1181
1182 bdev = bo->bdev;
1183
1184 virt = bo->start;
1185
1186 for (i = 0; i < bo->pgnr; i++) {
1187 isp_mmu_unmap(&bdev->mmu, virt, 1);
1188 virt += pgnr_to_size(1);
1189 }
1190
1191
1192
1193
1194
1195 isp_mmu_flush_tlb_range(&bdev->mmu, bo->start,
1196 (bo->pgnr << PAGE_SHIFT));
1197
1198 bo->status &= (~HMM_BO_BINDED);
1199
1200 mutex_unlock(&bo->mutex);
1201
1202 return;
1203
1204status_err:
1205 mutex_unlock(&bo->mutex);
1206 dev_err(atomisp_dev,
1207 "buffer vm or page not allocated or not binded yet.\n");
1208}
1209
1210int hmm_bo_binded(struct hmm_buffer_object *bo)
1211{
1212 int ret;
1213
1214 check_bo_null_return(bo, 0);
1215
1216 mutex_lock(&bo->mutex);
1217
1218 ret = bo->status & HMM_BO_BINDED;
1219
1220 mutex_unlock(&bo->mutex);
1221
1222 return ret;
1223}
1224
1225void *hmm_bo_vmap(struct hmm_buffer_object *bo, bool cached)
1226{
1227 struct page **pages;
1228 int i;
1229
1230 check_bo_null_return(bo, NULL);
1231
1232 mutex_lock(&bo->mutex);
1233 if (((bo->status & HMM_BO_VMAPED) && !cached) ||
1234 ((bo->status & HMM_BO_VMAPED_CACHED) && cached)) {
1235 mutex_unlock(&bo->mutex);
1236 return bo->vmap_addr;
1237 }
1238
1239
1240 if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
1241 vunmap(bo->vmap_addr);
1242 bo->vmap_addr = NULL;
1243 bo->status &= ~(HMM_BO_VMAPED | HMM_BO_VMAPED_CACHED);
1244 }
1245
1246 pages = kmalloc_array(bo->pgnr, sizeof(*pages), GFP_KERNEL);
1247 if (unlikely(!pages)) {
1248 mutex_unlock(&bo->mutex);
1249 return NULL;
1250 }
1251
1252 for (i = 0; i < bo->pgnr; i++)
1253 pages[i] = bo->page_obj[i].page;
1254
1255 bo->vmap_addr = vmap(pages, bo->pgnr, VM_MAP,
1256 cached ? PAGE_KERNEL : PAGE_KERNEL_NOCACHE);
1257 if (unlikely(!bo->vmap_addr)) {
1258 kfree(pages);
1259 mutex_unlock(&bo->mutex);
1260 dev_err(atomisp_dev, "vmap failed...\n");
1261 return NULL;
1262 }
1263 bo->status |= (cached ? HMM_BO_VMAPED_CACHED : HMM_BO_VMAPED);
1264
1265 kfree(pages);
1266
1267 mutex_unlock(&bo->mutex);
1268 return bo->vmap_addr;
1269}
1270
1271void hmm_bo_flush_vmap(struct hmm_buffer_object *bo)
1272{
1273 check_bo_null_return_void(bo);
1274
1275 mutex_lock(&bo->mutex);
1276 if (!(bo->status & HMM_BO_VMAPED_CACHED) || !bo->vmap_addr) {
1277 mutex_unlock(&bo->mutex);
1278 return;
1279 }
1280
1281 clflush_cache_range(bo->vmap_addr, bo->pgnr * PAGE_SIZE);
1282 mutex_unlock(&bo->mutex);
1283}
1284
1285void hmm_bo_vunmap(struct hmm_buffer_object *bo)
1286{
1287 check_bo_null_return_void(bo);
1288
1289 mutex_lock(&bo->mutex);
1290 if (bo->status & HMM_BO_VMAPED || bo->status & HMM_BO_VMAPED_CACHED) {
1291 vunmap(bo->vmap_addr);
1292 bo->vmap_addr = NULL;
1293 bo->status &= ~(HMM_BO_VMAPED | HMM_BO_VMAPED_CACHED);
1294 }
1295
1296 mutex_unlock(&bo->mutex);
1297 return;
1298}
1299
1300void hmm_bo_ref(struct hmm_buffer_object *bo)
1301{
1302 check_bo_null_return_void(bo);
1303
1304 kref_get(&bo->kref);
1305}
1306
1307static void kref_hmm_bo_release(struct kref *kref)
1308{
1309 if (!kref)
1310 return;
1311
1312 hmm_bo_release(kref_to_hmm_bo(kref));
1313}
1314
1315void hmm_bo_unref(struct hmm_buffer_object *bo)
1316{
1317 check_bo_null_return_void(bo);
1318
1319 kref_put(&bo->kref, kref_hmm_bo_release);
1320}
1321
1322static void hmm_bo_vm_open(struct vm_area_struct *vma)
1323{
1324 struct hmm_buffer_object *bo =
1325 (struct hmm_buffer_object *)vma->vm_private_data;
1326
1327 check_bo_null_return_void(bo);
1328
1329 hmm_bo_ref(bo);
1330
1331 mutex_lock(&bo->mutex);
1332
1333 bo->status |= HMM_BO_MMAPED;
1334
1335 bo->mmap_count++;
1336
1337 mutex_unlock(&bo->mutex);
1338}
1339
1340static void hmm_bo_vm_close(struct vm_area_struct *vma)
1341{
1342 struct hmm_buffer_object *bo =
1343 (struct hmm_buffer_object *)vma->vm_private_data;
1344
1345 check_bo_null_return_void(bo);
1346
1347 hmm_bo_unref(bo);
1348
1349 mutex_lock(&bo->mutex);
1350
1351 bo->mmap_count--;
1352
1353 if (!bo->mmap_count) {
1354 bo->status &= (~HMM_BO_MMAPED);
1355 vma->vm_private_data = NULL;
1356 }
1357
1358 mutex_unlock(&bo->mutex);
1359}
1360
1361static const struct vm_operations_struct hmm_bo_vm_ops = {
1362 .open = hmm_bo_vm_open,
1363 .close = hmm_bo_vm_close,
1364};
1365
1366
1367
1368
1369int hmm_bo_mmap(struct vm_area_struct *vma, struct hmm_buffer_object *bo)
1370{
1371 unsigned int start, end;
1372 unsigned int virt;
1373 unsigned int pgnr, i;
1374 unsigned int pfn;
1375
1376 check_bo_null_return(bo, -EINVAL);
1377
1378 check_bo_status_yes_goto(bo, HMM_BO_PAGE_ALLOCED, status_err);
1379
1380 pgnr = bo->pgnr;
1381 start = vma->vm_start;
1382 end = vma->vm_end;
1383
1384
1385
1386
1387
1388 if ((start + pgnr_to_size(pgnr)) != end) {
1389 dev_warn(atomisp_dev,
1390 "vma's address space size not equal to buffer object's size");
1391 return -EINVAL;
1392 }
1393
1394 virt = vma->vm_start;
1395 for (i = 0; i < pgnr; i++) {
1396 pfn = page_to_pfn(bo->page_obj[i].page);
1397 if (remap_pfn_range(vma, virt, pfn, PAGE_SIZE, PAGE_SHARED)) {
1398 dev_warn(atomisp_dev,
1399 "remap_pfn_range failed: virt = 0x%x, pfn = 0x%x, mapped_pgnr = %d\n",
1400 virt, pfn, 1);
1401 return -EINVAL;
1402 }
1403 virt += PAGE_SIZE;
1404 }
1405
1406 vma->vm_private_data = bo;
1407
1408 vma->vm_ops = &hmm_bo_vm_ops;
1409 vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP;
1410
1411
1412
1413
1414 hmm_bo_vm_open(vma);
1415
1416 return 0;
1417
1418status_err:
1419 dev_err(atomisp_dev, "buffer page not allocated yet.\n");
1420 return -EINVAL;
1421}
1422