1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17#include <linux/debugfs.h>
18#include <linux/dma-mapping.h>
19#include <linux/err.h>
20#include <linux/fs.h>
21#include <linux/list.h>
22#include <linux/init.h>
23#include <linux/slab.h>
24#include <linux/swap.h>
25
26#include "ion.h"
27
28static void *ion_page_pool_alloc_pages(struct ion_page_pool *pool)
29{
30 struct page *page = alloc_pages(pool->gfp_mask, pool->order);
31
32 if (!page)
33 return NULL;
34 return page;
35}
36
37static void ion_page_pool_free_pages(struct ion_page_pool *pool,
38 struct page *page)
39{
40 __free_pages(page, pool->order);
41}
42
43static int ion_page_pool_add(struct ion_page_pool *pool, struct page *page)
44{
45 mutex_lock(&pool->mutex);
46 if (PageHighMem(page)) {
47 list_add_tail(&page->lru, &pool->high_items);
48 pool->high_count++;
49 } else {
50 list_add_tail(&page->lru, &pool->low_items);
51 pool->low_count++;
52 }
53 mutex_unlock(&pool->mutex);
54 return 0;
55}
56
57static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high)
58{
59 struct page *page;
60
61 if (high) {
62 BUG_ON(!pool->high_count);
63 page = list_first_entry(&pool->high_items, struct page, lru);
64 pool->high_count--;
65 } else {
66 BUG_ON(!pool->low_count);
67 page = list_first_entry(&pool->low_items, struct page, lru);
68 pool->low_count--;
69 }
70
71 list_del(&page->lru);
72 return page;
73}
74
75struct page *ion_page_pool_alloc(struct ion_page_pool *pool)
76{
77 struct page *page = NULL;
78
79 BUG_ON(!pool);
80
81 mutex_lock(&pool->mutex);
82 if (pool->high_count)
83 page = ion_page_pool_remove(pool, true);
84 else if (pool->low_count)
85 page = ion_page_pool_remove(pool, false);
86 mutex_unlock(&pool->mutex);
87
88 if (!page)
89 page = ion_page_pool_alloc_pages(pool);
90
91 return page;
92}
93
94void ion_page_pool_free(struct ion_page_pool *pool, struct page *page)
95{
96 int ret;
97
98 BUG_ON(pool->order != compound_order(page));
99
100 ret = ion_page_pool_add(pool, page);
101 if (ret)
102 ion_page_pool_free_pages(pool, page);
103}
104
105static int ion_page_pool_total(struct ion_page_pool *pool, bool high)
106{
107 int count = pool->low_count;
108
109 if (high)
110 count += pool->high_count;
111
112 return count << pool->order;
113}
114
115int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,
116 int nr_to_scan)
117{
118 int freed = 0;
119 bool high;
120
121 if (current_is_kswapd())
122 high = true;
123 else
124 high = !!(gfp_mask & __GFP_HIGHMEM);
125
126 if (nr_to_scan == 0)
127 return ion_page_pool_total(pool, high);
128
129 while (freed < nr_to_scan) {
130 struct page *page;
131
132 mutex_lock(&pool->mutex);
133 if (pool->low_count) {
134 page = ion_page_pool_remove(pool, false);
135 } else if (high && pool->high_count) {
136 page = ion_page_pool_remove(pool, true);
137 } else {
138 mutex_unlock(&pool->mutex);
139 break;
140 }
141 mutex_unlock(&pool->mutex);
142 ion_page_pool_free_pages(pool, page);
143 freed += (1 << pool->order);
144 }
145
146 return freed;
147}
148
149struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order,
150 bool cached)
151{
152 struct ion_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL);
153
154 if (!pool)
155 return NULL;
156 pool->high_count = 0;
157 pool->low_count = 0;
158 INIT_LIST_HEAD(&pool->low_items);
159 INIT_LIST_HEAD(&pool->high_items);
160 pool->gfp_mask = gfp_mask | __GFP_COMP;
161 pool->order = order;
162 mutex_init(&pool->mutex);
163 plist_node_init(&pool->list, order);
164 if (cached)
165 pool->cached = true;
166
167 return pool;
168}
169
170void ion_page_pool_destroy(struct ion_page_pool *pool)
171{
172 kfree(pool);
173}
174
175static int __init ion_page_pool_init(void)
176{
177 return 0;
178}
179device_initcall(ion_page_pool_init);
180