1
2
3
4
5
6
7#include <linux/types.h>
8#include <linux/kernel.h>
9#include <linux/slab.h>
10
11#include <net/page_pool.h>
12#include <linux/dma-direction.h>
13#include <linux/dma-mapping.h>
14#include <linux/page-flags.h>
15#include <linux/mm.h>
16
17static int page_pool_init(struct page_pool *pool,
18 const struct page_pool_params *params)
19{
20 unsigned int ring_qsize = 1024;
21
22 memcpy(&pool->p, params, sizeof(pool->p));
23
24
25 if (pool->p.flags & ~(PP_FLAG_ALL))
26 return -EINVAL;
27
28 if (pool->p.pool_size)
29 ring_qsize = pool->p.pool_size;
30
31
32 if (ring_qsize > 32768)
33 return -E2BIG;
34
35
36
37
38
39 if ((pool->p.dma_dir != DMA_FROM_DEVICE) &&
40 (pool->p.dma_dir != DMA_BIDIRECTIONAL))
41 return -EINVAL;
42
43 if (ptr_ring_init(&pool->ring, ring_qsize, GFP_KERNEL) < 0)
44 return -ENOMEM;
45
46 return 0;
47}
48
49struct page_pool *page_pool_create(const struct page_pool_params *params)
50{
51 struct page_pool *pool;
52 int err = 0;
53
54 pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, params->nid);
55 if (!pool)
56 return ERR_PTR(-ENOMEM);
57
58 err = page_pool_init(pool, params);
59 if (err < 0) {
60 pr_warn("%s() gave up with errno %d\n", __func__, err);
61 kfree(pool);
62 return ERR_PTR(err);
63 }
64 return pool;
65}
66EXPORT_SYMBOL(page_pool_create);
67
68
69static struct page *__page_pool_get_cached(struct page_pool *pool)
70{
71 struct ptr_ring *r = &pool->ring;
72 struct page *page;
73
74
75 if (__ptr_ring_empty(r))
76 return NULL;
77
78
79 if (likely(in_serving_softirq())) {
80 if (likely(pool->alloc.count)) {
81
82 page = pool->alloc.cache[--pool->alloc.count];
83 return page;
84 }
85
86
87
88
89
90
91
92
93
94 spin_lock(&r->consumer_lock);
95 while ((page = __ptr_ring_consume(r))) {
96 if (pool->alloc.count == PP_ALLOC_CACHE_REFILL)
97 break;
98 pool->alloc.cache[pool->alloc.count++] = page;
99 }
100 spin_unlock(&r->consumer_lock);
101 return page;
102 }
103
104
105 page = ptr_ring_consume(&pool->ring);
106 return page;
107}
108
109
110noinline
111static struct page *__page_pool_alloc_pages_slow(struct page_pool *pool,
112 gfp_t _gfp)
113{
114 struct page *page;
115 gfp_t gfp = _gfp;
116 dma_addr_t dma;
117
118
119
120
121 if (pool->p.order)
122 gfp |= __GFP_COMP;
123
124
125
126
127
128
129
130
131
132 page = alloc_pages_node(pool->p.nid, gfp, pool->p.order);
133 if (!page)
134 return NULL;
135
136 if (!(pool->p.flags & PP_FLAG_DMA_MAP))
137 goto skip_dma_map;
138
139
140
141
142
143
144 dma = dma_map_page_attrs(pool->p.dev, page, 0,
145 (PAGE_SIZE << pool->p.order),
146 pool->p.dma_dir, DMA_ATTR_SKIP_CPU_SYNC);
147 if (dma_mapping_error(pool->p.dev, dma)) {
148 put_page(page);
149 return NULL;
150 }
151 page->dma_addr = dma;
152
153skip_dma_map:
154
155 return page;
156}
157
158
159
160
161struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp)
162{
163 struct page *page;
164
165
166 page = __page_pool_get_cached(pool);
167 if (page)
168 return page;
169
170
171 page = __page_pool_alloc_pages_slow(pool, gfp);
172 return page;
173}
174EXPORT_SYMBOL(page_pool_alloc_pages);
175
176
177static void __page_pool_clean_page(struct page_pool *pool,
178 struct page *page)
179{
180 dma_addr_t dma;
181
182 if (!(pool->p.flags & PP_FLAG_DMA_MAP))
183 return;
184
185 dma = page->dma_addr;
186
187 dma_unmap_page_attrs(pool->p.dev, dma,
188 PAGE_SIZE << pool->p.order, pool->p.dma_dir,
189 DMA_ATTR_SKIP_CPU_SYNC);
190 page->dma_addr = 0;
191}
192
193
194static void __page_pool_return_page(struct page_pool *pool, struct page *page)
195{
196 __page_pool_clean_page(pool, page);
197 put_page(page);
198
199
200
201
202}
203
204static bool __page_pool_recycle_into_ring(struct page_pool *pool,
205 struct page *page)
206{
207 int ret;
208
209 if (in_serving_softirq())
210 ret = ptr_ring_produce(&pool->ring, page);
211 else
212 ret = ptr_ring_produce_bh(&pool->ring, page);
213
214 return (ret == 0) ? true : false;
215}
216
217
218
219
220
221
222static bool __page_pool_recycle_direct(struct page *page,
223 struct page_pool *pool)
224{
225 if (unlikely(pool->alloc.count == PP_ALLOC_CACHE_SIZE))
226 return false;
227
228
229 pool->alloc.cache[pool->alloc.count++] = page;
230 return true;
231}
232
233void __page_pool_put_page(struct page_pool *pool,
234 struct page *page, bool allow_direct)
235{
236
237
238
239
240
241
242 if (likely(page_ref_count(page) == 1)) {
243
244
245 if (allow_direct && in_serving_softirq())
246 if (__page_pool_recycle_direct(page, pool))
247 return;
248
249 if (!__page_pool_recycle_into_ring(pool, page)) {
250
251 __page_pool_return_page(pool, page);
252 }
253 return;
254 }
255
256
257
258
259
260
261
262
263
264
265
266
267
268 __page_pool_clean_page(pool, page);
269 put_page(page);
270}
271EXPORT_SYMBOL(__page_pool_put_page);
272
273static void __page_pool_empty_ring(struct page_pool *pool)
274{
275 struct page *page;
276
277
278 while ((page = ptr_ring_consume_bh(&pool->ring))) {
279
280 if (!(page_ref_count(page) == 1))
281 pr_crit("%s() page_pool refcnt %d violation\n",
282 __func__, page_ref_count(page));
283
284 __page_pool_return_page(pool, page);
285 }
286}
287
288static void __page_pool_destroy_rcu(struct rcu_head *rcu)
289{
290 struct page_pool *pool;
291
292 pool = container_of(rcu, struct page_pool, rcu);
293
294 WARN(pool->alloc.count, "API usage violation");
295
296 __page_pool_empty_ring(pool);
297 ptr_ring_cleanup(&pool->ring, NULL);
298 kfree(pool);
299}
300
301
302void page_pool_destroy(struct page_pool *pool)
303{
304 struct page *page;
305
306
307
308
309
310 while (pool->alloc.count) {
311 page = pool->alloc.cache[--pool->alloc.count];
312 __page_pool_return_page(pool, page);
313 }
314
315
316
317
318 __page_pool_empty_ring(pool);
319
320
321 call_rcu(&pool->rcu, __page_pool_destroy_rcu);
322}
323EXPORT_SYMBOL(page_pool_destroy);
324