1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30#ifndef _NET_PAGE_POOL_H
31#define _NET_PAGE_POOL_H
32
33#include <linux/mm.h>
34#include <linux/ptr_ring.h>
35#include <linux/dma-direction.h>
36
37#define PP_FLAG_DMA_MAP BIT(0)
38
39
40#define PP_FLAG_DMA_SYNC_DEV BIT(1)
41
42
43
44
45
46
47
48#define PP_FLAG_PAGE_FRAG BIT(2)
49#define PP_FLAG_ALL (PP_FLAG_DMA_MAP |\
50 PP_FLAG_DMA_SYNC_DEV |\
51 PP_FLAG_PAGE_FRAG)
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67#define PP_ALLOC_CACHE_SIZE 128
68#define PP_ALLOC_CACHE_REFILL 64
69struct pp_alloc_cache {
70 u32 count;
71 struct page *cache[PP_ALLOC_CACHE_SIZE];
72};
73
74struct page_pool_params {
75 unsigned int flags;
76 unsigned int order;
77 unsigned int pool_size;
78 int nid;
79 struct device *dev;
80 enum dma_data_direction dma_dir;
81 unsigned int max_len;
82 unsigned int offset;
83};
84
85struct page_pool {
86 struct page_pool_params p;
87
88 struct delayed_work release_dw;
89 void (*disconnect)(void *);
90 unsigned long defer_start;
91 unsigned long defer_warn;
92
93 u32 pages_state_hold_cnt;
94 unsigned int frag_offset;
95 struct page *frag_page;
96 long frag_users;
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111 struct pp_alloc_cache alloc ____cacheline_aligned_in_smp;
112
113
114
115
116
117
118
119
120
121
122
123
124 struct ptr_ring ring;
125
126 atomic_t pages_state_release_cnt;
127
128
129
130
131
132 refcount_t user_cnt;
133
134 u64 destroy_cnt;
135};
136
137struct page *page_pool_alloc_pages(struct page_pool *pool, gfp_t gfp);
138
139static inline struct page *page_pool_dev_alloc_pages(struct page_pool *pool)
140{
141 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
142
143 return page_pool_alloc_pages(pool, gfp);
144}
145
146struct page *page_pool_alloc_frag(struct page_pool *pool, unsigned int *offset,
147 unsigned int size, gfp_t gfp);
148
149static inline struct page *page_pool_dev_alloc_frag(struct page_pool *pool,
150 unsigned int *offset,
151 unsigned int size)
152{
153 gfp_t gfp = (GFP_ATOMIC | __GFP_NOWARN);
154
155 return page_pool_alloc_frag(pool, offset, size, gfp);
156}
157
158
159
160
161static
162inline enum dma_data_direction page_pool_get_dma_dir(struct page_pool *pool)
163{
164 return pool->p.dma_dir;
165}
166
167bool page_pool_return_skb_page(struct page *page);
168
169struct page_pool *page_pool_create(const struct page_pool_params *params);
170
171#ifdef CONFIG_PAGE_POOL
172void page_pool_destroy(struct page_pool *pool);
173void page_pool_use_xdp_mem(struct page_pool *pool, void (*disconnect)(void *));
174void page_pool_release_page(struct page_pool *pool, struct page *page);
175void page_pool_put_page_bulk(struct page_pool *pool, void **data,
176 int count);
177#else
178static inline void page_pool_destroy(struct page_pool *pool)
179{
180}
181
182static inline void page_pool_use_xdp_mem(struct page_pool *pool,
183 void (*disconnect)(void *))
184{
185}
186static inline void page_pool_release_page(struct page_pool *pool,
187 struct page *page)
188{
189}
190
191static inline void page_pool_put_page_bulk(struct page_pool *pool, void **data,
192 int count)
193{
194}
195#endif
196
197void page_pool_put_page(struct page_pool *pool, struct page *page,
198 unsigned int dma_sync_size, bool allow_direct);
199
200
201static inline void page_pool_put_full_page(struct page_pool *pool,
202 struct page *page, bool allow_direct)
203{
204
205
206
207#ifdef CONFIG_PAGE_POOL
208 page_pool_put_page(pool, page, -1, allow_direct);
209#endif
210}
211
212
213static inline void page_pool_recycle_direct(struct page_pool *pool,
214 struct page *page)
215{
216 page_pool_put_full_page(pool, page, true);
217}
218
219#define PAGE_POOL_DMA_USE_PP_FRAG_COUNT \
220 (sizeof(dma_addr_t) > sizeof(unsigned long))
221
222static inline dma_addr_t page_pool_get_dma_addr(struct page *page)
223{
224 dma_addr_t ret = page->dma_addr;
225
226 if (PAGE_POOL_DMA_USE_PP_FRAG_COUNT)
227 ret |= (dma_addr_t)page->dma_addr_upper << 16 << 16;
228
229 return ret;
230}
231
232static inline void page_pool_set_dma_addr(struct page *page, dma_addr_t addr)
233{
234 page->dma_addr = addr;
235 if (PAGE_POOL_DMA_USE_PP_FRAG_COUNT)
236 page->dma_addr_upper = upper_32_bits(addr);
237}
238
239static inline void page_pool_set_frag_count(struct page *page, long nr)
240{
241 atomic_long_set(&page->pp_frag_count, nr);
242}
243
244static inline long page_pool_atomic_sub_frag_count_return(struct page *page,
245 long nr)
246{
247 long ret;
248
249
250
251
252
253
254 if (__builtin_constant_p(nr) &&
255 atomic_long_read(&page->pp_frag_count) == nr)
256 return 0;
257
258 ret = atomic_long_sub_return(nr, &page->pp_frag_count);
259 WARN_ON(ret < 0);
260 return ret;
261}
262
263static inline bool is_page_pool_compiled_in(void)
264{
265#ifdef CONFIG_PAGE_POOL
266 return true;
267#else
268 return false;
269#endif
270}
271
272static inline bool page_pool_put(struct page_pool *pool)
273{
274 return refcount_dec_and_test(&pool->user_cnt);
275}
276
277
278void page_pool_update_nid(struct page_pool *pool, int new_nid);
279static inline void page_pool_nid_changed(struct page_pool *pool, int new_nid)
280{
281 if (unlikely(pool->p.nid != new_nid))
282 page_pool_update_nid(pool, new_nid);
283}
284
285static inline void page_pool_ring_lock(struct page_pool *pool)
286 __acquires(&pool->ring.producer_lock)
287{
288 if (in_serving_softirq())
289 spin_lock(&pool->ring.producer_lock);
290 else
291 spin_lock_bh(&pool->ring.producer_lock);
292}
293
294static inline void page_pool_ring_unlock(struct page_pool *pool)
295 __releases(&pool->ring.producer_lock)
296{
297 if (in_serving_softirq())
298 spin_unlock(&pool->ring.producer_lock);
299 else
300 spin_unlock_bh(&pool->ring.producer_lock);
301}
302
303#endif
304