1
2
3
4
5
6
7
8
9
10
11
12
13#include <linux/mm.h>
14#include <linux/slab.h>
15#include <linux/highmem.h>
16#include <linux/kasan.h>
17#include <linux/kmemleak.h>
18#include <linux/export.h>
19#include <linux/mempool.h>
20#include <linux/writeback.h>
21#include "slab.h"
22
23#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB_DEBUG_ON)
24static void poison_error(mempool_t *pool, void *element, size_t size,
25 size_t byte)
26{
27 const int nr = pool->curr_nr;
28 const int start = max_t(int, byte - (BITS_PER_LONG / 8), 0);
29 const int end = min_t(int, byte + (BITS_PER_LONG / 8), size);
30 int i;
31
32 pr_err("BUG: mempool element poison mismatch\n");
33 pr_err("Mempool %p size %zu\n", pool, size);
34 pr_err(" nr=%d @ %p: %s0x", nr, element, start > 0 ? "... " : "");
35 for (i = start; i < end; i++)
36 pr_cont("%x ", *(u8 *)(element + i));
37 pr_cont("%s\n", end < size ? "..." : "");
38 dump_stack();
39}
40
41static void __check_element(mempool_t *pool, void *element, size_t size)
42{
43 u8 *obj = element;
44 size_t i;
45
46 for (i = 0; i < size; i++) {
47 u8 exp = (i < size - 1) ? POISON_FREE : POISON_END;
48
49 if (obj[i] != exp) {
50 poison_error(pool, element, size, i);
51 return;
52 }
53 }
54 memset(obj, POISON_INUSE, size);
55}
56
57static void check_element(mempool_t *pool, void *element)
58{
59
60 if (pool->free == mempool_free_slab || pool->free == mempool_kfree) {
61 __check_element(pool, element, ksize(element));
62 } else if (pool->free == mempool_free_pages) {
63
64 int order = (int)(long)pool->pool_data;
65 void *addr = kmap_atomic((struct page *)element);
66
67 __check_element(pool, addr, 1UL << (PAGE_SHIFT + order));
68 kunmap_atomic(addr);
69 }
70}
71
72static void __poison_element(void *element, size_t size)
73{
74 u8 *obj = element;
75
76 memset(obj, POISON_FREE, size - 1);
77 obj[size - 1] = POISON_END;
78}
79
80static void poison_element(mempool_t *pool, void *element)
81{
82
83 if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) {
84 __poison_element(element, ksize(element));
85 } else if (pool->alloc == mempool_alloc_pages) {
86
87 int order = (int)(long)pool->pool_data;
88 void *addr = kmap_atomic((struct page *)element);
89
90 __poison_element(addr, 1UL << (PAGE_SHIFT + order));
91 kunmap_atomic(addr);
92 }
93}
94#else
95static inline void check_element(mempool_t *pool, void *element)
96{
97}
98static inline void poison_element(mempool_t *pool, void *element)
99{
100}
101#endif
102
103static __always_inline void kasan_poison_element(mempool_t *pool, void *element)
104{
105 if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
106 kasan_slab_free_mempool(element);
107 else if (pool->alloc == mempool_alloc_pages)
108 kasan_poison_pages(element, (unsigned long)pool->pool_data,
109 false);
110}
111
112static void kasan_unpoison_element(mempool_t *pool, void *element)
113{
114 if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
115 kasan_unpoison_range(element, __ksize(element));
116 else if (pool->alloc == mempool_alloc_pages)
117 kasan_unpoison_pages(element, (unsigned long)pool->pool_data,
118 false);
119}
120
121static __always_inline void add_element(mempool_t *pool, void *element)
122{
123 BUG_ON(pool->curr_nr >= pool->min_nr);
124 poison_element(pool, element);
125 kasan_poison_element(pool, element);
126 pool->elements[pool->curr_nr++] = element;
127}
128
129static void *remove_element(mempool_t *pool)
130{
131 void *element = pool->elements[--pool->curr_nr];
132
133 BUG_ON(pool->curr_nr < 0);
134 kasan_unpoison_element(pool, element);
135 check_element(pool, element);
136 return element;
137}
138
139
140
141
142
143
144
145
146
147
148
149
150void mempool_exit(mempool_t *pool)
151{
152 while (pool->curr_nr) {
153 void *element = remove_element(pool);
154 pool->free(element, pool->pool_data);
155 }
156 kfree(pool->elements);
157 pool->elements = NULL;
158}
159EXPORT_SYMBOL(mempool_exit);
160
161
162
163
164
165
166
167
168
169void mempool_destroy(mempool_t *pool)
170{
171 if (unlikely(!pool))
172 return;
173
174 mempool_exit(pool);
175 kfree(pool);
176}
177EXPORT_SYMBOL(mempool_destroy);
178
179int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
180 mempool_free_t *free_fn, void *pool_data,
181 gfp_t gfp_mask, int node_id)
182{
183 spin_lock_init(&pool->lock);
184 pool->min_nr = min_nr;
185 pool->pool_data = pool_data;
186 pool->alloc = alloc_fn;
187 pool->free = free_fn;
188 init_waitqueue_head(&pool->wait);
189
190 pool->elements = kmalloc_array_node(min_nr, sizeof(void *),
191 gfp_mask, node_id);
192 if (!pool->elements)
193 return -ENOMEM;
194
195
196
197
198 while (pool->curr_nr < pool->min_nr) {
199 void *element;
200
201 element = pool->alloc(gfp_mask, pool->pool_data);
202 if (unlikely(!element)) {
203 mempool_exit(pool);
204 return -ENOMEM;
205 }
206 add_element(pool, element);
207 }
208
209 return 0;
210}
211EXPORT_SYMBOL(mempool_init_node);
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
228 mempool_free_t *free_fn, void *pool_data)
229{
230 return mempool_init_node(pool, min_nr, alloc_fn, free_fn,
231 pool_data, GFP_KERNEL, NUMA_NO_NODE);
232
233}
234EXPORT_SYMBOL(mempool_init);
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252mempool_t *mempool_create(int min_nr, mempool_alloc_t *alloc_fn,
253 mempool_free_t *free_fn, void *pool_data)
254{
255 return mempool_create_node(min_nr, alloc_fn, free_fn, pool_data,
256 GFP_KERNEL, NUMA_NO_NODE);
257}
258EXPORT_SYMBOL(mempool_create);
259
260mempool_t *mempool_create_node(int min_nr, mempool_alloc_t *alloc_fn,
261 mempool_free_t *free_fn, void *pool_data,
262 gfp_t gfp_mask, int node_id)
263{
264 mempool_t *pool;
265
266 pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);
267 if (!pool)
268 return NULL;
269
270 if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,
271 gfp_mask, node_id)) {
272 kfree(pool);
273 return NULL;
274 }
275
276 return pool;
277}
278EXPORT_SYMBOL(mempool_create_node);
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298int mempool_resize(mempool_t *pool, int new_min_nr)
299{
300 void *element;
301 void **new_elements;
302 unsigned long flags;
303
304 BUG_ON(new_min_nr <= 0);
305 might_sleep();
306
307 spin_lock_irqsave(&pool->lock, flags);
308 if (new_min_nr <= pool->min_nr) {
309 while (new_min_nr < pool->curr_nr) {
310 element = remove_element(pool);
311 spin_unlock_irqrestore(&pool->lock, flags);
312 pool->free(element, pool->pool_data);
313 spin_lock_irqsave(&pool->lock, flags);
314 }
315 pool->min_nr = new_min_nr;
316 goto out_unlock;
317 }
318 spin_unlock_irqrestore(&pool->lock, flags);
319
320
321 new_elements = kmalloc_array(new_min_nr, sizeof(*new_elements),
322 GFP_KERNEL);
323 if (!new_elements)
324 return -ENOMEM;
325
326 spin_lock_irqsave(&pool->lock, flags);
327 if (unlikely(new_min_nr <= pool->min_nr)) {
328
329 spin_unlock_irqrestore(&pool->lock, flags);
330 kfree(new_elements);
331 goto out;
332 }
333 memcpy(new_elements, pool->elements,
334 pool->curr_nr * sizeof(*new_elements));
335 kfree(pool->elements);
336 pool->elements = new_elements;
337 pool->min_nr = new_min_nr;
338
339 while (pool->curr_nr < pool->min_nr) {
340 spin_unlock_irqrestore(&pool->lock, flags);
341 element = pool->alloc(GFP_KERNEL, pool->pool_data);
342 if (!element)
343 goto out;
344 spin_lock_irqsave(&pool->lock, flags);
345 if (pool->curr_nr < pool->min_nr) {
346 add_element(pool, element);
347 } else {
348 spin_unlock_irqrestore(&pool->lock, flags);
349 pool->free(element, pool->pool_data);
350 goto out;
351 }
352 }
353out_unlock:
354 spin_unlock_irqrestore(&pool->lock, flags);
355out:
356 return 0;
357}
358EXPORT_SYMBOL(mempool_resize);
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
375{
376 void *element;
377 unsigned long flags;
378 wait_queue_entry_t wait;
379 gfp_t gfp_temp;
380
381 VM_WARN_ON_ONCE(gfp_mask & __GFP_ZERO);
382 might_sleep_if(gfp_mask & __GFP_DIRECT_RECLAIM);
383
384 gfp_mask |= __GFP_NOMEMALLOC;
385 gfp_mask |= __GFP_NORETRY;
386 gfp_mask |= __GFP_NOWARN;
387
388 gfp_temp = gfp_mask & ~(__GFP_DIRECT_RECLAIM|__GFP_IO);
389
390repeat_alloc:
391
392 element = pool->alloc(gfp_temp, pool->pool_data);
393 if (likely(element != NULL))
394 return element;
395
396 spin_lock_irqsave(&pool->lock, flags);
397 if (likely(pool->curr_nr)) {
398 element = remove_element(pool);
399 spin_unlock_irqrestore(&pool->lock, flags);
400
401 smp_wmb();
402
403
404
405
406 kmemleak_update_trace(element);
407 return element;
408 }
409
410
411
412
413
414 if (gfp_temp != gfp_mask) {
415 spin_unlock_irqrestore(&pool->lock, flags);
416 gfp_temp = gfp_mask;
417 goto repeat_alloc;
418 }
419
420
421 if (!(gfp_mask & __GFP_DIRECT_RECLAIM)) {
422 spin_unlock_irqrestore(&pool->lock, flags);
423 return NULL;
424 }
425
426
427 init_wait(&wait);
428 prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
429
430 spin_unlock_irqrestore(&pool->lock, flags);
431
432
433
434
435
436 io_schedule_timeout(5*HZ);
437
438 finish_wait(&pool->wait, &wait);
439 goto repeat_alloc;
440}
441EXPORT_SYMBOL(mempool_alloc);
442
443
444
445
446
447
448
449
450
451void mempool_free(void *element, mempool_t *pool)
452{
453 unsigned long flags;
454
455 if (unlikely(element == NULL))
456 return;
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472 smp_rmb();
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491 if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {
492 spin_lock_irqsave(&pool->lock, flags);
493 if (likely(pool->curr_nr < pool->min_nr)) {
494 add_element(pool, element);
495 spin_unlock_irqrestore(&pool->lock, flags);
496 wake_up(&pool->wait);
497 return;
498 }
499 spin_unlock_irqrestore(&pool->lock, flags);
500 }
501 pool->free(element, pool->pool_data);
502}
503EXPORT_SYMBOL(mempool_free);
504
505
506
507
508void *mempool_alloc_slab(gfp_t gfp_mask, void *pool_data)
509{
510 struct kmem_cache *mem = pool_data;
511 VM_BUG_ON(mem->ctor);
512 return kmem_cache_alloc(mem, gfp_mask);
513}
514EXPORT_SYMBOL(mempool_alloc_slab);
515
516void mempool_free_slab(void *element, void *pool_data)
517{
518 struct kmem_cache *mem = pool_data;
519 kmem_cache_free(mem, element);
520}
521EXPORT_SYMBOL(mempool_free_slab);
522
523
524
525
526
527void *mempool_kmalloc(gfp_t gfp_mask, void *pool_data)
528{
529 size_t size = (size_t)pool_data;
530 return kmalloc(size, gfp_mask);
531}
532EXPORT_SYMBOL(mempool_kmalloc);
533
534void mempool_kfree(void *element, void *pool_data)
535{
536 kfree(element);
537}
538EXPORT_SYMBOL(mempool_kfree);
539
540
541
542
543
544void *mempool_alloc_pages(gfp_t gfp_mask, void *pool_data)
545{
546 int order = (int)(long)pool_data;
547 return alloc_pages(gfp_mask, order);
548}
549EXPORT_SYMBOL(mempool_alloc_pages);
550
551void mempool_free_pages(void *element, void *pool_data)
552{
553 int order = (int)(long)pool_data;
554 __free_pages(element, order);
555}
556EXPORT_SYMBOL(mempool_free_pages);
557