1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include <linux/slab.h>
30#include <linux/export.h>
31#include <linux/bitmap.h>
32#include <linux/rculist.h>
33#include <linux/interrupt.h>
34#include <linux/genalloc.h>
35#include <linux/of_device.h>
36#include <linux/vmalloc.h>
37
38static inline size_t chunk_size(const struct gen_pool_chunk *chunk)
39{
40 return chunk->end_addr - chunk->start_addr + 1;
41}
42
43static int set_bits_ll(unsigned long *addr, unsigned long mask_to_set)
44{
45 unsigned long val, nval;
46
47 nval = *addr;
48 do {
49 val = nval;
50 if (val & mask_to_set)
51 return -EBUSY;
52 cpu_relax();
53 } while ((nval = cmpxchg(addr, val, val | mask_to_set)) != val);
54
55 return 0;
56}
57
58static int clear_bits_ll(unsigned long *addr, unsigned long mask_to_clear)
59{
60 unsigned long val, nval;
61
62 nval = *addr;
63 do {
64 val = nval;
65 if ((val & mask_to_clear) != mask_to_clear)
66 return -EBUSY;
67 cpu_relax();
68 } while ((nval = cmpxchg(addr, val, val & ~mask_to_clear)) != val);
69
70 return 0;
71}
72
73
74
75
76
77
78
79
80
81
82
83
84static int bitmap_set_ll(unsigned long *map, int start, int nr)
85{
86 unsigned long *p = map + BIT_WORD(start);
87 const int size = start + nr;
88 int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG);
89 unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start);
90
91 while (nr - bits_to_set >= 0) {
92 if (set_bits_ll(p, mask_to_set))
93 return nr;
94 nr -= bits_to_set;
95 bits_to_set = BITS_PER_LONG;
96 mask_to_set = ~0UL;
97 p++;
98 }
99 if (nr) {
100 mask_to_set &= BITMAP_LAST_WORD_MASK(size);
101 if (set_bits_ll(p, mask_to_set))
102 return nr;
103 }
104
105 return 0;
106}
107
108
109
110
111
112
113
114
115
116
117
118
119static int bitmap_clear_ll(unsigned long *map, int start, int nr)
120{
121 unsigned long *p = map + BIT_WORD(start);
122 const int size = start + nr;
123 int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG);
124 unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start);
125
126 while (nr - bits_to_clear >= 0) {
127 if (clear_bits_ll(p, mask_to_clear))
128 return nr;
129 nr -= bits_to_clear;
130 bits_to_clear = BITS_PER_LONG;
131 mask_to_clear = ~0UL;
132 p++;
133 }
134 if (nr) {
135 mask_to_clear &= BITMAP_LAST_WORD_MASK(size);
136 if (clear_bits_ll(p, mask_to_clear))
137 return nr;
138 }
139
140 return 0;
141}
142
143
144
145
146
147
148
149
150
151struct gen_pool *gen_pool_create(int min_alloc_order, int nid)
152{
153 struct gen_pool *pool;
154
155 pool = kmalloc_node(sizeof(struct gen_pool), GFP_KERNEL, nid);
156 if (pool != NULL) {
157 spin_lock_init(&pool->lock);
158 INIT_LIST_HEAD(&pool->chunks);
159 pool->min_alloc_order = min_alloc_order;
160 pool->algo = gen_pool_first_fit;
161 pool->data = NULL;
162 pool->name = NULL;
163 }
164 return pool;
165}
166EXPORT_SYMBOL(gen_pool_create);
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182int gen_pool_add_owner(struct gen_pool *pool, unsigned long virt, phys_addr_t phys,
183 size_t size, int nid, void *owner)
184{
185 struct gen_pool_chunk *chunk;
186 int nbits = size >> pool->min_alloc_order;
187 int nbytes = sizeof(struct gen_pool_chunk) +
188 BITS_TO_LONGS(nbits) * sizeof(long);
189
190 chunk = vzalloc_node(nbytes, nid);
191 if (unlikely(chunk == NULL))
192 return -ENOMEM;
193
194 chunk->phys_addr = phys;
195 chunk->start_addr = virt;
196 chunk->end_addr = virt + size - 1;
197 chunk->owner = owner;
198 atomic_long_set(&chunk->avail, size);
199
200 spin_lock(&pool->lock);
201 list_add_rcu(&chunk->next_chunk, &pool->chunks);
202 spin_unlock(&pool->lock);
203
204 return 0;
205}
206EXPORT_SYMBOL(gen_pool_add_owner);
207
208
209
210
211
212
213
214
215phys_addr_t gen_pool_virt_to_phys(struct gen_pool *pool, unsigned long addr)
216{
217 struct gen_pool_chunk *chunk;
218 phys_addr_t paddr = -1;
219
220 rcu_read_lock();
221 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) {
222 if (addr >= chunk->start_addr && addr <= chunk->end_addr) {
223 paddr = chunk->phys_addr + (addr - chunk->start_addr);
224 break;
225 }
226 }
227 rcu_read_unlock();
228
229 return paddr;
230}
231EXPORT_SYMBOL(gen_pool_virt_to_phys);
232
233
234
235
236
237
238
239
240void gen_pool_destroy(struct gen_pool *pool)
241{
242 struct list_head *_chunk, *_next_chunk;
243 struct gen_pool_chunk *chunk;
244 int order = pool->min_alloc_order;
245 int bit, end_bit;
246
247 list_for_each_safe(_chunk, _next_chunk, &pool->chunks) {
248 chunk = list_entry(_chunk, struct gen_pool_chunk, next_chunk);
249 list_del(&chunk->next_chunk);
250
251 end_bit = chunk_size(chunk) >> order;
252 bit = find_next_bit(chunk->bits, end_bit, 0);
253 BUG_ON(bit < end_bit);
254
255 vfree(chunk);
256 }
257 kfree_const(pool->name);
258 kfree(pool);
259}
260EXPORT_SYMBOL(gen_pool_destroy);
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275unsigned long gen_pool_alloc_algo_owner(struct gen_pool *pool, size_t size,
276 genpool_algo_t algo, void *data, void **owner)
277{
278 struct gen_pool_chunk *chunk;
279 unsigned long addr = 0;
280 int order = pool->min_alloc_order;
281 int nbits, start_bit, end_bit, remain;
282
283#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
284 BUG_ON(in_nmi());
285#endif
286
287 if (owner)
288 *owner = NULL;
289
290 if (size == 0)
291 return 0;
292
293 nbits = (size + (1UL << order) - 1) >> order;
294 rcu_read_lock();
295 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) {
296 if (size > atomic_long_read(&chunk->avail))
297 continue;
298
299 start_bit = 0;
300 end_bit = chunk_size(chunk) >> order;
301retry:
302 start_bit = algo(chunk->bits, end_bit, start_bit,
303 nbits, data, pool, chunk->start_addr);
304 if (start_bit >= end_bit)
305 continue;
306 remain = bitmap_set_ll(chunk->bits, start_bit, nbits);
307 if (remain) {
308 remain = bitmap_clear_ll(chunk->bits, start_bit,
309 nbits - remain);
310 BUG_ON(remain);
311 goto retry;
312 }
313
314 addr = chunk->start_addr + ((unsigned long)start_bit << order);
315 size = nbits << order;
316 atomic_long_sub(size, &chunk->avail);
317 if (owner)
318 *owner = chunk->owner;
319 break;
320 }
321 rcu_read_unlock();
322 return addr;
323}
324EXPORT_SYMBOL(gen_pool_alloc_algo_owner);
325
326
327
328
329
330
331
332
333
334
335
336
337void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma)
338{
339 unsigned long vaddr;
340
341 if (!pool)
342 return NULL;
343
344 vaddr = gen_pool_alloc(pool, size);
345 if (!vaddr)
346 return NULL;
347
348 if (dma)
349 *dma = gen_pool_virt_to_phys(pool, vaddr);
350
351 return (void *)vaddr;
352}
353EXPORT_SYMBOL(gen_pool_dma_alloc);
354
355
356
357
358
359
360
361
362
363
364
365
366void gen_pool_free_owner(struct gen_pool *pool, unsigned long addr, size_t size,
367 void **owner)
368{
369 struct gen_pool_chunk *chunk;
370 int order = pool->min_alloc_order;
371 int start_bit, nbits, remain;
372
373#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
374 BUG_ON(in_nmi());
375#endif
376
377 if (owner)
378 *owner = NULL;
379
380 nbits = (size + (1UL << order) - 1) >> order;
381 rcu_read_lock();
382 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) {
383 if (addr >= chunk->start_addr && addr <= chunk->end_addr) {
384 BUG_ON(addr + size - 1 > chunk->end_addr);
385 start_bit = (addr - chunk->start_addr) >> order;
386 remain = bitmap_clear_ll(chunk->bits, start_bit, nbits);
387 BUG_ON(remain);
388 size = nbits << order;
389 atomic_long_add(size, &chunk->avail);
390 if (owner)
391 *owner = chunk->owner;
392 rcu_read_unlock();
393 return;
394 }
395 }
396 rcu_read_unlock();
397 BUG();
398}
399EXPORT_SYMBOL(gen_pool_free_owner);
400
401
402
403
404
405
406
407
408
409
410void gen_pool_for_each_chunk(struct gen_pool *pool,
411 void (*func)(struct gen_pool *pool, struct gen_pool_chunk *chunk, void *data),
412 void *data)
413{
414 struct gen_pool_chunk *chunk;
415
416 rcu_read_lock();
417 list_for_each_entry_rcu(chunk, &(pool)->chunks, next_chunk)
418 func(pool, chunk, data);
419 rcu_read_unlock();
420}
421EXPORT_SYMBOL(gen_pool_for_each_chunk);
422
423
424
425
426
427
428
429
430
431
432bool addr_in_gen_pool(struct gen_pool *pool, unsigned long start,
433 size_t size)
434{
435 bool found = false;
436 unsigned long end = start + size - 1;
437 struct gen_pool_chunk *chunk;
438
439 rcu_read_lock();
440 list_for_each_entry_rcu(chunk, &(pool)->chunks, next_chunk) {
441 if (start >= chunk->start_addr && start <= chunk->end_addr) {
442 if (end <= chunk->end_addr) {
443 found = true;
444 break;
445 }
446 }
447 }
448 rcu_read_unlock();
449 return found;
450}
451
452
453
454
455
456
457
458size_t gen_pool_avail(struct gen_pool *pool)
459{
460 struct gen_pool_chunk *chunk;
461 size_t avail = 0;
462
463 rcu_read_lock();
464 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk)
465 avail += atomic_long_read(&chunk->avail);
466 rcu_read_unlock();
467 return avail;
468}
469EXPORT_SYMBOL_GPL(gen_pool_avail);
470
471
472
473
474
475
476
477size_t gen_pool_size(struct gen_pool *pool)
478{
479 struct gen_pool_chunk *chunk;
480 size_t size = 0;
481
482 rcu_read_lock();
483 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk)
484 size += chunk_size(chunk);
485 rcu_read_unlock();
486 return size;
487}
488EXPORT_SYMBOL_GPL(gen_pool_size);
489
490
491
492
493
494
495
496
497
498
499
500void gen_pool_set_algo(struct gen_pool *pool, genpool_algo_t algo, void *data)
501{
502 rcu_read_lock();
503
504 pool->algo = algo;
505 if (!pool->algo)
506 pool->algo = gen_pool_first_fit;
507
508 pool->data = data;
509
510 rcu_read_unlock();
511}
512EXPORT_SYMBOL(gen_pool_set_algo);
513
514
515
516
517
518
519
520
521
522
523
524unsigned long gen_pool_first_fit(unsigned long *map, unsigned long size,
525 unsigned long start, unsigned int nr, void *data,
526 struct gen_pool *pool, unsigned long start_addr)
527{
528 return bitmap_find_next_zero_area(map, size, start, nr, 0);
529}
530EXPORT_SYMBOL(gen_pool_first_fit);
531
532
533
534
535
536
537
538
539
540
541
542unsigned long gen_pool_first_fit_align(unsigned long *map, unsigned long size,
543 unsigned long start, unsigned int nr, void *data,
544 struct gen_pool *pool, unsigned long start_addr)
545{
546 struct genpool_data_align *alignment;
547 unsigned long align_mask, align_off;
548 int order;
549
550 alignment = data;
551 order = pool->min_alloc_order;
552 align_mask = ((alignment->align + (1UL << order) - 1) >> order) - 1;
553 align_off = (start_addr & (alignment->align - 1)) >> order;
554
555 return bitmap_find_next_zero_area_off(map, size, start, nr,
556 align_mask, align_off);
557}
558EXPORT_SYMBOL(gen_pool_first_fit_align);
559
560
561
562
563
564
565
566
567
568
569unsigned long gen_pool_fixed_alloc(unsigned long *map, unsigned long size,
570 unsigned long start, unsigned int nr, void *data,
571 struct gen_pool *pool, unsigned long start_addr)
572{
573 struct genpool_data_fixed *fixed_data;
574 int order;
575 unsigned long offset_bit;
576 unsigned long start_bit;
577
578 fixed_data = data;
579 order = pool->min_alloc_order;
580 offset_bit = fixed_data->offset >> order;
581 if (WARN_ON(fixed_data->offset & ((1UL << order) - 1)))
582 return size;
583
584 start_bit = bitmap_find_next_zero_area(map, size,
585 start + offset_bit, nr, 0);
586 if (start_bit != offset_bit)
587 start_bit = size;
588 return start_bit;
589}
590EXPORT_SYMBOL(gen_pool_fixed_alloc);
591
592
593
594
595
596
597
598
599
600
601
602
603unsigned long gen_pool_first_fit_order_align(unsigned long *map,
604 unsigned long size, unsigned long start,
605 unsigned int nr, void *data, struct gen_pool *pool,
606 unsigned long start_addr)
607{
608 unsigned long align_mask = roundup_pow_of_two(nr) - 1;
609
610 return bitmap_find_next_zero_area(map, size, start, nr, align_mask);
611}
612EXPORT_SYMBOL(gen_pool_first_fit_order_align);
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627unsigned long gen_pool_best_fit(unsigned long *map, unsigned long size,
628 unsigned long start, unsigned int nr, void *data,
629 struct gen_pool *pool, unsigned long start_addr)
630{
631 unsigned long start_bit = size;
632 unsigned long len = size + 1;
633 unsigned long index;
634
635 index = bitmap_find_next_zero_area(map, size, start, nr, 0);
636
637 while (index < size) {
638 int next_bit = find_next_bit(map, size, index + nr);
639 if ((next_bit - index) < len) {
640 len = next_bit - index;
641 start_bit = index;
642 if (len == nr)
643 return start_bit;
644 }
645 index = bitmap_find_next_zero_area(map, size,
646 next_bit + 1, nr, 0);
647 }
648
649 return start_bit;
650}
651EXPORT_SYMBOL(gen_pool_best_fit);
652
653static void devm_gen_pool_release(struct device *dev, void *res)
654{
655 gen_pool_destroy(*(struct gen_pool **)res);
656}
657
658static int devm_gen_pool_match(struct device *dev, void *res, void *data)
659{
660 struct gen_pool **p = res;
661
662
663 if (!data && !(*p)->name)
664 return 1;
665
666 if (!data || !(*p)->name)
667 return 0;
668
669 return !strcmp((*p)->name, data);
670}
671
672
673
674
675
676
677
678
679struct gen_pool *gen_pool_get(struct device *dev, const char *name)
680{
681 struct gen_pool **p;
682
683 p = devres_find(dev, devm_gen_pool_release, devm_gen_pool_match,
684 (void *)name);
685 if (!p)
686 return NULL;
687 return *p;
688}
689EXPORT_SYMBOL_GPL(gen_pool_get);
690
691
692
693
694
695
696
697
698
699
700
701
702struct gen_pool *devm_gen_pool_create(struct device *dev, int min_alloc_order,
703 int nid, const char *name)
704{
705 struct gen_pool **ptr, *pool;
706 const char *pool_name = NULL;
707
708
709 if (gen_pool_get(dev, name))
710 return ERR_PTR(-EINVAL);
711
712 if (name) {
713 pool_name = kstrdup_const(name, GFP_KERNEL);
714 if (!pool_name)
715 return ERR_PTR(-ENOMEM);
716 }
717
718 ptr = devres_alloc(devm_gen_pool_release, sizeof(*ptr), GFP_KERNEL);
719 if (!ptr)
720 goto free_pool_name;
721
722 pool = gen_pool_create(min_alloc_order, nid);
723 if (!pool)
724 goto free_devres;
725
726 *ptr = pool;
727 pool->name = pool_name;
728 devres_add(dev, ptr);
729
730 return pool;
731
732free_devres:
733 devres_free(ptr);
734free_pool_name:
735 kfree_const(pool_name);
736
737 return ERR_PTR(-ENOMEM);
738}
739EXPORT_SYMBOL(devm_gen_pool_create);
740
741#ifdef CONFIG_OF
742
743
744
745
746
747
748
749
750
751
752struct gen_pool *of_gen_pool_get(struct device_node *np,
753 const char *propname, int index)
754{
755 struct platform_device *pdev;
756 struct device_node *np_pool, *parent;
757 const char *name = NULL;
758 struct gen_pool *pool = NULL;
759
760 np_pool = of_parse_phandle(np, propname, index);
761 if (!np_pool)
762 return NULL;
763
764 pdev = of_find_device_by_node(np_pool);
765 if (!pdev) {
766
767 parent = of_get_parent(np_pool);
768 pdev = of_find_device_by_node(parent);
769 of_node_put(parent);
770
771 of_property_read_string(np_pool, "label", &name);
772 if (!name)
773 name = np_pool->name;
774 }
775 if (pdev)
776 pool = gen_pool_get(&pdev->dev, name);
777 of_node_put(np_pool);
778
779 return pool;
780}
781EXPORT_SYMBOL_GPL(of_gen_pool_get);
782#endif
783