1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include <linux/slab.h>
30#include <linux/export.h>
31#include <linux/bitmap.h>
32#include <linux/rculist.h>
33#include <linux/interrupt.h>
34#include <linux/genalloc.h>
35#include <linux/of_device.h>
36#include <linux/vmalloc.h>
37
38static inline size_t chunk_size(const struct gen_pool_chunk *chunk)
39{
40 return chunk->end_addr - chunk->start_addr + 1;
41}
42
43static int set_bits_ll(unsigned long *addr, unsigned long mask_to_set)
44{
45 unsigned long val, nval;
46
47 nval = *addr;
48 do {
49 val = nval;
50 if (val & mask_to_set)
51 return -EBUSY;
52 cpu_relax();
53 } while ((nval = cmpxchg(addr, val, val | mask_to_set)) != val);
54
55 return 0;
56}
57
58static int clear_bits_ll(unsigned long *addr, unsigned long mask_to_clear)
59{
60 unsigned long val, nval;
61
62 nval = *addr;
63 do {
64 val = nval;
65 if ((val & mask_to_clear) != mask_to_clear)
66 return -EBUSY;
67 cpu_relax();
68 } while ((nval = cmpxchg(addr, val, val & ~mask_to_clear)) != val);
69
70 return 0;
71}
72
73
74
75
76
77
78
79
80
81
82
83
84static int bitmap_set_ll(unsigned long *map, int start, int nr)
85{
86 unsigned long *p = map + BIT_WORD(start);
87 const int size = start + nr;
88 int bits_to_set = BITS_PER_LONG - (start % BITS_PER_LONG);
89 unsigned long mask_to_set = BITMAP_FIRST_WORD_MASK(start);
90
91 while (nr - bits_to_set >= 0) {
92 if (set_bits_ll(p, mask_to_set))
93 return nr;
94 nr -= bits_to_set;
95 bits_to_set = BITS_PER_LONG;
96 mask_to_set = ~0UL;
97 p++;
98 }
99 if (nr) {
100 mask_to_set &= BITMAP_LAST_WORD_MASK(size);
101 if (set_bits_ll(p, mask_to_set))
102 return nr;
103 }
104
105 return 0;
106}
107
108
109
110
111
112
113
114
115
116
117
118
119static int bitmap_clear_ll(unsigned long *map, int start, int nr)
120{
121 unsigned long *p = map + BIT_WORD(start);
122 const int size = start + nr;
123 int bits_to_clear = BITS_PER_LONG - (start % BITS_PER_LONG);
124 unsigned long mask_to_clear = BITMAP_FIRST_WORD_MASK(start);
125
126 while (nr - bits_to_clear >= 0) {
127 if (clear_bits_ll(p, mask_to_clear))
128 return nr;
129 nr -= bits_to_clear;
130 bits_to_clear = BITS_PER_LONG;
131 mask_to_clear = ~0UL;
132 p++;
133 }
134 if (nr) {
135 mask_to_clear &= BITMAP_LAST_WORD_MASK(size);
136 if (clear_bits_ll(p, mask_to_clear))
137 return nr;
138 }
139
140 return 0;
141}
142
143
144
145
146
147
148
149
150
151struct gen_pool *gen_pool_create(int min_alloc_order, int nid)
152{
153 struct gen_pool *pool;
154
155 pool = kmalloc_node(sizeof(struct gen_pool), GFP_KERNEL, nid);
156 if (pool != NULL) {
157 spin_lock_init(&pool->lock);
158 INIT_LIST_HEAD(&pool->chunks);
159 pool->min_alloc_order = min_alloc_order;
160 pool->algo = gen_pool_first_fit;
161 pool->data = NULL;
162 pool->name = NULL;
163 }
164 return pool;
165}
166EXPORT_SYMBOL(gen_pool_create);
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182int gen_pool_add_owner(struct gen_pool *pool, unsigned long virt, phys_addr_t phys,
183 size_t size, int nid, void *owner)
184{
185 struct gen_pool_chunk *chunk;
186 int nbits = size >> pool->min_alloc_order;
187 int nbytes = sizeof(struct gen_pool_chunk) +
188 BITS_TO_LONGS(nbits) * sizeof(long);
189
190 chunk = vzalloc_node(nbytes, nid);
191 if (unlikely(chunk == NULL))
192 return -ENOMEM;
193
194 chunk->phys_addr = phys;
195 chunk->start_addr = virt;
196 chunk->end_addr = virt + size - 1;
197 chunk->owner = owner;
198 atomic_long_set(&chunk->avail, size);
199
200 spin_lock(&pool->lock);
201 list_add_rcu(&chunk->next_chunk, &pool->chunks);
202 spin_unlock(&pool->lock);
203
204 return 0;
205}
206EXPORT_SYMBOL(gen_pool_add_owner);
207
208
209
210
211
212
213
214
215phys_addr_t gen_pool_virt_to_phys(struct gen_pool *pool, unsigned long addr)
216{
217 struct gen_pool_chunk *chunk;
218 phys_addr_t paddr = -1;
219
220 rcu_read_lock();
221 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) {
222 if (addr >= chunk->start_addr && addr <= chunk->end_addr) {
223 paddr = chunk->phys_addr + (addr - chunk->start_addr);
224 break;
225 }
226 }
227 rcu_read_unlock();
228
229 return paddr;
230}
231EXPORT_SYMBOL(gen_pool_virt_to_phys);
232
233
234
235
236
237
238
239
240void gen_pool_destroy(struct gen_pool *pool)
241{
242 struct list_head *_chunk, *_next_chunk;
243 struct gen_pool_chunk *chunk;
244 int order = pool->min_alloc_order;
245 int bit, end_bit;
246
247 list_for_each_safe(_chunk, _next_chunk, &pool->chunks) {
248 chunk = list_entry(_chunk, struct gen_pool_chunk, next_chunk);
249 list_del(&chunk->next_chunk);
250
251 end_bit = chunk_size(chunk) >> order;
252 bit = find_next_bit(chunk->bits, end_bit, 0);
253 BUG_ON(bit < end_bit);
254
255 vfree(chunk);
256 }
257 kfree_const(pool->name);
258 kfree(pool);
259}
260EXPORT_SYMBOL(gen_pool_destroy);
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275unsigned long gen_pool_alloc_algo_owner(struct gen_pool *pool, size_t size,
276 genpool_algo_t algo, void *data, void **owner)
277{
278 struct gen_pool_chunk *chunk;
279 unsigned long addr = 0;
280 int order = pool->min_alloc_order;
281 int nbits, start_bit, end_bit, remain;
282
283#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
284 BUG_ON(in_nmi());
285#endif
286
287 if (owner)
288 *owner = NULL;
289
290 if (size == 0)
291 return 0;
292
293 nbits = (size + (1UL << order) - 1) >> order;
294 rcu_read_lock();
295 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) {
296 if (size > atomic_long_read(&chunk->avail))
297 continue;
298
299 start_bit = 0;
300 end_bit = chunk_size(chunk) >> order;
301retry:
302 start_bit = algo(chunk->bits, end_bit, start_bit,
303 nbits, data, pool, chunk->start_addr);
304 if (start_bit >= end_bit)
305 continue;
306 remain = bitmap_set_ll(chunk->bits, start_bit, nbits);
307 if (remain) {
308 remain = bitmap_clear_ll(chunk->bits, start_bit,
309 nbits - remain);
310 BUG_ON(remain);
311 goto retry;
312 }
313
314 addr = chunk->start_addr + ((unsigned long)start_bit << order);
315 size = nbits << order;
316 atomic_long_sub(size, &chunk->avail);
317 if (owner)
318 *owner = chunk->owner;
319 break;
320 }
321 rcu_read_unlock();
322 return addr;
323}
324EXPORT_SYMBOL(gen_pool_alloc_algo_owner);
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339void *gen_pool_dma_alloc(struct gen_pool *pool, size_t size, dma_addr_t *dma)
340{
341 return gen_pool_dma_alloc_algo(pool, size, dma, pool->algo, pool->data);
342}
343EXPORT_SYMBOL(gen_pool_dma_alloc);
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360void *gen_pool_dma_alloc_algo(struct gen_pool *pool, size_t size,
361 dma_addr_t *dma, genpool_algo_t algo, void *data)
362{
363 unsigned long vaddr;
364
365 if (!pool)
366 return NULL;
367
368 vaddr = gen_pool_alloc_algo(pool, size, algo, data);
369 if (!vaddr)
370 return NULL;
371
372 if (dma)
373 *dma = gen_pool_virt_to_phys(pool, vaddr);
374
375 return (void *)vaddr;
376}
377EXPORT_SYMBOL(gen_pool_dma_alloc_algo);
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393void *gen_pool_dma_alloc_align(struct gen_pool *pool, size_t size,
394 dma_addr_t *dma, int align)
395{
396 struct genpool_data_align data = { .align = align };
397
398 return gen_pool_dma_alloc_algo(pool, size, dma,
399 gen_pool_first_fit_align, &data);
400}
401EXPORT_SYMBOL(gen_pool_dma_alloc_align);
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417void *gen_pool_dma_zalloc(struct gen_pool *pool, size_t size, dma_addr_t *dma)
418{
419 return gen_pool_dma_zalloc_algo(pool, size, dma, pool->algo, pool->data);
420}
421EXPORT_SYMBOL(gen_pool_dma_zalloc);
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438void *gen_pool_dma_zalloc_algo(struct gen_pool *pool, size_t size,
439 dma_addr_t *dma, genpool_algo_t algo, void *data)
440{
441 void *vaddr = gen_pool_dma_alloc_algo(pool, size, dma, algo, data);
442
443 if (vaddr)
444 memset(vaddr, 0, size);
445
446 return vaddr;
447}
448EXPORT_SYMBOL(gen_pool_dma_zalloc_algo);
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464void *gen_pool_dma_zalloc_align(struct gen_pool *pool, size_t size,
465 dma_addr_t *dma, int align)
466{
467 struct genpool_data_align data = { .align = align };
468
469 return gen_pool_dma_zalloc_algo(pool, size, dma,
470 gen_pool_first_fit_align, &data);
471}
472EXPORT_SYMBOL(gen_pool_dma_zalloc_align);
473
474
475
476
477
478
479
480
481
482
483
484
485void gen_pool_free_owner(struct gen_pool *pool, unsigned long addr, size_t size,
486 void **owner)
487{
488 struct gen_pool_chunk *chunk;
489 int order = pool->min_alloc_order;
490 int start_bit, nbits, remain;
491
492#ifndef CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG
493 BUG_ON(in_nmi());
494#endif
495
496 if (owner)
497 *owner = NULL;
498
499 nbits = (size + (1UL << order) - 1) >> order;
500 rcu_read_lock();
501 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk) {
502 if (addr >= chunk->start_addr && addr <= chunk->end_addr) {
503 BUG_ON(addr + size - 1 > chunk->end_addr);
504 start_bit = (addr - chunk->start_addr) >> order;
505 remain = bitmap_clear_ll(chunk->bits, start_bit, nbits);
506 BUG_ON(remain);
507 size = nbits << order;
508 atomic_long_add(size, &chunk->avail);
509 if (owner)
510 *owner = chunk->owner;
511 rcu_read_unlock();
512 return;
513 }
514 }
515 rcu_read_unlock();
516 BUG();
517}
518EXPORT_SYMBOL(gen_pool_free_owner);
519
520
521
522
523
524
525
526
527
528
529void gen_pool_for_each_chunk(struct gen_pool *pool,
530 void (*func)(struct gen_pool *pool, struct gen_pool_chunk *chunk, void *data),
531 void *data)
532{
533 struct gen_pool_chunk *chunk;
534
535 rcu_read_lock();
536 list_for_each_entry_rcu(chunk, &(pool)->chunks, next_chunk)
537 func(pool, chunk, data);
538 rcu_read_unlock();
539}
540EXPORT_SYMBOL(gen_pool_for_each_chunk);
541
542
543
544
545
546
547
548
549
550
551bool gen_pool_has_addr(struct gen_pool *pool, unsigned long start,
552 size_t size)
553{
554 bool found = false;
555 unsigned long end = start + size - 1;
556 struct gen_pool_chunk *chunk;
557
558 rcu_read_lock();
559 list_for_each_entry_rcu(chunk, &(pool)->chunks, next_chunk) {
560 if (start >= chunk->start_addr && start <= chunk->end_addr) {
561 if (end <= chunk->end_addr) {
562 found = true;
563 break;
564 }
565 }
566 }
567 rcu_read_unlock();
568 return found;
569}
570EXPORT_SYMBOL(gen_pool_has_addr);
571
572
573
574
575
576
577
578size_t gen_pool_avail(struct gen_pool *pool)
579{
580 struct gen_pool_chunk *chunk;
581 size_t avail = 0;
582
583 rcu_read_lock();
584 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk)
585 avail += atomic_long_read(&chunk->avail);
586 rcu_read_unlock();
587 return avail;
588}
589EXPORT_SYMBOL_GPL(gen_pool_avail);
590
591
592
593
594
595
596
597size_t gen_pool_size(struct gen_pool *pool)
598{
599 struct gen_pool_chunk *chunk;
600 size_t size = 0;
601
602 rcu_read_lock();
603 list_for_each_entry_rcu(chunk, &pool->chunks, next_chunk)
604 size += chunk_size(chunk);
605 rcu_read_unlock();
606 return size;
607}
608EXPORT_SYMBOL_GPL(gen_pool_size);
609
610
611
612
613
614
615
616
617
618
619
620void gen_pool_set_algo(struct gen_pool *pool, genpool_algo_t algo, void *data)
621{
622 rcu_read_lock();
623
624 pool->algo = algo;
625 if (!pool->algo)
626 pool->algo = gen_pool_first_fit;
627
628 pool->data = data;
629
630 rcu_read_unlock();
631}
632EXPORT_SYMBOL(gen_pool_set_algo);
633
634
635
636
637
638
639
640
641
642
643
644unsigned long gen_pool_first_fit(unsigned long *map, unsigned long size,
645 unsigned long start, unsigned int nr, void *data,
646 struct gen_pool *pool, unsigned long start_addr)
647{
648 return bitmap_find_next_zero_area(map, size, start, nr, 0);
649}
650EXPORT_SYMBOL(gen_pool_first_fit);
651
652
653
654
655
656
657
658
659
660
661
662unsigned long gen_pool_first_fit_align(unsigned long *map, unsigned long size,
663 unsigned long start, unsigned int nr, void *data,
664 struct gen_pool *pool, unsigned long start_addr)
665{
666 struct genpool_data_align *alignment;
667 unsigned long align_mask, align_off;
668 int order;
669
670 alignment = data;
671 order = pool->min_alloc_order;
672 align_mask = ((alignment->align + (1UL << order) - 1) >> order) - 1;
673 align_off = (start_addr & (alignment->align - 1)) >> order;
674
675 return bitmap_find_next_zero_area_off(map, size, start, nr,
676 align_mask, align_off);
677}
678EXPORT_SYMBOL(gen_pool_first_fit_align);
679
680
681
682
683
684
685
686
687
688
689unsigned long gen_pool_fixed_alloc(unsigned long *map, unsigned long size,
690 unsigned long start, unsigned int nr, void *data,
691 struct gen_pool *pool, unsigned long start_addr)
692{
693 struct genpool_data_fixed *fixed_data;
694 int order;
695 unsigned long offset_bit;
696 unsigned long start_bit;
697
698 fixed_data = data;
699 order = pool->min_alloc_order;
700 offset_bit = fixed_data->offset >> order;
701 if (WARN_ON(fixed_data->offset & ((1UL << order) - 1)))
702 return size;
703
704 start_bit = bitmap_find_next_zero_area(map, size,
705 start + offset_bit, nr, 0);
706 if (start_bit != offset_bit)
707 start_bit = size;
708 return start_bit;
709}
710EXPORT_SYMBOL(gen_pool_fixed_alloc);
711
712
713
714
715
716
717
718
719
720
721
722
723unsigned long gen_pool_first_fit_order_align(unsigned long *map,
724 unsigned long size, unsigned long start,
725 unsigned int nr, void *data, struct gen_pool *pool,
726 unsigned long start_addr)
727{
728 unsigned long align_mask = roundup_pow_of_two(nr) - 1;
729
730 return bitmap_find_next_zero_area(map, size, start, nr, align_mask);
731}
732EXPORT_SYMBOL(gen_pool_first_fit_order_align);
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747unsigned long gen_pool_best_fit(unsigned long *map, unsigned long size,
748 unsigned long start, unsigned int nr, void *data,
749 struct gen_pool *pool, unsigned long start_addr)
750{
751 unsigned long start_bit = size;
752 unsigned long len = size + 1;
753 unsigned long index;
754
755 index = bitmap_find_next_zero_area(map, size, start, nr, 0);
756
757 while (index < size) {
758 int next_bit = find_next_bit(map, size, index + nr);
759 if ((next_bit - index) < len) {
760 len = next_bit - index;
761 start_bit = index;
762 if (len == nr)
763 return start_bit;
764 }
765 index = bitmap_find_next_zero_area(map, size,
766 next_bit + 1, nr, 0);
767 }
768
769 return start_bit;
770}
771EXPORT_SYMBOL(gen_pool_best_fit);
772
773static void devm_gen_pool_release(struct device *dev, void *res)
774{
775 gen_pool_destroy(*(struct gen_pool **)res);
776}
777
778static int devm_gen_pool_match(struct device *dev, void *res, void *data)
779{
780 struct gen_pool **p = res;
781
782
783 if (!data && !(*p)->name)
784 return 1;
785
786 if (!data || !(*p)->name)
787 return 0;
788
789 return !strcmp((*p)->name, data);
790}
791
792
793
794
795
796
797
798
799struct gen_pool *gen_pool_get(struct device *dev, const char *name)
800{
801 struct gen_pool **p;
802
803 p = devres_find(dev, devm_gen_pool_release, devm_gen_pool_match,
804 (void *)name);
805 if (!p)
806 return NULL;
807 return *p;
808}
809EXPORT_SYMBOL_GPL(gen_pool_get);
810
811
812
813
814
815
816
817
818
819
820
821
822struct gen_pool *devm_gen_pool_create(struct device *dev, int min_alloc_order,
823 int nid, const char *name)
824{
825 struct gen_pool **ptr, *pool;
826 const char *pool_name = NULL;
827
828
829 if (gen_pool_get(dev, name))
830 return ERR_PTR(-EINVAL);
831
832 if (name) {
833 pool_name = kstrdup_const(name, GFP_KERNEL);
834 if (!pool_name)
835 return ERR_PTR(-ENOMEM);
836 }
837
838 ptr = devres_alloc(devm_gen_pool_release, sizeof(*ptr), GFP_KERNEL);
839 if (!ptr)
840 goto free_pool_name;
841
842 pool = gen_pool_create(min_alloc_order, nid);
843 if (!pool)
844 goto free_devres;
845
846 *ptr = pool;
847 pool->name = pool_name;
848 devres_add(dev, ptr);
849
850 return pool;
851
852free_devres:
853 devres_free(ptr);
854free_pool_name:
855 kfree_const(pool_name);
856
857 return ERR_PTR(-ENOMEM);
858}
859EXPORT_SYMBOL(devm_gen_pool_create);
860
861#ifdef CONFIG_OF
862
863
864
865
866
867
868
869
870
871
872struct gen_pool *of_gen_pool_get(struct device_node *np,
873 const char *propname, int index)
874{
875 struct platform_device *pdev;
876 struct device_node *np_pool, *parent;
877 const char *name = NULL;
878 struct gen_pool *pool = NULL;
879
880 np_pool = of_parse_phandle(np, propname, index);
881 if (!np_pool)
882 return NULL;
883
884 pdev = of_find_device_by_node(np_pool);
885 if (!pdev) {
886
887 parent = of_get_parent(np_pool);
888 pdev = of_find_device_by_node(parent);
889 of_node_put(parent);
890
891 of_property_read_string(np_pool, "label", &name);
892 if (!name)
893 name = np_pool->name;
894 }
895 if (pdev)
896 pool = gen_pool_get(&pdev->dev, name);
897 of_node_put(np_pool);
898
899 return pool;
900}
901EXPORT_SYMBOL_GPL(of_gen_pool_get);
902#endif
903