1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56#include <linux/bitmap.h>
57#include <linux/bootmem.h>
58#include <linux/err.h>
59#include <linux/list.h>
60#include <linux/log2.h>
61#include <linux/mm.h>
62#include <linux/module.h>
63#include <linux/mutex.h>
64#include <linux/percpu.h>
65#include <linux/pfn.h>
66#include <linux/slab.h>
67#include <linux/spinlock.h>
68#include <linux/vmalloc.h>
69#include <linux/workqueue.h>
70
71#include <asm/cacheflush.h>
72#include <asm/sections.h>
73#include <asm/tlbflush.h>
74#include <asm/io.h>
75
76#define PCPU_SLOT_BASE_SHIFT 5
77#define PCPU_DFL_MAP_ALLOC 16
78
79#ifdef CONFIG_SMP
80
81#ifndef __addr_to_pcpu_ptr
82#define __addr_to_pcpu_ptr(addr) \
83 (void __percpu *)((unsigned long)(addr) - \
84 (unsigned long)pcpu_base_addr + \
85 (unsigned long)__per_cpu_start)
86#endif
87#ifndef __pcpu_ptr_to_addr
88#define __pcpu_ptr_to_addr(ptr) \
89 (void __force *)((unsigned long)(ptr) + \
90 (unsigned long)pcpu_base_addr - \
91 (unsigned long)__per_cpu_start)
92#endif
93#else
94
95#define __addr_to_pcpu_ptr(addr) (void __percpu *)(addr)
96#define __pcpu_ptr_to_addr(ptr) (void __force *)(ptr)
97#endif
98
99struct pcpu_chunk {
100 struct list_head list;
101 int free_size;
102 int contig_hint;
103 void *base_addr;
104 int map_used;
105 int map_alloc;
106 int *map;
107 void *data;
108 bool immutable;
109 unsigned long populated[];
110};
111
112static int pcpu_unit_pages __read_mostly;
113static int pcpu_unit_size __read_mostly;
114static int pcpu_nr_units __read_mostly;
115static int pcpu_atom_size __read_mostly;
116static int pcpu_nr_slots __read_mostly;
117static size_t pcpu_chunk_struct_size __read_mostly;
118
119
120static unsigned int pcpu_first_unit_cpu __read_mostly;
121static unsigned int pcpu_last_unit_cpu __read_mostly;
122
123
124void *pcpu_base_addr __read_mostly;
125EXPORT_SYMBOL_GPL(pcpu_base_addr);
126
127static const int *pcpu_unit_map __read_mostly;
128const unsigned long *pcpu_unit_offsets __read_mostly;
129
130
131static int pcpu_nr_groups __read_mostly;
132static const unsigned long *pcpu_group_offsets __read_mostly;
133static const size_t *pcpu_group_sizes __read_mostly;
134
135
136
137
138
139
140static struct pcpu_chunk *pcpu_first_chunk;
141
142
143
144
145
146
147
148
149static struct pcpu_chunk *pcpu_reserved_chunk;
150static int pcpu_reserved_chunk_limit;
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176static DEFINE_MUTEX(pcpu_alloc_mutex);
177static DEFINE_SPINLOCK(pcpu_lock);
178
179static struct list_head *pcpu_slot __read_mostly;
180
181
182static void pcpu_reclaim(struct work_struct *work);
183static DECLARE_WORK(pcpu_reclaim_work, pcpu_reclaim);
184
185static bool pcpu_addr_in_first_chunk(void *addr)
186{
187 void *first_start = pcpu_first_chunk->base_addr;
188
189 return addr >= first_start && addr < first_start + pcpu_unit_size;
190}
191
192static bool pcpu_addr_in_reserved_chunk(void *addr)
193{
194 void *first_start = pcpu_first_chunk->base_addr;
195
196 return addr >= first_start &&
197 addr < first_start + pcpu_reserved_chunk_limit;
198}
199
200static int __pcpu_size_to_slot(int size)
201{
202 int highbit = fls(size);
203 return max(highbit - PCPU_SLOT_BASE_SHIFT + 2, 1);
204}
205
206static int pcpu_size_to_slot(int size)
207{
208 if (size == pcpu_unit_size)
209 return pcpu_nr_slots - 1;
210 return __pcpu_size_to_slot(size);
211}
212
213static int pcpu_chunk_slot(const struct pcpu_chunk *chunk)
214{
215 if (chunk->free_size < sizeof(int) || chunk->contig_hint < sizeof(int))
216 return 0;
217
218 return pcpu_size_to_slot(chunk->free_size);
219}
220
221
222static void pcpu_set_page_chunk(struct page *page, struct pcpu_chunk *pcpu)
223{
224 page->index = (unsigned long)pcpu;
225}
226
227
228static struct pcpu_chunk *pcpu_get_page_chunk(struct page *page)
229{
230 return (struct pcpu_chunk *)page->index;
231}
232
233static int __maybe_unused pcpu_page_idx(unsigned int cpu, int page_idx)
234{
235 return pcpu_unit_map[cpu] * pcpu_unit_pages + page_idx;
236}
237
238static unsigned long pcpu_chunk_addr(struct pcpu_chunk *chunk,
239 unsigned int cpu, int page_idx)
240{
241 return (unsigned long)chunk->base_addr + pcpu_unit_offsets[cpu] +
242 (page_idx << PAGE_SHIFT);
243}
244
245static void __maybe_unused pcpu_next_unpop(struct pcpu_chunk *chunk,
246 int *rs, int *re, int end)
247{
248 *rs = find_next_zero_bit(chunk->populated, end, *rs);
249 *re = find_next_bit(chunk->populated, end, *rs + 1);
250}
251
252static void __maybe_unused pcpu_next_pop(struct pcpu_chunk *chunk,
253 int *rs, int *re, int end)
254{
255 *rs = find_next_bit(chunk->populated, end, *rs);
256 *re = find_next_zero_bit(chunk->populated, end, *rs + 1);
257}
258
259
260
261
262
263
264
265#define pcpu_for_each_unpop_region(chunk, rs, re, start, end) \
266 for ((rs) = (start), pcpu_next_unpop((chunk), &(rs), &(re), (end)); \
267 (rs) < (re); \
268 (rs) = (re) + 1, pcpu_next_unpop((chunk), &(rs), &(re), (end)))
269
270#define pcpu_for_each_pop_region(chunk, rs, re, start, end) \
271 for ((rs) = (start), pcpu_next_pop((chunk), &(rs), &(re), (end)); \
272 (rs) < (re); \
273 (rs) = (re) + 1, pcpu_next_pop((chunk), &(rs), &(re), (end)))
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289static void *pcpu_mem_alloc(size_t size)
290{
291 if (WARN_ON_ONCE(!slab_is_available()))
292 return NULL;
293
294 if (size <= PAGE_SIZE)
295 return kzalloc(size, GFP_KERNEL);
296 else
297 return vzalloc(size);
298}
299
300
301
302
303
304
305
306
307static void pcpu_mem_free(void *ptr, size_t size)
308{
309 if (size <= PAGE_SIZE)
310 kfree(ptr);
311 else
312 vfree(ptr);
313}
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
329{
330 int nslot = pcpu_chunk_slot(chunk);
331
332 if (chunk != pcpu_reserved_chunk && oslot != nslot) {
333 if (oslot < nslot)
334 list_move(&chunk->list, &pcpu_slot[nslot]);
335 else
336 list_move_tail(&chunk->list, &pcpu_slot[nslot]);
337 }
338}
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354static int pcpu_need_to_extend(struct pcpu_chunk *chunk)
355{
356 int new_alloc;
357
358 if (chunk->map_alloc >= chunk->map_used + 2)
359 return 0;
360
361 new_alloc = PCPU_DFL_MAP_ALLOC;
362 while (new_alloc < chunk->map_used + 2)
363 new_alloc *= 2;
364
365 return new_alloc;
366}
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381static int pcpu_extend_area_map(struct pcpu_chunk *chunk, int new_alloc)
382{
383 int *old = NULL, *new = NULL;
384 size_t old_size = 0, new_size = new_alloc * sizeof(new[0]);
385 unsigned long flags;
386
387 new = pcpu_mem_alloc(new_size);
388 if (!new)
389 return -ENOMEM;
390
391
392 spin_lock_irqsave(&pcpu_lock, flags);
393
394 if (new_alloc <= chunk->map_alloc)
395 goto out_unlock;
396
397 old_size = chunk->map_alloc * sizeof(chunk->map[0]);
398 old = chunk->map;
399
400 memcpy(new, old, old_size);
401
402 chunk->map_alloc = new_alloc;
403 chunk->map = new;
404 new = NULL;
405
406out_unlock:
407 spin_unlock_irqrestore(&pcpu_lock, flags);
408
409
410
411
412
413 pcpu_mem_free(old, old_size);
414 pcpu_mem_free(new, new_size);
415
416 return 0;
417}
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439static void pcpu_split_block(struct pcpu_chunk *chunk, int i,
440 int head, int tail)
441{
442 int nr_extra = !!head + !!tail;
443
444 BUG_ON(chunk->map_alloc < chunk->map_used + nr_extra);
445
446
447 memmove(&chunk->map[i + nr_extra], &chunk->map[i],
448 sizeof(chunk->map[0]) * (chunk->map_used - i));
449 chunk->map_used += nr_extra;
450
451 if (head) {
452 chunk->map[i + 1] = chunk->map[i] - head;
453 chunk->map[i++] = head;
454 }
455 if (tail) {
456 chunk->map[i++] -= tail;
457 chunk->map[i] = tail;
458 }
459}
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480static int pcpu_alloc_area(struct pcpu_chunk *chunk, int size, int align)
481{
482 int oslot = pcpu_chunk_slot(chunk);
483 int max_contig = 0;
484 int i, off;
485
486 for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++])) {
487 bool is_last = i + 1 == chunk->map_used;
488 int head, tail;
489
490
491 head = ALIGN(off, align) - off;
492 BUG_ON(i == 0 && head != 0);
493
494 if (chunk->map[i] < 0)
495 continue;
496 if (chunk->map[i] < head + size) {
497 max_contig = max(chunk->map[i], max_contig);
498 continue;
499 }
500
501
502
503
504
505
506
507 if (head && (head < sizeof(int) || chunk->map[i - 1] > 0)) {
508 if (chunk->map[i - 1] > 0)
509 chunk->map[i - 1] += head;
510 else {
511 chunk->map[i - 1] -= head;
512 chunk->free_size -= head;
513 }
514 chunk->map[i] -= head;
515 off += head;
516 head = 0;
517 }
518
519
520 tail = chunk->map[i] - head - size;
521 if (tail < sizeof(int))
522 tail = 0;
523
524
525 if (head || tail) {
526 pcpu_split_block(chunk, i, head, tail);
527 if (head) {
528 i++;
529 off += head;
530 max_contig = max(chunk->map[i - 1], max_contig);
531 }
532 if (tail)
533 max_contig = max(chunk->map[i + 1], max_contig);
534 }
535
536
537 if (is_last)
538 chunk->contig_hint = max_contig;
539 else
540 chunk->contig_hint = max(chunk->contig_hint,
541 max_contig);
542
543 chunk->free_size -= chunk->map[i];
544 chunk->map[i] = -chunk->map[i];
545
546 pcpu_chunk_relocate(chunk, oslot);
547 return off;
548 }
549
550 chunk->contig_hint = max_contig;
551 pcpu_chunk_relocate(chunk, oslot);
552
553
554 return -1;
555}
556
557
558
559
560
561
562
563
564
565
566
567
568
569static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme)
570{
571 int oslot = pcpu_chunk_slot(chunk);
572 int i, off;
573
574 for (i = 0, off = 0; i < chunk->map_used; off += abs(chunk->map[i++]))
575 if (off == freeme)
576 break;
577 BUG_ON(off != freeme);
578 BUG_ON(chunk->map[i] > 0);
579
580 chunk->map[i] = -chunk->map[i];
581 chunk->free_size += chunk->map[i];
582
583
584 if (i > 0 && chunk->map[i - 1] >= 0) {
585 chunk->map[i - 1] += chunk->map[i];
586 chunk->map_used--;
587 memmove(&chunk->map[i], &chunk->map[i + 1],
588 (chunk->map_used - i) * sizeof(chunk->map[0]));
589 i--;
590 }
591
592 if (i + 1 < chunk->map_used && chunk->map[i + 1] >= 0) {
593 chunk->map[i] += chunk->map[i + 1];
594 chunk->map_used--;
595 memmove(&chunk->map[i + 1], &chunk->map[i + 2],
596 (chunk->map_used - (i + 1)) * sizeof(chunk->map[0]));
597 }
598
599 chunk->contig_hint = max(chunk->map[i], chunk->contig_hint);
600 pcpu_chunk_relocate(chunk, oslot);
601}
602
603static struct pcpu_chunk *pcpu_alloc_chunk(void)
604{
605 struct pcpu_chunk *chunk;
606
607 chunk = pcpu_mem_alloc(pcpu_chunk_struct_size);
608 if (!chunk)
609 return NULL;
610
611 chunk->map = pcpu_mem_alloc(PCPU_DFL_MAP_ALLOC * sizeof(chunk->map[0]));
612 if (!chunk->map) {
613 kfree(chunk);
614 return NULL;
615 }
616
617 chunk->map_alloc = PCPU_DFL_MAP_ALLOC;
618 chunk->map[chunk->map_used++] = pcpu_unit_size;
619
620 INIT_LIST_HEAD(&chunk->list);
621 chunk->free_size = pcpu_unit_size;
622 chunk->contig_hint = pcpu_unit_size;
623
624 return chunk;
625}
626
627static void pcpu_free_chunk(struct pcpu_chunk *chunk)
628{
629 if (!chunk)
630 return;
631 pcpu_mem_free(chunk->map, chunk->map_alloc * sizeof(chunk->map[0]));
632 kfree(chunk);
633}
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size);
651static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size);
652static struct pcpu_chunk *pcpu_create_chunk(void);
653static void pcpu_destroy_chunk(struct pcpu_chunk *chunk);
654static struct page *pcpu_addr_to_page(void *addr);
655static int __init pcpu_verify_alloc_info(const struct pcpu_alloc_info *ai);
656
657#ifdef CONFIG_NEED_PER_CPU_KM
658#include "percpu-km.c"
659#else
660#include "percpu-vm.c"
661#endif
662
663
664
665
666
667
668
669
670static struct pcpu_chunk *pcpu_chunk_addr_search(void *addr)
671{
672
673 if (pcpu_addr_in_first_chunk(addr)) {
674
675 if (pcpu_addr_in_reserved_chunk(addr))
676 return pcpu_reserved_chunk;
677 return pcpu_first_chunk;
678 }
679
680
681
682
683
684
685
686
687 addr += pcpu_unit_offsets[raw_smp_processor_id()];
688 return pcpu_get_page_chunk(pcpu_addr_to_page(addr));
689}
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705static void __percpu *pcpu_alloc(size_t size, size_t align, bool reserved)
706{
707 static int warn_limit = 10;
708 struct pcpu_chunk *chunk;
709 const char *err;
710 int slot, off, new_alloc;
711 unsigned long flags;
712
713 if (unlikely(!size || size > PCPU_MIN_UNIT_SIZE || align > PAGE_SIZE)) {
714 WARN(true, "illegal size (%zu) or align (%zu) for "
715 "percpu allocation\n", size, align);
716 return NULL;
717 }
718
719 mutex_lock(&pcpu_alloc_mutex);
720 spin_lock_irqsave(&pcpu_lock, flags);
721
722
723 if (reserved && pcpu_reserved_chunk) {
724 chunk = pcpu_reserved_chunk;
725
726 if (size > chunk->contig_hint) {
727 err = "alloc from reserved chunk failed";
728 goto fail_unlock;
729 }
730
731 while ((new_alloc = pcpu_need_to_extend(chunk))) {
732 spin_unlock_irqrestore(&pcpu_lock, flags);
733 if (pcpu_extend_area_map(chunk, new_alloc) < 0) {
734 err = "failed to extend area map of reserved chunk";
735 goto fail_unlock_mutex;
736 }
737 spin_lock_irqsave(&pcpu_lock, flags);
738 }
739
740 off = pcpu_alloc_area(chunk, size, align);
741 if (off >= 0)
742 goto area_found;
743
744 err = "alloc from reserved chunk failed";
745 goto fail_unlock;
746 }
747
748restart:
749
750 for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) {
751 list_for_each_entry(chunk, &pcpu_slot[slot], list) {
752 if (size > chunk->contig_hint)
753 continue;
754
755 new_alloc = pcpu_need_to_extend(chunk);
756 if (new_alloc) {
757 spin_unlock_irqrestore(&pcpu_lock, flags);
758 if (pcpu_extend_area_map(chunk,
759 new_alloc) < 0) {
760 err = "failed to extend area map";
761 goto fail_unlock_mutex;
762 }
763 spin_lock_irqsave(&pcpu_lock, flags);
764
765
766
767
768 goto restart;
769 }
770
771 off = pcpu_alloc_area(chunk, size, align);
772 if (off >= 0)
773 goto area_found;
774 }
775 }
776
777
778 spin_unlock_irqrestore(&pcpu_lock, flags);
779
780 chunk = pcpu_create_chunk();
781 if (!chunk) {
782 err = "failed to allocate new chunk";
783 goto fail_unlock_mutex;
784 }
785
786 spin_lock_irqsave(&pcpu_lock, flags);
787 pcpu_chunk_relocate(chunk, -1);
788 goto restart;
789
790area_found:
791 spin_unlock_irqrestore(&pcpu_lock, flags);
792
793
794 if (pcpu_populate_chunk(chunk, off, size)) {
795 spin_lock_irqsave(&pcpu_lock, flags);
796 pcpu_free_area(chunk, off);
797 err = "failed to populate";
798 goto fail_unlock;
799 }
800
801 mutex_unlock(&pcpu_alloc_mutex);
802
803
804 return __addr_to_pcpu_ptr(chunk->base_addr + off);
805
806fail_unlock:
807 spin_unlock_irqrestore(&pcpu_lock, flags);
808fail_unlock_mutex:
809 mutex_unlock(&pcpu_alloc_mutex);
810 if (warn_limit) {
811 pr_warning("PERCPU: allocation failed, size=%zu align=%zu, "
812 "%s\n", size, align, err);
813 dump_stack();
814 if (!--warn_limit)
815 pr_info("PERCPU: limit reached, disable warning\n");
816 }
817 return NULL;
818}
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834void __percpu *__alloc_percpu(size_t size, size_t align)
835{
836 return pcpu_alloc(size, align, false);
837}
838EXPORT_SYMBOL_GPL(__alloc_percpu);
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856void __percpu *__alloc_reserved_percpu(size_t size, size_t align)
857{
858 return pcpu_alloc(size, align, true);
859}
860
861
862
863
864
865
866
867
868
869
870static void pcpu_reclaim(struct work_struct *work)
871{
872 LIST_HEAD(todo);
873 struct list_head *head = &pcpu_slot[pcpu_nr_slots - 1];
874 struct pcpu_chunk *chunk, *next;
875
876 mutex_lock(&pcpu_alloc_mutex);
877 spin_lock_irq(&pcpu_lock);
878
879 list_for_each_entry_safe(chunk, next, head, list) {
880 WARN_ON(chunk->immutable);
881
882
883 if (chunk == list_first_entry(head, struct pcpu_chunk, list))
884 continue;
885
886 list_move(&chunk->list, &todo);
887 }
888
889 spin_unlock_irq(&pcpu_lock);
890
891 list_for_each_entry_safe(chunk, next, &todo, list) {
892 pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size);
893 pcpu_destroy_chunk(chunk);
894 }
895
896 mutex_unlock(&pcpu_alloc_mutex);
897}
898
899
900
901
902
903
904
905
906
907
908void free_percpu(void __percpu *ptr)
909{
910 void *addr;
911 struct pcpu_chunk *chunk;
912 unsigned long flags;
913 int off;
914
915 if (!ptr)
916 return;
917
918 addr = __pcpu_ptr_to_addr(ptr);
919
920 spin_lock_irqsave(&pcpu_lock, flags);
921
922 chunk = pcpu_chunk_addr_search(addr);
923 off = addr - chunk->base_addr;
924
925 pcpu_free_area(chunk, off);
926
927
928 if (chunk->free_size == pcpu_unit_size) {
929 struct pcpu_chunk *pos;
930
931 list_for_each_entry(pos, &pcpu_slot[pcpu_nr_slots - 1], list)
932 if (pos != chunk) {
933 schedule_work(&pcpu_reclaim_work);
934 break;
935 }
936 }
937
938 spin_unlock_irqrestore(&pcpu_lock, flags);
939}
940EXPORT_SYMBOL_GPL(free_percpu);
941
942
943
944
945
946
947
948
949
950
951
952
953bool is_kernel_percpu_address(unsigned long addr)
954{
955#ifdef CONFIG_SMP
956 const size_t static_size = __per_cpu_end - __per_cpu_start;
957 void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr);
958 unsigned int cpu;
959
960 for_each_possible_cpu(cpu) {
961 void *start = per_cpu_ptr(base, cpu);
962
963 if ((void *)addr >= start && (void *)addr < start + static_size)
964 return true;
965 }
966#endif
967
968 return false;
969}
970
971
972
973
974
975
976
977
978
979
980
981
982
983phys_addr_t per_cpu_ptr_to_phys(void *addr)
984{
985 void __percpu *base = __addr_to_pcpu_ptr(pcpu_base_addr);
986 bool in_first_chunk = false;
987 unsigned long first_start, first_end;
988 unsigned int cpu;
989
990
991
992
993
994
995 first_start = pcpu_chunk_addr(pcpu_first_chunk, pcpu_first_unit_cpu, 0);
996 first_end = pcpu_chunk_addr(pcpu_first_chunk, pcpu_last_unit_cpu,
997 pcpu_unit_pages);
998 if ((unsigned long)addr >= first_start &&
999 (unsigned long)addr < first_end) {
1000 for_each_possible_cpu(cpu) {
1001 void *start = per_cpu_ptr(base, cpu);
1002
1003 if (addr >= start && addr < start + pcpu_unit_size) {
1004 in_first_chunk = true;
1005 break;
1006 }
1007 }
1008 }
1009
1010 if (in_first_chunk) {
1011 if ((unsigned long)addr < VMALLOC_START ||
1012 (unsigned long)addr >= VMALLOC_END)
1013 return __pa(addr);
1014 else
1015 return page_to_phys(vmalloc_to_page(addr));
1016 } else
1017 return page_to_phys(pcpu_addr_to_page(addr));
1018}
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups,
1036 int nr_units)
1037{
1038 struct pcpu_alloc_info *ai;
1039 size_t base_size, ai_size;
1040 void *ptr;
1041 int unit;
1042
1043 base_size = ALIGN(sizeof(*ai) + nr_groups * sizeof(ai->groups[0]),
1044 __alignof__(ai->groups[0].cpu_map[0]));
1045 ai_size = base_size + nr_units * sizeof(ai->groups[0].cpu_map[0]);
1046
1047 ptr = alloc_bootmem_nopanic(PFN_ALIGN(ai_size));
1048 if (!ptr)
1049 return NULL;
1050 ai = ptr;
1051 ptr += base_size;
1052
1053 ai->groups[0].cpu_map = ptr;
1054
1055 for (unit = 0; unit < nr_units; unit++)
1056 ai->groups[0].cpu_map[unit] = NR_CPUS;
1057
1058 ai->nr_groups = nr_groups;
1059 ai->__ai_size = PFN_ALIGN(ai_size);
1060
1061 return ai;
1062}
1063
1064
1065
1066
1067
1068
1069
1070void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai)
1071{
1072 free_bootmem(__pa(ai), ai->__ai_size);
1073}
1074
1075
1076
1077
1078
1079
1080
1081
1082static void pcpu_dump_alloc_info(const char *lvl,
1083 const struct pcpu_alloc_info *ai)
1084{
1085 int group_width = 1, cpu_width = 1, width;
1086 char empty_str[] = "--------";
1087 int alloc = 0, alloc_end = 0;
1088 int group, v;
1089 int upa, apl;
1090
1091 v = ai->nr_groups;
1092 while (v /= 10)
1093 group_width++;
1094
1095 v = num_possible_cpus();
1096 while (v /= 10)
1097 cpu_width++;
1098 empty_str[min_t(int, cpu_width, sizeof(empty_str) - 1)] = '\0';
1099
1100 upa = ai->alloc_size / ai->unit_size;
1101 width = upa * (cpu_width + 1) + group_width + 3;
1102 apl = rounddown_pow_of_two(max(60 / width, 1));
1103
1104 printk("%spcpu-alloc: s%zu r%zu d%zu u%zu alloc=%zu*%zu",
1105 lvl, ai->static_size, ai->reserved_size, ai->dyn_size,
1106 ai->unit_size, ai->alloc_size / ai->atom_size, ai->atom_size);
1107
1108 for (group = 0; group < ai->nr_groups; group++) {
1109 const struct pcpu_group_info *gi = &ai->groups[group];
1110 int unit = 0, unit_end = 0;
1111
1112 BUG_ON(gi->nr_units % upa);
1113 for (alloc_end += gi->nr_units / upa;
1114 alloc < alloc_end; alloc++) {
1115 if (!(alloc % apl)) {
1116 printk("\n");
1117 printk("%spcpu-alloc: ", lvl);
1118 }
1119 printk("[%0*d] ", group_width, group);
1120
1121 for (unit_end += upa; unit < unit_end; unit++)
1122 if (gi->cpu_map[unit] != NR_CPUS)
1123 printk("%0*d ", cpu_width,
1124 gi->cpu_map[unit]);
1125 else
1126 printk("%s ", empty_str);
1127 }
1128 }
1129 printk("\n");
1130}
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
1189 void *base_addr)
1190{
1191 static char cpus_buf[4096] __initdata;
1192 static int smap[PERCPU_DYNAMIC_EARLY_SLOTS] __initdata;
1193 static int dmap[PERCPU_DYNAMIC_EARLY_SLOTS] __initdata;
1194 size_t dyn_size = ai->dyn_size;
1195 size_t size_sum = ai->static_size + ai->reserved_size + dyn_size;
1196 struct pcpu_chunk *schunk, *dchunk = NULL;
1197 unsigned long *group_offsets;
1198 size_t *group_sizes;
1199 unsigned long *unit_off;
1200 unsigned int cpu;
1201 int *unit_map;
1202 int group, unit, i;
1203
1204 cpumask_scnprintf(cpus_buf, sizeof(cpus_buf), cpu_possible_mask);
1205
1206#define PCPU_SETUP_BUG_ON(cond) do { \
1207 if (unlikely(cond)) { \
1208 pr_emerg("PERCPU: failed to initialize, %s", #cond); \
1209 pr_emerg("PERCPU: cpu_possible_mask=%s\n", cpus_buf); \
1210 pcpu_dump_alloc_info(KERN_EMERG, ai); \
1211 BUG(); \
1212 } \
1213} while (0)
1214
1215
1216 PCPU_SETUP_BUG_ON(ai->nr_groups <= 0);
1217#ifdef CONFIG_SMP
1218 PCPU_SETUP_BUG_ON(!ai->static_size);
1219#endif
1220 PCPU_SETUP_BUG_ON(!base_addr);
1221 PCPU_SETUP_BUG_ON(ai->unit_size < size_sum);
1222 PCPU_SETUP_BUG_ON(ai->unit_size & ~PAGE_MASK);
1223 PCPU_SETUP_BUG_ON(ai->unit_size < PCPU_MIN_UNIT_SIZE);
1224 PCPU_SETUP_BUG_ON(ai->dyn_size < PERCPU_DYNAMIC_EARLY_SIZE);
1225 PCPU_SETUP_BUG_ON(pcpu_verify_alloc_info(ai) < 0);
1226
1227
1228 group_offsets = alloc_bootmem(ai->nr_groups * sizeof(group_offsets[0]));
1229 group_sizes = alloc_bootmem(ai->nr_groups * sizeof(group_sizes[0]));
1230 unit_map = alloc_bootmem(nr_cpu_ids * sizeof(unit_map[0]));
1231 unit_off = alloc_bootmem(nr_cpu_ids * sizeof(unit_off[0]));
1232
1233 for (cpu = 0; cpu < nr_cpu_ids; cpu++)
1234 unit_map[cpu] = UINT_MAX;
1235 pcpu_first_unit_cpu = NR_CPUS;
1236
1237 for (group = 0, unit = 0; group < ai->nr_groups; group++, unit += i) {
1238 const struct pcpu_group_info *gi = &ai->groups[group];
1239
1240 group_offsets[group] = gi->base_offset;
1241 group_sizes[group] = gi->nr_units * ai->unit_size;
1242
1243 for (i = 0; i < gi->nr_units; i++) {
1244 cpu = gi->cpu_map[i];
1245 if (cpu == NR_CPUS)
1246 continue;
1247
1248 PCPU_SETUP_BUG_ON(cpu > nr_cpu_ids);
1249 PCPU_SETUP_BUG_ON(!cpu_possible(cpu));
1250 PCPU_SETUP_BUG_ON(unit_map[cpu] != UINT_MAX);
1251
1252 unit_map[cpu] = unit + i;
1253 unit_off[cpu] = gi->base_offset + i * ai->unit_size;
1254
1255 if (pcpu_first_unit_cpu == NR_CPUS)
1256 pcpu_first_unit_cpu = cpu;
1257 pcpu_last_unit_cpu = cpu;
1258 }
1259 }
1260 pcpu_nr_units = unit;
1261
1262 for_each_possible_cpu(cpu)
1263 PCPU_SETUP_BUG_ON(unit_map[cpu] == UINT_MAX);
1264
1265
1266#undef PCPU_SETUP_BUG_ON
1267 pcpu_dump_alloc_info(KERN_DEBUG, ai);
1268
1269 pcpu_nr_groups = ai->nr_groups;
1270 pcpu_group_offsets = group_offsets;
1271 pcpu_group_sizes = group_sizes;
1272 pcpu_unit_map = unit_map;
1273 pcpu_unit_offsets = unit_off;
1274
1275
1276 pcpu_unit_pages = ai->unit_size >> PAGE_SHIFT;
1277 pcpu_unit_size = pcpu_unit_pages << PAGE_SHIFT;
1278 pcpu_atom_size = ai->atom_size;
1279 pcpu_chunk_struct_size = sizeof(struct pcpu_chunk) +
1280 BITS_TO_LONGS(pcpu_unit_pages) * sizeof(unsigned long);
1281
1282
1283
1284
1285
1286 pcpu_nr_slots = __pcpu_size_to_slot(pcpu_unit_size) + 2;
1287 pcpu_slot = alloc_bootmem(pcpu_nr_slots * sizeof(pcpu_slot[0]));
1288 for (i = 0; i < pcpu_nr_slots; i++)
1289 INIT_LIST_HEAD(&pcpu_slot[i]);
1290
1291
1292
1293
1294
1295
1296
1297
1298 schunk = alloc_bootmem(pcpu_chunk_struct_size);
1299 INIT_LIST_HEAD(&schunk->list);
1300 schunk->base_addr = base_addr;
1301 schunk->map = smap;
1302 schunk->map_alloc = ARRAY_SIZE(smap);
1303 schunk->immutable = true;
1304 bitmap_fill(schunk->populated, pcpu_unit_pages);
1305
1306 if (ai->reserved_size) {
1307 schunk->free_size = ai->reserved_size;
1308 pcpu_reserved_chunk = schunk;
1309 pcpu_reserved_chunk_limit = ai->static_size + ai->reserved_size;
1310 } else {
1311 schunk->free_size = dyn_size;
1312 dyn_size = 0;
1313 }
1314 schunk->contig_hint = schunk->free_size;
1315
1316 schunk->map[schunk->map_used++] = -ai->static_size;
1317 if (schunk->free_size)
1318 schunk->map[schunk->map_used++] = schunk->free_size;
1319
1320
1321 if (dyn_size) {
1322 dchunk = alloc_bootmem(pcpu_chunk_struct_size);
1323 INIT_LIST_HEAD(&dchunk->list);
1324 dchunk->base_addr = base_addr;
1325 dchunk->map = dmap;
1326 dchunk->map_alloc = ARRAY_SIZE(dmap);
1327 dchunk->immutable = true;
1328 bitmap_fill(dchunk->populated, pcpu_unit_pages);
1329
1330 dchunk->contig_hint = dchunk->free_size = dyn_size;
1331 dchunk->map[dchunk->map_used++] = -pcpu_reserved_chunk_limit;
1332 dchunk->map[dchunk->map_used++] = dchunk->free_size;
1333 }
1334
1335
1336 pcpu_first_chunk = dchunk ?: schunk;
1337 pcpu_chunk_relocate(pcpu_first_chunk, -1);
1338
1339
1340 pcpu_base_addr = base_addr;
1341 return 0;
1342}
1343
1344#ifdef CONFIG_SMP
1345
1346const char *pcpu_fc_names[PCPU_FC_NR] __initdata = {
1347 [PCPU_FC_AUTO] = "auto",
1348 [PCPU_FC_EMBED] = "embed",
1349 [PCPU_FC_PAGE] = "page",
1350};
1351
1352enum pcpu_fc pcpu_chosen_fc __initdata = PCPU_FC_AUTO;
1353
1354static int __init percpu_alloc_setup(char *str)
1355{
1356 if (0)
1357 ;
1358#ifdef CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK
1359 else if (!strcmp(str, "embed"))
1360 pcpu_chosen_fc = PCPU_FC_EMBED;
1361#endif
1362#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
1363 else if (!strcmp(str, "page"))
1364 pcpu_chosen_fc = PCPU_FC_PAGE;
1365#endif
1366 else
1367 pr_warning("PERCPU: unknown allocator %s specified\n", str);
1368
1369 return 0;
1370}
1371early_param("percpu_alloc", percpu_alloc_setup);
1372
1373
1374
1375
1376
1377
1378#if defined(CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK) || \
1379 !defined(CONFIG_HAVE_SETUP_PER_CPU_AREA)
1380#define BUILD_EMBED_FIRST_CHUNK
1381#endif
1382
1383
1384#if defined(CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK)
1385#define BUILD_PAGE_FIRST_CHUNK
1386#endif
1387
1388
1389#if defined(BUILD_EMBED_FIRST_CHUNK) || defined(BUILD_PAGE_FIRST_CHUNK)
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411static struct pcpu_alloc_info * __init pcpu_build_alloc_info(
1412 size_t reserved_size, size_t dyn_size,
1413 size_t atom_size,
1414 pcpu_fc_cpu_distance_fn_t cpu_distance_fn)
1415{
1416 static int group_map[NR_CPUS] __initdata;
1417 static int group_cnt[NR_CPUS] __initdata;
1418 const size_t static_size = __per_cpu_end - __per_cpu_start;
1419 int nr_groups = 1, nr_units = 0;
1420 size_t size_sum, min_unit_size, alloc_size;
1421 int upa, max_upa, uninitialized_var(best_upa);
1422 int last_allocs, group, unit;
1423 unsigned int cpu, tcpu;
1424 struct pcpu_alloc_info *ai;
1425 unsigned int *cpu_map;
1426
1427
1428 memset(group_map, 0, sizeof(group_map));
1429 memset(group_cnt, 0, sizeof(group_cnt));
1430
1431
1432 size_sum = PFN_ALIGN(static_size + reserved_size +
1433 max_t(size_t, dyn_size, PERCPU_DYNAMIC_EARLY_SIZE));
1434 dyn_size = size_sum - static_size - reserved_size;
1435
1436
1437
1438
1439
1440
1441
1442 min_unit_size = max_t(size_t, size_sum, PCPU_MIN_UNIT_SIZE);
1443
1444 alloc_size = roundup(min_unit_size, atom_size);
1445 upa = alloc_size / min_unit_size;
1446 while (alloc_size % upa || ((alloc_size / upa) & ~PAGE_MASK))
1447 upa--;
1448 max_upa = upa;
1449
1450
1451 for_each_possible_cpu(cpu) {
1452 group = 0;
1453 next_group:
1454 for_each_possible_cpu(tcpu) {
1455 if (cpu == tcpu)
1456 break;
1457 if (group_map[tcpu] == group && cpu_distance_fn &&
1458 (cpu_distance_fn(cpu, tcpu) > LOCAL_DISTANCE ||
1459 cpu_distance_fn(tcpu, cpu) > LOCAL_DISTANCE)) {
1460 group++;
1461 nr_groups = max(nr_groups, group + 1);
1462 goto next_group;
1463 }
1464 }
1465 group_map[cpu] = group;
1466 group_cnt[group]++;
1467 }
1468
1469
1470
1471
1472
1473
1474 last_allocs = INT_MAX;
1475 for (upa = max_upa; upa; upa--) {
1476 int allocs = 0, wasted = 0;
1477
1478 if (alloc_size % upa || ((alloc_size / upa) & ~PAGE_MASK))
1479 continue;
1480
1481 for (group = 0; group < nr_groups; group++) {
1482 int this_allocs = DIV_ROUND_UP(group_cnt[group], upa);
1483 allocs += this_allocs;
1484 wasted += this_allocs * upa - group_cnt[group];
1485 }
1486
1487
1488
1489
1490
1491
1492 if (wasted > num_possible_cpus() / 3)
1493 continue;
1494
1495
1496 if (allocs > last_allocs)
1497 break;
1498 last_allocs = allocs;
1499 best_upa = upa;
1500 }
1501 upa = best_upa;
1502
1503
1504 for (group = 0; group < nr_groups; group++)
1505 nr_units += roundup(group_cnt[group], upa);
1506
1507 ai = pcpu_alloc_alloc_info(nr_groups, nr_units);
1508 if (!ai)
1509 return ERR_PTR(-ENOMEM);
1510 cpu_map = ai->groups[0].cpu_map;
1511
1512 for (group = 0; group < nr_groups; group++) {
1513 ai->groups[group].cpu_map = cpu_map;
1514 cpu_map += roundup(group_cnt[group], upa);
1515 }
1516
1517 ai->static_size = static_size;
1518 ai->reserved_size = reserved_size;
1519 ai->dyn_size = dyn_size;
1520 ai->unit_size = alloc_size / upa;
1521 ai->atom_size = atom_size;
1522 ai->alloc_size = alloc_size;
1523
1524 for (group = 0, unit = 0; group_cnt[group]; group++) {
1525 struct pcpu_group_info *gi = &ai->groups[group];
1526
1527
1528
1529
1530
1531
1532 gi->base_offset = unit * ai->unit_size;
1533
1534 for_each_possible_cpu(cpu)
1535 if (group_map[cpu] == group)
1536 gi->cpu_map[gi->nr_units++] = cpu;
1537 gi->nr_units = roundup(gi->nr_units, upa);
1538 unit += gi->nr_units;
1539 }
1540 BUG_ON(unit != nr_units);
1541
1542 return ai;
1543}
1544#endif
1545
1546#if defined(BUILD_EMBED_FIRST_CHUNK)
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579int __init pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size,
1580 size_t atom_size,
1581 pcpu_fc_cpu_distance_fn_t cpu_distance_fn,
1582 pcpu_fc_alloc_fn_t alloc_fn,
1583 pcpu_fc_free_fn_t free_fn)
1584{
1585 void *base = (void *)ULONG_MAX;
1586 void **areas = NULL;
1587 struct pcpu_alloc_info *ai;
1588 size_t size_sum, areas_size, max_distance;
1589 int group, i, rc;
1590
1591 ai = pcpu_build_alloc_info(reserved_size, dyn_size, atom_size,
1592 cpu_distance_fn);
1593 if (IS_ERR(ai))
1594 return PTR_ERR(ai);
1595
1596 size_sum = ai->static_size + ai->reserved_size + ai->dyn_size;
1597 areas_size = PFN_ALIGN(ai->nr_groups * sizeof(void *));
1598
1599 areas = alloc_bootmem_nopanic(areas_size);
1600 if (!areas) {
1601 rc = -ENOMEM;
1602 goto out_free;
1603 }
1604
1605
1606 for (group = 0; group < ai->nr_groups; group++) {
1607 struct pcpu_group_info *gi = &ai->groups[group];
1608 unsigned int cpu = NR_CPUS;
1609 void *ptr;
1610
1611 for (i = 0; i < gi->nr_units && cpu == NR_CPUS; i++)
1612 cpu = gi->cpu_map[i];
1613 BUG_ON(cpu == NR_CPUS);
1614
1615
1616 ptr = alloc_fn(cpu, gi->nr_units * ai->unit_size, atom_size);
1617 if (!ptr) {
1618 rc = -ENOMEM;
1619 goto out_free_areas;
1620 }
1621 areas[group] = ptr;
1622
1623 base = min(ptr, base);
1624
1625 for (i = 0; i < gi->nr_units; i++, ptr += ai->unit_size) {
1626 if (gi->cpu_map[i] == NR_CPUS) {
1627
1628 free_fn(ptr, ai->unit_size);
1629 continue;
1630 }
1631
1632 memcpy(ptr, __per_cpu_load, ai->static_size);
1633 free_fn(ptr + size_sum, ai->unit_size - size_sum);
1634 }
1635 }
1636
1637
1638 max_distance = 0;
1639 for (group = 0; group < ai->nr_groups; group++) {
1640 ai->groups[group].base_offset = areas[group] - base;
1641 max_distance = max_t(size_t, max_distance,
1642 ai->groups[group].base_offset);
1643 }
1644 max_distance += ai->unit_size;
1645
1646
1647 if (max_distance > (VMALLOC_END - VMALLOC_START) * 3 / 4) {
1648 pr_warning("PERCPU: max_distance=0x%zx too large for vmalloc "
1649 "space 0x%lx\n",
1650 max_distance, VMALLOC_END - VMALLOC_START);
1651#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
1652
1653 rc = -EINVAL;
1654 goto out_free;
1655#endif
1656 }
1657
1658 pr_info("PERCPU: Embedded %zu pages/cpu @%p s%zu r%zu d%zu u%zu\n",
1659 PFN_DOWN(size_sum), base, ai->static_size, ai->reserved_size,
1660 ai->dyn_size, ai->unit_size);
1661
1662 rc = pcpu_setup_first_chunk(ai, base);
1663 goto out_free;
1664
1665out_free_areas:
1666 for (group = 0; group < ai->nr_groups; group++)
1667 free_fn(areas[group],
1668 ai->groups[group].nr_units * ai->unit_size);
1669out_free:
1670 pcpu_free_alloc_info(ai);
1671 if (areas)
1672 free_bootmem(__pa(areas), areas_size);
1673 return rc;
1674}
1675#endif
1676
1677#ifdef BUILD_PAGE_FIRST_CHUNK
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694int __init pcpu_page_first_chunk(size_t reserved_size,
1695 pcpu_fc_alloc_fn_t alloc_fn,
1696 pcpu_fc_free_fn_t free_fn,
1697 pcpu_fc_populate_pte_fn_t populate_pte_fn)
1698{
1699 static struct vm_struct vm;
1700 struct pcpu_alloc_info *ai;
1701 char psize_str[16];
1702 int unit_pages;
1703 size_t pages_size;
1704 struct page **pages;
1705 int unit, i, j, rc;
1706
1707 snprintf(psize_str, sizeof(psize_str), "%luK", PAGE_SIZE >> 10);
1708
1709 ai = pcpu_build_alloc_info(reserved_size, 0, PAGE_SIZE, NULL);
1710 if (IS_ERR(ai))
1711 return PTR_ERR(ai);
1712 BUG_ON(ai->nr_groups != 1);
1713 BUG_ON(ai->groups[0].nr_units != num_possible_cpus());
1714
1715 unit_pages = ai->unit_size >> PAGE_SHIFT;
1716
1717
1718 pages_size = PFN_ALIGN(unit_pages * num_possible_cpus() *
1719 sizeof(pages[0]));
1720 pages = alloc_bootmem(pages_size);
1721
1722
1723 j = 0;
1724 for (unit = 0; unit < num_possible_cpus(); unit++)
1725 for (i = 0; i < unit_pages; i++) {
1726 unsigned int cpu = ai->groups[0].cpu_map[unit];
1727 void *ptr;
1728
1729 ptr = alloc_fn(cpu, PAGE_SIZE, PAGE_SIZE);
1730 if (!ptr) {
1731 pr_warning("PERCPU: failed to allocate %s page "
1732 "for cpu%u\n", psize_str, cpu);
1733 goto enomem;
1734 }
1735 pages[j++] = virt_to_page(ptr);
1736 }
1737
1738
1739 vm.flags = VM_ALLOC;
1740 vm.size = num_possible_cpus() * ai->unit_size;
1741 vm_area_register_early(&vm, PAGE_SIZE);
1742
1743 for (unit = 0; unit < num_possible_cpus(); unit++) {
1744 unsigned long unit_addr =
1745 (unsigned long)vm.addr + unit * ai->unit_size;
1746
1747 for (i = 0; i < unit_pages; i++)
1748 populate_pte_fn(unit_addr + (i << PAGE_SHIFT));
1749
1750
1751 rc = __pcpu_map_pages(unit_addr, &pages[unit * unit_pages],
1752 unit_pages);
1753 if (rc < 0)
1754 panic("failed to map percpu area, err=%d\n", rc);
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765 memcpy((void *)unit_addr, __per_cpu_load, ai->static_size);
1766 }
1767
1768
1769 pr_info("PERCPU: %d %s pages/cpu @%p s%zu r%zu d%zu\n",
1770 unit_pages, psize_str, vm.addr, ai->static_size,
1771 ai->reserved_size, ai->dyn_size);
1772
1773 rc = pcpu_setup_first_chunk(ai, vm.addr);
1774 goto out_free_ar;
1775
1776enomem:
1777 while (--j >= 0)
1778 free_fn(page_address(pages[j]), PAGE_SIZE);
1779 rc = -ENOMEM;
1780out_free_ar:
1781 free_bootmem(__pa(pages), pages_size);
1782 pcpu_free_alloc_info(ai);
1783 return rc;
1784}
1785#endif
1786
1787#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
1801EXPORT_SYMBOL(__per_cpu_offset);
1802
1803static void * __init pcpu_dfl_fc_alloc(unsigned int cpu, size_t size,
1804 size_t align)
1805{
1806 return __alloc_bootmem_nopanic(size, align, __pa(MAX_DMA_ADDRESS));
1807}
1808
1809static void __init pcpu_dfl_fc_free(void *ptr, size_t size)
1810{
1811 free_bootmem(__pa(ptr), size);
1812}
1813
1814void __init setup_per_cpu_areas(void)
1815{
1816 unsigned long delta;
1817 unsigned int cpu;
1818 int rc;
1819
1820
1821
1822
1823
1824 rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
1825 PERCPU_DYNAMIC_RESERVE, PAGE_SIZE, NULL,
1826 pcpu_dfl_fc_alloc, pcpu_dfl_fc_free);
1827 if (rc < 0)
1828 panic("Failed to initialize percpu areas.");
1829
1830 delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
1831 for_each_possible_cpu(cpu)
1832 __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
1833}
1834#endif
1835
1836#else
1837
1838
1839
1840
1841
1842
1843
1844
1845void __init setup_per_cpu_areas(void)
1846{
1847 const size_t unit_size =
1848 roundup_pow_of_two(max_t(size_t, PCPU_MIN_UNIT_SIZE,
1849 PERCPU_DYNAMIC_RESERVE));
1850 struct pcpu_alloc_info *ai;
1851 void *fc;
1852
1853 ai = pcpu_alloc_alloc_info(1, 1);
1854 fc = __alloc_bootmem(unit_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
1855 if (!ai || !fc)
1856 panic("Failed to allocate memory for percpu areas.");
1857
1858 ai->dyn_size = unit_size;
1859 ai->unit_size = unit_size;
1860 ai->atom_size = unit_size;
1861 ai->alloc_size = unit_size;
1862 ai->groups[0].nr_units = 1;
1863 ai->groups[0].cpu_map[0] = 0;
1864
1865 if (pcpu_setup_first_chunk(ai, fc) < 0)
1866 panic("Failed to initialize percpu areas.");
1867}
1868
1869#endif
1870
1871
1872
1873
1874
1875
1876
1877void __init percpu_init_late(void)
1878{
1879 struct pcpu_chunk *target_chunks[] =
1880 { pcpu_first_chunk, pcpu_reserved_chunk, NULL };
1881 struct pcpu_chunk *chunk;
1882 unsigned long flags;
1883 int i;
1884
1885 for (i = 0; (chunk = target_chunks[i]); i++) {
1886 int *map;
1887 const size_t size = PERCPU_DYNAMIC_EARLY_SLOTS * sizeof(map[0]);
1888
1889 BUILD_BUG_ON(size > PAGE_SIZE);
1890
1891 map = pcpu_mem_alloc(size);
1892 BUG_ON(!map);
1893
1894 spin_lock_irqsave(&pcpu_lock, flags);
1895 memcpy(map, chunk->map, size);
1896 chunk->map = map;
1897 spin_unlock_irqrestore(&pcpu_lock, flags);
1898 }
1899}
1900