1
2#ifndef __LINUX_CPUMASK_H
3#define __LINUX_CPUMASK_H
4
5
6
7
8
9
10#include <linux/kernel.h>
11#include <linux/threads.h>
12#include <linux/bitmap.h>
13#include <linux/atomic.h>
14#include <linux/bug.h>
15
16
17typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
18
19
20
21
22
23
24
25
26#define cpumask_bits(maskp) ((maskp)->bits)
27
28
29
30
31
32
33
34#define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
35
36#if NR_CPUS == 1
37#define nr_cpu_ids 1U
38#else
39extern unsigned int nr_cpu_ids;
40#endif
41
42#ifdef CONFIG_CPUMASK_OFFSTACK
43
44
45#define nr_cpumask_bits nr_cpu_ids
46#else
47#define nr_cpumask_bits ((unsigned int)NR_CPUS)
48#endif
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90extern struct cpumask __cpu_possible_mask;
91extern struct cpumask __cpu_online_mask;
92extern struct cpumask __cpu_present_mask;
93extern struct cpumask __cpu_active_mask;
94#define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
95#define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
96#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
97#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
98
99extern atomic_t __num_online_cpus;
100
101#if NR_CPUS > 1
102
103
104
105
106
107
108
109
110static inline unsigned int num_online_cpus(void)
111{
112 return atomic_read(&__num_online_cpus);
113}
114#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
115#define num_present_cpus() cpumask_weight(cpu_present_mask)
116#define num_active_cpus() cpumask_weight(cpu_active_mask)
117#define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask)
118#define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask)
119#define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask)
120#define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask)
121#else
122#define num_online_cpus() 1U
123#define num_possible_cpus() 1U
124#define num_present_cpus() 1U
125#define num_active_cpus() 1U
126#define cpu_online(cpu) ((cpu) == 0)
127#define cpu_possible(cpu) ((cpu) == 0)
128#define cpu_present(cpu) ((cpu) == 0)
129#define cpu_active(cpu) ((cpu) == 0)
130#endif
131
132extern cpumask_t cpus_booted_once_mask;
133
134static inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
135{
136#ifdef CONFIG_DEBUG_PER_CPU_MAPS
137 WARN_ON_ONCE(cpu >= bits);
138#endif
139}
140
141
142static inline unsigned int cpumask_check(unsigned int cpu)
143{
144 cpu_max_bits_warn(cpu, nr_cpumask_bits);
145 return cpu;
146}
147
148#if NR_CPUS == 1
149
150static inline unsigned int cpumask_first(const struct cpumask *srcp)
151{
152 return 0;
153}
154
155static inline unsigned int cpumask_last(const struct cpumask *srcp)
156{
157 return 0;
158}
159
160
161static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
162{
163 return n+1;
164}
165
166static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
167{
168 return n+1;
169}
170
171static inline unsigned int cpumask_next_and(int n,
172 const struct cpumask *srcp,
173 const struct cpumask *andp)
174{
175 return n+1;
176}
177
178static inline unsigned int cpumask_next_wrap(int n, const struct cpumask *mask,
179 int start, bool wrap)
180{
181
182 return (wrap && n == 0);
183}
184
185
186static inline unsigned int cpumask_any_but(const struct cpumask *mask,
187 unsigned int cpu)
188{
189 return 1;
190}
191
192static inline unsigned int cpumask_local_spread(unsigned int i, int node)
193{
194 return 0;
195}
196
197#define for_each_cpu(cpu, mask) \
198 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
199#define for_each_cpu_not(cpu, mask) \
200 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
201#define for_each_cpu_wrap(cpu, mask, start) \
202 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)(start))
203#define for_each_cpu_and(cpu, mask1, mask2) \
204 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask1, (void)mask2)
205#else
206
207
208
209
210
211
212static inline unsigned int cpumask_first(const struct cpumask *srcp)
213{
214 return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
215}
216
217
218
219
220
221
222
223static inline unsigned int cpumask_last(const struct cpumask *srcp)
224{
225 return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits);
226}
227
228unsigned int cpumask_next(int n, const struct cpumask *srcp);
229
230
231
232
233
234
235
236
237static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
238{
239
240 if (n != -1)
241 cpumask_check(n);
242 return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
243}
244
245int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
246int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
247unsigned int cpumask_local_spread(unsigned int i, int node);
248
249
250
251
252
253
254
255
256#define for_each_cpu(cpu, mask) \
257 for ((cpu) = -1; \
258 (cpu) = cpumask_next((cpu), (mask)), \
259 (cpu) < nr_cpu_ids;)
260
261
262
263
264
265
266
267
268#define for_each_cpu_not(cpu, mask) \
269 for ((cpu) = -1; \
270 (cpu) = cpumask_next_zero((cpu), (mask)), \
271 (cpu) < nr_cpu_ids;)
272
273extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
274
275
276
277
278
279
280
281
282
283
284
285#define for_each_cpu_wrap(cpu, mask, start) \
286 for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \
287 (cpu) < nr_cpumask_bits; \
288 (cpu) = cpumask_next_wrap((cpu), (mask), (start), true))
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304#define for_each_cpu_and(cpu, mask1, mask2) \
305 for ((cpu) = -1; \
306 (cpu) = cpumask_next_and((cpu), (mask1), (mask2)), \
307 (cpu) < nr_cpu_ids;)
308#endif
309
310#define CPU_BITS_NONE \
311{ \
312 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
313}
314
315#define CPU_BITS_CPU0 \
316{ \
317 [0] = 1UL \
318}
319
320
321
322
323
324
325static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
326{
327 set_bit(cpumask_check(cpu), cpumask_bits(dstp));
328}
329
330static inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
331{
332 __set_bit(cpumask_check(cpu), cpumask_bits(dstp));
333}
334
335
336
337
338
339
340
341static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
342{
343 clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
344}
345
346static inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
347{
348 __clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
349}
350
351
352
353
354
355
356
357
358static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
359{
360 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
361}
362
363
364
365
366
367
368
369
370
371
372static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
373{
374 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
375}
376
377
378
379
380
381
382
383
384
385
386static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
387{
388 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
389}
390
391
392
393
394
395static inline void cpumask_setall(struct cpumask *dstp)
396{
397 bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
398}
399
400
401
402
403
404static inline void cpumask_clear(struct cpumask *dstp)
405{
406 bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
407}
408
409
410
411
412
413
414
415
416
417static inline int cpumask_and(struct cpumask *dstp,
418 const struct cpumask *src1p,
419 const struct cpumask *src2p)
420{
421 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
422 cpumask_bits(src2p), nr_cpumask_bits);
423}
424
425
426
427
428
429
430
431static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
432 const struct cpumask *src2p)
433{
434 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
435 cpumask_bits(src2p), nr_cpumask_bits);
436}
437
438
439
440
441
442
443
444static inline void cpumask_xor(struct cpumask *dstp,
445 const struct cpumask *src1p,
446 const struct cpumask *src2p)
447{
448 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
449 cpumask_bits(src2p), nr_cpumask_bits);
450}
451
452
453
454
455
456
457
458
459
460static inline int cpumask_andnot(struct cpumask *dstp,
461 const struct cpumask *src1p,
462 const struct cpumask *src2p)
463{
464 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
465 cpumask_bits(src2p), nr_cpumask_bits);
466}
467
468
469
470
471
472
473static inline void cpumask_complement(struct cpumask *dstp,
474 const struct cpumask *srcp)
475{
476 bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
477 nr_cpumask_bits);
478}
479
480
481
482
483
484
485static inline bool cpumask_equal(const struct cpumask *src1p,
486 const struct cpumask *src2p)
487{
488 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
489 nr_cpumask_bits);
490}
491
492
493
494
495
496
497
498static inline bool cpumask_or_equal(const struct cpumask *src1p,
499 const struct cpumask *src2p,
500 const struct cpumask *src3p)
501{
502 return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
503 cpumask_bits(src3p), nr_cpumask_bits);
504}
505
506
507
508
509
510
511static inline bool cpumask_intersects(const struct cpumask *src1p,
512 const struct cpumask *src2p)
513{
514 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
515 nr_cpumask_bits);
516}
517
518
519
520
521
522
523
524
525static inline int cpumask_subset(const struct cpumask *src1p,
526 const struct cpumask *src2p)
527{
528 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
529 nr_cpumask_bits);
530}
531
532
533
534
535
536static inline bool cpumask_empty(const struct cpumask *srcp)
537{
538 return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
539}
540
541
542
543
544
545static inline bool cpumask_full(const struct cpumask *srcp)
546{
547 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
548}
549
550
551
552
553
554static inline unsigned int cpumask_weight(const struct cpumask *srcp)
555{
556 return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
557}
558
559
560
561
562
563
564
565static inline void cpumask_shift_right(struct cpumask *dstp,
566 const struct cpumask *srcp, int n)
567{
568 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
569 nr_cpumask_bits);
570}
571
572
573
574
575
576
577
578static inline void cpumask_shift_left(struct cpumask *dstp,
579 const struct cpumask *srcp, int n)
580{
581 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
582 nr_cpumask_bits);
583}
584
585
586
587
588
589
590static inline void cpumask_copy(struct cpumask *dstp,
591 const struct cpumask *srcp)
592{
593 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
594}
595
596
597
598
599
600
601
602#define cpumask_any(srcp) cpumask_first(srcp)
603
604
605
606
607
608
609
610
611#define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p))
612
613
614
615
616
617
618
619
620#define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
621
622
623
624
625
626#define cpumask_of(cpu) (get_cpu_mask(cpu))
627
628
629
630
631
632
633
634
635
636static inline int cpumask_parse_user(const char __user *buf, int len,
637 struct cpumask *dstp)
638{
639 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
640}
641
642
643
644
645
646
647
648
649
650static inline int cpumask_parselist_user(const char __user *buf, int len,
651 struct cpumask *dstp)
652{
653 return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
654 nr_cpumask_bits);
655}
656
657
658
659
660
661
662
663
664static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
665{
666 return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
667}
668
669
670
671
672
673
674
675
676static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
677{
678 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
679}
680
681
682
683
684static inline unsigned int cpumask_size(void)
685{
686 return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
687}
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729#ifdef CONFIG_CPUMASK_OFFSTACK
730typedef struct cpumask *cpumask_var_t;
731
732#define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
733#define __cpumask_var_read_mostly __read_mostly
734
735bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
736bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
737bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
738bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
739void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
740void free_cpumask_var(cpumask_var_t mask);
741void free_bootmem_cpumask_var(cpumask_var_t mask);
742
743static inline bool cpumask_available(cpumask_var_t mask)
744{
745 return mask != NULL;
746}
747
748#else
749typedef struct cpumask cpumask_var_t[1];
750
751#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
752#define __cpumask_var_read_mostly
753
754static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
755{
756 return true;
757}
758
759static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
760 int node)
761{
762 return true;
763}
764
765static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
766{
767 cpumask_clear(*mask);
768 return true;
769}
770
771static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
772 int node)
773{
774 cpumask_clear(*mask);
775 return true;
776}
777
778static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
779{
780}
781
782static inline void free_cpumask_var(cpumask_var_t mask)
783{
784}
785
786static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
787{
788}
789
790static inline bool cpumask_available(cpumask_var_t mask)
791{
792 return true;
793}
794#endif
795
796
797
798extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
799#define cpu_all_mask to_cpumask(cpu_all_bits)
800
801
802#define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
803
804#define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
805#define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
806#define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
807
808
809void init_cpu_present(const struct cpumask *src);
810void init_cpu_possible(const struct cpumask *src);
811void init_cpu_online(const struct cpumask *src);
812
813static inline void reset_cpu_possible_mask(void)
814{
815 bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
816}
817
818static inline void
819set_cpu_possible(unsigned int cpu, bool possible)
820{
821 if (possible)
822 cpumask_set_cpu(cpu, &__cpu_possible_mask);
823 else
824 cpumask_clear_cpu(cpu, &__cpu_possible_mask);
825}
826
827static inline void
828set_cpu_present(unsigned int cpu, bool present)
829{
830 if (present)
831 cpumask_set_cpu(cpu, &__cpu_present_mask);
832 else
833 cpumask_clear_cpu(cpu, &__cpu_present_mask);
834}
835
836void set_cpu_online(unsigned int cpu, bool online);
837
838static inline void
839set_cpu_active(unsigned int cpu, bool active)
840{
841 if (active)
842 cpumask_set_cpu(cpu, &__cpu_active_mask);
843 else
844 cpumask_clear_cpu(cpu, &__cpu_active_mask);
845}
846
847
848
849
850
851
852
853
854
855
856
857
858#define to_cpumask(bitmap) \
859 ((struct cpumask *)(1 ? (bitmap) \
860 : (void *)sizeof(__check_is_bitmap(bitmap))))
861
862static inline int __check_is_bitmap(const unsigned long *bitmap)
863{
864 return 1;
865}
866
867
868
869
870
871
872
873
874extern const unsigned long
875 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
876
877static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
878{
879 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
880 p -= cpu / BITS_PER_LONG;
881 return to_cpumask(p);
882}
883
884#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
885
886#if NR_CPUS <= BITS_PER_LONG
887#define CPU_BITS_ALL \
888{ \
889 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
890}
891
892#else
893
894#define CPU_BITS_ALL \
895{ \
896 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
897 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
898}
899#endif
900
901
902
903
904
905
906
907
908
909
910
911static inline ssize_t
912cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
913{
914 return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
915 nr_cpu_ids);
916}
917
918#if NR_CPUS <= BITS_PER_LONG
919#define CPU_MASK_ALL \
920(cpumask_t) { { \
921 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
922} }
923#else
924#define CPU_MASK_ALL \
925(cpumask_t) { { \
926 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
927 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
928} }
929#endif
930
931#define CPU_MASK_NONE \
932(cpumask_t) { { \
933 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
934} }
935
936#define CPU_MASK_CPU0 \
937(cpumask_t) { { \
938 [0] = 1UL \
939} }
940
941#endif
942