1#ifndef __LINUX_CPUMASK_H
2#define __LINUX_CPUMASK_H
3
4
5
6
7
8
9#include <linux/kernel.h>
10#include <linux/threads.h>
11#include <linux/bitmap.h>
12
13typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
14
15
16
17
18
19
20
21
22#define cpumask_bits(maskp) ((maskp)->bits)
23
24#if NR_CPUS == 1
25#define nr_cpu_ids 1
26#else
27extern int nr_cpu_ids;
28#endif
29
30#ifdef CONFIG_CPUMASK_OFFSTACK
31
32
33#define nr_cpumask_bits nr_cpu_ids
34#else
35#define nr_cpumask_bits NR_CPUS
36#endif
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78extern const struct cpumask *const cpu_possible_mask;
79extern const struct cpumask *const cpu_online_mask;
80extern const struct cpumask *const cpu_present_mask;
81extern const struct cpumask *const cpu_active_mask;
82
83#if NR_CPUS > 1
84#define num_online_cpus() cpumask_weight(cpu_online_mask)
85#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
86#define num_present_cpus() cpumask_weight(cpu_present_mask)
87#define num_active_cpus() cpumask_weight(cpu_active_mask)
88#define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask)
89#define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask)
90#define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask)
91#define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask)
92#else
93#define num_online_cpus() 1U
94#define num_possible_cpus() 1U
95#define num_present_cpus() 1U
96#define num_active_cpus() 1U
97#define cpu_online(cpu) ((cpu) == 0)
98#define cpu_possible(cpu) ((cpu) == 0)
99#define cpu_present(cpu) ((cpu) == 0)
100#define cpu_active(cpu) ((cpu) == 0)
101#endif
102
103
104static inline unsigned int cpumask_check(unsigned int cpu)
105{
106#ifdef CONFIG_DEBUG_PER_CPU_MAPS
107 WARN_ON_ONCE(cpu >= nr_cpumask_bits);
108#endif
109 return cpu;
110}
111
112#if NR_CPUS == 1
113
114static inline unsigned int cpumask_first(const struct cpumask *srcp)
115{
116 return 0;
117}
118
119
120static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
121{
122 return n+1;
123}
124
125static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
126{
127 return n+1;
128}
129
130static inline unsigned int cpumask_next_and(int n,
131 const struct cpumask *srcp,
132 const struct cpumask *andp)
133{
134 return n+1;
135}
136
137
138static inline unsigned int cpumask_any_but(const struct cpumask *mask,
139 unsigned int cpu)
140{
141 return 1;
142}
143
144#define for_each_cpu(cpu, mask) \
145 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
146#define for_each_cpu_not(cpu, mask) \
147 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
148#define for_each_cpu_and(cpu, mask, and) \
149 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and)
150#else
151
152
153
154
155
156
157static inline unsigned int cpumask_first(const struct cpumask *srcp)
158{
159 return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
160}
161
162
163
164
165
166
167
168
169static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
170{
171
172 if (n != -1)
173 cpumask_check(n);
174 return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
175}
176
177
178
179
180
181
182
183
184static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
185{
186
187 if (n != -1)
188 cpumask_check(n);
189 return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
190}
191
192int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
193int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
194
195
196
197
198
199
200
201
202#define for_each_cpu(cpu, mask) \
203 for ((cpu) = -1; \
204 (cpu) = cpumask_next((cpu), (mask)), \
205 (cpu) < nr_cpu_ids;)
206
207
208
209
210
211
212
213
214#define for_each_cpu_not(cpu, mask) \
215 for ((cpu) = -1; \
216 (cpu) = cpumask_next_zero((cpu), (mask)), \
217 (cpu) < nr_cpu_ids;)
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233#define for_each_cpu_and(cpu, mask, and) \
234 for ((cpu) = -1; \
235 (cpu) = cpumask_next_and((cpu), (mask), (and)), \
236 (cpu) < nr_cpu_ids;)
237#endif
238
239#define CPU_BITS_NONE \
240{ \
241 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
242}
243
244#define CPU_BITS_CPU0 \
245{ \
246 [0] = 1UL \
247}
248
249
250
251
252
253
254static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
255{
256 set_bit(cpumask_check(cpu), cpumask_bits(dstp));
257}
258
259
260
261
262
263
264static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
265{
266 clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
267}
268
269
270
271
272
273
274
275
276#define cpumask_test_cpu(cpu, cpumask) \
277 test_bit(cpumask_check(cpu), cpumask_bits((cpumask)))
278
279
280
281
282
283
284
285
286static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
287{
288 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
289}
290
291
292
293
294
295
296
297
298static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
299{
300 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
301}
302
303
304
305
306
307static inline void cpumask_setall(struct cpumask *dstp)
308{
309 bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
310}
311
312
313
314
315
316static inline void cpumask_clear(struct cpumask *dstp)
317{
318 bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
319}
320
321
322
323
324
325
326
327static inline int cpumask_and(struct cpumask *dstp,
328 const struct cpumask *src1p,
329 const struct cpumask *src2p)
330{
331 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
332 cpumask_bits(src2p), nr_cpumask_bits);
333}
334
335
336
337
338
339
340
341static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
342 const struct cpumask *src2p)
343{
344 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
345 cpumask_bits(src2p), nr_cpumask_bits);
346}
347
348
349
350
351
352
353
354static inline void cpumask_xor(struct cpumask *dstp,
355 const struct cpumask *src1p,
356 const struct cpumask *src2p)
357{
358 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
359 cpumask_bits(src2p), nr_cpumask_bits);
360}
361
362
363
364
365
366
367
368static inline int cpumask_andnot(struct cpumask *dstp,
369 const struct cpumask *src1p,
370 const struct cpumask *src2p)
371{
372 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
373 cpumask_bits(src2p), nr_cpumask_bits);
374}
375
376
377
378
379
380
381static inline void cpumask_complement(struct cpumask *dstp,
382 const struct cpumask *srcp)
383{
384 bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
385 nr_cpumask_bits);
386}
387
388
389
390
391
392
393static inline bool cpumask_equal(const struct cpumask *src1p,
394 const struct cpumask *src2p)
395{
396 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
397 nr_cpumask_bits);
398}
399
400
401
402
403
404
405static inline bool cpumask_intersects(const struct cpumask *src1p,
406 const struct cpumask *src2p)
407{
408 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
409 nr_cpumask_bits);
410}
411
412
413
414
415
416
417static inline int cpumask_subset(const struct cpumask *src1p,
418 const struct cpumask *src2p)
419{
420 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
421 nr_cpumask_bits);
422}
423
424
425
426
427
428static inline bool cpumask_empty(const struct cpumask *srcp)
429{
430 return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
431}
432
433
434
435
436
437static inline bool cpumask_full(const struct cpumask *srcp)
438{
439 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
440}
441
442
443
444
445
446static inline unsigned int cpumask_weight(const struct cpumask *srcp)
447{
448 return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
449}
450
451
452
453
454
455
456
457static inline void cpumask_shift_right(struct cpumask *dstp,
458 const struct cpumask *srcp, int n)
459{
460 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
461 nr_cpumask_bits);
462}
463
464
465
466
467
468
469
470static inline void cpumask_shift_left(struct cpumask *dstp,
471 const struct cpumask *srcp, int n)
472{
473 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
474 nr_cpumask_bits);
475}
476
477
478
479
480
481
482static inline void cpumask_copy(struct cpumask *dstp,
483 const struct cpumask *srcp)
484{
485 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
486}
487
488
489
490
491
492
493
494#define cpumask_any(srcp) cpumask_first(srcp)
495
496
497
498
499
500
501
502
503#define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p))
504
505
506
507
508
509
510
511
512#define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
513
514
515
516
517
518#define cpumask_of(cpu) (get_cpu_mask(cpu))
519
520
521
522
523
524
525
526
527
528
529static inline int cpumask_scnprintf(char *buf, int len,
530 const struct cpumask *srcp)
531{
532 return bitmap_scnprintf(buf, len, cpumask_bits(srcp), nr_cpumask_bits);
533}
534
535
536
537
538
539
540
541
542
543static inline int cpumask_parse_user(const char __user *buf, int len,
544 struct cpumask *dstp)
545{
546 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
547}
548
549
550
551
552
553
554
555
556
557
558static inline int cpulist_scnprintf(char *buf, int len,
559 const struct cpumask *srcp)
560{
561 return bitmap_scnlistprintf(buf, len, cpumask_bits(srcp),
562 nr_cpumask_bits);
563}
564
565
566
567
568
569
570
571
572
573static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
574{
575 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
576}
577
578
579
580
581
582
583static inline size_t cpumask_size(void)
584{
585
586
587 return BITS_TO_LONGS(NR_CPUS) * sizeof(long);
588}
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606#ifdef CONFIG_CPUMASK_OFFSTACK
607typedef struct cpumask *cpumask_var_t;
608
609bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
610bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
611bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
612bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
613void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
614void free_cpumask_var(cpumask_var_t mask);
615void free_bootmem_cpumask_var(cpumask_var_t mask);
616
617#else
618typedef struct cpumask cpumask_var_t[1];
619
620static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
621{
622 return true;
623}
624
625static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
626 int node)
627{
628 return true;
629}
630
631static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
632{
633 cpumask_clear(*mask);
634 return true;
635}
636
637static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
638 int node)
639{
640 cpumask_clear(*mask);
641 return true;
642}
643
644static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
645{
646}
647
648static inline void free_cpumask_var(cpumask_var_t mask)
649{
650}
651
652static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
653{
654}
655#endif
656
657
658
659extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
660#define cpu_all_mask to_cpumask(cpu_all_bits)
661
662
663#define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
664
665#define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
666#define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
667#define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
668
669
670void set_cpu_possible(unsigned int cpu, bool possible);
671void set_cpu_present(unsigned int cpu, bool present);
672void set_cpu_online(unsigned int cpu, bool online);
673void set_cpu_active(unsigned int cpu, bool active);
674void init_cpu_present(const struct cpumask *src);
675void init_cpu_possible(const struct cpumask *src);
676void init_cpu_online(const struct cpumask *src);
677
678
679
680
681
682
683
684
685
686
687
688#define to_cpumask(bitmap) \
689 ((struct cpumask *)(1 ? (bitmap) \
690 : (void *)sizeof(__check_is_bitmap(bitmap))))
691
692static inline int __check_is_bitmap(const unsigned long *bitmap)
693{
694 return 1;
695}
696
697
698
699
700
701
702
703
704extern const unsigned long
705 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
706
707static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
708{
709 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
710 p -= cpu / BITS_PER_LONG;
711 return to_cpumask(p);
712}
713
714#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
715
716#if NR_CPUS <= BITS_PER_LONG
717#define CPU_BITS_ALL \
718{ \
719 [BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD \
720}
721
722#else
723
724#define CPU_BITS_ALL \
725{ \
726 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
727 [BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD \
728}
729#endif
730
731
732
733
734
735
736#ifndef CONFIG_DISABLE_OBSOLETE_CPUMASK_FUNCTIONS
737
738#define cpu_possible_map (*(cpumask_t *)cpu_possible_mask)
739#define cpu_online_map (*(cpumask_t *)cpu_online_mask)
740#define cpu_present_map (*(cpumask_t *)cpu_present_mask)
741#define cpu_active_map (*(cpumask_t *)cpu_active_mask)
742
743#define cpumask_of_cpu(cpu) (*get_cpu_mask(cpu))
744
745#define CPU_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(NR_CPUS)
746
747#if NR_CPUS <= BITS_PER_LONG
748
749#define CPU_MASK_ALL \
750(cpumask_t) { { \
751 [BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD \
752} }
753
754#else
755
756#define CPU_MASK_ALL \
757(cpumask_t) { { \
758 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
759 [BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD \
760} }
761
762#endif
763
764#define CPU_MASK_NONE \
765(cpumask_t) { { \
766 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
767} }
768
769#define CPU_MASK_CPU0 \
770(cpumask_t) { { \
771 [0] = 1UL \
772} }
773
774#if NR_CPUS == 1
775#define first_cpu(src) ({ (void)(src); 0; })
776#define next_cpu(n, src) ({ (void)(src); 1; })
777#define any_online_cpu(mask) 0
778#define for_each_cpu_mask(cpu, mask) \
779 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
780#else
781int __first_cpu(const cpumask_t *srcp);
782int __next_cpu(int n, const cpumask_t *srcp);
783int __any_online_cpu(const cpumask_t *mask);
784
785#define first_cpu(src) __first_cpu(&(src))
786#define next_cpu(n, src) __next_cpu((n), &(src))
787#define any_online_cpu(mask) __any_online_cpu(&(mask))
788#define for_each_cpu_mask(cpu, mask) \
789 for ((cpu) = -1; \
790 (cpu) = next_cpu((cpu), (mask)), \
791 (cpu) < NR_CPUS; )
792#endif
793
794#if NR_CPUS <= 64
795
796#define for_each_cpu_mask_nr(cpu, mask) for_each_cpu_mask(cpu, mask)
797
798#else
799
800int __next_cpu_nr(int n, const cpumask_t *srcp);
801#define for_each_cpu_mask_nr(cpu, mask) \
802 for ((cpu) = -1; \
803 (cpu) = __next_cpu_nr((cpu), &(mask)), \
804 (cpu) < nr_cpu_ids; )
805
806#endif
807
808#define cpus_addr(src) ((src).bits)
809
810#define cpu_set(cpu, dst) __cpu_set((cpu), &(dst))
811static inline void __cpu_set(int cpu, volatile cpumask_t *dstp)
812{
813 set_bit(cpu, dstp->bits);
814}
815
816#define cpu_clear(cpu, dst) __cpu_clear((cpu), &(dst))
817static inline void __cpu_clear(int cpu, volatile cpumask_t *dstp)
818{
819 clear_bit(cpu, dstp->bits);
820}
821
822#define cpus_setall(dst) __cpus_setall(&(dst), NR_CPUS)
823static inline void __cpus_setall(cpumask_t *dstp, int nbits)
824{
825 bitmap_fill(dstp->bits, nbits);
826}
827
828#define cpus_clear(dst) __cpus_clear(&(dst), NR_CPUS)
829static inline void __cpus_clear(cpumask_t *dstp, int nbits)
830{
831 bitmap_zero(dstp->bits, nbits);
832}
833
834
835#define cpu_isset(cpu, cpumask) test_bit((cpu), (cpumask).bits)
836
837#define cpu_test_and_set(cpu, cpumask) __cpu_test_and_set((cpu), &(cpumask))
838static inline int __cpu_test_and_set(int cpu, cpumask_t *addr)
839{
840 return test_and_set_bit(cpu, addr->bits);
841}
842
843#define cpus_and(dst, src1, src2) __cpus_and(&(dst), &(src1), &(src2), NR_CPUS)
844static inline int __cpus_and(cpumask_t *dstp, const cpumask_t *src1p,
845 const cpumask_t *src2p, int nbits)
846{
847 return bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
848}
849
850#define cpus_or(dst, src1, src2) __cpus_or(&(dst), &(src1), &(src2), NR_CPUS)
851static inline void __cpus_or(cpumask_t *dstp, const cpumask_t *src1p,
852 const cpumask_t *src2p, int nbits)
853{
854 bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
855}
856
857#define cpus_xor(dst, src1, src2) __cpus_xor(&(dst), &(src1), &(src2), NR_CPUS)
858static inline void __cpus_xor(cpumask_t *dstp, const cpumask_t *src1p,
859 const cpumask_t *src2p, int nbits)
860{
861 bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
862}
863
864#define cpus_andnot(dst, src1, src2) \
865 __cpus_andnot(&(dst), &(src1), &(src2), NR_CPUS)
866static inline int __cpus_andnot(cpumask_t *dstp, const cpumask_t *src1p,
867 const cpumask_t *src2p, int nbits)
868{
869 return bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
870}
871
872#define cpus_equal(src1, src2) __cpus_equal(&(src1), &(src2), NR_CPUS)
873static inline int __cpus_equal(const cpumask_t *src1p,
874 const cpumask_t *src2p, int nbits)
875{
876 return bitmap_equal(src1p->bits, src2p->bits, nbits);
877}
878
879#define cpus_intersects(src1, src2) __cpus_intersects(&(src1), &(src2), NR_CPUS)
880static inline int __cpus_intersects(const cpumask_t *src1p,
881 const cpumask_t *src2p, int nbits)
882{
883 return bitmap_intersects(src1p->bits, src2p->bits, nbits);
884}
885
886#define cpus_subset(src1, src2) __cpus_subset(&(src1), &(src2), NR_CPUS)
887static inline int __cpus_subset(const cpumask_t *src1p,
888 const cpumask_t *src2p, int nbits)
889{
890 return bitmap_subset(src1p->bits, src2p->bits, nbits);
891}
892
893#define cpus_empty(src) __cpus_empty(&(src), NR_CPUS)
894static inline int __cpus_empty(const cpumask_t *srcp, int nbits)
895{
896 return bitmap_empty(srcp->bits, nbits);
897}
898
899#define cpus_weight(cpumask) __cpus_weight(&(cpumask), NR_CPUS)
900static inline int __cpus_weight(const cpumask_t *srcp, int nbits)
901{
902 return bitmap_weight(srcp->bits, nbits);
903}
904
905#define cpus_shift_left(dst, src, n) \
906 __cpus_shift_left(&(dst), &(src), (n), NR_CPUS)
907static inline void __cpus_shift_left(cpumask_t *dstp,
908 const cpumask_t *srcp, int n, int nbits)
909{
910 bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
911}
912#endif
913
914#endif
915