1
2#ifndef __LINUX_CPUMASK_H
3#define __LINUX_CPUMASK_H
4
5
6
7
8
9
10#include <linux/kernel.h>
11#include <linux/threads.h>
12#include <linux/bitmap.h>
13#include <linux/bug.h>
14
15
16typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
17
18
19
20
21
22
23
24
25#define cpumask_bits(maskp) ((maskp)->bits)
26
27
28
29
30
31
32
33#define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
34
35#if NR_CPUS == 1
36#define nr_cpu_ids 1U
37#else
38extern unsigned int nr_cpu_ids;
39#endif
40
41#ifdef CONFIG_CPUMASK_OFFSTACK
42
43
44#define nr_cpumask_bits nr_cpu_ids
45#else
46#define nr_cpumask_bits ((unsigned int)NR_CPUS)
47#endif
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89extern struct cpumask __cpu_possible_mask;
90extern struct cpumask __cpu_online_mask;
91extern struct cpumask __cpu_present_mask;
92extern struct cpumask __cpu_active_mask;
93#define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
94#define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
95#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
96#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
97
98#if NR_CPUS > 1
99#define num_online_cpus() cpumask_weight(cpu_online_mask)
100#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
101#define num_present_cpus() cpumask_weight(cpu_present_mask)
102#define num_active_cpus() cpumask_weight(cpu_active_mask)
103#define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask)
104#define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask)
105#define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask)
106#define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask)
107#else
108#define num_online_cpus() 1U
109#define num_possible_cpus() 1U
110#define num_present_cpus() 1U
111#define num_active_cpus() 1U
112#define cpu_online(cpu) ((cpu) == 0)
113#define cpu_possible(cpu) ((cpu) == 0)
114#define cpu_present(cpu) ((cpu) == 0)
115#define cpu_active(cpu) ((cpu) == 0)
116#endif
117
118static inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
119{
120#ifdef CONFIG_DEBUG_PER_CPU_MAPS
121 WARN_ON_ONCE(cpu >= bits);
122#endif
123}
124
125
126static inline unsigned int cpumask_check(unsigned int cpu)
127{
128 cpu_max_bits_warn(cpu, nr_cpumask_bits);
129 return cpu;
130}
131
132#if NR_CPUS == 1
133
134static inline unsigned int cpumask_first(const struct cpumask *srcp)
135{
136 return 0;
137}
138
139static inline unsigned int cpumask_last(const struct cpumask *srcp)
140{
141 return 0;
142}
143
144
145static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
146{
147 return n+1;
148}
149
150static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
151{
152 return n+1;
153}
154
155static inline unsigned int cpumask_next_and(int n,
156 const struct cpumask *srcp,
157 const struct cpumask *andp)
158{
159 return n+1;
160}
161
162static inline unsigned int cpumask_next_wrap(int n, const struct cpumask *mask,
163 int start, bool wrap)
164{
165
166 return (wrap && n == 0);
167}
168
169
170static inline unsigned int cpumask_any_but(const struct cpumask *mask,
171 unsigned int cpu)
172{
173 return 1;
174}
175
176static inline unsigned int cpumask_local_spread(unsigned int i, int node)
177{
178 return 0;
179}
180
181#define for_each_cpu(cpu, mask) \
182 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
183#define for_each_cpu_not(cpu, mask) \
184 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
185#define for_each_cpu_wrap(cpu, mask, start) \
186 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)(start))
187#define for_each_cpu_and(cpu, mask, and) \
188 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and)
189#else
190
191
192
193
194
195
196static inline unsigned int cpumask_first(const struct cpumask *srcp)
197{
198 return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
199}
200
201
202
203
204
205
206
207static inline unsigned int cpumask_last(const struct cpumask *srcp)
208{
209 return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits);
210}
211
212unsigned int cpumask_next(int n, const struct cpumask *srcp);
213
214
215
216
217
218
219
220
221static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
222{
223
224 if (n != -1)
225 cpumask_check(n);
226 return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
227}
228
229int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
230int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
231unsigned int cpumask_local_spread(unsigned int i, int node);
232
233
234
235
236
237
238
239
240#define for_each_cpu(cpu, mask) \
241 for ((cpu) = -1; \
242 (cpu) = cpumask_next((cpu), (mask)), \
243 (cpu) < nr_cpu_ids;)
244
245
246
247
248
249
250
251
252#define for_each_cpu_not(cpu, mask) \
253 for ((cpu) = -1; \
254 (cpu) = cpumask_next_zero((cpu), (mask)), \
255 (cpu) < nr_cpu_ids;)
256
257extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
258
259
260
261
262
263
264
265
266
267
268
269#define for_each_cpu_wrap(cpu, mask, start) \
270 for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \
271 (cpu) < nr_cpumask_bits; \
272 (cpu) = cpumask_next_wrap((cpu), (mask), (start), true))
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288#define for_each_cpu_and(cpu, mask, and) \
289 for ((cpu) = -1; \
290 (cpu) = cpumask_next_and((cpu), (mask), (and)), \
291 (cpu) < nr_cpu_ids;)
292#endif
293
294#define CPU_BITS_NONE \
295{ \
296 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
297}
298
299#define CPU_BITS_CPU0 \
300{ \
301 [0] = 1UL \
302}
303
304
305
306
307
308
309static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
310{
311 set_bit(cpumask_check(cpu), cpumask_bits(dstp));
312}
313
314static inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
315{
316 __set_bit(cpumask_check(cpu), cpumask_bits(dstp));
317}
318
319
320
321
322
323
324
325static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
326{
327 clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
328}
329
330static inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
331{
332 __clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
333}
334
335
336
337
338
339
340
341
342static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
343{
344 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
345}
346
347
348
349
350
351
352
353
354
355
356static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
357{
358 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
359}
360
361
362
363
364
365
366
367
368
369
370static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
371{
372 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
373}
374
375
376
377
378
379static inline void cpumask_setall(struct cpumask *dstp)
380{
381 bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
382}
383
384
385
386
387
388static inline void cpumask_clear(struct cpumask *dstp)
389{
390 bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
391}
392
393
394
395
396
397
398
399
400
401static inline int cpumask_and(struct cpumask *dstp,
402 const struct cpumask *src1p,
403 const struct cpumask *src2p)
404{
405 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
406 cpumask_bits(src2p), nr_cpumask_bits);
407}
408
409
410
411
412
413
414
415static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
416 const struct cpumask *src2p)
417{
418 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
419 cpumask_bits(src2p), nr_cpumask_bits);
420}
421
422
423
424
425
426
427
428static inline void cpumask_xor(struct cpumask *dstp,
429 const struct cpumask *src1p,
430 const struct cpumask *src2p)
431{
432 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
433 cpumask_bits(src2p), nr_cpumask_bits);
434}
435
436
437
438
439
440
441
442
443
444static inline int cpumask_andnot(struct cpumask *dstp,
445 const struct cpumask *src1p,
446 const struct cpumask *src2p)
447{
448 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
449 cpumask_bits(src2p), nr_cpumask_bits);
450}
451
452
453
454
455
456
457static inline void cpumask_complement(struct cpumask *dstp,
458 const struct cpumask *srcp)
459{
460 bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
461 nr_cpumask_bits);
462}
463
464
465
466
467
468
469static inline bool cpumask_equal(const struct cpumask *src1p,
470 const struct cpumask *src2p)
471{
472 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
473 nr_cpumask_bits);
474}
475
476
477
478
479
480
481static inline bool cpumask_intersects(const struct cpumask *src1p,
482 const struct cpumask *src2p)
483{
484 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
485 nr_cpumask_bits);
486}
487
488
489
490
491
492
493
494
495static inline int cpumask_subset(const struct cpumask *src1p,
496 const struct cpumask *src2p)
497{
498 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
499 nr_cpumask_bits);
500}
501
502
503
504
505
506static inline bool cpumask_empty(const struct cpumask *srcp)
507{
508 return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
509}
510
511
512
513
514
515static inline bool cpumask_full(const struct cpumask *srcp)
516{
517 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
518}
519
520
521
522
523
524static inline unsigned int cpumask_weight(const struct cpumask *srcp)
525{
526 return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
527}
528
529
530
531
532
533
534
535static inline void cpumask_shift_right(struct cpumask *dstp,
536 const struct cpumask *srcp, int n)
537{
538 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
539 nr_cpumask_bits);
540}
541
542
543
544
545
546
547
548static inline void cpumask_shift_left(struct cpumask *dstp,
549 const struct cpumask *srcp, int n)
550{
551 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
552 nr_cpumask_bits);
553}
554
555
556
557
558
559
560static inline void cpumask_copy(struct cpumask *dstp,
561 const struct cpumask *srcp)
562{
563 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
564}
565
566
567
568
569
570
571
572#define cpumask_any(srcp) cpumask_first(srcp)
573
574
575
576
577
578
579
580
581#define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p))
582
583
584
585
586
587
588
589
590#define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
591
592
593
594
595
596#define cpumask_of(cpu) (get_cpu_mask(cpu))
597
598
599
600
601
602
603
604
605
606static inline int cpumask_parse_user(const char __user *buf, int len,
607 struct cpumask *dstp)
608{
609 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
610}
611
612
613
614
615
616
617
618
619
620static inline int cpumask_parselist_user(const char __user *buf, int len,
621 struct cpumask *dstp)
622{
623 return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
624 nr_cpumask_bits);
625}
626
627
628
629
630
631
632
633
634static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
635{
636 unsigned int len = strchrnul(buf, '\n') - buf;
637
638 return bitmap_parse(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
639}
640
641
642
643
644
645
646
647
648static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
649{
650 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
651}
652
653
654
655
656static inline unsigned int cpumask_size(void)
657{
658 return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
659}
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701#ifdef CONFIG_CPUMASK_OFFSTACK
702typedef struct cpumask *cpumask_var_t;
703
704#define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
705#define __cpumask_var_read_mostly __read_mostly
706
707bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
708bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
709bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
710bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
711void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
712void free_cpumask_var(cpumask_var_t mask);
713void free_bootmem_cpumask_var(cpumask_var_t mask);
714
715static inline bool cpumask_available(cpumask_var_t mask)
716{
717 return mask != NULL;
718}
719
720#else
721typedef struct cpumask cpumask_var_t[1];
722
723#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
724#define __cpumask_var_read_mostly
725
726static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
727{
728 return true;
729}
730
731static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
732 int node)
733{
734 return true;
735}
736
737static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
738{
739 cpumask_clear(*mask);
740 return true;
741}
742
743static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
744 int node)
745{
746 cpumask_clear(*mask);
747 return true;
748}
749
750static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
751{
752}
753
754static inline void free_cpumask_var(cpumask_var_t mask)
755{
756}
757
758static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
759{
760}
761
762static inline bool cpumask_available(cpumask_var_t mask)
763{
764 return true;
765}
766#endif
767
768
769
770extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
771#define cpu_all_mask to_cpumask(cpu_all_bits)
772
773
774#define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
775
776#define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
777#define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
778#define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
779
780
781void init_cpu_present(const struct cpumask *src);
782void init_cpu_possible(const struct cpumask *src);
783void init_cpu_online(const struct cpumask *src);
784
785static inline void reset_cpu_possible_mask(void)
786{
787 bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
788}
789
790static inline void
791set_cpu_possible(unsigned int cpu, bool possible)
792{
793 if (possible)
794 cpumask_set_cpu(cpu, &__cpu_possible_mask);
795 else
796 cpumask_clear_cpu(cpu, &__cpu_possible_mask);
797}
798
799static inline void
800set_cpu_present(unsigned int cpu, bool present)
801{
802 if (present)
803 cpumask_set_cpu(cpu, &__cpu_present_mask);
804 else
805 cpumask_clear_cpu(cpu, &__cpu_present_mask);
806}
807
808static inline void
809set_cpu_online(unsigned int cpu, bool online)
810{
811 if (online)
812 cpumask_set_cpu(cpu, &__cpu_online_mask);
813 else
814 cpumask_clear_cpu(cpu, &__cpu_online_mask);
815}
816
817static inline void
818set_cpu_active(unsigned int cpu, bool active)
819{
820 if (active)
821 cpumask_set_cpu(cpu, &__cpu_active_mask);
822 else
823 cpumask_clear_cpu(cpu, &__cpu_active_mask);
824}
825
826
827
828
829
830
831
832
833
834
835
836
837#define to_cpumask(bitmap) \
838 ((struct cpumask *)(1 ? (bitmap) \
839 : (void *)sizeof(__check_is_bitmap(bitmap))))
840
841static inline int __check_is_bitmap(const unsigned long *bitmap)
842{
843 return 1;
844}
845
846
847
848
849
850
851
852
853extern const unsigned long
854 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
855
856static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
857{
858 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
859 p -= cpu / BITS_PER_LONG;
860 return to_cpumask(p);
861}
862
863#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
864
865#if NR_CPUS <= BITS_PER_LONG
866#define CPU_BITS_ALL \
867{ \
868 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
869}
870
871#else
872
873#define CPU_BITS_ALL \
874{ \
875 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
876 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
877}
878#endif
879
880
881
882
883
884
885
886
887
888
889
890static inline ssize_t
891cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
892{
893 return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
894 nr_cpu_ids);
895}
896
897#if NR_CPUS <= BITS_PER_LONG
898#define CPU_MASK_ALL \
899(cpumask_t) { { \
900 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
901} }
902#else
903#define CPU_MASK_ALL \
904(cpumask_t) { { \
905 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
906 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
907} }
908#endif
909
910#define CPU_MASK_NONE \
911(cpumask_t) { { \
912 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
913} }
914
915#define CPU_MASK_CPU0 \
916(cpumask_t) { { \
917 [0] = 1UL \
918} }
919
920#endif
921