1
2#ifndef __LINUX_CPUMASK_H
3#define __LINUX_CPUMASK_H
4
5
6
7
8
9
10#include <linux/kernel.h>
11#include <linux/threads.h>
12#include <linux/bitmap.h>
13#include <linux/bug.h>
14
15
16typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
17
18
19
20
21
22
23
24
25#define cpumask_bits(maskp) ((maskp)->bits)
26
27
28
29
30
31
32
33#define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
34
35#if NR_CPUS == 1
36#define nr_cpu_ids 1U
37#else
38extern unsigned int nr_cpu_ids;
39#endif
40
41#ifdef CONFIG_CPUMASK_OFFSTACK
42
43
44#define nr_cpumask_bits nr_cpu_ids
45#else
46#define nr_cpumask_bits ((unsigned int)NR_CPUS)
47#endif
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89extern struct cpumask __cpu_possible_mask;
90extern struct cpumask __cpu_online_mask;
91extern struct cpumask __cpu_present_mask;
92extern struct cpumask __cpu_active_mask;
93#define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
94#define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
95#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
96#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
97
98#if NR_CPUS > 1
99#define num_online_cpus() cpumask_weight(cpu_online_mask)
100#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
101#define num_present_cpus() cpumask_weight(cpu_present_mask)
102#define num_active_cpus() cpumask_weight(cpu_active_mask)
103#define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask)
104#define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask)
105#define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask)
106#define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask)
107#else
108#define num_online_cpus() 1U
109#define num_possible_cpus() 1U
110#define num_present_cpus() 1U
111#define num_active_cpus() 1U
112#define cpu_online(cpu) ((cpu) == 0)
113#define cpu_possible(cpu) ((cpu) == 0)
114#define cpu_present(cpu) ((cpu) == 0)
115#define cpu_active(cpu) ((cpu) == 0)
116#endif
117
118static inline void cpu_max_bits_warn(unsigned int cpu, unsigned int bits)
119{
120#ifdef CONFIG_DEBUG_PER_CPU_MAPS
121 WARN_ON_ONCE(cpu >= bits);
122#endif
123}
124
125
126static inline unsigned int cpumask_check(unsigned int cpu)
127{
128 cpu_max_bits_warn(cpu, nr_cpumask_bits);
129 return cpu;
130}
131
132#if NR_CPUS == 1
133
134static inline unsigned int cpumask_first(const struct cpumask *srcp)
135{
136 return 0;
137}
138
139static inline unsigned int cpumask_last(const struct cpumask *srcp)
140{
141 return 0;
142}
143
144
145static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
146{
147 return n+1;
148}
149
150static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
151{
152 return n+1;
153}
154
155static inline unsigned int cpumask_next_and(int n,
156 const struct cpumask *srcp,
157 const struct cpumask *andp)
158{
159 return n+1;
160}
161
162
163static inline unsigned int cpumask_any_but(const struct cpumask *mask,
164 unsigned int cpu)
165{
166 return 1;
167}
168
169static inline unsigned int cpumask_local_spread(unsigned int i, int node)
170{
171 return 0;
172}
173
174#define for_each_cpu(cpu, mask) \
175 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
176#define for_each_cpu_not(cpu, mask) \
177 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
178#define for_each_cpu_wrap(cpu, mask, start) \
179 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)(start))
180#define for_each_cpu_and(cpu, mask, and) \
181 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and)
182#else
183
184
185
186
187
188
189static inline unsigned int cpumask_first(const struct cpumask *srcp)
190{
191 return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
192}
193
194
195
196
197
198
199
200static inline unsigned int cpumask_last(const struct cpumask *srcp)
201{
202 return find_last_bit(cpumask_bits(srcp), nr_cpumask_bits);
203}
204
205unsigned int cpumask_next(int n, const struct cpumask *srcp);
206
207
208
209
210
211
212
213
214static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
215{
216
217 if (n != -1)
218 cpumask_check(n);
219 return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
220}
221
222int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
223int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
224unsigned int cpumask_local_spread(unsigned int i, int node);
225
226
227
228
229
230
231
232
233#define for_each_cpu(cpu, mask) \
234 for ((cpu) = -1; \
235 (cpu) = cpumask_next((cpu), (mask)), \
236 (cpu) < nr_cpu_ids;)
237
238
239
240
241
242
243
244
245#define for_each_cpu_not(cpu, mask) \
246 for ((cpu) = -1; \
247 (cpu) = cpumask_next_zero((cpu), (mask)), \
248 (cpu) < nr_cpu_ids;)
249
250extern int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap);
251
252
253
254
255
256
257
258
259
260
261
262#define for_each_cpu_wrap(cpu, mask, start) \
263 for ((cpu) = cpumask_next_wrap((start)-1, (mask), (start), false); \
264 (cpu) < nr_cpumask_bits; \
265 (cpu) = cpumask_next_wrap((cpu), (mask), (start), true))
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281#define for_each_cpu_and(cpu, mask, and) \
282 for ((cpu) = -1; \
283 (cpu) = cpumask_next_and((cpu), (mask), (and)), \
284 (cpu) < nr_cpu_ids;)
285#endif
286
287#define CPU_BITS_NONE \
288{ \
289 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
290}
291
292#define CPU_BITS_CPU0 \
293{ \
294 [0] = 1UL \
295}
296
297
298
299
300
301
302static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
303{
304 set_bit(cpumask_check(cpu), cpumask_bits(dstp));
305}
306
307static inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
308{
309 __set_bit(cpumask_check(cpu), cpumask_bits(dstp));
310}
311
312
313
314
315
316
317
318static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
319{
320 clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
321}
322
323static inline void __cpumask_clear_cpu(int cpu, struct cpumask *dstp)
324{
325 __clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
326}
327
328
329
330
331
332
333
334
335static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
336{
337 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
338}
339
340
341
342
343
344
345
346
347
348
349static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
350{
351 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
352}
353
354
355
356
357
358
359
360
361
362
363static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
364{
365 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
366}
367
368
369
370
371
372static inline void cpumask_setall(struct cpumask *dstp)
373{
374 bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
375}
376
377
378
379
380
381static inline void cpumask_clear(struct cpumask *dstp)
382{
383 bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
384}
385
386
387
388
389
390
391
392
393
394static inline int cpumask_and(struct cpumask *dstp,
395 const struct cpumask *src1p,
396 const struct cpumask *src2p)
397{
398 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
399 cpumask_bits(src2p), nr_cpumask_bits);
400}
401
402
403
404
405
406
407
408static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
409 const struct cpumask *src2p)
410{
411 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
412 cpumask_bits(src2p), nr_cpumask_bits);
413}
414
415
416
417
418
419
420
421static inline void cpumask_xor(struct cpumask *dstp,
422 const struct cpumask *src1p,
423 const struct cpumask *src2p)
424{
425 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
426 cpumask_bits(src2p), nr_cpumask_bits);
427}
428
429
430
431
432
433
434
435
436
437static inline int cpumask_andnot(struct cpumask *dstp,
438 const struct cpumask *src1p,
439 const struct cpumask *src2p)
440{
441 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
442 cpumask_bits(src2p), nr_cpumask_bits);
443}
444
445
446
447
448
449
450static inline void cpumask_complement(struct cpumask *dstp,
451 const struct cpumask *srcp)
452{
453 bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
454 nr_cpumask_bits);
455}
456
457
458
459
460
461
462static inline bool cpumask_equal(const struct cpumask *src1p,
463 const struct cpumask *src2p)
464{
465 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
466 nr_cpumask_bits);
467}
468
469
470
471
472
473
474static inline bool cpumask_intersects(const struct cpumask *src1p,
475 const struct cpumask *src2p)
476{
477 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
478 nr_cpumask_bits);
479}
480
481
482
483
484
485
486
487
488static inline int cpumask_subset(const struct cpumask *src1p,
489 const struct cpumask *src2p)
490{
491 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
492 nr_cpumask_bits);
493}
494
495
496
497
498
499static inline bool cpumask_empty(const struct cpumask *srcp)
500{
501 return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
502}
503
504
505
506
507
508static inline bool cpumask_full(const struct cpumask *srcp)
509{
510 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
511}
512
513
514
515
516
517static inline unsigned int cpumask_weight(const struct cpumask *srcp)
518{
519 return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
520}
521
522
523
524
525
526
527
528static inline void cpumask_shift_right(struct cpumask *dstp,
529 const struct cpumask *srcp, int n)
530{
531 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
532 nr_cpumask_bits);
533}
534
535
536
537
538
539
540
541static inline void cpumask_shift_left(struct cpumask *dstp,
542 const struct cpumask *srcp, int n)
543{
544 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
545 nr_cpumask_bits);
546}
547
548
549
550
551
552
553static inline void cpumask_copy(struct cpumask *dstp,
554 const struct cpumask *srcp)
555{
556 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
557}
558
559
560
561
562
563
564
565#define cpumask_any(srcp) cpumask_first(srcp)
566
567
568
569
570
571
572
573
574#define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p))
575
576
577
578
579
580
581
582
583#define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
584
585
586
587
588
589#define cpumask_of(cpu) (get_cpu_mask(cpu))
590
591
592
593
594
595
596
597
598
599static inline int cpumask_parse_user(const char __user *buf, int len,
600 struct cpumask *dstp)
601{
602 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
603}
604
605
606
607
608
609
610
611
612
613static inline int cpumask_parselist_user(const char __user *buf, int len,
614 struct cpumask *dstp)
615{
616 return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
617 nr_cpumask_bits);
618}
619
620
621
622
623
624
625
626
627static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
628{
629 char *nl = strchr(buf, '\n');
630 unsigned int len = nl ? (unsigned int)(nl - buf) : strlen(buf);
631
632 return bitmap_parse(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
633}
634
635
636
637
638
639
640
641
642static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
643{
644 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
645}
646
647
648
649
650static inline unsigned int cpumask_size(void)
651{
652 return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
653}
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695#ifdef CONFIG_CPUMASK_OFFSTACK
696typedef struct cpumask *cpumask_var_t;
697
698#define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
699#define __cpumask_var_read_mostly __read_mostly
700
701bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
702bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
703bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
704bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
705void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
706void free_cpumask_var(cpumask_var_t mask);
707void free_bootmem_cpumask_var(cpumask_var_t mask);
708
709static inline bool cpumask_available(cpumask_var_t mask)
710{
711 return mask != NULL;
712}
713
714#else
715typedef struct cpumask cpumask_var_t[1];
716
717#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
718#define __cpumask_var_read_mostly
719
720static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
721{
722 return true;
723}
724
725static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
726 int node)
727{
728 return true;
729}
730
731static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
732{
733 cpumask_clear(*mask);
734 return true;
735}
736
737static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
738 int node)
739{
740 cpumask_clear(*mask);
741 return true;
742}
743
744static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
745{
746}
747
748static inline void free_cpumask_var(cpumask_var_t mask)
749{
750}
751
752static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
753{
754}
755
756static inline bool cpumask_available(cpumask_var_t mask)
757{
758 return true;
759}
760#endif
761
762
763
764extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
765#define cpu_all_mask to_cpumask(cpu_all_bits)
766
767
768#define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
769
770#define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
771#define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
772#define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
773
774
775void init_cpu_present(const struct cpumask *src);
776void init_cpu_possible(const struct cpumask *src);
777void init_cpu_online(const struct cpumask *src);
778
779static inline void reset_cpu_possible_mask(void)
780{
781 bitmap_zero(cpumask_bits(&__cpu_possible_mask), NR_CPUS);
782}
783
784static inline void
785set_cpu_possible(unsigned int cpu, bool possible)
786{
787 if (possible)
788 cpumask_set_cpu(cpu, &__cpu_possible_mask);
789 else
790 cpumask_clear_cpu(cpu, &__cpu_possible_mask);
791}
792
793static inline void
794set_cpu_present(unsigned int cpu, bool present)
795{
796 if (present)
797 cpumask_set_cpu(cpu, &__cpu_present_mask);
798 else
799 cpumask_clear_cpu(cpu, &__cpu_present_mask);
800}
801
802static inline void
803set_cpu_online(unsigned int cpu, bool online)
804{
805 if (online)
806 cpumask_set_cpu(cpu, &__cpu_online_mask);
807 else
808 cpumask_clear_cpu(cpu, &__cpu_online_mask);
809}
810
811static inline void
812set_cpu_active(unsigned int cpu, bool active)
813{
814 if (active)
815 cpumask_set_cpu(cpu, &__cpu_active_mask);
816 else
817 cpumask_clear_cpu(cpu, &__cpu_active_mask);
818}
819
820
821
822
823
824
825
826
827
828
829
830
831#define to_cpumask(bitmap) \
832 ((struct cpumask *)(1 ? (bitmap) \
833 : (void *)sizeof(__check_is_bitmap(bitmap))))
834
835static inline int __check_is_bitmap(const unsigned long *bitmap)
836{
837 return 1;
838}
839
840
841
842
843
844
845
846
847extern const unsigned long
848 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
849
850static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
851{
852 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
853 p -= cpu / BITS_PER_LONG;
854 return to_cpumask(p);
855}
856
857#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
858
859#if NR_CPUS <= BITS_PER_LONG
860#define CPU_BITS_ALL \
861{ \
862 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
863}
864
865#else
866
867#define CPU_BITS_ALL \
868{ \
869 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
870 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
871}
872#endif
873
874
875
876
877
878
879
880
881
882
883
884static inline ssize_t
885cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
886{
887 return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
888 nr_cpu_ids);
889}
890
891#if NR_CPUS <= BITS_PER_LONG
892#define CPU_MASK_ALL \
893(cpumask_t) { { \
894 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
895} }
896#else
897#define CPU_MASK_ALL \
898(cpumask_t) { { \
899 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
900 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
901} }
902#endif
903
904#define CPU_MASK_NONE \
905(cpumask_t) { { \
906 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
907} }
908
909#define CPU_MASK_CPU0 \
910(cpumask_t) { { \
911 [0] = 1UL \
912} }
913
914#endif
915