1#ifndef __LINUX_CPUMASK_H
2#define __LINUX_CPUMASK_H
3
4
5
6
7
8
9#include <linux/kernel.h>
10#include <linux/threads.h>
11#include <linux/bitmap.h>
12#include <linux/bug.h>
13
14
15typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
16
17
18
19
20
21
22
23
24#define cpumask_bits(maskp) ((maskp)->bits)
25
26
27
28
29
30
31
32#define cpumask_pr_args(maskp) nr_cpu_ids, cpumask_bits(maskp)
33
34#if NR_CPUS == 1
35#define nr_cpu_ids 1
36#else
37extern int nr_cpu_ids;
38#endif
39
40#ifdef CONFIG_CPUMASK_OFFSTACK
41
42
43#define nr_cpumask_bits nr_cpu_ids
44#else
45#define nr_cpumask_bits NR_CPUS
46#endif
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88extern struct cpumask __cpu_possible_mask;
89extern struct cpumask __cpu_online_mask;
90extern struct cpumask __cpu_present_mask;
91extern struct cpumask __cpu_active_mask;
92#define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask)
93#define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask)
94#define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask)
95#define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask)
96
97#if NR_CPUS > 1
98#define num_online_cpus() cpumask_weight(cpu_online_mask)
99#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
100#define num_present_cpus() cpumask_weight(cpu_present_mask)
101#define num_active_cpus() cpumask_weight(cpu_active_mask)
102#define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask)
103#define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask)
104#define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask)
105#define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask)
106#else
107#define num_online_cpus() 1U
108#define num_possible_cpus() 1U
109#define num_present_cpus() 1U
110#define num_active_cpus() 1U
111#define cpu_online(cpu) ((cpu) == 0)
112#define cpu_possible(cpu) ((cpu) == 0)
113#define cpu_present(cpu) ((cpu) == 0)
114#define cpu_active(cpu) ((cpu) == 0)
115#endif
116
117
118static inline unsigned int cpumask_check(unsigned int cpu)
119{
120#ifdef CONFIG_DEBUG_PER_CPU_MAPS
121 WARN_ON_ONCE(cpu >= nr_cpumask_bits);
122#endif
123 return cpu;
124}
125
126#if NR_CPUS == 1
127
128static inline unsigned int cpumask_first(const struct cpumask *srcp)
129{
130 return 0;
131}
132
133
134static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
135{
136 return n+1;
137}
138
139static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
140{
141 return n+1;
142}
143
144static inline unsigned int cpumask_next_and(int n,
145 const struct cpumask *srcp,
146 const struct cpumask *andp)
147{
148 return n+1;
149}
150
151
152static inline unsigned int cpumask_any_but(const struct cpumask *mask,
153 unsigned int cpu)
154{
155 return 1;
156}
157
158static inline unsigned int cpumask_local_spread(unsigned int i, int node)
159{
160 return 0;
161}
162
163#define for_each_cpu(cpu, mask) \
164 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
165#define for_each_cpu_not(cpu, mask) \
166 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
167#define for_each_cpu_and(cpu, mask, and) \
168 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and)
169#else
170
171
172
173
174
175
176static inline unsigned int cpumask_first(const struct cpumask *srcp)
177{
178 return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
179}
180
181
182
183
184
185
186
187
188static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
189{
190
191 if (n != -1)
192 cpumask_check(n);
193 return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
194}
195
196
197
198
199
200
201
202
203static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
204{
205
206 if (n != -1)
207 cpumask_check(n);
208 return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
209}
210
211int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
212int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
213unsigned int cpumask_local_spread(unsigned int i, int node);
214
215
216
217
218
219
220
221
222#define for_each_cpu(cpu, mask) \
223 for ((cpu) = -1; \
224 (cpu) = cpumask_next((cpu), (mask)), \
225 (cpu) < nr_cpu_ids;)
226
227
228
229
230
231
232
233
234#define for_each_cpu_not(cpu, mask) \
235 for ((cpu) = -1; \
236 (cpu) = cpumask_next_zero((cpu), (mask)), \
237 (cpu) < nr_cpu_ids;)
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253#define for_each_cpu_and(cpu, mask, and) \
254 for ((cpu) = -1; \
255 (cpu) = cpumask_next_and((cpu), (mask), (and)), \
256 (cpu) < nr_cpu_ids;)
257#endif
258
259#define CPU_BITS_NONE \
260{ \
261 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
262}
263
264#define CPU_BITS_CPU0 \
265{ \
266 [0] = 1UL \
267}
268
269
270
271
272
273
274static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
275{
276 set_bit(cpumask_check(cpu), cpumask_bits(dstp));
277}
278
279
280
281
282
283
284static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
285{
286 clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
287}
288
289
290
291
292
293
294
295
296static inline int cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
297{
298 return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
299}
300
301
302
303
304
305
306
307
308
309
310static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
311{
312 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
313}
314
315
316
317
318
319
320
321
322
323
324static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
325{
326 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
327}
328
329
330
331
332
333static inline void cpumask_setall(struct cpumask *dstp)
334{
335 bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
336}
337
338
339
340
341
342static inline void cpumask_clear(struct cpumask *dstp)
343{
344 bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
345}
346
347
348
349
350
351
352
353
354
355static inline int cpumask_and(struct cpumask *dstp,
356 const struct cpumask *src1p,
357 const struct cpumask *src2p)
358{
359 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
360 cpumask_bits(src2p), nr_cpumask_bits);
361}
362
363
364
365
366
367
368
369static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
370 const struct cpumask *src2p)
371{
372 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
373 cpumask_bits(src2p), nr_cpumask_bits);
374}
375
376
377
378
379
380
381
382static inline void cpumask_xor(struct cpumask *dstp,
383 const struct cpumask *src1p,
384 const struct cpumask *src2p)
385{
386 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
387 cpumask_bits(src2p), nr_cpumask_bits);
388}
389
390
391
392
393
394
395
396
397
398static inline int cpumask_andnot(struct cpumask *dstp,
399 const struct cpumask *src1p,
400 const struct cpumask *src2p)
401{
402 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
403 cpumask_bits(src2p), nr_cpumask_bits);
404}
405
406
407
408
409
410
411static inline void cpumask_complement(struct cpumask *dstp,
412 const struct cpumask *srcp)
413{
414 bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
415 nr_cpumask_bits);
416}
417
418
419
420
421
422
423static inline bool cpumask_equal(const struct cpumask *src1p,
424 const struct cpumask *src2p)
425{
426 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
427 nr_cpumask_bits);
428}
429
430
431
432
433
434
435static inline bool cpumask_intersects(const struct cpumask *src1p,
436 const struct cpumask *src2p)
437{
438 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
439 nr_cpumask_bits);
440}
441
442
443
444
445
446
447
448
449static inline int cpumask_subset(const struct cpumask *src1p,
450 const struct cpumask *src2p)
451{
452 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
453 nr_cpumask_bits);
454}
455
456
457
458
459
460static inline bool cpumask_empty(const struct cpumask *srcp)
461{
462 return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
463}
464
465
466
467
468
469static inline bool cpumask_full(const struct cpumask *srcp)
470{
471 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
472}
473
474
475
476
477
478static inline unsigned int cpumask_weight(const struct cpumask *srcp)
479{
480 return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
481}
482
483
484
485
486
487
488
489static inline void cpumask_shift_right(struct cpumask *dstp,
490 const struct cpumask *srcp, int n)
491{
492 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
493 nr_cpumask_bits);
494}
495
496
497
498
499
500
501
502static inline void cpumask_shift_left(struct cpumask *dstp,
503 const struct cpumask *srcp, int n)
504{
505 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
506 nr_cpumask_bits);
507}
508
509
510
511
512
513
514static inline void cpumask_copy(struct cpumask *dstp,
515 const struct cpumask *srcp)
516{
517 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
518}
519
520
521
522
523
524
525
526#define cpumask_any(srcp) cpumask_first(srcp)
527
528
529
530
531
532
533
534
535#define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p))
536
537
538
539
540
541
542
543
544#define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
545
546
547
548
549
550#define cpumask_of(cpu) (get_cpu_mask(cpu))
551
552
553
554
555
556
557
558
559
560static inline int cpumask_parse_user(const char __user *buf, int len,
561 struct cpumask *dstp)
562{
563 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpu_ids);
564}
565
566
567
568
569
570
571
572
573
574static inline int cpumask_parselist_user(const char __user *buf, int len,
575 struct cpumask *dstp)
576{
577 return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
578 nr_cpu_ids);
579}
580
581
582
583
584
585
586
587
588static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
589{
590 char *nl = strchr(buf, '\n');
591 unsigned int len = nl ? (unsigned int)(nl - buf) : strlen(buf);
592
593 return bitmap_parse(buf, len, cpumask_bits(dstp), nr_cpu_ids);
594}
595
596
597
598
599
600
601
602
603static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
604{
605 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpu_ids);
606}
607
608
609
610
611static inline size_t cpumask_size(void)
612{
613 return BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long);
614}
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653#ifdef CONFIG_CPUMASK_OFFSTACK
654typedef struct cpumask *cpumask_var_t;
655
656#define this_cpu_cpumask_var_ptr(x) this_cpu_read(x)
657
658bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
659bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
660bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
661bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
662void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
663void free_cpumask_var(cpumask_var_t mask);
664void free_bootmem_cpumask_var(cpumask_var_t mask);
665
666#else
667typedef struct cpumask cpumask_var_t[1];
668
669#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
670
671static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
672{
673 return true;
674}
675
676static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
677 int node)
678{
679 return true;
680}
681
682static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
683{
684 cpumask_clear(*mask);
685 return true;
686}
687
688static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
689 int node)
690{
691 cpumask_clear(*mask);
692 return true;
693}
694
695static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
696{
697}
698
699static inline void free_cpumask_var(cpumask_var_t mask)
700{
701}
702
703static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
704{
705}
706#endif
707
708
709
710extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
711#define cpu_all_mask to_cpumask(cpu_all_bits)
712
713
714#define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
715
716#define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
717#define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
718#define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
719
720
721void init_cpu_present(const struct cpumask *src);
722void init_cpu_possible(const struct cpumask *src);
723void init_cpu_online(const struct cpumask *src);
724
725static inline void
726set_cpu_possible(unsigned int cpu, bool possible)
727{
728 if (possible)
729 cpumask_set_cpu(cpu, &__cpu_possible_mask);
730 else
731 cpumask_clear_cpu(cpu, &__cpu_possible_mask);
732}
733
734static inline void
735set_cpu_present(unsigned int cpu, bool present)
736{
737 if (present)
738 cpumask_set_cpu(cpu, &__cpu_present_mask);
739 else
740 cpumask_clear_cpu(cpu, &__cpu_present_mask);
741}
742
743static inline void
744set_cpu_online(unsigned int cpu, bool online)
745{
746 if (online) {
747 cpumask_set_cpu(cpu, &__cpu_online_mask);
748 cpumask_set_cpu(cpu, &__cpu_active_mask);
749 } else {
750 cpumask_clear_cpu(cpu, &__cpu_online_mask);
751 }
752}
753
754static inline void
755set_cpu_active(unsigned int cpu, bool active)
756{
757 if (active)
758 cpumask_set_cpu(cpu, &__cpu_active_mask);
759 else
760 cpumask_clear_cpu(cpu, &__cpu_active_mask);
761}
762
763
764
765
766
767
768
769
770
771
772
773
774#define to_cpumask(bitmap) \
775 ((struct cpumask *)(1 ? (bitmap) \
776 : (void *)sizeof(__check_is_bitmap(bitmap))))
777
778static inline int __check_is_bitmap(const unsigned long *bitmap)
779{
780 return 1;
781}
782
783
784
785
786
787
788
789
790extern const unsigned long
791 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
792
793static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
794{
795 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
796 p -= cpu / BITS_PER_LONG;
797 return to_cpumask(p);
798}
799
800#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
801
802#if NR_CPUS <= BITS_PER_LONG
803#define CPU_BITS_ALL \
804{ \
805 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
806}
807
808#else
809
810#define CPU_BITS_ALL \
811{ \
812 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
813 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
814}
815#endif
816
817
818
819
820
821
822
823
824
825
826
827static inline ssize_t
828cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
829{
830 return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
831 nr_cpu_ids);
832}
833
834#if NR_CPUS <= BITS_PER_LONG
835#define CPU_MASK_ALL \
836(cpumask_t) { { \
837 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
838} }
839#else
840#define CPU_MASK_ALL \
841(cpumask_t) { { \
842 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
843 [BITS_TO_LONGS(NR_CPUS)-1] = BITMAP_LAST_WORD_MASK(NR_CPUS) \
844} }
845#endif
846
847#define CPU_MASK_NONE \
848(cpumask_t) { { \
849 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
850} }
851
852#define CPU_MASK_CPU0 \
853(cpumask_t) { { \
854 [0] = 1UL \
855} }
856
857#endif
858