1#ifndef __LINUX_CPUMASK_H
2#define __LINUX_CPUMASK_H
3
4
5
6
7
8
9#include <linux/kernel.h>
10#include <linux/threads.h>
11#include <linux/bitmap.h>
12
13typedef struct cpumask { DECLARE_BITMAP(bits, NR_CPUS); } cpumask_t;
14
15
16
17
18
19
20
21
22#define cpumask_bits(maskp) ((maskp)->bits)
23
24#if NR_CPUS == 1
25#define nr_cpu_ids 1
26#else
27extern int nr_cpu_ids;
28#endif
29
30#ifdef CONFIG_CPUMASK_OFFSTACK
31
32
33#define nr_cpumask_bits nr_cpu_ids
34#else
35#define nr_cpumask_bits NR_CPUS
36#endif
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78extern const struct cpumask *const cpu_possible_mask;
79extern const struct cpumask *const cpu_online_mask;
80extern const struct cpumask *const cpu_present_mask;
81extern const struct cpumask *const cpu_active_mask;
82
83#if NR_CPUS > 1
84#define num_online_cpus() cpumask_weight(cpu_online_mask)
85#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
86#define num_present_cpus() cpumask_weight(cpu_present_mask)
87#define cpu_online(cpu) cpumask_test_cpu((cpu), cpu_online_mask)
88#define cpu_possible(cpu) cpumask_test_cpu((cpu), cpu_possible_mask)
89#define cpu_present(cpu) cpumask_test_cpu((cpu), cpu_present_mask)
90#define cpu_active(cpu) cpumask_test_cpu((cpu), cpu_active_mask)
91#else
92#define num_online_cpus() 1
93#define num_possible_cpus() 1
94#define num_present_cpus() 1
95#define cpu_online(cpu) ((cpu) == 0)
96#define cpu_possible(cpu) ((cpu) == 0)
97#define cpu_present(cpu) ((cpu) == 0)
98#define cpu_active(cpu) ((cpu) == 0)
99#endif
100
101
102static inline unsigned int cpumask_check(unsigned int cpu)
103{
104#ifdef CONFIG_DEBUG_PER_CPU_MAPS
105 WARN_ON_ONCE(cpu >= nr_cpumask_bits);
106#endif
107 return cpu;
108}
109
110#if NR_CPUS == 1
111
112static inline unsigned int cpumask_first(const struct cpumask *srcp)
113{
114 return 0;
115}
116
117
118static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
119{
120 return n+1;
121}
122
123static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
124{
125 return n+1;
126}
127
128static inline unsigned int cpumask_next_and(int n,
129 const struct cpumask *srcp,
130 const struct cpumask *andp)
131{
132 return n+1;
133}
134
135
136static inline unsigned int cpumask_any_but(const struct cpumask *mask,
137 unsigned int cpu)
138{
139 return 1;
140}
141
142#define for_each_cpu(cpu, mask) \
143 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
144#define for_each_cpu_and(cpu, mask, and) \
145 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask, (void)and)
146#else
147
148
149
150
151
152
153static inline unsigned int cpumask_first(const struct cpumask *srcp)
154{
155 return find_first_bit(cpumask_bits(srcp), nr_cpumask_bits);
156}
157
158
159
160
161
162
163
164
165static inline unsigned int cpumask_next(int n, const struct cpumask *srcp)
166{
167
168 if (n != -1)
169 cpumask_check(n);
170 return find_next_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
171}
172
173
174
175
176
177
178
179
180static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
181{
182
183 if (n != -1)
184 cpumask_check(n);
185 return find_next_zero_bit(cpumask_bits(srcp), nr_cpumask_bits, n+1);
186}
187
188int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *);
189int cpumask_any_but(const struct cpumask *mask, unsigned int cpu);
190
191
192
193
194
195
196
197
198#define for_each_cpu(cpu, mask) \
199 for ((cpu) = -1; \
200 (cpu) = cpumask_next((cpu), (mask)), \
201 (cpu) < nr_cpu_ids;)
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217#define for_each_cpu_and(cpu, mask, and) \
218 for ((cpu) = -1; \
219 (cpu) = cpumask_next_and((cpu), (mask), (and)), \
220 (cpu) < nr_cpu_ids;)
221#endif
222
223#define CPU_BITS_NONE \
224{ \
225 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
226}
227
228#define CPU_BITS_CPU0 \
229{ \
230 [0] = 1UL \
231}
232
233
234
235
236
237
238static inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
239{
240 set_bit(cpumask_check(cpu), cpumask_bits(dstp));
241}
242
243
244
245
246
247
248static inline void cpumask_clear_cpu(int cpu, struct cpumask *dstp)
249{
250 clear_bit(cpumask_check(cpu), cpumask_bits(dstp));
251}
252
253
254
255
256
257
258
259
260#define cpumask_test_cpu(cpu, cpumask) \
261 test_bit(cpumask_check(cpu), cpumask_bits((cpumask)))
262
263
264
265
266
267
268
269
270static inline int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
271{
272 return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
273}
274
275
276
277
278
279
280
281
282static inline int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
283{
284 return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
285}
286
287
288
289
290
291static inline void cpumask_setall(struct cpumask *dstp)
292{
293 bitmap_fill(cpumask_bits(dstp), nr_cpumask_bits);
294}
295
296
297
298
299
300static inline void cpumask_clear(struct cpumask *dstp)
301{
302 bitmap_zero(cpumask_bits(dstp), nr_cpumask_bits);
303}
304
305
306
307
308
309
310
311static inline int cpumask_and(struct cpumask *dstp,
312 const struct cpumask *src1p,
313 const struct cpumask *src2p)
314{
315 return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
316 cpumask_bits(src2p), nr_cpumask_bits);
317}
318
319
320
321
322
323
324
325static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
326 const struct cpumask *src2p)
327{
328 bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
329 cpumask_bits(src2p), nr_cpumask_bits);
330}
331
332
333
334
335
336
337
338static inline void cpumask_xor(struct cpumask *dstp,
339 const struct cpumask *src1p,
340 const struct cpumask *src2p)
341{
342 bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
343 cpumask_bits(src2p), nr_cpumask_bits);
344}
345
346
347
348
349
350
351
352static inline int cpumask_andnot(struct cpumask *dstp,
353 const struct cpumask *src1p,
354 const struct cpumask *src2p)
355{
356 return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
357 cpumask_bits(src2p), nr_cpumask_bits);
358}
359
360
361
362
363
364
365static inline void cpumask_complement(struct cpumask *dstp,
366 const struct cpumask *srcp)
367{
368 bitmap_complement(cpumask_bits(dstp), cpumask_bits(srcp),
369 nr_cpumask_bits);
370}
371
372
373
374
375
376
377static inline bool cpumask_equal(const struct cpumask *src1p,
378 const struct cpumask *src2p)
379{
380 return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
381 nr_cpumask_bits);
382}
383
384
385
386
387
388
389static inline bool cpumask_intersects(const struct cpumask *src1p,
390 const struct cpumask *src2p)
391{
392 return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
393 nr_cpumask_bits);
394}
395
396
397
398
399
400
401static inline int cpumask_subset(const struct cpumask *src1p,
402 const struct cpumask *src2p)
403{
404 return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
405 nr_cpumask_bits);
406}
407
408
409
410
411
412static inline bool cpumask_empty(const struct cpumask *srcp)
413{
414 return bitmap_empty(cpumask_bits(srcp), nr_cpumask_bits);
415}
416
417
418
419
420
421static inline bool cpumask_full(const struct cpumask *srcp)
422{
423 return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
424}
425
426
427
428
429
430static inline unsigned int cpumask_weight(const struct cpumask *srcp)
431{
432 return bitmap_weight(cpumask_bits(srcp), nr_cpumask_bits);
433}
434
435
436
437
438
439
440
441static inline void cpumask_shift_right(struct cpumask *dstp,
442 const struct cpumask *srcp, int n)
443{
444 bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
445 nr_cpumask_bits);
446}
447
448
449
450
451
452
453
454static inline void cpumask_shift_left(struct cpumask *dstp,
455 const struct cpumask *srcp, int n)
456{
457 bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
458 nr_cpumask_bits);
459}
460
461
462
463
464
465
466static inline void cpumask_copy(struct cpumask *dstp,
467 const struct cpumask *srcp)
468{
469 bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), nr_cpumask_bits);
470}
471
472
473
474
475
476
477
478#define cpumask_any(srcp) cpumask_first(srcp)
479
480
481
482
483
484
485
486
487#define cpumask_first_and(src1p, src2p) cpumask_next_and(-1, (src1p), (src2p))
488
489
490
491
492
493
494
495
496#define cpumask_any_and(mask1, mask2) cpumask_first_and((mask1), (mask2))
497
498
499
500
501
502#define cpumask_of(cpu) (get_cpu_mask(cpu))
503
504
505
506
507
508
509
510
511
512
513static inline int cpumask_scnprintf(char *buf, int len,
514 const struct cpumask *srcp)
515{
516 return bitmap_scnprintf(buf, len, cpumask_bits(srcp), nr_cpumask_bits);
517}
518
519
520
521
522
523
524
525
526
527static inline int cpumask_parse_user(const char __user *buf, int len,
528 struct cpumask *dstp)
529{
530 return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
531}
532
533
534
535
536
537
538
539
540
541
542static inline int cpulist_scnprintf(char *buf, int len,
543 const struct cpumask *srcp)
544{
545 return bitmap_scnlistprintf(buf, len, cpumask_bits(srcp),
546 nr_cpumask_bits);
547}
548
549
550
551
552
553
554
555
556
557static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
558{
559 return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
560}
561
562
563
564
565
566
567static inline size_t cpumask_size(void)
568{
569
570
571 return BITS_TO_LONGS(NR_CPUS) * sizeof(long);
572}
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590#ifdef CONFIG_CPUMASK_OFFSTACK
591typedef struct cpumask *cpumask_var_t;
592
593bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
594bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
595bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
596bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags);
597void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
598void free_cpumask_var(cpumask_var_t mask);
599void free_bootmem_cpumask_var(cpumask_var_t mask);
600
601#else
602typedef struct cpumask cpumask_var_t[1];
603
604static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
605{
606 return true;
607}
608
609static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
610 int node)
611{
612 return true;
613}
614
615static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
616{
617 cpumask_clear(*mask);
618 return true;
619}
620
621static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
622 int node)
623{
624 cpumask_clear(*mask);
625 return true;
626}
627
628static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
629{
630}
631
632static inline void free_cpumask_var(cpumask_var_t mask)
633{
634}
635
636static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
637{
638}
639#endif
640
641
642
643extern const DECLARE_BITMAP(cpu_all_bits, NR_CPUS);
644#define cpu_all_mask to_cpumask(cpu_all_bits)
645
646
647#define cpu_none_mask to_cpumask(cpu_bit_bitmap[0])
648
649#define for_each_possible_cpu(cpu) for_each_cpu((cpu), cpu_possible_mask)
650#define for_each_online_cpu(cpu) for_each_cpu((cpu), cpu_online_mask)
651#define for_each_present_cpu(cpu) for_each_cpu((cpu), cpu_present_mask)
652
653
654void set_cpu_possible(unsigned int cpu, bool possible);
655void set_cpu_present(unsigned int cpu, bool present);
656void set_cpu_online(unsigned int cpu, bool online);
657void set_cpu_active(unsigned int cpu, bool active);
658void init_cpu_present(const struct cpumask *src);
659void init_cpu_possible(const struct cpumask *src);
660void init_cpu_online(const struct cpumask *src);
661
662
663
664
665
666
667
668
669
670
671
672#define to_cpumask(bitmap) \
673 ((struct cpumask *)(1 ? (bitmap) \
674 : (void *)sizeof(__check_is_bitmap(bitmap))))
675
676static inline int __check_is_bitmap(const unsigned long *bitmap)
677{
678 return 1;
679}
680
681
682
683
684
685
686
687
688extern const unsigned long
689 cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
690
691static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
692{
693 const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
694 p -= cpu / BITS_PER_LONG;
695 return to_cpumask(p);
696}
697
698#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu))
699
700#if NR_CPUS <= BITS_PER_LONG
701#define CPU_BITS_ALL \
702{ \
703 [BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD \
704}
705
706#else
707
708#define CPU_BITS_ALL \
709{ \
710 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
711 [BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD \
712}
713#endif
714
715
716
717
718
719
720#ifndef CONFIG_DISABLE_OBSOLETE_CPUMASK_FUNCTIONS
721
722#define cpu_possible_map (*(cpumask_t *)cpu_possible_mask)
723#define cpu_online_map (*(cpumask_t *)cpu_online_mask)
724#define cpu_present_map (*(cpumask_t *)cpu_present_mask)
725#define cpu_active_map (*(cpumask_t *)cpu_active_mask)
726
727#define cpumask_of_cpu(cpu) (*get_cpu_mask(cpu))
728
729#define CPU_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(NR_CPUS)
730
731#if NR_CPUS <= BITS_PER_LONG
732
733#define CPU_MASK_ALL \
734(cpumask_t) { { \
735 [BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD \
736} }
737
738#else
739
740#define CPU_MASK_ALL \
741(cpumask_t) { { \
742 [0 ... BITS_TO_LONGS(NR_CPUS)-2] = ~0UL, \
743 [BITS_TO_LONGS(NR_CPUS)-1] = CPU_MASK_LAST_WORD \
744} }
745
746#endif
747
748#define CPU_MASK_NONE \
749(cpumask_t) { { \
750 [0 ... BITS_TO_LONGS(NR_CPUS)-1] = 0UL \
751} }
752
753#define CPU_MASK_CPU0 \
754(cpumask_t) { { \
755 [0] = 1UL \
756} }
757
758#if NR_CPUS == 1
759#define first_cpu(src) ({ (void)(src); 0; })
760#define next_cpu(n, src) ({ (void)(src); 1; })
761#define any_online_cpu(mask) 0
762#define for_each_cpu_mask(cpu, mask) \
763 for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask)
764#else
765int __first_cpu(const cpumask_t *srcp);
766int __next_cpu(int n, const cpumask_t *srcp);
767int __any_online_cpu(const cpumask_t *mask);
768
769#define first_cpu(src) __first_cpu(&(src))
770#define next_cpu(n, src) __next_cpu((n), &(src))
771#define any_online_cpu(mask) __any_online_cpu(&(mask))
772#define for_each_cpu_mask(cpu, mask) \
773 for ((cpu) = -1; \
774 (cpu) = next_cpu((cpu), (mask)), \
775 (cpu) < NR_CPUS; )
776#endif
777
778#if NR_CPUS <= 64
779
780#define for_each_cpu_mask_nr(cpu, mask) for_each_cpu_mask(cpu, mask)
781
782#else
783
784int __next_cpu_nr(int n, const cpumask_t *srcp);
785#define for_each_cpu_mask_nr(cpu, mask) \
786 for ((cpu) = -1; \
787 (cpu) = __next_cpu_nr((cpu), &(mask)), \
788 (cpu) < nr_cpu_ids; )
789
790#endif
791
792#define cpus_addr(src) ((src).bits)
793
794#define cpu_set(cpu, dst) __cpu_set((cpu), &(dst))
795static inline void __cpu_set(int cpu, volatile cpumask_t *dstp)
796{
797 set_bit(cpu, dstp->bits);
798}
799
800#define cpu_clear(cpu, dst) __cpu_clear((cpu), &(dst))
801static inline void __cpu_clear(int cpu, volatile cpumask_t *dstp)
802{
803 clear_bit(cpu, dstp->bits);
804}
805
806#define cpus_setall(dst) __cpus_setall(&(dst), NR_CPUS)
807static inline void __cpus_setall(cpumask_t *dstp, int nbits)
808{
809 bitmap_fill(dstp->bits, nbits);
810}
811
812#define cpus_clear(dst) __cpus_clear(&(dst), NR_CPUS)
813static inline void __cpus_clear(cpumask_t *dstp, int nbits)
814{
815 bitmap_zero(dstp->bits, nbits);
816}
817
818
819#define cpu_isset(cpu, cpumask) test_bit((cpu), (cpumask).bits)
820
821#define cpu_test_and_set(cpu, cpumask) __cpu_test_and_set((cpu), &(cpumask))
822static inline int __cpu_test_and_set(int cpu, cpumask_t *addr)
823{
824 return test_and_set_bit(cpu, addr->bits);
825}
826
827#define cpus_and(dst, src1, src2) __cpus_and(&(dst), &(src1), &(src2), NR_CPUS)
828static inline int __cpus_and(cpumask_t *dstp, const cpumask_t *src1p,
829 const cpumask_t *src2p, int nbits)
830{
831 return bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
832}
833
834#define cpus_or(dst, src1, src2) __cpus_or(&(dst), &(src1), &(src2), NR_CPUS)
835static inline void __cpus_or(cpumask_t *dstp, const cpumask_t *src1p,
836 const cpumask_t *src2p, int nbits)
837{
838 bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
839}
840
841#define cpus_xor(dst, src1, src2) __cpus_xor(&(dst), &(src1), &(src2), NR_CPUS)
842static inline void __cpus_xor(cpumask_t *dstp, const cpumask_t *src1p,
843 const cpumask_t *src2p, int nbits)
844{
845 bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
846}
847
848#define cpus_andnot(dst, src1, src2) \
849 __cpus_andnot(&(dst), &(src1), &(src2), NR_CPUS)
850static inline int __cpus_andnot(cpumask_t *dstp, const cpumask_t *src1p,
851 const cpumask_t *src2p, int nbits)
852{
853 return bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
854}
855
856#define cpus_equal(src1, src2) __cpus_equal(&(src1), &(src2), NR_CPUS)
857static inline int __cpus_equal(const cpumask_t *src1p,
858 const cpumask_t *src2p, int nbits)
859{
860 return bitmap_equal(src1p->bits, src2p->bits, nbits);
861}
862
863#define cpus_intersects(src1, src2) __cpus_intersects(&(src1), &(src2), NR_CPUS)
864static inline int __cpus_intersects(const cpumask_t *src1p,
865 const cpumask_t *src2p, int nbits)
866{
867 return bitmap_intersects(src1p->bits, src2p->bits, nbits);
868}
869
870#define cpus_subset(src1, src2) __cpus_subset(&(src1), &(src2), NR_CPUS)
871static inline int __cpus_subset(const cpumask_t *src1p,
872 const cpumask_t *src2p, int nbits)
873{
874 return bitmap_subset(src1p->bits, src2p->bits, nbits);
875}
876
877#define cpus_empty(src) __cpus_empty(&(src), NR_CPUS)
878static inline int __cpus_empty(const cpumask_t *srcp, int nbits)
879{
880 return bitmap_empty(srcp->bits, nbits);
881}
882
883#define cpus_weight(cpumask) __cpus_weight(&(cpumask), NR_CPUS)
884static inline int __cpus_weight(const cpumask_t *srcp, int nbits)
885{
886 return bitmap_weight(srcp->bits, nbits);
887}
888
889#define cpus_shift_left(dst, src, n) \
890 __cpus_shift_left(&(dst), &(src), (n), NR_CPUS)
891static inline void __cpus_shift_left(cpumask_t *dstp,
892 const cpumask_t *srcp, int n, int nbits)
893{
894 bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
895}
896#endif
897
898#endif
899