1#ifndef __LINUX_NODEMASK_H
2#define __LINUX_NODEMASK_H
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90#include <linux/kernel.h>
91#include <linux/threads.h>
92#include <linux/bitmap.h>
93#include <linux/numa.h>
94
95typedef struct { DECLARE_BITMAP(bits, MAX_NUMNODES); } nodemask_t;
96extern nodemask_t _unused_nodemask_arg_;
97
98
99
100
101
102
103
104#define nodemask_pr_args(maskp) MAX_NUMNODES, (maskp)->bits
105
106
107
108
109
110
111
112
113
114
115#define node_set(node, dst) __node_set((node), &(dst))
116static __always_inline void __node_set(int node, volatile nodemask_t *dstp)
117{
118 set_bit(node, dstp->bits);
119}
120
121#define node_clear(node, dst) __node_clear((node), &(dst))
122static inline void __node_clear(int node, volatile nodemask_t *dstp)
123{
124 clear_bit(node, dstp->bits);
125}
126
127#define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES)
128static inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)
129{
130 bitmap_fill(dstp->bits, nbits);
131}
132
133#define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES)
134static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)
135{
136 bitmap_zero(dstp->bits, nbits);
137}
138
139
140#define node_isset(node, nodemask) test_bit((node), (nodemask).bits)
141
142#define node_test_and_set(node, nodemask) \
143 __node_test_and_set((node), &(nodemask))
144static inline int __node_test_and_set(int node, nodemask_t *addr)
145{
146 return test_and_set_bit(node, addr->bits);
147}
148
149#define nodes_and(dst, src1, src2) \
150 __nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES)
151static inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p,
152 const nodemask_t *src2p, unsigned int nbits)
153{
154 bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
155}
156
157#define nodes_or(dst, src1, src2) \
158 __nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES)
159static inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p,
160 const nodemask_t *src2p, unsigned int nbits)
161{
162 bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
163}
164
165#define nodes_xor(dst, src1, src2) \
166 __nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES)
167static inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p,
168 const nodemask_t *src2p, unsigned int nbits)
169{
170 bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
171}
172
173#define nodes_andnot(dst, src1, src2) \
174 __nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES)
175static inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p,
176 const nodemask_t *src2p, unsigned int nbits)
177{
178 bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
179}
180
181#define nodes_complement(dst, src) \
182 __nodes_complement(&(dst), &(src), MAX_NUMNODES)
183static inline void __nodes_complement(nodemask_t *dstp,
184 const nodemask_t *srcp, unsigned int nbits)
185{
186 bitmap_complement(dstp->bits, srcp->bits, nbits);
187}
188
189#define nodes_equal(src1, src2) \
190 __nodes_equal(&(src1), &(src2), MAX_NUMNODES)
191static inline int __nodes_equal(const nodemask_t *src1p,
192 const nodemask_t *src2p, unsigned int nbits)
193{
194 return bitmap_equal(src1p->bits, src2p->bits, nbits);
195}
196
197#define nodes_intersects(src1, src2) \
198 __nodes_intersects(&(src1), &(src2), MAX_NUMNODES)
199static inline int __nodes_intersects(const nodemask_t *src1p,
200 const nodemask_t *src2p, unsigned int nbits)
201{
202 return bitmap_intersects(src1p->bits, src2p->bits, nbits);
203}
204
205#define nodes_subset(src1, src2) \
206 __nodes_subset(&(src1), &(src2), MAX_NUMNODES)
207static inline int __nodes_subset(const nodemask_t *src1p,
208 const nodemask_t *src2p, unsigned int nbits)
209{
210 return bitmap_subset(src1p->bits, src2p->bits, nbits);
211}
212
213#define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES)
214static inline int __nodes_empty(const nodemask_t *srcp, unsigned int nbits)
215{
216 return bitmap_empty(srcp->bits, nbits);
217}
218
219#define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES)
220static inline int __nodes_full(const nodemask_t *srcp, unsigned int nbits)
221{
222 return bitmap_full(srcp->bits, nbits);
223}
224
225#define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES)
226static inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)
227{
228 return bitmap_weight(srcp->bits, nbits);
229}
230
231#define nodes_shift_right(dst, src, n) \
232 __nodes_shift_right(&(dst), &(src), (n), MAX_NUMNODES)
233static inline void __nodes_shift_right(nodemask_t *dstp,
234 const nodemask_t *srcp, int n, int nbits)
235{
236 bitmap_shift_right(dstp->bits, srcp->bits, n, nbits);
237}
238
239#define nodes_shift_left(dst, src, n) \
240 __nodes_shift_left(&(dst), &(src), (n), MAX_NUMNODES)
241static inline void __nodes_shift_left(nodemask_t *dstp,
242 const nodemask_t *srcp, int n, int nbits)
243{
244 bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
245}
246
247
248
249
250#define first_node(src) __first_node(&(src))
251static inline int __first_node(const nodemask_t *srcp)
252{
253 return min_t(int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES));
254}
255
256#define next_node(n, src) __next_node((n), &(src))
257static inline int __next_node(int n, const nodemask_t *srcp)
258{
259 return min_t(int,MAX_NUMNODES,find_next_bit(srcp->bits, MAX_NUMNODES, n+1));
260}
261
262static inline void init_nodemask_of_node(nodemask_t *mask, int node)
263{
264 nodes_clear(*mask);
265 node_set(node, *mask);
266}
267
268#define nodemask_of_node(node) \
269({ \
270 typeof(_unused_nodemask_arg_) m; \
271 if (sizeof(m) == sizeof(unsigned long)) { \
272 m.bits[0] = 1UL << (node); \
273 } else { \
274 init_nodemask_of_node(&m, (node)); \
275 } \
276 m; \
277})
278
279#define first_unset_node(mask) __first_unset_node(&(mask))
280static inline int __first_unset_node(const nodemask_t *maskp)
281{
282 return min_t(int,MAX_NUMNODES,
283 find_first_zero_bit(maskp->bits, MAX_NUMNODES));
284}
285
286#define NODE_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(MAX_NUMNODES)
287
288#if MAX_NUMNODES <= BITS_PER_LONG
289
290#define NODE_MASK_ALL \
291((nodemask_t) { { \
292 [BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD \
293} })
294
295#else
296
297#define NODE_MASK_ALL \
298((nodemask_t) { { \
299 [0 ... BITS_TO_LONGS(MAX_NUMNODES)-2] = ~0UL, \
300 [BITS_TO_LONGS(MAX_NUMNODES)-1] = NODE_MASK_LAST_WORD \
301} })
302
303#endif
304
305#define NODE_MASK_NONE \
306((nodemask_t) { { \
307 [0 ... BITS_TO_LONGS(MAX_NUMNODES)-1] = 0UL \
308} })
309
310#define nodes_addr(src) ((src).bits)
311
312#define nodemask_parse_user(ubuf, ulen, dst) \
313 __nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES)
314static inline int __nodemask_parse_user(const char __user *buf, int len,
315 nodemask_t *dstp, int nbits)
316{
317 return bitmap_parse_user(buf, len, dstp->bits, nbits);
318}
319
320#define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES)
321static inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits)
322{
323 return bitmap_parselist(buf, dstp->bits, nbits);
324}
325
326#define node_remap(oldbit, old, new) \
327 __node_remap((oldbit), &(old), &(new), MAX_NUMNODES)
328static inline int __node_remap(int oldbit,
329 const nodemask_t *oldp, const nodemask_t *newp, int nbits)
330{
331 return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits);
332}
333
334#define nodes_remap(dst, src, old, new) \
335 __nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES)
336static inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp,
337 const nodemask_t *oldp, const nodemask_t *newp, int nbits)
338{
339 bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits);
340}
341
342#define nodes_onto(dst, orig, relmap) \
343 __nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES)
344static inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp,
345 const nodemask_t *relmapp, int nbits)
346{
347 bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits);
348}
349
350#define nodes_fold(dst, orig, sz) \
351 __nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES)
352static inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp,
353 int sz, int nbits)
354{
355 bitmap_fold(dstp->bits, origp->bits, sz, nbits);
356}
357
358#if MAX_NUMNODES > 1
359#define for_each_node_mask(node, mask) \
360 for ((node) = first_node(mask); \
361 (node) < MAX_NUMNODES; \
362 (node) = next_node((node), (mask)))
363#else
364#define for_each_node_mask(node, mask) \
365 if (!nodes_empty(mask)) \
366 for ((node) = 0; (node) < 1; (node)++)
367#endif
368
369
370
371
372enum node_states {
373 N_POSSIBLE,
374 N_ONLINE,
375 N_NORMAL_MEMORY,
376#ifdef CONFIG_HIGHMEM
377 N_HIGH_MEMORY,
378#else
379 N_HIGH_MEMORY = N_NORMAL_MEMORY,
380#endif
381#ifdef CONFIG_MOVABLE_NODE
382 N_MEMORY,
383#else
384 N_MEMORY = N_HIGH_MEMORY,
385#endif
386 N_CPU,
387 NR_NODE_STATES
388};
389
390
391
392
393
394
395extern nodemask_t node_states[NR_NODE_STATES];
396
397#if MAX_NUMNODES > 1
398static inline int node_state(int node, enum node_states state)
399{
400 return node_isset(node, node_states[state]);
401}
402
403static inline void node_set_state(int node, enum node_states state)
404{
405 __node_set(node, &node_states[state]);
406}
407
408static inline void node_clear_state(int node, enum node_states state)
409{
410 __node_clear(node, &node_states[state]);
411}
412
413static inline int num_node_state(enum node_states state)
414{
415 return nodes_weight(node_states[state]);
416}
417
418#define for_each_node_state(__node, __state) \
419 for_each_node_mask((__node), node_states[__state])
420
421#define first_online_node first_node(node_states[N_ONLINE])
422#define first_memory_node first_node(node_states[N_MEMORY])
423static inline int next_online_node(int nid)
424{
425 return next_node(nid, node_states[N_ONLINE]);
426}
427static inline int next_memory_node(int nid)
428{
429 return next_node(nid, node_states[N_MEMORY]);
430}
431
432extern int nr_node_ids;
433extern int nr_online_nodes;
434
435static inline void node_set_online(int nid)
436{
437 node_set_state(nid, N_ONLINE);
438 nr_online_nodes = num_node_state(N_ONLINE);
439}
440
441static inline void node_set_offline(int nid)
442{
443 node_clear_state(nid, N_ONLINE);
444 nr_online_nodes = num_node_state(N_ONLINE);
445}
446
447#else
448
449static inline int node_state(int node, enum node_states state)
450{
451 return node == 0;
452}
453
454static inline void node_set_state(int node, enum node_states state)
455{
456}
457
458static inline void node_clear_state(int node, enum node_states state)
459{
460}
461
462static inline int num_node_state(enum node_states state)
463{
464 return 1;
465}
466
467#define for_each_node_state(node, __state) \
468 for ( (node) = 0; (node) == 0; (node) = 1)
469
470#define first_online_node 0
471#define first_memory_node 0
472#define next_online_node(nid) (MAX_NUMNODES)
473#define nr_node_ids 1
474#define nr_online_nodes 1
475
476#define node_set_online(node) node_set_state((node), N_ONLINE)
477#define node_set_offline(node) node_clear_state((node), N_ONLINE)
478
479#endif
480
481#if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
482extern int node_random(const nodemask_t *maskp);
483#else
484static inline int node_random(const nodemask_t *mask)
485{
486 return 0;
487}
488#endif
489
490#define node_online_map node_states[N_ONLINE]
491#define node_possible_map node_states[N_POSSIBLE]
492
493#define num_online_nodes() num_node_state(N_ONLINE)
494#define num_possible_nodes() num_node_state(N_POSSIBLE)
495#define node_online(node) node_state((node), N_ONLINE)
496#define node_possible(node) node_state((node), N_POSSIBLE)
497
498#define for_each_node(node) for_each_node_state(node, N_POSSIBLE)
499#define for_each_online_node(node) for_each_node_state(node, N_ONLINE)
500
501
502
503
504
505
506#if NODES_SHIFT > 8
507#define NODEMASK_ALLOC(type, name, gfp_flags) \
508 type *name = kmalloc(sizeof(*name), gfp_flags)
509#define NODEMASK_FREE(m) kfree(m)
510#else
511#define NODEMASK_ALLOC(type, name, gfp_flags) type _##name, *name = &_##name
512#define NODEMASK_FREE(m) do {} while (0)
513#endif
514
515
516struct nodemask_scratch {
517 nodemask_t mask1;
518 nodemask_t mask2;
519};
520
521#define NODEMASK_SCRATCH(x) \
522 NODEMASK_ALLOC(struct nodemask_scratch, x, \
523 GFP_KERNEL | __GFP_NORETRY)
524#define NODEMASK_SCRATCH_FREE(x) NODEMASK_FREE(x)
525
526
527#endif
528