1
2#ifndef _MM_PERCPU_INTERNAL_H
3#define _MM_PERCPU_INTERNAL_H
4
5#include <linux/types.h>
6#include <linux/percpu.h>
7
8
9
10
11
12
13struct pcpu_block_md {
14 int contig_hint;
15 int contig_hint_start;
16
17 int left_free;
18
19 int right_free;
20
21 int first_free;
22};
23
24struct pcpu_chunk {
25#ifdef CONFIG_PERCPU_STATS
26 int nr_alloc;
27 size_t max_alloc_size;
28#endif
29
30 struct list_head list;
31 int free_bytes;
32 int contig_bits;
33 int contig_bits_start;
34
35 void *base_addr;
36
37 unsigned long *alloc_map;
38 unsigned long *bound_map;
39 struct pcpu_block_md *md_blocks;
40
41 void *data;
42 int first_bit;
43 bool immutable;
44 int start_offset;
45
46
47 int end_offset;
48
49
50
51 int nr_pages;
52 int nr_populated;
53 int nr_empty_pop_pages;
54 unsigned long populated[];
55};
56
57extern spinlock_t pcpu_lock;
58
59extern struct list_head *pcpu_slot;
60extern int pcpu_nr_slots;
61extern int pcpu_nr_empty_pop_pages;
62
63extern struct pcpu_chunk *pcpu_first_chunk;
64extern struct pcpu_chunk *pcpu_reserved_chunk;
65
66
67
68
69
70
71
72
73static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk)
74{
75 return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE;
76}
77
78
79
80
81
82
83
84
85static inline int pcpu_nr_pages_to_map_bits(int pages)
86{
87 return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE;
88}
89
90
91
92
93
94
95
96
97static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk)
98{
99 return pcpu_nr_pages_to_map_bits(chunk->nr_pages);
100}
101
102#ifdef CONFIG_PERCPU_STATS
103
104#include <linux/spinlock.h>
105
106struct percpu_stats {
107 u64 nr_alloc;
108 u64 nr_dealloc;
109 u64 nr_cur_alloc;
110 u64 nr_max_alloc;
111 u32 nr_chunks;
112 u32 nr_max_chunks;
113 size_t min_alloc_size;
114 size_t max_alloc_size;
115};
116
117extern struct percpu_stats pcpu_stats;
118extern struct pcpu_alloc_info pcpu_stats_ai;
119
120
121
122
123static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
124{
125 memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info));
126
127
128 pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size;
129}
130
131
132
133
134
135
136
137
138
139static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
140{
141 lockdep_assert_held(&pcpu_lock);
142
143 pcpu_stats.nr_alloc++;
144 pcpu_stats.nr_cur_alloc++;
145 pcpu_stats.nr_max_alloc =
146 max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc);
147 pcpu_stats.min_alloc_size =
148 min(pcpu_stats.min_alloc_size, size);
149 pcpu_stats.max_alloc_size =
150 max(pcpu_stats.max_alloc_size, size);
151
152 chunk->nr_alloc++;
153 chunk->max_alloc_size = max(chunk->max_alloc_size, size);
154}
155
156
157
158
159
160
161
162
163static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
164{
165 lockdep_assert_held(&pcpu_lock);
166
167 pcpu_stats.nr_dealloc++;
168 pcpu_stats.nr_cur_alloc--;
169
170 chunk->nr_alloc--;
171}
172
173
174
175
176static inline void pcpu_stats_chunk_alloc(void)
177{
178 unsigned long flags;
179 spin_lock_irqsave(&pcpu_lock, flags);
180
181 pcpu_stats.nr_chunks++;
182 pcpu_stats.nr_max_chunks =
183 max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks);
184
185 spin_unlock_irqrestore(&pcpu_lock, flags);
186}
187
188
189
190
191static inline void pcpu_stats_chunk_dealloc(void)
192{
193 unsigned long flags;
194 spin_lock_irqsave(&pcpu_lock, flags);
195
196 pcpu_stats.nr_chunks--;
197
198 spin_unlock_irqrestore(&pcpu_lock, flags);
199}
200
201#else
202
203static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
204{
205}
206
207static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
208{
209}
210
211static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
212{
213}
214
215static inline void pcpu_stats_chunk_alloc(void)
216{
217}
218
219static inline void pcpu_stats_chunk_dealloc(void)
220{
221}
222
223#endif
224
225#endif
226