1
2#ifndef _MM_PERCPU_INTERNAL_H
3#define _MM_PERCPU_INTERNAL_H
4
5#include <linux/types.h>
6#include <linux/percpu.h>
7
8
9
10
11
12
13
14
15
16
17
18enum pcpu_chunk_type {
19 PCPU_CHUNK_ROOT,
20#ifdef CONFIG_MEMCG_KMEM
21 PCPU_CHUNK_MEMCG,
22#endif
23 PCPU_NR_CHUNK_TYPES,
24 PCPU_FAIL_ALLOC = PCPU_NR_CHUNK_TYPES
25};
26
27
28
29
30
31
32
33
34
35
36
37
38struct pcpu_block_md {
39 int scan_hint;
40 int scan_hint_start;
41
42 int contig_hint;
43 int contig_hint_start;
44
45 int left_free;
46
47 int right_free;
48
49 int first_free;
50 int nr_bits;
51};
52
53struct pcpu_chunk {
54#ifdef CONFIG_PERCPU_STATS
55 int nr_alloc;
56 size_t max_alloc_size;
57#endif
58
59 struct list_head list;
60 int free_bytes;
61 struct pcpu_block_md chunk_md;
62 void *base_addr;
63
64 unsigned long *alloc_map;
65 unsigned long *bound_map;
66 struct pcpu_block_md *md_blocks;
67
68 void *data;
69 bool immutable;
70 int start_offset;
71
72
73 int end_offset;
74
75
76#ifdef CONFIG_MEMCG_KMEM
77 struct obj_cgroup **obj_cgroups;
78#endif
79
80 int nr_pages;
81 int nr_populated;
82 int nr_empty_pop_pages;
83 unsigned long populated[];
84};
85
86extern spinlock_t pcpu_lock;
87
88extern struct list_head *pcpu_chunk_lists;
89extern int pcpu_nr_slots;
90extern int pcpu_nr_empty_pop_pages[];
91
92extern struct pcpu_chunk *pcpu_first_chunk;
93extern struct pcpu_chunk *pcpu_reserved_chunk;
94
95
96
97
98
99
100
101
102static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk)
103{
104 return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE;
105}
106
107
108
109
110
111
112
113
114static inline int pcpu_nr_pages_to_map_bits(int pages)
115{
116 return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE;
117}
118
119
120
121
122
123
124
125
126static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk)
127{
128 return pcpu_nr_pages_to_map_bits(chunk->nr_pages);
129}
130
131#ifdef CONFIG_MEMCG_KMEM
132static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk)
133{
134 if (chunk->obj_cgroups)
135 return PCPU_CHUNK_MEMCG;
136 return PCPU_CHUNK_ROOT;
137}
138
139static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type)
140{
141 return chunk_type == PCPU_CHUNK_MEMCG;
142}
143
144#else
145static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk)
146{
147 return PCPU_CHUNK_ROOT;
148}
149
150static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type)
151{
152 return false;
153}
154#endif
155
156static inline struct list_head *pcpu_chunk_list(enum pcpu_chunk_type chunk_type)
157{
158 return &pcpu_chunk_lists[pcpu_nr_slots *
159 pcpu_is_memcg_chunk(chunk_type)];
160}
161
162#ifdef CONFIG_PERCPU_STATS
163
164#include <linux/spinlock.h>
165
166struct percpu_stats {
167 u64 nr_alloc;
168 u64 nr_dealloc;
169 u64 nr_cur_alloc;
170 u64 nr_max_alloc;
171 u32 nr_chunks;
172 u32 nr_max_chunks;
173 size_t min_alloc_size;
174 size_t max_alloc_size;
175};
176
177extern struct percpu_stats pcpu_stats;
178extern struct pcpu_alloc_info pcpu_stats_ai;
179
180
181
182
183static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
184{
185 memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info));
186
187
188 pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size;
189}
190
191
192
193
194
195
196
197
198
199static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
200{
201 lockdep_assert_held(&pcpu_lock);
202
203 pcpu_stats.nr_alloc++;
204 pcpu_stats.nr_cur_alloc++;
205 pcpu_stats.nr_max_alloc =
206 max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc);
207 pcpu_stats.min_alloc_size =
208 min(pcpu_stats.min_alloc_size, size);
209 pcpu_stats.max_alloc_size =
210 max(pcpu_stats.max_alloc_size, size);
211
212 chunk->nr_alloc++;
213 chunk->max_alloc_size = max(chunk->max_alloc_size, size);
214}
215
216
217
218
219
220
221
222
223static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
224{
225 lockdep_assert_held(&pcpu_lock);
226
227 pcpu_stats.nr_dealloc++;
228 pcpu_stats.nr_cur_alloc--;
229
230 chunk->nr_alloc--;
231}
232
233
234
235
236static inline void pcpu_stats_chunk_alloc(void)
237{
238 unsigned long flags;
239 spin_lock_irqsave(&pcpu_lock, flags);
240
241 pcpu_stats.nr_chunks++;
242 pcpu_stats.nr_max_chunks =
243 max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks);
244
245 spin_unlock_irqrestore(&pcpu_lock, flags);
246}
247
248
249
250
251static inline void pcpu_stats_chunk_dealloc(void)
252{
253 unsigned long flags;
254 spin_lock_irqsave(&pcpu_lock, flags);
255
256 pcpu_stats.nr_chunks--;
257
258 spin_unlock_irqrestore(&pcpu_lock, flags);
259}
260
261#else
262
263static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
264{
265}
266
267static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
268{
269}
270
271static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
272{
273}
274
275static inline void pcpu_stats_chunk_alloc(void)
276{
277}
278
279static inline void pcpu_stats_chunk_dealloc(void)
280{
281}
282
283#endif
284
285#endif
286