1
2#ifndef _MM_PERCPU_INTERNAL_H
3#define _MM_PERCPU_INTERNAL_H
4
5#include <linux/types.h>
6#include <linux/percpu.h>
7
8
9
10
11
12
13
14
15
16
17
18
19struct pcpu_block_md {
20 int scan_hint;
21 int scan_hint_start;
22
23 int contig_hint;
24 int contig_hint_start;
25
26 int left_free;
27
28 int right_free;
29
30 int first_free;
31 int nr_bits;
32};
33
34struct pcpu_chunk {
35#ifdef CONFIG_PERCPU_STATS
36 int nr_alloc;
37 size_t max_alloc_size;
38#endif
39
40 struct list_head list;
41 int free_bytes;
42 struct pcpu_block_md chunk_md;
43 void *base_addr;
44
45 unsigned long *alloc_map;
46 unsigned long *bound_map;
47 struct pcpu_block_md *md_blocks;
48
49 void *data;
50 bool immutable;
51 bool isolated;
52
53 int start_offset;
54
55
56 int end_offset;
57
58
59#ifdef CONFIG_MEMCG_KMEM
60 struct obj_cgroup **obj_cgroups;
61#endif
62
63 int nr_pages;
64 int nr_populated;
65 int nr_empty_pop_pages;
66 unsigned long populated[];
67};
68
69extern spinlock_t pcpu_lock;
70
71extern struct list_head *pcpu_chunk_lists;
72extern int pcpu_nr_slots;
73extern int pcpu_sidelined_slot;
74extern int pcpu_to_depopulate_slot;
75extern int pcpu_nr_empty_pop_pages;
76
77extern struct pcpu_chunk *pcpu_first_chunk;
78extern struct pcpu_chunk *pcpu_reserved_chunk;
79
80
81
82
83
84
85
86
87static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk)
88{
89 return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE;
90}
91
92
93
94
95
96
97
98
99static inline int pcpu_nr_pages_to_map_bits(int pages)
100{
101 return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE;
102}
103
104
105
106
107
108
109
110
111static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk)
112{
113 return pcpu_nr_pages_to_map_bits(chunk->nr_pages);
114}
115
116#ifdef CONFIG_PERCPU_STATS
117
118#include <linux/spinlock.h>
119
120struct percpu_stats {
121 u64 nr_alloc;
122 u64 nr_dealloc;
123 u64 nr_cur_alloc;
124 u64 nr_max_alloc;
125 u32 nr_chunks;
126 u32 nr_max_chunks;
127 size_t min_alloc_size;
128 size_t max_alloc_size;
129};
130
131extern struct percpu_stats pcpu_stats;
132extern struct pcpu_alloc_info pcpu_stats_ai;
133
134
135
136
137static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
138{
139 memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info));
140
141
142 pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size;
143}
144
145
146
147
148
149
150
151
152
153static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
154{
155 lockdep_assert_held(&pcpu_lock);
156
157 pcpu_stats.nr_alloc++;
158 pcpu_stats.nr_cur_alloc++;
159 pcpu_stats.nr_max_alloc =
160 max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc);
161 pcpu_stats.min_alloc_size =
162 min(pcpu_stats.min_alloc_size, size);
163 pcpu_stats.max_alloc_size =
164 max(pcpu_stats.max_alloc_size, size);
165
166 chunk->nr_alloc++;
167 chunk->max_alloc_size = max(chunk->max_alloc_size, size);
168}
169
170
171
172
173
174
175
176
177static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
178{
179 lockdep_assert_held(&pcpu_lock);
180
181 pcpu_stats.nr_dealloc++;
182 pcpu_stats.nr_cur_alloc--;
183
184 chunk->nr_alloc--;
185}
186
187
188
189
190static inline void pcpu_stats_chunk_alloc(void)
191{
192 unsigned long flags;
193 spin_lock_irqsave(&pcpu_lock, flags);
194
195 pcpu_stats.nr_chunks++;
196 pcpu_stats.nr_max_chunks =
197 max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks);
198
199 spin_unlock_irqrestore(&pcpu_lock, flags);
200}
201
202
203
204
205static inline void pcpu_stats_chunk_dealloc(void)
206{
207 unsigned long flags;
208 spin_lock_irqsave(&pcpu_lock, flags);
209
210 pcpu_stats.nr_chunks--;
211
212 spin_unlock_irqrestore(&pcpu_lock, flags);
213}
214
215#else
216
217static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
218{
219}
220
221static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
222{
223}
224
225static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
226{
227}
228
229static inline void pcpu_stats_chunk_alloc(void)
230{
231}
232
233static inline void pcpu_stats_chunk_dealloc(void)
234{
235}
236
237#endif
238
239#endif
240