1#ifndef _LINUX_SLUB_DEF_H
2#define _LINUX_SLUB_DEF_H
3
4
5
6
7
8
9#include <linux/types.h>
10#include <linux/gfp.h>
11#include <linux/bug.h>
12#include <linux/workqueue.h>
13#include <linux/kobject.h>
14
15#include <linux/kmemleak.h>
16
17enum stat_item {
18 ALLOC_FASTPATH,
19 ALLOC_SLOWPATH,
20 FREE_FASTPATH,
21 FREE_SLOWPATH,
22 FREE_FROZEN,
23 FREE_ADD_PARTIAL,
24 FREE_REMOVE_PARTIAL,
25 ALLOC_FROM_PARTIAL,
26 ALLOC_SLAB,
27 ALLOC_REFILL,
28 ALLOC_NODE_MISMATCH,
29 FREE_SLAB,
30 CPUSLAB_FLUSH,
31 DEACTIVATE_FULL,
32 DEACTIVATE_EMPTY,
33 DEACTIVATE_TO_HEAD,
34 DEACTIVATE_TO_TAIL,
35 DEACTIVATE_REMOTE_FREES,
36 DEACTIVATE_BYPASS,
37 ORDER_FALLBACK,
38 CMPXCHG_DOUBLE_CPU_FAIL,
39 CMPXCHG_DOUBLE_FAIL,
40 CPU_PARTIAL_ALLOC,
41 CPU_PARTIAL_FREE,
42 CPU_PARTIAL_NODE,
43 CPU_PARTIAL_DRAIN,
44 NR_SLUB_STAT_ITEMS };
45
46struct kmem_cache_cpu {
47 void **freelist;
48 unsigned long tid;
49 struct page *page;
50 struct page *partial;
51 int node;
52#ifdef CONFIG_SLUB_STATS
53 unsigned stat[NR_SLUB_STAT_ITEMS];
54#endif
55};
56
57struct kmem_cache_node {
58 spinlock_t list_lock;
59 unsigned long nr_partial;
60 struct list_head partial;
61#ifdef CONFIG_SLUB_DEBUG
62 atomic_long_t nr_slabs;
63 atomic_long_t total_objects;
64 struct list_head full;
65#endif
66};
67
68
69
70
71
72
73struct kmem_cache_order_objects {
74 unsigned long x;
75};
76
77
78
79
80struct kmem_cache {
81 struct kmem_cache_cpu __percpu *cpu_slab;
82
83 unsigned long flags;
84 unsigned long min_partial;
85 int size;
86 int objsize;
87 int offset;
88 int cpu_partial;
89 struct kmem_cache_order_objects oo;
90
91
92 struct kmem_cache_order_objects max;
93 struct kmem_cache_order_objects min;
94 gfp_t allocflags;
95 int refcount;
96 void (*ctor)(void *);
97 int inuse;
98 int align;
99 int reserved;
100 const char *name;
101 struct list_head list;
102#ifdef CONFIG_SYSFS
103 struct kobject kobj;
104#endif
105
106#ifdef CONFIG_NUMA
107
108
109
110 int remote_node_defrag_ratio;
111#endif
112 struct kmem_cache_node *node[MAX_NUMNODES];
113};
114
115
116
117
118#if defined(ARCH_DMA_MINALIGN) && ARCH_DMA_MINALIGN > 8
119#define KMALLOC_MIN_SIZE ARCH_DMA_MINALIGN
120#else
121#define KMALLOC_MIN_SIZE 8
122#endif
123
124#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
125
126
127
128
129
130
131
132
133
134
135#define SLUB_MAX_SIZE (2 * PAGE_SIZE)
136
137#define SLUB_PAGE_SHIFT (PAGE_SHIFT + 2)
138
139#ifdef CONFIG_ZONE_DMA
140#define SLUB_DMA __GFP_DMA
141#else
142
143#define SLUB_DMA (__force gfp_t)0
144#endif
145
146
147
148
149
150extern struct kmem_cache *kmalloc_caches[SLUB_PAGE_SHIFT];
151
152
153
154
155
156static __always_inline int kmalloc_index(size_t size)
157{
158 if (!size)
159 return 0;
160
161 if (size <= KMALLOC_MIN_SIZE)
162 return KMALLOC_SHIFT_LOW;
163
164 if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
165 return 1;
166 if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
167 return 2;
168 if (size <= 8) return 3;
169 if (size <= 16) return 4;
170 if (size <= 32) return 5;
171 if (size <= 64) return 6;
172 if (size <= 128) return 7;
173 if (size <= 256) return 8;
174 if (size <= 512) return 9;
175 if (size <= 1024) return 10;
176 if (size <= 2 * 1024) return 11;
177 if (size <= 4 * 1024) return 12;
178
179
180
181
182
183 if (size <= 8 * 1024) return 13;
184 if (size <= 16 * 1024) return 14;
185 if (size <= 32 * 1024) return 15;
186 if (size <= 64 * 1024) return 16;
187 if (size <= 128 * 1024) return 17;
188 if (size <= 256 * 1024) return 18;
189 if (size <= 512 * 1024) return 19;
190 if (size <= 1024 * 1024) return 20;
191 if (size <= 2 * 1024 * 1024) return 21;
192 BUG();
193 return -1;
194
195
196
197
198
199
200
201
202}
203
204
205
206
207
208
209
210static __always_inline struct kmem_cache *kmalloc_slab(size_t size)
211{
212 int index = kmalloc_index(size);
213
214 if (index == 0)
215 return NULL;
216
217 return kmalloc_caches[index];
218}
219
220void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
221void *__kmalloc(size_t size, gfp_t flags);
222
223static __always_inline void *
224kmalloc_order(size_t size, gfp_t flags, unsigned int order)
225{
226 void *ret = (void *) __get_free_pages(flags | __GFP_COMP, order);
227 kmemleak_alloc(ret, size, 1, flags);
228 return ret;
229}
230
231
232
233
234
235#ifdef CONFIG_SLUB_DEBUG
236extern bool verify_mem_not_deleted(const void *x);
237#else
238static inline bool verify_mem_not_deleted(const void *x)
239{
240 return true;
241}
242#endif
243
244#ifdef CONFIG_TRACING
245extern void *
246kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size);
247extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order);
248#else
249static __always_inline void *
250kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
251{
252 return kmem_cache_alloc(s, gfpflags);
253}
254
255static __always_inline void *
256kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
257{
258 return kmalloc_order(size, flags, order);
259}
260#endif
261
262static __always_inline void *kmalloc_large(size_t size, gfp_t flags)
263{
264 unsigned int order = get_order(size);
265 return kmalloc_order_trace(size, flags, order);
266}
267
268static __always_inline void *kmalloc(size_t size, gfp_t flags)
269{
270 if (__builtin_constant_p(size)) {
271 if (size > SLUB_MAX_SIZE)
272 return kmalloc_large(size, flags);
273
274 if (!(flags & SLUB_DMA)) {
275 struct kmem_cache *s = kmalloc_slab(size);
276
277 if (!s)
278 return ZERO_SIZE_PTR;
279
280 return kmem_cache_alloc_trace(s, flags, size);
281 }
282 }
283 return __kmalloc(size, flags);
284}
285
286#ifdef CONFIG_NUMA
287void *__kmalloc_node(size_t size, gfp_t flags, int node);
288void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
289
290#ifdef CONFIG_TRACING
291extern void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
292 gfp_t gfpflags,
293 int node, size_t size);
294#else
295static __always_inline void *
296kmem_cache_alloc_node_trace(struct kmem_cache *s,
297 gfp_t gfpflags,
298 int node, size_t size)
299{
300 return kmem_cache_alloc_node(s, gfpflags, node);
301}
302#endif
303
304static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
305{
306 if (__builtin_constant_p(size) &&
307 size <= SLUB_MAX_SIZE && !(flags & SLUB_DMA)) {
308 struct kmem_cache *s = kmalloc_slab(size);
309
310 if (!s)
311 return ZERO_SIZE_PTR;
312
313 return kmem_cache_alloc_node_trace(s, flags, node, size);
314 }
315 return __kmalloc_node(size, flags, node);
316}
317#endif
318
319#endif
320