1#ifndef _LINUX_SLUB_DEF_H
2#define _LINUX_SLUB_DEF_H
3
4
5
6
7
8
9#include <linux/types.h>
10#include <linux/gfp.h>
11#include <linux/workqueue.h>
12#include <linux/kobject.h>
13#include <linux/kmemtrace.h>
14#include <linux/kmemleak.h>
15
16enum stat_item {
17 ALLOC_FASTPATH,
18 ALLOC_SLOWPATH,
19 FREE_FASTPATH,
20 FREE_SLOWPATH,
21 FREE_FROZEN,
22 FREE_ADD_PARTIAL,
23 FREE_REMOVE_PARTIAL,
24 ALLOC_FROM_PARTIAL,
25 ALLOC_SLAB,
26 ALLOC_REFILL,
27 FREE_SLAB,
28 CPUSLAB_FLUSH,
29 DEACTIVATE_FULL,
30 DEACTIVATE_EMPTY,
31 DEACTIVATE_TO_HEAD,
32 DEACTIVATE_TO_TAIL,
33 DEACTIVATE_REMOTE_FREES,
34 ORDER_FALLBACK,
35 NR_SLUB_STAT_ITEMS };
36
37struct kmem_cache_cpu {
38 void **freelist;
39 struct page *page;
40 int node;
41 unsigned int offset;
42 unsigned int objsize;
43#ifdef CONFIG_SLUB_STATS
44 unsigned stat[NR_SLUB_STAT_ITEMS];
45#endif
46};
47
48struct kmem_cache_node {
49 spinlock_t list_lock;
50 unsigned long nr_partial;
51 struct list_head partial;
52#ifdef CONFIG_SLUB_DEBUG
53 atomic_long_t nr_slabs;
54 atomic_long_t total_objects;
55 struct list_head full;
56#endif
57};
58
59
60
61
62
63
64struct kmem_cache_order_objects {
65 unsigned long x;
66};
67
68
69
70
71struct kmem_cache {
72
73 unsigned long flags;
74 int size;
75 int objsize;
76 int offset;
77 struct kmem_cache_order_objects oo;
78
79
80
81
82
83 struct kmem_cache_node local_node;
84
85
86 struct kmem_cache_order_objects max;
87 struct kmem_cache_order_objects min;
88 gfp_t allocflags;
89 int refcount;
90 void (*ctor)(void *);
91 int inuse;
92 int align;
93 unsigned long min_partial;
94 const char *name;
95 struct list_head list;
96#ifdef CONFIG_SLUB_DEBUG
97 struct kobject kobj;
98#endif
99
100#ifdef CONFIG_NUMA
101
102
103
104 int remote_node_defrag_ratio;
105 struct kmem_cache_node *node[MAX_NUMNODES];
106#endif
107#ifdef CONFIG_SMP
108 struct kmem_cache_cpu *cpu_slab[NR_CPUS];
109#else
110 struct kmem_cache_cpu cpu_slab;
111#endif
112};
113
114
115
116
117#if defined(ARCH_KMALLOC_MINALIGN) && ARCH_KMALLOC_MINALIGN > 8
118#define KMALLOC_MIN_SIZE ARCH_KMALLOC_MINALIGN
119#else
120#define KMALLOC_MIN_SIZE 8
121#endif
122
123#define KMALLOC_SHIFT_LOW ilog2(KMALLOC_MIN_SIZE)
124
125
126
127
128
129
130
131
132
133
134#define SLUB_MAX_SIZE (2 * PAGE_SIZE)
135
136#define SLUB_PAGE_SHIFT (PAGE_SHIFT + 2)
137
138
139
140
141
142extern struct kmem_cache kmalloc_caches[SLUB_PAGE_SHIFT];
143
144
145
146
147
148static __always_inline int kmalloc_index(size_t size)
149{
150 if (!size)
151 return 0;
152
153 if (size <= KMALLOC_MIN_SIZE)
154 return KMALLOC_SHIFT_LOW;
155
156 if (KMALLOC_MIN_SIZE <= 32 && size > 64 && size <= 96)
157 return 1;
158 if (KMALLOC_MIN_SIZE <= 64 && size > 128 && size <= 192)
159 return 2;
160 if (size <= 8) return 3;
161 if (size <= 16) return 4;
162 if (size <= 32) return 5;
163 if (size <= 64) return 6;
164 if (size <= 128) return 7;
165 if (size <= 256) return 8;
166 if (size <= 512) return 9;
167 if (size <= 1024) return 10;
168 if (size <= 2 * 1024) return 11;
169 if (size <= 4 * 1024) return 12;
170
171
172
173
174 if (size <= 8 * 1024) return 13;
175 if (size <= 16 * 1024) return 14;
176 if (size <= 32 * 1024) return 15;
177 if (size <= 64 * 1024) return 16;
178 if (size <= 128 * 1024) return 17;
179 if (size <= 256 * 1024) return 18;
180 if (size <= 512 * 1024) return 19;
181 if (size <= 1024 * 1024) return 20;
182 if (size <= 2 * 1024 * 1024) return 21;
183 return -1;
184
185
186
187
188
189
190
191
192}
193
194
195
196
197
198
199
200static __always_inline struct kmem_cache *kmalloc_slab(size_t size)
201{
202 int index = kmalloc_index(size);
203
204 if (index == 0)
205 return NULL;
206
207 return &kmalloc_caches[index];
208}
209
210#ifdef CONFIG_ZONE_DMA
211#define SLUB_DMA __GFP_DMA
212#else
213
214#define SLUB_DMA (__force gfp_t)0
215#endif
216
217void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
218void *__kmalloc(size_t size, gfp_t flags);
219
220#ifdef CONFIG_KMEMTRACE
221extern void *kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags);
222#else
223static __always_inline void *
224kmem_cache_alloc_notrace(struct kmem_cache *s, gfp_t gfpflags)
225{
226 return kmem_cache_alloc(s, gfpflags);
227}
228#endif
229
230static __always_inline void *kmalloc_large(size_t size, gfp_t flags)
231{
232 unsigned int order = get_order(size);
233 void *ret = (void *) __get_free_pages(flags | __GFP_COMP, order);
234
235 kmemleak_alloc(ret, size, 1, flags);
236 trace_kmalloc(_THIS_IP_, ret, size, PAGE_SIZE << order, flags);
237
238 return ret;
239}
240
241static __always_inline void *kmalloc(size_t size, gfp_t flags)
242{
243 void *ret;
244
245 if (__builtin_constant_p(size)) {
246 if (size > SLUB_MAX_SIZE)
247 return kmalloc_large(size, flags);
248
249 if (!(flags & SLUB_DMA)) {
250 struct kmem_cache *s = kmalloc_slab(size);
251
252 if (!s)
253 return ZERO_SIZE_PTR;
254
255 ret = kmem_cache_alloc_notrace(s, flags);
256
257 trace_kmalloc(_THIS_IP_, ret, size, s->size, flags);
258
259 return ret;
260 }
261 }
262 return __kmalloc(size, flags);
263}
264
265#ifdef CONFIG_NUMA
266void *__kmalloc_node(size_t size, gfp_t flags, int node);
267void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
268
269#ifdef CONFIG_KMEMTRACE
270extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *s,
271 gfp_t gfpflags,
272 int node);
273#else
274static __always_inline void *
275kmem_cache_alloc_node_notrace(struct kmem_cache *s,
276 gfp_t gfpflags,
277 int node)
278{
279 return kmem_cache_alloc_node(s, gfpflags, node);
280}
281#endif
282
283static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
284{
285 void *ret;
286
287 if (__builtin_constant_p(size) &&
288 size <= SLUB_MAX_SIZE && !(flags & SLUB_DMA)) {
289 struct kmem_cache *s = kmalloc_slab(size);
290
291 if (!s)
292 return ZERO_SIZE_PTR;
293
294 ret = kmem_cache_alloc_node_notrace(s, flags, node);
295
296 trace_kmalloc_node(_THIS_IP_, ret,
297 size, s->size, flags, node);
298
299 return ret;
300 }
301 return __kmalloc_node(size, flags, node);
302}
303#endif
304
305#endif
306