1
2#ifndef MM_SLAB_H
3#define MM_SLAB_H
4
5
6
7
8#ifdef CONFIG_SLOB
9
10
11
12
13
14
15
16
17
18
19
20struct kmem_cache {
21 unsigned int object_size;
22 unsigned int size;
23 unsigned int align;
24 slab_flags_t flags;
25 unsigned int useroffset;
26 unsigned int usersize;
27 const char *name;
28 int refcount;
29 void (*ctor)(void *);
30 struct list_head list;
31};
32
33#endif
34
35#ifdef CONFIG_SLAB
36#include <linux/slab_def.h>
37#endif
38
39#ifdef CONFIG_SLUB
40#include <linux/slub_def.h>
41#endif
42
43#include <linux/memcontrol.h>
44#include <linux/fault-inject.h>
45#include <linux/kasan.h>
46#include <linux/kmemleak.h>
47#include <linux/random.h>
48#include <linux/sched/mm.h>
49
50
51
52
53
54
55
56
57
58enum slab_state {
59 DOWN,
60 PARTIAL,
61 PARTIAL_NODE,
62 UP,
63 FULL
64};
65
66extern enum slab_state slab_state;
67
68
69extern struct mutex slab_mutex;
70
71
72extern struct list_head slab_caches;
73
74
75extern struct kmem_cache *kmem_cache;
76
77
78extern const struct kmalloc_info_struct {
79 const char *name;
80 unsigned int size;
81} kmalloc_info[];
82
83#ifndef CONFIG_SLOB
84
85void setup_kmalloc_cache_index_table(void);
86void create_kmalloc_caches(slab_flags_t);
87
88
89struct kmem_cache *kmalloc_slab(size_t, gfp_t);
90#endif
91
92
93
94int __kmem_cache_create(struct kmem_cache *, slab_flags_t flags);
95
96struct kmem_cache *create_kmalloc_cache(const char *name, unsigned int size,
97 slab_flags_t flags, unsigned int useroffset,
98 unsigned int usersize);
99extern void create_boot_cache(struct kmem_cache *, const char *name,
100 unsigned int size, slab_flags_t flags,
101 unsigned int useroffset, unsigned int usersize);
102
103int slab_unmergeable(struct kmem_cache *s);
104struct kmem_cache *find_mergeable(unsigned size, unsigned align,
105 slab_flags_t flags, const char *name, void (*ctor)(void *));
106#ifndef CONFIG_SLOB
107struct kmem_cache *
108__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
109 slab_flags_t flags, void (*ctor)(void *));
110
111slab_flags_t kmem_cache_flags(unsigned int object_size,
112 slab_flags_t flags, const char *name,
113 void (*ctor)(void *));
114#else
115static inline struct kmem_cache *
116__kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
117 slab_flags_t flags, void (*ctor)(void *))
118{ return NULL; }
119
120static inline slab_flags_t kmem_cache_flags(unsigned int object_size,
121 slab_flags_t flags, const char *name,
122 void (*ctor)(void *))
123{
124 return flags;
125}
126#endif
127
128
129
130#define SLAB_CORE_FLAGS (SLAB_HWCACHE_ALIGN | SLAB_CACHE_DMA | \
131 SLAB_CACHE_DMA32 | SLAB_PANIC | \
132 SLAB_TYPESAFE_BY_RCU | SLAB_DEBUG_OBJECTS )
133
134#if defined(CONFIG_DEBUG_SLAB)
135#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
136#elif defined(CONFIG_SLUB_DEBUG)
137#define SLAB_DEBUG_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
138 SLAB_TRACE | SLAB_CONSISTENCY_CHECKS)
139#else
140#define SLAB_DEBUG_FLAGS (0)
141#endif
142
143#if defined(CONFIG_SLAB)
144#define SLAB_CACHE_FLAGS (SLAB_MEM_SPREAD | SLAB_NOLEAKTRACE | \
145 SLAB_RECLAIM_ACCOUNT | SLAB_TEMPORARY | \
146 SLAB_ACCOUNT)
147#elif defined(CONFIG_SLUB)
148#define SLAB_CACHE_FLAGS (SLAB_NOLEAKTRACE | SLAB_RECLAIM_ACCOUNT | \
149 SLAB_TEMPORARY | SLAB_ACCOUNT)
150#else
151#define SLAB_CACHE_FLAGS (0)
152#endif
153
154
155#define CACHE_CREATE_MASK (SLAB_CORE_FLAGS | SLAB_DEBUG_FLAGS | SLAB_CACHE_FLAGS)
156
157
158#define SLAB_FLAGS_PERMITTED (SLAB_CORE_FLAGS | \
159 SLAB_RED_ZONE | \
160 SLAB_POISON | \
161 SLAB_STORE_USER | \
162 SLAB_TRACE | \
163 SLAB_CONSISTENCY_CHECKS | \
164 SLAB_MEM_SPREAD | \
165 SLAB_NOLEAKTRACE | \
166 SLAB_RECLAIM_ACCOUNT | \
167 SLAB_TEMPORARY | \
168 SLAB_ACCOUNT)
169
170bool __kmem_cache_empty(struct kmem_cache *);
171int __kmem_cache_shutdown(struct kmem_cache *);
172void __kmem_cache_release(struct kmem_cache *);
173int __kmem_cache_shrink(struct kmem_cache *);
174void __kmemcg_cache_deactivate(struct kmem_cache *s);
175void slab_kmem_cache_release(struct kmem_cache *);
176
177struct seq_file;
178struct file;
179
180struct slabinfo {
181 unsigned long active_objs;
182 unsigned long num_objs;
183 unsigned long active_slabs;
184 unsigned long num_slabs;
185 unsigned long shared_avail;
186 unsigned int limit;
187 unsigned int batchcount;
188 unsigned int shared;
189 unsigned int objects_per_slab;
190 unsigned int cache_order;
191};
192
193void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo);
194void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s);
195ssize_t slabinfo_write(struct file *file, const char __user *buffer,
196 size_t count, loff_t *ppos);
197
198
199
200
201
202
203
204void __kmem_cache_free_bulk(struct kmem_cache *, size_t, void **);
205int __kmem_cache_alloc_bulk(struct kmem_cache *, gfp_t, size_t, void **);
206
207#ifdef CONFIG_MEMCG_KMEM
208
209
210extern struct list_head slab_root_caches;
211#define root_caches_node memcg_params.__root_caches_node
212
213
214
215
216
217#define for_each_memcg_cache(iter, root) \
218 list_for_each_entry(iter, &(root)->memcg_params.children, \
219 memcg_params.children_node)
220
221static inline bool is_root_cache(struct kmem_cache *s)
222{
223 return !s->memcg_params.root_cache;
224}
225
226static inline bool slab_equal_or_root(struct kmem_cache *s,
227 struct kmem_cache *p)
228{
229 return p == s || p == s->memcg_params.root_cache;
230}
231
232
233
234
235
236
237static inline const char *cache_name(struct kmem_cache *s)
238{
239 if (!is_root_cache(s))
240 s = s->memcg_params.root_cache;
241 return s->name;
242}
243
244
245
246
247
248
249static inline struct kmem_cache *
250cache_from_memcg_idx(struct kmem_cache *s, int idx)
251{
252 struct kmem_cache *cachep;
253 struct memcg_cache_array *arr;
254
255 rcu_read_lock();
256 arr = rcu_dereference(s->memcg_params.memcg_caches);
257
258
259
260
261
262
263 cachep = READ_ONCE(arr->entries[idx]);
264 rcu_read_unlock();
265
266 return cachep;
267}
268
269static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
270{
271 if (is_root_cache(s))
272 return s;
273 return s->memcg_params.root_cache;
274}
275
276static __always_inline int memcg_charge_slab(struct page *page,
277 gfp_t gfp, int order,
278 struct kmem_cache *s)
279{
280 if (is_root_cache(s))
281 return 0;
282 return memcg_kmem_charge_memcg(page, gfp, order, s->memcg_params.memcg);
283}
284
285static __always_inline void memcg_uncharge_slab(struct page *page, int order,
286 struct kmem_cache *s)
287{
288 memcg_kmem_uncharge(page, order);
289}
290
291extern void slab_init_memcg_params(struct kmem_cache *);
292extern void memcg_link_cache(struct kmem_cache *s);
293extern void slab_deactivate_memcg_cache_rcu_sched(struct kmem_cache *s,
294 void (*deact_fn)(struct kmem_cache *));
295
296#else
297
298
299#define slab_root_caches slab_caches
300#define root_caches_node list
301
302#define for_each_memcg_cache(iter, root) \
303 for ((void)(iter), (void)(root); 0; )
304
305static inline bool is_root_cache(struct kmem_cache *s)
306{
307 return true;
308}
309
310static inline bool slab_equal_or_root(struct kmem_cache *s,
311 struct kmem_cache *p)
312{
313 return true;
314}
315
316static inline const char *cache_name(struct kmem_cache *s)
317{
318 return s->name;
319}
320
321static inline struct kmem_cache *
322cache_from_memcg_idx(struct kmem_cache *s, int idx)
323{
324 return NULL;
325}
326
327static inline struct kmem_cache *memcg_root_cache(struct kmem_cache *s)
328{
329 return s;
330}
331
332static inline int memcg_charge_slab(struct page *page, gfp_t gfp, int order,
333 struct kmem_cache *s)
334{
335 return 0;
336}
337
338static inline void memcg_uncharge_slab(struct page *page, int order,
339 struct kmem_cache *s)
340{
341}
342
343static inline void slab_init_memcg_params(struct kmem_cache *s)
344{
345}
346
347static inline void memcg_link_cache(struct kmem_cache *s)
348{
349}
350
351#endif
352
353static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
354{
355 struct kmem_cache *cachep;
356 struct page *page;
357
358
359
360
361
362
363
364
365 if (!memcg_kmem_enabled() &&
366 !unlikely(s->flags & SLAB_CONSISTENCY_CHECKS))
367 return s;
368
369 page = virt_to_head_page(x);
370 cachep = page->slab_cache;
371 if (slab_equal_or_root(cachep, s))
372 return cachep;
373
374 pr_err("%s: Wrong slab cache. %s but object is from %s\n",
375 __func__, s->name, cachep->name);
376 WARN_ON_ONCE(1);
377 return s;
378}
379
380static inline size_t slab_ksize(const struct kmem_cache *s)
381{
382#ifndef CONFIG_SLUB
383 return s->object_size;
384
385#else
386# ifdef CONFIG_SLUB_DEBUG
387
388
389
390
391 if (s->flags & (SLAB_RED_ZONE | SLAB_POISON))
392 return s->object_size;
393# endif
394 if (s->flags & SLAB_KASAN)
395 return s->object_size;
396
397
398
399
400
401 if (s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_STORE_USER))
402 return s->inuse;
403
404
405
406 return s->size;
407#endif
408}
409
410static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s,
411 gfp_t flags)
412{
413 flags &= gfp_allowed_mask;
414
415 fs_reclaim_acquire(flags);
416 fs_reclaim_release(flags);
417
418 might_sleep_if(gfpflags_allow_blocking(flags));
419
420 if (should_failslab(s, flags))
421 return NULL;
422
423 if (memcg_kmem_enabled() &&
424 ((flags & __GFP_ACCOUNT) || (s->flags & SLAB_ACCOUNT)))
425 return memcg_kmem_get_cache(s);
426
427 return s;
428}
429
430static inline void slab_post_alloc_hook(struct kmem_cache *s, gfp_t flags,
431 size_t size, void **p)
432{
433 size_t i;
434
435 flags &= gfp_allowed_mask;
436 for (i = 0; i < size; i++) {
437 p[i] = kasan_slab_alloc(s, p[i], flags);
438
439 kmemleak_alloc_recursive(p[i], s->object_size, 1,
440 s->flags, flags);
441 }
442
443 if (memcg_kmem_enabled())
444 memcg_kmem_put_cache(s);
445}
446
447#ifndef CONFIG_SLOB
448
449
450
451struct kmem_cache_node {
452 spinlock_t list_lock;
453
454#ifdef CONFIG_SLAB
455 struct list_head slabs_partial;
456 struct list_head slabs_full;
457 struct list_head slabs_free;
458 unsigned long total_slabs;
459 unsigned long free_slabs;
460 unsigned long free_objects;
461 unsigned int free_limit;
462 unsigned int colour_next;
463 struct array_cache *shared;
464 struct alien_cache **alien;
465 unsigned long next_reap;
466 int free_touched;
467#endif
468
469#ifdef CONFIG_SLUB
470 unsigned long nr_partial;
471 struct list_head partial;
472#ifdef CONFIG_SLUB_DEBUG
473 atomic_long_t nr_slabs;
474 atomic_long_t total_objects;
475 struct list_head full;
476#endif
477#endif
478
479};
480
481static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
482{
483 return s->node[node];
484}
485
486
487
488
489
490#define for_each_kmem_cache_node(__s, __node, __n) \
491 for (__node = 0; __node < nr_node_ids; __node++) \
492 if ((__n = get_node(__s, __node)))
493
494#endif
495
496void *slab_start(struct seq_file *m, loff_t *pos);
497void *slab_next(struct seq_file *m, void *p, loff_t *pos);
498void slab_stop(struct seq_file *m, void *p);
499void *memcg_slab_start(struct seq_file *m, loff_t *pos);
500void *memcg_slab_next(struct seq_file *m, void *p, loff_t *pos);
501void memcg_slab_stop(struct seq_file *m, void *p);
502int memcg_slab_show(struct seq_file *m, void *p);
503
504#if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
505void dump_unreclaimable_slab(void);
506#else
507static inline void dump_unreclaimable_slab(void)
508{
509}
510#endif
511
512void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr);
513
514#ifdef CONFIG_SLAB_FREELIST_RANDOM
515int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
516 gfp_t gfp);
517void cache_random_seq_destroy(struct kmem_cache *cachep);
518#else
519static inline int cache_random_seq_create(struct kmem_cache *cachep,
520 unsigned int count, gfp_t gfp)
521{
522 return 0;
523}
524static inline void cache_random_seq_destroy(struct kmem_cache *cachep) { }
525#endif
526
527#endif
528