1
2
3
4
5
6
7#include <linux/slab.h>
8
9#include <linux/mm.h>
10#include <linux/poison.h>
11#include <linux/interrupt.h>
12#include <linux/memory.h>
13#include <linux/cache.h>
14#include <linux/compiler.h>
15#include <linux/module.h>
16#include <linux/cpu.h>
17#include <linux/uaccess.h>
18#include <linux/seq_file.h>
19#include <linux/proc_fs.h>
20#include <asm/cacheflush.h>
21#include <asm/tlbflush.h>
22#include <asm/page.h>
23#include <linux/memcontrol.h>
24
25#define CREATE_TRACE_POINTS
26#include <trace/events/kmem.h>
27
28#include "slab.h"
29
30enum slab_state slab_state;
31LIST_HEAD(slab_caches);
32DEFINE_MUTEX(slab_mutex);
33struct kmem_cache *kmem_cache;
34
35#ifdef CONFIG_HARDENED_USERCOPY
36bool usercopy_fallback __ro_after_init =
37 IS_ENABLED(CONFIG_HARDENED_USERCOPY_FALLBACK);
38module_param(usercopy_fallback, bool, 0400);
39MODULE_PARM_DESC(usercopy_fallback,
40 "WARN instead of reject usercopy whitelist violations");
41#endif
42
43static LIST_HEAD(slab_caches_to_rcu_destroy);
44static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work);
45static DECLARE_WORK(slab_caches_to_rcu_destroy_work,
46 slab_caches_to_rcu_destroy_workfn);
47
48
49
50
51#define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
52 SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \
53 SLAB_FAILSLAB | SLAB_KASAN)
54
55#define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \
56 SLAB_CACHE_DMA32 | SLAB_ACCOUNT)
57
58
59
60
61static bool slab_nomerge = !IS_ENABLED(CONFIG_SLAB_MERGE_DEFAULT);
62
63static int __init setup_slab_nomerge(char *str)
64{
65 slab_nomerge = true;
66 return 1;
67}
68
69#ifdef CONFIG_SLUB
70__setup_param("slub_nomerge", slub_nomerge, setup_slab_nomerge, 0);
71#endif
72
73__setup("slab_nomerge", setup_slab_nomerge);
74
75
76
77
78unsigned int kmem_cache_size(struct kmem_cache *s)
79{
80 return s->object_size;
81}
82EXPORT_SYMBOL(kmem_cache_size);
83
84#ifdef CONFIG_DEBUG_VM
85static int kmem_cache_sanity_check(const char *name, unsigned int size)
86{
87 if (!name || in_interrupt() || size < sizeof(void *) ||
88 size > KMALLOC_MAX_SIZE) {
89 pr_err("kmem_cache_create(%s) integrity check failed\n", name);
90 return -EINVAL;
91 }
92
93 WARN_ON(strchr(name, ' '));
94 return 0;
95}
96#else
97static inline int kmem_cache_sanity_check(const char *name, unsigned int size)
98{
99 return 0;
100}
101#endif
102
103void __kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p)
104{
105 size_t i;
106
107 for (i = 0; i < nr; i++) {
108 if (s)
109 kmem_cache_free(s, p[i]);
110 else
111 kfree(p[i]);
112 }
113}
114
115int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr,
116 void **p)
117{
118 size_t i;
119
120 for (i = 0; i < nr; i++) {
121 void *x = p[i] = kmem_cache_alloc(s, flags);
122 if (!x) {
123 __kmem_cache_free_bulk(s, i, p);
124 return 0;
125 }
126 }
127 return i;
128}
129
130#ifdef CONFIG_MEMCG_KMEM
131
132LIST_HEAD(slab_root_caches);
133
134void slab_init_memcg_params(struct kmem_cache *s)
135{
136 s->memcg_params.root_cache = NULL;
137 RCU_INIT_POINTER(s->memcg_params.memcg_caches, NULL);
138 INIT_LIST_HEAD(&s->memcg_params.children);
139 s->memcg_params.dying = false;
140}
141
142static int init_memcg_params(struct kmem_cache *s,
143 struct mem_cgroup *memcg, struct kmem_cache *root_cache)
144{
145 struct memcg_cache_array *arr;
146
147 if (root_cache) {
148 s->memcg_params.root_cache = root_cache;
149 s->memcg_params.memcg = memcg;
150 INIT_LIST_HEAD(&s->memcg_params.children_node);
151 INIT_LIST_HEAD(&s->memcg_params.kmem_caches_node);
152 return 0;
153 }
154
155 slab_init_memcg_params(s);
156
157 if (!memcg_nr_cache_ids)
158 return 0;
159
160 arr = kvzalloc(sizeof(struct memcg_cache_array) +
161 memcg_nr_cache_ids * sizeof(void *),
162 GFP_KERNEL);
163 if (!arr)
164 return -ENOMEM;
165
166 RCU_INIT_POINTER(s->memcg_params.memcg_caches, arr);
167 return 0;
168}
169
170static void destroy_memcg_params(struct kmem_cache *s)
171{
172 if (is_root_cache(s))
173 kvfree(rcu_access_pointer(s->memcg_params.memcg_caches));
174}
175
176static void free_memcg_params(struct rcu_head *rcu)
177{
178 struct memcg_cache_array *old;
179
180 old = container_of(rcu, struct memcg_cache_array, rcu);
181 kvfree(old);
182}
183
184static int update_memcg_params(struct kmem_cache *s, int new_array_size)
185{
186 struct memcg_cache_array *old, *new;
187
188 new = kvzalloc(sizeof(struct memcg_cache_array) +
189 new_array_size * sizeof(void *), GFP_KERNEL);
190 if (!new)
191 return -ENOMEM;
192
193 old = rcu_dereference_protected(s->memcg_params.memcg_caches,
194 lockdep_is_held(&slab_mutex));
195 if (old)
196 memcpy(new->entries, old->entries,
197 memcg_nr_cache_ids * sizeof(void *));
198
199 rcu_assign_pointer(s->memcg_params.memcg_caches, new);
200 if (old)
201 call_rcu(&old->rcu, free_memcg_params);
202 return 0;
203}
204
205int memcg_update_all_caches(int num_memcgs)
206{
207 struct kmem_cache *s;
208 int ret = 0;
209
210 mutex_lock(&slab_mutex);
211 list_for_each_entry(s, &slab_root_caches, root_caches_node) {
212 ret = update_memcg_params(s, num_memcgs);
213
214
215
216
217 if (ret)
218 break;
219 }
220 mutex_unlock(&slab_mutex);
221 return ret;
222}
223
224void memcg_link_cache(struct kmem_cache *s)
225{
226 if (is_root_cache(s)) {
227 list_add(&s->root_caches_node, &slab_root_caches);
228 } else {
229 list_add(&s->memcg_params.children_node,
230 &s->memcg_params.root_cache->memcg_params.children);
231 list_add(&s->memcg_params.kmem_caches_node,
232 &s->memcg_params.memcg->kmem_caches);
233 }
234}
235
236static void memcg_unlink_cache(struct kmem_cache *s)
237{
238 if (is_root_cache(s)) {
239 list_del(&s->root_caches_node);
240 } else {
241 list_del(&s->memcg_params.children_node);
242 list_del(&s->memcg_params.kmem_caches_node);
243 }
244}
245#else
246static inline int init_memcg_params(struct kmem_cache *s,
247 struct mem_cgroup *memcg, struct kmem_cache *root_cache)
248{
249 return 0;
250}
251
252static inline void destroy_memcg_params(struct kmem_cache *s)
253{
254}
255
256static inline void memcg_unlink_cache(struct kmem_cache *s)
257{
258}
259#endif
260
261
262
263
264
265static unsigned int calculate_alignment(slab_flags_t flags,
266 unsigned int align, unsigned int size)
267{
268
269
270
271
272
273
274
275 if (flags & SLAB_HWCACHE_ALIGN) {
276 unsigned int ralign;
277
278 ralign = cache_line_size();
279 while (size <= ralign / 2)
280 ralign /= 2;
281 align = max(align, ralign);
282 }
283
284 if (align < ARCH_SLAB_MINALIGN)
285 align = ARCH_SLAB_MINALIGN;
286
287 return ALIGN(align, sizeof(void *));
288}
289
290
291
292
293int slab_unmergeable(struct kmem_cache *s)
294{
295 if (slab_nomerge || (s->flags & SLAB_NEVER_MERGE))
296 return 1;
297
298 if (!is_root_cache(s))
299 return 1;
300
301 if (s->ctor)
302 return 1;
303
304 if (s->usersize)
305 return 1;
306
307
308
309
310 if (s->refcount < 0)
311 return 1;
312
313 return 0;
314}
315
316struct kmem_cache *find_mergeable(unsigned int size, unsigned int align,
317 slab_flags_t flags, const char *name, void (*ctor)(void *))
318{
319 struct kmem_cache *s;
320
321 if (slab_nomerge)
322 return NULL;
323
324 if (ctor)
325 return NULL;
326
327 size = ALIGN(size, sizeof(void *));
328 align = calculate_alignment(flags, align, size);
329 size = ALIGN(size, align);
330 flags = kmem_cache_flags(size, flags, name, NULL);
331
332 if (flags & SLAB_NEVER_MERGE)
333 return NULL;
334
335 list_for_each_entry_reverse(s, &slab_root_caches, root_caches_node) {
336 if (slab_unmergeable(s))
337 continue;
338
339 if (size > s->size)
340 continue;
341
342 if ((flags & SLAB_MERGE_SAME) != (s->flags & SLAB_MERGE_SAME))
343 continue;
344
345
346
347
348 if ((s->size & ~(align - 1)) != s->size)
349 continue;
350
351 if (s->size - size >= sizeof(void *))
352 continue;
353
354 if (IS_ENABLED(CONFIG_SLAB) && align &&
355 (align > s->align || s->align % align))
356 continue;
357
358 return s;
359 }
360 return NULL;
361}
362
363static struct kmem_cache *create_cache(const char *name,
364 unsigned int object_size, unsigned int align,
365 slab_flags_t flags, unsigned int useroffset,
366 unsigned int usersize, void (*ctor)(void *),
367 struct mem_cgroup *memcg, struct kmem_cache *root_cache)
368{
369 struct kmem_cache *s;
370 int err;
371
372 if (WARN_ON(useroffset + usersize > object_size))
373 useroffset = usersize = 0;
374
375 err = -ENOMEM;
376 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
377 if (!s)
378 goto out;
379
380 s->name = name;
381 s->size = s->object_size = object_size;
382 s->align = align;
383 s->ctor = ctor;
384 s->useroffset = useroffset;
385 s->usersize = usersize;
386
387 err = init_memcg_params(s, memcg, root_cache);
388 if (err)
389 goto out_free_cache;
390
391 err = __kmem_cache_create(s, flags);
392 if (err)
393 goto out_free_cache;
394
395 s->refcount = 1;
396 list_add(&s->list, &slab_caches);
397 memcg_link_cache(s);
398out:
399 if (err)
400 return ERR_PTR(err);
401 return s;
402
403out_free_cache:
404 destroy_memcg_params(s);
405 kmem_cache_free(kmem_cache, s);
406 goto out;
407}
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437struct kmem_cache *
438kmem_cache_create_usercopy(const char *name,
439 unsigned int size, unsigned int align,
440 slab_flags_t flags,
441 unsigned int useroffset, unsigned int usersize,
442 void (*ctor)(void *))
443{
444 struct kmem_cache *s = NULL;
445 const char *cache_name;
446 int err;
447
448 get_online_cpus();
449 get_online_mems();
450 memcg_get_cache_ids();
451
452 mutex_lock(&slab_mutex);
453
454 err = kmem_cache_sanity_check(name, size);
455 if (err) {
456 goto out_unlock;
457 }
458
459
460 if (flags & ~SLAB_FLAGS_PERMITTED) {
461 err = -EINVAL;
462 goto out_unlock;
463 }
464
465
466
467
468
469
470
471 flags &= CACHE_CREATE_MASK;
472
473
474 if (WARN_ON(!usersize && useroffset) ||
475 WARN_ON(size < usersize || size - usersize < useroffset))
476 usersize = useroffset = 0;
477
478 if (!usersize)
479 s = __kmem_cache_alias(name, size, align, flags, ctor);
480 if (s)
481 goto out_unlock;
482
483 cache_name = kstrdup_const(name, GFP_KERNEL);
484 if (!cache_name) {
485 err = -ENOMEM;
486 goto out_unlock;
487 }
488
489 s = create_cache(cache_name, size,
490 calculate_alignment(flags, align, size),
491 flags, useroffset, usersize, ctor, NULL, NULL);
492 if (IS_ERR(s)) {
493 err = PTR_ERR(s);
494 kfree_const(cache_name);
495 }
496
497out_unlock:
498 mutex_unlock(&slab_mutex);
499
500 memcg_put_cache_ids();
501 put_online_mems();
502 put_online_cpus();
503
504 if (err) {
505 if (flags & SLAB_PANIC)
506 panic("kmem_cache_create: Failed to create slab '%s'. Error %d\n",
507 name, err);
508 else {
509 pr_warn("kmem_cache_create(%s) failed with error %d\n",
510 name, err);
511 dump_stack();
512 }
513 return NULL;
514 }
515 return s;
516}
517EXPORT_SYMBOL(kmem_cache_create_usercopy);
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544struct kmem_cache *
545kmem_cache_create(const char *name, unsigned int size, unsigned int align,
546 slab_flags_t flags, void (*ctor)(void *))
547{
548 return kmem_cache_create_usercopy(name, size, align, flags, 0, 0,
549 ctor);
550}
551EXPORT_SYMBOL(kmem_cache_create);
552
553static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work)
554{
555 LIST_HEAD(to_destroy);
556 struct kmem_cache *s, *s2;
557
558
559
560
561
562
563
564
565
566
567 mutex_lock(&slab_mutex);
568 list_splice_init(&slab_caches_to_rcu_destroy, &to_destroy);
569 mutex_unlock(&slab_mutex);
570
571 if (list_empty(&to_destroy))
572 return;
573
574 rcu_barrier();
575
576 list_for_each_entry_safe(s, s2, &to_destroy, list) {
577#ifdef SLAB_SUPPORTS_SYSFS
578 sysfs_slab_release(s);
579#else
580 slab_kmem_cache_release(s);
581#endif
582 }
583}
584
585static int shutdown_cache(struct kmem_cache *s)
586{
587
588 kasan_cache_shutdown(s);
589
590 if (__kmem_cache_shutdown(s) != 0)
591 return -EBUSY;
592
593 memcg_unlink_cache(s);
594 list_del(&s->list);
595
596 if (s->flags & SLAB_TYPESAFE_BY_RCU) {
597#ifdef SLAB_SUPPORTS_SYSFS
598 sysfs_slab_unlink(s);
599#endif
600 list_add_tail(&s->list, &slab_caches_to_rcu_destroy);
601 schedule_work(&slab_caches_to_rcu_destroy_work);
602 } else {
603#ifdef SLAB_SUPPORTS_SYSFS
604 sysfs_slab_unlink(s);
605 sysfs_slab_release(s);
606#else
607 slab_kmem_cache_release(s);
608#endif
609 }
610
611 return 0;
612}
613
614#ifdef CONFIG_MEMCG_KMEM
615
616
617
618
619
620
621
622
623
624void memcg_create_kmem_cache(struct mem_cgroup *memcg,
625 struct kmem_cache *root_cache)
626{
627 static char memcg_name_buf[NAME_MAX + 1];
628 struct cgroup_subsys_state *css = &memcg->css;
629 struct memcg_cache_array *arr;
630 struct kmem_cache *s = NULL;
631 char *cache_name;
632 int idx;
633
634 get_online_cpus();
635 get_online_mems();
636
637 mutex_lock(&slab_mutex);
638
639
640
641
642
643 if (memcg->kmem_state != KMEM_ONLINE || root_cache->memcg_params.dying)
644 goto out_unlock;
645
646 idx = memcg_cache_id(memcg);
647 arr = rcu_dereference_protected(root_cache->memcg_params.memcg_caches,
648 lockdep_is_held(&slab_mutex));
649
650
651
652
653
654
655 if (arr->entries[idx])
656 goto out_unlock;
657
658 cgroup_name(css->cgroup, memcg_name_buf, sizeof(memcg_name_buf));
659 cache_name = kasprintf(GFP_KERNEL, "%s(%llu:%s)", root_cache->name,
660 css->serial_nr, memcg_name_buf);
661 if (!cache_name)
662 goto out_unlock;
663
664 s = create_cache(cache_name, root_cache->object_size,
665 root_cache->align,
666 root_cache->flags & CACHE_CREATE_MASK,
667 root_cache->useroffset, root_cache->usersize,
668 root_cache->ctor, memcg, root_cache);
669
670
671
672
673
674 if (IS_ERR(s)) {
675 kfree(cache_name);
676 goto out_unlock;
677 }
678
679
680
681
682
683
684 smp_wmb();
685 arr->entries[idx] = s;
686
687out_unlock:
688 mutex_unlock(&slab_mutex);
689
690 put_online_mems();
691 put_online_cpus();
692}
693
694static void kmemcg_deactivate_workfn(struct work_struct *work)
695{
696 struct kmem_cache *s = container_of(work, struct kmem_cache,
697 memcg_params.deact_work);
698
699 get_online_cpus();
700 get_online_mems();
701
702 mutex_lock(&slab_mutex);
703
704 s->memcg_params.deact_fn(s);
705
706 mutex_unlock(&slab_mutex);
707
708 put_online_mems();
709 put_online_cpus();
710
711
712 css_put(&s->memcg_params.memcg->css);
713}
714
715static void kmemcg_deactivate_rcufn(struct rcu_head *head)
716{
717 struct kmem_cache *s = container_of(head, struct kmem_cache,
718 memcg_params.deact_rcu_head);
719
720
721
722
723
724
725 INIT_WORK(&s->memcg_params.deact_work, kmemcg_deactivate_workfn);
726 queue_work(memcg_kmem_cache_wq, &s->memcg_params.deact_work);
727}
728
729
730
731
732
733
734
735
736
737
738
739
740void slab_deactivate_memcg_cache_rcu_sched(struct kmem_cache *s,
741 void (*deact_fn)(struct kmem_cache *))
742{
743 if (WARN_ON_ONCE(is_root_cache(s)) ||
744 WARN_ON_ONCE(s->memcg_params.deact_fn))
745 return;
746
747 if (s->memcg_params.root_cache->memcg_params.dying)
748 return;
749
750
751 css_get(&s->memcg_params.memcg->css);
752
753 s->memcg_params.deact_fn = deact_fn;
754 call_rcu(&s->memcg_params.deact_rcu_head, kmemcg_deactivate_rcufn);
755}
756
757void memcg_deactivate_kmem_caches(struct mem_cgroup *memcg)
758{
759 int idx;
760 struct memcg_cache_array *arr;
761 struct kmem_cache *s, *c;
762
763 idx = memcg_cache_id(memcg);
764
765 get_online_cpus();
766 get_online_mems();
767
768 mutex_lock(&slab_mutex);
769 list_for_each_entry(s, &slab_root_caches, root_caches_node) {
770 arr = rcu_dereference_protected(s->memcg_params.memcg_caches,
771 lockdep_is_held(&slab_mutex));
772 c = arr->entries[idx];
773 if (!c)
774 continue;
775
776 __kmemcg_cache_deactivate(c);
777 arr->entries[idx] = NULL;
778 }
779 mutex_unlock(&slab_mutex);
780
781 put_online_mems();
782 put_online_cpus();
783}
784
785void memcg_destroy_kmem_caches(struct mem_cgroup *memcg)
786{
787 struct kmem_cache *s, *s2;
788
789 get_online_cpus();
790 get_online_mems();
791
792 mutex_lock(&slab_mutex);
793 list_for_each_entry_safe(s, s2, &memcg->kmem_caches,
794 memcg_params.kmem_caches_node) {
795
796
797
798
799 BUG_ON(shutdown_cache(s));
800 }
801 mutex_unlock(&slab_mutex);
802
803 put_online_mems();
804 put_online_cpus();
805}
806
807static int shutdown_memcg_caches(struct kmem_cache *s)
808{
809 struct memcg_cache_array *arr;
810 struct kmem_cache *c, *c2;
811 LIST_HEAD(busy);
812 int i;
813
814 BUG_ON(!is_root_cache(s));
815
816
817
818
819
820 arr = rcu_dereference_protected(s->memcg_params.memcg_caches,
821 lockdep_is_held(&slab_mutex));
822 for_each_memcg_cache_index(i) {
823 c = arr->entries[i];
824 if (!c)
825 continue;
826 if (shutdown_cache(c))
827
828
829
830
831
832 list_move(&c->memcg_params.children_node, &busy);
833 else
834
835
836
837
838
839
840 arr->entries[i] = NULL;
841 }
842
843
844
845
846
847 list_for_each_entry_safe(c, c2, &s->memcg_params.children,
848 memcg_params.children_node)
849 shutdown_cache(c);
850
851 list_splice(&busy, &s->memcg_params.children);
852
853
854
855
856
857 if (!list_empty(&s->memcg_params.children))
858 return -EBUSY;
859 return 0;
860}
861
862static void flush_memcg_workqueue(struct kmem_cache *s)
863{
864 mutex_lock(&slab_mutex);
865 s->memcg_params.dying = true;
866 mutex_unlock(&slab_mutex);
867
868
869
870
871
872 if (IS_ENABLED(CONFIG_SLUB))
873 rcu_barrier();
874
875
876
877
878
879
880 flush_workqueue(memcg_kmem_cache_wq);
881}
882#else
883static inline int shutdown_memcg_caches(struct kmem_cache *s)
884{
885 return 0;
886}
887
888static inline void flush_memcg_workqueue(struct kmem_cache *s)
889{
890}
891#endif
892
893void slab_kmem_cache_release(struct kmem_cache *s)
894{
895 __kmem_cache_release(s);
896 destroy_memcg_params(s);
897 kfree_const(s->name);
898 kmem_cache_free(kmem_cache, s);
899}
900
901void kmem_cache_destroy(struct kmem_cache *s)
902{
903 int err;
904
905 if (unlikely(!s))
906 return;
907
908 flush_memcg_workqueue(s);
909
910 get_online_cpus();
911 get_online_mems();
912
913 mutex_lock(&slab_mutex);
914
915 s->refcount--;
916 if (s->refcount)
917 goto out_unlock;
918
919 err = shutdown_memcg_caches(s);
920 if (!err)
921 err = shutdown_cache(s);
922
923 if (err) {
924 pr_err("kmem_cache_destroy %s: Slab cache still has objects\n",
925 s->name);
926 dump_stack();
927 }
928out_unlock:
929 mutex_unlock(&slab_mutex);
930
931 put_online_mems();
932 put_online_cpus();
933}
934EXPORT_SYMBOL(kmem_cache_destroy);
935
936
937
938
939
940
941
942
943
944
945int kmem_cache_shrink(struct kmem_cache *cachep)
946{
947 int ret;
948
949 get_online_cpus();
950 get_online_mems();
951 kasan_cache_shrink(cachep);
952 ret = __kmem_cache_shrink(cachep);
953 put_online_mems();
954 put_online_cpus();
955 return ret;
956}
957EXPORT_SYMBOL(kmem_cache_shrink);
958
959bool slab_is_available(void)
960{
961 return slab_state >= UP;
962}
963
964#ifndef CONFIG_SLOB
965
966void __init create_boot_cache(struct kmem_cache *s, const char *name,
967 unsigned int size, slab_flags_t flags,
968 unsigned int useroffset, unsigned int usersize)
969{
970 int err;
971
972 s->name = name;
973 s->size = s->object_size = size;
974 s->align = calculate_alignment(flags, ARCH_KMALLOC_MINALIGN, size);
975 s->useroffset = useroffset;
976 s->usersize = usersize;
977
978 slab_init_memcg_params(s);
979
980 err = __kmem_cache_create(s, flags);
981
982 if (err)
983 panic("Creation of kmalloc slab %s size=%u failed. Reason %d\n",
984 name, size, err);
985
986 s->refcount = -1;
987}
988
989struct kmem_cache *__init create_kmalloc_cache(const char *name,
990 unsigned int size, slab_flags_t flags,
991 unsigned int useroffset, unsigned int usersize)
992{
993 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
994
995 if (!s)
996 panic("Out of memory when creating slab %s\n", name);
997
998 create_boot_cache(s, name, size, flags, useroffset, usersize);
999 list_add(&s->list, &slab_caches);
1000 memcg_link_cache(s);
1001 s->refcount = 1;
1002 return s;
1003}
1004
1005struct kmem_cache *
1006kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1] __ro_after_init;
1007EXPORT_SYMBOL(kmalloc_caches);
1008
1009
1010
1011
1012
1013
1014
1015static u8 size_index[24] __ro_after_init = {
1016 3,
1017 4,
1018 5,
1019 5,
1020 6,
1021 6,
1022 6,
1023 6,
1024 1,
1025 1,
1026 1,
1027 1,
1028 7,
1029 7,
1030 7,
1031 7,
1032 2,
1033 2,
1034 2,
1035 2,
1036 2,
1037 2,
1038 2,
1039 2
1040};
1041
1042static inline unsigned int size_index_elem(unsigned int bytes)
1043{
1044 return (bytes - 1) / 8;
1045}
1046
1047
1048
1049
1050
1051struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
1052{
1053 unsigned int index;
1054
1055 if (size <= 192) {
1056 if (!size)
1057 return ZERO_SIZE_PTR;
1058
1059 index = size_index[size_index_elem(size)];
1060 } else {
1061 if (WARN_ON_ONCE(size > KMALLOC_MAX_CACHE_SIZE))
1062 return NULL;
1063 index = fls(size - 1);
1064 }
1065
1066 return kmalloc_caches[kmalloc_type(flags)][index];
1067}
1068
1069
1070
1071
1072
1073
1074const struct kmalloc_info_struct kmalloc_info[] __initconst = {
1075 {NULL, 0}, {"kmalloc-96", 96},
1076 {"kmalloc-192", 192}, {"kmalloc-8", 8},
1077 {"kmalloc-16", 16}, {"kmalloc-32", 32},
1078 {"kmalloc-64", 64}, {"kmalloc-128", 128},
1079 {"kmalloc-256", 256}, {"kmalloc-512", 512},
1080 {"kmalloc-1k", 1024}, {"kmalloc-2k", 2048},
1081 {"kmalloc-4k", 4096}, {"kmalloc-8k", 8192},
1082 {"kmalloc-16k", 16384}, {"kmalloc-32k", 32768},
1083 {"kmalloc-64k", 65536}, {"kmalloc-128k", 131072},
1084 {"kmalloc-256k", 262144}, {"kmalloc-512k", 524288},
1085 {"kmalloc-1M", 1048576}, {"kmalloc-2M", 2097152},
1086 {"kmalloc-4M", 4194304}, {"kmalloc-8M", 8388608},
1087 {"kmalloc-16M", 16777216}, {"kmalloc-32M", 33554432},
1088 {"kmalloc-64M", 67108864}
1089};
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102void __init setup_kmalloc_cache_index_table(void)
1103{
1104 unsigned int i;
1105
1106 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
1107 (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
1108
1109 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
1110 unsigned int elem = size_index_elem(i);
1111
1112 if (elem >= ARRAY_SIZE(size_index))
1113 break;
1114 size_index[elem] = KMALLOC_SHIFT_LOW;
1115 }
1116
1117 if (KMALLOC_MIN_SIZE >= 64) {
1118
1119
1120
1121
1122 for (i = 64 + 8; i <= 96; i += 8)
1123 size_index[size_index_elem(i)] = 7;
1124
1125 }
1126
1127 if (KMALLOC_MIN_SIZE >= 128) {
1128
1129
1130
1131
1132
1133 for (i = 128 + 8; i <= 192; i += 8)
1134 size_index[size_index_elem(i)] = 8;
1135 }
1136}
1137
1138static const char *
1139kmalloc_cache_name(const char *prefix, unsigned int size)
1140{
1141
1142 static const char units[3] = "\0kM";
1143 int idx = 0;
1144
1145 while (size >= 1024 && (size % 1024 == 0)) {
1146 size /= 1024;
1147 idx++;
1148 }
1149
1150 return kasprintf(GFP_NOWAIT, "%s-%u%c", prefix, size, units[idx]);
1151}
1152
1153static void __init
1154new_kmalloc_cache(int idx, int type, slab_flags_t flags)
1155{
1156 const char *name;
1157
1158 if (type == KMALLOC_RECLAIM) {
1159 flags |= SLAB_RECLAIM_ACCOUNT;
1160 name = kmalloc_cache_name("kmalloc-rcl",
1161 kmalloc_info[idx].size);
1162 BUG_ON(!name);
1163 } else {
1164 name = kmalloc_info[idx].name;
1165 }
1166
1167 kmalloc_caches[type][idx] = create_kmalloc_cache(name,
1168 kmalloc_info[idx].size, flags, 0,
1169 kmalloc_info[idx].size);
1170}
1171
1172
1173
1174
1175
1176
1177void __init create_kmalloc_caches(slab_flags_t flags)
1178{
1179 int i, type;
1180
1181 for (type = KMALLOC_NORMAL; type <= KMALLOC_RECLAIM; type++) {
1182 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
1183 if (!kmalloc_caches[type][i])
1184 new_kmalloc_cache(i, type, flags);
1185
1186
1187
1188
1189
1190
1191 if (KMALLOC_MIN_SIZE <= 32 && i == 6 &&
1192 !kmalloc_caches[type][1])
1193 new_kmalloc_cache(1, type, flags);
1194 if (KMALLOC_MIN_SIZE <= 64 && i == 7 &&
1195 !kmalloc_caches[type][2])
1196 new_kmalloc_cache(2, type, flags);
1197 }
1198 }
1199
1200
1201 slab_state = UP;
1202
1203#ifdef CONFIG_ZONE_DMA
1204 for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
1205 struct kmem_cache *s = kmalloc_caches[KMALLOC_NORMAL][i];
1206
1207 if (s) {
1208 unsigned int size = kmalloc_size(i);
1209 const char *n = kmalloc_cache_name("dma-kmalloc", size);
1210
1211 BUG_ON(!n);
1212 kmalloc_caches[KMALLOC_DMA][i] = create_kmalloc_cache(
1213 n, size, SLAB_CACHE_DMA | flags, 0, 0);
1214 }
1215 }
1216#endif
1217}
1218#endif
1219
1220
1221
1222
1223
1224
1225void *kmalloc_order(size_t size, gfp_t flags, unsigned int order)
1226{
1227 void *ret;
1228 struct page *page;
1229
1230 flags |= __GFP_COMP;
1231 page = alloc_pages(flags, order);
1232 ret = page ? page_address(page) : NULL;
1233 ret = kasan_kmalloc_large(ret, size, flags);
1234
1235 kmemleak_alloc(ret, size, 1, flags);
1236 return ret;
1237}
1238EXPORT_SYMBOL(kmalloc_order);
1239
1240#ifdef CONFIG_TRACING
1241void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
1242{
1243 void *ret = kmalloc_order(size, flags, order);
1244 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags);
1245 return ret;
1246}
1247EXPORT_SYMBOL(kmalloc_order_trace);
1248#endif
1249
1250#ifdef CONFIG_SLAB_FREELIST_RANDOM
1251
1252static void freelist_randomize(struct rnd_state *state, unsigned int *list,
1253 unsigned int count)
1254{
1255 unsigned int rand;
1256 unsigned int i;
1257
1258 for (i = 0; i < count; i++)
1259 list[i] = i;
1260
1261
1262 for (i = count - 1; i > 0; i--) {
1263 rand = prandom_u32_state(state);
1264 rand %= (i + 1);
1265 swap(list[i], list[rand]);
1266 }
1267}
1268
1269
1270int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
1271 gfp_t gfp)
1272{
1273 struct rnd_state state;
1274
1275 if (count < 2 || cachep->random_seq)
1276 return 0;
1277
1278 cachep->random_seq = kcalloc(count, sizeof(unsigned int), gfp);
1279 if (!cachep->random_seq)
1280 return -ENOMEM;
1281
1282
1283 prandom_seed_state(&state, get_random_long());
1284
1285 freelist_randomize(&state, cachep->random_seq, count);
1286 return 0;
1287}
1288
1289
1290void cache_random_seq_destroy(struct kmem_cache *cachep)
1291{
1292 kfree(cachep->random_seq);
1293 cachep->random_seq = NULL;
1294}
1295#endif
1296
1297#if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
1298#ifdef CONFIG_SLAB
1299#define SLABINFO_RIGHTS (0600)
1300#else
1301#define SLABINFO_RIGHTS (0400)
1302#endif
1303
1304static void print_slabinfo_header(struct seq_file *m)
1305{
1306
1307
1308
1309
1310#ifdef CONFIG_DEBUG_SLAB
1311 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
1312#else
1313 seq_puts(m, "slabinfo - version: 2.1\n");
1314#endif
1315 seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>");
1316 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
1317 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
1318#ifdef CONFIG_DEBUG_SLAB
1319 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> <error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
1320 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
1321#endif
1322 seq_putc(m, '\n');
1323}
1324
1325void *slab_start(struct seq_file *m, loff_t *pos)
1326{
1327 mutex_lock(&slab_mutex);
1328 return seq_list_start(&slab_root_caches, *pos);
1329}
1330
1331void *slab_next(struct seq_file *m, void *p, loff_t *pos)
1332{
1333 return seq_list_next(p, &slab_root_caches, pos);
1334}
1335
1336void slab_stop(struct seq_file *m, void *p)
1337{
1338 mutex_unlock(&slab_mutex);
1339}
1340
1341static void
1342memcg_accumulate_slabinfo(struct kmem_cache *s, struct slabinfo *info)
1343{
1344 struct kmem_cache *c;
1345 struct slabinfo sinfo;
1346
1347 if (!is_root_cache(s))
1348 return;
1349
1350 for_each_memcg_cache(c, s) {
1351 memset(&sinfo, 0, sizeof(sinfo));
1352 get_slabinfo(c, &sinfo);
1353
1354 info->active_slabs += sinfo.active_slabs;
1355 info->num_slabs += sinfo.num_slabs;
1356 info->shared_avail += sinfo.shared_avail;
1357 info->active_objs += sinfo.active_objs;
1358 info->num_objs += sinfo.num_objs;
1359 }
1360}
1361
1362static void cache_show(struct kmem_cache *s, struct seq_file *m)
1363{
1364 struct slabinfo sinfo;
1365
1366 memset(&sinfo, 0, sizeof(sinfo));
1367 get_slabinfo(s, &sinfo);
1368
1369 memcg_accumulate_slabinfo(s, &sinfo);
1370
1371 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
1372 cache_name(s), sinfo.active_objs, sinfo.num_objs, s->size,
1373 sinfo.objects_per_slab, (1 << sinfo.cache_order));
1374
1375 seq_printf(m, " : tunables %4u %4u %4u",
1376 sinfo.limit, sinfo.batchcount, sinfo.shared);
1377 seq_printf(m, " : slabdata %6lu %6lu %6lu",
1378 sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail);
1379 slabinfo_show_stats(m, s);
1380 seq_putc(m, '\n');
1381}
1382
1383static int slab_show(struct seq_file *m, void *p)
1384{
1385 struct kmem_cache *s = list_entry(p, struct kmem_cache, root_caches_node);
1386
1387 if (p == slab_root_caches.next)
1388 print_slabinfo_header(m);
1389 cache_show(s, m);
1390 return 0;
1391}
1392
1393void dump_unreclaimable_slab(void)
1394{
1395 struct kmem_cache *s, *s2;
1396 struct slabinfo sinfo;
1397
1398
1399
1400
1401
1402
1403
1404
1405 if (!mutex_trylock(&slab_mutex)) {
1406 pr_warn("excessive unreclaimable slab but cannot dump stats\n");
1407 return;
1408 }
1409
1410 pr_info("Unreclaimable slab info:\n");
1411 pr_info("Name Used Total\n");
1412
1413 list_for_each_entry_safe(s, s2, &slab_caches, list) {
1414 if (!is_root_cache(s) || (s->flags & SLAB_RECLAIM_ACCOUNT))
1415 continue;
1416
1417 get_slabinfo(s, &sinfo);
1418
1419 if (sinfo.num_objs > 0)
1420 pr_info("%-17s %10luKB %10luKB\n", cache_name(s),
1421 (sinfo.active_objs * s->size) / 1024,
1422 (sinfo.num_objs * s->size) / 1024);
1423 }
1424 mutex_unlock(&slab_mutex);
1425}
1426
1427#if defined(CONFIG_MEMCG)
1428void *memcg_slab_start(struct seq_file *m, loff_t *pos)
1429{
1430 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
1431
1432 mutex_lock(&slab_mutex);
1433 return seq_list_start(&memcg->kmem_caches, *pos);
1434}
1435
1436void *memcg_slab_next(struct seq_file *m, void *p, loff_t *pos)
1437{
1438 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
1439
1440 return seq_list_next(p, &memcg->kmem_caches, pos);
1441}
1442
1443void memcg_slab_stop(struct seq_file *m, void *p)
1444{
1445 mutex_unlock(&slab_mutex);
1446}
1447
1448int memcg_slab_show(struct seq_file *m, void *p)
1449{
1450 struct kmem_cache *s = list_entry(p, struct kmem_cache,
1451 memcg_params.kmem_caches_node);
1452 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
1453
1454 if (p == memcg->kmem_caches.next)
1455 print_slabinfo_header(m);
1456 cache_show(s, m);
1457 return 0;
1458}
1459#endif
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474static const struct seq_operations slabinfo_op = {
1475 .start = slab_start,
1476 .next = slab_next,
1477 .stop = slab_stop,
1478 .show = slab_show,
1479};
1480
1481static int slabinfo_open(struct inode *inode, struct file *file)
1482{
1483 return seq_open(file, &slabinfo_op);
1484}
1485
1486static const struct file_operations proc_slabinfo_operations = {
1487 .open = slabinfo_open,
1488 .read = seq_read,
1489 .write = slabinfo_write,
1490 .llseek = seq_lseek,
1491 .release = seq_release,
1492};
1493
1494static int __init slab_proc_init(void)
1495{
1496 proc_create("slabinfo", SLABINFO_RIGHTS, NULL,
1497 &proc_slabinfo_operations);
1498 return 0;
1499}
1500module_init(slab_proc_init);
1501#endif
1502
1503static __always_inline void *__do_krealloc(const void *p, size_t new_size,
1504 gfp_t flags)
1505{
1506 void *ret;
1507 size_t ks = 0;
1508
1509 if (p)
1510 ks = ksize(p);
1511
1512 if (ks >= new_size) {
1513 p = kasan_krealloc((void *)p, new_size, flags);
1514 return (void *)p;
1515 }
1516
1517 ret = kmalloc_track_caller(new_size, flags);
1518 if (ret && p)
1519 memcpy(ret, p, ks);
1520
1521 return ret;
1522}
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536void *__krealloc(const void *p, size_t new_size, gfp_t flags)
1537{
1538 if (unlikely(!new_size))
1539 return ZERO_SIZE_PTR;
1540
1541 return __do_krealloc(p, new_size, flags);
1542
1543}
1544EXPORT_SYMBOL(__krealloc);
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559void *krealloc(const void *p, size_t new_size, gfp_t flags)
1560{
1561 void *ret;
1562
1563 if (unlikely(!new_size)) {
1564 kfree(p);
1565 return ZERO_SIZE_PTR;
1566 }
1567
1568 ret = __do_krealloc(p, new_size, flags);
1569 if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret))
1570 kfree(p);
1571
1572 return ret;
1573}
1574EXPORT_SYMBOL(krealloc);
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587void kzfree(const void *p)
1588{
1589 size_t ks;
1590 void *mem = (void *)p;
1591
1592 if (unlikely(ZERO_OR_NULL_PTR(mem)))
1593 return;
1594 ks = ksize(mem);
1595 memset(mem, 0, ks);
1596 kfree(mem);
1597}
1598EXPORT_SYMBOL(kzfree);
1599
1600
1601EXPORT_TRACEPOINT_SYMBOL(kmalloc);
1602EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
1603EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
1604EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
1605EXPORT_TRACEPOINT_SYMBOL(kfree);
1606EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);
1607
1608int should_failslab(struct kmem_cache *s, gfp_t gfpflags)
1609{
1610 if (__should_failslab(s, gfpflags))
1611 return -ENOMEM;
1612 return 0;
1613}
1614ALLOW_ERROR_INJECTION(should_failslab, ERRNO);
1615