1
2
3
4
5
6
7#include <linux/slab.h>
8
9#include <linux/mm.h>
10#include <linux/poison.h>
11#include <linux/interrupt.h>
12#include <linux/memory.h>
13#include <linux/cache.h>
14#include <linux/compiler.h>
15#include <linux/kfence.h>
16#include <linux/module.h>
17#include <linux/cpu.h>
18#include <linux/uaccess.h>
19#include <linux/seq_file.h>
20#include <linux/proc_fs.h>
21#include <linux/debugfs.h>
22#include <linux/kasan.h>
23#include <asm/cacheflush.h>
24#include <asm/tlbflush.h>
25#include <asm/page.h>
26#include <linux/memcontrol.h>
27
28#define CREATE_TRACE_POINTS
29#include <trace/events/kmem.h>
30
31#include "internal.h"
32
33#include "slab.h"
34
35enum slab_state slab_state;
36LIST_HEAD(slab_caches);
37DEFINE_MUTEX(slab_mutex);
38struct kmem_cache *kmem_cache;
39
40#ifdef CONFIG_HARDENED_USERCOPY
41bool usercopy_fallback __ro_after_init =
42 IS_ENABLED(CONFIG_HARDENED_USERCOPY_FALLBACK);
43module_param(usercopy_fallback, bool, 0400);
44MODULE_PARM_DESC(usercopy_fallback,
45 "WARN instead of reject usercopy whitelist violations");
46#endif
47
48static LIST_HEAD(slab_caches_to_rcu_destroy);
49static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work);
50static DECLARE_WORK(slab_caches_to_rcu_destroy_work,
51 slab_caches_to_rcu_destroy_workfn);
52
53
54
55
56#define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
57 SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \
58 SLAB_FAILSLAB | kasan_never_merge())
59
60#define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \
61 SLAB_CACHE_DMA32 | SLAB_ACCOUNT)
62
63
64
65
66static bool slab_nomerge = !IS_ENABLED(CONFIG_SLAB_MERGE_DEFAULT);
67
68static int __init setup_slab_nomerge(char *str)
69{
70 slab_nomerge = true;
71 return 1;
72}
73
74static int __init setup_slab_merge(char *str)
75{
76 slab_nomerge = false;
77 return 1;
78}
79
80#ifdef CONFIG_SLUB
81__setup_param("slub_nomerge", slub_nomerge, setup_slab_nomerge, 0);
82__setup_param("slub_merge", slub_merge, setup_slab_merge, 0);
83#endif
84
85__setup("slab_nomerge", setup_slab_nomerge);
86__setup("slab_merge", setup_slab_merge);
87
88
89
90
91unsigned int kmem_cache_size(struct kmem_cache *s)
92{
93 return s->object_size;
94}
95EXPORT_SYMBOL(kmem_cache_size);
96
97#ifdef CONFIG_DEBUG_VM
98static int kmem_cache_sanity_check(const char *name, unsigned int size)
99{
100 if (!name || in_interrupt() || size > KMALLOC_MAX_SIZE) {
101 pr_err("kmem_cache_create(%s) integrity check failed\n", name);
102 return -EINVAL;
103 }
104
105 WARN_ON(strchr(name, ' '));
106 return 0;
107}
108#else
109static inline int kmem_cache_sanity_check(const char *name, unsigned int size)
110{
111 return 0;
112}
113#endif
114
115void __kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p)
116{
117 size_t i;
118
119 for (i = 0; i < nr; i++) {
120 if (s)
121 kmem_cache_free(s, p[i]);
122 else
123 kfree(p[i]);
124 }
125}
126
127int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr,
128 void **p)
129{
130 size_t i;
131
132 for (i = 0; i < nr; i++) {
133 void *x = p[i] = kmem_cache_alloc(s, flags);
134 if (!x) {
135 __kmem_cache_free_bulk(s, i, p);
136 return 0;
137 }
138 }
139 return i;
140}
141
142
143
144
145
146static unsigned int calculate_alignment(slab_flags_t flags,
147 unsigned int align, unsigned int size)
148{
149
150
151
152
153
154
155
156 if (flags & SLAB_HWCACHE_ALIGN) {
157 unsigned int ralign;
158
159 ralign = cache_line_size();
160 while (size <= ralign / 2)
161 ralign /= 2;
162 align = max(align, ralign);
163 }
164
165 if (align < ARCH_SLAB_MINALIGN)
166 align = ARCH_SLAB_MINALIGN;
167
168 return ALIGN(align, sizeof(void *));
169}
170
171
172
173
174int slab_unmergeable(struct kmem_cache *s)
175{
176 if (slab_nomerge || (s->flags & SLAB_NEVER_MERGE))
177 return 1;
178
179 if (s->ctor)
180 return 1;
181
182 if (s->usersize)
183 return 1;
184
185
186
187
188 if (s->refcount < 0)
189 return 1;
190
191 return 0;
192}
193
194struct kmem_cache *find_mergeable(unsigned int size, unsigned int align,
195 slab_flags_t flags, const char *name, void (*ctor)(void *))
196{
197 struct kmem_cache *s;
198
199 if (slab_nomerge)
200 return NULL;
201
202 if (ctor)
203 return NULL;
204
205 size = ALIGN(size, sizeof(void *));
206 align = calculate_alignment(flags, align, size);
207 size = ALIGN(size, align);
208 flags = kmem_cache_flags(size, flags, name);
209
210 if (flags & SLAB_NEVER_MERGE)
211 return NULL;
212
213 list_for_each_entry_reverse(s, &slab_caches, list) {
214 if (slab_unmergeable(s))
215 continue;
216
217 if (size > s->size)
218 continue;
219
220 if ((flags & SLAB_MERGE_SAME) != (s->flags & SLAB_MERGE_SAME))
221 continue;
222
223
224
225
226 if ((s->size & ~(align - 1)) != s->size)
227 continue;
228
229 if (s->size - size >= sizeof(void *))
230 continue;
231
232 if (IS_ENABLED(CONFIG_SLAB) && align &&
233 (align > s->align || s->align % align))
234 continue;
235
236 return s;
237 }
238 return NULL;
239}
240
241static struct kmem_cache *create_cache(const char *name,
242 unsigned int object_size, unsigned int align,
243 slab_flags_t flags, unsigned int useroffset,
244 unsigned int usersize, void (*ctor)(void *),
245 struct kmem_cache *root_cache)
246{
247 struct kmem_cache *s;
248 int err;
249
250 if (WARN_ON(useroffset + usersize > object_size))
251 useroffset = usersize = 0;
252
253 err = -ENOMEM;
254 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
255 if (!s)
256 goto out;
257
258 s->name = name;
259 s->size = s->object_size = object_size;
260 s->align = align;
261 s->ctor = ctor;
262 s->useroffset = useroffset;
263 s->usersize = usersize;
264
265 err = __kmem_cache_create(s, flags);
266 if (err)
267 goto out_free_cache;
268
269 s->refcount = 1;
270 list_add(&s->list, &slab_caches);
271out:
272 if (err)
273 return ERR_PTR(err);
274 return s;
275
276out_free_cache:
277 kmem_cache_free(kmem_cache, s);
278 goto out;
279}
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309struct kmem_cache *
310kmem_cache_create_usercopy(const char *name,
311 unsigned int size, unsigned int align,
312 slab_flags_t flags,
313 unsigned int useroffset, unsigned int usersize,
314 void (*ctor)(void *))
315{
316 struct kmem_cache *s = NULL;
317 const char *cache_name;
318 int err;
319
320#ifdef CONFIG_SLUB_DEBUG
321
322
323
324
325
326 if (flags & SLAB_DEBUG_FLAGS)
327 static_branch_enable(&slub_debug_enabled);
328#endif
329
330 mutex_lock(&slab_mutex);
331
332 err = kmem_cache_sanity_check(name, size);
333 if (err) {
334 goto out_unlock;
335 }
336
337
338 if (flags & ~SLAB_FLAGS_PERMITTED) {
339 err = -EINVAL;
340 goto out_unlock;
341 }
342
343
344
345
346
347
348
349 flags &= CACHE_CREATE_MASK;
350
351
352 if (WARN_ON(!usersize && useroffset) ||
353 WARN_ON(size < usersize || size - usersize < useroffset))
354 usersize = useroffset = 0;
355
356 if (!usersize)
357 s = __kmem_cache_alias(name, size, align, flags, ctor);
358 if (s)
359 goto out_unlock;
360
361 cache_name = kstrdup_const(name, GFP_KERNEL);
362 if (!cache_name) {
363 err = -ENOMEM;
364 goto out_unlock;
365 }
366
367 s = create_cache(cache_name, size,
368 calculate_alignment(flags, align, size),
369 flags, useroffset, usersize, ctor, NULL);
370 if (IS_ERR(s)) {
371 err = PTR_ERR(s);
372 kfree_const(cache_name);
373 }
374
375out_unlock:
376 mutex_unlock(&slab_mutex);
377
378 if (err) {
379 if (flags & SLAB_PANIC)
380 panic("%s: Failed to create slab '%s'. Error %d\n",
381 __func__, name, err);
382 else {
383 pr_warn("%s(%s) failed with error %d\n",
384 __func__, name, err);
385 dump_stack();
386 }
387 return NULL;
388 }
389 return s;
390}
391EXPORT_SYMBOL(kmem_cache_create_usercopy);
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418struct kmem_cache *
419kmem_cache_create(const char *name, unsigned int size, unsigned int align,
420 slab_flags_t flags, void (*ctor)(void *))
421{
422 return kmem_cache_create_usercopy(name, size, align, flags, 0, 0,
423 ctor);
424}
425EXPORT_SYMBOL(kmem_cache_create);
426
427static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work)
428{
429 LIST_HEAD(to_destroy);
430 struct kmem_cache *s, *s2;
431
432
433
434
435
436
437
438
439
440
441 mutex_lock(&slab_mutex);
442 list_splice_init(&slab_caches_to_rcu_destroy, &to_destroy);
443 mutex_unlock(&slab_mutex);
444
445 if (list_empty(&to_destroy))
446 return;
447
448 rcu_barrier();
449
450 list_for_each_entry_safe(s, s2, &to_destroy, list) {
451 debugfs_slab_release(s);
452 kfence_shutdown_cache(s);
453#ifdef SLAB_SUPPORTS_SYSFS
454 sysfs_slab_release(s);
455#else
456 slab_kmem_cache_release(s);
457#endif
458 }
459}
460
461static int shutdown_cache(struct kmem_cache *s)
462{
463
464 kasan_cache_shutdown(s);
465
466 if (__kmem_cache_shutdown(s) != 0)
467 return -EBUSY;
468
469 list_del(&s->list);
470
471 if (s->flags & SLAB_TYPESAFE_BY_RCU) {
472#ifdef SLAB_SUPPORTS_SYSFS
473 sysfs_slab_unlink(s);
474#endif
475 list_add_tail(&s->list, &slab_caches_to_rcu_destroy);
476 schedule_work(&slab_caches_to_rcu_destroy_work);
477 } else {
478 kfence_shutdown_cache(s);
479 debugfs_slab_release(s);
480#ifdef SLAB_SUPPORTS_SYSFS
481 sysfs_slab_unlink(s);
482 sysfs_slab_release(s);
483#else
484 slab_kmem_cache_release(s);
485#endif
486 }
487
488 return 0;
489}
490
491void slab_kmem_cache_release(struct kmem_cache *s)
492{
493 __kmem_cache_release(s);
494 kfree_const(s->name);
495 kmem_cache_free(kmem_cache, s);
496}
497
498void kmem_cache_destroy(struct kmem_cache *s)
499{
500 int err;
501
502 if (unlikely(!s))
503 return;
504
505 cpus_read_lock();
506 mutex_lock(&slab_mutex);
507
508 s->refcount--;
509 if (s->refcount)
510 goto out_unlock;
511
512 err = shutdown_cache(s);
513 if (err) {
514 pr_err("%s %s: Slab cache still has objects\n",
515 __func__, s->name);
516 dump_stack();
517 }
518out_unlock:
519 mutex_unlock(&slab_mutex);
520 cpus_read_unlock();
521}
522EXPORT_SYMBOL(kmem_cache_destroy);
523
524
525
526
527
528
529
530
531
532
533int kmem_cache_shrink(struct kmem_cache *cachep)
534{
535 int ret;
536
537
538 kasan_cache_shrink(cachep);
539 ret = __kmem_cache_shrink(cachep);
540
541 return ret;
542}
543EXPORT_SYMBOL(kmem_cache_shrink);
544
545bool slab_is_available(void)
546{
547 return slab_state >= UP;
548}
549
550#ifdef CONFIG_PRINTK
551
552
553
554
555
556
557
558
559bool kmem_valid_obj(void *object)
560{
561 struct page *page;
562
563
564 if (object < (void *)PAGE_SIZE || !virt_addr_valid(object))
565 return false;
566 page = virt_to_head_page(object);
567 return PageSlab(page);
568}
569EXPORT_SYMBOL_GPL(kmem_valid_obj);
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586void kmem_dump_obj(void *object)
587{
588 char *cp = IS_ENABLED(CONFIG_MMU) ? "" : "/vmalloc";
589 int i;
590 struct page *page;
591 unsigned long ptroffset;
592 struct kmem_obj_info kp = { };
593
594 if (WARN_ON_ONCE(!virt_addr_valid(object)))
595 return;
596 page = virt_to_head_page(object);
597 if (WARN_ON_ONCE(!PageSlab(page))) {
598 pr_cont(" non-slab memory.\n");
599 return;
600 }
601 kmem_obj_info(&kp, object, page);
602 if (kp.kp_slab_cache)
603 pr_cont(" slab%s %s", cp, kp.kp_slab_cache->name);
604 else
605 pr_cont(" slab%s", cp);
606 if (kp.kp_objp)
607 pr_cont(" start %px", kp.kp_objp);
608 if (kp.kp_data_offset)
609 pr_cont(" data offset %lu", kp.kp_data_offset);
610 if (kp.kp_objp) {
611 ptroffset = ((char *)object - (char *)kp.kp_objp) - kp.kp_data_offset;
612 pr_cont(" pointer offset %lu", ptroffset);
613 }
614 if (kp.kp_slab_cache && kp.kp_slab_cache->usersize)
615 pr_cont(" size %u", kp.kp_slab_cache->usersize);
616 if (kp.kp_ret)
617 pr_cont(" allocated at %pS\n", kp.kp_ret);
618 else
619 pr_cont("\n");
620 for (i = 0; i < ARRAY_SIZE(kp.kp_stack); i++) {
621 if (!kp.kp_stack[i])
622 break;
623 pr_info(" %pS\n", kp.kp_stack[i]);
624 }
625
626 if (kp.kp_free_stack[0])
627 pr_cont(" Free path:\n");
628
629 for (i = 0; i < ARRAY_SIZE(kp.kp_free_stack); i++) {
630 if (!kp.kp_free_stack[i])
631 break;
632 pr_info(" %pS\n", kp.kp_free_stack[i]);
633 }
634
635}
636EXPORT_SYMBOL_GPL(kmem_dump_obj);
637#endif
638
639#ifndef CONFIG_SLOB
640
641void __init create_boot_cache(struct kmem_cache *s, const char *name,
642 unsigned int size, slab_flags_t flags,
643 unsigned int useroffset, unsigned int usersize)
644{
645 int err;
646 unsigned int align = ARCH_KMALLOC_MINALIGN;
647
648 s->name = name;
649 s->size = s->object_size = size;
650
651
652
653
654
655 if (is_power_of_2(size))
656 align = max(align, size);
657 s->align = calculate_alignment(flags, align, size);
658
659 s->useroffset = useroffset;
660 s->usersize = usersize;
661
662 err = __kmem_cache_create(s, flags);
663
664 if (err)
665 panic("Creation of kmalloc slab %s size=%u failed. Reason %d\n",
666 name, size, err);
667
668 s->refcount = -1;
669}
670
671struct kmem_cache *__init create_kmalloc_cache(const char *name,
672 unsigned int size, slab_flags_t flags,
673 unsigned int useroffset, unsigned int usersize)
674{
675 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
676
677 if (!s)
678 panic("Out of memory when creating slab %s\n", name);
679
680 create_boot_cache(s, name, size, flags, useroffset, usersize);
681 kasan_cache_create_kmalloc(s);
682 list_add(&s->list, &slab_caches);
683 s->refcount = 1;
684 return s;
685}
686
687struct kmem_cache *
688kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1] __ro_after_init =
689{ };
690EXPORT_SYMBOL(kmalloc_caches);
691
692
693
694
695
696
697
698static u8 size_index[24] __ro_after_init = {
699 3,
700 4,
701 5,
702 5,
703 6,
704 6,
705 6,
706 6,
707 1,
708 1,
709 1,
710 1,
711 7,
712 7,
713 7,
714 7,
715 2,
716 2,
717 2,
718 2,
719 2,
720 2,
721 2,
722 2
723};
724
725static inline unsigned int size_index_elem(unsigned int bytes)
726{
727 return (bytes - 1) / 8;
728}
729
730
731
732
733
734struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
735{
736 unsigned int index;
737
738 if (size <= 192) {
739 if (!size)
740 return ZERO_SIZE_PTR;
741
742 index = size_index[size_index_elem(size)];
743 } else {
744 if (WARN_ON_ONCE(size > KMALLOC_MAX_CACHE_SIZE))
745 return NULL;
746 index = fls(size - 1);
747 }
748
749 return kmalloc_caches[kmalloc_type(flags)][index];
750}
751
752#ifdef CONFIG_ZONE_DMA
753#define KMALLOC_DMA_NAME(sz) .name[KMALLOC_DMA] = "dma-kmalloc-" #sz,
754#else
755#define KMALLOC_DMA_NAME(sz)
756#endif
757
758#ifdef CONFIG_MEMCG_KMEM
759#define KMALLOC_CGROUP_NAME(sz) .name[KMALLOC_CGROUP] = "kmalloc-cg-" #sz,
760#else
761#define KMALLOC_CGROUP_NAME(sz)
762#endif
763
764#define INIT_KMALLOC_INFO(__size, __short_size) \
765{ \
766 .name[KMALLOC_NORMAL] = "kmalloc-" #__short_size, \
767 .name[KMALLOC_RECLAIM] = "kmalloc-rcl-" #__short_size, \
768 KMALLOC_CGROUP_NAME(__short_size) \
769 KMALLOC_DMA_NAME(__short_size) \
770 .size = __size, \
771}
772
773
774
775
776
777
778const struct kmalloc_info_struct kmalloc_info[] __initconst = {
779 INIT_KMALLOC_INFO(0, 0),
780 INIT_KMALLOC_INFO(96, 96),
781 INIT_KMALLOC_INFO(192, 192),
782 INIT_KMALLOC_INFO(8, 8),
783 INIT_KMALLOC_INFO(16, 16),
784 INIT_KMALLOC_INFO(32, 32),
785 INIT_KMALLOC_INFO(64, 64),
786 INIT_KMALLOC_INFO(128, 128),
787 INIT_KMALLOC_INFO(256, 256),
788 INIT_KMALLOC_INFO(512, 512),
789 INIT_KMALLOC_INFO(1024, 1k),
790 INIT_KMALLOC_INFO(2048, 2k),
791 INIT_KMALLOC_INFO(4096, 4k),
792 INIT_KMALLOC_INFO(8192, 8k),
793 INIT_KMALLOC_INFO(16384, 16k),
794 INIT_KMALLOC_INFO(32768, 32k),
795 INIT_KMALLOC_INFO(65536, 64k),
796 INIT_KMALLOC_INFO(131072, 128k),
797 INIT_KMALLOC_INFO(262144, 256k),
798 INIT_KMALLOC_INFO(524288, 512k),
799 INIT_KMALLOC_INFO(1048576, 1M),
800 INIT_KMALLOC_INFO(2097152, 2M),
801 INIT_KMALLOC_INFO(4194304, 4M),
802 INIT_KMALLOC_INFO(8388608, 8M),
803 INIT_KMALLOC_INFO(16777216, 16M),
804 INIT_KMALLOC_INFO(33554432, 32M)
805};
806
807
808
809
810
811
812
813
814
815
816
817
818void __init setup_kmalloc_cache_index_table(void)
819{
820 unsigned int i;
821
822 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
823 (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
824
825 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
826 unsigned int elem = size_index_elem(i);
827
828 if (elem >= ARRAY_SIZE(size_index))
829 break;
830 size_index[elem] = KMALLOC_SHIFT_LOW;
831 }
832
833 if (KMALLOC_MIN_SIZE >= 64) {
834
835
836
837
838 for (i = 64 + 8; i <= 96; i += 8)
839 size_index[size_index_elem(i)] = 7;
840
841 }
842
843 if (KMALLOC_MIN_SIZE >= 128) {
844
845
846
847
848
849 for (i = 128 + 8; i <= 192; i += 8)
850 size_index[size_index_elem(i)] = 8;
851 }
852}
853
854static void __init
855new_kmalloc_cache(int idx, enum kmalloc_cache_type type, slab_flags_t flags)
856{
857 if (type == KMALLOC_RECLAIM) {
858 flags |= SLAB_RECLAIM_ACCOUNT;
859 } else if (IS_ENABLED(CONFIG_MEMCG_KMEM) && (type == KMALLOC_CGROUP)) {
860 if (cgroup_memory_nokmem) {
861 kmalloc_caches[type][idx] = kmalloc_caches[KMALLOC_NORMAL][idx];
862 return;
863 }
864 flags |= SLAB_ACCOUNT;
865 }
866
867 kmalloc_caches[type][idx] = create_kmalloc_cache(
868 kmalloc_info[idx].name[type],
869 kmalloc_info[idx].size, flags, 0,
870 kmalloc_info[idx].size);
871
872
873
874
875
876 if (IS_ENABLED(CONFIG_MEMCG_KMEM) && (type == KMALLOC_NORMAL))
877 kmalloc_caches[type][idx]->refcount = -1;
878}
879
880
881
882
883
884
885void __init create_kmalloc_caches(slab_flags_t flags)
886{
887 int i;
888 enum kmalloc_cache_type type;
889
890
891
892
893 for (type = KMALLOC_NORMAL; type <= KMALLOC_RECLAIM; type++) {
894 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
895 if (!kmalloc_caches[type][i])
896 new_kmalloc_cache(i, type, flags);
897
898
899
900
901
902
903 if (KMALLOC_MIN_SIZE <= 32 && i == 6 &&
904 !kmalloc_caches[type][1])
905 new_kmalloc_cache(1, type, flags);
906 if (KMALLOC_MIN_SIZE <= 64 && i == 7 &&
907 !kmalloc_caches[type][2])
908 new_kmalloc_cache(2, type, flags);
909 }
910 }
911
912
913 slab_state = UP;
914
915#ifdef CONFIG_ZONE_DMA
916 for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
917 struct kmem_cache *s = kmalloc_caches[KMALLOC_NORMAL][i];
918
919 if (s) {
920 kmalloc_caches[KMALLOC_DMA][i] = create_kmalloc_cache(
921 kmalloc_info[i].name[KMALLOC_DMA],
922 kmalloc_info[i].size,
923 SLAB_CACHE_DMA | flags, 0,
924 kmalloc_info[i].size);
925 }
926 }
927#endif
928}
929#endif
930
931gfp_t kmalloc_fix_flags(gfp_t flags)
932{
933 gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK;
934
935 flags &= ~GFP_SLAB_BUG_MASK;
936 pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n",
937 invalid_mask, &invalid_mask, flags, &flags);
938 dump_stack();
939
940 return flags;
941}
942
943
944
945
946
947
948void *kmalloc_order(size_t size, gfp_t flags, unsigned int order)
949{
950 void *ret = NULL;
951 struct page *page;
952
953 if (unlikely(flags & GFP_SLAB_BUG_MASK))
954 flags = kmalloc_fix_flags(flags);
955
956 flags |= __GFP_COMP;
957 page = alloc_pages(flags, order);
958 if (likely(page)) {
959 ret = page_address(page);
960 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
961 PAGE_SIZE << order);
962 }
963 ret = kasan_kmalloc_large(ret, size, flags);
964
965 kmemleak_alloc(ret, size, 1, flags);
966 return ret;
967}
968EXPORT_SYMBOL(kmalloc_order);
969
970#ifdef CONFIG_TRACING
971void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
972{
973 void *ret = kmalloc_order(size, flags, order);
974 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags);
975 return ret;
976}
977EXPORT_SYMBOL(kmalloc_order_trace);
978#endif
979
980#ifdef CONFIG_SLAB_FREELIST_RANDOM
981
982static void freelist_randomize(struct rnd_state *state, unsigned int *list,
983 unsigned int count)
984{
985 unsigned int rand;
986 unsigned int i;
987
988 for (i = 0; i < count; i++)
989 list[i] = i;
990
991
992 for (i = count - 1; i > 0; i--) {
993 rand = prandom_u32_state(state);
994 rand %= (i + 1);
995 swap(list[i], list[rand]);
996 }
997}
998
999
1000int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
1001 gfp_t gfp)
1002{
1003 struct rnd_state state;
1004
1005 if (count < 2 || cachep->random_seq)
1006 return 0;
1007
1008 cachep->random_seq = kcalloc(count, sizeof(unsigned int), gfp);
1009 if (!cachep->random_seq)
1010 return -ENOMEM;
1011
1012
1013 prandom_seed_state(&state, get_random_long());
1014
1015 freelist_randomize(&state, cachep->random_seq, count);
1016 return 0;
1017}
1018
1019
1020void cache_random_seq_destroy(struct kmem_cache *cachep)
1021{
1022 kfree(cachep->random_seq);
1023 cachep->random_seq = NULL;
1024}
1025#endif
1026
1027#if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
1028#ifdef CONFIG_SLAB
1029#define SLABINFO_RIGHTS (0600)
1030#else
1031#define SLABINFO_RIGHTS (0400)
1032#endif
1033
1034static void print_slabinfo_header(struct seq_file *m)
1035{
1036
1037
1038
1039
1040#ifdef CONFIG_DEBUG_SLAB
1041 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
1042#else
1043 seq_puts(m, "slabinfo - version: 2.1\n");
1044#endif
1045 seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>");
1046 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
1047 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
1048#ifdef CONFIG_DEBUG_SLAB
1049 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> <error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
1050 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
1051#endif
1052 seq_putc(m, '\n');
1053}
1054
1055void *slab_start(struct seq_file *m, loff_t *pos)
1056{
1057 mutex_lock(&slab_mutex);
1058 return seq_list_start(&slab_caches, *pos);
1059}
1060
1061void *slab_next(struct seq_file *m, void *p, loff_t *pos)
1062{
1063 return seq_list_next(p, &slab_caches, pos);
1064}
1065
1066void slab_stop(struct seq_file *m, void *p)
1067{
1068 mutex_unlock(&slab_mutex);
1069}
1070
1071static void cache_show(struct kmem_cache *s, struct seq_file *m)
1072{
1073 struct slabinfo sinfo;
1074
1075 memset(&sinfo, 0, sizeof(sinfo));
1076 get_slabinfo(s, &sinfo);
1077
1078 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
1079 s->name, sinfo.active_objs, sinfo.num_objs, s->size,
1080 sinfo.objects_per_slab, (1 << sinfo.cache_order));
1081
1082 seq_printf(m, " : tunables %4u %4u %4u",
1083 sinfo.limit, sinfo.batchcount, sinfo.shared);
1084 seq_printf(m, " : slabdata %6lu %6lu %6lu",
1085 sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail);
1086 slabinfo_show_stats(m, s);
1087 seq_putc(m, '\n');
1088}
1089
1090static int slab_show(struct seq_file *m, void *p)
1091{
1092 struct kmem_cache *s = list_entry(p, struct kmem_cache, list);
1093
1094 if (p == slab_caches.next)
1095 print_slabinfo_header(m);
1096 cache_show(s, m);
1097 return 0;
1098}
1099
1100void dump_unreclaimable_slab(void)
1101{
1102 struct kmem_cache *s;
1103 struct slabinfo sinfo;
1104
1105
1106
1107
1108
1109
1110
1111
1112 if (!mutex_trylock(&slab_mutex)) {
1113 pr_warn("excessive unreclaimable slab but cannot dump stats\n");
1114 return;
1115 }
1116
1117 pr_info("Unreclaimable slab info:\n");
1118 pr_info("Name Used Total\n");
1119
1120 list_for_each_entry(s, &slab_caches, list) {
1121 if (s->flags & SLAB_RECLAIM_ACCOUNT)
1122 continue;
1123
1124 get_slabinfo(s, &sinfo);
1125
1126 if (sinfo.num_objs > 0)
1127 pr_info("%-17s %10luKB %10luKB\n", s->name,
1128 (sinfo.active_objs * s->size) / 1024,
1129 (sinfo.num_objs * s->size) / 1024);
1130 }
1131 mutex_unlock(&slab_mutex);
1132}
1133
1134#if defined(CONFIG_MEMCG_KMEM)
1135int memcg_slab_show(struct seq_file *m, void *p)
1136{
1137
1138
1139
1140
1141 return 0;
1142}
1143#endif
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158static const struct seq_operations slabinfo_op = {
1159 .start = slab_start,
1160 .next = slab_next,
1161 .stop = slab_stop,
1162 .show = slab_show,
1163};
1164
1165static int slabinfo_open(struct inode *inode, struct file *file)
1166{
1167 return seq_open(file, &slabinfo_op);
1168}
1169
1170static const struct proc_ops slabinfo_proc_ops = {
1171 .proc_flags = PROC_ENTRY_PERMANENT,
1172 .proc_open = slabinfo_open,
1173 .proc_read = seq_read,
1174 .proc_write = slabinfo_write,
1175 .proc_lseek = seq_lseek,
1176 .proc_release = seq_release,
1177};
1178
1179static int __init slab_proc_init(void)
1180{
1181 proc_create("slabinfo", SLABINFO_RIGHTS, NULL, &slabinfo_proc_ops);
1182 return 0;
1183}
1184module_init(slab_proc_init);
1185
1186#endif
1187
1188static __always_inline void *__do_krealloc(const void *p, size_t new_size,
1189 gfp_t flags)
1190{
1191 void *ret;
1192 size_t ks;
1193
1194
1195 if (likely(!ZERO_OR_NULL_PTR(p))) {
1196 if (!kasan_check_byte(p))
1197 return NULL;
1198 ks = kfence_ksize(p) ?: __ksize(p);
1199 } else
1200 ks = 0;
1201
1202
1203 if (ks >= new_size) {
1204 p = kasan_krealloc((void *)p, new_size, flags);
1205 return (void *)p;
1206 }
1207
1208 ret = kmalloc_track_caller(new_size, flags);
1209 if (ret && p) {
1210
1211 kasan_disable_current();
1212 memcpy(ret, kasan_reset_tag(p), ks);
1213 kasan_enable_current();
1214 }
1215
1216 return ret;
1217}
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232void *krealloc(const void *p, size_t new_size, gfp_t flags)
1233{
1234 void *ret;
1235
1236 if (unlikely(!new_size)) {
1237 kfree(p);
1238 return ZERO_SIZE_PTR;
1239 }
1240
1241 ret = __do_krealloc(p, new_size, flags);
1242 if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret))
1243 kfree(p);
1244
1245 return ret;
1246}
1247EXPORT_SYMBOL(krealloc);
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260void kfree_sensitive(const void *p)
1261{
1262 size_t ks;
1263 void *mem = (void *)p;
1264
1265 ks = ksize(mem);
1266 if (ks)
1267 memzero_explicit(mem, ks);
1268 kfree(mem);
1269}
1270EXPORT_SYMBOL(kfree_sensitive);
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286size_t ksize(const void *objp)
1287{
1288 size_t size;
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305 if (unlikely(ZERO_OR_NULL_PTR(objp)) || !kasan_check_byte(objp))
1306 return 0;
1307
1308 size = kfence_ksize(objp) ?: __ksize(objp);
1309
1310
1311
1312
1313 kasan_unpoison_range(objp, size);
1314 return size;
1315}
1316EXPORT_SYMBOL(ksize);
1317
1318
1319EXPORT_TRACEPOINT_SYMBOL(kmalloc);
1320EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
1321EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
1322EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
1323EXPORT_TRACEPOINT_SYMBOL(kfree);
1324EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);
1325
1326int should_failslab(struct kmem_cache *s, gfp_t gfpflags)
1327{
1328 if (__should_failslab(s, gfpflags))
1329 return -ENOMEM;
1330 return 0;
1331}
1332ALLOW_ERROR_INJECTION(should_failslab, ERRNO);
1333