1
2
3
4
5
6
7#include <linux/slab.h>
8
9#include <linux/mm.h>
10#include <linux/poison.h>
11#include <linux/interrupt.h>
12#include <linux/memory.h>
13#include <linux/cache.h>
14#include <linux/compiler.h>
15#include <linux/kfence.h>
16#include <linux/module.h>
17#include <linux/cpu.h>
18#include <linux/uaccess.h>
19#include <linux/seq_file.h>
20#include <linux/proc_fs.h>
21#include <linux/debugfs.h>
22#include <linux/kasan.h>
23#include <asm/cacheflush.h>
24#include <asm/tlbflush.h>
25#include <asm/page.h>
26#include <linux/memcontrol.h>
27
28#define CREATE_TRACE_POINTS
29#include <trace/events/kmem.h>
30
31#include "internal.h"
32
33#include "slab.h"
34
35enum slab_state slab_state;
36LIST_HEAD(slab_caches);
37DEFINE_MUTEX(slab_mutex);
38struct kmem_cache *kmem_cache;
39
40#ifdef CONFIG_HARDENED_USERCOPY
41bool usercopy_fallback __ro_after_init =
42 IS_ENABLED(CONFIG_HARDENED_USERCOPY_FALLBACK);
43module_param(usercopy_fallback, bool, 0400);
44MODULE_PARM_DESC(usercopy_fallback,
45 "WARN instead of reject usercopy whitelist violations");
46#endif
47
48static LIST_HEAD(slab_caches_to_rcu_destroy);
49static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work);
50static DECLARE_WORK(slab_caches_to_rcu_destroy_work,
51 slab_caches_to_rcu_destroy_workfn);
52
53
54
55
56#define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
57 SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \
58 SLAB_FAILSLAB | kasan_never_merge())
59
60#define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \
61 SLAB_CACHE_DMA32 | SLAB_ACCOUNT)
62
63
64
65
66static bool slab_nomerge = !IS_ENABLED(CONFIG_SLAB_MERGE_DEFAULT);
67
68static int __init setup_slab_nomerge(char *str)
69{
70 slab_nomerge = true;
71 return 1;
72}
73
74static int __init setup_slab_merge(char *str)
75{
76 slab_nomerge = false;
77 return 1;
78}
79
80#ifdef CONFIG_SLUB
81__setup_param("slub_nomerge", slub_nomerge, setup_slab_nomerge, 0);
82__setup_param("slub_merge", slub_merge, setup_slab_merge, 0);
83#endif
84
85__setup("slab_nomerge", setup_slab_nomerge);
86__setup("slab_merge", setup_slab_merge);
87
88
89
90
91unsigned int kmem_cache_size(struct kmem_cache *s)
92{
93 return s->object_size;
94}
95EXPORT_SYMBOL(kmem_cache_size);
96
97#ifdef CONFIG_DEBUG_VM
98static int kmem_cache_sanity_check(const char *name, unsigned int size)
99{
100 if (!name || in_interrupt() || size > KMALLOC_MAX_SIZE) {
101 pr_err("kmem_cache_create(%s) integrity check failed\n", name);
102 return -EINVAL;
103 }
104
105 WARN_ON(strchr(name, ' '));
106 return 0;
107}
108#else
109static inline int kmem_cache_sanity_check(const char *name, unsigned int size)
110{
111 return 0;
112}
113#endif
114
115void __kmem_cache_free_bulk(struct kmem_cache *s, size_t nr, void **p)
116{
117 size_t i;
118
119 for (i = 0; i < nr; i++) {
120 if (s)
121 kmem_cache_free(s, p[i]);
122 else
123 kfree(p[i]);
124 }
125}
126
127int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t nr,
128 void **p)
129{
130 size_t i;
131
132 for (i = 0; i < nr; i++) {
133 void *x = p[i] = kmem_cache_alloc(s, flags);
134 if (!x) {
135 __kmem_cache_free_bulk(s, i, p);
136 return 0;
137 }
138 }
139 return i;
140}
141
142
143
144
145
146static unsigned int calculate_alignment(slab_flags_t flags,
147 unsigned int align, unsigned int size)
148{
149
150
151
152
153
154
155
156 if (flags & SLAB_HWCACHE_ALIGN) {
157 unsigned int ralign;
158
159 ralign = cache_line_size();
160 while (size <= ralign / 2)
161 ralign /= 2;
162 align = max(align, ralign);
163 }
164
165 if (align < ARCH_SLAB_MINALIGN)
166 align = ARCH_SLAB_MINALIGN;
167
168 return ALIGN(align, sizeof(void *));
169}
170
171
172
173
174int slab_unmergeable(struct kmem_cache *s)
175{
176 if (slab_nomerge || (s->flags & SLAB_NEVER_MERGE))
177 return 1;
178
179 if (s->ctor)
180 return 1;
181
182 if (s->usersize)
183 return 1;
184
185
186
187
188 if (s->refcount < 0)
189 return 1;
190
191 return 0;
192}
193
194struct kmem_cache *find_mergeable(unsigned int size, unsigned int align,
195 slab_flags_t flags, const char *name, void (*ctor)(void *))
196{
197 struct kmem_cache *s;
198
199 if (slab_nomerge)
200 return NULL;
201
202 if (ctor)
203 return NULL;
204
205 size = ALIGN(size, sizeof(void *));
206 align = calculate_alignment(flags, align, size);
207 size = ALIGN(size, align);
208 flags = kmem_cache_flags(size, flags, name);
209
210 if (flags & SLAB_NEVER_MERGE)
211 return NULL;
212
213 list_for_each_entry_reverse(s, &slab_caches, list) {
214 if (slab_unmergeable(s))
215 continue;
216
217 if (size > s->size)
218 continue;
219
220 if ((flags & SLAB_MERGE_SAME) != (s->flags & SLAB_MERGE_SAME))
221 continue;
222
223
224
225
226 if ((s->size & ~(align - 1)) != s->size)
227 continue;
228
229 if (s->size - size >= sizeof(void *))
230 continue;
231
232 if (IS_ENABLED(CONFIG_SLAB) && align &&
233 (align > s->align || s->align % align))
234 continue;
235
236 return s;
237 }
238 return NULL;
239}
240
241static struct kmem_cache *create_cache(const char *name,
242 unsigned int object_size, unsigned int align,
243 slab_flags_t flags, unsigned int useroffset,
244 unsigned int usersize, void (*ctor)(void *),
245 struct kmem_cache *root_cache)
246{
247 struct kmem_cache *s;
248 int err;
249
250 if (WARN_ON(useroffset + usersize > object_size))
251 useroffset = usersize = 0;
252
253 err = -ENOMEM;
254 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
255 if (!s)
256 goto out;
257
258 s->name = name;
259 s->size = s->object_size = object_size;
260 s->align = align;
261 s->ctor = ctor;
262 s->useroffset = useroffset;
263 s->usersize = usersize;
264
265 err = __kmem_cache_create(s, flags);
266 if (err)
267 goto out_free_cache;
268
269 s->refcount = 1;
270 list_add(&s->list, &slab_caches);
271out:
272 if (err)
273 return ERR_PTR(err);
274 return s;
275
276out_free_cache:
277 kmem_cache_free(kmem_cache, s);
278 goto out;
279}
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309struct kmem_cache *
310kmem_cache_create_usercopy(const char *name,
311 unsigned int size, unsigned int align,
312 slab_flags_t flags,
313 unsigned int useroffset, unsigned int usersize,
314 void (*ctor)(void *))
315{
316 struct kmem_cache *s = NULL;
317 const char *cache_name;
318 int err;
319
320#ifdef CONFIG_SLUB_DEBUG
321
322
323
324
325
326 if (flags & SLAB_DEBUG_FLAGS)
327 static_branch_enable(&slub_debug_enabled);
328#endif
329
330 mutex_lock(&slab_mutex);
331
332 err = kmem_cache_sanity_check(name, size);
333 if (err) {
334 goto out_unlock;
335 }
336
337
338 if (flags & ~SLAB_FLAGS_PERMITTED) {
339 err = -EINVAL;
340 goto out_unlock;
341 }
342
343
344
345
346
347
348
349 flags &= CACHE_CREATE_MASK;
350
351
352 if (WARN_ON(!usersize && useroffset) ||
353 WARN_ON(size < usersize || size - usersize < useroffset))
354 usersize = useroffset = 0;
355
356 if (!usersize)
357 s = __kmem_cache_alias(name, size, align, flags, ctor);
358 if (s)
359 goto out_unlock;
360
361 cache_name = kstrdup_const(name, GFP_KERNEL);
362 if (!cache_name) {
363 err = -ENOMEM;
364 goto out_unlock;
365 }
366
367 s = create_cache(cache_name, size,
368 calculate_alignment(flags, align, size),
369 flags, useroffset, usersize, ctor, NULL);
370 if (IS_ERR(s)) {
371 err = PTR_ERR(s);
372 kfree_const(cache_name);
373 }
374
375out_unlock:
376 mutex_unlock(&slab_mutex);
377
378 if (err) {
379 if (flags & SLAB_PANIC)
380 panic("%s: Failed to create slab '%s'. Error %d\n",
381 __func__, name, err);
382 else {
383 pr_warn("%s(%s) failed with error %d\n",
384 __func__, name, err);
385 dump_stack();
386 }
387 return NULL;
388 }
389 return s;
390}
391EXPORT_SYMBOL(kmem_cache_create_usercopy);
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418struct kmem_cache *
419kmem_cache_create(const char *name, unsigned int size, unsigned int align,
420 slab_flags_t flags, void (*ctor)(void *))
421{
422 return kmem_cache_create_usercopy(name, size, align, flags, 0, 0,
423 ctor);
424}
425EXPORT_SYMBOL(kmem_cache_create);
426
427static void slab_caches_to_rcu_destroy_workfn(struct work_struct *work)
428{
429 LIST_HEAD(to_destroy);
430 struct kmem_cache *s, *s2;
431
432
433
434
435
436
437
438
439
440
441 mutex_lock(&slab_mutex);
442 list_splice_init(&slab_caches_to_rcu_destroy, &to_destroy);
443 mutex_unlock(&slab_mutex);
444
445 if (list_empty(&to_destroy))
446 return;
447
448 rcu_barrier();
449
450 list_for_each_entry_safe(s, s2, &to_destroy, list) {
451 debugfs_slab_release(s);
452 kfence_shutdown_cache(s);
453#ifdef SLAB_SUPPORTS_SYSFS
454 sysfs_slab_release(s);
455#else
456 slab_kmem_cache_release(s);
457#endif
458 }
459}
460
461static int shutdown_cache(struct kmem_cache *s)
462{
463
464 kasan_cache_shutdown(s);
465
466 if (__kmem_cache_shutdown(s) != 0)
467 return -EBUSY;
468
469 list_del(&s->list);
470
471 if (s->flags & SLAB_TYPESAFE_BY_RCU) {
472#ifdef SLAB_SUPPORTS_SYSFS
473 sysfs_slab_unlink(s);
474#endif
475 list_add_tail(&s->list, &slab_caches_to_rcu_destroy);
476 schedule_work(&slab_caches_to_rcu_destroy_work);
477 } else {
478 kfence_shutdown_cache(s);
479 debugfs_slab_release(s);
480#ifdef SLAB_SUPPORTS_SYSFS
481 sysfs_slab_unlink(s);
482 sysfs_slab_release(s);
483#else
484 slab_kmem_cache_release(s);
485#endif
486 }
487
488 return 0;
489}
490
491void slab_kmem_cache_release(struct kmem_cache *s)
492{
493 __kmem_cache_release(s);
494 kfree_const(s->name);
495 kmem_cache_free(kmem_cache, s);
496}
497
498void kmem_cache_destroy(struct kmem_cache *s)
499{
500 int err;
501
502 if (unlikely(!s))
503 return;
504
505 mutex_lock(&slab_mutex);
506
507 s->refcount--;
508 if (s->refcount)
509 goto out_unlock;
510
511 err = shutdown_cache(s);
512 if (err) {
513 pr_err("%s %s: Slab cache still has objects\n",
514 __func__, s->name);
515 dump_stack();
516 }
517out_unlock:
518 mutex_unlock(&slab_mutex);
519}
520EXPORT_SYMBOL(kmem_cache_destroy);
521
522
523
524
525
526
527
528
529
530
531int kmem_cache_shrink(struct kmem_cache *cachep)
532{
533 int ret;
534
535
536 kasan_cache_shrink(cachep);
537 ret = __kmem_cache_shrink(cachep);
538
539 return ret;
540}
541EXPORT_SYMBOL(kmem_cache_shrink);
542
543bool slab_is_available(void)
544{
545 return slab_state >= UP;
546}
547
548#ifdef CONFIG_PRINTK
549
550
551
552
553
554
555
556
557bool kmem_valid_obj(void *object)
558{
559 struct page *page;
560
561
562 if (object < (void *)PAGE_SIZE || !virt_addr_valid(object))
563 return false;
564 page = virt_to_head_page(object);
565 return PageSlab(page);
566}
567EXPORT_SYMBOL_GPL(kmem_valid_obj);
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584void kmem_dump_obj(void *object)
585{
586 char *cp = IS_ENABLED(CONFIG_MMU) ? "" : "/vmalloc";
587 int i;
588 struct page *page;
589 unsigned long ptroffset;
590 struct kmem_obj_info kp = { };
591
592 if (WARN_ON_ONCE(!virt_addr_valid(object)))
593 return;
594 page = virt_to_head_page(object);
595 if (WARN_ON_ONCE(!PageSlab(page))) {
596 pr_cont(" non-slab memory.\n");
597 return;
598 }
599 kmem_obj_info(&kp, object, page);
600 if (kp.kp_slab_cache)
601 pr_cont(" slab%s %s", cp, kp.kp_slab_cache->name);
602 else
603 pr_cont(" slab%s", cp);
604 if (kp.kp_objp)
605 pr_cont(" start %px", kp.kp_objp);
606 if (kp.kp_data_offset)
607 pr_cont(" data offset %lu", kp.kp_data_offset);
608 if (kp.kp_objp) {
609 ptroffset = ((char *)object - (char *)kp.kp_objp) - kp.kp_data_offset;
610 pr_cont(" pointer offset %lu", ptroffset);
611 }
612 if (kp.kp_slab_cache && kp.kp_slab_cache->usersize)
613 pr_cont(" size %u", kp.kp_slab_cache->usersize);
614 if (kp.kp_ret)
615 pr_cont(" allocated at %pS\n", kp.kp_ret);
616 else
617 pr_cont("\n");
618 for (i = 0; i < ARRAY_SIZE(kp.kp_stack); i++) {
619 if (!kp.kp_stack[i])
620 break;
621 pr_info(" %pS\n", kp.kp_stack[i]);
622 }
623
624 if (kp.kp_free_stack[0])
625 pr_cont(" Free path:\n");
626
627 for (i = 0; i < ARRAY_SIZE(kp.kp_free_stack); i++) {
628 if (!kp.kp_free_stack[i])
629 break;
630 pr_info(" %pS\n", kp.kp_free_stack[i]);
631 }
632
633}
634EXPORT_SYMBOL_GPL(kmem_dump_obj);
635#endif
636
637#ifndef CONFIG_SLOB
638
639void __init create_boot_cache(struct kmem_cache *s, const char *name,
640 unsigned int size, slab_flags_t flags,
641 unsigned int useroffset, unsigned int usersize)
642{
643 int err;
644 unsigned int align = ARCH_KMALLOC_MINALIGN;
645
646 s->name = name;
647 s->size = s->object_size = size;
648
649
650
651
652
653 if (is_power_of_2(size))
654 align = max(align, size);
655 s->align = calculate_alignment(flags, align, size);
656
657 s->useroffset = useroffset;
658 s->usersize = usersize;
659
660 err = __kmem_cache_create(s, flags);
661
662 if (err)
663 panic("Creation of kmalloc slab %s size=%u failed. Reason %d\n",
664 name, size, err);
665
666 s->refcount = -1;
667}
668
669struct kmem_cache *__init create_kmalloc_cache(const char *name,
670 unsigned int size, slab_flags_t flags,
671 unsigned int useroffset, unsigned int usersize)
672{
673 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
674
675 if (!s)
676 panic("Out of memory when creating slab %s\n", name);
677
678 create_boot_cache(s, name, size, flags, useroffset, usersize);
679 kasan_cache_create_kmalloc(s);
680 list_add(&s->list, &slab_caches);
681 s->refcount = 1;
682 return s;
683}
684
685struct kmem_cache *
686kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1] __ro_after_init =
687{ };
688EXPORT_SYMBOL(kmalloc_caches);
689
690
691
692
693
694
695
696static u8 size_index[24] __ro_after_init = {
697 3,
698 4,
699 5,
700 5,
701 6,
702 6,
703 6,
704 6,
705 1,
706 1,
707 1,
708 1,
709 7,
710 7,
711 7,
712 7,
713 2,
714 2,
715 2,
716 2,
717 2,
718 2,
719 2,
720 2
721};
722
723static inline unsigned int size_index_elem(unsigned int bytes)
724{
725 return (bytes - 1) / 8;
726}
727
728
729
730
731
732struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
733{
734 unsigned int index;
735
736 if (size <= 192) {
737 if (!size)
738 return ZERO_SIZE_PTR;
739
740 index = size_index[size_index_elem(size)];
741 } else {
742 if (WARN_ON_ONCE(size > KMALLOC_MAX_CACHE_SIZE))
743 return NULL;
744 index = fls(size - 1);
745 }
746
747 return kmalloc_caches[kmalloc_type(flags)][index];
748}
749
750#ifdef CONFIG_ZONE_DMA
751#define KMALLOC_DMA_NAME(sz) .name[KMALLOC_DMA] = "dma-kmalloc-" #sz,
752#else
753#define KMALLOC_DMA_NAME(sz)
754#endif
755
756#ifdef CONFIG_MEMCG_KMEM
757#define KMALLOC_CGROUP_NAME(sz) .name[KMALLOC_CGROUP] = "kmalloc-cg-" #sz,
758#else
759#define KMALLOC_CGROUP_NAME(sz)
760#endif
761
762#define INIT_KMALLOC_INFO(__size, __short_size) \
763{ \
764 .name[KMALLOC_NORMAL] = "kmalloc-" #__short_size, \
765 .name[KMALLOC_RECLAIM] = "kmalloc-rcl-" #__short_size, \
766 KMALLOC_CGROUP_NAME(__short_size) \
767 KMALLOC_DMA_NAME(__short_size) \
768 .size = __size, \
769}
770
771
772
773
774
775
776const struct kmalloc_info_struct kmalloc_info[] __initconst = {
777 INIT_KMALLOC_INFO(0, 0),
778 INIT_KMALLOC_INFO(96, 96),
779 INIT_KMALLOC_INFO(192, 192),
780 INIT_KMALLOC_INFO(8, 8),
781 INIT_KMALLOC_INFO(16, 16),
782 INIT_KMALLOC_INFO(32, 32),
783 INIT_KMALLOC_INFO(64, 64),
784 INIT_KMALLOC_INFO(128, 128),
785 INIT_KMALLOC_INFO(256, 256),
786 INIT_KMALLOC_INFO(512, 512),
787 INIT_KMALLOC_INFO(1024, 1k),
788 INIT_KMALLOC_INFO(2048, 2k),
789 INIT_KMALLOC_INFO(4096, 4k),
790 INIT_KMALLOC_INFO(8192, 8k),
791 INIT_KMALLOC_INFO(16384, 16k),
792 INIT_KMALLOC_INFO(32768, 32k),
793 INIT_KMALLOC_INFO(65536, 64k),
794 INIT_KMALLOC_INFO(131072, 128k),
795 INIT_KMALLOC_INFO(262144, 256k),
796 INIT_KMALLOC_INFO(524288, 512k),
797 INIT_KMALLOC_INFO(1048576, 1M),
798 INIT_KMALLOC_INFO(2097152, 2M),
799 INIT_KMALLOC_INFO(4194304, 4M),
800 INIT_KMALLOC_INFO(8388608, 8M),
801 INIT_KMALLOC_INFO(16777216, 16M),
802 INIT_KMALLOC_INFO(33554432, 32M)
803};
804
805
806
807
808
809
810
811
812
813
814
815
816void __init setup_kmalloc_cache_index_table(void)
817{
818 unsigned int i;
819
820 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
821 (KMALLOC_MIN_SIZE & (KMALLOC_MIN_SIZE - 1)));
822
823 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
824 unsigned int elem = size_index_elem(i);
825
826 if (elem >= ARRAY_SIZE(size_index))
827 break;
828 size_index[elem] = KMALLOC_SHIFT_LOW;
829 }
830
831 if (KMALLOC_MIN_SIZE >= 64) {
832
833
834
835
836 for (i = 64 + 8; i <= 96; i += 8)
837 size_index[size_index_elem(i)] = 7;
838
839 }
840
841 if (KMALLOC_MIN_SIZE >= 128) {
842
843
844
845
846
847 for (i = 128 + 8; i <= 192; i += 8)
848 size_index[size_index_elem(i)] = 8;
849 }
850}
851
852static void __init
853new_kmalloc_cache(int idx, enum kmalloc_cache_type type, slab_flags_t flags)
854{
855 if (type == KMALLOC_RECLAIM) {
856 flags |= SLAB_RECLAIM_ACCOUNT;
857 } else if (IS_ENABLED(CONFIG_MEMCG_KMEM) && (type == KMALLOC_CGROUP)) {
858 if (cgroup_memory_nokmem) {
859 kmalloc_caches[type][idx] = kmalloc_caches[KMALLOC_NORMAL][idx];
860 return;
861 }
862 flags |= SLAB_ACCOUNT;
863 }
864
865 kmalloc_caches[type][idx] = create_kmalloc_cache(
866 kmalloc_info[idx].name[type],
867 kmalloc_info[idx].size, flags, 0,
868 kmalloc_info[idx].size);
869
870
871
872
873
874 if (IS_ENABLED(CONFIG_MEMCG_KMEM) && (type == KMALLOC_NORMAL))
875 kmalloc_caches[type][idx]->refcount = -1;
876}
877
878
879
880
881
882
883void __init create_kmalloc_caches(slab_flags_t flags)
884{
885 int i;
886 enum kmalloc_cache_type type;
887
888
889
890
891 for (type = KMALLOC_NORMAL; type <= KMALLOC_RECLAIM; type++) {
892 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
893 if (!kmalloc_caches[type][i])
894 new_kmalloc_cache(i, type, flags);
895
896
897
898
899
900
901 if (KMALLOC_MIN_SIZE <= 32 && i == 6 &&
902 !kmalloc_caches[type][1])
903 new_kmalloc_cache(1, type, flags);
904 if (KMALLOC_MIN_SIZE <= 64 && i == 7 &&
905 !kmalloc_caches[type][2])
906 new_kmalloc_cache(2, type, flags);
907 }
908 }
909
910
911 slab_state = UP;
912
913#ifdef CONFIG_ZONE_DMA
914 for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
915 struct kmem_cache *s = kmalloc_caches[KMALLOC_NORMAL][i];
916
917 if (s) {
918 kmalloc_caches[KMALLOC_DMA][i] = create_kmalloc_cache(
919 kmalloc_info[i].name[KMALLOC_DMA],
920 kmalloc_info[i].size,
921 SLAB_CACHE_DMA | flags, 0,
922 kmalloc_info[i].size);
923 }
924 }
925#endif
926}
927#endif
928
929gfp_t kmalloc_fix_flags(gfp_t flags)
930{
931 gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK;
932
933 flags &= ~GFP_SLAB_BUG_MASK;
934 pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n",
935 invalid_mask, &invalid_mask, flags, &flags);
936 dump_stack();
937
938 return flags;
939}
940
941
942
943
944
945
946void *kmalloc_order(size_t size, gfp_t flags, unsigned int order)
947{
948 void *ret = NULL;
949 struct page *page;
950
951 if (unlikely(flags & GFP_SLAB_BUG_MASK))
952 flags = kmalloc_fix_flags(flags);
953
954 flags |= __GFP_COMP;
955 page = alloc_pages(flags, order);
956 if (likely(page)) {
957 ret = page_address(page);
958 mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
959 PAGE_SIZE << order);
960 }
961 ret = kasan_kmalloc_large(ret, size, flags);
962
963 kmemleak_alloc(ret, size, 1, flags);
964 return ret;
965}
966EXPORT_SYMBOL(kmalloc_order);
967
968#ifdef CONFIG_TRACING
969void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order)
970{
971 void *ret = kmalloc_order(size, flags, order);
972 trace_kmalloc(_RET_IP_, ret, size, PAGE_SIZE << order, flags);
973 return ret;
974}
975EXPORT_SYMBOL(kmalloc_order_trace);
976#endif
977
978#ifdef CONFIG_SLAB_FREELIST_RANDOM
979
980static void freelist_randomize(struct rnd_state *state, unsigned int *list,
981 unsigned int count)
982{
983 unsigned int rand;
984 unsigned int i;
985
986 for (i = 0; i < count; i++)
987 list[i] = i;
988
989
990 for (i = count - 1; i > 0; i--) {
991 rand = prandom_u32_state(state);
992 rand %= (i + 1);
993 swap(list[i], list[rand]);
994 }
995}
996
997
998int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
999 gfp_t gfp)
1000{
1001 struct rnd_state state;
1002
1003 if (count < 2 || cachep->random_seq)
1004 return 0;
1005
1006 cachep->random_seq = kcalloc(count, sizeof(unsigned int), gfp);
1007 if (!cachep->random_seq)
1008 return -ENOMEM;
1009
1010
1011 prandom_seed_state(&state, get_random_long());
1012
1013 freelist_randomize(&state, cachep->random_seq, count);
1014 return 0;
1015}
1016
1017
1018void cache_random_seq_destroy(struct kmem_cache *cachep)
1019{
1020 kfree(cachep->random_seq);
1021 cachep->random_seq = NULL;
1022}
1023#endif
1024
1025#if defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG)
1026#ifdef CONFIG_SLAB
1027#define SLABINFO_RIGHTS (0600)
1028#else
1029#define SLABINFO_RIGHTS (0400)
1030#endif
1031
1032static void print_slabinfo_header(struct seq_file *m)
1033{
1034
1035
1036
1037
1038#ifdef CONFIG_DEBUG_SLAB
1039 seq_puts(m, "slabinfo - version: 2.1 (statistics)\n");
1040#else
1041 seq_puts(m, "slabinfo - version: 2.1\n");
1042#endif
1043 seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>");
1044 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
1045 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
1046#ifdef CONFIG_DEBUG_SLAB
1047 seq_puts(m, " : globalstat <listallocs> <maxobjs> <grown> <reaped> <error> <maxfreeable> <nodeallocs> <remotefrees> <alienoverflow>");
1048 seq_puts(m, " : cpustat <allochit> <allocmiss> <freehit> <freemiss>");
1049#endif
1050 seq_putc(m, '\n');
1051}
1052
1053void *slab_start(struct seq_file *m, loff_t *pos)
1054{
1055 mutex_lock(&slab_mutex);
1056 return seq_list_start(&slab_caches, *pos);
1057}
1058
1059void *slab_next(struct seq_file *m, void *p, loff_t *pos)
1060{
1061 return seq_list_next(p, &slab_caches, pos);
1062}
1063
1064void slab_stop(struct seq_file *m, void *p)
1065{
1066 mutex_unlock(&slab_mutex);
1067}
1068
1069static void cache_show(struct kmem_cache *s, struct seq_file *m)
1070{
1071 struct slabinfo sinfo;
1072
1073 memset(&sinfo, 0, sizeof(sinfo));
1074 get_slabinfo(s, &sinfo);
1075
1076 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
1077 s->name, sinfo.active_objs, sinfo.num_objs, s->size,
1078 sinfo.objects_per_slab, (1 << sinfo.cache_order));
1079
1080 seq_printf(m, " : tunables %4u %4u %4u",
1081 sinfo.limit, sinfo.batchcount, sinfo.shared);
1082 seq_printf(m, " : slabdata %6lu %6lu %6lu",
1083 sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail);
1084 slabinfo_show_stats(m, s);
1085 seq_putc(m, '\n');
1086}
1087
1088static int slab_show(struct seq_file *m, void *p)
1089{
1090 struct kmem_cache *s = list_entry(p, struct kmem_cache, list);
1091
1092 if (p == slab_caches.next)
1093 print_slabinfo_header(m);
1094 cache_show(s, m);
1095 return 0;
1096}
1097
1098void dump_unreclaimable_slab(void)
1099{
1100 struct kmem_cache *s;
1101 struct slabinfo sinfo;
1102
1103
1104
1105
1106
1107
1108
1109
1110 if (!mutex_trylock(&slab_mutex)) {
1111 pr_warn("excessive unreclaimable slab but cannot dump stats\n");
1112 return;
1113 }
1114
1115 pr_info("Unreclaimable slab info:\n");
1116 pr_info("Name Used Total\n");
1117
1118 list_for_each_entry(s, &slab_caches, list) {
1119 if (s->flags & SLAB_RECLAIM_ACCOUNT)
1120 continue;
1121
1122 get_slabinfo(s, &sinfo);
1123
1124 if (sinfo.num_objs > 0)
1125 pr_info("%-17s %10luKB %10luKB\n", s->name,
1126 (sinfo.active_objs * s->size) / 1024,
1127 (sinfo.num_objs * s->size) / 1024);
1128 }
1129 mutex_unlock(&slab_mutex);
1130}
1131
1132#if defined(CONFIG_MEMCG_KMEM)
1133int memcg_slab_show(struct seq_file *m, void *p)
1134{
1135
1136
1137
1138
1139 return 0;
1140}
1141#endif
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156static const struct seq_operations slabinfo_op = {
1157 .start = slab_start,
1158 .next = slab_next,
1159 .stop = slab_stop,
1160 .show = slab_show,
1161};
1162
1163static int slabinfo_open(struct inode *inode, struct file *file)
1164{
1165 return seq_open(file, &slabinfo_op);
1166}
1167
1168static const struct proc_ops slabinfo_proc_ops = {
1169 .proc_flags = PROC_ENTRY_PERMANENT,
1170 .proc_open = slabinfo_open,
1171 .proc_read = seq_read,
1172 .proc_write = slabinfo_write,
1173 .proc_lseek = seq_lseek,
1174 .proc_release = seq_release,
1175};
1176
1177static int __init slab_proc_init(void)
1178{
1179 proc_create("slabinfo", SLABINFO_RIGHTS, NULL, &slabinfo_proc_ops);
1180 return 0;
1181}
1182module_init(slab_proc_init);
1183
1184#endif
1185
1186static __always_inline void *__do_krealloc(const void *p, size_t new_size,
1187 gfp_t flags)
1188{
1189 void *ret;
1190 size_t ks;
1191
1192
1193 if (likely(!ZERO_OR_NULL_PTR(p))) {
1194 if (!kasan_check_byte(p))
1195 return NULL;
1196 ks = kfence_ksize(p) ?: __ksize(p);
1197 } else
1198 ks = 0;
1199
1200
1201 if (ks >= new_size) {
1202 p = kasan_krealloc((void *)p, new_size, flags);
1203 return (void *)p;
1204 }
1205
1206 ret = kmalloc_track_caller(new_size, flags);
1207 if (ret && p) {
1208
1209 kasan_disable_current();
1210 memcpy(ret, kasan_reset_tag(p), ks);
1211 kasan_enable_current();
1212 }
1213
1214 return ret;
1215}
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230void *krealloc(const void *p, size_t new_size, gfp_t flags)
1231{
1232 void *ret;
1233
1234 if (unlikely(!new_size)) {
1235 kfree(p);
1236 return ZERO_SIZE_PTR;
1237 }
1238
1239 ret = __do_krealloc(p, new_size, flags);
1240 if (ret && kasan_reset_tag(p) != kasan_reset_tag(ret))
1241 kfree(p);
1242
1243 return ret;
1244}
1245EXPORT_SYMBOL(krealloc);
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258void kfree_sensitive(const void *p)
1259{
1260 size_t ks;
1261 void *mem = (void *)p;
1262
1263 ks = ksize(mem);
1264 if (ks)
1265 memzero_explicit(mem, ks);
1266 kfree(mem);
1267}
1268EXPORT_SYMBOL(kfree_sensitive);
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284size_t ksize(const void *objp)
1285{
1286 size_t size;
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303 if (unlikely(ZERO_OR_NULL_PTR(objp)) || !kasan_check_byte(objp))
1304 return 0;
1305
1306 size = kfence_ksize(objp) ?: __ksize(objp);
1307
1308
1309
1310
1311 kasan_unpoison_range(objp, size);
1312 return size;
1313}
1314EXPORT_SYMBOL(ksize);
1315
1316
1317EXPORT_TRACEPOINT_SYMBOL(kmalloc);
1318EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
1319EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
1320EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
1321EXPORT_TRACEPOINT_SYMBOL(kfree);
1322EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);
1323
1324int should_failslab(struct kmem_cache *s, gfp_t gfpflags)
1325{
1326 if (__should_failslab(s, gfpflags))
1327 return -ENOMEM;
1328 return 0;
1329}
1330ALLOW_ERROR_INJECTION(should_failslab, ERRNO);
1331