1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
59
60#include <linux/init.h>
61#include <linux/kernel.h>
62#include <linux/list.h>
63#include <linux/sched/signal.h>
64#include <linux/sched/task.h>
65#include <linux/sched/task_stack.h>
66#include <linux/jiffies.h>
67#include <linux/delay.h>
68#include <linux/export.h>
69#include <linux/kthread.h>
70#include <linux/rbtree.h>
71#include <linux/fs.h>
72#include <linux/debugfs.h>
73#include <linux/seq_file.h>
74#include <linux/cpumask.h>
75#include <linux/spinlock.h>
76#include <linux/module.h>
77#include <linux/mutex.h>
78#include <linux/rcupdate.h>
79#include <linux/stacktrace.h>
80#include <linux/cache.h>
81#include <linux/percpu.h>
82#include <linux/memblock.h>
83#include <linux/pfn.h>
84#include <linux/mmzone.h>
85#include <linux/slab.h>
86#include <linux/thread_info.h>
87#include <linux/err.h>
88#include <linux/uaccess.h>
89#include <linux/string.h>
90#include <linux/nodemask.h>
91#include <linux/mm.h>
92#include <linux/workqueue.h>
93#include <linux/crc32.h>
94
95#include <asm/sections.h>
96#include <asm/processor.h>
97#include <linux/atomic.h>
98
99#include <linux/kasan.h>
100#include <linux/kfence.h>
101#include <linux/kmemleak.h>
102#include <linux/memory_hotplug.h>
103
104
105
106
107#define MAX_TRACE 16
108#define MSECS_MIN_AGE 5000
109#define SECS_FIRST_SCAN 60
110#define SECS_SCAN_WAIT 600
111#define MAX_SCAN_SIZE 4096
112
113#define BYTES_PER_POINTER sizeof(void *)
114
115
116#define gfp_kmemleak_mask(gfp) (((gfp) & (GFP_KERNEL | GFP_ATOMIC)) | \
117 __GFP_NORETRY | __GFP_NOMEMALLOC | \
118 __GFP_NOWARN)
119
120
121struct kmemleak_scan_area {
122 struct hlist_node node;
123 unsigned long start;
124 size_t size;
125};
126
127#define KMEMLEAK_GREY 0
128#define KMEMLEAK_BLACK -1
129
130
131
132
133
134
135
136
137
138struct kmemleak_object {
139 raw_spinlock_t lock;
140 unsigned int flags;
141 struct list_head object_list;
142 struct list_head gray_list;
143 struct rb_node rb_node;
144 struct rcu_head rcu;
145
146 atomic_t use_count;
147 unsigned long pointer;
148 size_t size;
149
150 unsigned long excess_ref;
151
152 int min_count;
153
154 int count;
155
156 u32 checksum;
157
158 struct hlist_head area_list;
159 unsigned long trace[MAX_TRACE];
160 unsigned int trace_len;
161 unsigned long jiffies;
162 pid_t pid;
163 char comm[TASK_COMM_LEN];
164};
165
166
167#define OBJECT_ALLOCATED (1 << 0)
168
169#define OBJECT_REPORTED (1 << 1)
170
171#define OBJECT_NO_SCAN (1 << 2)
172
173#define OBJECT_FULL_SCAN (1 << 3)
174
175#define HEX_PREFIX " "
176
177#define HEX_ROW_SIZE 16
178
179#define HEX_GROUP_SIZE 1
180
181#define HEX_ASCII 1
182
183#define HEX_MAX_LINES 2
184
185
186static LIST_HEAD(object_list);
187
188static LIST_HEAD(gray_list);
189
190static struct kmemleak_object mem_pool[CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE];
191static int mem_pool_free_count = ARRAY_SIZE(mem_pool);
192static LIST_HEAD(mem_pool_free_list);
193
194static struct rb_root object_tree_root = RB_ROOT;
195
196static DEFINE_RAW_SPINLOCK(kmemleak_lock);
197
198
199static struct kmem_cache *object_cache;
200static struct kmem_cache *scan_area_cache;
201
202
203static int kmemleak_enabled = 1;
204
205static int kmemleak_free_enabled = 1;
206
207static int kmemleak_initialized;
208
209static int kmemleak_warning;
210
211static int kmemleak_error;
212
213
214static unsigned long min_addr = ULONG_MAX;
215static unsigned long max_addr;
216
217static struct task_struct *scan_thread;
218
219static unsigned long jiffies_min_age;
220static unsigned long jiffies_last_scan;
221
222static unsigned long jiffies_scan_wait;
223
224static int kmemleak_stack_scan = 1;
225
226static DEFINE_MUTEX(scan_mutex);
227
228static int kmemleak_skip_disable;
229
230static bool kmemleak_found_leaks;
231
232static bool kmemleak_verbose;
233module_param_named(verbose, kmemleak_verbose, bool, 0600);
234
235static void kmemleak_disable(void);
236
237
238
239
240#define kmemleak_warn(x...) do { \
241 pr_warn(x); \
242 dump_stack(); \
243 kmemleak_warning = 1; \
244} while (0)
245
246
247
248
249
250
251#define kmemleak_stop(x...) do { \
252 kmemleak_warn(x); \
253 kmemleak_disable(); \
254} while (0)
255
256#define warn_or_seq_printf(seq, fmt, ...) do { \
257 if (seq) \
258 seq_printf(seq, fmt, ##__VA_ARGS__); \
259 else \
260 pr_warn(fmt, ##__VA_ARGS__); \
261} while (0)
262
263static void warn_or_seq_hex_dump(struct seq_file *seq, int prefix_type,
264 int rowsize, int groupsize, const void *buf,
265 size_t len, bool ascii)
266{
267 if (seq)
268 seq_hex_dump(seq, HEX_PREFIX, prefix_type, rowsize, groupsize,
269 buf, len, ascii);
270 else
271 print_hex_dump(KERN_WARNING, pr_fmt(HEX_PREFIX), prefix_type,
272 rowsize, groupsize, buf, len, ascii);
273}
274
275
276
277
278
279
280
281static void hex_dump_object(struct seq_file *seq,
282 struct kmemleak_object *object)
283{
284 const u8 *ptr = (const u8 *)object->pointer;
285 size_t len;
286
287
288 len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);
289
290 warn_or_seq_printf(seq, " hex dump (first %zu bytes):\n", len);
291 kasan_disable_current();
292 warn_or_seq_hex_dump(seq, DUMP_PREFIX_NONE, HEX_ROW_SIZE,
293 HEX_GROUP_SIZE, kasan_reset_tag((void *)ptr), len, HEX_ASCII);
294 kasan_enable_current();
295}
296
297
298
299
300
301
302
303
304
305
306
307static bool color_white(const struct kmemleak_object *object)
308{
309 return object->count != KMEMLEAK_BLACK &&
310 object->count < object->min_count;
311}
312
313static bool color_gray(const struct kmemleak_object *object)
314{
315 return object->min_count != KMEMLEAK_BLACK &&
316 object->count >= object->min_count;
317}
318
319
320
321
322
323
324static bool unreferenced_object(struct kmemleak_object *object)
325{
326 return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
327 time_before_eq(object->jiffies + jiffies_min_age,
328 jiffies_last_scan);
329}
330
331
332
333
334
335static void print_unreferenced(struct seq_file *seq,
336 struct kmemleak_object *object)
337{
338 int i;
339 unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);
340
341 warn_or_seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
342 object->pointer, object->size);
343 warn_or_seq_printf(seq, " comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
344 object->comm, object->pid, object->jiffies,
345 msecs_age / 1000, msecs_age % 1000);
346 hex_dump_object(seq, object);
347 warn_or_seq_printf(seq, " backtrace:\n");
348
349 for (i = 0; i < object->trace_len; i++) {
350 void *ptr = (void *)object->trace[i];
351 warn_or_seq_printf(seq, " [<%p>] %pS\n", ptr, ptr);
352 }
353}
354
355
356
357
358
359
360static void dump_object_info(struct kmemleak_object *object)
361{
362 pr_notice("Object 0x%08lx (size %zu):\n",
363 object->pointer, object->size);
364 pr_notice(" comm \"%s\", pid %d, jiffies %lu\n",
365 object->comm, object->pid, object->jiffies);
366 pr_notice(" min_count = %d\n", object->min_count);
367 pr_notice(" count = %d\n", object->count);
368 pr_notice(" flags = 0x%x\n", object->flags);
369 pr_notice(" checksum = %u\n", object->checksum);
370 pr_notice(" backtrace:\n");
371 stack_trace_print(object->trace, object->trace_len, 4);
372}
373
374
375
376
377
378
379
380static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
381{
382 struct rb_node *rb = object_tree_root.rb_node;
383
384 while (rb) {
385 struct kmemleak_object *object =
386 rb_entry(rb, struct kmemleak_object, rb_node);
387 if (ptr < object->pointer)
388 rb = object->rb_node.rb_left;
389 else if (object->pointer + object->size <= ptr)
390 rb = object->rb_node.rb_right;
391 else if (object->pointer == ptr || alias)
392 return object;
393 else {
394 kmemleak_warn("Found object by alias at 0x%08lx\n",
395 ptr);
396 dump_object_info(object);
397 break;
398 }
399 }
400 return NULL;
401}
402
403
404
405
406
407
408
409static int get_object(struct kmemleak_object *object)
410{
411 return atomic_inc_not_zero(&object->use_count);
412}
413
414
415
416
417static struct kmemleak_object *mem_pool_alloc(gfp_t gfp)
418{
419 unsigned long flags;
420 struct kmemleak_object *object;
421
422
423 if (object_cache) {
424 object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
425 if (object)
426 return object;
427 }
428
429
430 raw_spin_lock_irqsave(&kmemleak_lock, flags);
431 object = list_first_entry_or_null(&mem_pool_free_list,
432 typeof(*object), object_list);
433 if (object)
434 list_del(&object->object_list);
435 else if (mem_pool_free_count)
436 object = &mem_pool[--mem_pool_free_count];
437 else
438 pr_warn_once("Memory pool empty, consider increasing CONFIG_DEBUG_KMEMLEAK_MEM_POOL_SIZE\n");
439 raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
440
441 return object;
442}
443
444
445
446
447static void mem_pool_free(struct kmemleak_object *object)
448{
449 unsigned long flags;
450
451 if (object < mem_pool || object >= mem_pool + ARRAY_SIZE(mem_pool)) {
452 kmem_cache_free(object_cache, object);
453 return;
454 }
455
456
457 raw_spin_lock_irqsave(&kmemleak_lock, flags);
458 list_add(&object->object_list, &mem_pool_free_list);
459 raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
460}
461
462
463
464
465static void free_object_rcu(struct rcu_head *rcu)
466{
467 struct hlist_node *tmp;
468 struct kmemleak_scan_area *area;
469 struct kmemleak_object *object =
470 container_of(rcu, struct kmemleak_object, rcu);
471
472
473
474
475
476 hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {
477 hlist_del(&area->node);
478 kmem_cache_free(scan_area_cache, area);
479 }
480 mem_pool_free(object);
481}
482
483
484
485
486
487
488
489
490static void put_object(struct kmemleak_object *object)
491{
492 if (!atomic_dec_and_test(&object->use_count))
493 return;
494
495
496 WARN_ON(object->flags & OBJECT_ALLOCATED);
497
498
499
500
501
502
503 if (object_cache)
504 call_rcu(&object->rcu, free_object_rcu);
505 else
506 free_object_rcu(&object->rcu);
507}
508
509
510
511
512static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
513{
514 unsigned long flags;
515 struct kmemleak_object *object;
516
517 rcu_read_lock();
518 raw_spin_lock_irqsave(&kmemleak_lock, flags);
519 object = lookup_object(ptr, alias);
520 raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
521
522
523 if (object && !get_object(object))
524 object = NULL;
525 rcu_read_unlock();
526
527 return object;
528}
529
530
531
532
533
534static void __remove_object(struct kmemleak_object *object)
535{
536 rb_erase(&object->rb_node, &object_tree_root);
537 list_del_rcu(&object->object_list);
538}
539
540
541
542
543
544
545static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int alias)
546{
547 unsigned long flags;
548 struct kmemleak_object *object;
549
550 raw_spin_lock_irqsave(&kmemleak_lock, flags);
551 object = lookup_object(ptr, alias);
552 if (object)
553 __remove_object(object);
554 raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
555
556 return object;
557}
558
559
560
561
562static int __save_stack_trace(unsigned long *trace)
563{
564 return stack_trace_save(trace, MAX_TRACE, 2);
565}
566
567
568
569
570
571static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
572 int min_count, gfp_t gfp)
573{
574 unsigned long flags;
575 struct kmemleak_object *object, *parent;
576 struct rb_node **link, *rb_parent;
577 unsigned long untagged_ptr;
578
579 object = mem_pool_alloc(gfp);
580 if (!object) {
581 pr_warn("Cannot allocate a kmemleak_object structure\n");
582 kmemleak_disable();
583 return NULL;
584 }
585
586 INIT_LIST_HEAD(&object->object_list);
587 INIT_LIST_HEAD(&object->gray_list);
588 INIT_HLIST_HEAD(&object->area_list);
589 raw_spin_lock_init(&object->lock);
590 atomic_set(&object->use_count, 1);
591 object->flags = OBJECT_ALLOCATED;
592 object->pointer = ptr;
593 object->size = kfence_ksize((void *)ptr) ?: size;
594 object->excess_ref = 0;
595 object->min_count = min_count;
596 object->count = 0;
597 object->jiffies = jiffies;
598 object->checksum = 0;
599
600
601 if (in_irq()) {
602 object->pid = 0;
603 strncpy(object->comm, "hardirq", sizeof(object->comm));
604 } else if (in_serving_softirq()) {
605 object->pid = 0;
606 strncpy(object->comm, "softirq", sizeof(object->comm));
607 } else {
608 object->pid = current->pid;
609
610
611
612
613
614
615 strncpy(object->comm, current->comm, sizeof(object->comm));
616 }
617
618
619 object->trace_len = __save_stack_trace(object->trace);
620
621 raw_spin_lock_irqsave(&kmemleak_lock, flags);
622
623 untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
624 min_addr = min(min_addr, untagged_ptr);
625 max_addr = max(max_addr, untagged_ptr + size);
626 link = &object_tree_root.rb_node;
627 rb_parent = NULL;
628 while (*link) {
629 rb_parent = *link;
630 parent = rb_entry(rb_parent, struct kmemleak_object, rb_node);
631 if (ptr + size <= parent->pointer)
632 link = &parent->rb_node.rb_left;
633 else if (parent->pointer + parent->size <= ptr)
634 link = &parent->rb_node.rb_right;
635 else {
636 kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
637 ptr);
638
639
640
641
642 dump_object_info(parent);
643 kmem_cache_free(object_cache, object);
644 object = NULL;
645 goto out;
646 }
647 }
648 rb_link_node(&object->rb_node, rb_parent, link);
649 rb_insert_color(&object->rb_node, &object_tree_root);
650
651 list_add_tail_rcu(&object->object_list, &object_list);
652out:
653 raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
654 return object;
655}
656
657
658
659
660static void __delete_object(struct kmemleak_object *object)
661{
662 unsigned long flags;
663
664 WARN_ON(!(object->flags & OBJECT_ALLOCATED));
665 WARN_ON(atomic_read(&object->use_count) < 1);
666
667
668
669
670
671 raw_spin_lock_irqsave(&object->lock, flags);
672 object->flags &= ~OBJECT_ALLOCATED;
673 raw_spin_unlock_irqrestore(&object->lock, flags);
674 put_object(object);
675}
676
677
678
679
680
681static void delete_object_full(unsigned long ptr)
682{
683 struct kmemleak_object *object;
684
685 object = find_and_remove_object(ptr, 0);
686 if (!object) {
687#ifdef DEBUG
688 kmemleak_warn("Freeing unknown object at 0x%08lx\n",
689 ptr);
690#endif
691 return;
692 }
693 __delete_object(object);
694}
695
696
697
698
699
700
701static void delete_object_part(unsigned long ptr, size_t size)
702{
703 struct kmemleak_object *object;
704 unsigned long start, end;
705
706 object = find_and_remove_object(ptr, 1);
707 if (!object) {
708#ifdef DEBUG
709 kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
710 ptr, size);
711#endif
712 return;
713 }
714
715
716
717
718
719
720 start = object->pointer;
721 end = object->pointer + object->size;
722 if (ptr > start)
723 create_object(start, ptr - start, object->min_count,
724 GFP_KERNEL);
725 if (ptr + size < end)
726 create_object(ptr + size, end - ptr - size, object->min_count,
727 GFP_KERNEL);
728
729 __delete_object(object);
730}
731
732static void __paint_it(struct kmemleak_object *object, int color)
733{
734 object->min_count = color;
735 if (color == KMEMLEAK_BLACK)
736 object->flags |= OBJECT_NO_SCAN;
737}
738
739static void paint_it(struct kmemleak_object *object, int color)
740{
741 unsigned long flags;
742
743 raw_spin_lock_irqsave(&object->lock, flags);
744 __paint_it(object, color);
745 raw_spin_unlock_irqrestore(&object->lock, flags);
746}
747
748static void paint_ptr(unsigned long ptr, int color)
749{
750 struct kmemleak_object *object;
751
752 object = find_and_get_object(ptr, 0);
753 if (!object) {
754 kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
755 ptr,
756 (color == KMEMLEAK_GREY) ? "Grey" :
757 (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
758 return;
759 }
760 paint_it(object, color);
761 put_object(object);
762}
763
764
765
766
767
768static void make_gray_object(unsigned long ptr)
769{
770 paint_ptr(ptr, KMEMLEAK_GREY);
771}
772
773
774
775
776
777static void make_black_object(unsigned long ptr)
778{
779 paint_ptr(ptr, KMEMLEAK_BLACK);
780}
781
782
783
784
785
786static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
787{
788 unsigned long flags;
789 struct kmemleak_object *object;
790 struct kmemleak_scan_area *area = NULL;
791
792 object = find_and_get_object(ptr, 1);
793 if (!object) {
794 kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
795 ptr);
796 return;
797 }
798
799 if (scan_area_cache)
800 area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
801
802 raw_spin_lock_irqsave(&object->lock, flags);
803 if (!area) {
804 pr_warn_once("Cannot allocate a scan area, scanning the full object\n");
805
806 object->flags |= OBJECT_FULL_SCAN;
807 goto out_unlock;
808 }
809 if (size == SIZE_MAX) {
810 size = object->pointer + object->size - ptr;
811 } else if (ptr + size > object->pointer + object->size) {
812 kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
813 dump_object_info(object);
814 kmem_cache_free(scan_area_cache, area);
815 goto out_unlock;
816 }
817
818 INIT_HLIST_NODE(&area->node);
819 area->start = ptr;
820 area->size = size;
821
822 hlist_add_head(&area->node, &object->area_list);
823out_unlock:
824 raw_spin_unlock_irqrestore(&object->lock, flags);
825 put_object(object);
826}
827
828
829
830
831
832
833
834static void object_set_excess_ref(unsigned long ptr, unsigned long excess_ref)
835{
836 unsigned long flags;
837 struct kmemleak_object *object;
838
839 object = find_and_get_object(ptr, 0);
840 if (!object) {
841 kmemleak_warn("Setting excess_ref on unknown object at 0x%08lx\n",
842 ptr);
843 return;
844 }
845
846 raw_spin_lock_irqsave(&object->lock, flags);
847 object->excess_ref = excess_ref;
848 raw_spin_unlock_irqrestore(&object->lock, flags);
849 put_object(object);
850}
851
852
853
854
855
856
857static void object_no_scan(unsigned long ptr)
858{
859 unsigned long flags;
860 struct kmemleak_object *object;
861
862 object = find_and_get_object(ptr, 0);
863 if (!object) {
864 kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
865 return;
866 }
867
868 raw_spin_lock_irqsave(&object->lock, flags);
869 object->flags |= OBJECT_NO_SCAN;
870 raw_spin_unlock_irqrestore(&object->lock, flags);
871 put_object(object);
872}
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
889 gfp_t gfp)
890{
891 pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);
892
893 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
894 create_object((unsigned long)ptr, size, min_count, gfp);
895}
896EXPORT_SYMBOL_GPL(kmemleak_alloc);
897
898
899
900
901
902
903
904
905
906
907void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size,
908 gfp_t gfp)
909{
910 unsigned int cpu;
911
912 pr_debug("%s(0x%p, %zu)\n", __func__, ptr, size);
913
914
915
916
917
918 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
919 for_each_possible_cpu(cpu)
920 create_object((unsigned long)per_cpu_ptr(ptr, cpu),
921 size, 0, gfp);
922}
923EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
924
925
926
927
928
929
930
931
932
933
934void __ref kmemleak_vmalloc(const struct vm_struct *area, size_t size, gfp_t gfp)
935{
936 pr_debug("%s(0x%p, %zu)\n", __func__, area, size);
937
938
939
940
941
942 if (kmemleak_enabled) {
943 create_object((unsigned long)area->addr, size, 2, gfp);
944 object_set_excess_ref((unsigned long)area,
945 (unsigned long)area->addr);
946 }
947}
948EXPORT_SYMBOL_GPL(kmemleak_vmalloc);
949
950
951
952
953
954
955
956
957void __ref kmemleak_free(const void *ptr)
958{
959 pr_debug("%s(0x%p)\n", __func__, ptr);
960
961 if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
962 delete_object_full((unsigned long)ptr);
963}
964EXPORT_SYMBOL_GPL(kmemleak_free);
965
966
967
968
969
970
971
972
973
974
975void __ref kmemleak_free_part(const void *ptr, size_t size)
976{
977 pr_debug("%s(0x%p)\n", __func__, ptr);
978
979 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
980 delete_object_part((unsigned long)ptr, size);
981}
982EXPORT_SYMBOL_GPL(kmemleak_free_part);
983
984
985
986
987
988
989
990
991void __ref kmemleak_free_percpu(const void __percpu *ptr)
992{
993 unsigned int cpu;
994
995 pr_debug("%s(0x%p)\n", __func__, ptr);
996
997 if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
998 for_each_possible_cpu(cpu)
999 delete_object_full((unsigned long)per_cpu_ptr(ptr,
1000 cpu));
1001}
1002EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
1003
1004
1005
1006
1007
1008
1009
1010
1011void __ref kmemleak_update_trace(const void *ptr)
1012{
1013 struct kmemleak_object *object;
1014 unsigned long flags;
1015
1016 pr_debug("%s(0x%p)\n", __func__, ptr);
1017
1018 if (!kmemleak_enabled || IS_ERR_OR_NULL(ptr))
1019 return;
1020
1021 object = find_and_get_object((unsigned long)ptr, 1);
1022 if (!object) {
1023#ifdef DEBUG
1024 kmemleak_warn("Updating stack trace for unknown object at %p\n",
1025 ptr);
1026#endif
1027 return;
1028 }
1029
1030 raw_spin_lock_irqsave(&object->lock, flags);
1031 object->trace_len = __save_stack_trace(object->trace);
1032 raw_spin_unlock_irqrestore(&object->lock, flags);
1033
1034 put_object(object);
1035}
1036EXPORT_SYMBOL(kmemleak_update_trace);
1037
1038
1039
1040
1041
1042
1043
1044
1045void __ref kmemleak_not_leak(const void *ptr)
1046{
1047 pr_debug("%s(0x%p)\n", __func__, ptr);
1048
1049 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1050 make_gray_object((unsigned long)ptr);
1051}
1052EXPORT_SYMBOL(kmemleak_not_leak);
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063void __ref kmemleak_ignore(const void *ptr)
1064{
1065 pr_debug("%s(0x%p)\n", __func__, ptr);
1066
1067 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1068 make_black_object((unsigned long)ptr);
1069}
1070EXPORT_SYMBOL(kmemleak_ignore);
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
1084{
1085 pr_debug("%s(0x%p)\n", __func__, ptr);
1086
1087 if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
1088 add_scan_area((unsigned long)ptr, size, gfp);
1089}
1090EXPORT_SYMBOL(kmemleak_scan_area);
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101void __ref kmemleak_no_scan(const void *ptr)
1102{
1103 pr_debug("%s(0x%p)\n", __func__, ptr);
1104
1105 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1106 object_no_scan((unsigned long)ptr);
1107}
1108EXPORT_SYMBOL(kmemleak_no_scan);
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, int min_count,
1120 gfp_t gfp)
1121{
1122 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1123 kmemleak_alloc(__va(phys), size, min_count, gfp);
1124}
1125EXPORT_SYMBOL(kmemleak_alloc_phys);
1126
1127
1128
1129
1130
1131
1132
1133
1134void __ref kmemleak_free_part_phys(phys_addr_t phys, size_t size)
1135{
1136 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1137 kmemleak_free_part(__va(phys), size);
1138}
1139EXPORT_SYMBOL(kmemleak_free_part_phys);
1140
1141
1142
1143
1144
1145
1146void __ref kmemleak_not_leak_phys(phys_addr_t phys)
1147{
1148 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1149 kmemleak_not_leak(__va(phys));
1150}
1151EXPORT_SYMBOL(kmemleak_not_leak_phys);
1152
1153
1154
1155
1156
1157
1158void __ref kmemleak_ignore_phys(phys_addr_t phys)
1159{
1160 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1161 kmemleak_ignore(__va(phys));
1162}
1163EXPORT_SYMBOL(kmemleak_ignore_phys);
1164
1165
1166
1167
1168static bool update_checksum(struct kmemleak_object *object)
1169{
1170 u32 old_csum = object->checksum;
1171
1172 kasan_disable_current();
1173 kcsan_disable_current();
1174 object->checksum = crc32(0, kasan_reset_tag((void *)object->pointer), object->size);
1175 kasan_enable_current();
1176 kcsan_enable_current();
1177
1178 return object->checksum != old_csum;
1179}
1180
1181
1182
1183
1184static void update_refs(struct kmemleak_object *object)
1185{
1186 if (!color_white(object)) {
1187
1188 return;
1189 }
1190
1191
1192
1193
1194
1195
1196
1197 object->count++;
1198 if (color_gray(object)) {
1199
1200 WARN_ON(!get_object(object));
1201 list_add_tail(&object->gray_list, &gray_list);
1202 }
1203}
1204
1205
1206
1207
1208
1209static int scan_should_stop(void)
1210{
1211 if (!kmemleak_enabled)
1212 return 1;
1213
1214
1215
1216
1217
1218 if (current->mm)
1219 return signal_pending(current);
1220 else
1221 return kthread_should_stop();
1222
1223 return 0;
1224}
1225
1226
1227
1228
1229
1230static void scan_block(void *_start, void *_end,
1231 struct kmemleak_object *scanned)
1232{
1233 unsigned long *ptr;
1234 unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
1235 unsigned long *end = _end - (BYTES_PER_POINTER - 1);
1236 unsigned long flags;
1237 unsigned long untagged_ptr;
1238
1239 raw_spin_lock_irqsave(&kmemleak_lock, flags);
1240 for (ptr = start; ptr < end; ptr++) {
1241 struct kmemleak_object *object;
1242 unsigned long pointer;
1243 unsigned long excess_ref;
1244
1245 if (scan_should_stop())
1246 break;
1247
1248 kasan_disable_current();
1249 pointer = *(unsigned long *)kasan_reset_tag((void *)ptr);
1250 kasan_enable_current();
1251
1252 untagged_ptr = (unsigned long)kasan_reset_tag((void *)pointer);
1253 if (untagged_ptr < min_addr || untagged_ptr >= max_addr)
1254 continue;
1255
1256
1257
1258
1259
1260
1261
1262 object = lookup_object(pointer, 1);
1263 if (!object)
1264 continue;
1265 if (object == scanned)
1266
1267 continue;
1268
1269
1270
1271
1272
1273
1274 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1275
1276 if (color_gray(object)) {
1277 excess_ref = object->excess_ref;
1278
1279 } else {
1280 excess_ref = 0;
1281 update_refs(object);
1282 }
1283 raw_spin_unlock(&object->lock);
1284
1285 if (excess_ref) {
1286 object = lookup_object(excess_ref, 0);
1287 if (!object)
1288 continue;
1289 if (object == scanned)
1290
1291 continue;
1292 raw_spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1293 update_refs(object);
1294 raw_spin_unlock(&object->lock);
1295 }
1296 }
1297 raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
1298}
1299
1300
1301
1302
1303#ifdef CONFIG_SMP
1304static void scan_large_block(void *start, void *end)
1305{
1306 void *next;
1307
1308 while (start < end) {
1309 next = min(start + MAX_SCAN_SIZE, end);
1310 scan_block(start, next, NULL);
1311 start = next;
1312 cond_resched();
1313 }
1314}
1315#endif
1316
1317
1318
1319
1320
1321static void scan_object(struct kmemleak_object *object)
1322{
1323 struct kmemleak_scan_area *area;
1324 unsigned long flags;
1325
1326
1327
1328
1329
1330 raw_spin_lock_irqsave(&object->lock, flags);
1331 if (object->flags & OBJECT_NO_SCAN)
1332 goto out;
1333 if (!(object->flags & OBJECT_ALLOCATED))
1334
1335 goto out;
1336 if (hlist_empty(&object->area_list) ||
1337 object->flags & OBJECT_FULL_SCAN) {
1338 void *start = (void *)object->pointer;
1339 void *end = (void *)(object->pointer + object->size);
1340 void *next;
1341
1342 do {
1343 next = min(start + MAX_SCAN_SIZE, end);
1344 scan_block(start, next, object);
1345
1346 start = next;
1347 if (start >= end)
1348 break;
1349
1350 raw_spin_unlock_irqrestore(&object->lock, flags);
1351 cond_resched();
1352 raw_spin_lock_irqsave(&object->lock, flags);
1353 } while (object->flags & OBJECT_ALLOCATED);
1354 } else
1355 hlist_for_each_entry(area, &object->area_list, node)
1356 scan_block((void *)area->start,
1357 (void *)(area->start + area->size),
1358 object);
1359out:
1360 raw_spin_unlock_irqrestore(&object->lock, flags);
1361}
1362
1363
1364
1365
1366
1367static void scan_gray_list(void)
1368{
1369 struct kmemleak_object *object, *tmp;
1370
1371
1372
1373
1374
1375
1376 object = list_entry(gray_list.next, typeof(*object), gray_list);
1377 while (&object->gray_list != &gray_list) {
1378 cond_resched();
1379
1380
1381 if (!scan_should_stop())
1382 scan_object(object);
1383
1384 tmp = list_entry(object->gray_list.next, typeof(*object),
1385 gray_list);
1386
1387
1388 list_del(&object->gray_list);
1389 put_object(object);
1390
1391 object = tmp;
1392 }
1393 WARN_ON(!list_empty(&gray_list));
1394}
1395
1396
1397
1398
1399
1400
1401static void kmemleak_scan(void)
1402{
1403 unsigned long flags;
1404 struct kmemleak_object *object;
1405 int i;
1406 int new_leaks = 0;
1407
1408 jiffies_last_scan = jiffies;
1409
1410
1411 rcu_read_lock();
1412 list_for_each_entry_rcu(object, &object_list, object_list) {
1413 raw_spin_lock_irqsave(&object->lock, flags);
1414#ifdef DEBUG
1415
1416
1417
1418
1419 if (atomic_read(&object->use_count) > 1) {
1420 pr_debug("object->use_count = %d\n",
1421 atomic_read(&object->use_count));
1422 dump_object_info(object);
1423 }
1424#endif
1425
1426 object->count = 0;
1427 if (color_gray(object) && get_object(object))
1428 list_add_tail(&object->gray_list, &gray_list);
1429
1430 raw_spin_unlock_irqrestore(&object->lock, flags);
1431 }
1432 rcu_read_unlock();
1433
1434#ifdef CONFIG_SMP
1435
1436 for_each_possible_cpu(i)
1437 scan_large_block(__per_cpu_start + per_cpu_offset(i),
1438 __per_cpu_end + per_cpu_offset(i));
1439#endif
1440
1441
1442
1443
1444 get_online_mems();
1445 for_each_online_node(i) {
1446 unsigned long start_pfn = node_start_pfn(i);
1447 unsigned long end_pfn = node_end_pfn(i);
1448 unsigned long pfn;
1449
1450 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1451 struct page *page = pfn_to_online_page(pfn);
1452
1453 if (!page)
1454 continue;
1455
1456
1457 if (page_to_nid(page) != i)
1458 continue;
1459
1460 if (page_count(page) == 0)
1461 continue;
1462 scan_block(page, page + 1, NULL);
1463 if (!(pfn & 63))
1464 cond_resched();
1465 }
1466 }
1467 put_online_mems();
1468
1469
1470
1471
1472 if (kmemleak_stack_scan) {
1473 struct task_struct *p, *g;
1474
1475 rcu_read_lock();
1476 for_each_process_thread(g, p) {
1477 void *stack = try_get_task_stack(p);
1478 if (stack) {
1479 scan_block(stack, stack + THREAD_SIZE, NULL);
1480 put_task_stack(p);
1481 }
1482 }
1483 rcu_read_unlock();
1484 }
1485
1486
1487
1488
1489
1490 scan_gray_list();
1491
1492
1493
1494
1495
1496 rcu_read_lock();
1497 list_for_each_entry_rcu(object, &object_list, object_list) {
1498 raw_spin_lock_irqsave(&object->lock, flags);
1499 if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
1500 && update_checksum(object) && get_object(object)) {
1501
1502 object->count = object->min_count;
1503 list_add_tail(&object->gray_list, &gray_list);
1504 }
1505 raw_spin_unlock_irqrestore(&object->lock, flags);
1506 }
1507 rcu_read_unlock();
1508
1509
1510
1511
1512 scan_gray_list();
1513
1514
1515
1516
1517 if (scan_should_stop())
1518 return;
1519
1520
1521
1522
1523 rcu_read_lock();
1524 list_for_each_entry_rcu(object, &object_list, object_list) {
1525 raw_spin_lock_irqsave(&object->lock, flags);
1526 if (unreferenced_object(object) &&
1527 !(object->flags & OBJECT_REPORTED)) {
1528 object->flags |= OBJECT_REPORTED;
1529
1530 if (kmemleak_verbose)
1531 print_unreferenced(NULL, object);
1532
1533 new_leaks++;
1534 }
1535 raw_spin_unlock_irqrestore(&object->lock, flags);
1536 }
1537 rcu_read_unlock();
1538
1539 if (new_leaks) {
1540 kmemleak_found_leaks = true;
1541
1542 pr_info("%d new suspected memory leaks (see /sys/kernel/debug/kmemleak)\n",
1543 new_leaks);
1544 }
1545
1546}
1547
1548
1549
1550
1551
1552static int kmemleak_scan_thread(void *arg)
1553{
1554 static int first_run = IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN);
1555
1556 pr_info("Automatic memory scanning thread started\n");
1557 set_user_nice(current, 10);
1558
1559
1560
1561
1562 if (first_run) {
1563 signed long timeout = msecs_to_jiffies(SECS_FIRST_SCAN * 1000);
1564 first_run = 0;
1565 while (timeout && !kthread_should_stop())
1566 timeout = schedule_timeout_interruptible(timeout);
1567 }
1568
1569 while (!kthread_should_stop()) {
1570 signed long timeout = READ_ONCE(jiffies_scan_wait);
1571
1572 mutex_lock(&scan_mutex);
1573 kmemleak_scan();
1574 mutex_unlock(&scan_mutex);
1575
1576
1577 while (timeout && !kthread_should_stop())
1578 timeout = schedule_timeout_interruptible(timeout);
1579 }
1580
1581 pr_info("Automatic memory scanning thread ended\n");
1582
1583 return 0;
1584}
1585
1586
1587
1588
1589
1590static void start_scan_thread(void)
1591{
1592 if (scan_thread)
1593 return;
1594 scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
1595 if (IS_ERR(scan_thread)) {
1596 pr_warn("Failed to create the scan thread\n");
1597 scan_thread = NULL;
1598 }
1599}
1600
1601
1602
1603
1604static void stop_scan_thread(void)
1605{
1606 if (scan_thread) {
1607 kthread_stop(scan_thread);
1608 scan_thread = NULL;
1609 }
1610}
1611
1612
1613
1614
1615
1616
1617static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
1618{
1619 struct kmemleak_object *object;
1620 loff_t n = *pos;
1621 int err;
1622
1623 err = mutex_lock_interruptible(&scan_mutex);
1624 if (err < 0)
1625 return ERR_PTR(err);
1626
1627 rcu_read_lock();
1628 list_for_each_entry_rcu(object, &object_list, object_list) {
1629 if (n-- > 0)
1630 continue;
1631 if (get_object(object))
1632 goto out;
1633 }
1634 object = NULL;
1635out:
1636 return object;
1637}
1638
1639
1640
1641
1642
1643static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1644{
1645 struct kmemleak_object *prev_obj = v;
1646 struct kmemleak_object *next_obj = NULL;
1647 struct kmemleak_object *obj = prev_obj;
1648
1649 ++(*pos);
1650
1651 list_for_each_entry_continue_rcu(obj, &object_list, object_list) {
1652 if (get_object(obj)) {
1653 next_obj = obj;
1654 break;
1655 }
1656 }
1657
1658 put_object(prev_obj);
1659 return next_obj;
1660}
1661
1662
1663
1664
1665static void kmemleak_seq_stop(struct seq_file *seq, void *v)
1666{
1667 if (!IS_ERR(v)) {
1668
1669
1670
1671
1672 rcu_read_unlock();
1673 mutex_unlock(&scan_mutex);
1674 if (v)
1675 put_object(v);
1676 }
1677}
1678
1679
1680
1681
1682static int kmemleak_seq_show(struct seq_file *seq, void *v)
1683{
1684 struct kmemleak_object *object = v;
1685 unsigned long flags;
1686
1687 raw_spin_lock_irqsave(&object->lock, flags);
1688 if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
1689 print_unreferenced(seq, object);
1690 raw_spin_unlock_irqrestore(&object->lock, flags);
1691 return 0;
1692}
1693
1694static const struct seq_operations kmemleak_seq_ops = {
1695 .start = kmemleak_seq_start,
1696 .next = kmemleak_seq_next,
1697 .stop = kmemleak_seq_stop,
1698 .show = kmemleak_seq_show,
1699};
1700
1701static int kmemleak_open(struct inode *inode, struct file *file)
1702{
1703 return seq_open(file, &kmemleak_seq_ops);
1704}
1705
1706static int dump_str_object_info(const char *str)
1707{
1708 unsigned long flags;
1709 struct kmemleak_object *object;
1710 unsigned long addr;
1711
1712 if (kstrtoul(str, 0, &addr))
1713 return -EINVAL;
1714 object = find_and_get_object(addr, 0);
1715 if (!object) {
1716 pr_info("Unknown object at 0x%08lx\n", addr);
1717 return -EINVAL;
1718 }
1719
1720 raw_spin_lock_irqsave(&object->lock, flags);
1721 dump_object_info(object);
1722 raw_spin_unlock_irqrestore(&object->lock, flags);
1723
1724 put_object(object);
1725 return 0;
1726}
1727
1728
1729
1730
1731
1732
1733
1734static void kmemleak_clear(void)
1735{
1736 struct kmemleak_object *object;
1737 unsigned long flags;
1738
1739 rcu_read_lock();
1740 list_for_each_entry_rcu(object, &object_list, object_list) {
1741 raw_spin_lock_irqsave(&object->lock, flags);
1742 if ((object->flags & OBJECT_REPORTED) &&
1743 unreferenced_object(object))
1744 __paint_it(object, KMEMLEAK_GREY);
1745 raw_spin_unlock_irqrestore(&object->lock, flags);
1746 }
1747 rcu_read_unlock();
1748
1749 kmemleak_found_leaks = false;
1750}
1751
1752static void __kmemleak_do_cleanup(void);
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
1771 size_t size, loff_t *ppos)
1772{
1773 char buf[64];
1774 int buf_size;
1775 int ret;
1776
1777 buf_size = min(size, (sizeof(buf) - 1));
1778 if (strncpy_from_user(buf, user_buf, buf_size) < 0)
1779 return -EFAULT;
1780 buf[buf_size] = 0;
1781
1782 ret = mutex_lock_interruptible(&scan_mutex);
1783 if (ret < 0)
1784 return ret;
1785
1786 if (strncmp(buf, "clear", 5) == 0) {
1787 if (kmemleak_enabled)
1788 kmemleak_clear();
1789 else
1790 __kmemleak_do_cleanup();
1791 goto out;
1792 }
1793
1794 if (!kmemleak_enabled) {
1795 ret = -EPERM;
1796 goto out;
1797 }
1798
1799 if (strncmp(buf, "off", 3) == 0)
1800 kmemleak_disable();
1801 else if (strncmp(buf, "stack=on", 8) == 0)
1802 kmemleak_stack_scan = 1;
1803 else if (strncmp(buf, "stack=off", 9) == 0)
1804 kmemleak_stack_scan = 0;
1805 else if (strncmp(buf, "scan=on", 7) == 0)
1806 start_scan_thread();
1807 else if (strncmp(buf, "scan=off", 8) == 0)
1808 stop_scan_thread();
1809 else if (strncmp(buf, "scan=", 5) == 0) {
1810 unsigned secs;
1811 unsigned long msecs;
1812
1813 ret = kstrtouint(buf + 5, 0, &secs);
1814 if (ret < 0)
1815 goto out;
1816
1817 msecs = secs * MSEC_PER_SEC;
1818 if (msecs > UINT_MAX)
1819 msecs = UINT_MAX;
1820
1821 stop_scan_thread();
1822 if (msecs) {
1823 WRITE_ONCE(jiffies_scan_wait, msecs_to_jiffies(msecs));
1824 start_scan_thread();
1825 }
1826 } else if (strncmp(buf, "scan", 4) == 0)
1827 kmemleak_scan();
1828 else if (strncmp(buf, "dump=", 5) == 0)
1829 ret = dump_str_object_info(buf + 5);
1830 else
1831 ret = -EINVAL;
1832
1833out:
1834 mutex_unlock(&scan_mutex);
1835 if (ret < 0)
1836 return ret;
1837
1838
1839 *ppos += size;
1840 return size;
1841}
1842
1843static const struct file_operations kmemleak_fops = {
1844 .owner = THIS_MODULE,
1845 .open = kmemleak_open,
1846 .read = seq_read,
1847 .write = kmemleak_write,
1848 .llseek = seq_lseek,
1849 .release = seq_release,
1850};
1851
1852static void __kmemleak_do_cleanup(void)
1853{
1854 struct kmemleak_object *object, *tmp;
1855
1856
1857
1858
1859
1860 list_for_each_entry_safe(object, tmp, &object_list, object_list) {
1861 __remove_object(object);
1862 __delete_object(object);
1863 }
1864}
1865
1866
1867
1868
1869
1870
1871static void kmemleak_do_cleanup(struct work_struct *work)
1872{
1873 stop_scan_thread();
1874
1875 mutex_lock(&scan_mutex);
1876
1877
1878
1879
1880
1881
1882 kmemleak_free_enabled = 0;
1883 mutex_unlock(&scan_mutex);
1884
1885 if (!kmemleak_found_leaks)
1886 __kmemleak_do_cleanup();
1887 else
1888 pr_info("Kmemleak disabled without freeing internal data. Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\".\n");
1889}
1890
1891static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
1892
1893
1894
1895
1896
1897static void kmemleak_disable(void)
1898{
1899
1900 if (cmpxchg(&kmemleak_error, 0, 1))
1901 return;
1902
1903
1904 kmemleak_enabled = 0;
1905
1906
1907 if (kmemleak_initialized)
1908 schedule_work(&cleanup_work);
1909 else
1910 kmemleak_free_enabled = 0;
1911
1912 pr_info("Kernel memory leak detector disabled\n");
1913}
1914
1915
1916
1917
1918static int __init kmemleak_boot_config(char *str)
1919{
1920 if (!str)
1921 return -EINVAL;
1922 if (strcmp(str, "off") == 0)
1923 kmemleak_disable();
1924 else if (strcmp(str, "on") == 0)
1925 kmemleak_skip_disable = 1;
1926 else
1927 return -EINVAL;
1928 return 0;
1929}
1930early_param("kmemleak", kmemleak_boot_config);
1931
1932
1933
1934
1935void __init kmemleak_init(void)
1936{
1937#ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
1938 if (!kmemleak_skip_disable) {
1939 kmemleak_disable();
1940 return;
1941 }
1942#endif
1943
1944 if (kmemleak_error)
1945 return;
1946
1947 jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
1948 jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
1949
1950 object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
1951 scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
1952
1953
1954 create_object((unsigned long)_sdata, _edata - _sdata,
1955 KMEMLEAK_GREY, GFP_ATOMIC);
1956 create_object((unsigned long)__bss_start, __bss_stop - __bss_start,
1957 KMEMLEAK_GREY, GFP_ATOMIC);
1958
1959 if (&__start_ro_after_init < &_sdata || &__end_ro_after_init > &_edata)
1960 create_object((unsigned long)__start_ro_after_init,
1961 __end_ro_after_init - __start_ro_after_init,
1962 KMEMLEAK_GREY, GFP_ATOMIC);
1963}
1964
1965
1966
1967
1968static int __init kmemleak_late_init(void)
1969{
1970 kmemleak_initialized = 1;
1971
1972 debugfs_create_file("kmemleak", 0644, NULL, NULL, &kmemleak_fops);
1973
1974 if (kmemleak_error) {
1975
1976
1977
1978
1979
1980
1981 schedule_work(&cleanup_work);
1982 return -ENOMEM;
1983 }
1984
1985 if (IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN)) {
1986 mutex_lock(&scan_mutex);
1987 start_scan_thread();
1988 mutex_unlock(&scan_mutex);
1989 }
1990
1991 pr_info("Kernel memory leak detector initialized (mem pool available: %d)\n",
1992 mem_pool_free_count);
1993
1994 return 0;
1995}
1996late_initcall(kmemleak_late_init);
1997