1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
59
60#include <linux/init.h>
61#include <linux/kernel.h>
62#include <linux/list.h>
63#include <linux/sched/signal.h>
64#include <linux/sched/task.h>
65#include <linux/sched/task_stack.h>
66#include <linux/jiffies.h>
67#include <linux/delay.h>
68#include <linux/export.h>
69#include <linux/kthread.h>
70#include <linux/rbtree.h>
71#include <linux/fs.h>
72#include <linux/debugfs.h>
73#include <linux/seq_file.h>
74#include <linux/cpumask.h>
75#include <linux/spinlock.h>
76#include <linux/module.h>
77#include <linux/mutex.h>
78#include <linux/rcupdate.h>
79#include <linux/stacktrace.h>
80#include <linux/cache.h>
81#include <linux/percpu.h>
82#include <linux/memblock.h>
83#include <linux/pfn.h>
84#include <linux/mmzone.h>
85#include <linux/slab.h>
86#include <linux/thread_info.h>
87#include <linux/err.h>
88#include <linux/uaccess.h>
89#include <linux/string.h>
90#include <linux/nodemask.h>
91#include <linux/mm.h>
92#include <linux/workqueue.h>
93#include <linux/crc32.h>
94
95#include <asm/sections.h>
96#include <asm/processor.h>
97#include <linux/atomic.h>
98
99#include <linux/kasan.h>
100#include <linux/kmemleak.h>
101#include <linux/memory_hotplug.h>
102
103
104
105
106#define MAX_TRACE 16
107#define MSECS_MIN_AGE 5000
108#define SECS_FIRST_SCAN 60
109#define SECS_SCAN_WAIT 600
110#define MAX_SCAN_SIZE 4096
111
112#define BYTES_PER_POINTER sizeof(void *)
113
114
115#define gfp_kmemleak_mask(gfp) (((gfp) & (GFP_KERNEL | GFP_ATOMIC)) | \
116 __GFP_NORETRY | __GFP_NOMEMALLOC | \
117 __GFP_NOWARN)
118
119
120struct kmemleak_scan_area {
121 struct hlist_node node;
122 unsigned long start;
123 size_t size;
124};
125
126#define KMEMLEAK_GREY 0
127#define KMEMLEAK_BLACK -1
128
129
130
131
132
133
134
135
136
137struct kmemleak_object {
138 spinlock_t lock;
139 unsigned int flags;
140 struct list_head object_list;
141 struct list_head gray_list;
142 struct rb_node rb_node;
143 struct rcu_head rcu;
144
145 atomic_t use_count;
146 unsigned long pointer;
147 size_t size;
148
149 unsigned long excess_ref;
150
151 int min_count;
152
153 int count;
154
155 u32 checksum;
156
157 struct hlist_head area_list;
158 unsigned long trace[MAX_TRACE];
159 unsigned int trace_len;
160 unsigned long jiffies;
161 pid_t pid;
162 char comm[TASK_COMM_LEN];
163};
164
165
166#define OBJECT_ALLOCATED (1 << 0)
167
168#define OBJECT_REPORTED (1 << 1)
169
170#define OBJECT_NO_SCAN (1 << 2)
171
172#define HEX_PREFIX " "
173
174#define HEX_ROW_SIZE 16
175
176#define HEX_GROUP_SIZE 1
177
178#define HEX_ASCII 1
179
180#define HEX_MAX_LINES 2
181
182
183static LIST_HEAD(object_list);
184
185static LIST_HEAD(gray_list);
186
187static struct rb_root object_tree_root = RB_ROOT;
188
189static DEFINE_RWLOCK(kmemleak_lock);
190
191
192static struct kmem_cache *object_cache;
193static struct kmem_cache *scan_area_cache;
194
195
196static int kmemleak_enabled;
197
198static int kmemleak_free_enabled;
199
200static int kmemleak_initialized;
201
202static int kmemleak_early_log = 1;
203
204static int kmemleak_warning;
205
206static int kmemleak_error;
207
208
209static unsigned long min_addr = ULONG_MAX;
210static unsigned long max_addr;
211
212static struct task_struct *scan_thread;
213
214static unsigned long jiffies_min_age;
215static unsigned long jiffies_last_scan;
216
217static signed long jiffies_scan_wait;
218
219static int kmemleak_stack_scan = 1;
220
221static DEFINE_MUTEX(scan_mutex);
222
223static int kmemleak_skip_disable;
224
225static bool kmemleak_found_leaks;
226
227static bool kmemleak_verbose;
228module_param_named(verbose, kmemleak_verbose, bool, 0600);
229
230
231
232
233
234
235
236
237
238
239enum {
240 KMEMLEAK_ALLOC,
241 KMEMLEAK_ALLOC_PERCPU,
242 KMEMLEAK_FREE,
243 KMEMLEAK_FREE_PART,
244 KMEMLEAK_FREE_PERCPU,
245 KMEMLEAK_NOT_LEAK,
246 KMEMLEAK_IGNORE,
247 KMEMLEAK_SCAN_AREA,
248 KMEMLEAK_NO_SCAN,
249 KMEMLEAK_SET_EXCESS_REF
250};
251
252
253
254
255
256struct early_log {
257 int op_type;
258 int min_count;
259 const void *ptr;
260 union {
261 size_t size;
262 unsigned long excess_ref;
263 };
264 unsigned long trace[MAX_TRACE];
265 unsigned int trace_len;
266};
267
268
269static struct early_log
270 early_log[CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE] __initdata;
271static int crt_early_log __initdata;
272
273static void kmemleak_disable(void);
274
275
276
277
278#define kmemleak_warn(x...) do { \
279 pr_warn(x); \
280 dump_stack(); \
281 kmemleak_warning = 1; \
282} while (0)
283
284
285
286
287
288
289#define kmemleak_stop(x...) do { \
290 kmemleak_warn(x); \
291 kmemleak_disable(); \
292} while (0)
293
294#define warn_or_seq_printf(seq, fmt, ...) do { \
295 if (seq) \
296 seq_printf(seq, fmt, ##__VA_ARGS__); \
297 else \
298 pr_warn(fmt, ##__VA_ARGS__); \
299} while (0)
300
301static void warn_or_seq_hex_dump(struct seq_file *seq, int prefix_type,
302 int rowsize, int groupsize, const void *buf,
303 size_t len, bool ascii)
304{
305 if (seq)
306 seq_hex_dump(seq, HEX_PREFIX, prefix_type, rowsize, groupsize,
307 buf, len, ascii);
308 else
309 print_hex_dump(KERN_WARNING, pr_fmt(HEX_PREFIX), prefix_type,
310 rowsize, groupsize, buf, len, ascii);
311}
312
313
314
315
316
317
318
319static void hex_dump_object(struct seq_file *seq,
320 struct kmemleak_object *object)
321{
322 const u8 *ptr = (const u8 *)object->pointer;
323 size_t len;
324
325
326 len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);
327
328 warn_or_seq_printf(seq, " hex dump (first %zu bytes):\n", len);
329 kasan_disable_current();
330 warn_or_seq_hex_dump(seq, DUMP_PREFIX_NONE, HEX_ROW_SIZE,
331 HEX_GROUP_SIZE, ptr, len, HEX_ASCII);
332 kasan_enable_current();
333}
334
335
336
337
338
339
340
341
342
343
344
345static bool color_white(const struct kmemleak_object *object)
346{
347 return object->count != KMEMLEAK_BLACK &&
348 object->count < object->min_count;
349}
350
351static bool color_gray(const struct kmemleak_object *object)
352{
353 return object->min_count != KMEMLEAK_BLACK &&
354 object->count >= object->min_count;
355}
356
357
358
359
360
361
362static bool unreferenced_object(struct kmemleak_object *object)
363{
364 return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
365 time_before_eq(object->jiffies + jiffies_min_age,
366 jiffies_last_scan);
367}
368
369
370
371
372
373static void print_unreferenced(struct seq_file *seq,
374 struct kmemleak_object *object)
375{
376 int i;
377 unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);
378
379 warn_or_seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
380 object->pointer, object->size);
381 warn_or_seq_printf(seq, " comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
382 object->comm, object->pid, object->jiffies,
383 msecs_age / 1000, msecs_age % 1000);
384 hex_dump_object(seq, object);
385 warn_or_seq_printf(seq, " backtrace:\n");
386
387 for (i = 0; i < object->trace_len; i++) {
388 void *ptr = (void *)object->trace[i];
389 warn_or_seq_printf(seq, " [<%p>] %pS\n", ptr, ptr);
390 }
391}
392
393
394
395
396
397
398static void dump_object_info(struct kmemleak_object *object)
399{
400 pr_notice("Object 0x%08lx (size %zu):\n",
401 object->pointer, object->size);
402 pr_notice(" comm \"%s\", pid %d, jiffies %lu\n",
403 object->comm, object->pid, object->jiffies);
404 pr_notice(" min_count = %d\n", object->min_count);
405 pr_notice(" count = %d\n", object->count);
406 pr_notice(" flags = 0x%x\n", object->flags);
407 pr_notice(" checksum = %u\n", object->checksum);
408 pr_notice(" backtrace:\n");
409 stack_trace_print(object->trace, object->trace_len, 4);
410}
411
412
413
414
415
416
417
418static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
419{
420 struct rb_node *rb = object_tree_root.rb_node;
421
422 while (rb) {
423 struct kmemleak_object *object =
424 rb_entry(rb, struct kmemleak_object, rb_node);
425 if (ptr < object->pointer)
426 rb = object->rb_node.rb_left;
427 else if (object->pointer + object->size <= ptr)
428 rb = object->rb_node.rb_right;
429 else if (object->pointer == ptr || alias)
430 return object;
431 else {
432 kmemleak_warn("Found object by alias at 0x%08lx\n",
433 ptr);
434 dump_object_info(object);
435 break;
436 }
437 }
438 return NULL;
439}
440
441
442
443
444
445
446
447static int get_object(struct kmemleak_object *object)
448{
449 return atomic_inc_not_zero(&object->use_count);
450}
451
452
453
454
455static void free_object_rcu(struct rcu_head *rcu)
456{
457 struct hlist_node *tmp;
458 struct kmemleak_scan_area *area;
459 struct kmemleak_object *object =
460 container_of(rcu, struct kmemleak_object, rcu);
461
462
463
464
465
466 hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {
467 hlist_del(&area->node);
468 kmem_cache_free(scan_area_cache, area);
469 }
470 kmem_cache_free(object_cache, object);
471}
472
473
474
475
476
477
478
479
480static void put_object(struct kmemleak_object *object)
481{
482 if (!atomic_dec_and_test(&object->use_count))
483 return;
484
485
486 WARN_ON(object->flags & OBJECT_ALLOCATED);
487
488 call_rcu(&object->rcu, free_object_rcu);
489}
490
491
492
493
494static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
495{
496 unsigned long flags;
497 struct kmemleak_object *object;
498
499 rcu_read_lock();
500 read_lock_irqsave(&kmemleak_lock, flags);
501 object = lookup_object(ptr, alias);
502 read_unlock_irqrestore(&kmemleak_lock, flags);
503
504
505 if (object && !get_object(object))
506 object = NULL;
507 rcu_read_unlock();
508
509 return object;
510}
511
512
513
514
515
516
517static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int alias)
518{
519 unsigned long flags;
520 struct kmemleak_object *object;
521
522 write_lock_irqsave(&kmemleak_lock, flags);
523 object = lookup_object(ptr, alias);
524 if (object) {
525 rb_erase(&object->rb_node, &object_tree_root);
526 list_del_rcu(&object->object_list);
527 }
528 write_unlock_irqrestore(&kmemleak_lock, flags);
529
530 return object;
531}
532
533
534
535
536static int __save_stack_trace(unsigned long *trace)
537{
538 return stack_trace_save(trace, MAX_TRACE, 2);
539}
540
541
542
543
544
545static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
546 int min_count, gfp_t gfp)
547{
548 unsigned long flags;
549 struct kmemleak_object *object, *parent;
550 struct rb_node **link, *rb_parent;
551 unsigned long untagged_ptr;
552
553 object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
554 if (!object) {
555 pr_warn("Cannot allocate a kmemleak_object structure\n");
556 kmemleak_disable();
557 return NULL;
558 }
559
560 INIT_LIST_HEAD(&object->object_list);
561 INIT_LIST_HEAD(&object->gray_list);
562 INIT_HLIST_HEAD(&object->area_list);
563 spin_lock_init(&object->lock);
564 atomic_set(&object->use_count, 1);
565 object->flags = OBJECT_ALLOCATED;
566 object->pointer = ptr;
567 object->size = size;
568 object->excess_ref = 0;
569 object->min_count = min_count;
570 object->count = 0;
571 object->jiffies = jiffies;
572 object->checksum = 0;
573
574
575 if (in_irq()) {
576 object->pid = 0;
577 strncpy(object->comm, "hardirq", sizeof(object->comm));
578 } else if (in_serving_softirq()) {
579 object->pid = 0;
580 strncpy(object->comm, "softirq", sizeof(object->comm));
581 } else {
582 object->pid = current->pid;
583
584
585
586
587
588
589 strncpy(object->comm, current->comm, sizeof(object->comm));
590 }
591
592
593 object->trace_len = __save_stack_trace(object->trace);
594
595 write_lock_irqsave(&kmemleak_lock, flags);
596
597 untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
598 min_addr = min(min_addr, untagged_ptr);
599 max_addr = max(max_addr, untagged_ptr + size);
600 link = &object_tree_root.rb_node;
601 rb_parent = NULL;
602 while (*link) {
603 rb_parent = *link;
604 parent = rb_entry(rb_parent, struct kmemleak_object, rb_node);
605 if (ptr + size <= parent->pointer)
606 link = &parent->rb_node.rb_left;
607 else if (parent->pointer + parent->size <= ptr)
608 link = &parent->rb_node.rb_right;
609 else {
610 kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
611 ptr);
612
613
614
615
616 dump_object_info(parent);
617 kmem_cache_free(object_cache, object);
618 object = NULL;
619 goto out;
620 }
621 }
622 rb_link_node(&object->rb_node, rb_parent, link);
623 rb_insert_color(&object->rb_node, &object_tree_root);
624
625 list_add_tail_rcu(&object->object_list, &object_list);
626out:
627 write_unlock_irqrestore(&kmemleak_lock, flags);
628 return object;
629}
630
631
632
633
634static void __delete_object(struct kmemleak_object *object)
635{
636 unsigned long flags;
637
638 WARN_ON(!(object->flags & OBJECT_ALLOCATED));
639 WARN_ON(atomic_read(&object->use_count) < 1);
640
641
642
643
644
645 spin_lock_irqsave(&object->lock, flags);
646 object->flags &= ~OBJECT_ALLOCATED;
647 spin_unlock_irqrestore(&object->lock, flags);
648 put_object(object);
649}
650
651
652
653
654
655static void delete_object_full(unsigned long ptr)
656{
657 struct kmemleak_object *object;
658
659 object = find_and_remove_object(ptr, 0);
660 if (!object) {
661#ifdef DEBUG
662 kmemleak_warn("Freeing unknown object at 0x%08lx\n",
663 ptr);
664#endif
665 return;
666 }
667 __delete_object(object);
668}
669
670
671
672
673
674
675static void delete_object_part(unsigned long ptr, size_t size)
676{
677 struct kmemleak_object *object;
678 unsigned long start, end;
679
680 object = find_and_remove_object(ptr, 1);
681 if (!object) {
682#ifdef DEBUG
683 kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
684 ptr, size);
685#endif
686 return;
687 }
688
689
690
691
692
693
694
695
696 start = object->pointer;
697 end = object->pointer + object->size;
698 if (ptr > start)
699 create_object(start, ptr - start, object->min_count,
700 GFP_KERNEL);
701 if (ptr + size < end)
702 create_object(ptr + size, end - ptr - size, object->min_count,
703 GFP_KERNEL);
704
705 __delete_object(object);
706}
707
708static void __paint_it(struct kmemleak_object *object, int color)
709{
710 object->min_count = color;
711 if (color == KMEMLEAK_BLACK)
712 object->flags |= OBJECT_NO_SCAN;
713}
714
715static void paint_it(struct kmemleak_object *object, int color)
716{
717 unsigned long flags;
718
719 spin_lock_irqsave(&object->lock, flags);
720 __paint_it(object, color);
721 spin_unlock_irqrestore(&object->lock, flags);
722}
723
724static void paint_ptr(unsigned long ptr, int color)
725{
726 struct kmemleak_object *object;
727
728 object = find_and_get_object(ptr, 0);
729 if (!object) {
730 kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
731 ptr,
732 (color == KMEMLEAK_GREY) ? "Grey" :
733 (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
734 return;
735 }
736 paint_it(object, color);
737 put_object(object);
738}
739
740
741
742
743
744static void make_gray_object(unsigned long ptr)
745{
746 paint_ptr(ptr, KMEMLEAK_GREY);
747}
748
749
750
751
752
753static void make_black_object(unsigned long ptr)
754{
755 paint_ptr(ptr, KMEMLEAK_BLACK);
756}
757
758
759
760
761
762static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
763{
764 unsigned long flags;
765 struct kmemleak_object *object;
766 struct kmemleak_scan_area *area;
767
768 object = find_and_get_object(ptr, 1);
769 if (!object) {
770 kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
771 ptr);
772 return;
773 }
774
775 area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
776 if (!area) {
777 pr_warn("Cannot allocate a scan area\n");
778 goto out;
779 }
780
781 spin_lock_irqsave(&object->lock, flags);
782 if (size == SIZE_MAX) {
783 size = object->pointer + object->size - ptr;
784 } else if (ptr + size > object->pointer + object->size) {
785 kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
786 dump_object_info(object);
787 kmem_cache_free(scan_area_cache, area);
788 goto out_unlock;
789 }
790
791 INIT_HLIST_NODE(&area->node);
792 area->start = ptr;
793 area->size = size;
794
795 hlist_add_head(&area->node, &object->area_list);
796out_unlock:
797 spin_unlock_irqrestore(&object->lock, flags);
798out:
799 put_object(object);
800}
801
802
803
804
805
806
807
808static void object_set_excess_ref(unsigned long ptr, unsigned long excess_ref)
809{
810 unsigned long flags;
811 struct kmemleak_object *object;
812
813 object = find_and_get_object(ptr, 0);
814 if (!object) {
815 kmemleak_warn("Setting excess_ref on unknown object at 0x%08lx\n",
816 ptr);
817 return;
818 }
819
820 spin_lock_irqsave(&object->lock, flags);
821 object->excess_ref = excess_ref;
822 spin_unlock_irqrestore(&object->lock, flags);
823 put_object(object);
824}
825
826
827
828
829
830
831static void object_no_scan(unsigned long ptr)
832{
833 unsigned long flags;
834 struct kmemleak_object *object;
835
836 object = find_and_get_object(ptr, 0);
837 if (!object) {
838 kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
839 return;
840 }
841
842 spin_lock_irqsave(&object->lock, flags);
843 object->flags |= OBJECT_NO_SCAN;
844 spin_unlock_irqrestore(&object->lock, flags);
845 put_object(object);
846}
847
848
849
850
851
852static void __init log_early(int op_type, const void *ptr, size_t size,
853 int min_count)
854{
855 unsigned long flags;
856 struct early_log *log;
857
858 if (kmemleak_error) {
859
860 crt_early_log++;
861 return;
862 }
863
864 if (crt_early_log >= ARRAY_SIZE(early_log)) {
865 crt_early_log++;
866 kmemleak_disable();
867 return;
868 }
869
870
871
872
873
874 local_irq_save(flags);
875 log = &early_log[crt_early_log];
876 log->op_type = op_type;
877 log->ptr = ptr;
878 log->size = size;
879 log->min_count = min_count;
880 log->trace_len = __save_stack_trace(log->trace);
881 crt_early_log++;
882 local_irq_restore(flags);
883}
884
885
886
887
888static void early_alloc(struct early_log *log)
889{
890 struct kmemleak_object *object;
891 unsigned long flags;
892 int i;
893
894 if (!kmemleak_enabled || !log->ptr || IS_ERR(log->ptr))
895 return;
896
897
898
899
900 rcu_read_lock();
901 object = create_object((unsigned long)log->ptr, log->size,
902 log->min_count, GFP_ATOMIC);
903 if (!object)
904 goto out;
905 spin_lock_irqsave(&object->lock, flags);
906 for (i = 0; i < log->trace_len; i++)
907 object->trace[i] = log->trace[i];
908 object->trace_len = log->trace_len;
909 spin_unlock_irqrestore(&object->lock, flags);
910out:
911 rcu_read_unlock();
912}
913
914
915
916
917static void early_alloc_percpu(struct early_log *log)
918{
919 unsigned int cpu;
920 const void __percpu *ptr = log->ptr;
921
922 for_each_possible_cpu(cpu) {
923 log->ptr = per_cpu_ptr(ptr, cpu);
924 early_alloc(log);
925 }
926}
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
943 gfp_t gfp)
944{
945 pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);
946
947 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
948 create_object((unsigned long)ptr, size, min_count, gfp);
949 else if (kmemleak_early_log)
950 log_early(KMEMLEAK_ALLOC, ptr, size, min_count);
951}
952EXPORT_SYMBOL_GPL(kmemleak_alloc);
953
954
955
956
957
958
959
960
961
962
963void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size,
964 gfp_t gfp)
965{
966 unsigned int cpu;
967
968 pr_debug("%s(0x%p, %zu)\n", __func__, ptr, size);
969
970
971
972
973
974 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
975 for_each_possible_cpu(cpu)
976 create_object((unsigned long)per_cpu_ptr(ptr, cpu),
977 size, 0, gfp);
978 else if (kmemleak_early_log)
979 log_early(KMEMLEAK_ALLOC_PERCPU, ptr, size, 0);
980}
981EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
982
983
984
985
986
987
988
989
990
991
992void __ref kmemleak_vmalloc(const struct vm_struct *area, size_t size, gfp_t gfp)
993{
994 pr_debug("%s(0x%p, %zu)\n", __func__, area, size);
995
996
997
998
999
1000 if (kmemleak_enabled) {
1001 create_object((unsigned long)area->addr, size, 2, gfp);
1002 object_set_excess_ref((unsigned long)area,
1003 (unsigned long)area->addr);
1004 } else if (kmemleak_early_log) {
1005 log_early(KMEMLEAK_ALLOC, area->addr, size, 2);
1006
1007 log_early(KMEMLEAK_SET_EXCESS_REF,
1008 area, (unsigned long)area->addr, 0);
1009 }
1010}
1011EXPORT_SYMBOL_GPL(kmemleak_vmalloc);
1012
1013
1014
1015
1016
1017
1018
1019
1020void __ref kmemleak_free(const void *ptr)
1021{
1022 pr_debug("%s(0x%p)\n", __func__, ptr);
1023
1024 if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
1025 delete_object_full((unsigned long)ptr);
1026 else if (kmemleak_early_log)
1027 log_early(KMEMLEAK_FREE, ptr, 0, 0);
1028}
1029EXPORT_SYMBOL_GPL(kmemleak_free);
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040void __ref kmemleak_free_part(const void *ptr, size_t size)
1041{
1042 pr_debug("%s(0x%p)\n", __func__, ptr);
1043
1044 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1045 delete_object_part((unsigned long)ptr, size);
1046 else if (kmemleak_early_log)
1047 log_early(KMEMLEAK_FREE_PART, ptr, size, 0);
1048}
1049EXPORT_SYMBOL_GPL(kmemleak_free_part);
1050
1051
1052
1053
1054
1055
1056
1057
1058void __ref kmemleak_free_percpu(const void __percpu *ptr)
1059{
1060 unsigned int cpu;
1061
1062 pr_debug("%s(0x%p)\n", __func__, ptr);
1063
1064 if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
1065 for_each_possible_cpu(cpu)
1066 delete_object_full((unsigned long)per_cpu_ptr(ptr,
1067 cpu));
1068 else if (kmemleak_early_log)
1069 log_early(KMEMLEAK_FREE_PERCPU, ptr, 0, 0);
1070}
1071EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
1072
1073
1074
1075
1076
1077
1078
1079
1080void __ref kmemleak_update_trace(const void *ptr)
1081{
1082 struct kmemleak_object *object;
1083 unsigned long flags;
1084
1085 pr_debug("%s(0x%p)\n", __func__, ptr);
1086
1087 if (!kmemleak_enabled || IS_ERR_OR_NULL(ptr))
1088 return;
1089
1090 object = find_and_get_object((unsigned long)ptr, 1);
1091 if (!object) {
1092#ifdef DEBUG
1093 kmemleak_warn("Updating stack trace for unknown object at %p\n",
1094 ptr);
1095#endif
1096 return;
1097 }
1098
1099 spin_lock_irqsave(&object->lock, flags);
1100 object->trace_len = __save_stack_trace(object->trace);
1101 spin_unlock_irqrestore(&object->lock, flags);
1102
1103 put_object(object);
1104}
1105EXPORT_SYMBOL(kmemleak_update_trace);
1106
1107
1108
1109
1110
1111
1112
1113
1114void __ref kmemleak_not_leak(const void *ptr)
1115{
1116 pr_debug("%s(0x%p)\n", __func__, ptr);
1117
1118 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1119 make_gray_object((unsigned long)ptr);
1120 else if (kmemleak_early_log)
1121 log_early(KMEMLEAK_NOT_LEAK, ptr, 0, 0);
1122}
1123EXPORT_SYMBOL(kmemleak_not_leak);
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134void __ref kmemleak_ignore(const void *ptr)
1135{
1136 pr_debug("%s(0x%p)\n", __func__, ptr);
1137
1138 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1139 make_black_object((unsigned long)ptr);
1140 else if (kmemleak_early_log)
1141 log_early(KMEMLEAK_IGNORE, ptr, 0, 0);
1142}
1143EXPORT_SYMBOL(kmemleak_ignore);
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
1157{
1158 pr_debug("%s(0x%p)\n", __func__, ptr);
1159
1160 if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
1161 add_scan_area((unsigned long)ptr, size, gfp);
1162 else if (kmemleak_early_log)
1163 log_early(KMEMLEAK_SCAN_AREA, ptr, size, 0);
1164}
1165EXPORT_SYMBOL(kmemleak_scan_area);
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176void __ref kmemleak_no_scan(const void *ptr)
1177{
1178 pr_debug("%s(0x%p)\n", __func__, ptr);
1179
1180 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1181 object_no_scan((unsigned long)ptr);
1182 else if (kmemleak_early_log)
1183 log_early(KMEMLEAK_NO_SCAN, ptr, 0, 0);
1184}
1185EXPORT_SYMBOL(kmemleak_no_scan);
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, int min_count,
1197 gfp_t gfp)
1198{
1199 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1200 kmemleak_alloc(__va(phys), size, min_count, gfp);
1201}
1202EXPORT_SYMBOL(kmemleak_alloc_phys);
1203
1204
1205
1206
1207
1208
1209
1210
1211void __ref kmemleak_free_part_phys(phys_addr_t phys, size_t size)
1212{
1213 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1214 kmemleak_free_part(__va(phys), size);
1215}
1216EXPORT_SYMBOL(kmemleak_free_part_phys);
1217
1218
1219
1220
1221
1222
1223void __ref kmemleak_not_leak_phys(phys_addr_t phys)
1224{
1225 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1226 kmemleak_not_leak(__va(phys));
1227}
1228EXPORT_SYMBOL(kmemleak_not_leak_phys);
1229
1230
1231
1232
1233
1234
1235void __ref kmemleak_ignore_phys(phys_addr_t phys)
1236{
1237 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1238 kmemleak_ignore(__va(phys));
1239}
1240EXPORT_SYMBOL(kmemleak_ignore_phys);
1241
1242
1243
1244
1245static bool update_checksum(struct kmemleak_object *object)
1246{
1247 u32 old_csum = object->checksum;
1248
1249 kasan_disable_current();
1250 object->checksum = crc32(0, (void *)object->pointer, object->size);
1251 kasan_enable_current();
1252
1253 return object->checksum != old_csum;
1254}
1255
1256
1257
1258
1259static void update_refs(struct kmemleak_object *object)
1260{
1261 if (!color_white(object)) {
1262
1263 return;
1264 }
1265
1266
1267
1268
1269
1270
1271
1272 object->count++;
1273 if (color_gray(object)) {
1274
1275 WARN_ON(!get_object(object));
1276 list_add_tail(&object->gray_list, &gray_list);
1277 }
1278}
1279
1280
1281
1282
1283
1284static int scan_should_stop(void)
1285{
1286 if (!kmemleak_enabled)
1287 return 1;
1288
1289
1290
1291
1292
1293 if (current->mm)
1294 return signal_pending(current);
1295 else
1296 return kthread_should_stop();
1297
1298 return 0;
1299}
1300
1301
1302
1303
1304
1305static void scan_block(void *_start, void *_end,
1306 struct kmemleak_object *scanned)
1307{
1308 unsigned long *ptr;
1309 unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
1310 unsigned long *end = _end - (BYTES_PER_POINTER - 1);
1311 unsigned long flags;
1312 unsigned long untagged_ptr;
1313
1314 read_lock_irqsave(&kmemleak_lock, flags);
1315 for (ptr = start; ptr < end; ptr++) {
1316 struct kmemleak_object *object;
1317 unsigned long pointer;
1318 unsigned long excess_ref;
1319
1320 if (scan_should_stop())
1321 break;
1322
1323 kasan_disable_current();
1324 pointer = *ptr;
1325 kasan_enable_current();
1326
1327 untagged_ptr = (unsigned long)kasan_reset_tag((void *)pointer);
1328 if (untagged_ptr < min_addr || untagged_ptr >= max_addr)
1329 continue;
1330
1331
1332
1333
1334
1335
1336
1337 object = lookup_object(pointer, 1);
1338 if (!object)
1339 continue;
1340 if (object == scanned)
1341
1342 continue;
1343
1344
1345
1346
1347
1348
1349 spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1350
1351 if (color_gray(object)) {
1352 excess_ref = object->excess_ref;
1353
1354 } else {
1355 excess_ref = 0;
1356 update_refs(object);
1357 }
1358 spin_unlock(&object->lock);
1359
1360 if (excess_ref) {
1361 object = lookup_object(excess_ref, 0);
1362 if (!object)
1363 continue;
1364 if (object == scanned)
1365
1366 continue;
1367 spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1368 update_refs(object);
1369 spin_unlock(&object->lock);
1370 }
1371 }
1372 read_unlock_irqrestore(&kmemleak_lock, flags);
1373}
1374
1375
1376
1377
1378#ifdef CONFIG_SMP
1379static void scan_large_block(void *start, void *end)
1380{
1381 void *next;
1382
1383 while (start < end) {
1384 next = min(start + MAX_SCAN_SIZE, end);
1385 scan_block(start, next, NULL);
1386 start = next;
1387 cond_resched();
1388 }
1389}
1390#endif
1391
1392
1393
1394
1395
1396static void scan_object(struct kmemleak_object *object)
1397{
1398 struct kmemleak_scan_area *area;
1399 unsigned long flags;
1400
1401
1402
1403
1404
1405 spin_lock_irqsave(&object->lock, flags);
1406 if (object->flags & OBJECT_NO_SCAN)
1407 goto out;
1408 if (!(object->flags & OBJECT_ALLOCATED))
1409
1410 goto out;
1411 if (hlist_empty(&object->area_list)) {
1412 void *start = (void *)object->pointer;
1413 void *end = (void *)(object->pointer + object->size);
1414 void *next;
1415
1416 do {
1417 next = min(start + MAX_SCAN_SIZE, end);
1418 scan_block(start, next, object);
1419
1420 start = next;
1421 if (start >= end)
1422 break;
1423
1424 spin_unlock_irqrestore(&object->lock, flags);
1425 cond_resched();
1426 spin_lock_irqsave(&object->lock, flags);
1427 } while (object->flags & OBJECT_ALLOCATED);
1428 } else
1429 hlist_for_each_entry(area, &object->area_list, node)
1430 scan_block((void *)area->start,
1431 (void *)(area->start + area->size),
1432 object);
1433out:
1434 spin_unlock_irqrestore(&object->lock, flags);
1435}
1436
1437
1438
1439
1440
1441static void scan_gray_list(void)
1442{
1443 struct kmemleak_object *object, *tmp;
1444
1445
1446
1447
1448
1449
1450 object = list_entry(gray_list.next, typeof(*object), gray_list);
1451 while (&object->gray_list != &gray_list) {
1452 cond_resched();
1453
1454
1455 if (!scan_should_stop())
1456 scan_object(object);
1457
1458 tmp = list_entry(object->gray_list.next, typeof(*object),
1459 gray_list);
1460
1461
1462 list_del(&object->gray_list);
1463 put_object(object);
1464
1465 object = tmp;
1466 }
1467 WARN_ON(!list_empty(&gray_list));
1468}
1469
1470
1471
1472
1473
1474
1475static void kmemleak_scan(void)
1476{
1477 unsigned long flags;
1478 struct kmemleak_object *object;
1479 int i;
1480 int new_leaks = 0;
1481
1482 jiffies_last_scan = jiffies;
1483
1484
1485 rcu_read_lock();
1486 list_for_each_entry_rcu(object, &object_list, object_list) {
1487 spin_lock_irqsave(&object->lock, flags);
1488#ifdef DEBUG
1489
1490
1491
1492
1493 if (atomic_read(&object->use_count) > 1) {
1494 pr_debug("object->use_count = %d\n",
1495 atomic_read(&object->use_count));
1496 dump_object_info(object);
1497 }
1498#endif
1499
1500 object->count = 0;
1501 if (color_gray(object) && get_object(object))
1502 list_add_tail(&object->gray_list, &gray_list);
1503
1504 spin_unlock_irqrestore(&object->lock, flags);
1505 }
1506 rcu_read_unlock();
1507
1508#ifdef CONFIG_SMP
1509
1510 for_each_possible_cpu(i)
1511 scan_large_block(__per_cpu_start + per_cpu_offset(i),
1512 __per_cpu_end + per_cpu_offset(i));
1513#endif
1514
1515
1516
1517
1518 get_online_mems();
1519 for_each_online_node(i) {
1520 unsigned long start_pfn = node_start_pfn(i);
1521 unsigned long end_pfn = node_end_pfn(i);
1522 unsigned long pfn;
1523
1524 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1525 struct page *page = pfn_to_online_page(pfn);
1526
1527 if (!page)
1528 continue;
1529
1530
1531 if (page_to_nid(page) != i)
1532 continue;
1533
1534 if (page_count(page) == 0)
1535 continue;
1536 scan_block(page, page + 1, NULL);
1537 if (!(pfn & 63))
1538 cond_resched();
1539 }
1540 }
1541 put_online_mems();
1542
1543
1544
1545
1546 if (kmemleak_stack_scan) {
1547 struct task_struct *p, *g;
1548
1549 read_lock(&tasklist_lock);
1550 do_each_thread(g, p) {
1551 void *stack = try_get_task_stack(p);
1552 if (stack) {
1553 scan_block(stack, stack + THREAD_SIZE, NULL);
1554 put_task_stack(p);
1555 }
1556 } while_each_thread(g, p);
1557 read_unlock(&tasklist_lock);
1558 }
1559
1560
1561
1562
1563
1564 scan_gray_list();
1565
1566
1567
1568
1569
1570 rcu_read_lock();
1571 list_for_each_entry_rcu(object, &object_list, object_list) {
1572 spin_lock_irqsave(&object->lock, flags);
1573 if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
1574 && update_checksum(object) && get_object(object)) {
1575
1576 object->count = object->min_count;
1577 list_add_tail(&object->gray_list, &gray_list);
1578 }
1579 spin_unlock_irqrestore(&object->lock, flags);
1580 }
1581 rcu_read_unlock();
1582
1583
1584
1585
1586 scan_gray_list();
1587
1588
1589
1590
1591 if (scan_should_stop())
1592 return;
1593
1594
1595
1596
1597 rcu_read_lock();
1598 list_for_each_entry_rcu(object, &object_list, object_list) {
1599 spin_lock_irqsave(&object->lock, flags);
1600 if (unreferenced_object(object) &&
1601 !(object->flags & OBJECT_REPORTED)) {
1602 object->flags |= OBJECT_REPORTED;
1603
1604 if (kmemleak_verbose)
1605 print_unreferenced(NULL, object);
1606
1607 new_leaks++;
1608 }
1609 spin_unlock_irqrestore(&object->lock, flags);
1610 }
1611 rcu_read_unlock();
1612
1613 if (new_leaks) {
1614 kmemleak_found_leaks = true;
1615
1616 pr_info("%d new suspected memory leaks (see /sys/kernel/debug/kmemleak)\n",
1617 new_leaks);
1618 }
1619
1620}
1621
1622
1623
1624
1625
1626static int kmemleak_scan_thread(void *arg)
1627{
1628 static int first_run = IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN);
1629
1630 pr_info("Automatic memory scanning thread started\n");
1631 set_user_nice(current, 10);
1632
1633
1634
1635
1636 if (first_run) {
1637 signed long timeout = msecs_to_jiffies(SECS_FIRST_SCAN * 1000);
1638 first_run = 0;
1639 while (timeout && !kthread_should_stop())
1640 timeout = schedule_timeout_interruptible(timeout);
1641 }
1642
1643 while (!kthread_should_stop()) {
1644 signed long timeout = jiffies_scan_wait;
1645
1646 mutex_lock(&scan_mutex);
1647 kmemleak_scan();
1648 mutex_unlock(&scan_mutex);
1649
1650
1651 while (timeout && !kthread_should_stop())
1652 timeout = schedule_timeout_interruptible(timeout);
1653 }
1654
1655 pr_info("Automatic memory scanning thread ended\n");
1656
1657 return 0;
1658}
1659
1660
1661
1662
1663
1664static void start_scan_thread(void)
1665{
1666 if (scan_thread)
1667 return;
1668 scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
1669 if (IS_ERR(scan_thread)) {
1670 pr_warn("Failed to create the scan thread\n");
1671 scan_thread = NULL;
1672 }
1673}
1674
1675
1676
1677
1678static void stop_scan_thread(void)
1679{
1680 if (scan_thread) {
1681 kthread_stop(scan_thread);
1682 scan_thread = NULL;
1683 }
1684}
1685
1686
1687
1688
1689
1690
1691static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
1692{
1693 struct kmemleak_object *object;
1694 loff_t n = *pos;
1695 int err;
1696
1697 err = mutex_lock_interruptible(&scan_mutex);
1698 if (err < 0)
1699 return ERR_PTR(err);
1700
1701 rcu_read_lock();
1702 list_for_each_entry_rcu(object, &object_list, object_list) {
1703 if (n-- > 0)
1704 continue;
1705 if (get_object(object))
1706 goto out;
1707 }
1708 object = NULL;
1709out:
1710 return object;
1711}
1712
1713
1714
1715
1716
1717static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1718{
1719 struct kmemleak_object *prev_obj = v;
1720 struct kmemleak_object *next_obj = NULL;
1721 struct kmemleak_object *obj = prev_obj;
1722
1723 ++(*pos);
1724
1725 list_for_each_entry_continue_rcu(obj, &object_list, object_list) {
1726 if (get_object(obj)) {
1727 next_obj = obj;
1728 break;
1729 }
1730 }
1731
1732 put_object(prev_obj);
1733 return next_obj;
1734}
1735
1736
1737
1738
1739static void kmemleak_seq_stop(struct seq_file *seq, void *v)
1740{
1741 if (!IS_ERR(v)) {
1742
1743
1744
1745
1746 rcu_read_unlock();
1747 mutex_unlock(&scan_mutex);
1748 if (v)
1749 put_object(v);
1750 }
1751}
1752
1753
1754
1755
1756static int kmemleak_seq_show(struct seq_file *seq, void *v)
1757{
1758 struct kmemleak_object *object = v;
1759 unsigned long flags;
1760
1761 spin_lock_irqsave(&object->lock, flags);
1762 if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
1763 print_unreferenced(seq, object);
1764 spin_unlock_irqrestore(&object->lock, flags);
1765 return 0;
1766}
1767
1768static const struct seq_operations kmemleak_seq_ops = {
1769 .start = kmemleak_seq_start,
1770 .next = kmemleak_seq_next,
1771 .stop = kmemleak_seq_stop,
1772 .show = kmemleak_seq_show,
1773};
1774
1775static int kmemleak_open(struct inode *inode, struct file *file)
1776{
1777 return seq_open(file, &kmemleak_seq_ops);
1778}
1779
1780static int dump_str_object_info(const char *str)
1781{
1782 unsigned long flags;
1783 struct kmemleak_object *object;
1784 unsigned long addr;
1785
1786 if (kstrtoul(str, 0, &addr))
1787 return -EINVAL;
1788 object = find_and_get_object(addr, 0);
1789 if (!object) {
1790 pr_info("Unknown object at 0x%08lx\n", addr);
1791 return -EINVAL;
1792 }
1793
1794 spin_lock_irqsave(&object->lock, flags);
1795 dump_object_info(object);
1796 spin_unlock_irqrestore(&object->lock, flags);
1797
1798 put_object(object);
1799 return 0;
1800}
1801
1802
1803
1804
1805
1806
1807
1808static void kmemleak_clear(void)
1809{
1810 struct kmemleak_object *object;
1811 unsigned long flags;
1812
1813 rcu_read_lock();
1814 list_for_each_entry_rcu(object, &object_list, object_list) {
1815 spin_lock_irqsave(&object->lock, flags);
1816 if ((object->flags & OBJECT_REPORTED) &&
1817 unreferenced_object(object))
1818 __paint_it(object, KMEMLEAK_GREY);
1819 spin_unlock_irqrestore(&object->lock, flags);
1820 }
1821 rcu_read_unlock();
1822
1823 kmemleak_found_leaks = false;
1824}
1825
1826static void __kmemleak_do_cleanup(void);
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
1845 size_t size, loff_t *ppos)
1846{
1847 char buf[64];
1848 int buf_size;
1849 int ret;
1850
1851 buf_size = min(size, (sizeof(buf) - 1));
1852 if (strncpy_from_user(buf, user_buf, buf_size) < 0)
1853 return -EFAULT;
1854 buf[buf_size] = 0;
1855
1856 ret = mutex_lock_interruptible(&scan_mutex);
1857 if (ret < 0)
1858 return ret;
1859
1860 if (strncmp(buf, "clear", 5) == 0) {
1861 if (kmemleak_enabled)
1862 kmemleak_clear();
1863 else
1864 __kmemleak_do_cleanup();
1865 goto out;
1866 }
1867
1868 if (!kmemleak_enabled) {
1869 ret = -EPERM;
1870 goto out;
1871 }
1872
1873 if (strncmp(buf, "off", 3) == 0)
1874 kmemleak_disable();
1875 else if (strncmp(buf, "stack=on", 8) == 0)
1876 kmemleak_stack_scan = 1;
1877 else if (strncmp(buf, "stack=off", 9) == 0)
1878 kmemleak_stack_scan = 0;
1879 else if (strncmp(buf, "scan=on", 7) == 0)
1880 start_scan_thread();
1881 else if (strncmp(buf, "scan=off", 8) == 0)
1882 stop_scan_thread();
1883 else if (strncmp(buf, "scan=", 5) == 0) {
1884 unsigned long secs;
1885
1886 ret = kstrtoul(buf + 5, 0, &secs);
1887 if (ret < 0)
1888 goto out;
1889 stop_scan_thread();
1890 if (secs) {
1891 jiffies_scan_wait = msecs_to_jiffies(secs * 1000);
1892 start_scan_thread();
1893 }
1894 } else if (strncmp(buf, "scan", 4) == 0)
1895 kmemleak_scan();
1896 else if (strncmp(buf, "dump=", 5) == 0)
1897 ret = dump_str_object_info(buf + 5);
1898 else
1899 ret = -EINVAL;
1900
1901out:
1902 mutex_unlock(&scan_mutex);
1903 if (ret < 0)
1904 return ret;
1905
1906
1907 *ppos += size;
1908 return size;
1909}
1910
1911static const struct file_operations kmemleak_fops = {
1912 .owner = THIS_MODULE,
1913 .open = kmemleak_open,
1914 .read = seq_read,
1915 .write = kmemleak_write,
1916 .llseek = seq_lseek,
1917 .release = seq_release,
1918};
1919
1920static void __kmemleak_do_cleanup(void)
1921{
1922 struct kmemleak_object *object;
1923
1924 rcu_read_lock();
1925 list_for_each_entry_rcu(object, &object_list, object_list)
1926 delete_object_full(object->pointer);
1927 rcu_read_unlock();
1928}
1929
1930
1931
1932
1933
1934
1935static void kmemleak_do_cleanup(struct work_struct *work)
1936{
1937 stop_scan_thread();
1938
1939 mutex_lock(&scan_mutex);
1940
1941
1942
1943
1944
1945
1946 kmemleak_free_enabled = 0;
1947 mutex_unlock(&scan_mutex);
1948
1949 if (!kmemleak_found_leaks)
1950 __kmemleak_do_cleanup();
1951 else
1952 pr_info("Kmemleak disabled without freeing internal data. Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\".\n");
1953}
1954
1955static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
1956
1957
1958
1959
1960
1961static void kmemleak_disable(void)
1962{
1963
1964 if (cmpxchg(&kmemleak_error, 0, 1))
1965 return;
1966
1967
1968 kmemleak_enabled = 0;
1969 kmemleak_early_log = 0;
1970
1971
1972 if (kmemleak_initialized)
1973 schedule_work(&cleanup_work);
1974 else
1975 kmemleak_free_enabled = 0;
1976
1977 pr_info("Kernel memory leak detector disabled\n");
1978}
1979
1980
1981
1982
1983static int __init kmemleak_boot_config(char *str)
1984{
1985 if (!str)
1986 return -EINVAL;
1987 if (strcmp(str, "off") == 0)
1988 kmemleak_disable();
1989 else if (strcmp(str, "on") == 0)
1990 kmemleak_skip_disable = 1;
1991 else
1992 return -EINVAL;
1993 return 0;
1994}
1995early_param("kmemleak", kmemleak_boot_config);
1996
1997static void __init print_log_trace(struct early_log *log)
1998{
1999 pr_notice("Early log backtrace:\n");
2000 stack_trace_print(log->trace, log->trace_len, 2);
2001}
2002
2003
2004
2005
2006void __init kmemleak_init(void)
2007{
2008 int i;
2009 unsigned long flags;
2010
2011#ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
2012 if (!kmemleak_skip_disable) {
2013 kmemleak_disable();
2014 return;
2015 }
2016#endif
2017
2018 jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
2019 jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
2020
2021 object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
2022 scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
2023
2024 if (crt_early_log > ARRAY_SIZE(early_log))
2025 pr_warn("Early log buffer exceeded (%d), please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n",
2026 crt_early_log);
2027
2028
2029 local_irq_save(flags);
2030 kmemleak_early_log = 0;
2031 if (kmemleak_error) {
2032 local_irq_restore(flags);
2033 return;
2034 } else {
2035 kmemleak_enabled = 1;
2036 kmemleak_free_enabled = 1;
2037 }
2038 local_irq_restore(flags);
2039
2040
2041 create_object((unsigned long)_sdata, _edata - _sdata,
2042 KMEMLEAK_GREY, GFP_ATOMIC);
2043 create_object((unsigned long)__bss_start, __bss_stop - __bss_start,
2044 KMEMLEAK_GREY, GFP_ATOMIC);
2045
2046 if (__start_ro_after_init < _sdata || __end_ro_after_init > _edata)
2047 create_object((unsigned long)__start_ro_after_init,
2048 __end_ro_after_init - __start_ro_after_init,
2049 KMEMLEAK_GREY, GFP_ATOMIC);
2050
2051
2052
2053
2054
2055
2056 for (i = 0; i < crt_early_log; i++) {
2057 struct early_log *log = &early_log[i];
2058
2059 switch (log->op_type) {
2060 case KMEMLEAK_ALLOC:
2061 early_alloc(log);
2062 break;
2063 case KMEMLEAK_ALLOC_PERCPU:
2064 early_alloc_percpu(log);
2065 break;
2066 case KMEMLEAK_FREE:
2067 kmemleak_free(log->ptr);
2068 break;
2069 case KMEMLEAK_FREE_PART:
2070 kmemleak_free_part(log->ptr, log->size);
2071 break;
2072 case KMEMLEAK_FREE_PERCPU:
2073 kmemleak_free_percpu(log->ptr);
2074 break;
2075 case KMEMLEAK_NOT_LEAK:
2076 kmemleak_not_leak(log->ptr);
2077 break;
2078 case KMEMLEAK_IGNORE:
2079 kmemleak_ignore(log->ptr);
2080 break;
2081 case KMEMLEAK_SCAN_AREA:
2082 kmemleak_scan_area(log->ptr, log->size, GFP_KERNEL);
2083 break;
2084 case KMEMLEAK_NO_SCAN:
2085 kmemleak_no_scan(log->ptr);
2086 break;
2087 case KMEMLEAK_SET_EXCESS_REF:
2088 object_set_excess_ref((unsigned long)log->ptr,
2089 log->excess_ref);
2090 break;
2091 default:
2092 kmemleak_warn("Unknown early log operation: %d\n",
2093 log->op_type);
2094 }
2095
2096 if (kmemleak_warning) {
2097 print_log_trace(log);
2098 kmemleak_warning = 0;
2099 }
2100 }
2101}
2102
2103
2104
2105
2106static int __init kmemleak_late_init(void)
2107{
2108 kmemleak_initialized = 1;
2109
2110 debugfs_create_file("kmemleak", 0644, NULL, NULL, &kmemleak_fops);
2111
2112 if (kmemleak_error) {
2113
2114
2115
2116
2117
2118
2119 schedule_work(&cleanup_work);
2120 return -ENOMEM;
2121 }
2122
2123 if (IS_ENABLED(CONFIG_DEBUG_KMEMLEAK_AUTO_SCAN)) {
2124 mutex_lock(&scan_mutex);
2125 start_scan_thread();
2126 mutex_unlock(&scan_mutex);
2127 }
2128
2129 pr_info("Kernel memory leak detector initialized\n");
2130
2131 return 0;
2132}
2133late_initcall(kmemleak_late_init);
2134