1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
72
73#include <linux/init.h>
74#include <linux/kernel.h>
75#include <linux/list.h>
76#include <linux/sched/signal.h>
77#include <linux/sched/task.h>
78#include <linux/sched/task_stack.h>
79#include <linux/jiffies.h>
80#include <linux/delay.h>
81#include <linux/export.h>
82#include <linux/kthread.h>
83#include <linux/rbtree.h>
84#include <linux/fs.h>
85#include <linux/debugfs.h>
86#include <linux/seq_file.h>
87#include <linux/cpumask.h>
88#include <linux/spinlock.h>
89#include <linux/mutex.h>
90#include <linux/rcupdate.h>
91#include <linux/stacktrace.h>
92#include <linux/cache.h>
93#include <linux/percpu.h>
94#include <linux/bootmem.h>
95#include <linux/pfn.h>
96#include <linux/mmzone.h>
97#include <linux/slab.h>
98#include <linux/thread_info.h>
99#include <linux/err.h>
100#include <linux/uaccess.h>
101#include <linux/string.h>
102#include <linux/nodemask.h>
103#include <linux/mm.h>
104#include <linux/workqueue.h>
105#include <linux/crc32.h>
106
107#include <asm/sections.h>
108#include <asm/processor.h>
109#include <linux/atomic.h>
110
111#include <linux/kasan.h>
112#include <linux/kmemleak.h>
113#include <linux/memory_hotplug.h>
114
115
116
117
118#define MAX_TRACE 16
119#define MSECS_MIN_AGE 5000
120#define SECS_FIRST_SCAN 60
121#define SECS_SCAN_WAIT 600
122#define MAX_SCAN_SIZE 4096
123
124#define BYTES_PER_POINTER sizeof(void *)
125
126
127#define gfp_kmemleak_mask(gfp) (((gfp) & (GFP_KERNEL | GFP_ATOMIC)) | \
128 __GFP_NORETRY | __GFP_NOMEMALLOC | \
129 __GFP_NOWARN | __GFP_NOFAIL)
130
131
132struct kmemleak_scan_area {
133 struct hlist_node node;
134 unsigned long start;
135 size_t size;
136};
137
138#define KMEMLEAK_GREY 0
139#define KMEMLEAK_BLACK -1
140
141
142
143
144
145
146
147
148
149struct kmemleak_object {
150 spinlock_t lock;
151 unsigned int flags;
152 struct list_head object_list;
153 struct list_head gray_list;
154 struct rb_node rb_node;
155 struct rcu_head rcu;
156
157 atomic_t use_count;
158 unsigned long pointer;
159 size_t size;
160
161 unsigned long excess_ref;
162
163 int min_count;
164
165 int count;
166
167 u32 checksum;
168
169 struct hlist_head area_list;
170 unsigned long trace[MAX_TRACE];
171 unsigned int trace_len;
172 unsigned long jiffies;
173 pid_t pid;
174 char comm[TASK_COMM_LEN];
175};
176
177
178#define OBJECT_ALLOCATED (1 << 0)
179
180#define OBJECT_REPORTED (1 << 1)
181
182#define OBJECT_NO_SCAN (1 << 2)
183
184
185#define HEX_ROW_SIZE 16
186
187#define HEX_GROUP_SIZE 1
188
189#define HEX_ASCII 1
190
191#define HEX_MAX_LINES 2
192
193
194static LIST_HEAD(object_list);
195
196static LIST_HEAD(gray_list);
197
198static struct rb_root object_tree_root = RB_ROOT;
199
200static DEFINE_RWLOCK(kmemleak_lock);
201
202
203static struct kmem_cache *object_cache;
204static struct kmem_cache *scan_area_cache;
205
206
207static int kmemleak_enabled;
208
209static int kmemleak_free_enabled;
210
211static int kmemleak_initialized;
212
213static int kmemleak_early_log = 1;
214
215static int kmemleak_warning;
216
217static int kmemleak_error;
218
219
220static unsigned long min_addr = ULONG_MAX;
221static unsigned long max_addr;
222
223static struct task_struct *scan_thread;
224
225static unsigned long jiffies_min_age;
226static unsigned long jiffies_last_scan;
227
228static signed long jiffies_scan_wait;
229
230static int kmemleak_stack_scan = 1;
231
232static DEFINE_MUTEX(scan_mutex);
233
234static int kmemleak_skip_disable;
235
236static bool kmemleak_found_leaks;
237
238
239
240
241
242
243
244
245
246
247enum {
248 KMEMLEAK_ALLOC,
249 KMEMLEAK_ALLOC_PERCPU,
250 KMEMLEAK_FREE,
251 KMEMLEAK_FREE_PART,
252 KMEMLEAK_FREE_PERCPU,
253 KMEMLEAK_NOT_LEAK,
254 KMEMLEAK_IGNORE,
255 KMEMLEAK_SCAN_AREA,
256 KMEMLEAK_NO_SCAN,
257 KMEMLEAK_SET_EXCESS_REF
258};
259
260
261
262
263
264struct early_log {
265 int op_type;
266 int min_count;
267 const void *ptr;
268 union {
269 size_t size;
270 unsigned long excess_ref;
271 };
272 unsigned long trace[MAX_TRACE];
273 unsigned int trace_len;
274};
275
276
277static struct early_log
278 early_log[CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE] __initdata;
279static int crt_early_log __initdata;
280
281static void kmemleak_disable(void);
282
283
284
285
286#define kmemleak_warn(x...) do { \
287 pr_warn(x); \
288 dump_stack(); \
289 kmemleak_warning = 1; \
290} while (0)
291
292
293
294
295
296
297#define kmemleak_stop(x...) do { \
298 kmemleak_warn(x); \
299 kmemleak_disable(); \
300} while (0)
301
302
303
304
305
306
307
308static void hex_dump_object(struct seq_file *seq,
309 struct kmemleak_object *object)
310{
311 const u8 *ptr = (const u8 *)object->pointer;
312 size_t len;
313
314
315 len = min_t(size_t, object->size, HEX_MAX_LINES * HEX_ROW_SIZE);
316
317 seq_printf(seq, " hex dump (first %zu bytes):\n", len);
318 kasan_disable_current();
319 seq_hex_dump(seq, " ", DUMP_PREFIX_NONE, HEX_ROW_SIZE,
320 HEX_GROUP_SIZE, ptr, len, HEX_ASCII);
321 kasan_enable_current();
322}
323
324
325
326
327
328
329
330
331
332
333
334static bool color_white(const struct kmemleak_object *object)
335{
336 return object->count != KMEMLEAK_BLACK &&
337 object->count < object->min_count;
338}
339
340static bool color_gray(const struct kmemleak_object *object)
341{
342 return object->min_count != KMEMLEAK_BLACK &&
343 object->count >= object->min_count;
344}
345
346
347
348
349
350
351static bool unreferenced_object(struct kmemleak_object *object)
352{
353 return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
354 time_before_eq(object->jiffies + jiffies_min_age,
355 jiffies_last_scan);
356}
357
358
359
360
361
362static void print_unreferenced(struct seq_file *seq,
363 struct kmemleak_object *object)
364{
365 int i;
366 unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);
367
368 seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
369 object->pointer, object->size);
370 seq_printf(seq, " comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
371 object->comm, object->pid, object->jiffies,
372 msecs_age / 1000, msecs_age % 1000);
373 hex_dump_object(seq, object);
374 seq_printf(seq, " backtrace:\n");
375
376 for (i = 0; i < object->trace_len; i++) {
377 void *ptr = (void *)object->trace[i];
378 seq_printf(seq, " [<%p>] %pS\n", ptr, ptr);
379 }
380}
381
382
383
384
385
386
387static void dump_object_info(struct kmemleak_object *object)
388{
389 struct stack_trace trace;
390
391 trace.nr_entries = object->trace_len;
392 trace.entries = object->trace;
393
394 pr_notice("Object 0x%08lx (size %zu):\n",
395 object->pointer, object->size);
396 pr_notice(" comm \"%s\", pid %d, jiffies %lu\n",
397 object->comm, object->pid, object->jiffies);
398 pr_notice(" min_count = %d\n", object->min_count);
399 pr_notice(" count = %d\n", object->count);
400 pr_notice(" flags = 0x%x\n", object->flags);
401 pr_notice(" checksum = %u\n", object->checksum);
402 pr_notice(" backtrace:\n");
403 print_stack_trace(&trace, 4);
404}
405
406
407
408
409
410
411
412static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
413{
414 struct rb_node *rb = object_tree_root.rb_node;
415
416 while (rb) {
417 struct kmemleak_object *object =
418 rb_entry(rb, struct kmemleak_object, rb_node);
419 if (ptr < object->pointer)
420 rb = object->rb_node.rb_left;
421 else if (object->pointer + object->size <= ptr)
422 rb = object->rb_node.rb_right;
423 else if (object->pointer == ptr || alias)
424 return object;
425 else {
426 kmemleak_warn("Found object by alias at 0x%08lx\n",
427 ptr);
428 dump_object_info(object);
429 break;
430 }
431 }
432 return NULL;
433}
434
435
436
437
438
439
440
441static int get_object(struct kmemleak_object *object)
442{
443 return atomic_inc_not_zero(&object->use_count);
444}
445
446
447
448
449static void free_object_rcu(struct rcu_head *rcu)
450{
451 struct hlist_node *tmp;
452 struct kmemleak_scan_area *area;
453 struct kmemleak_object *object =
454 container_of(rcu, struct kmemleak_object, rcu);
455
456
457
458
459
460 hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {
461 hlist_del(&area->node);
462 kmem_cache_free(scan_area_cache, area);
463 }
464 kmem_cache_free(object_cache, object);
465}
466
467
468
469
470
471
472
473
474static void put_object(struct kmemleak_object *object)
475{
476 if (!atomic_dec_and_test(&object->use_count))
477 return;
478
479
480 WARN_ON(object->flags & OBJECT_ALLOCATED);
481
482 call_rcu(&object->rcu, free_object_rcu);
483}
484
485
486
487
488static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
489{
490 unsigned long flags;
491 struct kmemleak_object *object;
492
493 rcu_read_lock();
494 read_lock_irqsave(&kmemleak_lock, flags);
495 object = lookup_object(ptr, alias);
496 read_unlock_irqrestore(&kmemleak_lock, flags);
497
498
499 if (object && !get_object(object))
500 object = NULL;
501 rcu_read_unlock();
502
503 return object;
504}
505
506
507
508
509
510
511static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int alias)
512{
513 unsigned long flags;
514 struct kmemleak_object *object;
515
516 write_lock_irqsave(&kmemleak_lock, flags);
517 object = lookup_object(ptr, alias);
518 if (object) {
519 rb_erase(&object->rb_node, &object_tree_root);
520 list_del_rcu(&object->object_list);
521 }
522 write_unlock_irqrestore(&kmemleak_lock, flags);
523
524 return object;
525}
526
527
528
529
530static int __save_stack_trace(unsigned long *trace)
531{
532 struct stack_trace stack_trace;
533
534 stack_trace.max_entries = MAX_TRACE;
535 stack_trace.nr_entries = 0;
536 stack_trace.entries = trace;
537 stack_trace.skip = 2;
538 save_stack_trace(&stack_trace);
539
540 return stack_trace.nr_entries;
541}
542
543
544
545
546
547static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
548 int min_count, gfp_t gfp)
549{
550 unsigned long flags;
551 struct kmemleak_object *object, *parent;
552 struct rb_node **link, *rb_parent;
553
554 object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
555 if (!object) {
556 pr_warn("Cannot allocate a kmemleak_object structure\n");
557 kmemleak_disable();
558 return NULL;
559 }
560
561 INIT_LIST_HEAD(&object->object_list);
562 INIT_LIST_HEAD(&object->gray_list);
563 INIT_HLIST_HEAD(&object->area_list);
564 spin_lock_init(&object->lock);
565 atomic_set(&object->use_count, 1);
566 object->flags = OBJECT_ALLOCATED;
567 object->pointer = ptr;
568 object->size = size;
569 object->excess_ref = 0;
570 object->min_count = min_count;
571 object->count = 0;
572 object->jiffies = jiffies;
573 object->checksum = 0;
574
575
576 if (in_irq()) {
577 object->pid = 0;
578 strncpy(object->comm, "hardirq", sizeof(object->comm));
579 } else if (in_softirq()) {
580 object->pid = 0;
581 strncpy(object->comm, "softirq", sizeof(object->comm));
582 } else {
583 object->pid = current->pid;
584
585
586
587
588
589
590 strncpy(object->comm, current->comm, sizeof(object->comm));
591 }
592
593
594 object->trace_len = __save_stack_trace(object->trace);
595
596 write_lock_irqsave(&kmemleak_lock, flags);
597
598 min_addr = min(min_addr, ptr);
599 max_addr = max(max_addr, ptr + size);
600 link = &object_tree_root.rb_node;
601 rb_parent = NULL;
602 while (*link) {
603 rb_parent = *link;
604 parent = rb_entry(rb_parent, struct kmemleak_object, rb_node);
605 if (ptr + size <= parent->pointer)
606 link = &parent->rb_node.rb_left;
607 else if (parent->pointer + parent->size <= ptr)
608 link = &parent->rb_node.rb_right;
609 else {
610 kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
611 ptr);
612
613
614
615
616 dump_object_info(parent);
617 kmem_cache_free(object_cache, object);
618 object = NULL;
619 goto out;
620 }
621 }
622 rb_link_node(&object->rb_node, rb_parent, link);
623 rb_insert_color(&object->rb_node, &object_tree_root);
624
625 list_add_tail_rcu(&object->object_list, &object_list);
626out:
627 write_unlock_irqrestore(&kmemleak_lock, flags);
628 return object;
629}
630
631
632
633
634static void __delete_object(struct kmemleak_object *object)
635{
636 unsigned long flags;
637
638 WARN_ON(!(object->flags & OBJECT_ALLOCATED));
639 WARN_ON(atomic_read(&object->use_count) < 1);
640
641
642
643
644
645 spin_lock_irqsave(&object->lock, flags);
646 object->flags &= ~OBJECT_ALLOCATED;
647 spin_unlock_irqrestore(&object->lock, flags);
648 put_object(object);
649}
650
651
652
653
654
655static void delete_object_full(unsigned long ptr)
656{
657 struct kmemleak_object *object;
658
659 object = find_and_remove_object(ptr, 0);
660 if (!object) {
661#ifdef DEBUG
662 kmemleak_warn("Freeing unknown object at 0x%08lx\n",
663 ptr);
664#endif
665 return;
666 }
667 __delete_object(object);
668}
669
670
671
672
673
674
675static void delete_object_part(unsigned long ptr, size_t size)
676{
677 struct kmemleak_object *object;
678 unsigned long start, end;
679
680 object = find_and_remove_object(ptr, 1);
681 if (!object) {
682#ifdef DEBUG
683 kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
684 ptr, size);
685#endif
686 return;
687 }
688
689
690
691
692
693
694
695
696 start = object->pointer;
697 end = object->pointer + object->size;
698 if (ptr > start)
699 create_object(start, ptr - start, object->min_count,
700 GFP_KERNEL);
701 if (ptr + size < end)
702 create_object(ptr + size, end - ptr - size, object->min_count,
703 GFP_KERNEL);
704
705 __delete_object(object);
706}
707
708static void __paint_it(struct kmemleak_object *object, int color)
709{
710 object->min_count = color;
711 if (color == KMEMLEAK_BLACK)
712 object->flags |= OBJECT_NO_SCAN;
713}
714
715static void paint_it(struct kmemleak_object *object, int color)
716{
717 unsigned long flags;
718
719 spin_lock_irqsave(&object->lock, flags);
720 __paint_it(object, color);
721 spin_unlock_irqrestore(&object->lock, flags);
722}
723
724static void paint_ptr(unsigned long ptr, int color)
725{
726 struct kmemleak_object *object;
727
728 object = find_and_get_object(ptr, 0);
729 if (!object) {
730 kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
731 ptr,
732 (color == KMEMLEAK_GREY) ? "Grey" :
733 (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
734 return;
735 }
736 paint_it(object, color);
737 put_object(object);
738}
739
740
741
742
743
744static void make_gray_object(unsigned long ptr)
745{
746 paint_ptr(ptr, KMEMLEAK_GREY);
747}
748
749
750
751
752
753static void make_black_object(unsigned long ptr)
754{
755 paint_ptr(ptr, KMEMLEAK_BLACK);
756}
757
758
759
760
761
762static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
763{
764 unsigned long flags;
765 struct kmemleak_object *object;
766 struct kmemleak_scan_area *area;
767
768 object = find_and_get_object(ptr, 1);
769 if (!object) {
770 kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
771 ptr);
772 return;
773 }
774
775 area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
776 if (!area) {
777 pr_warn("Cannot allocate a scan area\n");
778 goto out;
779 }
780
781 spin_lock_irqsave(&object->lock, flags);
782 if (size == SIZE_MAX) {
783 size = object->pointer + object->size - ptr;
784 } else if (ptr + size > object->pointer + object->size) {
785 kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
786 dump_object_info(object);
787 kmem_cache_free(scan_area_cache, area);
788 goto out_unlock;
789 }
790
791 INIT_HLIST_NODE(&area->node);
792 area->start = ptr;
793 area->size = size;
794
795 hlist_add_head(&area->node, &object->area_list);
796out_unlock:
797 spin_unlock_irqrestore(&object->lock, flags);
798out:
799 put_object(object);
800}
801
802
803
804
805
806
807
808static void object_set_excess_ref(unsigned long ptr, unsigned long excess_ref)
809{
810 unsigned long flags;
811 struct kmemleak_object *object;
812
813 object = find_and_get_object(ptr, 0);
814 if (!object) {
815 kmemleak_warn("Setting excess_ref on unknown object at 0x%08lx\n",
816 ptr);
817 return;
818 }
819
820 spin_lock_irqsave(&object->lock, flags);
821 object->excess_ref = excess_ref;
822 spin_unlock_irqrestore(&object->lock, flags);
823 put_object(object);
824}
825
826
827
828
829
830
831static void object_no_scan(unsigned long ptr)
832{
833 unsigned long flags;
834 struct kmemleak_object *object;
835
836 object = find_and_get_object(ptr, 0);
837 if (!object) {
838 kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
839 return;
840 }
841
842 spin_lock_irqsave(&object->lock, flags);
843 object->flags |= OBJECT_NO_SCAN;
844 spin_unlock_irqrestore(&object->lock, flags);
845 put_object(object);
846}
847
848
849
850
851
852static void __init log_early(int op_type, const void *ptr, size_t size,
853 int min_count)
854{
855 unsigned long flags;
856 struct early_log *log;
857
858 if (kmemleak_error) {
859
860 crt_early_log++;
861 return;
862 }
863
864 if (crt_early_log >= ARRAY_SIZE(early_log)) {
865 crt_early_log++;
866 kmemleak_disable();
867 return;
868 }
869
870
871
872
873
874 local_irq_save(flags);
875 log = &early_log[crt_early_log];
876 log->op_type = op_type;
877 log->ptr = ptr;
878 log->size = size;
879 log->min_count = min_count;
880 log->trace_len = __save_stack_trace(log->trace);
881 crt_early_log++;
882 local_irq_restore(flags);
883}
884
885
886
887
888static void early_alloc(struct early_log *log)
889{
890 struct kmemleak_object *object;
891 unsigned long flags;
892 int i;
893
894 if (!kmemleak_enabled || !log->ptr || IS_ERR(log->ptr))
895 return;
896
897
898
899
900 rcu_read_lock();
901 object = create_object((unsigned long)log->ptr, log->size,
902 log->min_count, GFP_ATOMIC);
903 if (!object)
904 goto out;
905 spin_lock_irqsave(&object->lock, flags);
906 for (i = 0; i < log->trace_len; i++)
907 object->trace[i] = log->trace[i];
908 object->trace_len = log->trace_len;
909 spin_unlock_irqrestore(&object->lock, flags);
910out:
911 rcu_read_unlock();
912}
913
914
915
916
917static void early_alloc_percpu(struct early_log *log)
918{
919 unsigned int cpu;
920 const void __percpu *ptr = log->ptr;
921
922 for_each_possible_cpu(cpu) {
923 log->ptr = per_cpu_ptr(ptr, cpu);
924 early_alloc(log);
925 }
926}
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
943 gfp_t gfp)
944{
945 pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);
946
947 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
948 create_object((unsigned long)ptr, size, min_count, gfp);
949 else if (kmemleak_early_log)
950 log_early(KMEMLEAK_ALLOC, ptr, size, min_count);
951}
952EXPORT_SYMBOL_GPL(kmemleak_alloc);
953
954
955
956
957
958
959
960
961
962
963void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size,
964 gfp_t gfp)
965{
966 unsigned int cpu;
967
968 pr_debug("%s(0x%p, %zu)\n", __func__, ptr, size);
969
970
971
972
973
974 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
975 for_each_possible_cpu(cpu)
976 create_object((unsigned long)per_cpu_ptr(ptr, cpu),
977 size, 0, gfp);
978 else if (kmemleak_early_log)
979 log_early(KMEMLEAK_ALLOC_PERCPU, ptr, size, 0);
980}
981EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
982
983
984
985
986
987
988
989
990
991
992void __ref kmemleak_vmalloc(const struct vm_struct *area, size_t size, gfp_t gfp)
993{
994 pr_debug("%s(0x%p, %zu)\n", __func__, area, size);
995
996
997
998
999
1000 if (kmemleak_enabled) {
1001 create_object((unsigned long)area->addr, size, 2, gfp);
1002 object_set_excess_ref((unsigned long)area,
1003 (unsigned long)area->addr);
1004 } else if (kmemleak_early_log) {
1005 log_early(KMEMLEAK_ALLOC, area->addr, size, 2);
1006
1007 log_early(KMEMLEAK_SET_EXCESS_REF,
1008 area, (unsigned long)area->addr, 0);
1009 }
1010}
1011EXPORT_SYMBOL_GPL(kmemleak_vmalloc);
1012
1013
1014
1015
1016
1017
1018
1019
1020void __ref kmemleak_free(const void *ptr)
1021{
1022 pr_debug("%s(0x%p)\n", __func__, ptr);
1023
1024 if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
1025 delete_object_full((unsigned long)ptr);
1026 else if (kmemleak_early_log)
1027 log_early(KMEMLEAK_FREE, ptr, 0, 0);
1028}
1029EXPORT_SYMBOL_GPL(kmemleak_free);
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040void __ref kmemleak_free_part(const void *ptr, size_t size)
1041{
1042 pr_debug("%s(0x%p)\n", __func__, ptr);
1043
1044 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1045 delete_object_part((unsigned long)ptr, size);
1046 else if (kmemleak_early_log)
1047 log_early(KMEMLEAK_FREE_PART, ptr, size, 0);
1048}
1049EXPORT_SYMBOL_GPL(kmemleak_free_part);
1050
1051
1052
1053
1054
1055
1056
1057
1058void __ref kmemleak_free_percpu(const void __percpu *ptr)
1059{
1060 unsigned int cpu;
1061
1062 pr_debug("%s(0x%p)\n", __func__, ptr);
1063
1064 if (kmemleak_free_enabled && ptr && !IS_ERR(ptr))
1065 for_each_possible_cpu(cpu)
1066 delete_object_full((unsigned long)per_cpu_ptr(ptr,
1067 cpu));
1068 else if (kmemleak_early_log)
1069 log_early(KMEMLEAK_FREE_PERCPU, ptr, 0, 0);
1070}
1071EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
1072
1073
1074
1075
1076
1077
1078
1079
1080void __ref kmemleak_update_trace(const void *ptr)
1081{
1082 struct kmemleak_object *object;
1083 unsigned long flags;
1084
1085 pr_debug("%s(0x%p)\n", __func__, ptr);
1086
1087 if (!kmemleak_enabled || IS_ERR_OR_NULL(ptr))
1088 return;
1089
1090 object = find_and_get_object((unsigned long)ptr, 1);
1091 if (!object) {
1092#ifdef DEBUG
1093 kmemleak_warn("Updating stack trace for unknown object at %p\n",
1094 ptr);
1095#endif
1096 return;
1097 }
1098
1099 spin_lock_irqsave(&object->lock, flags);
1100 object->trace_len = __save_stack_trace(object->trace);
1101 spin_unlock_irqrestore(&object->lock, flags);
1102
1103 put_object(object);
1104}
1105EXPORT_SYMBOL(kmemleak_update_trace);
1106
1107
1108
1109
1110
1111
1112
1113
1114void __ref kmemleak_not_leak(const void *ptr)
1115{
1116 pr_debug("%s(0x%p)\n", __func__, ptr);
1117
1118 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1119 make_gray_object((unsigned long)ptr);
1120 else if (kmemleak_early_log)
1121 log_early(KMEMLEAK_NOT_LEAK, ptr, 0, 0);
1122}
1123EXPORT_SYMBOL(kmemleak_not_leak);
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134void __ref kmemleak_ignore(const void *ptr)
1135{
1136 pr_debug("%s(0x%p)\n", __func__, ptr);
1137
1138 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1139 make_black_object((unsigned long)ptr);
1140 else if (kmemleak_early_log)
1141 log_early(KMEMLEAK_IGNORE, ptr, 0, 0);
1142}
1143EXPORT_SYMBOL(kmemleak_ignore);
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
1157{
1158 pr_debug("%s(0x%p)\n", __func__, ptr);
1159
1160 if (kmemleak_enabled && ptr && size && !IS_ERR(ptr))
1161 add_scan_area((unsigned long)ptr, size, gfp);
1162 else if (kmemleak_early_log)
1163 log_early(KMEMLEAK_SCAN_AREA, ptr, size, 0);
1164}
1165EXPORT_SYMBOL(kmemleak_scan_area);
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176void __ref kmemleak_no_scan(const void *ptr)
1177{
1178 pr_debug("%s(0x%p)\n", __func__, ptr);
1179
1180 if (kmemleak_enabled && ptr && !IS_ERR(ptr))
1181 object_no_scan((unsigned long)ptr);
1182 else if (kmemleak_early_log)
1183 log_early(KMEMLEAK_NO_SCAN, ptr, 0, 0);
1184}
1185EXPORT_SYMBOL(kmemleak_no_scan);
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196void __ref kmemleak_alloc_phys(phys_addr_t phys, size_t size, int min_count,
1197 gfp_t gfp)
1198{
1199 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1200 kmemleak_alloc(__va(phys), size, min_count, gfp);
1201}
1202EXPORT_SYMBOL(kmemleak_alloc_phys);
1203
1204
1205
1206
1207
1208
1209
1210
1211void __ref kmemleak_free_part_phys(phys_addr_t phys, size_t size)
1212{
1213 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1214 kmemleak_free_part(__va(phys), size);
1215}
1216EXPORT_SYMBOL(kmemleak_free_part_phys);
1217
1218
1219
1220
1221
1222
1223void __ref kmemleak_not_leak_phys(phys_addr_t phys)
1224{
1225 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1226 kmemleak_not_leak(__va(phys));
1227}
1228EXPORT_SYMBOL(kmemleak_not_leak_phys);
1229
1230
1231
1232
1233
1234
1235void __ref kmemleak_ignore_phys(phys_addr_t phys)
1236{
1237 if (!IS_ENABLED(CONFIG_HIGHMEM) || PHYS_PFN(phys) < max_low_pfn)
1238 kmemleak_ignore(__va(phys));
1239}
1240EXPORT_SYMBOL(kmemleak_ignore_phys);
1241
1242
1243
1244
1245static bool update_checksum(struct kmemleak_object *object)
1246{
1247 u32 old_csum = object->checksum;
1248
1249 kasan_disable_current();
1250 object->checksum = crc32(0, (void *)object->pointer, object->size);
1251 kasan_enable_current();
1252
1253 return object->checksum != old_csum;
1254}
1255
1256
1257
1258
1259static void update_refs(struct kmemleak_object *object)
1260{
1261 if (!color_white(object)) {
1262
1263 return;
1264 }
1265
1266
1267
1268
1269
1270
1271
1272 object->count++;
1273 if (color_gray(object)) {
1274
1275 WARN_ON(!get_object(object));
1276 list_add_tail(&object->gray_list, &gray_list);
1277 }
1278}
1279
1280
1281
1282
1283
1284static int scan_should_stop(void)
1285{
1286 if (!kmemleak_enabled)
1287 return 1;
1288
1289
1290
1291
1292
1293 if (current->mm)
1294 return signal_pending(current);
1295 else
1296 return kthread_should_stop();
1297
1298 return 0;
1299}
1300
1301
1302
1303
1304
1305static void scan_block(void *_start, void *_end,
1306 struct kmemleak_object *scanned)
1307{
1308 unsigned long *ptr;
1309 unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
1310 unsigned long *end = _end - (BYTES_PER_POINTER - 1);
1311 unsigned long flags;
1312
1313 read_lock_irqsave(&kmemleak_lock, flags);
1314 for (ptr = start; ptr < end; ptr++) {
1315 struct kmemleak_object *object;
1316 unsigned long pointer;
1317 unsigned long excess_ref;
1318
1319 if (scan_should_stop())
1320 break;
1321
1322 kasan_disable_current();
1323 pointer = *ptr;
1324 kasan_enable_current();
1325
1326 if (pointer < min_addr || pointer >= max_addr)
1327 continue;
1328
1329
1330
1331
1332
1333
1334
1335 object = lookup_object(pointer, 1);
1336 if (!object)
1337 continue;
1338 if (object == scanned)
1339
1340 continue;
1341
1342
1343
1344
1345
1346
1347 spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1348
1349 if (color_gray(object)) {
1350 excess_ref = object->excess_ref;
1351
1352 } else {
1353 excess_ref = 0;
1354 update_refs(object);
1355 }
1356 spin_unlock(&object->lock);
1357
1358 if (excess_ref) {
1359 object = lookup_object(excess_ref, 0);
1360 if (!object)
1361 continue;
1362 if (object == scanned)
1363
1364 continue;
1365 spin_lock_nested(&object->lock, SINGLE_DEPTH_NESTING);
1366 update_refs(object);
1367 spin_unlock(&object->lock);
1368 }
1369 }
1370 read_unlock_irqrestore(&kmemleak_lock, flags);
1371}
1372
1373
1374
1375
1376static void scan_large_block(void *start, void *end)
1377{
1378 void *next;
1379
1380 while (start < end) {
1381 next = min(start + MAX_SCAN_SIZE, end);
1382 scan_block(start, next, NULL);
1383 start = next;
1384 cond_resched();
1385 }
1386}
1387
1388
1389
1390
1391
1392static void scan_object(struct kmemleak_object *object)
1393{
1394 struct kmemleak_scan_area *area;
1395 unsigned long flags;
1396
1397
1398
1399
1400
1401 spin_lock_irqsave(&object->lock, flags);
1402 if (object->flags & OBJECT_NO_SCAN)
1403 goto out;
1404 if (!(object->flags & OBJECT_ALLOCATED))
1405
1406 goto out;
1407 if (hlist_empty(&object->area_list)) {
1408 void *start = (void *)object->pointer;
1409 void *end = (void *)(object->pointer + object->size);
1410 void *next;
1411
1412 do {
1413 next = min(start + MAX_SCAN_SIZE, end);
1414 scan_block(start, next, object);
1415
1416 start = next;
1417 if (start >= end)
1418 break;
1419
1420 spin_unlock_irqrestore(&object->lock, flags);
1421 cond_resched();
1422 spin_lock_irqsave(&object->lock, flags);
1423 } while (object->flags & OBJECT_ALLOCATED);
1424 } else
1425 hlist_for_each_entry(area, &object->area_list, node)
1426 scan_block((void *)area->start,
1427 (void *)(area->start + area->size),
1428 object);
1429out:
1430 spin_unlock_irqrestore(&object->lock, flags);
1431}
1432
1433
1434
1435
1436
1437static void scan_gray_list(void)
1438{
1439 struct kmemleak_object *object, *tmp;
1440
1441
1442
1443
1444
1445
1446 object = list_entry(gray_list.next, typeof(*object), gray_list);
1447 while (&object->gray_list != &gray_list) {
1448 cond_resched();
1449
1450
1451 if (!scan_should_stop())
1452 scan_object(object);
1453
1454 tmp = list_entry(object->gray_list.next, typeof(*object),
1455 gray_list);
1456
1457
1458 list_del(&object->gray_list);
1459 put_object(object);
1460
1461 object = tmp;
1462 }
1463 WARN_ON(!list_empty(&gray_list));
1464}
1465
1466
1467
1468
1469
1470
1471static void kmemleak_scan(void)
1472{
1473 unsigned long flags;
1474 struct kmemleak_object *object;
1475 int i;
1476 int new_leaks = 0;
1477
1478 jiffies_last_scan = jiffies;
1479
1480
1481 rcu_read_lock();
1482 list_for_each_entry_rcu(object, &object_list, object_list) {
1483 spin_lock_irqsave(&object->lock, flags);
1484#ifdef DEBUG
1485
1486
1487
1488
1489 if (atomic_read(&object->use_count) > 1) {
1490 pr_debug("object->use_count = %d\n",
1491 atomic_read(&object->use_count));
1492 dump_object_info(object);
1493 }
1494#endif
1495
1496 object->count = 0;
1497 if (color_gray(object) && get_object(object))
1498 list_add_tail(&object->gray_list, &gray_list);
1499
1500 spin_unlock_irqrestore(&object->lock, flags);
1501 }
1502 rcu_read_unlock();
1503
1504
1505 scan_large_block(_sdata, _edata);
1506 scan_large_block(__bss_start, __bss_stop);
1507 scan_large_block(__start_ro_after_init, __end_ro_after_init);
1508
1509#ifdef CONFIG_SMP
1510
1511 for_each_possible_cpu(i)
1512 scan_large_block(__per_cpu_start + per_cpu_offset(i),
1513 __per_cpu_end + per_cpu_offset(i));
1514#endif
1515
1516
1517
1518
1519 get_online_mems();
1520 for_each_online_node(i) {
1521 unsigned long start_pfn = node_start_pfn(i);
1522 unsigned long end_pfn = node_end_pfn(i);
1523 unsigned long pfn;
1524
1525 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1526 struct page *page;
1527
1528 if (!pfn_valid(pfn))
1529 continue;
1530 page = pfn_to_page(pfn);
1531
1532 if (page_count(page) == 0)
1533 continue;
1534 scan_block(page, page + 1, NULL);
1535 if (!(pfn & 63))
1536 cond_resched();
1537 }
1538 }
1539 put_online_mems();
1540
1541
1542
1543
1544 if (kmemleak_stack_scan) {
1545 struct task_struct *p, *g;
1546
1547 read_lock(&tasklist_lock);
1548 do_each_thread(g, p) {
1549 void *stack = try_get_task_stack(p);
1550 if (stack) {
1551 scan_block(stack, stack + THREAD_SIZE, NULL);
1552 put_task_stack(p);
1553 }
1554 } while_each_thread(g, p);
1555 read_unlock(&tasklist_lock);
1556 }
1557
1558
1559
1560
1561
1562 scan_gray_list();
1563
1564
1565
1566
1567
1568 rcu_read_lock();
1569 list_for_each_entry_rcu(object, &object_list, object_list) {
1570 spin_lock_irqsave(&object->lock, flags);
1571 if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
1572 && update_checksum(object) && get_object(object)) {
1573
1574 object->count = object->min_count;
1575 list_add_tail(&object->gray_list, &gray_list);
1576 }
1577 spin_unlock_irqrestore(&object->lock, flags);
1578 }
1579 rcu_read_unlock();
1580
1581
1582
1583
1584 scan_gray_list();
1585
1586
1587
1588
1589 if (scan_should_stop())
1590 return;
1591
1592
1593
1594
1595 rcu_read_lock();
1596 list_for_each_entry_rcu(object, &object_list, object_list) {
1597 spin_lock_irqsave(&object->lock, flags);
1598 if (unreferenced_object(object) &&
1599 !(object->flags & OBJECT_REPORTED)) {
1600 object->flags |= OBJECT_REPORTED;
1601 new_leaks++;
1602 }
1603 spin_unlock_irqrestore(&object->lock, flags);
1604 }
1605 rcu_read_unlock();
1606
1607 if (new_leaks) {
1608 kmemleak_found_leaks = true;
1609
1610 pr_info("%d new suspected memory leaks (see /sys/kernel/debug/kmemleak)\n",
1611 new_leaks);
1612 }
1613
1614}
1615
1616
1617
1618
1619
1620static int kmemleak_scan_thread(void *arg)
1621{
1622 static int first_run = 1;
1623
1624 pr_info("Automatic memory scanning thread started\n");
1625 set_user_nice(current, 10);
1626
1627
1628
1629
1630 if (first_run) {
1631 signed long timeout = msecs_to_jiffies(SECS_FIRST_SCAN * 1000);
1632 first_run = 0;
1633 while (timeout && !kthread_should_stop())
1634 timeout = schedule_timeout_interruptible(timeout);
1635 }
1636
1637 while (!kthread_should_stop()) {
1638 signed long timeout = jiffies_scan_wait;
1639
1640 mutex_lock(&scan_mutex);
1641 kmemleak_scan();
1642 mutex_unlock(&scan_mutex);
1643
1644
1645 while (timeout && !kthread_should_stop())
1646 timeout = schedule_timeout_interruptible(timeout);
1647 }
1648
1649 pr_info("Automatic memory scanning thread ended\n");
1650
1651 return 0;
1652}
1653
1654
1655
1656
1657
1658static void start_scan_thread(void)
1659{
1660 if (scan_thread)
1661 return;
1662 scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
1663 if (IS_ERR(scan_thread)) {
1664 pr_warn("Failed to create the scan thread\n");
1665 scan_thread = NULL;
1666 }
1667}
1668
1669
1670
1671
1672static void stop_scan_thread(void)
1673{
1674 if (scan_thread) {
1675 kthread_stop(scan_thread);
1676 scan_thread = NULL;
1677 }
1678}
1679
1680
1681
1682
1683
1684
1685static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
1686{
1687 struct kmemleak_object *object;
1688 loff_t n = *pos;
1689 int err;
1690
1691 err = mutex_lock_interruptible(&scan_mutex);
1692 if (err < 0)
1693 return ERR_PTR(err);
1694
1695 rcu_read_lock();
1696 list_for_each_entry_rcu(object, &object_list, object_list) {
1697 if (n-- > 0)
1698 continue;
1699 if (get_object(object))
1700 goto out;
1701 }
1702 object = NULL;
1703out:
1704 return object;
1705}
1706
1707
1708
1709
1710
1711static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1712{
1713 struct kmemleak_object *prev_obj = v;
1714 struct kmemleak_object *next_obj = NULL;
1715 struct kmemleak_object *obj = prev_obj;
1716
1717 ++(*pos);
1718
1719 list_for_each_entry_continue_rcu(obj, &object_list, object_list) {
1720 if (get_object(obj)) {
1721 next_obj = obj;
1722 break;
1723 }
1724 }
1725
1726 put_object(prev_obj);
1727 return next_obj;
1728}
1729
1730
1731
1732
1733static void kmemleak_seq_stop(struct seq_file *seq, void *v)
1734{
1735 if (!IS_ERR(v)) {
1736
1737
1738
1739
1740 rcu_read_unlock();
1741 mutex_unlock(&scan_mutex);
1742 if (v)
1743 put_object(v);
1744 }
1745}
1746
1747
1748
1749
1750static int kmemleak_seq_show(struct seq_file *seq, void *v)
1751{
1752 struct kmemleak_object *object = v;
1753 unsigned long flags;
1754
1755 spin_lock_irqsave(&object->lock, flags);
1756 if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
1757 print_unreferenced(seq, object);
1758 spin_unlock_irqrestore(&object->lock, flags);
1759 return 0;
1760}
1761
1762static const struct seq_operations kmemleak_seq_ops = {
1763 .start = kmemleak_seq_start,
1764 .next = kmemleak_seq_next,
1765 .stop = kmemleak_seq_stop,
1766 .show = kmemleak_seq_show,
1767};
1768
1769static int kmemleak_open(struct inode *inode, struct file *file)
1770{
1771 return seq_open(file, &kmemleak_seq_ops);
1772}
1773
1774static int dump_str_object_info(const char *str)
1775{
1776 unsigned long flags;
1777 struct kmemleak_object *object;
1778 unsigned long addr;
1779
1780 if (kstrtoul(str, 0, &addr))
1781 return -EINVAL;
1782 object = find_and_get_object(addr, 0);
1783 if (!object) {
1784 pr_info("Unknown object at 0x%08lx\n", addr);
1785 return -EINVAL;
1786 }
1787
1788 spin_lock_irqsave(&object->lock, flags);
1789 dump_object_info(object);
1790 spin_unlock_irqrestore(&object->lock, flags);
1791
1792 put_object(object);
1793 return 0;
1794}
1795
1796
1797
1798
1799
1800
1801
1802static void kmemleak_clear(void)
1803{
1804 struct kmemleak_object *object;
1805 unsigned long flags;
1806
1807 rcu_read_lock();
1808 list_for_each_entry_rcu(object, &object_list, object_list) {
1809 spin_lock_irqsave(&object->lock, flags);
1810 if ((object->flags & OBJECT_REPORTED) &&
1811 unreferenced_object(object))
1812 __paint_it(object, KMEMLEAK_GREY);
1813 spin_unlock_irqrestore(&object->lock, flags);
1814 }
1815 rcu_read_unlock();
1816
1817 kmemleak_found_leaks = false;
1818}
1819
1820static void __kmemleak_do_cleanup(void);
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
1839 size_t size, loff_t *ppos)
1840{
1841 char buf[64];
1842 int buf_size;
1843 int ret;
1844
1845 buf_size = min(size, (sizeof(buf) - 1));
1846 if (strncpy_from_user(buf, user_buf, buf_size) < 0)
1847 return -EFAULT;
1848 buf[buf_size] = 0;
1849
1850 ret = mutex_lock_interruptible(&scan_mutex);
1851 if (ret < 0)
1852 return ret;
1853
1854 if (strncmp(buf, "clear", 5) == 0) {
1855 if (kmemleak_enabled)
1856 kmemleak_clear();
1857 else
1858 __kmemleak_do_cleanup();
1859 goto out;
1860 }
1861
1862 if (!kmemleak_enabled) {
1863 ret = -EBUSY;
1864 goto out;
1865 }
1866
1867 if (strncmp(buf, "off", 3) == 0)
1868 kmemleak_disable();
1869 else if (strncmp(buf, "stack=on", 8) == 0)
1870 kmemleak_stack_scan = 1;
1871 else if (strncmp(buf, "stack=off", 9) == 0)
1872 kmemleak_stack_scan = 0;
1873 else if (strncmp(buf, "scan=on", 7) == 0)
1874 start_scan_thread();
1875 else if (strncmp(buf, "scan=off", 8) == 0)
1876 stop_scan_thread();
1877 else if (strncmp(buf, "scan=", 5) == 0) {
1878 unsigned long secs;
1879
1880 ret = kstrtoul(buf + 5, 0, &secs);
1881 if (ret < 0)
1882 goto out;
1883 stop_scan_thread();
1884 if (secs) {
1885 jiffies_scan_wait = msecs_to_jiffies(secs * 1000);
1886 start_scan_thread();
1887 }
1888 } else if (strncmp(buf, "scan", 4) == 0)
1889 kmemleak_scan();
1890 else if (strncmp(buf, "dump=", 5) == 0)
1891 ret = dump_str_object_info(buf + 5);
1892 else
1893 ret = -EINVAL;
1894
1895out:
1896 mutex_unlock(&scan_mutex);
1897 if (ret < 0)
1898 return ret;
1899
1900
1901 *ppos += size;
1902 return size;
1903}
1904
1905static const struct file_operations kmemleak_fops = {
1906 .owner = THIS_MODULE,
1907 .open = kmemleak_open,
1908 .read = seq_read,
1909 .write = kmemleak_write,
1910 .llseek = seq_lseek,
1911 .release = seq_release,
1912};
1913
1914static void __kmemleak_do_cleanup(void)
1915{
1916 struct kmemleak_object *object;
1917
1918 rcu_read_lock();
1919 list_for_each_entry_rcu(object, &object_list, object_list)
1920 delete_object_full(object->pointer);
1921 rcu_read_unlock();
1922}
1923
1924
1925
1926
1927
1928
1929static void kmemleak_do_cleanup(struct work_struct *work)
1930{
1931 stop_scan_thread();
1932
1933 mutex_lock(&scan_mutex);
1934
1935
1936
1937
1938
1939
1940 kmemleak_free_enabled = 0;
1941 mutex_unlock(&scan_mutex);
1942
1943 if (!kmemleak_found_leaks)
1944 __kmemleak_do_cleanup();
1945 else
1946 pr_info("Kmemleak disabled without freeing internal data. Reclaim the memory with \"echo clear > /sys/kernel/debug/kmemleak\".\n");
1947}
1948
1949static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
1950
1951
1952
1953
1954
1955static void kmemleak_disable(void)
1956{
1957
1958 if (cmpxchg(&kmemleak_error, 0, 1))
1959 return;
1960
1961
1962 kmemleak_enabled = 0;
1963
1964
1965 if (kmemleak_initialized)
1966 schedule_work(&cleanup_work);
1967 else
1968 kmemleak_free_enabled = 0;
1969
1970 pr_info("Kernel memory leak detector disabled\n");
1971}
1972
1973
1974
1975
1976static int __init kmemleak_boot_config(char *str)
1977{
1978 if (!str)
1979 return -EINVAL;
1980 if (strcmp(str, "off") == 0)
1981 kmemleak_disable();
1982 else if (strcmp(str, "on") == 0)
1983 kmemleak_skip_disable = 1;
1984 else
1985 return -EINVAL;
1986 return 0;
1987}
1988early_param("kmemleak", kmemleak_boot_config);
1989
1990static void __init print_log_trace(struct early_log *log)
1991{
1992 struct stack_trace trace;
1993
1994 trace.nr_entries = log->trace_len;
1995 trace.entries = log->trace;
1996
1997 pr_notice("Early log backtrace:\n");
1998 print_stack_trace(&trace, 2);
1999}
2000
2001
2002
2003
2004void __init kmemleak_init(void)
2005{
2006 int i;
2007 unsigned long flags;
2008
2009#ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
2010 if (!kmemleak_skip_disable) {
2011 kmemleak_early_log = 0;
2012 kmemleak_disable();
2013 return;
2014 }
2015#endif
2016
2017 jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
2018 jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
2019
2020 object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
2021 scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
2022
2023 if (crt_early_log > ARRAY_SIZE(early_log))
2024 pr_warn("Early log buffer exceeded (%d), please increase DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n",
2025 crt_early_log);
2026
2027
2028 local_irq_save(flags);
2029 kmemleak_early_log = 0;
2030 if (kmemleak_error) {
2031 local_irq_restore(flags);
2032 return;
2033 } else {
2034 kmemleak_enabled = 1;
2035 kmemleak_free_enabled = 1;
2036 }
2037 local_irq_restore(flags);
2038
2039
2040
2041
2042
2043
2044 for (i = 0; i < crt_early_log; i++) {
2045 struct early_log *log = &early_log[i];
2046
2047 switch (log->op_type) {
2048 case KMEMLEAK_ALLOC:
2049 early_alloc(log);
2050 break;
2051 case KMEMLEAK_ALLOC_PERCPU:
2052 early_alloc_percpu(log);
2053 break;
2054 case KMEMLEAK_FREE:
2055 kmemleak_free(log->ptr);
2056 break;
2057 case KMEMLEAK_FREE_PART:
2058 kmemleak_free_part(log->ptr, log->size);
2059 break;
2060 case KMEMLEAK_FREE_PERCPU:
2061 kmemleak_free_percpu(log->ptr);
2062 break;
2063 case KMEMLEAK_NOT_LEAK:
2064 kmemleak_not_leak(log->ptr);
2065 break;
2066 case KMEMLEAK_IGNORE:
2067 kmemleak_ignore(log->ptr);
2068 break;
2069 case KMEMLEAK_SCAN_AREA:
2070 kmemleak_scan_area(log->ptr, log->size, GFP_KERNEL);
2071 break;
2072 case KMEMLEAK_NO_SCAN:
2073 kmemleak_no_scan(log->ptr);
2074 break;
2075 case KMEMLEAK_SET_EXCESS_REF:
2076 object_set_excess_ref((unsigned long)log->ptr,
2077 log->excess_ref);
2078 break;
2079 default:
2080 kmemleak_warn("Unknown early log operation: %d\n",
2081 log->op_type);
2082 }
2083
2084 if (kmemleak_warning) {
2085 print_log_trace(log);
2086 kmemleak_warning = 0;
2087 }
2088 }
2089}
2090
2091
2092
2093
2094static int __init kmemleak_late_init(void)
2095{
2096 struct dentry *dentry;
2097
2098 kmemleak_initialized = 1;
2099
2100 dentry = debugfs_create_file("kmemleak", 0644, NULL, NULL,
2101 &kmemleak_fops);
2102 if (!dentry)
2103 pr_warn("Failed to create the debugfs kmemleak file\n");
2104
2105 if (kmemleak_error) {
2106
2107
2108
2109
2110
2111
2112 schedule_work(&cleanup_work);
2113 return -ENOMEM;
2114 }
2115
2116 mutex_lock(&scan_mutex);
2117 start_scan_thread();
2118 mutex_unlock(&scan_mutex);
2119
2120 pr_info("Kernel memory leak detector initialized\n");
2121
2122 return 0;
2123}
2124late_initcall(kmemleak_late_init);
2125