1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
65
66#include <linux/init.h>
67#include <linux/kernel.h>
68#include <linux/list.h>
69#include <linux/sched.h>
70#include <linux/jiffies.h>
71#include <linux/delay.h>
72#include <linux/export.h>
73#include <linux/kthread.h>
74#include <linux/rbtree.h>
75#include <linux/fs.h>
76#include <linux/debugfs.h>
77#include <linux/seq_file.h>
78#include <linux/cpumask.h>
79#include <linux/spinlock.h>
80#include <linux/mutex.h>
81#include <linux/rcupdate.h>
82#include <linux/stacktrace.h>
83#include <linux/cache.h>
84#include <linux/percpu.h>
85#include <linux/hardirq.h>
86#include <linux/mmzone.h>
87#include <linux/slab.h>
88#include <linux/thread_info.h>
89#include <linux/err.h>
90#include <linux/uaccess.h>
91#include <linux/string.h>
92#include <linux/nodemask.h>
93#include <linux/mm.h>
94#include <linux/workqueue.h>
95#include <linux/crc32.h>
96
97#include <asm/sections.h>
98#include <asm/processor.h>
99#include <linux/atomic.h>
100
101#include <linux/kmemcheck.h>
102#include <linux/kmemleak.h>
103#include <linux/memory_hotplug.h>
104
105
106
107
108#define MAX_TRACE 16
109#define MSECS_MIN_AGE 5000
110#define SECS_FIRST_SCAN 60
111#define SECS_SCAN_WAIT 600
112#define MAX_SCAN_SIZE 4096
113
114#define BYTES_PER_POINTER sizeof(void *)
115
116
117#define gfp_kmemleak_mask(gfp) (((gfp) & (GFP_KERNEL | GFP_ATOMIC)) | \
118 __GFP_NORETRY | __GFP_NOMEMALLOC | \
119 __GFP_NOWARN)
120
121
122struct kmemleak_scan_area {
123 struct hlist_node node;
124 unsigned long start;
125 size_t size;
126};
127
128#define KMEMLEAK_GREY 0
129#define KMEMLEAK_BLACK -1
130
131
132
133
134
135
136
137
138
139struct kmemleak_object {
140 spinlock_t lock;
141 unsigned long flags;
142 struct list_head object_list;
143 struct list_head gray_list;
144 struct rb_node rb_node;
145 struct rcu_head rcu;
146
147 atomic_t use_count;
148 unsigned long pointer;
149 size_t size;
150
151 int min_count;
152
153 int count;
154
155 u32 checksum;
156
157 struct hlist_head area_list;
158 unsigned long trace[MAX_TRACE];
159 unsigned int trace_len;
160 unsigned long jiffies;
161 pid_t pid;
162 char comm[TASK_COMM_LEN];
163};
164
165
166#define OBJECT_ALLOCATED (1 << 0)
167
168#define OBJECT_REPORTED (1 << 1)
169
170#define OBJECT_NO_SCAN (1 << 2)
171
172
173#define HEX_ROW_SIZE 16
174
175#define HEX_GROUP_SIZE 1
176
177#define HEX_ASCII 1
178
179#define HEX_MAX_LINES 2
180
181
182static LIST_HEAD(object_list);
183
184static LIST_HEAD(gray_list);
185
186static struct rb_root object_tree_root = RB_ROOT;
187
188static DEFINE_RWLOCK(kmemleak_lock);
189
190
191static struct kmem_cache *object_cache;
192static struct kmem_cache *scan_area_cache;
193
194
195static atomic_t kmemleak_enabled = ATOMIC_INIT(0);
196
197static atomic_t kmemleak_initialized = ATOMIC_INIT(0);
198
199static atomic_t kmemleak_early_log = ATOMIC_INIT(1);
200
201static atomic_t kmemleak_warning = ATOMIC_INIT(0);
202
203static atomic_t kmemleak_error = ATOMIC_INIT(0);
204
205
206static unsigned long min_addr = ULONG_MAX;
207static unsigned long max_addr;
208
209static struct task_struct *scan_thread;
210
211static unsigned long jiffies_min_age;
212static unsigned long jiffies_last_scan;
213
214static signed long jiffies_scan_wait;
215
216static int kmemleak_stack_scan = 1;
217
218static DEFINE_MUTEX(scan_mutex);
219
220static int kmemleak_skip_disable;
221
222
223
224
225
226
227
228
229
230
231
232enum {
233 KMEMLEAK_ALLOC,
234 KMEMLEAK_ALLOC_PERCPU,
235 KMEMLEAK_FREE,
236 KMEMLEAK_FREE_PART,
237 KMEMLEAK_FREE_PERCPU,
238 KMEMLEAK_NOT_LEAK,
239 KMEMLEAK_IGNORE,
240 KMEMLEAK_SCAN_AREA,
241 KMEMLEAK_NO_SCAN
242};
243
244
245
246
247
248struct early_log {
249 int op_type;
250 const void *ptr;
251 size_t size;
252 int min_count;
253 unsigned long trace[MAX_TRACE];
254 unsigned int trace_len;
255};
256
257
258static struct early_log
259 early_log[CONFIG_DEBUG_KMEMLEAK_EARLY_LOG_SIZE] __initdata;
260static int crt_early_log __initdata;
261
262static void kmemleak_disable(void);
263
264
265
266
267#define kmemleak_warn(x...) do { \
268 pr_warning(x); \
269 dump_stack(); \
270 atomic_set(&kmemleak_warning, 1); \
271} while (0)
272
273
274
275
276
277
278#define kmemleak_stop(x...) do { \
279 kmemleak_warn(x); \
280 kmemleak_disable(); \
281} while (0)
282
283
284
285
286
287
288
289static void hex_dump_object(struct seq_file *seq,
290 struct kmemleak_object *object)
291{
292 const u8 *ptr = (const u8 *)object->pointer;
293 int i, len, remaining;
294 unsigned char linebuf[HEX_ROW_SIZE * 5];
295
296
297 remaining = len =
298 min(object->size, (size_t)(HEX_MAX_LINES * HEX_ROW_SIZE));
299
300 seq_printf(seq, " hex dump (first %d bytes):\n", len);
301 for (i = 0; i < len; i += HEX_ROW_SIZE) {
302 int linelen = min(remaining, HEX_ROW_SIZE);
303
304 remaining -= HEX_ROW_SIZE;
305 hex_dump_to_buffer(ptr + i, linelen, HEX_ROW_SIZE,
306 HEX_GROUP_SIZE, linebuf, sizeof(linebuf),
307 HEX_ASCII);
308 seq_printf(seq, " %s\n", linebuf);
309 }
310}
311
312
313
314
315
316
317
318
319
320
321
322static bool color_white(const struct kmemleak_object *object)
323{
324 return object->count != KMEMLEAK_BLACK &&
325 object->count < object->min_count;
326}
327
328static bool color_gray(const struct kmemleak_object *object)
329{
330 return object->min_count != KMEMLEAK_BLACK &&
331 object->count >= object->min_count;
332}
333
334
335
336
337
338
339static bool unreferenced_object(struct kmemleak_object *object)
340{
341 return (color_white(object) && object->flags & OBJECT_ALLOCATED) &&
342 time_before_eq(object->jiffies + jiffies_min_age,
343 jiffies_last_scan);
344}
345
346
347
348
349
350static void print_unreferenced(struct seq_file *seq,
351 struct kmemleak_object *object)
352{
353 int i;
354 unsigned int msecs_age = jiffies_to_msecs(jiffies - object->jiffies);
355
356 seq_printf(seq, "unreferenced object 0x%08lx (size %zu):\n",
357 object->pointer, object->size);
358 seq_printf(seq, " comm \"%s\", pid %d, jiffies %lu (age %d.%03ds)\n",
359 object->comm, object->pid, object->jiffies,
360 msecs_age / 1000, msecs_age % 1000);
361 hex_dump_object(seq, object);
362 seq_printf(seq, " backtrace:\n");
363
364 for (i = 0; i < object->trace_len; i++) {
365 void *ptr = (void *)object->trace[i];
366 seq_printf(seq, " [<%p>] %pS\n", ptr, ptr);
367 }
368}
369
370
371
372
373
374
375static void dump_object_info(struct kmemleak_object *object)
376{
377 struct stack_trace trace;
378
379 trace.nr_entries = object->trace_len;
380 trace.entries = object->trace;
381
382 pr_notice("Object 0x%08lx (size %zu):\n",
383 object->pointer, object->size);
384 pr_notice(" comm \"%s\", pid %d, jiffies %lu\n",
385 object->comm, object->pid, object->jiffies);
386 pr_notice(" min_count = %d\n", object->min_count);
387 pr_notice(" count = %d\n", object->count);
388 pr_notice(" flags = 0x%lx\n", object->flags);
389 pr_notice(" checksum = %d\n", object->checksum);
390 pr_notice(" backtrace:\n");
391 print_stack_trace(&trace, 4);
392}
393
394
395
396
397
398
399
400static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
401{
402 struct rb_node *rb = object_tree_root.rb_node;
403
404 while (rb) {
405 struct kmemleak_object *object =
406 rb_entry(rb, struct kmemleak_object, rb_node);
407 if (ptr < object->pointer)
408 rb = object->rb_node.rb_left;
409 else if (object->pointer + object->size <= ptr)
410 rb = object->rb_node.rb_right;
411 else if (object->pointer == ptr || alias)
412 return object;
413 else {
414 kmemleak_warn("Found object by alias at 0x%08lx\n",
415 ptr);
416 dump_object_info(object);
417 break;
418 }
419 }
420 return NULL;
421}
422
423
424
425
426
427
428
429static int get_object(struct kmemleak_object *object)
430{
431 return atomic_inc_not_zero(&object->use_count);
432}
433
434
435
436
437static void free_object_rcu(struct rcu_head *rcu)
438{
439 struct hlist_node *tmp;
440 struct kmemleak_scan_area *area;
441 struct kmemleak_object *object =
442 container_of(rcu, struct kmemleak_object, rcu);
443
444
445
446
447
448 hlist_for_each_entry_safe(area, tmp, &object->area_list, node) {
449 hlist_del(&area->node);
450 kmem_cache_free(scan_area_cache, area);
451 }
452 kmem_cache_free(object_cache, object);
453}
454
455
456
457
458
459
460
461
462static void put_object(struct kmemleak_object *object)
463{
464 if (!atomic_dec_and_test(&object->use_count))
465 return;
466
467
468 WARN_ON(object->flags & OBJECT_ALLOCATED);
469
470 call_rcu(&object->rcu, free_object_rcu);
471}
472
473
474
475
476static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
477{
478 unsigned long flags;
479 struct kmemleak_object *object = NULL;
480
481 rcu_read_lock();
482 read_lock_irqsave(&kmemleak_lock, flags);
483 if (ptr >= min_addr && ptr < max_addr)
484 object = lookup_object(ptr, alias);
485 read_unlock_irqrestore(&kmemleak_lock, flags);
486
487
488 if (object && !get_object(object))
489 object = NULL;
490 rcu_read_unlock();
491
492 return object;
493}
494
495
496
497
498static int __save_stack_trace(unsigned long *trace)
499{
500 struct stack_trace stack_trace;
501
502 stack_trace.max_entries = MAX_TRACE;
503 stack_trace.nr_entries = 0;
504 stack_trace.entries = trace;
505 stack_trace.skip = 2;
506 save_stack_trace(&stack_trace);
507
508 return stack_trace.nr_entries;
509}
510
511
512
513
514
515static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
516 int min_count, gfp_t gfp)
517{
518 unsigned long flags;
519 struct kmemleak_object *object, *parent;
520 struct rb_node **link, *rb_parent;
521
522 object = kmem_cache_alloc(object_cache, gfp_kmemleak_mask(gfp));
523 if (!object) {
524 pr_warning("Cannot allocate a kmemleak_object structure\n");
525 kmemleak_disable();
526 return NULL;
527 }
528
529 INIT_LIST_HEAD(&object->object_list);
530 INIT_LIST_HEAD(&object->gray_list);
531 INIT_HLIST_HEAD(&object->area_list);
532 spin_lock_init(&object->lock);
533 atomic_set(&object->use_count, 1);
534 object->flags = OBJECT_ALLOCATED;
535 object->pointer = ptr;
536 object->size = size;
537 object->min_count = min_count;
538 object->count = 0;
539 object->jiffies = jiffies;
540 object->checksum = 0;
541
542
543 if (in_irq()) {
544 object->pid = 0;
545 strncpy(object->comm, "hardirq", sizeof(object->comm));
546 } else if (in_softirq()) {
547 object->pid = 0;
548 strncpy(object->comm, "softirq", sizeof(object->comm));
549 } else {
550 object->pid = current->pid;
551
552
553
554
555
556
557 strncpy(object->comm, current->comm, sizeof(object->comm));
558 }
559
560
561 object->trace_len = __save_stack_trace(object->trace);
562
563 write_lock_irqsave(&kmemleak_lock, flags);
564
565 min_addr = min(min_addr, ptr);
566 max_addr = max(max_addr, ptr + size);
567 link = &object_tree_root.rb_node;
568 rb_parent = NULL;
569 while (*link) {
570 rb_parent = *link;
571 parent = rb_entry(rb_parent, struct kmemleak_object, rb_node);
572 if (ptr + size <= parent->pointer)
573 link = &parent->rb_node.rb_left;
574 else if (parent->pointer + parent->size <= ptr)
575 link = &parent->rb_node.rb_right;
576 else {
577 kmemleak_stop("Cannot insert 0x%lx into the object "
578 "search tree (overlaps existing)\n",
579 ptr);
580 kmem_cache_free(object_cache, object);
581 object = parent;
582 spin_lock(&object->lock);
583 dump_object_info(object);
584 spin_unlock(&object->lock);
585 goto out;
586 }
587 }
588 rb_link_node(&object->rb_node, rb_parent, link);
589 rb_insert_color(&object->rb_node, &object_tree_root);
590
591 list_add_tail_rcu(&object->object_list, &object_list);
592out:
593 write_unlock_irqrestore(&kmemleak_lock, flags);
594 return object;
595}
596
597
598
599
600
601static void __delete_object(struct kmemleak_object *object)
602{
603 unsigned long flags;
604
605 write_lock_irqsave(&kmemleak_lock, flags);
606 rb_erase(&object->rb_node, &object_tree_root);
607 list_del_rcu(&object->object_list);
608 write_unlock_irqrestore(&kmemleak_lock, flags);
609
610 WARN_ON(!(object->flags & OBJECT_ALLOCATED));
611 WARN_ON(atomic_read(&object->use_count) < 2);
612
613
614
615
616
617 spin_lock_irqsave(&object->lock, flags);
618 object->flags &= ~OBJECT_ALLOCATED;
619 spin_unlock_irqrestore(&object->lock, flags);
620 put_object(object);
621}
622
623
624
625
626
627static void delete_object_full(unsigned long ptr)
628{
629 struct kmemleak_object *object;
630
631 object = find_and_get_object(ptr, 0);
632 if (!object) {
633#ifdef DEBUG
634 kmemleak_warn("Freeing unknown object at 0x%08lx\n",
635 ptr);
636#endif
637 return;
638 }
639 __delete_object(object);
640 put_object(object);
641}
642
643
644
645
646
647
648static void delete_object_part(unsigned long ptr, size_t size)
649{
650 struct kmemleak_object *object;
651 unsigned long start, end;
652
653 object = find_and_get_object(ptr, 1);
654 if (!object) {
655#ifdef DEBUG
656 kmemleak_warn("Partially freeing unknown object at 0x%08lx "
657 "(size %zu)\n", ptr, size);
658#endif
659 return;
660 }
661 __delete_object(object);
662
663
664
665
666
667
668
669
670 start = object->pointer;
671 end = object->pointer + object->size;
672 if (ptr > start)
673 create_object(start, ptr - start, object->min_count,
674 GFP_KERNEL);
675 if (ptr + size < end)
676 create_object(ptr + size, end - ptr - size, object->min_count,
677 GFP_KERNEL);
678
679 put_object(object);
680}
681
682static void __paint_it(struct kmemleak_object *object, int color)
683{
684 object->min_count = color;
685 if (color == KMEMLEAK_BLACK)
686 object->flags |= OBJECT_NO_SCAN;
687}
688
689static void paint_it(struct kmemleak_object *object, int color)
690{
691 unsigned long flags;
692
693 spin_lock_irqsave(&object->lock, flags);
694 __paint_it(object, color);
695 spin_unlock_irqrestore(&object->lock, flags);
696}
697
698static void paint_ptr(unsigned long ptr, int color)
699{
700 struct kmemleak_object *object;
701
702 object = find_and_get_object(ptr, 0);
703 if (!object) {
704 kmemleak_warn("Trying to color unknown object "
705 "at 0x%08lx as %s\n", ptr,
706 (color == KMEMLEAK_GREY) ? "Grey" :
707 (color == KMEMLEAK_BLACK) ? "Black" : "Unknown");
708 return;
709 }
710 paint_it(object, color);
711 put_object(object);
712}
713
714
715
716
717
718static void make_gray_object(unsigned long ptr)
719{
720 paint_ptr(ptr, KMEMLEAK_GREY);
721}
722
723
724
725
726
727static void make_black_object(unsigned long ptr)
728{
729 paint_ptr(ptr, KMEMLEAK_BLACK);
730}
731
732
733
734
735
736static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
737{
738 unsigned long flags;
739 struct kmemleak_object *object;
740 struct kmemleak_scan_area *area;
741
742 object = find_and_get_object(ptr, 1);
743 if (!object) {
744 kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
745 ptr);
746 return;
747 }
748
749 area = kmem_cache_alloc(scan_area_cache, gfp_kmemleak_mask(gfp));
750 if (!area) {
751 pr_warning("Cannot allocate a scan area\n");
752 goto out;
753 }
754
755 spin_lock_irqsave(&object->lock, flags);
756 if (ptr + size > object->pointer + object->size) {
757 kmemleak_warn("Scan area larger than object 0x%08lx\n", ptr);
758 dump_object_info(object);
759 kmem_cache_free(scan_area_cache, area);
760 goto out_unlock;
761 }
762
763 INIT_HLIST_NODE(&area->node);
764 area->start = ptr;
765 area->size = size;
766
767 hlist_add_head(&area->node, &object->area_list);
768out_unlock:
769 spin_unlock_irqrestore(&object->lock, flags);
770out:
771 put_object(object);
772}
773
774
775
776
777
778
779static void object_no_scan(unsigned long ptr)
780{
781 unsigned long flags;
782 struct kmemleak_object *object;
783
784 object = find_and_get_object(ptr, 0);
785 if (!object) {
786 kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
787 return;
788 }
789
790 spin_lock_irqsave(&object->lock, flags);
791 object->flags |= OBJECT_NO_SCAN;
792 spin_unlock_irqrestore(&object->lock, flags);
793 put_object(object);
794}
795
796
797
798
799
800static void __init log_early(int op_type, const void *ptr, size_t size,
801 int min_count)
802{
803 unsigned long flags;
804 struct early_log *log;
805
806 if (atomic_read(&kmemleak_error)) {
807
808 crt_early_log++;
809 return;
810 }
811
812 if (crt_early_log >= ARRAY_SIZE(early_log)) {
813 kmemleak_disable();
814 return;
815 }
816
817
818
819
820
821 local_irq_save(flags);
822 log = &early_log[crt_early_log];
823 log->op_type = op_type;
824 log->ptr = ptr;
825 log->size = size;
826 log->min_count = min_count;
827 log->trace_len = __save_stack_trace(log->trace);
828 crt_early_log++;
829 local_irq_restore(flags);
830}
831
832
833
834
835static void early_alloc(struct early_log *log)
836{
837 struct kmemleak_object *object;
838 unsigned long flags;
839 int i;
840
841 if (!atomic_read(&kmemleak_enabled) || !log->ptr || IS_ERR(log->ptr))
842 return;
843
844
845
846
847 rcu_read_lock();
848 object = create_object((unsigned long)log->ptr, log->size,
849 log->min_count, GFP_ATOMIC);
850 if (!object)
851 goto out;
852 spin_lock_irqsave(&object->lock, flags);
853 for (i = 0; i < log->trace_len; i++)
854 object->trace[i] = log->trace[i];
855 object->trace_len = log->trace_len;
856 spin_unlock_irqrestore(&object->lock, flags);
857out:
858 rcu_read_unlock();
859}
860
861
862
863
864static void early_alloc_percpu(struct early_log *log)
865{
866 unsigned int cpu;
867 const void __percpu *ptr = log->ptr;
868
869 for_each_possible_cpu(cpu) {
870 log->ptr = per_cpu_ptr(ptr, cpu);
871 early_alloc(log);
872 }
873}
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889void __ref kmemleak_alloc(const void *ptr, size_t size, int min_count,
890 gfp_t gfp)
891{
892 pr_debug("%s(0x%p, %zu, %d)\n", __func__, ptr, size, min_count);
893
894 if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
895 create_object((unsigned long)ptr, size, min_count, gfp);
896 else if (atomic_read(&kmemleak_early_log))
897 log_early(KMEMLEAK_ALLOC, ptr, size, min_count);
898}
899EXPORT_SYMBOL_GPL(kmemleak_alloc);
900
901
902
903
904
905
906
907
908
909
910void __ref kmemleak_alloc_percpu(const void __percpu *ptr, size_t size)
911{
912 unsigned int cpu;
913
914 pr_debug("%s(0x%p, %zu)\n", __func__, ptr, size);
915
916
917
918
919
920 if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
921 for_each_possible_cpu(cpu)
922 create_object((unsigned long)per_cpu_ptr(ptr, cpu),
923 size, 0, GFP_KERNEL);
924 else if (atomic_read(&kmemleak_early_log))
925 log_early(KMEMLEAK_ALLOC_PERCPU, ptr, size, 0);
926}
927EXPORT_SYMBOL_GPL(kmemleak_alloc_percpu);
928
929
930
931
932
933
934
935
936void __ref kmemleak_free(const void *ptr)
937{
938 pr_debug("%s(0x%p)\n", __func__, ptr);
939
940 if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
941 delete_object_full((unsigned long)ptr);
942 else if (atomic_read(&kmemleak_early_log))
943 log_early(KMEMLEAK_FREE, ptr, 0, 0);
944}
945EXPORT_SYMBOL_GPL(kmemleak_free);
946
947
948
949
950
951
952
953
954
955
956void __ref kmemleak_free_part(const void *ptr, size_t size)
957{
958 pr_debug("%s(0x%p)\n", __func__, ptr);
959
960 if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
961 delete_object_part((unsigned long)ptr, size);
962 else if (atomic_read(&kmemleak_early_log))
963 log_early(KMEMLEAK_FREE_PART, ptr, size, 0);
964}
965EXPORT_SYMBOL_GPL(kmemleak_free_part);
966
967
968
969
970
971
972
973
974void __ref kmemleak_free_percpu(const void __percpu *ptr)
975{
976 unsigned int cpu;
977
978 pr_debug("%s(0x%p)\n", __func__, ptr);
979
980 if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
981 for_each_possible_cpu(cpu)
982 delete_object_full((unsigned long)per_cpu_ptr(ptr,
983 cpu));
984 else if (atomic_read(&kmemleak_early_log))
985 log_early(KMEMLEAK_FREE_PERCPU, ptr, 0, 0);
986}
987EXPORT_SYMBOL_GPL(kmemleak_free_percpu);
988
989
990
991
992
993
994
995
996void __ref kmemleak_not_leak(const void *ptr)
997{
998 pr_debug("%s(0x%p)\n", __func__, ptr);
999
1000 if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
1001 make_gray_object((unsigned long)ptr);
1002 else if (atomic_read(&kmemleak_early_log))
1003 log_early(KMEMLEAK_NOT_LEAK, ptr, 0, 0);
1004}
1005EXPORT_SYMBOL(kmemleak_not_leak);
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016void __ref kmemleak_ignore(const void *ptr)
1017{
1018 pr_debug("%s(0x%p)\n", __func__, ptr);
1019
1020 if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
1021 make_black_object((unsigned long)ptr);
1022 else if (atomic_read(&kmemleak_early_log))
1023 log_early(KMEMLEAK_IGNORE, ptr, 0, 0);
1024}
1025EXPORT_SYMBOL(kmemleak_ignore);
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038void __ref kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp)
1039{
1040 pr_debug("%s(0x%p)\n", __func__, ptr);
1041
1042 if (atomic_read(&kmemleak_enabled) && ptr && size && !IS_ERR(ptr))
1043 add_scan_area((unsigned long)ptr, size, gfp);
1044 else if (atomic_read(&kmemleak_early_log))
1045 log_early(KMEMLEAK_SCAN_AREA, ptr, size, 0);
1046}
1047EXPORT_SYMBOL(kmemleak_scan_area);
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058void __ref kmemleak_no_scan(const void *ptr)
1059{
1060 pr_debug("%s(0x%p)\n", __func__, ptr);
1061
1062 if (atomic_read(&kmemleak_enabled) && ptr && !IS_ERR(ptr))
1063 object_no_scan((unsigned long)ptr);
1064 else if (atomic_read(&kmemleak_early_log))
1065 log_early(KMEMLEAK_NO_SCAN, ptr, 0, 0);
1066}
1067EXPORT_SYMBOL(kmemleak_no_scan);
1068
1069
1070
1071
1072static bool update_checksum(struct kmemleak_object *object)
1073{
1074 u32 old_csum = object->checksum;
1075
1076 if (!kmemcheck_is_obj_initialized(object->pointer, object->size))
1077 return false;
1078
1079 object->checksum = crc32(0, (void *)object->pointer, object->size);
1080 return object->checksum != old_csum;
1081}
1082
1083
1084
1085
1086
1087static int scan_should_stop(void)
1088{
1089 if (!atomic_read(&kmemleak_enabled))
1090 return 1;
1091
1092
1093
1094
1095
1096 if (current->mm)
1097 return signal_pending(current);
1098 else
1099 return kthread_should_stop();
1100
1101 return 0;
1102}
1103
1104
1105
1106
1107
1108static void scan_block(void *_start, void *_end,
1109 struct kmemleak_object *scanned, int allow_resched)
1110{
1111 unsigned long *ptr;
1112 unsigned long *start = PTR_ALIGN(_start, BYTES_PER_POINTER);
1113 unsigned long *end = _end - (BYTES_PER_POINTER - 1);
1114
1115 for (ptr = start; ptr < end; ptr++) {
1116 struct kmemleak_object *object;
1117 unsigned long flags;
1118 unsigned long pointer;
1119
1120 if (allow_resched)
1121 cond_resched();
1122 if (scan_should_stop())
1123 break;
1124
1125
1126 if (!kmemcheck_is_obj_initialized((unsigned long)ptr,
1127 BYTES_PER_POINTER))
1128 continue;
1129
1130 pointer = *ptr;
1131
1132 object = find_and_get_object(pointer, 1);
1133 if (!object)
1134 continue;
1135 if (object == scanned) {
1136
1137 put_object(object);
1138 continue;
1139 }
1140
1141
1142
1143
1144
1145
1146 spin_lock_irqsave_nested(&object->lock, flags,
1147 SINGLE_DEPTH_NESTING);
1148 if (!color_white(object)) {
1149
1150 spin_unlock_irqrestore(&object->lock, flags);
1151 put_object(object);
1152 continue;
1153 }
1154
1155
1156
1157
1158
1159
1160
1161 object->count++;
1162 if (color_gray(object)) {
1163 list_add_tail(&object->gray_list, &gray_list);
1164 spin_unlock_irqrestore(&object->lock, flags);
1165 continue;
1166 }
1167
1168 spin_unlock_irqrestore(&object->lock, flags);
1169 put_object(object);
1170 }
1171}
1172
1173
1174
1175
1176
1177static void scan_object(struct kmemleak_object *object)
1178{
1179 struct kmemleak_scan_area *area;
1180 unsigned long flags;
1181
1182
1183
1184
1185
1186 spin_lock_irqsave(&object->lock, flags);
1187 if (object->flags & OBJECT_NO_SCAN)
1188 goto out;
1189 if (!(object->flags & OBJECT_ALLOCATED))
1190
1191 goto out;
1192 if (hlist_empty(&object->area_list)) {
1193 void *start = (void *)object->pointer;
1194 void *end = (void *)(object->pointer + object->size);
1195
1196 while (start < end && (object->flags & OBJECT_ALLOCATED) &&
1197 !(object->flags & OBJECT_NO_SCAN)) {
1198 scan_block(start, min(start + MAX_SCAN_SIZE, end),
1199 object, 0);
1200 start += MAX_SCAN_SIZE;
1201
1202 spin_unlock_irqrestore(&object->lock, flags);
1203 cond_resched();
1204 spin_lock_irqsave(&object->lock, flags);
1205 }
1206 } else
1207 hlist_for_each_entry(area, &object->area_list, node)
1208 scan_block((void *)area->start,
1209 (void *)(area->start + area->size),
1210 object, 0);
1211out:
1212 spin_unlock_irqrestore(&object->lock, flags);
1213}
1214
1215
1216
1217
1218
1219static void scan_gray_list(void)
1220{
1221 struct kmemleak_object *object, *tmp;
1222
1223
1224
1225
1226
1227
1228 object = list_entry(gray_list.next, typeof(*object), gray_list);
1229 while (&object->gray_list != &gray_list) {
1230 cond_resched();
1231
1232
1233 if (!scan_should_stop())
1234 scan_object(object);
1235
1236 tmp = list_entry(object->gray_list.next, typeof(*object),
1237 gray_list);
1238
1239
1240 list_del(&object->gray_list);
1241 put_object(object);
1242
1243 object = tmp;
1244 }
1245 WARN_ON(!list_empty(&gray_list));
1246}
1247
1248
1249
1250
1251
1252
1253static void kmemleak_scan(void)
1254{
1255 unsigned long flags;
1256 struct kmemleak_object *object;
1257 int i;
1258 int new_leaks = 0;
1259
1260 jiffies_last_scan = jiffies;
1261
1262
1263 rcu_read_lock();
1264 list_for_each_entry_rcu(object, &object_list, object_list) {
1265 spin_lock_irqsave(&object->lock, flags);
1266#ifdef DEBUG
1267
1268
1269
1270
1271 if (atomic_read(&object->use_count) > 1) {
1272 pr_debug("object->use_count = %d\n",
1273 atomic_read(&object->use_count));
1274 dump_object_info(object);
1275 }
1276#endif
1277
1278 object->count = 0;
1279 if (color_gray(object) && get_object(object))
1280 list_add_tail(&object->gray_list, &gray_list);
1281
1282 spin_unlock_irqrestore(&object->lock, flags);
1283 }
1284 rcu_read_unlock();
1285
1286
1287 scan_block(_sdata, _edata, NULL, 1);
1288 scan_block(__bss_start, __bss_stop, NULL, 1);
1289
1290#ifdef CONFIG_SMP
1291
1292 for_each_possible_cpu(i)
1293 scan_block(__per_cpu_start + per_cpu_offset(i),
1294 __per_cpu_end + per_cpu_offset(i), NULL, 1);
1295#endif
1296
1297
1298
1299
1300 lock_memory_hotplug();
1301 for_each_online_node(i) {
1302 unsigned long start_pfn = node_start_pfn(i);
1303 unsigned long end_pfn = node_end_pfn(i);
1304 unsigned long pfn;
1305
1306 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
1307 struct page *page;
1308
1309 if (!pfn_valid(pfn))
1310 continue;
1311 page = pfn_to_page(pfn);
1312
1313 if (page_count(page) == 0)
1314 continue;
1315 scan_block(page, page + 1, NULL, 1);
1316 }
1317 }
1318 unlock_memory_hotplug();
1319
1320
1321
1322
1323 if (kmemleak_stack_scan) {
1324 struct task_struct *p, *g;
1325
1326 read_lock(&tasklist_lock);
1327 do_each_thread(g, p) {
1328 scan_block(task_stack_page(p), task_stack_page(p) +
1329 THREAD_SIZE, NULL, 0);
1330 } while_each_thread(g, p);
1331 read_unlock(&tasklist_lock);
1332 }
1333
1334
1335
1336
1337
1338 scan_gray_list();
1339
1340
1341
1342
1343
1344 rcu_read_lock();
1345 list_for_each_entry_rcu(object, &object_list, object_list) {
1346 spin_lock_irqsave(&object->lock, flags);
1347 if (color_white(object) && (object->flags & OBJECT_ALLOCATED)
1348 && update_checksum(object) && get_object(object)) {
1349
1350 object->count = object->min_count;
1351 list_add_tail(&object->gray_list, &gray_list);
1352 }
1353 spin_unlock_irqrestore(&object->lock, flags);
1354 }
1355 rcu_read_unlock();
1356
1357
1358
1359
1360 scan_gray_list();
1361
1362
1363
1364
1365 if (scan_should_stop())
1366 return;
1367
1368
1369
1370
1371 rcu_read_lock();
1372 list_for_each_entry_rcu(object, &object_list, object_list) {
1373 spin_lock_irqsave(&object->lock, flags);
1374 if (unreferenced_object(object) &&
1375 !(object->flags & OBJECT_REPORTED)) {
1376 object->flags |= OBJECT_REPORTED;
1377 new_leaks++;
1378 }
1379 spin_unlock_irqrestore(&object->lock, flags);
1380 }
1381 rcu_read_unlock();
1382
1383 if (new_leaks)
1384 pr_info("%d new suspected memory leaks (see "
1385 "/sys/kernel/debug/kmemleak)\n", new_leaks);
1386
1387}
1388
1389
1390
1391
1392
1393static int kmemleak_scan_thread(void *arg)
1394{
1395 static int first_run = 1;
1396
1397 pr_info("Automatic memory scanning thread started\n");
1398 set_user_nice(current, 10);
1399
1400
1401
1402
1403 if (first_run) {
1404 first_run = 0;
1405 ssleep(SECS_FIRST_SCAN);
1406 }
1407
1408 while (!kthread_should_stop()) {
1409 signed long timeout = jiffies_scan_wait;
1410
1411 mutex_lock(&scan_mutex);
1412 kmemleak_scan();
1413 mutex_unlock(&scan_mutex);
1414
1415
1416 while (timeout && !kthread_should_stop())
1417 timeout = schedule_timeout_interruptible(timeout);
1418 }
1419
1420 pr_info("Automatic memory scanning thread ended\n");
1421
1422 return 0;
1423}
1424
1425
1426
1427
1428
1429static void start_scan_thread(void)
1430{
1431 if (scan_thread)
1432 return;
1433 scan_thread = kthread_run(kmemleak_scan_thread, NULL, "kmemleak");
1434 if (IS_ERR(scan_thread)) {
1435 pr_warning("Failed to create the scan thread\n");
1436 scan_thread = NULL;
1437 }
1438}
1439
1440
1441
1442
1443
1444static void stop_scan_thread(void)
1445{
1446 if (scan_thread) {
1447 kthread_stop(scan_thread);
1448 scan_thread = NULL;
1449 }
1450}
1451
1452
1453
1454
1455
1456
1457static void *kmemleak_seq_start(struct seq_file *seq, loff_t *pos)
1458{
1459 struct kmemleak_object *object;
1460 loff_t n = *pos;
1461 int err;
1462
1463 err = mutex_lock_interruptible(&scan_mutex);
1464 if (err < 0)
1465 return ERR_PTR(err);
1466
1467 rcu_read_lock();
1468 list_for_each_entry_rcu(object, &object_list, object_list) {
1469 if (n-- > 0)
1470 continue;
1471 if (get_object(object))
1472 goto out;
1473 }
1474 object = NULL;
1475out:
1476 return object;
1477}
1478
1479
1480
1481
1482
1483static void *kmemleak_seq_next(struct seq_file *seq, void *v, loff_t *pos)
1484{
1485 struct kmemleak_object *prev_obj = v;
1486 struct kmemleak_object *next_obj = NULL;
1487 struct kmemleak_object *obj = prev_obj;
1488
1489 ++(*pos);
1490
1491 list_for_each_entry_continue_rcu(obj, &object_list, object_list) {
1492 if (get_object(obj)) {
1493 next_obj = obj;
1494 break;
1495 }
1496 }
1497
1498 put_object(prev_obj);
1499 return next_obj;
1500}
1501
1502
1503
1504
1505static void kmemleak_seq_stop(struct seq_file *seq, void *v)
1506{
1507 if (!IS_ERR(v)) {
1508
1509
1510
1511
1512 rcu_read_unlock();
1513 mutex_unlock(&scan_mutex);
1514 if (v)
1515 put_object(v);
1516 }
1517}
1518
1519
1520
1521
1522static int kmemleak_seq_show(struct seq_file *seq, void *v)
1523{
1524 struct kmemleak_object *object = v;
1525 unsigned long flags;
1526
1527 spin_lock_irqsave(&object->lock, flags);
1528 if ((object->flags & OBJECT_REPORTED) && unreferenced_object(object))
1529 print_unreferenced(seq, object);
1530 spin_unlock_irqrestore(&object->lock, flags);
1531 return 0;
1532}
1533
1534static const struct seq_operations kmemleak_seq_ops = {
1535 .start = kmemleak_seq_start,
1536 .next = kmemleak_seq_next,
1537 .stop = kmemleak_seq_stop,
1538 .show = kmemleak_seq_show,
1539};
1540
1541static int kmemleak_open(struct inode *inode, struct file *file)
1542{
1543 return seq_open(file, &kmemleak_seq_ops);
1544}
1545
1546static int kmemleak_release(struct inode *inode, struct file *file)
1547{
1548 return seq_release(inode, file);
1549}
1550
1551static int dump_str_object_info(const char *str)
1552{
1553 unsigned long flags;
1554 struct kmemleak_object *object;
1555 unsigned long addr;
1556
1557 if (kstrtoul(str, 0, &addr))
1558 return -EINVAL;
1559 object = find_and_get_object(addr, 0);
1560 if (!object) {
1561 pr_info("Unknown object at 0x%08lx\n", addr);
1562 return -EINVAL;
1563 }
1564
1565 spin_lock_irqsave(&object->lock, flags);
1566 dump_object_info(object);
1567 spin_unlock_irqrestore(&object->lock, flags);
1568
1569 put_object(object);
1570 return 0;
1571}
1572
1573
1574
1575
1576
1577
1578
1579static void kmemleak_clear(void)
1580{
1581 struct kmemleak_object *object;
1582 unsigned long flags;
1583
1584 rcu_read_lock();
1585 list_for_each_entry_rcu(object, &object_list, object_list) {
1586 spin_lock_irqsave(&object->lock, flags);
1587 if ((object->flags & OBJECT_REPORTED) &&
1588 unreferenced_object(object))
1589 __paint_it(object, KMEMLEAK_GREY);
1590 spin_unlock_irqrestore(&object->lock, flags);
1591 }
1592 rcu_read_unlock();
1593}
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610static ssize_t kmemleak_write(struct file *file, const char __user *user_buf,
1611 size_t size, loff_t *ppos)
1612{
1613 char buf[64];
1614 int buf_size;
1615 int ret;
1616
1617 if (!atomic_read(&kmemleak_enabled))
1618 return -EBUSY;
1619
1620 buf_size = min(size, (sizeof(buf) - 1));
1621 if (strncpy_from_user(buf, user_buf, buf_size) < 0)
1622 return -EFAULT;
1623 buf[buf_size] = 0;
1624
1625 ret = mutex_lock_interruptible(&scan_mutex);
1626 if (ret < 0)
1627 return ret;
1628
1629 if (strncmp(buf, "off", 3) == 0)
1630 kmemleak_disable();
1631 else if (strncmp(buf, "stack=on", 8) == 0)
1632 kmemleak_stack_scan = 1;
1633 else if (strncmp(buf, "stack=off", 9) == 0)
1634 kmemleak_stack_scan = 0;
1635 else if (strncmp(buf, "scan=on", 7) == 0)
1636 start_scan_thread();
1637 else if (strncmp(buf, "scan=off", 8) == 0)
1638 stop_scan_thread();
1639 else if (strncmp(buf, "scan=", 5) == 0) {
1640 unsigned long secs;
1641
1642 ret = kstrtoul(buf + 5, 0, &secs);
1643 if (ret < 0)
1644 goto out;
1645 stop_scan_thread();
1646 if (secs) {
1647 jiffies_scan_wait = msecs_to_jiffies(secs * 1000);
1648 start_scan_thread();
1649 }
1650 } else if (strncmp(buf, "scan", 4) == 0)
1651 kmemleak_scan();
1652 else if (strncmp(buf, "clear", 5) == 0)
1653 kmemleak_clear();
1654 else if (strncmp(buf, "dump=", 5) == 0)
1655 ret = dump_str_object_info(buf + 5);
1656 else
1657 ret = -EINVAL;
1658
1659out:
1660 mutex_unlock(&scan_mutex);
1661 if (ret < 0)
1662 return ret;
1663
1664
1665 *ppos += size;
1666 return size;
1667}
1668
1669static const struct file_operations kmemleak_fops = {
1670 .owner = THIS_MODULE,
1671 .open = kmemleak_open,
1672 .read = seq_read,
1673 .write = kmemleak_write,
1674 .llseek = seq_lseek,
1675 .release = kmemleak_release,
1676};
1677
1678
1679
1680
1681
1682
1683static void kmemleak_do_cleanup(struct work_struct *work)
1684{
1685 struct kmemleak_object *object;
1686 bool cleanup = scan_thread == NULL;
1687
1688 mutex_lock(&scan_mutex);
1689 stop_scan_thread();
1690
1691 if (cleanup) {
1692 rcu_read_lock();
1693 list_for_each_entry_rcu(object, &object_list, object_list)
1694 delete_object_full(object->pointer);
1695 rcu_read_unlock();
1696 }
1697 mutex_unlock(&scan_mutex);
1698}
1699
1700static DECLARE_WORK(cleanup_work, kmemleak_do_cleanup);
1701
1702
1703
1704
1705
1706static void kmemleak_disable(void)
1707{
1708
1709 if (atomic_cmpxchg(&kmemleak_error, 0, 1))
1710 return;
1711
1712
1713 atomic_set(&kmemleak_enabled, 0);
1714
1715
1716 if (atomic_read(&kmemleak_initialized))
1717 schedule_work(&cleanup_work);
1718
1719 pr_info("Kernel memory leak detector disabled\n");
1720}
1721
1722
1723
1724
1725static int kmemleak_boot_config(char *str)
1726{
1727 if (!str)
1728 return -EINVAL;
1729 if (strcmp(str, "off") == 0)
1730 kmemleak_disable();
1731 else if (strcmp(str, "on") == 0)
1732 kmemleak_skip_disable = 1;
1733 else
1734 return -EINVAL;
1735 return 0;
1736}
1737early_param("kmemleak", kmemleak_boot_config);
1738
1739static void __init print_log_trace(struct early_log *log)
1740{
1741 struct stack_trace trace;
1742
1743 trace.nr_entries = log->trace_len;
1744 trace.entries = log->trace;
1745
1746 pr_notice("Early log backtrace:\n");
1747 print_stack_trace(&trace, 2);
1748}
1749
1750
1751
1752
1753void __init kmemleak_init(void)
1754{
1755 int i;
1756 unsigned long flags;
1757
1758#ifdef CONFIG_DEBUG_KMEMLEAK_DEFAULT_OFF
1759 if (!kmemleak_skip_disable) {
1760 atomic_set(&kmemleak_early_log, 0);
1761 kmemleak_disable();
1762 return;
1763 }
1764#endif
1765
1766 jiffies_min_age = msecs_to_jiffies(MSECS_MIN_AGE);
1767 jiffies_scan_wait = msecs_to_jiffies(SECS_SCAN_WAIT * 1000);
1768
1769 object_cache = KMEM_CACHE(kmemleak_object, SLAB_NOLEAKTRACE);
1770 scan_area_cache = KMEM_CACHE(kmemleak_scan_area, SLAB_NOLEAKTRACE);
1771
1772 if (crt_early_log >= ARRAY_SIZE(early_log))
1773 pr_warning("Early log buffer exceeded (%d), please increase "
1774 "DEBUG_KMEMLEAK_EARLY_LOG_SIZE\n", crt_early_log);
1775
1776
1777 local_irq_save(flags);
1778 atomic_set(&kmemleak_early_log, 0);
1779 if (atomic_read(&kmemleak_error)) {
1780 local_irq_restore(flags);
1781 return;
1782 } else
1783 atomic_set(&kmemleak_enabled, 1);
1784 local_irq_restore(flags);
1785
1786
1787
1788
1789
1790
1791 for (i = 0; i < crt_early_log; i++) {
1792 struct early_log *log = &early_log[i];
1793
1794 switch (log->op_type) {
1795 case KMEMLEAK_ALLOC:
1796 early_alloc(log);
1797 break;
1798 case KMEMLEAK_ALLOC_PERCPU:
1799 early_alloc_percpu(log);
1800 break;
1801 case KMEMLEAK_FREE:
1802 kmemleak_free(log->ptr);
1803 break;
1804 case KMEMLEAK_FREE_PART:
1805 kmemleak_free_part(log->ptr, log->size);
1806 break;
1807 case KMEMLEAK_FREE_PERCPU:
1808 kmemleak_free_percpu(log->ptr);
1809 break;
1810 case KMEMLEAK_NOT_LEAK:
1811 kmemleak_not_leak(log->ptr);
1812 break;
1813 case KMEMLEAK_IGNORE:
1814 kmemleak_ignore(log->ptr);
1815 break;
1816 case KMEMLEAK_SCAN_AREA:
1817 kmemleak_scan_area(log->ptr, log->size, GFP_KERNEL);
1818 break;
1819 case KMEMLEAK_NO_SCAN:
1820 kmemleak_no_scan(log->ptr);
1821 break;
1822 default:
1823 kmemleak_warn("Unknown early log operation: %d\n",
1824 log->op_type);
1825 }
1826
1827 if (atomic_read(&kmemleak_warning)) {
1828 print_log_trace(log);
1829 atomic_set(&kmemleak_warning, 0);
1830 }
1831 }
1832}
1833
1834
1835
1836
1837static int __init kmemleak_late_init(void)
1838{
1839 struct dentry *dentry;
1840
1841 atomic_set(&kmemleak_initialized, 1);
1842
1843 if (atomic_read(&kmemleak_error)) {
1844
1845
1846
1847
1848
1849
1850 schedule_work(&cleanup_work);
1851 return -ENOMEM;
1852 }
1853
1854 dentry = debugfs_create_file("kmemleak", S_IRUGO, NULL, NULL,
1855 &kmemleak_fops);
1856 if (!dentry)
1857 pr_warning("Failed to create the debugfs kmemleak file\n");
1858 mutex_lock(&scan_mutex);
1859 start_scan_thread();
1860 mutex_unlock(&scan_mutex);
1861
1862 pr_info("Kernel memory leak detector initialized\n");
1863
1864 return 0;
1865}
1866late_initcall(kmemleak_late_init);
1867