1
2
3
4
5
6
7
8
9
10
11#define pr_fmt(fmt) "ODEBUG: " fmt
12
13#include <linux/debugobjects.h>
14#include <linux/interrupt.h>
15#include <linux/sched.h>
16#include <linux/sched/task_stack.h>
17#include <linux/seq_file.h>
18#include <linux/debugfs.h>
19#include <linux/slab.h>
20#include <linux/hash.h>
21#include <linux/kmemleak.h>
22
23#define ODEBUG_HASH_BITS 14
24#define ODEBUG_HASH_SIZE (1 << ODEBUG_HASH_BITS)
25
26#define ODEBUG_POOL_SIZE 1024
27#define ODEBUG_POOL_MIN_LEVEL 256
28#define ODEBUG_POOL_PERCPU_SIZE 64
29#define ODEBUG_BATCH_SIZE 16
30
31#define ODEBUG_CHUNK_SHIFT PAGE_SHIFT
32#define ODEBUG_CHUNK_SIZE (1 << ODEBUG_CHUNK_SHIFT)
33#define ODEBUG_CHUNK_MASK (~(ODEBUG_CHUNK_SIZE - 1))
34
35
36
37
38
39
40#define ODEBUG_FREE_WORK_MAX 1024
41#define ODEBUG_FREE_WORK_DELAY DIV_ROUND_UP(HZ, 10)
42
43struct debug_bucket {
44 struct hlist_head list;
45 raw_spinlock_t lock;
46};
47
48
49
50
51
52struct debug_percpu_free {
53 struct hlist_head free_objs;
54 int obj_free;
55};
56
57static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
58
59static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE];
60
61static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata;
62
63static DEFINE_RAW_SPINLOCK(pool_lock);
64
65static HLIST_HEAD(obj_pool);
66static HLIST_HEAD(obj_to_free);
67
68
69
70
71
72
73
74
75static int obj_pool_min_free = ODEBUG_POOL_SIZE;
76static int obj_pool_free = ODEBUG_POOL_SIZE;
77static int obj_pool_used;
78static int obj_pool_max_used;
79static bool obj_freeing;
80
81static int obj_nr_tofree;
82
83static int debug_objects_maxchain __read_mostly;
84static int __maybe_unused debug_objects_maxchecked __read_mostly;
85static int debug_objects_fixups __read_mostly;
86static int debug_objects_warnings __read_mostly;
87static int debug_objects_enabled __read_mostly
88 = CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT;
89static int debug_objects_pool_size __read_mostly
90 = ODEBUG_POOL_SIZE;
91static int debug_objects_pool_min_level __read_mostly
92 = ODEBUG_POOL_MIN_LEVEL;
93static struct debug_obj_descr *descr_test __read_mostly;
94static struct kmem_cache *obj_cache __read_mostly;
95
96
97
98
99static int debug_objects_allocated;
100static int debug_objects_freed;
101
102static void free_obj_work(struct work_struct *work);
103static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work);
104
105static int __init enable_object_debug(char *str)
106{
107 debug_objects_enabled = 1;
108 return 0;
109}
110
111static int __init disable_object_debug(char *str)
112{
113 debug_objects_enabled = 0;
114 return 0;
115}
116
117early_param("debug_objects", enable_object_debug);
118early_param("no_debug_objects", disable_object_debug);
119
120static const char *obj_states[ODEBUG_STATE_MAX] = {
121 [ODEBUG_STATE_NONE] = "none",
122 [ODEBUG_STATE_INIT] = "initialized",
123 [ODEBUG_STATE_INACTIVE] = "inactive",
124 [ODEBUG_STATE_ACTIVE] = "active",
125 [ODEBUG_STATE_DESTROYED] = "destroyed",
126 [ODEBUG_STATE_NOTAVAILABLE] = "not available",
127};
128
129static void fill_pool(void)
130{
131 gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
132 struct debug_obj *obj;
133 unsigned long flags;
134
135 if (likely(obj_pool_free >= debug_objects_pool_min_level))
136 return;
137
138
139
140
141
142 while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
143 raw_spin_lock_irqsave(&pool_lock, flags);
144
145
146
147
148 while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
149 obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
150 hlist_del(&obj->node);
151 obj_nr_tofree--;
152 hlist_add_head(&obj->node, &obj_pool);
153 obj_pool_free++;
154 }
155 raw_spin_unlock_irqrestore(&pool_lock, flags);
156 }
157
158 if (unlikely(!obj_cache))
159 return;
160
161 while (obj_pool_free < debug_objects_pool_min_level) {
162 struct debug_obj *new[ODEBUG_BATCH_SIZE];
163 int cnt;
164
165 for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
166 new[cnt] = kmem_cache_zalloc(obj_cache, gfp);
167 if (!new[cnt])
168 break;
169 }
170 if (!cnt)
171 return;
172
173 raw_spin_lock_irqsave(&pool_lock, flags);
174 while (cnt) {
175 hlist_add_head(&new[--cnt]->node, &obj_pool);
176 debug_objects_allocated++;
177 obj_pool_free++;
178 }
179 raw_spin_unlock_irqrestore(&pool_lock, flags);
180 }
181}
182
183
184
185
186static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b)
187{
188 struct debug_obj *obj;
189 int cnt = 0;
190
191 hlist_for_each_entry(obj, &b->list, node) {
192 cnt++;
193 if (obj->object == addr)
194 return obj;
195 }
196 if (cnt > debug_objects_maxchain)
197 debug_objects_maxchain = cnt;
198
199 return NULL;
200}
201
202
203
204
205static struct debug_obj *__alloc_object(struct hlist_head *list)
206{
207 struct debug_obj *obj = NULL;
208
209 if (list->first) {
210 obj = hlist_entry(list->first, typeof(*obj), node);
211 hlist_del(&obj->node);
212 }
213
214 return obj;
215}
216
217
218
219
220
221static struct debug_obj *
222alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
223{
224 struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool);
225 struct debug_obj *obj;
226
227 if (likely(obj_cache)) {
228 obj = __alloc_object(&percpu_pool->free_objs);
229 if (obj) {
230 percpu_pool->obj_free--;
231 goto init_obj;
232 }
233 }
234
235 raw_spin_lock(&pool_lock);
236 obj = __alloc_object(&obj_pool);
237 if (obj) {
238 obj_pool_used++;
239 obj_pool_free--;
240
241
242
243
244
245 if (likely(obj_cache)) {
246 int i;
247
248 for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
249 struct debug_obj *obj2;
250
251 obj2 = __alloc_object(&obj_pool);
252 if (!obj2)
253 break;
254 hlist_add_head(&obj2->node,
255 &percpu_pool->free_objs);
256 percpu_pool->obj_free++;
257 obj_pool_used++;
258 obj_pool_free--;
259 }
260 }
261
262 if (obj_pool_used > obj_pool_max_used)
263 obj_pool_max_used = obj_pool_used;
264
265 if (obj_pool_free < obj_pool_min_free)
266 obj_pool_min_free = obj_pool_free;
267 }
268 raw_spin_unlock(&pool_lock);
269
270init_obj:
271 if (obj) {
272 obj->object = addr;
273 obj->descr = descr;
274 obj->state = ODEBUG_STATE_NONE;
275 obj->astate = 0;
276 hlist_add_head(&obj->node, &b->list);
277 }
278 return obj;
279}
280
281
282
283
284
285
286
287static void free_obj_work(struct work_struct *work)
288{
289 struct hlist_node *tmp;
290 struct debug_obj *obj;
291 unsigned long flags;
292 HLIST_HEAD(tofree);
293
294 WRITE_ONCE(obj_freeing, false);
295 if (!raw_spin_trylock_irqsave(&pool_lock, flags))
296 return;
297
298 if (obj_pool_free >= debug_objects_pool_size)
299 goto free_objs;
300
301
302
303
304
305
306
307
308 while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
309 obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
310 hlist_del(&obj->node);
311 hlist_add_head(&obj->node, &obj_pool);
312 obj_pool_free++;
313 obj_nr_tofree--;
314 }
315 raw_spin_unlock_irqrestore(&pool_lock, flags);
316 return;
317
318free_objs:
319
320
321
322
323
324 if (obj_nr_tofree) {
325 hlist_move_list(&obj_to_free, &tofree);
326 debug_objects_freed += obj_nr_tofree;
327 obj_nr_tofree = 0;
328 }
329 raw_spin_unlock_irqrestore(&pool_lock, flags);
330
331 hlist_for_each_entry_safe(obj, tmp, &tofree, node) {
332 hlist_del(&obj->node);
333 kmem_cache_free(obj_cache, obj);
334 }
335}
336
337static void __free_object(struct debug_obj *obj)
338{
339 struct debug_obj *objs[ODEBUG_BATCH_SIZE];
340 struct debug_percpu_free *percpu_pool;
341 int lookahead_count = 0;
342 unsigned long flags;
343 bool work;
344
345 local_irq_save(flags);
346 if (!obj_cache)
347 goto free_to_obj_pool;
348
349
350
351
352 percpu_pool = this_cpu_ptr(&percpu_obj_pool);
353 if (percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) {
354 hlist_add_head(&obj->node, &percpu_pool->free_objs);
355 percpu_pool->obj_free++;
356 local_irq_restore(flags);
357 return;
358 }
359
360
361
362
363
364 for (; lookahead_count < ODEBUG_BATCH_SIZE; lookahead_count++) {
365 objs[lookahead_count] = __alloc_object(&percpu_pool->free_objs);
366 if (!objs[lookahead_count])
367 break;
368 percpu_pool->obj_free--;
369 }
370
371free_to_obj_pool:
372 raw_spin_lock(&pool_lock);
373 work = (obj_pool_free > debug_objects_pool_size) && obj_cache &&
374 (obj_nr_tofree < ODEBUG_FREE_WORK_MAX);
375 obj_pool_used--;
376
377 if (work) {
378 obj_nr_tofree++;
379 hlist_add_head(&obj->node, &obj_to_free);
380 if (lookahead_count) {
381 obj_nr_tofree += lookahead_count;
382 obj_pool_used -= lookahead_count;
383 while (lookahead_count) {
384 hlist_add_head(&objs[--lookahead_count]->node,
385 &obj_to_free);
386 }
387 }
388
389 if ((obj_pool_free > debug_objects_pool_size) &&
390 (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) {
391 int i;
392
393
394
395
396 for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
397 obj = __alloc_object(&obj_pool);
398 hlist_add_head(&obj->node, &obj_to_free);
399 obj_pool_free--;
400 obj_nr_tofree++;
401 }
402 }
403 } else {
404 obj_pool_free++;
405 hlist_add_head(&obj->node, &obj_pool);
406 if (lookahead_count) {
407 obj_pool_free += lookahead_count;
408 obj_pool_used -= lookahead_count;
409 while (lookahead_count) {
410 hlist_add_head(&objs[--lookahead_count]->node,
411 &obj_pool);
412 }
413 }
414 }
415 raw_spin_unlock(&pool_lock);
416 local_irq_restore(flags);
417}
418
419
420
421
422
423static void free_object(struct debug_obj *obj)
424{
425 __free_object(obj);
426 if (!obj_freeing && obj_nr_tofree) {
427 WRITE_ONCE(obj_freeing, true);
428 schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
429 }
430}
431
432
433
434
435
436static void debug_objects_oom(void)
437{
438 struct debug_bucket *db = obj_hash;
439 struct hlist_node *tmp;
440 HLIST_HEAD(freelist);
441 struct debug_obj *obj;
442 unsigned long flags;
443 int i;
444
445 pr_warn("Out of memory. ODEBUG disabled\n");
446
447 for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
448 raw_spin_lock_irqsave(&db->lock, flags);
449 hlist_move_list(&db->list, &freelist);
450 raw_spin_unlock_irqrestore(&db->lock, flags);
451
452
453 hlist_for_each_entry_safe(obj, tmp, &freelist, node) {
454 hlist_del(&obj->node);
455 free_object(obj);
456 }
457 }
458}
459
460
461
462
463
464static struct debug_bucket *get_bucket(unsigned long addr)
465{
466 unsigned long hash;
467
468 hash = hash_long((addr >> ODEBUG_CHUNK_SHIFT), ODEBUG_HASH_BITS);
469 return &obj_hash[hash];
470}
471
472static void debug_print_object(struct debug_obj *obj, char *msg)
473{
474 struct debug_obj_descr *descr = obj->descr;
475 static int limit;
476
477 if (limit < 5 && descr != descr_test) {
478 void *hint = descr->debug_hint ?
479 descr->debug_hint(obj->object) : NULL;
480 limit++;
481 WARN(1, KERN_ERR "ODEBUG: %s %s (active state %u) "
482 "object type: %s hint: %pS\n",
483 msg, obj_states[obj->state], obj->astate,
484 descr->name, hint);
485 }
486 debug_objects_warnings++;
487}
488
489
490
491
492
493static bool
494debug_object_fixup(bool (*fixup)(void *addr, enum debug_obj_state state),
495 void * addr, enum debug_obj_state state)
496{
497 if (fixup && fixup(addr, state)) {
498 debug_objects_fixups++;
499 return true;
500 }
501 return false;
502}
503
504static void debug_object_is_on_stack(void *addr, int onstack)
505{
506 int is_on_stack;
507 static int limit;
508
509 if (limit > 4)
510 return;
511
512 is_on_stack = object_is_on_stack(addr);
513 if (is_on_stack == onstack)
514 return;
515
516 limit++;
517 if (is_on_stack)
518 pr_warn("object %p is on stack %p, but NOT annotated.\n", addr,
519 task_stack_page(current));
520 else
521 pr_warn("object %p is NOT on stack %p, but annotated.\n", addr,
522 task_stack_page(current));
523
524 WARN_ON(1);
525}
526
527static void
528__debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack)
529{
530 enum debug_obj_state state;
531 bool check_stack = false;
532 struct debug_bucket *db;
533 struct debug_obj *obj;
534 unsigned long flags;
535
536 fill_pool();
537
538 db = get_bucket((unsigned long) addr);
539
540 raw_spin_lock_irqsave(&db->lock, flags);
541
542 obj = lookup_object(addr, db);
543 if (!obj) {
544 obj = alloc_object(addr, db, descr);
545 if (!obj) {
546 debug_objects_enabled = 0;
547 raw_spin_unlock_irqrestore(&db->lock, flags);
548 debug_objects_oom();
549 return;
550 }
551 check_stack = true;
552 }
553
554 switch (obj->state) {
555 case ODEBUG_STATE_NONE:
556 case ODEBUG_STATE_INIT:
557 case ODEBUG_STATE_INACTIVE:
558 obj->state = ODEBUG_STATE_INIT;
559 break;
560
561 case ODEBUG_STATE_ACTIVE:
562 state = obj->state;
563 raw_spin_unlock_irqrestore(&db->lock, flags);
564 debug_print_object(obj, "init");
565 debug_object_fixup(descr->fixup_init, addr, state);
566 return;
567
568 case ODEBUG_STATE_DESTROYED:
569 raw_spin_unlock_irqrestore(&db->lock, flags);
570 debug_print_object(obj, "init");
571 return;
572 default:
573 break;
574 }
575
576 raw_spin_unlock_irqrestore(&db->lock, flags);
577 if (check_stack)
578 debug_object_is_on_stack(addr, onstack);
579}
580
581
582
583
584
585
586void debug_object_init(void *addr, struct debug_obj_descr *descr)
587{
588 if (!debug_objects_enabled)
589 return;
590
591 __debug_object_init(addr, descr, 0);
592}
593EXPORT_SYMBOL_GPL(debug_object_init);
594
595
596
597
598
599
600
601void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr)
602{
603 if (!debug_objects_enabled)
604 return;
605
606 __debug_object_init(addr, descr, 1);
607}
608EXPORT_SYMBOL_GPL(debug_object_init_on_stack);
609
610
611
612
613
614
615
616int debug_object_activate(void *addr, struct debug_obj_descr *descr)
617{
618 enum debug_obj_state state;
619 struct debug_bucket *db;
620 struct debug_obj *obj;
621 unsigned long flags;
622 int ret;
623 struct debug_obj o = { .object = addr,
624 .state = ODEBUG_STATE_NOTAVAILABLE,
625 .descr = descr };
626
627 if (!debug_objects_enabled)
628 return 0;
629
630 db = get_bucket((unsigned long) addr);
631
632 raw_spin_lock_irqsave(&db->lock, flags);
633
634 obj = lookup_object(addr, db);
635 if (obj) {
636 bool print_object = false;
637
638 switch (obj->state) {
639 case ODEBUG_STATE_INIT:
640 case ODEBUG_STATE_INACTIVE:
641 obj->state = ODEBUG_STATE_ACTIVE;
642 ret = 0;
643 break;
644
645 case ODEBUG_STATE_ACTIVE:
646 state = obj->state;
647 raw_spin_unlock_irqrestore(&db->lock, flags);
648 debug_print_object(obj, "activate");
649 ret = debug_object_fixup(descr->fixup_activate, addr, state);
650 return ret ? 0 : -EINVAL;
651
652 case ODEBUG_STATE_DESTROYED:
653 print_object = true;
654 ret = -EINVAL;
655 break;
656 default:
657 ret = 0;
658 break;
659 }
660 raw_spin_unlock_irqrestore(&db->lock, flags);
661 if (print_object)
662 debug_print_object(obj, "activate");
663 return ret;
664 }
665
666 raw_spin_unlock_irqrestore(&db->lock, flags);
667
668
669
670
671
672
673
674
675 if (descr->is_static_object && descr->is_static_object(addr)) {
676
677 debug_object_init(addr, descr);
678 debug_object_activate(addr, descr);
679 } else {
680 debug_print_object(&o, "activate");
681 ret = debug_object_fixup(descr->fixup_activate, addr,
682 ODEBUG_STATE_NOTAVAILABLE);
683 return ret ? 0 : -EINVAL;
684 }
685 return 0;
686}
687EXPORT_SYMBOL_GPL(debug_object_activate);
688
689
690
691
692
693
694void debug_object_deactivate(void *addr, struct debug_obj_descr *descr)
695{
696 struct debug_bucket *db;
697 struct debug_obj *obj;
698 unsigned long flags;
699 bool print_object = false;
700
701 if (!debug_objects_enabled)
702 return;
703
704 db = get_bucket((unsigned long) addr);
705
706 raw_spin_lock_irqsave(&db->lock, flags);
707
708 obj = lookup_object(addr, db);
709 if (obj) {
710 switch (obj->state) {
711 case ODEBUG_STATE_INIT:
712 case ODEBUG_STATE_INACTIVE:
713 case ODEBUG_STATE_ACTIVE:
714 if (!obj->astate)
715 obj->state = ODEBUG_STATE_INACTIVE;
716 else
717 print_object = true;
718 break;
719
720 case ODEBUG_STATE_DESTROYED:
721 print_object = true;
722 break;
723 default:
724 break;
725 }
726 }
727
728 raw_spin_unlock_irqrestore(&db->lock, flags);
729 if (!obj) {
730 struct debug_obj o = { .object = addr,
731 .state = ODEBUG_STATE_NOTAVAILABLE,
732 .descr = descr };
733
734 debug_print_object(&o, "deactivate");
735 } else if (print_object) {
736 debug_print_object(obj, "deactivate");
737 }
738}
739EXPORT_SYMBOL_GPL(debug_object_deactivate);
740
741
742
743
744
745
746void debug_object_destroy(void *addr, struct debug_obj_descr *descr)
747{
748 enum debug_obj_state state;
749 struct debug_bucket *db;
750 struct debug_obj *obj;
751 unsigned long flags;
752 bool print_object = false;
753
754 if (!debug_objects_enabled)
755 return;
756
757 db = get_bucket((unsigned long) addr);
758
759 raw_spin_lock_irqsave(&db->lock, flags);
760
761 obj = lookup_object(addr, db);
762 if (!obj)
763 goto out_unlock;
764
765 switch (obj->state) {
766 case ODEBUG_STATE_NONE:
767 case ODEBUG_STATE_INIT:
768 case ODEBUG_STATE_INACTIVE:
769 obj->state = ODEBUG_STATE_DESTROYED;
770 break;
771 case ODEBUG_STATE_ACTIVE:
772 state = obj->state;
773 raw_spin_unlock_irqrestore(&db->lock, flags);
774 debug_print_object(obj, "destroy");
775 debug_object_fixup(descr->fixup_destroy, addr, state);
776 return;
777
778 case ODEBUG_STATE_DESTROYED:
779 print_object = true;
780 break;
781 default:
782 break;
783 }
784out_unlock:
785 raw_spin_unlock_irqrestore(&db->lock, flags);
786 if (print_object)
787 debug_print_object(obj, "destroy");
788}
789EXPORT_SYMBOL_GPL(debug_object_destroy);
790
791
792
793
794
795
796void debug_object_free(void *addr, struct debug_obj_descr *descr)
797{
798 enum debug_obj_state state;
799 struct debug_bucket *db;
800 struct debug_obj *obj;
801 unsigned long flags;
802
803 if (!debug_objects_enabled)
804 return;
805
806 db = get_bucket((unsigned long) addr);
807
808 raw_spin_lock_irqsave(&db->lock, flags);
809
810 obj = lookup_object(addr, db);
811 if (!obj)
812 goto out_unlock;
813
814 switch (obj->state) {
815 case ODEBUG_STATE_ACTIVE:
816 state = obj->state;
817 raw_spin_unlock_irqrestore(&db->lock, flags);
818 debug_print_object(obj, "free");
819 debug_object_fixup(descr->fixup_free, addr, state);
820 return;
821 default:
822 hlist_del(&obj->node);
823 raw_spin_unlock_irqrestore(&db->lock, flags);
824 free_object(obj);
825 return;
826 }
827out_unlock:
828 raw_spin_unlock_irqrestore(&db->lock, flags);
829}
830EXPORT_SYMBOL_GPL(debug_object_free);
831
832
833
834
835
836
837void debug_object_assert_init(void *addr, struct debug_obj_descr *descr)
838{
839 struct debug_bucket *db;
840 struct debug_obj *obj;
841 unsigned long flags;
842
843 if (!debug_objects_enabled)
844 return;
845
846 db = get_bucket((unsigned long) addr);
847
848 raw_spin_lock_irqsave(&db->lock, flags);
849
850 obj = lookup_object(addr, db);
851 if (!obj) {
852 struct debug_obj o = { .object = addr,
853 .state = ODEBUG_STATE_NOTAVAILABLE,
854 .descr = descr };
855
856 raw_spin_unlock_irqrestore(&db->lock, flags);
857
858
859
860
861
862 if (descr->is_static_object && descr->is_static_object(addr)) {
863
864 debug_object_init(addr, descr);
865 } else {
866 debug_print_object(&o, "assert_init");
867 debug_object_fixup(descr->fixup_assert_init, addr,
868 ODEBUG_STATE_NOTAVAILABLE);
869 }
870 return;
871 }
872
873 raw_spin_unlock_irqrestore(&db->lock, flags);
874}
875EXPORT_SYMBOL_GPL(debug_object_assert_init);
876
877
878
879
880
881
882
883
884void
885debug_object_active_state(void *addr, struct debug_obj_descr *descr,
886 unsigned int expect, unsigned int next)
887{
888 struct debug_bucket *db;
889 struct debug_obj *obj;
890 unsigned long flags;
891 bool print_object = false;
892
893 if (!debug_objects_enabled)
894 return;
895
896 db = get_bucket((unsigned long) addr);
897
898 raw_spin_lock_irqsave(&db->lock, flags);
899
900 obj = lookup_object(addr, db);
901 if (obj) {
902 switch (obj->state) {
903 case ODEBUG_STATE_ACTIVE:
904 if (obj->astate == expect)
905 obj->astate = next;
906 else
907 print_object = true;
908 break;
909
910 default:
911 print_object = true;
912 break;
913 }
914 }
915
916 raw_spin_unlock_irqrestore(&db->lock, flags);
917 if (!obj) {
918 struct debug_obj o = { .object = addr,
919 .state = ODEBUG_STATE_NOTAVAILABLE,
920 .descr = descr };
921
922 debug_print_object(&o, "active_state");
923 } else if (print_object) {
924 debug_print_object(obj, "active_state");
925 }
926}
927EXPORT_SYMBOL_GPL(debug_object_active_state);
928
929#ifdef CONFIG_DEBUG_OBJECTS_FREE
930static void __debug_check_no_obj_freed(const void *address, unsigned long size)
931{
932 unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
933 struct debug_obj_descr *descr;
934 enum debug_obj_state state;
935 struct debug_bucket *db;
936 struct hlist_node *tmp;
937 struct debug_obj *obj;
938 int cnt, objs_checked = 0;
939
940 saddr = (unsigned long) address;
941 eaddr = saddr + size;
942 paddr = saddr & ODEBUG_CHUNK_MASK;
943 chunks = ((eaddr - paddr) + (ODEBUG_CHUNK_SIZE - 1));
944 chunks >>= ODEBUG_CHUNK_SHIFT;
945
946 for (;chunks > 0; chunks--, paddr += ODEBUG_CHUNK_SIZE) {
947 db = get_bucket(paddr);
948
949repeat:
950 cnt = 0;
951 raw_spin_lock_irqsave(&db->lock, flags);
952 hlist_for_each_entry_safe(obj, tmp, &db->list, node) {
953 cnt++;
954 oaddr = (unsigned long) obj->object;
955 if (oaddr < saddr || oaddr >= eaddr)
956 continue;
957
958 switch (obj->state) {
959 case ODEBUG_STATE_ACTIVE:
960 descr = obj->descr;
961 state = obj->state;
962 raw_spin_unlock_irqrestore(&db->lock, flags);
963 debug_print_object(obj, "free");
964 debug_object_fixup(descr->fixup_free,
965 (void *) oaddr, state);
966 goto repeat;
967 default:
968 hlist_del(&obj->node);
969 __free_object(obj);
970 break;
971 }
972 }
973 raw_spin_unlock_irqrestore(&db->lock, flags);
974
975 if (cnt > debug_objects_maxchain)
976 debug_objects_maxchain = cnt;
977
978 objs_checked += cnt;
979 }
980
981 if (objs_checked > debug_objects_maxchecked)
982 debug_objects_maxchecked = objs_checked;
983
984
985 if (!obj_freeing && obj_nr_tofree) {
986 WRITE_ONCE(obj_freeing, true);
987 schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
988 }
989}
990
991void debug_check_no_obj_freed(const void *address, unsigned long size)
992{
993 if (debug_objects_enabled)
994 __debug_check_no_obj_freed(address, size);
995}
996#endif
997
998#ifdef CONFIG_DEBUG_FS
999
1000static int debug_stats_show(struct seq_file *m, void *v)
1001{
1002 int cpu, obj_percpu_free = 0;
1003
1004 for_each_possible_cpu(cpu)
1005 obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu);
1006
1007 seq_printf(m, "max_chain :%d\n", debug_objects_maxchain);
1008 seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked);
1009 seq_printf(m, "warnings :%d\n", debug_objects_warnings);
1010 seq_printf(m, "fixups :%d\n", debug_objects_fixups);
1011 seq_printf(m, "pool_free :%d\n", obj_pool_free + obj_percpu_free);
1012 seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
1013 seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
1014 seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free);
1015 seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
1016 seq_printf(m, "on_free_list :%d\n", obj_nr_tofree);
1017 seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
1018 seq_printf(m, "objs_freed :%d\n", debug_objects_freed);
1019 return 0;
1020}
1021
1022static int debug_stats_open(struct inode *inode, struct file *filp)
1023{
1024 return single_open(filp, debug_stats_show, NULL);
1025}
1026
1027static const struct file_operations debug_stats_fops = {
1028 .open = debug_stats_open,
1029 .read = seq_read,
1030 .llseek = seq_lseek,
1031 .release = single_release,
1032};
1033
1034static int __init debug_objects_init_debugfs(void)
1035{
1036 struct dentry *dbgdir;
1037
1038 if (!debug_objects_enabled)
1039 return 0;
1040
1041 dbgdir = debugfs_create_dir("debug_objects", NULL);
1042
1043 debugfs_create_file("stats", 0444, dbgdir, NULL, &debug_stats_fops);
1044
1045 return 0;
1046}
1047__initcall(debug_objects_init_debugfs);
1048
1049#else
1050static inline void debug_objects_init_debugfs(void) { }
1051#endif
1052
1053#ifdef CONFIG_DEBUG_OBJECTS_SELFTEST
1054
1055
1056struct self_test {
1057 unsigned long dummy1[6];
1058 int static_init;
1059 unsigned long dummy2[3];
1060};
1061
1062static __initdata struct debug_obj_descr descr_type_test;
1063
1064static bool __init is_static_object(void *addr)
1065{
1066 struct self_test *obj = addr;
1067
1068 return obj->static_init;
1069}
1070
1071
1072
1073
1074
1075static bool __init fixup_init(void *addr, enum debug_obj_state state)
1076{
1077 struct self_test *obj = addr;
1078
1079 switch (state) {
1080 case ODEBUG_STATE_ACTIVE:
1081 debug_object_deactivate(obj, &descr_type_test);
1082 debug_object_init(obj, &descr_type_test);
1083 return true;
1084 default:
1085 return false;
1086 }
1087}
1088
1089
1090
1091
1092
1093
1094static bool __init fixup_activate(void *addr, enum debug_obj_state state)
1095{
1096 struct self_test *obj = addr;
1097
1098 switch (state) {
1099 case ODEBUG_STATE_NOTAVAILABLE:
1100 return true;
1101 case ODEBUG_STATE_ACTIVE:
1102 debug_object_deactivate(obj, &descr_type_test);
1103 debug_object_activate(obj, &descr_type_test);
1104 return true;
1105
1106 default:
1107 return false;
1108 }
1109}
1110
1111
1112
1113
1114
1115static bool __init fixup_destroy(void *addr, enum debug_obj_state state)
1116{
1117 struct self_test *obj = addr;
1118
1119 switch (state) {
1120 case ODEBUG_STATE_ACTIVE:
1121 debug_object_deactivate(obj, &descr_type_test);
1122 debug_object_destroy(obj, &descr_type_test);
1123 return true;
1124 default:
1125 return false;
1126 }
1127}
1128
1129
1130
1131
1132
1133static bool __init fixup_free(void *addr, enum debug_obj_state state)
1134{
1135 struct self_test *obj = addr;
1136
1137 switch (state) {
1138 case ODEBUG_STATE_ACTIVE:
1139 debug_object_deactivate(obj, &descr_type_test);
1140 debug_object_free(obj, &descr_type_test);
1141 return true;
1142 default:
1143 return false;
1144 }
1145}
1146
1147static int __init
1148check_results(void *addr, enum debug_obj_state state, int fixups, int warnings)
1149{
1150 struct debug_bucket *db;
1151 struct debug_obj *obj;
1152 unsigned long flags;
1153 int res = -EINVAL;
1154
1155 db = get_bucket((unsigned long) addr);
1156
1157 raw_spin_lock_irqsave(&db->lock, flags);
1158
1159 obj = lookup_object(addr, db);
1160 if (!obj && state != ODEBUG_STATE_NONE) {
1161 WARN(1, KERN_ERR "ODEBUG: selftest object not found\n");
1162 goto out;
1163 }
1164 if (obj && obj->state != state) {
1165 WARN(1, KERN_ERR "ODEBUG: selftest wrong state: %d != %d\n",
1166 obj->state, state);
1167 goto out;
1168 }
1169 if (fixups != debug_objects_fixups) {
1170 WARN(1, KERN_ERR "ODEBUG: selftest fixups failed %d != %d\n",
1171 fixups, debug_objects_fixups);
1172 goto out;
1173 }
1174 if (warnings != debug_objects_warnings) {
1175 WARN(1, KERN_ERR "ODEBUG: selftest warnings failed %d != %d\n",
1176 warnings, debug_objects_warnings);
1177 goto out;
1178 }
1179 res = 0;
1180out:
1181 raw_spin_unlock_irqrestore(&db->lock, flags);
1182 if (res)
1183 debug_objects_enabled = 0;
1184 return res;
1185}
1186
1187static __initdata struct debug_obj_descr descr_type_test = {
1188 .name = "selftest",
1189 .is_static_object = is_static_object,
1190 .fixup_init = fixup_init,
1191 .fixup_activate = fixup_activate,
1192 .fixup_destroy = fixup_destroy,
1193 .fixup_free = fixup_free,
1194};
1195
1196static __initdata struct self_test obj = { .static_init = 0 };
1197
1198static void __init debug_objects_selftest(void)
1199{
1200 int fixups, oldfixups, warnings, oldwarnings;
1201 unsigned long flags;
1202
1203 local_irq_save(flags);
1204
1205 fixups = oldfixups = debug_objects_fixups;
1206 warnings = oldwarnings = debug_objects_warnings;
1207 descr_test = &descr_type_test;
1208
1209 debug_object_init(&obj, &descr_type_test);
1210 if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
1211 goto out;
1212 debug_object_activate(&obj, &descr_type_test);
1213 if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1214 goto out;
1215 debug_object_activate(&obj, &descr_type_test);
1216 if (check_results(&obj, ODEBUG_STATE_ACTIVE, ++fixups, ++warnings))
1217 goto out;
1218 debug_object_deactivate(&obj, &descr_type_test);
1219 if (check_results(&obj, ODEBUG_STATE_INACTIVE, fixups, warnings))
1220 goto out;
1221 debug_object_destroy(&obj, &descr_type_test);
1222 if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, warnings))
1223 goto out;
1224 debug_object_init(&obj, &descr_type_test);
1225 if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1226 goto out;
1227 debug_object_activate(&obj, &descr_type_test);
1228 if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1229 goto out;
1230 debug_object_deactivate(&obj, &descr_type_test);
1231 if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1232 goto out;
1233 debug_object_free(&obj, &descr_type_test);
1234 if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
1235 goto out;
1236
1237 obj.static_init = 1;
1238 debug_object_activate(&obj, &descr_type_test);
1239 if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1240 goto out;
1241 debug_object_init(&obj, &descr_type_test);
1242 if (check_results(&obj, ODEBUG_STATE_INIT, ++fixups, ++warnings))
1243 goto out;
1244 debug_object_free(&obj, &descr_type_test);
1245 if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
1246 goto out;
1247
1248#ifdef CONFIG_DEBUG_OBJECTS_FREE
1249 debug_object_init(&obj, &descr_type_test);
1250 if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
1251 goto out;
1252 debug_object_activate(&obj, &descr_type_test);
1253 if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1254 goto out;
1255 __debug_check_no_obj_freed(&obj, sizeof(obj));
1256 if (check_results(&obj, ODEBUG_STATE_NONE, ++fixups, ++warnings))
1257 goto out;
1258#endif
1259 pr_info("selftest passed\n");
1260
1261out:
1262 debug_objects_fixups = oldfixups;
1263 debug_objects_warnings = oldwarnings;
1264 descr_test = NULL;
1265
1266 local_irq_restore(flags);
1267}
1268#else
1269static inline void debug_objects_selftest(void) { }
1270#endif
1271
1272
1273
1274
1275
1276
1277void __init debug_objects_early_init(void)
1278{
1279 int i;
1280
1281 for (i = 0; i < ODEBUG_HASH_SIZE; i++)
1282 raw_spin_lock_init(&obj_hash[i].lock);
1283
1284 for (i = 0; i < ODEBUG_POOL_SIZE; i++)
1285 hlist_add_head(&obj_static_pool[i].node, &obj_pool);
1286}
1287
1288
1289
1290
1291static int __init debug_objects_replace_static_objects(void)
1292{
1293 struct debug_bucket *db = obj_hash;
1294 struct hlist_node *tmp;
1295 struct debug_obj *obj, *new;
1296 HLIST_HEAD(objects);
1297 int i, cnt = 0;
1298
1299 for (i = 0; i < ODEBUG_POOL_SIZE; i++) {
1300 obj = kmem_cache_zalloc(obj_cache, GFP_KERNEL);
1301 if (!obj)
1302 goto free;
1303 hlist_add_head(&obj->node, &objects);
1304 }
1305
1306
1307
1308
1309
1310
1311
1312
1313 hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
1314 hlist_del(&obj->node);
1315
1316 hlist_move_list(&objects, &obj_pool);
1317
1318
1319 for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
1320 hlist_move_list(&db->list, &objects);
1321
1322 hlist_for_each_entry(obj, &objects, node) {
1323 new = hlist_entry(obj_pool.first, typeof(*obj), node);
1324 hlist_del(&new->node);
1325
1326 *new = *obj;
1327 hlist_add_head(&new->node, &db->list);
1328 cnt++;
1329 }
1330 }
1331
1332 pr_debug("%d of %d active objects replaced\n",
1333 cnt, obj_pool_used);
1334 return 0;
1335free:
1336 hlist_for_each_entry_safe(obj, tmp, &objects, node) {
1337 hlist_del(&obj->node);
1338 kmem_cache_free(obj_cache, obj);
1339 }
1340 return -ENOMEM;
1341}
1342
1343
1344
1345
1346
1347
1348
1349void __init debug_objects_mem_init(void)
1350{
1351 int cpu, extras;
1352
1353 if (!debug_objects_enabled)
1354 return;
1355
1356
1357
1358
1359
1360
1361
1362 for_each_possible_cpu(cpu)
1363 INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu));
1364
1365 obj_cache = kmem_cache_create("debug_objects_cache",
1366 sizeof (struct debug_obj), 0,
1367 SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE,
1368 NULL);
1369
1370 if (!obj_cache || debug_objects_replace_static_objects()) {
1371 debug_objects_enabled = 0;
1372 kmem_cache_destroy(obj_cache);
1373 pr_warn("out of memory.\n");
1374 } else
1375 debug_objects_selftest();
1376
1377
1378
1379
1380
1381 extras = num_possible_cpus() * ODEBUG_BATCH_SIZE;
1382 debug_objects_pool_size += extras;
1383 debug_objects_pool_min_level += extras;
1384}
1385