1
2
3
4
5
6
7
8#define pr_fmt(fmt) "DMA-API: " fmt
9
10#include <linux/sched/task_stack.h>
11#include <linux/scatterlist.h>
12#include <linux/dma-map-ops.h>
13#include <linux/sched/task.h>
14#include <linux/stacktrace.h>
15#include <linux/spinlock.h>
16#include <linux/vmalloc.h>
17#include <linux/debugfs.h>
18#include <linux/uaccess.h>
19#include <linux/export.h>
20#include <linux/device.h>
21#include <linux/types.h>
22#include <linux/sched.h>
23#include <linux/ctype.h>
24#include <linux/list.h>
25#include <linux/slab.h>
26#include <asm/sections.h>
27#include "debug.h"
28
29#define HASH_SIZE 16384ULL
30#define HASH_FN_SHIFT 13
31#define HASH_FN_MASK (HASH_SIZE - 1)
32
33#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
34
35#define DMA_DEBUG_DYNAMIC_ENTRIES (PAGE_SIZE / sizeof(struct dma_debug_entry))
36
37enum {
38 dma_debug_single,
39 dma_debug_sg,
40 dma_debug_coherent,
41 dma_debug_resource,
42};
43
44enum map_err_types {
45 MAP_ERR_CHECK_NOT_APPLICABLE,
46 MAP_ERR_NOT_CHECKED,
47 MAP_ERR_CHECKED,
48};
49
50#define DMA_DEBUG_STACKTRACE_ENTRIES 5
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66struct dma_debug_entry {
67 struct list_head list;
68 struct device *dev;
69 u64 dev_addr;
70 u64 size;
71 int type;
72 int direction;
73 int sg_call_ents;
74 int sg_mapped_ents;
75 unsigned long pfn;
76 size_t offset;
77 enum map_err_types map_err_type;
78#ifdef CONFIG_STACKTRACE
79 unsigned int stack_len;
80 unsigned long stack_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
81#endif
82} ____cacheline_aligned_in_smp;
83
84typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *);
85
86struct hash_bucket {
87 struct list_head list;
88 spinlock_t lock;
89};
90
91
92static struct hash_bucket dma_entry_hash[HASH_SIZE];
93
94static LIST_HEAD(free_entries);
95
96static DEFINE_SPINLOCK(free_entries_lock);
97
98
99static bool global_disable __read_mostly;
100
101
102static bool dma_debug_initialized __read_mostly;
103
104static inline bool dma_debug_disabled(void)
105{
106 return global_disable || !dma_debug_initialized;
107}
108
109
110static u32 error_count;
111
112
113static u32 show_all_errors __read_mostly;
114
115static u32 show_num_errors = 1;
116
117static u32 num_free_entries;
118static u32 min_free_entries;
119static u32 nr_total_entries;
120
121
122static u32 nr_prealloc_entries = PREALLOC_DMA_DEBUG_ENTRIES;
123
124
125
126#define NAME_MAX_LEN 64
127
128static char current_driver_name[NAME_MAX_LEN] __read_mostly;
129static struct device_driver *current_driver __read_mostly;
130
131static DEFINE_RWLOCK(driver_name_lock);
132
133static const char *const maperr2str[] = {
134 [MAP_ERR_CHECK_NOT_APPLICABLE] = "dma map error check not applicable",
135 [MAP_ERR_NOT_CHECKED] = "dma map error not checked",
136 [MAP_ERR_CHECKED] = "dma map error checked",
137};
138
139static const char *type2name[] = {
140 [dma_debug_single] = "single",
141 [dma_debug_sg] = "scather-gather",
142 [dma_debug_coherent] = "coherent",
143 [dma_debug_resource] = "resource",
144};
145
146static const char *dir2name[] = {
147 [DMA_BIDIRECTIONAL] = "DMA_BIDIRECTIONAL",
148 [DMA_TO_DEVICE] = "DMA_TO_DEVICE",
149 [DMA_FROM_DEVICE] = "DMA_FROM_DEVICE",
150 [DMA_NONE] = "DMA_NONE",
151};
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166static inline void dump_entry_trace(struct dma_debug_entry *entry)
167{
168#ifdef CONFIG_STACKTRACE
169 if (entry) {
170 pr_warn("Mapped at:\n");
171 stack_trace_print(entry->stack_entries, entry->stack_len, 0);
172 }
173#endif
174}
175
176static bool driver_filter(struct device *dev)
177{
178 struct device_driver *drv;
179 unsigned long flags;
180 bool ret;
181
182
183 if (likely(!current_driver_name[0]))
184 return true;
185
186
187 if (current_driver && dev && dev->driver == current_driver)
188 return true;
189
190
191 if (!dev)
192 return false;
193
194 if (current_driver || !current_driver_name[0])
195 return false;
196
197
198 drv = dev->driver;
199 if (!drv)
200 return false;
201
202
203 read_lock_irqsave(&driver_name_lock, flags);
204
205 ret = false;
206 if (drv->name &&
207 strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {
208 current_driver = drv;
209 ret = true;
210 }
211
212 read_unlock_irqrestore(&driver_name_lock, flags);
213
214 return ret;
215}
216
217#define err_printk(dev, entry, format, arg...) do { \
218 error_count += 1; \
219 if (driver_filter(dev) && \
220 (show_all_errors || show_num_errors > 0)) { \
221 WARN(1, pr_fmt("%s %s: ") format, \
222 dev ? dev_driver_string(dev) : "NULL", \
223 dev ? dev_name(dev) : "NULL", ## arg); \
224 dump_entry_trace(entry); \
225 } \
226 if (!show_all_errors && show_num_errors > 0) \
227 show_num_errors -= 1; \
228 } while (0);
229
230
231
232
233
234
235
236static int hash_fn(struct dma_debug_entry *entry)
237{
238
239
240
241
242 return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
243}
244
245
246
247
248static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
249 unsigned long *flags)
250 __acquires(&dma_entry_hash[idx].lock)
251{
252 int idx = hash_fn(entry);
253 unsigned long __flags;
254
255 spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
256 *flags = __flags;
257 return &dma_entry_hash[idx];
258}
259
260
261
262
263static void put_hash_bucket(struct hash_bucket *bucket,
264 unsigned long flags)
265 __releases(&bucket->lock)
266{
267 spin_unlock_irqrestore(&bucket->lock, flags);
268}
269
270static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b)
271{
272 return ((a->dev_addr == b->dev_addr) &&
273 (a->dev == b->dev)) ? true : false;
274}
275
276static bool containing_match(struct dma_debug_entry *a,
277 struct dma_debug_entry *b)
278{
279 if (a->dev != b->dev)
280 return false;
281
282 if ((b->dev_addr <= a->dev_addr) &&
283 ((b->dev_addr + b->size) >= (a->dev_addr + a->size)))
284 return true;
285
286 return false;
287}
288
289
290
291
292static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket,
293 struct dma_debug_entry *ref,
294 match_fn match)
295{
296 struct dma_debug_entry *entry, *ret = NULL;
297 int matches = 0, match_lvl, last_lvl = -1;
298
299 list_for_each_entry(entry, &bucket->list, list) {
300 if (!match(ref, entry))
301 continue;
302
303
304
305
306
307
308
309
310
311
312
313 matches += 1;
314 match_lvl = 0;
315 entry->size == ref->size ? ++match_lvl : 0;
316 entry->type == ref->type ? ++match_lvl : 0;
317 entry->direction == ref->direction ? ++match_lvl : 0;
318 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;
319
320 if (match_lvl == 4) {
321
322 return entry;
323 } else if (match_lvl > last_lvl) {
324
325
326
327
328 last_lvl = match_lvl;
329 ret = entry;
330 }
331 }
332
333
334
335
336
337 ret = (matches == 1) ? ret : NULL;
338
339 return ret;
340}
341
342static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket,
343 struct dma_debug_entry *ref)
344{
345 return __hash_bucket_find(bucket, ref, exact_match);
346}
347
348static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket,
349 struct dma_debug_entry *ref,
350 unsigned long *flags)
351{
352
353 unsigned int max_range = dma_get_max_seg_size(ref->dev);
354 struct dma_debug_entry *entry, index = *ref;
355 unsigned int range = 0;
356
357 while (range <= max_range) {
358 entry = __hash_bucket_find(*bucket, ref, containing_match);
359
360 if (entry)
361 return entry;
362
363
364
365
366 put_hash_bucket(*bucket, *flags);
367 range += (1 << HASH_FN_SHIFT);
368 index.dev_addr -= (1 << HASH_FN_SHIFT);
369 *bucket = get_hash_bucket(&index, flags);
370 }
371
372 return NULL;
373}
374
375
376
377
378static void hash_bucket_add(struct hash_bucket *bucket,
379 struct dma_debug_entry *entry)
380{
381 list_add_tail(&entry->list, &bucket->list);
382}
383
384
385
386
387static void hash_bucket_del(struct dma_debug_entry *entry)
388{
389 list_del(&entry->list);
390}
391
392static unsigned long long phys_addr(struct dma_debug_entry *entry)
393{
394 if (entry->type == dma_debug_resource)
395 return __pfn_to_phys(entry->pfn) + entry->offset;
396
397 return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset;
398}
399
400
401
402
403void debug_dma_dump_mappings(struct device *dev)
404{
405 int idx;
406
407 for (idx = 0; idx < HASH_SIZE; idx++) {
408 struct hash_bucket *bucket = &dma_entry_hash[idx];
409 struct dma_debug_entry *entry;
410 unsigned long flags;
411
412 spin_lock_irqsave(&bucket->lock, flags);
413
414 list_for_each_entry(entry, &bucket->list, list) {
415 if (!dev || dev == entry->dev) {
416 dev_info(entry->dev,
417 "%s idx %d P=%Lx N=%lx D=%Lx L=%Lx %s %s\n",
418 type2name[entry->type], idx,
419 phys_addr(entry), entry->pfn,
420 entry->dev_addr, entry->size,
421 dir2name[entry->direction],
422 maperr2str[entry->map_err_type]);
423 }
424 }
425
426 spin_unlock_irqrestore(&bucket->lock, flags);
427 cond_resched();
428 }
429}
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451static RADIX_TREE(dma_active_cacheline, GFP_NOWAIT);
452static DEFINE_SPINLOCK(radix_lock);
453#define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1)
454#define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT)
455#define CACHELINES_PER_PAGE (1 << CACHELINE_PER_PAGE_SHIFT)
456
457static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)
458{
459 return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) +
460 (entry->offset >> L1_CACHE_SHIFT);
461}
462
463static int active_cacheline_read_overlap(phys_addr_t cln)
464{
465 int overlap = 0, i;
466
467 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
468 if (radix_tree_tag_get(&dma_active_cacheline, cln, i))
469 overlap |= 1 << i;
470 return overlap;
471}
472
473static int active_cacheline_set_overlap(phys_addr_t cln, int overlap)
474{
475 int i;
476
477 if (overlap > ACTIVE_CACHELINE_MAX_OVERLAP || overlap < 0)
478 return overlap;
479
480 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
481 if (overlap & 1 << i)
482 radix_tree_tag_set(&dma_active_cacheline, cln, i);
483 else
484 radix_tree_tag_clear(&dma_active_cacheline, cln, i);
485
486 return overlap;
487}
488
489static void active_cacheline_inc_overlap(phys_addr_t cln)
490{
491 int overlap = active_cacheline_read_overlap(cln);
492
493 overlap = active_cacheline_set_overlap(cln, ++overlap);
494
495
496
497
498 WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
499 pr_fmt("exceeded %d overlapping mappings of cacheline %pa\n"),
500 ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
501}
502
503static int active_cacheline_dec_overlap(phys_addr_t cln)
504{
505 int overlap = active_cacheline_read_overlap(cln);
506
507 return active_cacheline_set_overlap(cln, --overlap);
508}
509
510static int active_cacheline_insert(struct dma_debug_entry *entry)
511{
512 phys_addr_t cln = to_cacheline_number(entry);
513 unsigned long flags;
514 int rc;
515
516
517
518
519
520 if (entry->direction == DMA_TO_DEVICE)
521 return 0;
522
523 spin_lock_irqsave(&radix_lock, flags);
524 rc = radix_tree_insert(&dma_active_cacheline, cln, entry);
525 if (rc == -EEXIST)
526 active_cacheline_inc_overlap(cln);
527 spin_unlock_irqrestore(&radix_lock, flags);
528
529 return rc;
530}
531
532static void active_cacheline_remove(struct dma_debug_entry *entry)
533{
534 phys_addr_t cln = to_cacheline_number(entry);
535 unsigned long flags;
536
537
538 if (entry->direction == DMA_TO_DEVICE)
539 return;
540
541 spin_lock_irqsave(&radix_lock, flags);
542
543
544
545
546 if (active_cacheline_dec_overlap(cln) < 0)
547 radix_tree_delete(&dma_active_cacheline, cln);
548 spin_unlock_irqrestore(&radix_lock, flags);
549}
550
551
552
553
554
555static void add_dma_entry(struct dma_debug_entry *entry)
556{
557 struct hash_bucket *bucket;
558 unsigned long flags;
559 int rc;
560
561 bucket = get_hash_bucket(entry, &flags);
562 hash_bucket_add(bucket, entry);
563 put_hash_bucket(bucket, flags);
564
565 rc = active_cacheline_insert(entry);
566 if (rc == -ENOMEM) {
567 pr_err("cacheline tracking ENOMEM, dma-debug disabled\n");
568 global_disable = true;
569 } else if (rc == -EEXIST) {
570 pr_err("cacheline tracking EEXIST, overlapping mappings aren't supported\n");
571 }
572}
573
574static int dma_debug_create_entries(gfp_t gfp)
575{
576 struct dma_debug_entry *entry;
577 int i;
578
579 entry = (void *)get_zeroed_page(gfp);
580 if (!entry)
581 return -ENOMEM;
582
583 for (i = 0; i < DMA_DEBUG_DYNAMIC_ENTRIES; i++)
584 list_add_tail(&entry[i].list, &free_entries);
585
586 num_free_entries += DMA_DEBUG_DYNAMIC_ENTRIES;
587 nr_total_entries += DMA_DEBUG_DYNAMIC_ENTRIES;
588
589 return 0;
590}
591
592static struct dma_debug_entry *__dma_entry_alloc(void)
593{
594 struct dma_debug_entry *entry;
595
596 entry = list_entry(free_entries.next, struct dma_debug_entry, list);
597 list_del(&entry->list);
598 memset(entry, 0, sizeof(*entry));
599
600 num_free_entries -= 1;
601 if (num_free_entries < min_free_entries)
602 min_free_entries = num_free_entries;
603
604 return entry;
605}
606
607static void __dma_entry_alloc_check_leak(void)
608{
609 u32 tmp = nr_total_entries % nr_prealloc_entries;
610
611
612 if (tmp < DMA_DEBUG_DYNAMIC_ENTRIES) {
613 pr_info("dma_debug_entry pool grown to %u (%u00%%)\n",
614 nr_total_entries,
615 (nr_total_entries / nr_prealloc_entries));
616 }
617}
618
619
620
621
622
623
624static struct dma_debug_entry *dma_entry_alloc(void)
625{
626 struct dma_debug_entry *entry;
627 unsigned long flags;
628
629 spin_lock_irqsave(&free_entries_lock, flags);
630 if (num_free_entries == 0) {
631 if (dma_debug_create_entries(GFP_ATOMIC)) {
632 global_disable = true;
633 spin_unlock_irqrestore(&free_entries_lock, flags);
634 pr_err("debugging out of memory - disabling\n");
635 return NULL;
636 }
637 __dma_entry_alloc_check_leak();
638 }
639
640 entry = __dma_entry_alloc();
641
642 spin_unlock_irqrestore(&free_entries_lock, flags);
643
644#ifdef CONFIG_STACKTRACE
645 entry->stack_len = stack_trace_save(entry->stack_entries,
646 ARRAY_SIZE(entry->stack_entries),
647 1);
648#endif
649 return entry;
650}
651
652static void dma_entry_free(struct dma_debug_entry *entry)
653{
654 unsigned long flags;
655
656 active_cacheline_remove(entry);
657
658
659
660
661
662 spin_lock_irqsave(&free_entries_lock, flags);
663 list_add(&entry->list, &free_entries);
664 num_free_entries += 1;
665 spin_unlock_irqrestore(&free_entries_lock, flags);
666}
667
668
669
670
671
672
673
674
675
676static ssize_t filter_read(struct file *file, char __user *user_buf,
677 size_t count, loff_t *ppos)
678{
679 char buf[NAME_MAX_LEN + 1];
680 unsigned long flags;
681 int len;
682
683 if (!current_driver_name[0])
684 return 0;
685
686
687
688
689
690
691 read_lock_irqsave(&driver_name_lock, flags);
692 len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);
693 read_unlock_irqrestore(&driver_name_lock, flags);
694
695 return simple_read_from_buffer(user_buf, count, ppos, buf, len);
696}
697
698static ssize_t filter_write(struct file *file, const char __user *userbuf,
699 size_t count, loff_t *ppos)
700{
701 char buf[NAME_MAX_LEN];
702 unsigned long flags;
703 size_t len;
704 int i;
705
706
707
708
709
710
711
712 len = min(count, (size_t)(NAME_MAX_LEN - 1));
713 if (copy_from_user(buf, userbuf, len))
714 return -EFAULT;
715
716 buf[len] = 0;
717
718 write_lock_irqsave(&driver_name_lock, flags);
719
720
721
722
723
724
725
726
727
728 if (!isalnum(buf[0])) {
729
730
731
732
733
734 if (current_driver_name[0])
735 pr_info("switching off dma-debug driver filter\n");
736 current_driver_name[0] = 0;
737 current_driver = NULL;
738 goto out_unlock;
739 }
740
741
742
743
744
745 for (i = 0; i < NAME_MAX_LEN - 1; ++i) {
746 current_driver_name[i] = buf[i];
747 if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)
748 break;
749 }
750 current_driver_name[i] = 0;
751 current_driver = NULL;
752
753 pr_info("enable driver filter for driver [%s]\n",
754 current_driver_name);
755
756out_unlock:
757 write_unlock_irqrestore(&driver_name_lock, flags);
758
759 return count;
760}
761
762static const struct file_operations filter_fops = {
763 .read = filter_read,
764 .write = filter_write,
765 .llseek = default_llseek,
766};
767
768static int dump_show(struct seq_file *seq, void *v)
769{
770 int idx;
771
772 for (idx = 0; idx < HASH_SIZE; idx++) {
773 struct hash_bucket *bucket = &dma_entry_hash[idx];
774 struct dma_debug_entry *entry;
775 unsigned long flags;
776
777 spin_lock_irqsave(&bucket->lock, flags);
778 list_for_each_entry(entry, &bucket->list, list) {
779 seq_printf(seq,
780 "%s %s %s idx %d P=%llx N=%lx D=%llx L=%llx %s %s\n",
781 dev_name(entry->dev),
782 dev_driver_string(entry->dev),
783 type2name[entry->type], idx,
784 phys_addr(entry), entry->pfn,
785 entry->dev_addr, entry->size,
786 dir2name[entry->direction],
787 maperr2str[entry->map_err_type]);
788 }
789 spin_unlock_irqrestore(&bucket->lock, flags);
790 }
791 return 0;
792}
793DEFINE_SHOW_ATTRIBUTE(dump);
794
795static void dma_debug_fs_init(void)
796{
797 struct dentry *dentry = debugfs_create_dir("dma-api", NULL);
798
799 debugfs_create_bool("disabled", 0444, dentry, &global_disable);
800 debugfs_create_u32("error_count", 0444, dentry, &error_count);
801 debugfs_create_u32("all_errors", 0644, dentry, &show_all_errors);
802 debugfs_create_u32("num_errors", 0644, dentry, &show_num_errors);
803 debugfs_create_u32("num_free_entries", 0444, dentry, &num_free_entries);
804 debugfs_create_u32("min_free_entries", 0444, dentry, &min_free_entries);
805 debugfs_create_u32("nr_total_entries", 0444, dentry, &nr_total_entries);
806 debugfs_create_file("driver_filter", 0644, dentry, NULL, &filter_fops);
807 debugfs_create_file("dump", 0444, dentry, NULL, &dump_fops);
808}
809
810static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
811{
812 struct dma_debug_entry *entry;
813 unsigned long flags;
814 int count = 0, i;
815
816 for (i = 0; i < HASH_SIZE; ++i) {
817 spin_lock_irqsave(&dma_entry_hash[i].lock, flags);
818 list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
819 if (entry->dev == dev) {
820 count += 1;
821 *out_entry = entry;
822 }
823 }
824 spin_unlock_irqrestore(&dma_entry_hash[i].lock, flags);
825 }
826
827 return count;
828}
829
830static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
831{
832 struct device *dev = data;
833 struct dma_debug_entry *entry;
834 int count;
835
836 if (dma_debug_disabled())
837 return 0;
838
839 switch (action) {
840 case BUS_NOTIFY_UNBOUND_DRIVER:
841 count = device_dma_allocations(dev, &entry);
842 if (count == 0)
843 break;
844 err_printk(dev, entry, "device driver has pending "
845 "DMA allocations while released from device "
846 "[count=%d]\n"
847 "One of leaked entries details: "
848 "[device address=0x%016llx] [size=%llu bytes] "
849 "[mapped with %s] [mapped as %s]\n",
850 count, entry->dev_addr, entry->size,
851 dir2name[entry->direction], type2name[entry->type]);
852 break;
853 default:
854 break;
855 }
856
857 return 0;
858}
859
860void dma_debug_add_bus(struct bus_type *bus)
861{
862 struct notifier_block *nb;
863
864 if (dma_debug_disabled())
865 return;
866
867 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
868 if (nb == NULL) {
869 pr_err("dma_debug_add_bus: out of memory\n");
870 return;
871 }
872
873 nb->notifier_call = dma_debug_device_change;
874
875 bus_register_notifier(bus, nb);
876}
877
878static int dma_debug_init(void)
879{
880 int i, nr_pages;
881
882
883
884
885 if (global_disable)
886 return 0;
887
888 for (i = 0; i < HASH_SIZE; ++i) {
889 INIT_LIST_HEAD(&dma_entry_hash[i].list);
890 spin_lock_init(&dma_entry_hash[i].lock);
891 }
892
893 dma_debug_fs_init();
894
895 nr_pages = DIV_ROUND_UP(nr_prealloc_entries, DMA_DEBUG_DYNAMIC_ENTRIES);
896 for (i = 0; i < nr_pages; ++i)
897 dma_debug_create_entries(GFP_KERNEL);
898 if (num_free_entries >= nr_prealloc_entries) {
899 pr_info("preallocated %d debug entries\n", nr_total_entries);
900 } else if (num_free_entries > 0) {
901 pr_warn("%d debug entries requested but only %d allocated\n",
902 nr_prealloc_entries, nr_total_entries);
903 } else {
904 pr_err("debugging out of memory error - disabled\n");
905 global_disable = true;
906
907 return 0;
908 }
909 min_free_entries = num_free_entries;
910
911 dma_debug_initialized = true;
912
913 pr_info("debugging enabled by kernel config\n");
914 return 0;
915}
916core_initcall(dma_debug_init);
917
918static __init int dma_debug_cmdline(char *str)
919{
920 if (!str)
921 return -EINVAL;
922
923 if (strncmp(str, "off", 3) == 0) {
924 pr_info("debugging disabled on kernel command line\n");
925 global_disable = true;
926 }
927
928 return 0;
929}
930
931static __init int dma_debug_entries_cmdline(char *str)
932{
933 if (!str)
934 return -EINVAL;
935 if (!get_option(&str, &nr_prealloc_entries))
936 nr_prealloc_entries = PREALLOC_DMA_DEBUG_ENTRIES;
937 return 0;
938}
939
940__setup("dma_debug=", dma_debug_cmdline);
941__setup("dma_debug_entries=", dma_debug_entries_cmdline);
942
943static void check_unmap(struct dma_debug_entry *ref)
944{
945 struct dma_debug_entry *entry;
946 struct hash_bucket *bucket;
947 unsigned long flags;
948
949 bucket = get_hash_bucket(ref, &flags);
950 entry = bucket_find_exact(bucket, ref);
951
952 if (!entry) {
953
954 put_hash_bucket(bucket, flags);
955
956 if (dma_mapping_error(ref->dev, ref->dev_addr)) {
957 err_printk(ref->dev, NULL,
958 "device driver tries to free an "
959 "invalid DMA memory address\n");
960 } else {
961 err_printk(ref->dev, NULL,
962 "device driver tries to free DMA "
963 "memory it has not allocated [device "
964 "address=0x%016llx] [size=%llu bytes]\n",
965 ref->dev_addr, ref->size);
966 }
967 return;
968 }
969
970 if (ref->size != entry->size) {
971 err_printk(ref->dev, entry, "device driver frees "
972 "DMA memory with different size "
973 "[device address=0x%016llx] [map size=%llu bytes] "
974 "[unmap size=%llu bytes]\n",
975 ref->dev_addr, entry->size, ref->size);
976 }
977
978 if (ref->type != entry->type) {
979 err_printk(ref->dev, entry, "device driver frees "
980 "DMA memory with wrong function "
981 "[device address=0x%016llx] [size=%llu bytes] "
982 "[mapped as %s] [unmapped as %s]\n",
983 ref->dev_addr, ref->size,
984 type2name[entry->type], type2name[ref->type]);
985 } else if ((entry->type == dma_debug_coherent) &&
986 (phys_addr(ref) != phys_addr(entry))) {
987 err_printk(ref->dev, entry, "device driver frees "
988 "DMA memory with different CPU address "
989 "[device address=0x%016llx] [size=%llu bytes] "
990 "[cpu alloc address=0x%016llx] "
991 "[cpu free address=0x%016llx]",
992 ref->dev_addr, ref->size,
993 phys_addr(entry),
994 phys_addr(ref));
995 }
996
997 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
998 ref->sg_call_ents != entry->sg_call_ents) {
999 err_printk(ref->dev, entry, "device driver frees "
1000 "DMA sg list with different entry count "
1001 "[map count=%d] [unmap count=%d]\n",
1002 entry->sg_call_ents, ref->sg_call_ents);
1003 }
1004
1005
1006
1007
1008
1009 if (ref->direction != entry->direction) {
1010 err_printk(ref->dev, entry, "device driver frees "
1011 "DMA memory with different direction "
1012 "[device address=0x%016llx] [size=%llu bytes] "
1013 "[mapped with %s] [unmapped with %s]\n",
1014 ref->dev_addr, ref->size,
1015 dir2name[entry->direction],
1016 dir2name[ref->direction]);
1017 }
1018
1019
1020
1021
1022
1023
1024 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1025 err_printk(ref->dev, entry,
1026 "device driver failed to check map error"
1027 "[device address=0x%016llx] [size=%llu bytes] "
1028 "[mapped as %s]",
1029 ref->dev_addr, ref->size,
1030 type2name[entry->type]);
1031 }
1032
1033 hash_bucket_del(entry);
1034 dma_entry_free(entry);
1035
1036 put_hash_bucket(bucket, flags);
1037}
1038
1039static void check_for_stack(struct device *dev,
1040 struct page *page, size_t offset)
1041{
1042 void *addr;
1043 struct vm_struct *stack_vm_area = task_stack_vm_area(current);
1044
1045 if (!stack_vm_area) {
1046
1047 if (PageHighMem(page))
1048 return;
1049 addr = page_address(page) + offset;
1050 if (object_is_on_stack(addr))
1051 err_printk(dev, NULL, "device driver maps memory from stack [addr=%p]\n", addr);
1052 } else {
1053
1054 int i;
1055
1056 for (i = 0; i < stack_vm_area->nr_pages; i++) {
1057 if (page != stack_vm_area->pages[i])
1058 continue;
1059
1060 addr = (u8 *)current->stack + i * PAGE_SIZE + offset;
1061 err_printk(dev, NULL, "device driver maps memory from stack [probable addr=%p]\n", addr);
1062 break;
1063 }
1064 }
1065}
1066
1067static inline bool overlap(void *addr, unsigned long len, void *start, void *end)
1068{
1069 unsigned long a1 = (unsigned long)addr;
1070 unsigned long b1 = a1 + len;
1071 unsigned long a2 = (unsigned long)start;
1072 unsigned long b2 = (unsigned long)end;
1073
1074 return !(b1 <= a2 || a1 >= b2);
1075}
1076
1077static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
1078{
1079 if (overlap(addr, len, _stext, _etext) ||
1080 overlap(addr, len, __start_rodata, __end_rodata))
1081 err_printk(dev, NULL, "device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
1082}
1083
1084static void check_sync(struct device *dev,
1085 struct dma_debug_entry *ref,
1086 bool to_cpu)
1087{
1088 struct dma_debug_entry *entry;
1089 struct hash_bucket *bucket;
1090 unsigned long flags;
1091
1092 bucket = get_hash_bucket(ref, &flags);
1093
1094 entry = bucket_find_contain(&bucket, ref, &flags);
1095
1096 if (!entry) {
1097 err_printk(dev, NULL, "device driver tries "
1098 "to sync DMA memory it has not allocated "
1099 "[device address=0x%016llx] [size=%llu bytes]\n",
1100 (unsigned long long)ref->dev_addr, ref->size);
1101 goto out;
1102 }
1103
1104 if (ref->size > entry->size) {
1105 err_printk(dev, entry, "device driver syncs"
1106 " DMA memory outside allocated range "
1107 "[device address=0x%016llx] "
1108 "[allocation size=%llu bytes] "
1109 "[sync offset+size=%llu]\n",
1110 entry->dev_addr, entry->size,
1111 ref->size);
1112 }
1113
1114 if (entry->direction == DMA_BIDIRECTIONAL)
1115 goto out;
1116
1117 if (ref->direction != entry->direction) {
1118 err_printk(dev, entry, "device driver syncs "
1119 "DMA memory with different direction "
1120 "[device address=0x%016llx] [size=%llu bytes] "
1121 "[mapped with %s] [synced with %s]\n",
1122 (unsigned long long)ref->dev_addr, entry->size,
1123 dir2name[entry->direction],
1124 dir2name[ref->direction]);
1125 }
1126
1127 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
1128 !(ref->direction == DMA_TO_DEVICE))
1129 err_printk(dev, entry, "device driver syncs "
1130 "device read-only DMA memory for cpu "
1131 "[device address=0x%016llx] [size=%llu bytes] "
1132 "[mapped with %s] [synced with %s]\n",
1133 (unsigned long long)ref->dev_addr, entry->size,
1134 dir2name[entry->direction],
1135 dir2name[ref->direction]);
1136
1137 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
1138 !(ref->direction == DMA_FROM_DEVICE))
1139 err_printk(dev, entry, "device driver syncs "
1140 "device write-only DMA memory to device "
1141 "[device address=0x%016llx] [size=%llu bytes] "
1142 "[mapped with %s] [synced with %s]\n",
1143 (unsigned long long)ref->dev_addr, entry->size,
1144 dir2name[entry->direction],
1145 dir2name[ref->direction]);
1146
1147 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1148 ref->sg_call_ents != entry->sg_call_ents) {
1149 err_printk(ref->dev, entry, "device driver syncs "
1150 "DMA sg list with different entry count "
1151 "[map count=%d] [sync count=%d]\n",
1152 entry->sg_call_ents, ref->sg_call_ents);
1153 }
1154
1155out:
1156 put_hash_bucket(bucket, flags);
1157}
1158
1159static void check_sg_segment(struct device *dev, struct scatterlist *sg)
1160{
1161#ifdef CONFIG_DMA_API_DEBUG_SG
1162 unsigned int max_seg = dma_get_max_seg_size(dev);
1163 u64 start, end, boundary = dma_get_seg_boundary(dev);
1164
1165
1166
1167
1168
1169 if (sg->length > max_seg)
1170 err_printk(dev, NULL, "mapping sg segment longer than device claims to support [len=%u] [max=%u]\n",
1171 sg->length, max_seg);
1172
1173
1174
1175
1176
1177 start = sg_dma_address(sg);
1178 end = start + sg_dma_len(sg) - 1;
1179 if ((start ^ end) & ~boundary)
1180 err_printk(dev, NULL, "mapping sg segment across boundary [start=0x%016llx] [end=0x%016llx] [boundary=0x%016llx]\n",
1181 start, end, boundary);
1182#endif
1183}
1184
1185void debug_dma_map_single(struct device *dev, const void *addr,
1186 unsigned long len)
1187{
1188 if (unlikely(dma_debug_disabled()))
1189 return;
1190
1191 if (!virt_addr_valid(addr))
1192 err_printk(dev, NULL, "device driver maps memory from invalid area [addr=%p] [len=%lu]\n",
1193 addr, len);
1194
1195 if (is_vmalloc_addr(addr))
1196 err_printk(dev, NULL, "device driver maps memory from vmalloc area [addr=%p] [len=%lu]\n",
1197 addr, len);
1198}
1199EXPORT_SYMBOL(debug_dma_map_single);
1200
1201void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
1202 size_t size, int direction, dma_addr_t dma_addr)
1203{
1204 struct dma_debug_entry *entry;
1205
1206 if (unlikely(dma_debug_disabled()))
1207 return;
1208
1209 if (dma_mapping_error(dev, dma_addr))
1210 return;
1211
1212 entry = dma_entry_alloc();
1213 if (!entry)
1214 return;
1215
1216 entry->dev = dev;
1217 entry->type = dma_debug_single;
1218 entry->pfn = page_to_pfn(page);
1219 entry->offset = offset;
1220 entry->dev_addr = dma_addr;
1221 entry->size = size;
1222 entry->direction = direction;
1223 entry->map_err_type = MAP_ERR_NOT_CHECKED;
1224
1225 check_for_stack(dev, page, offset);
1226
1227 if (!PageHighMem(page)) {
1228 void *addr = page_address(page) + offset;
1229
1230 check_for_illegal_area(dev, addr, size);
1231 }
1232
1233 add_dma_entry(entry);
1234}
1235
1236void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
1237{
1238 struct dma_debug_entry ref;
1239 struct dma_debug_entry *entry;
1240 struct hash_bucket *bucket;
1241 unsigned long flags;
1242
1243 if (unlikely(dma_debug_disabled()))
1244 return;
1245
1246 ref.dev = dev;
1247 ref.dev_addr = dma_addr;
1248 bucket = get_hash_bucket(&ref, &flags);
1249
1250 list_for_each_entry(entry, &bucket->list, list) {
1251 if (!exact_match(&ref, entry))
1252 continue;
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1265 entry->map_err_type = MAP_ERR_CHECKED;
1266 break;
1267 }
1268 }
1269
1270 put_hash_bucket(bucket, flags);
1271}
1272EXPORT_SYMBOL(debug_dma_mapping_error);
1273
1274void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
1275 size_t size, int direction)
1276{
1277 struct dma_debug_entry ref = {
1278 .type = dma_debug_single,
1279 .dev = dev,
1280 .dev_addr = addr,
1281 .size = size,
1282 .direction = direction,
1283 };
1284
1285 if (unlikely(dma_debug_disabled()))
1286 return;
1287 check_unmap(&ref);
1288}
1289
1290void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
1291 int nents, int mapped_ents, int direction)
1292{
1293 struct dma_debug_entry *entry;
1294 struct scatterlist *s;
1295 int i;
1296
1297 if (unlikely(dma_debug_disabled()))
1298 return;
1299
1300 for_each_sg(sg, s, mapped_ents, i) {
1301 entry = dma_entry_alloc();
1302 if (!entry)
1303 return;
1304
1305 entry->type = dma_debug_sg;
1306 entry->dev = dev;
1307 entry->pfn = page_to_pfn(sg_page(s));
1308 entry->offset = s->offset;
1309 entry->size = sg_dma_len(s);
1310 entry->dev_addr = sg_dma_address(s);
1311 entry->direction = direction;
1312 entry->sg_call_ents = nents;
1313 entry->sg_mapped_ents = mapped_ents;
1314
1315 check_for_stack(dev, sg_page(s), s->offset);
1316
1317 if (!PageHighMem(sg_page(s))) {
1318 check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s));
1319 }
1320
1321 check_sg_segment(dev, s);
1322
1323 add_dma_entry(entry);
1324 }
1325}
1326
1327static int get_nr_mapped_entries(struct device *dev,
1328 struct dma_debug_entry *ref)
1329{
1330 struct dma_debug_entry *entry;
1331 struct hash_bucket *bucket;
1332 unsigned long flags;
1333 int mapped_ents;
1334
1335 bucket = get_hash_bucket(ref, &flags);
1336 entry = bucket_find_exact(bucket, ref);
1337 mapped_ents = 0;
1338
1339 if (entry)
1340 mapped_ents = entry->sg_mapped_ents;
1341 put_hash_bucket(bucket, flags);
1342
1343 return mapped_ents;
1344}
1345
1346void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
1347 int nelems, int dir)
1348{
1349 struct scatterlist *s;
1350 int mapped_ents = 0, i;
1351
1352 if (unlikely(dma_debug_disabled()))
1353 return;
1354
1355 for_each_sg(sglist, s, nelems, i) {
1356
1357 struct dma_debug_entry ref = {
1358 .type = dma_debug_sg,
1359 .dev = dev,
1360 .pfn = page_to_pfn(sg_page(s)),
1361 .offset = s->offset,
1362 .dev_addr = sg_dma_address(s),
1363 .size = sg_dma_len(s),
1364 .direction = dir,
1365 .sg_call_ents = nelems,
1366 };
1367
1368 if (mapped_ents && i >= mapped_ents)
1369 break;
1370
1371 if (!i)
1372 mapped_ents = get_nr_mapped_entries(dev, &ref);
1373
1374 check_unmap(&ref);
1375 }
1376}
1377
1378void debug_dma_alloc_coherent(struct device *dev, size_t size,
1379 dma_addr_t dma_addr, void *virt)
1380{
1381 struct dma_debug_entry *entry;
1382
1383 if (unlikely(dma_debug_disabled()))
1384 return;
1385
1386 if (unlikely(virt == NULL))
1387 return;
1388
1389
1390 if (!is_vmalloc_addr(virt) && !virt_addr_valid(virt))
1391 return;
1392
1393 entry = dma_entry_alloc();
1394 if (!entry)
1395 return;
1396
1397 entry->type = dma_debug_coherent;
1398 entry->dev = dev;
1399 entry->offset = offset_in_page(virt);
1400 entry->size = size;
1401 entry->dev_addr = dma_addr;
1402 entry->direction = DMA_BIDIRECTIONAL;
1403
1404 if (is_vmalloc_addr(virt))
1405 entry->pfn = vmalloc_to_pfn(virt);
1406 else
1407 entry->pfn = page_to_pfn(virt_to_page(virt));
1408
1409 add_dma_entry(entry);
1410}
1411
1412void debug_dma_free_coherent(struct device *dev, size_t size,
1413 void *virt, dma_addr_t addr)
1414{
1415 struct dma_debug_entry ref = {
1416 .type = dma_debug_coherent,
1417 .dev = dev,
1418 .offset = offset_in_page(virt),
1419 .dev_addr = addr,
1420 .size = size,
1421 .direction = DMA_BIDIRECTIONAL,
1422 };
1423
1424
1425 if (!is_vmalloc_addr(virt) && !virt_addr_valid(virt))
1426 return;
1427
1428 if (is_vmalloc_addr(virt))
1429 ref.pfn = vmalloc_to_pfn(virt);
1430 else
1431 ref.pfn = page_to_pfn(virt_to_page(virt));
1432
1433 if (unlikely(dma_debug_disabled()))
1434 return;
1435
1436 check_unmap(&ref);
1437}
1438
1439void debug_dma_map_resource(struct device *dev, phys_addr_t addr, size_t size,
1440 int direction, dma_addr_t dma_addr)
1441{
1442 struct dma_debug_entry *entry;
1443
1444 if (unlikely(dma_debug_disabled()))
1445 return;
1446
1447 entry = dma_entry_alloc();
1448 if (!entry)
1449 return;
1450
1451 entry->type = dma_debug_resource;
1452 entry->dev = dev;
1453 entry->pfn = PHYS_PFN(addr);
1454 entry->offset = offset_in_page(addr);
1455 entry->size = size;
1456 entry->dev_addr = dma_addr;
1457 entry->direction = direction;
1458 entry->map_err_type = MAP_ERR_NOT_CHECKED;
1459
1460 add_dma_entry(entry);
1461}
1462
1463void debug_dma_unmap_resource(struct device *dev, dma_addr_t dma_addr,
1464 size_t size, int direction)
1465{
1466 struct dma_debug_entry ref = {
1467 .type = dma_debug_resource,
1468 .dev = dev,
1469 .dev_addr = dma_addr,
1470 .size = size,
1471 .direction = direction,
1472 };
1473
1474 if (unlikely(dma_debug_disabled()))
1475 return;
1476
1477 check_unmap(&ref);
1478}
1479
1480void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
1481 size_t size, int direction)
1482{
1483 struct dma_debug_entry ref;
1484
1485 if (unlikely(dma_debug_disabled()))
1486 return;
1487
1488 ref.type = dma_debug_single;
1489 ref.dev = dev;
1490 ref.dev_addr = dma_handle;
1491 ref.size = size;
1492 ref.direction = direction;
1493 ref.sg_call_ents = 0;
1494
1495 check_sync(dev, &ref, true);
1496}
1497
1498void debug_dma_sync_single_for_device(struct device *dev,
1499 dma_addr_t dma_handle, size_t size,
1500 int direction)
1501{
1502 struct dma_debug_entry ref;
1503
1504 if (unlikely(dma_debug_disabled()))
1505 return;
1506
1507 ref.type = dma_debug_single;
1508 ref.dev = dev;
1509 ref.dev_addr = dma_handle;
1510 ref.size = size;
1511 ref.direction = direction;
1512 ref.sg_call_ents = 0;
1513
1514 check_sync(dev, &ref, false);
1515}
1516
1517void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1518 int nelems, int direction)
1519{
1520 struct scatterlist *s;
1521 int mapped_ents = 0, i;
1522
1523 if (unlikely(dma_debug_disabled()))
1524 return;
1525
1526 for_each_sg(sg, s, nelems, i) {
1527
1528 struct dma_debug_entry ref = {
1529 .type = dma_debug_sg,
1530 .dev = dev,
1531 .pfn = page_to_pfn(sg_page(s)),
1532 .offset = s->offset,
1533 .dev_addr = sg_dma_address(s),
1534 .size = sg_dma_len(s),
1535 .direction = direction,
1536 .sg_call_ents = nelems,
1537 };
1538
1539 if (!i)
1540 mapped_ents = get_nr_mapped_entries(dev, &ref);
1541
1542 if (i >= mapped_ents)
1543 break;
1544
1545 check_sync(dev, &ref, true);
1546 }
1547}
1548
1549void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1550 int nelems, int direction)
1551{
1552 struct scatterlist *s;
1553 int mapped_ents = 0, i;
1554
1555 if (unlikely(dma_debug_disabled()))
1556 return;
1557
1558 for_each_sg(sg, s, nelems, i) {
1559
1560 struct dma_debug_entry ref = {
1561 .type = dma_debug_sg,
1562 .dev = dev,
1563 .pfn = page_to_pfn(sg_page(s)),
1564 .offset = s->offset,
1565 .dev_addr = sg_dma_address(s),
1566 .size = sg_dma_len(s),
1567 .direction = direction,
1568 .sg_call_ents = nelems,
1569 };
1570 if (!i)
1571 mapped_ents = get_nr_mapped_entries(dev, &ref);
1572
1573 if (i >= mapped_ents)
1574 break;
1575
1576 check_sync(dev, &ref, false);
1577 }
1578}
1579
1580static int __init dma_debug_driver_setup(char *str)
1581{
1582 int i;
1583
1584 for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) {
1585 current_driver_name[i] = *str;
1586 if (*str == 0)
1587 break;
1588 }
1589
1590 if (current_driver_name[0])
1591 pr_info("enable driver filter for driver [%s]\n",
1592 current_driver_name);
1593
1594
1595 return 1;
1596}
1597__setup("dma_debug_driver=", dma_debug_driver_setup);
1598