1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/sched/task_stack.h>
21#include <linux/scatterlist.h>
22#include <linux/dma-mapping.h>
23#include <linux/sched/task.h>
24#include <linux/stacktrace.h>
25#include <linux/dma-debug.h>
26#include <linux/spinlock.h>
27#include <linux/vmalloc.h>
28#include <linux/debugfs.h>
29#include <linux/uaccess.h>
30#include <linux/export.h>
31#include <linux/device.h>
32#include <linux/types.h>
33#include <linux/sched.h>
34#include <linux/ctype.h>
35#include <linux/list.h>
36#include <linux/slab.h>
37
38#include <asm/sections.h>
39
40#define HASH_SIZE 1024ULL
41#define HASH_FN_SHIFT 13
42#define HASH_FN_MASK (HASH_SIZE - 1)
43
44enum {
45 dma_debug_single,
46 dma_debug_page,
47 dma_debug_sg,
48 dma_debug_coherent,
49 dma_debug_resource,
50};
51
52enum map_err_types {
53 MAP_ERR_CHECK_NOT_APPLICABLE,
54 MAP_ERR_NOT_CHECKED,
55 MAP_ERR_CHECKED,
56};
57
58#define DMA_DEBUG_STACKTRACE_ENTRIES 5
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74struct dma_debug_entry {
75 struct list_head list;
76 struct device *dev;
77 int type;
78 unsigned long pfn;
79 size_t offset;
80 u64 dev_addr;
81 u64 size;
82 int direction;
83 int sg_call_ents;
84 int sg_mapped_ents;
85 enum map_err_types map_err_type;
86#ifdef CONFIG_STACKTRACE
87 struct stack_trace stacktrace;
88 unsigned long st_entries[DMA_DEBUG_STACKTRACE_ENTRIES];
89#endif
90};
91
92typedef bool (*match_fn)(struct dma_debug_entry *, struct dma_debug_entry *);
93
94struct hash_bucket {
95 struct list_head list;
96 spinlock_t lock;
97} ____cacheline_aligned_in_smp;
98
99
100static struct hash_bucket dma_entry_hash[HASH_SIZE];
101
102static LIST_HEAD(free_entries);
103
104static DEFINE_SPINLOCK(free_entries_lock);
105
106
107static bool global_disable __read_mostly;
108
109
110static bool dma_debug_initialized __read_mostly;
111
112static inline bool dma_debug_disabled(void)
113{
114 return global_disable || !dma_debug_initialized;
115}
116
117
118static u32 error_count;
119
120
121static u32 show_all_errors __read_mostly;
122
123static u32 show_num_errors = 1;
124
125static u32 num_free_entries;
126static u32 min_free_entries;
127static u32 nr_total_entries;
128
129
130static u32 req_entries;
131
132
133static struct dentry *dma_debug_dent __read_mostly;
134static struct dentry *global_disable_dent __read_mostly;
135static struct dentry *error_count_dent __read_mostly;
136static struct dentry *show_all_errors_dent __read_mostly;
137static struct dentry *show_num_errors_dent __read_mostly;
138static struct dentry *num_free_entries_dent __read_mostly;
139static struct dentry *min_free_entries_dent __read_mostly;
140static struct dentry *filter_dent __read_mostly;
141
142
143
144#define NAME_MAX_LEN 64
145
146static char current_driver_name[NAME_MAX_LEN] __read_mostly;
147static struct device_driver *current_driver __read_mostly;
148
149static DEFINE_RWLOCK(driver_name_lock);
150
151static const char *const maperr2str[] = {
152 [MAP_ERR_CHECK_NOT_APPLICABLE] = "dma map error check not applicable",
153 [MAP_ERR_NOT_CHECKED] = "dma map error not checked",
154 [MAP_ERR_CHECKED] = "dma map error checked",
155};
156
157static const char *type2name[5] = { "single", "page",
158 "scather-gather", "coherent",
159 "resource" };
160
161static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
162 "DMA_FROM_DEVICE", "DMA_NONE" };
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177static inline void dump_entry_trace(struct dma_debug_entry *entry)
178{
179#ifdef CONFIG_STACKTRACE
180 if (entry) {
181 pr_warning("Mapped at:\n");
182 print_stack_trace(&entry->stacktrace, 0);
183 }
184#endif
185}
186
187static bool driver_filter(struct device *dev)
188{
189 struct device_driver *drv;
190 unsigned long flags;
191 bool ret;
192
193
194 if (likely(!current_driver_name[0]))
195 return true;
196
197
198 if (current_driver && dev && dev->driver == current_driver)
199 return true;
200
201
202 if (!dev)
203 return false;
204
205 if (current_driver || !current_driver_name[0])
206 return false;
207
208
209 drv = dev->driver;
210 if (!drv)
211 return false;
212
213
214 read_lock_irqsave(&driver_name_lock, flags);
215
216 ret = false;
217 if (drv->name &&
218 strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) {
219 current_driver = drv;
220 ret = true;
221 }
222
223 read_unlock_irqrestore(&driver_name_lock, flags);
224
225 return ret;
226}
227
228#define err_printk(dev, entry, format, arg...) do { \
229 error_count += 1; \
230 if (driver_filter(dev) && \
231 (show_all_errors || show_num_errors > 0)) { \
232 WARN(1, "%s %s: " format, \
233 dev ? dev_driver_string(dev) : "NULL", \
234 dev ? dev_name(dev) : "NULL", ## arg); \
235 dump_entry_trace(entry); \
236 } \
237 if (!show_all_errors && show_num_errors > 0) \
238 show_num_errors -= 1; \
239 } while (0);
240
241
242
243
244
245
246
247static int hash_fn(struct dma_debug_entry *entry)
248{
249
250
251
252
253 return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK;
254}
255
256
257
258
259static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry,
260 unsigned long *flags)
261 __acquires(&dma_entry_hash[idx].lock)
262{
263 int idx = hash_fn(entry);
264 unsigned long __flags;
265
266 spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags);
267 *flags = __flags;
268 return &dma_entry_hash[idx];
269}
270
271
272
273
274static void put_hash_bucket(struct hash_bucket *bucket,
275 unsigned long *flags)
276 __releases(&bucket->lock)
277{
278 unsigned long __flags = *flags;
279
280 spin_unlock_irqrestore(&bucket->lock, __flags);
281}
282
283static bool exact_match(struct dma_debug_entry *a, struct dma_debug_entry *b)
284{
285 return ((a->dev_addr == b->dev_addr) &&
286 (a->dev == b->dev)) ? true : false;
287}
288
289static bool containing_match(struct dma_debug_entry *a,
290 struct dma_debug_entry *b)
291{
292 if (a->dev != b->dev)
293 return false;
294
295 if ((b->dev_addr <= a->dev_addr) &&
296 ((b->dev_addr + b->size) >= (a->dev_addr + a->size)))
297 return true;
298
299 return false;
300}
301
302
303
304
305static struct dma_debug_entry *__hash_bucket_find(struct hash_bucket *bucket,
306 struct dma_debug_entry *ref,
307 match_fn match)
308{
309 struct dma_debug_entry *entry, *ret = NULL;
310 int matches = 0, match_lvl, last_lvl = -1;
311
312 list_for_each_entry(entry, &bucket->list, list) {
313 if (!match(ref, entry))
314 continue;
315
316
317
318
319
320
321
322
323
324
325
326 matches += 1;
327 match_lvl = 0;
328 entry->size == ref->size ? ++match_lvl : 0;
329 entry->type == ref->type ? ++match_lvl : 0;
330 entry->direction == ref->direction ? ++match_lvl : 0;
331 entry->sg_call_ents == ref->sg_call_ents ? ++match_lvl : 0;
332
333 if (match_lvl == 4) {
334
335 return entry;
336 } else if (match_lvl > last_lvl) {
337
338
339
340
341 last_lvl = match_lvl;
342 ret = entry;
343 }
344 }
345
346
347
348
349
350 ret = (matches == 1) ? ret : NULL;
351
352 return ret;
353}
354
355static struct dma_debug_entry *bucket_find_exact(struct hash_bucket *bucket,
356 struct dma_debug_entry *ref)
357{
358 return __hash_bucket_find(bucket, ref, exact_match);
359}
360
361static struct dma_debug_entry *bucket_find_contain(struct hash_bucket **bucket,
362 struct dma_debug_entry *ref,
363 unsigned long *flags)
364{
365
366 unsigned int max_range = dma_get_max_seg_size(ref->dev);
367 struct dma_debug_entry *entry, index = *ref;
368 unsigned int range = 0;
369
370 while (range <= max_range) {
371 entry = __hash_bucket_find(*bucket, ref, containing_match);
372
373 if (entry)
374 return entry;
375
376
377
378
379 put_hash_bucket(*bucket, flags);
380 range += (1 << HASH_FN_SHIFT);
381 index.dev_addr -= (1 << HASH_FN_SHIFT);
382 *bucket = get_hash_bucket(&index, flags);
383 }
384
385 return NULL;
386}
387
388
389
390
391static void hash_bucket_add(struct hash_bucket *bucket,
392 struct dma_debug_entry *entry)
393{
394 list_add_tail(&entry->list, &bucket->list);
395}
396
397
398
399
400static void hash_bucket_del(struct dma_debug_entry *entry)
401{
402 list_del(&entry->list);
403}
404
405static unsigned long long phys_addr(struct dma_debug_entry *entry)
406{
407 if (entry->type == dma_debug_resource)
408 return __pfn_to_phys(entry->pfn) + entry->offset;
409
410 return page_to_phys(pfn_to_page(entry->pfn)) + entry->offset;
411}
412
413
414
415
416void debug_dma_dump_mappings(struct device *dev)
417{
418 int idx;
419
420 for (idx = 0; idx < HASH_SIZE; idx++) {
421 struct hash_bucket *bucket = &dma_entry_hash[idx];
422 struct dma_debug_entry *entry;
423 unsigned long flags;
424
425 spin_lock_irqsave(&bucket->lock, flags);
426
427 list_for_each_entry(entry, &bucket->list, list) {
428 if (!dev || dev == entry->dev) {
429 dev_info(entry->dev,
430 "%s idx %d P=%Lx N=%lx D=%Lx L=%Lx %s %s\n",
431 type2name[entry->type], idx,
432 phys_addr(entry), entry->pfn,
433 entry->dev_addr, entry->size,
434 dir2name[entry->direction],
435 maperr2str[entry->map_err_type]);
436 }
437 }
438
439 spin_unlock_irqrestore(&bucket->lock, flags);
440 }
441}
442EXPORT_SYMBOL(debug_dma_dump_mappings);
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467static RADIX_TREE(dma_active_cacheline, GFP_NOWAIT);
468static DEFINE_SPINLOCK(radix_lock);
469#define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1)
470#define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT)
471#define CACHELINES_PER_PAGE (1 << CACHELINE_PER_PAGE_SHIFT)
472
473static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)
474{
475 return (entry->pfn << CACHELINE_PER_PAGE_SHIFT) +
476 (entry->offset >> L1_CACHE_SHIFT);
477}
478
479static int active_cacheline_read_overlap(phys_addr_t cln)
480{
481 int overlap = 0, i;
482
483 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
484 if (radix_tree_tag_get(&dma_active_cacheline, cln, i))
485 overlap |= 1 << i;
486 return overlap;
487}
488
489static int active_cacheline_set_overlap(phys_addr_t cln, int overlap)
490{
491 int i;
492
493 if (overlap > ACTIVE_CACHELINE_MAX_OVERLAP || overlap < 0)
494 return overlap;
495
496 for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
497 if (overlap & 1 << i)
498 radix_tree_tag_set(&dma_active_cacheline, cln, i);
499 else
500 radix_tree_tag_clear(&dma_active_cacheline, cln, i);
501
502 return overlap;
503}
504
505static void active_cacheline_inc_overlap(phys_addr_t cln)
506{
507 int overlap = active_cacheline_read_overlap(cln);
508
509 overlap = active_cacheline_set_overlap(cln, ++overlap);
510
511
512
513
514
515
516
517 WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
518 "DMA-API: exceeded %d overlapping mappings of cacheline %pa\n",
519 ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
520}
521
522static int active_cacheline_dec_overlap(phys_addr_t cln)
523{
524 int overlap = active_cacheline_read_overlap(cln);
525
526 return active_cacheline_set_overlap(cln, --overlap);
527}
528
529static int active_cacheline_insert(struct dma_debug_entry *entry)
530{
531 phys_addr_t cln = to_cacheline_number(entry);
532 unsigned long flags;
533 int rc;
534
535
536
537
538
539 if (entry->direction == DMA_TO_DEVICE)
540 return 0;
541
542 spin_lock_irqsave(&radix_lock, flags);
543 rc = radix_tree_insert(&dma_active_cacheline, cln, entry);
544 if (rc == -EEXIST)
545 active_cacheline_inc_overlap(cln);
546 spin_unlock_irqrestore(&radix_lock, flags);
547
548 return rc;
549}
550
551static void active_cacheline_remove(struct dma_debug_entry *entry)
552{
553 phys_addr_t cln = to_cacheline_number(entry);
554 unsigned long flags;
555
556
557 if (entry->direction == DMA_TO_DEVICE)
558 return;
559
560 spin_lock_irqsave(&radix_lock, flags);
561
562
563
564
565 if (active_cacheline_dec_overlap(cln) < 0)
566 radix_tree_delete(&dma_active_cacheline, cln);
567 spin_unlock_irqrestore(&radix_lock, flags);
568}
569
570
571
572
573
574
575
576
577
578void debug_dma_assert_idle(struct page *page)
579{
580 static struct dma_debug_entry *ents[CACHELINES_PER_PAGE];
581 struct dma_debug_entry *entry = NULL;
582 void **results = (void **) &ents;
583 unsigned int nents, i;
584 unsigned long flags;
585 phys_addr_t cln;
586
587 if (dma_debug_disabled())
588 return;
589
590 if (!page)
591 return;
592
593 cln = (phys_addr_t) page_to_pfn(page) << CACHELINE_PER_PAGE_SHIFT;
594 spin_lock_irqsave(&radix_lock, flags);
595 nents = radix_tree_gang_lookup(&dma_active_cacheline, results, cln,
596 CACHELINES_PER_PAGE);
597 for (i = 0; i < nents; i++) {
598 phys_addr_t ent_cln = to_cacheline_number(ents[i]);
599
600 if (ent_cln == cln) {
601 entry = ents[i];
602 break;
603 } else if (ent_cln >= cln + CACHELINES_PER_PAGE)
604 break;
605 }
606 spin_unlock_irqrestore(&radix_lock, flags);
607
608 if (!entry)
609 return;
610
611 cln = to_cacheline_number(entry);
612 err_printk(entry->dev, entry,
613 "DMA-API: cpu touching an active dma mapped cacheline [cln=%pa]\n",
614 &cln);
615}
616
617
618
619
620
621static void add_dma_entry(struct dma_debug_entry *entry)
622{
623 struct hash_bucket *bucket;
624 unsigned long flags;
625 int rc;
626
627 bucket = get_hash_bucket(entry, &flags);
628 hash_bucket_add(bucket, entry);
629 put_hash_bucket(bucket, &flags);
630
631 rc = active_cacheline_insert(entry);
632 if (rc == -ENOMEM) {
633 pr_err("DMA-API: cacheline tracking ENOMEM, dma-debug disabled\n");
634 global_disable = true;
635 }
636
637
638
639
640}
641
642static struct dma_debug_entry *__dma_entry_alloc(void)
643{
644 struct dma_debug_entry *entry;
645
646 entry = list_entry(free_entries.next, struct dma_debug_entry, list);
647 list_del(&entry->list);
648 memset(entry, 0, sizeof(*entry));
649
650 num_free_entries -= 1;
651 if (num_free_entries < min_free_entries)
652 min_free_entries = num_free_entries;
653
654 return entry;
655}
656
657
658
659
660
661
662static struct dma_debug_entry *dma_entry_alloc(void)
663{
664 struct dma_debug_entry *entry;
665 unsigned long flags;
666
667 spin_lock_irqsave(&free_entries_lock, flags);
668
669 if (list_empty(&free_entries)) {
670 global_disable = true;
671 spin_unlock_irqrestore(&free_entries_lock, flags);
672 pr_err("DMA-API: debugging out of memory - disabling\n");
673 return NULL;
674 }
675
676 entry = __dma_entry_alloc();
677
678 spin_unlock_irqrestore(&free_entries_lock, flags);
679
680#ifdef CONFIG_STACKTRACE
681 entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES;
682 entry->stacktrace.entries = entry->st_entries;
683 entry->stacktrace.skip = 2;
684 save_stack_trace(&entry->stacktrace);
685#endif
686
687 return entry;
688}
689
690static void dma_entry_free(struct dma_debug_entry *entry)
691{
692 unsigned long flags;
693
694 active_cacheline_remove(entry);
695
696
697
698
699
700 spin_lock_irqsave(&free_entries_lock, flags);
701 list_add(&entry->list, &free_entries);
702 num_free_entries += 1;
703 spin_unlock_irqrestore(&free_entries_lock, flags);
704}
705
706int dma_debug_resize_entries(u32 num_entries)
707{
708 int i, delta, ret = 0;
709 unsigned long flags;
710 struct dma_debug_entry *entry;
711 LIST_HEAD(tmp);
712
713 spin_lock_irqsave(&free_entries_lock, flags);
714
715 if (nr_total_entries < num_entries) {
716 delta = num_entries - nr_total_entries;
717
718 spin_unlock_irqrestore(&free_entries_lock, flags);
719
720 for (i = 0; i < delta; i++) {
721 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
722 if (!entry)
723 break;
724
725 list_add_tail(&entry->list, &tmp);
726 }
727
728 spin_lock_irqsave(&free_entries_lock, flags);
729
730 list_splice(&tmp, &free_entries);
731 nr_total_entries += i;
732 num_free_entries += i;
733 } else {
734 delta = nr_total_entries - num_entries;
735
736 for (i = 0; i < delta && !list_empty(&free_entries); i++) {
737 entry = __dma_entry_alloc();
738 kfree(entry);
739 }
740
741 nr_total_entries -= i;
742 }
743
744 if (nr_total_entries != num_entries)
745 ret = 1;
746
747 spin_unlock_irqrestore(&free_entries_lock, flags);
748
749 return ret;
750}
751EXPORT_SYMBOL(dma_debug_resize_entries);
752
753
754
755
756
757
758
759
760
761static int prealloc_memory(u32 num_entries)
762{
763 struct dma_debug_entry *entry, *next_entry;
764 int i;
765
766 for (i = 0; i < num_entries; ++i) {
767 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
768 if (!entry)
769 goto out_err;
770
771 list_add_tail(&entry->list, &free_entries);
772 }
773
774 num_free_entries = num_entries;
775 min_free_entries = num_entries;
776
777 pr_info("DMA-API: preallocated %d debug entries\n", num_entries);
778
779 return 0;
780
781out_err:
782
783 list_for_each_entry_safe(entry, next_entry, &free_entries, list) {
784 list_del(&entry->list);
785 kfree(entry);
786 }
787
788 return -ENOMEM;
789}
790
791static ssize_t filter_read(struct file *file, char __user *user_buf,
792 size_t count, loff_t *ppos)
793{
794 char buf[NAME_MAX_LEN + 1];
795 unsigned long flags;
796 int len;
797
798 if (!current_driver_name[0])
799 return 0;
800
801
802
803
804
805
806 read_lock_irqsave(&driver_name_lock, flags);
807 len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name);
808 read_unlock_irqrestore(&driver_name_lock, flags);
809
810 return simple_read_from_buffer(user_buf, count, ppos, buf, len);
811}
812
813static ssize_t filter_write(struct file *file, const char __user *userbuf,
814 size_t count, loff_t *ppos)
815{
816 char buf[NAME_MAX_LEN];
817 unsigned long flags;
818 size_t len;
819 int i;
820
821
822
823
824
825
826
827 len = min(count, (size_t)(NAME_MAX_LEN - 1));
828 if (copy_from_user(buf, userbuf, len))
829 return -EFAULT;
830
831 buf[len] = 0;
832
833 write_lock_irqsave(&driver_name_lock, flags);
834
835
836
837
838
839
840
841
842
843 if (!isalnum(buf[0])) {
844
845
846
847
848
849 if (current_driver_name[0])
850 pr_info("DMA-API: switching off dma-debug driver filter\n");
851 current_driver_name[0] = 0;
852 current_driver = NULL;
853 goto out_unlock;
854 }
855
856
857
858
859
860 for (i = 0; i < NAME_MAX_LEN - 1; ++i) {
861 current_driver_name[i] = buf[i];
862 if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0)
863 break;
864 }
865 current_driver_name[i] = 0;
866 current_driver = NULL;
867
868 pr_info("DMA-API: enable driver filter for driver [%s]\n",
869 current_driver_name);
870
871out_unlock:
872 write_unlock_irqrestore(&driver_name_lock, flags);
873
874 return count;
875}
876
877static const struct file_operations filter_fops = {
878 .read = filter_read,
879 .write = filter_write,
880 .llseek = default_llseek,
881};
882
883static int dma_debug_fs_init(void)
884{
885 dma_debug_dent = debugfs_create_dir("dma-api", NULL);
886 if (!dma_debug_dent) {
887 pr_err("DMA-API: can not create debugfs directory\n");
888 return -ENOMEM;
889 }
890
891 global_disable_dent = debugfs_create_bool("disabled", 0444,
892 dma_debug_dent,
893 &global_disable);
894 if (!global_disable_dent)
895 goto out_err;
896
897 error_count_dent = debugfs_create_u32("error_count", 0444,
898 dma_debug_dent, &error_count);
899 if (!error_count_dent)
900 goto out_err;
901
902 show_all_errors_dent = debugfs_create_u32("all_errors", 0644,
903 dma_debug_dent,
904 &show_all_errors);
905 if (!show_all_errors_dent)
906 goto out_err;
907
908 show_num_errors_dent = debugfs_create_u32("num_errors", 0644,
909 dma_debug_dent,
910 &show_num_errors);
911 if (!show_num_errors_dent)
912 goto out_err;
913
914 num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444,
915 dma_debug_dent,
916 &num_free_entries);
917 if (!num_free_entries_dent)
918 goto out_err;
919
920 min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444,
921 dma_debug_dent,
922 &min_free_entries);
923 if (!min_free_entries_dent)
924 goto out_err;
925
926 filter_dent = debugfs_create_file("driver_filter", 0644,
927 dma_debug_dent, NULL, &filter_fops);
928 if (!filter_dent)
929 goto out_err;
930
931 return 0;
932
933out_err:
934 debugfs_remove_recursive(dma_debug_dent);
935
936 return -ENOMEM;
937}
938
939static int device_dma_allocations(struct device *dev, struct dma_debug_entry **out_entry)
940{
941 struct dma_debug_entry *entry;
942 unsigned long flags;
943 int count = 0, i;
944
945 for (i = 0; i < HASH_SIZE; ++i) {
946 spin_lock_irqsave(&dma_entry_hash[i].lock, flags);
947 list_for_each_entry(entry, &dma_entry_hash[i].list, list) {
948 if (entry->dev == dev) {
949 count += 1;
950 *out_entry = entry;
951 }
952 }
953 spin_unlock_irqrestore(&dma_entry_hash[i].lock, flags);
954 }
955
956 return count;
957}
958
959static int dma_debug_device_change(struct notifier_block *nb, unsigned long action, void *data)
960{
961 struct device *dev = data;
962 struct dma_debug_entry *uninitialized_var(entry);
963 int count;
964
965 if (dma_debug_disabled())
966 return 0;
967
968 switch (action) {
969 case BUS_NOTIFY_UNBOUND_DRIVER:
970 count = device_dma_allocations(dev, &entry);
971 if (count == 0)
972 break;
973 err_printk(dev, entry, "DMA-API: device driver has pending "
974 "DMA allocations while released from device "
975 "[count=%d]\n"
976 "One of leaked entries details: "
977 "[device address=0x%016llx] [size=%llu bytes] "
978 "[mapped with %s] [mapped as %s]\n",
979 count, entry->dev_addr, entry->size,
980 dir2name[entry->direction], type2name[entry->type]);
981 break;
982 default:
983 break;
984 }
985
986 return 0;
987}
988
989void dma_debug_add_bus(struct bus_type *bus)
990{
991 struct notifier_block *nb;
992
993 if (dma_debug_disabled())
994 return;
995
996 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL);
997 if (nb == NULL) {
998 pr_err("dma_debug_add_bus: out of memory\n");
999 return;
1000 }
1001
1002 nb->notifier_call = dma_debug_device_change;
1003
1004 bus_register_notifier(bus, nb);
1005}
1006
1007
1008
1009
1010void dma_debug_init(u32 num_entries)
1011{
1012 int i;
1013
1014
1015
1016
1017 if (global_disable)
1018 return;
1019
1020 for (i = 0; i < HASH_SIZE; ++i) {
1021 INIT_LIST_HEAD(&dma_entry_hash[i].list);
1022 spin_lock_init(&dma_entry_hash[i].lock);
1023 }
1024
1025 if (dma_debug_fs_init() != 0) {
1026 pr_err("DMA-API: error creating debugfs entries - disabling\n");
1027 global_disable = true;
1028
1029 return;
1030 }
1031
1032 if (req_entries)
1033 num_entries = req_entries;
1034
1035 if (prealloc_memory(num_entries) != 0) {
1036 pr_err("DMA-API: debugging out of memory error - disabled\n");
1037 global_disable = true;
1038
1039 return;
1040 }
1041
1042 nr_total_entries = num_free_entries;
1043
1044 dma_debug_initialized = true;
1045
1046 pr_info("DMA-API: debugging enabled by kernel config\n");
1047}
1048
1049static __init int dma_debug_cmdline(char *str)
1050{
1051 if (!str)
1052 return -EINVAL;
1053
1054 if (strncmp(str, "off", 3) == 0) {
1055 pr_info("DMA-API: debugging disabled on kernel command line\n");
1056 global_disable = true;
1057 }
1058
1059 return 0;
1060}
1061
1062static __init int dma_debug_entries_cmdline(char *str)
1063{
1064 int res;
1065
1066 if (!str)
1067 return -EINVAL;
1068
1069 res = get_option(&str, &req_entries);
1070
1071 if (!res)
1072 req_entries = 0;
1073
1074 return 0;
1075}
1076
1077__setup("dma_debug=", dma_debug_cmdline);
1078__setup("dma_debug_entries=", dma_debug_entries_cmdline);
1079
1080static void check_unmap(struct dma_debug_entry *ref)
1081{
1082 struct dma_debug_entry *entry;
1083 struct hash_bucket *bucket;
1084 unsigned long flags;
1085
1086 bucket = get_hash_bucket(ref, &flags);
1087 entry = bucket_find_exact(bucket, ref);
1088
1089 if (!entry) {
1090
1091 put_hash_bucket(bucket, &flags);
1092
1093 if (dma_mapping_error(ref->dev, ref->dev_addr)) {
1094 err_printk(ref->dev, NULL,
1095 "DMA-API: device driver tries to free an "
1096 "invalid DMA memory address\n");
1097 } else {
1098 err_printk(ref->dev, NULL,
1099 "DMA-API: device driver tries to free DMA "
1100 "memory it has not allocated [device "
1101 "address=0x%016llx] [size=%llu bytes]\n",
1102 ref->dev_addr, ref->size);
1103 }
1104 return;
1105 }
1106
1107 if (ref->size != entry->size) {
1108 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1109 "DMA memory with different size "
1110 "[device address=0x%016llx] [map size=%llu bytes] "
1111 "[unmap size=%llu bytes]\n",
1112 ref->dev_addr, entry->size, ref->size);
1113 }
1114
1115 if (ref->type != entry->type) {
1116 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1117 "DMA memory with wrong function "
1118 "[device address=0x%016llx] [size=%llu bytes] "
1119 "[mapped as %s] [unmapped as %s]\n",
1120 ref->dev_addr, ref->size,
1121 type2name[entry->type], type2name[ref->type]);
1122 } else if ((entry->type == dma_debug_coherent) &&
1123 (phys_addr(ref) != phys_addr(entry))) {
1124 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1125 "DMA memory with different CPU address "
1126 "[device address=0x%016llx] [size=%llu bytes] "
1127 "[cpu alloc address=0x%016llx] "
1128 "[cpu free address=0x%016llx]",
1129 ref->dev_addr, ref->size,
1130 phys_addr(entry),
1131 phys_addr(ref));
1132 }
1133
1134 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1135 ref->sg_call_ents != entry->sg_call_ents) {
1136 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1137 "DMA sg list with different entry count "
1138 "[map count=%d] [unmap count=%d]\n",
1139 entry->sg_call_ents, ref->sg_call_ents);
1140 }
1141
1142
1143
1144
1145
1146 if (ref->direction != entry->direction) {
1147 err_printk(ref->dev, entry, "DMA-API: device driver frees "
1148 "DMA memory with different direction "
1149 "[device address=0x%016llx] [size=%llu bytes] "
1150 "[mapped with %s] [unmapped with %s]\n",
1151 ref->dev_addr, ref->size,
1152 dir2name[entry->direction],
1153 dir2name[ref->direction]);
1154 }
1155
1156
1157
1158
1159
1160
1161 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1162 err_printk(ref->dev, entry,
1163 "DMA-API: device driver failed to check map error"
1164 "[device address=0x%016llx] [size=%llu bytes] "
1165 "[mapped as %s]",
1166 ref->dev_addr, ref->size,
1167 type2name[entry->type]);
1168 }
1169
1170 hash_bucket_del(entry);
1171 dma_entry_free(entry);
1172
1173 put_hash_bucket(bucket, &flags);
1174}
1175
1176static void check_for_stack(struct device *dev,
1177 struct page *page, size_t offset)
1178{
1179 void *addr;
1180 struct vm_struct *stack_vm_area = task_stack_vm_area(current);
1181
1182 if (!stack_vm_area) {
1183
1184 if (PageHighMem(page))
1185 return;
1186 addr = page_address(page) + offset;
1187 if (object_is_on_stack(addr))
1188 err_printk(dev, NULL, "DMA-API: device driver maps memory from stack [addr=%p]\n", addr);
1189 } else {
1190
1191 int i;
1192
1193 for (i = 0; i < stack_vm_area->nr_pages; i++) {
1194 if (page != stack_vm_area->pages[i])
1195 continue;
1196
1197 addr = (u8 *)current->stack + i * PAGE_SIZE + offset;
1198 err_printk(dev, NULL, "DMA-API: device driver maps memory from stack [probable addr=%p]\n", addr);
1199 break;
1200 }
1201 }
1202}
1203
1204static inline bool overlap(void *addr, unsigned long len, void *start, void *end)
1205{
1206 unsigned long a1 = (unsigned long)addr;
1207 unsigned long b1 = a1 + len;
1208 unsigned long a2 = (unsigned long)start;
1209 unsigned long b2 = (unsigned long)end;
1210
1211 return !(b1 <= a2 || a1 >= b2);
1212}
1213
1214static void check_for_illegal_area(struct device *dev, void *addr, unsigned long len)
1215{
1216 if (overlap(addr, len, _stext, _etext) ||
1217 overlap(addr, len, __start_rodata, __end_rodata))
1218 err_printk(dev, NULL, "DMA-API: device driver maps memory from kernel text or rodata [addr=%p] [len=%lu]\n", addr, len);
1219}
1220
1221static void check_sync(struct device *dev,
1222 struct dma_debug_entry *ref,
1223 bool to_cpu)
1224{
1225 struct dma_debug_entry *entry;
1226 struct hash_bucket *bucket;
1227 unsigned long flags;
1228
1229 bucket = get_hash_bucket(ref, &flags);
1230
1231 entry = bucket_find_contain(&bucket, ref, &flags);
1232
1233 if (!entry) {
1234 err_printk(dev, NULL, "DMA-API: device driver tries "
1235 "to sync DMA memory it has not allocated "
1236 "[device address=0x%016llx] [size=%llu bytes]\n",
1237 (unsigned long long)ref->dev_addr, ref->size);
1238 goto out;
1239 }
1240
1241 if (ref->size > entry->size) {
1242 err_printk(dev, entry, "DMA-API: device driver syncs"
1243 " DMA memory outside allocated range "
1244 "[device address=0x%016llx] "
1245 "[allocation size=%llu bytes] "
1246 "[sync offset+size=%llu]\n",
1247 entry->dev_addr, entry->size,
1248 ref->size);
1249 }
1250
1251 if (entry->direction == DMA_BIDIRECTIONAL)
1252 goto out;
1253
1254 if (ref->direction != entry->direction) {
1255 err_printk(dev, entry, "DMA-API: device driver syncs "
1256 "DMA memory with different direction "
1257 "[device address=0x%016llx] [size=%llu bytes] "
1258 "[mapped with %s] [synced with %s]\n",
1259 (unsigned long long)ref->dev_addr, entry->size,
1260 dir2name[entry->direction],
1261 dir2name[ref->direction]);
1262 }
1263
1264 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) &&
1265 !(ref->direction == DMA_TO_DEVICE))
1266 err_printk(dev, entry, "DMA-API: device driver syncs "
1267 "device read-only DMA memory for cpu "
1268 "[device address=0x%016llx] [size=%llu bytes] "
1269 "[mapped with %s] [synced with %s]\n",
1270 (unsigned long long)ref->dev_addr, entry->size,
1271 dir2name[entry->direction],
1272 dir2name[ref->direction]);
1273
1274 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) &&
1275 !(ref->direction == DMA_FROM_DEVICE))
1276 err_printk(dev, entry, "DMA-API: device driver syncs "
1277 "device write-only DMA memory to device "
1278 "[device address=0x%016llx] [size=%llu bytes] "
1279 "[mapped with %s] [synced with %s]\n",
1280 (unsigned long long)ref->dev_addr, entry->size,
1281 dir2name[entry->direction],
1282 dir2name[ref->direction]);
1283
1284 if (ref->sg_call_ents && ref->type == dma_debug_sg &&
1285 ref->sg_call_ents != entry->sg_call_ents) {
1286 err_printk(ref->dev, entry, "DMA-API: device driver syncs "
1287 "DMA sg list with different entry count "
1288 "[map count=%d] [sync count=%d]\n",
1289 entry->sg_call_ents, ref->sg_call_ents);
1290 }
1291
1292out:
1293 put_hash_bucket(bucket, &flags);
1294}
1295
1296void debug_dma_map_page(struct device *dev, struct page *page, size_t offset,
1297 size_t size, int direction, dma_addr_t dma_addr,
1298 bool map_single)
1299{
1300 struct dma_debug_entry *entry;
1301
1302 if (unlikely(dma_debug_disabled()))
1303 return;
1304
1305 if (dma_mapping_error(dev, dma_addr))
1306 return;
1307
1308 entry = dma_entry_alloc();
1309 if (!entry)
1310 return;
1311
1312 entry->dev = dev;
1313 entry->type = dma_debug_page;
1314 entry->pfn = page_to_pfn(page);
1315 entry->offset = offset,
1316 entry->dev_addr = dma_addr;
1317 entry->size = size;
1318 entry->direction = direction;
1319 entry->map_err_type = MAP_ERR_NOT_CHECKED;
1320
1321 if (map_single)
1322 entry->type = dma_debug_single;
1323
1324 check_for_stack(dev, page, offset);
1325
1326 if (!PageHighMem(page)) {
1327 void *addr = page_address(page) + offset;
1328
1329 check_for_illegal_area(dev, addr, size);
1330 }
1331
1332 add_dma_entry(entry);
1333}
1334EXPORT_SYMBOL(debug_dma_map_page);
1335
1336void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
1337{
1338 struct dma_debug_entry ref;
1339 struct dma_debug_entry *entry;
1340 struct hash_bucket *bucket;
1341 unsigned long flags;
1342
1343 if (unlikely(dma_debug_disabled()))
1344 return;
1345
1346 ref.dev = dev;
1347 ref.dev_addr = dma_addr;
1348 bucket = get_hash_bucket(&ref, &flags);
1349
1350 list_for_each_entry(entry, &bucket->list, list) {
1351 if (!exact_match(&ref, entry))
1352 continue;
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364 if (entry->map_err_type == MAP_ERR_NOT_CHECKED) {
1365 entry->map_err_type = MAP_ERR_CHECKED;
1366 break;
1367 }
1368 }
1369
1370 put_hash_bucket(bucket, &flags);
1371}
1372EXPORT_SYMBOL(debug_dma_mapping_error);
1373
1374void debug_dma_unmap_page(struct device *dev, dma_addr_t addr,
1375 size_t size, int direction, bool map_single)
1376{
1377 struct dma_debug_entry ref = {
1378 .type = dma_debug_page,
1379 .dev = dev,
1380 .dev_addr = addr,
1381 .size = size,
1382 .direction = direction,
1383 };
1384
1385 if (unlikely(dma_debug_disabled()))
1386 return;
1387
1388 if (map_single)
1389 ref.type = dma_debug_single;
1390
1391 check_unmap(&ref);
1392}
1393EXPORT_SYMBOL(debug_dma_unmap_page);
1394
1395void debug_dma_map_sg(struct device *dev, struct scatterlist *sg,
1396 int nents, int mapped_ents, int direction)
1397{
1398 struct dma_debug_entry *entry;
1399 struct scatterlist *s;
1400 int i;
1401
1402 if (unlikely(dma_debug_disabled()))
1403 return;
1404
1405 for_each_sg(sg, s, mapped_ents, i) {
1406 entry = dma_entry_alloc();
1407 if (!entry)
1408 return;
1409
1410 entry->type = dma_debug_sg;
1411 entry->dev = dev;
1412 entry->pfn = page_to_pfn(sg_page(s));
1413 entry->offset = s->offset,
1414 entry->size = sg_dma_len(s);
1415 entry->dev_addr = sg_dma_address(s);
1416 entry->direction = direction;
1417 entry->sg_call_ents = nents;
1418 entry->sg_mapped_ents = mapped_ents;
1419
1420 check_for_stack(dev, sg_page(s), s->offset);
1421
1422 if (!PageHighMem(sg_page(s))) {
1423 check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s));
1424 }
1425
1426 add_dma_entry(entry);
1427 }
1428}
1429EXPORT_SYMBOL(debug_dma_map_sg);
1430
1431static int get_nr_mapped_entries(struct device *dev,
1432 struct dma_debug_entry *ref)
1433{
1434 struct dma_debug_entry *entry;
1435 struct hash_bucket *bucket;
1436 unsigned long flags;
1437 int mapped_ents;
1438
1439 bucket = get_hash_bucket(ref, &flags);
1440 entry = bucket_find_exact(bucket, ref);
1441 mapped_ents = 0;
1442
1443 if (entry)
1444 mapped_ents = entry->sg_mapped_ents;
1445 put_hash_bucket(bucket, &flags);
1446
1447 return mapped_ents;
1448}
1449
1450void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
1451 int nelems, int dir)
1452{
1453 struct scatterlist *s;
1454 int mapped_ents = 0, i;
1455
1456 if (unlikely(dma_debug_disabled()))
1457 return;
1458
1459 for_each_sg(sglist, s, nelems, i) {
1460
1461 struct dma_debug_entry ref = {
1462 .type = dma_debug_sg,
1463 .dev = dev,
1464 .pfn = page_to_pfn(sg_page(s)),
1465 .offset = s->offset,
1466 .dev_addr = sg_dma_address(s),
1467 .size = sg_dma_len(s),
1468 .direction = dir,
1469 .sg_call_ents = nelems,
1470 };
1471
1472 if (mapped_ents && i >= mapped_ents)
1473 break;
1474
1475 if (!i)
1476 mapped_ents = get_nr_mapped_entries(dev, &ref);
1477
1478 check_unmap(&ref);
1479 }
1480}
1481EXPORT_SYMBOL(debug_dma_unmap_sg);
1482
1483void debug_dma_alloc_coherent(struct device *dev, size_t size,
1484 dma_addr_t dma_addr, void *virt)
1485{
1486 struct dma_debug_entry *entry;
1487
1488 if (unlikely(dma_debug_disabled()))
1489 return;
1490
1491 if (unlikely(virt == NULL))
1492 return;
1493
1494
1495 if (!is_vmalloc_addr(virt) && !virt_addr_valid(virt))
1496 return;
1497
1498 entry = dma_entry_alloc();
1499 if (!entry)
1500 return;
1501
1502 entry->type = dma_debug_coherent;
1503 entry->dev = dev;
1504 entry->offset = offset_in_page(virt);
1505 entry->size = size;
1506 entry->dev_addr = dma_addr;
1507 entry->direction = DMA_BIDIRECTIONAL;
1508
1509 if (is_vmalloc_addr(virt))
1510 entry->pfn = vmalloc_to_pfn(virt);
1511 else
1512 entry->pfn = page_to_pfn(virt_to_page(virt));
1513
1514 add_dma_entry(entry);
1515}
1516EXPORT_SYMBOL(debug_dma_alloc_coherent);
1517
1518void debug_dma_free_coherent(struct device *dev, size_t size,
1519 void *virt, dma_addr_t addr)
1520{
1521 struct dma_debug_entry ref = {
1522 .type = dma_debug_coherent,
1523 .dev = dev,
1524 .offset = offset_in_page(virt),
1525 .dev_addr = addr,
1526 .size = size,
1527 .direction = DMA_BIDIRECTIONAL,
1528 };
1529
1530
1531 if (!is_vmalloc_addr(virt) && !virt_addr_valid(virt))
1532 return;
1533
1534 if (is_vmalloc_addr(virt))
1535 ref.pfn = vmalloc_to_pfn(virt);
1536 else
1537 ref.pfn = page_to_pfn(virt_to_page(virt));
1538
1539 if (unlikely(dma_debug_disabled()))
1540 return;
1541
1542 check_unmap(&ref);
1543}
1544EXPORT_SYMBOL(debug_dma_free_coherent);
1545
1546void debug_dma_map_resource(struct device *dev, phys_addr_t addr, size_t size,
1547 int direction, dma_addr_t dma_addr)
1548{
1549 struct dma_debug_entry *entry;
1550
1551 if (unlikely(dma_debug_disabled()))
1552 return;
1553
1554 entry = dma_entry_alloc();
1555 if (!entry)
1556 return;
1557
1558 entry->type = dma_debug_resource;
1559 entry->dev = dev;
1560 entry->pfn = PHYS_PFN(addr);
1561 entry->offset = offset_in_page(addr);
1562 entry->size = size;
1563 entry->dev_addr = dma_addr;
1564 entry->direction = direction;
1565 entry->map_err_type = MAP_ERR_NOT_CHECKED;
1566
1567 add_dma_entry(entry);
1568}
1569EXPORT_SYMBOL(debug_dma_map_resource);
1570
1571void debug_dma_unmap_resource(struct device *dev, dma_addr_t dma_addr,
1572 size_t size, int direction)
1573{
1574 struct dma_debug_entry ref = {
1575 .type = dma_debug_resource,
1576 .dev = dev,
1577 .dev_addr = dma_addr,
1578 .size = size,
1579 .direction = direction,
1580 };
1581
1582 if (unlikely(dma_debug_disabled()))
1583 return;
1584
1585 check_unmap(&ref);
1586}
1587EXPORT_SYMBOL(debug_dma_unmap_resource);
1588
1589void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
1590 size_t size, int direction)
1591{
1592 struct dma_debug_entry ref;
1593
1594 if (unlikely(dma_debug_disabled()))
1595 return;
1596
1597 ref.type = dma_debug_single;
1598 ref.dev = dev;
1599 ref.dev_addr = dma_handle;
1600 ref.size = size;
1601 ref.direction = direction;
1602 ref.sg_call_ents = 0;
1603
1604 check_sync(dev, &ref, true);
1605}
1606EXPORT_SYMBOL(debug_dma_sync_single_for_cpu);
1607
1608void debug_dma_sync_single_for_device(struct device *dev,
1609 dma_addr_t dma_handle, size_t size,
1610 int direction)
1611{
1612 struct dma_debug_entry ref;
1613
1614 if (unlikely(dma_debug_disabled()))
1615 return;
1616
1617 ref.type = dma_debug_single;
1618 ref.dev = dev;
1619 ref.dev_addr = dma_handle;
1620 ref.size = size;
1621 ref.direction = direction;
1622 ref.sg_call_ents = 0;
1623
1624 check_sync(dev, &ref, false);
1625}
1626EXPORT_SYMBOL(debug_dma_sync_single_for_device);
1627
1628void debug_dma_sync_single_range_for_cpu(struct device *dev,
1629 dma_addr_t dma_handle,
1630 unsigned long offset, size_t size,
1631 int direction)
1632{
1633 struct dma_debug_entry ref;
1634
1635 if (unlikely(dma_debug_disabled()))
1636 return;
1637
1638 ref.type = dma_debug_single;
1639 ref.dev = dev;
1640 ref.dev_addr = dma_handle;
1641 ref.size = offset + size;
1642 ref.direction = direction;
1643 ref.sg_call_ents = 0;
1644
1645 check_sync(dev, &ref, true);
1646}
1647EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu);
1648
1649void debug_dma_sync_single_range_for_device(struct device *dev,
1650 dma_addr_t dma_handle,
1651 unsigned long offset,
1652 size_t size, int direction)
1653{
1654 struct dma_debug_entry ref;
1655
1656 if (unlikely(dma_debug_disabled()))
1657 return;
1658
1659 ref.type = dma_debug_single;
1660 ref.dev = dev;
1661 ref.dev_addr = dma_handle;
1662 ref.size = offset + size;
1663 ref.direction = direction;
1664 ref.sg_call_ents = 0;
1665
1666 check_sync(dev, &ref, false);
1667}
1668EXPORT_SYMBOL(debug_dma_sync_single_range_for_device);
1669
1670void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
1671 int nelems, int direction)
1672{
1673 struct scatterlist *s;
1674 int mapped_ents = 0, i;
1675
1676 if (unlikely(dma_debug_disabled()))
1677 return;
1678
1679 for_each_sg(sg, s, nelems, i) {
1680
1681 struct dma_debug_entry ref = {
1682 .type = dma_debug_sg,
1683 .dev = dev,
1684 .pfn = page_to_pfn(sg_page(s)),
1685 .offset = s->offset,
1686 .dev_addr = sg_dma_address(s),
1687 .size = sg_dma_len(s),
1688 .direction = direction,
1689 .sg_call_ents = nelems,
1690 };
1691
1692 if (!i)
1693 mapped_ents = get_nr_mapped_entries(dev, &ref);
1694
1695 if (i >= mapped_ents)
1696 break;
1697
1698 check_sync(dev, &ref, true);
1699 }
1700}
1701EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu);
1702
1703void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
1704 int nelems, int direction)
1705{
1706 struct scatterlist *s;
1707 int mapped_ents = 0, i;
1708
1709 if (unlikely(dma_debug_disabled()))
1710 return;
1711
1712 for_each_sg(sg, s, nelems, i) {
1713
1714 struct dma_debug_entry ref = {
1715 .type = dma_debug_sg,
1716 .dev = dev,
1717 .pfn = page_to_pfn(sg_page(s)),
1718 .offset = s->offset,
1719 .dev_addr = sg_dma_address(s),
1720 .size = sg_dma_len(s),
1721 .direction = direction,
1722 .sg_call_ents = nelems,
1723 };
1724 if (!i)
1725 mapped_ents = get_nr_mapped_entries(dev, &ref);
1726
1727 if (i >= mapped_ents)
1728 break;
1729
1730 check_sync(dev, &ref, false);
1731 }
1732}
1733EXPORT_SYMBOL(debug_dma_sync_sg_for_device);
1734
1735static int __init dma_debug_driver_setup(char *str)
1736{
1737 int i;
1738
1739 for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) {
1740 current_driver_name[i] = *str;
1741 if (*str == 0)
1742 break;
1743 }
1744
1745 if (current_driver_name[0])
1746 pr_info("DMA-API: enable driver filter for driver [%s]\n",
1747 current_driver_name);
1748
1749
1750 return 1;
1751}
1752__setup("dma_debug_driver=", dma_debug_driver_setup);
1753