2 * Copyright (C) 2008 Advanced Micro Devices, Inc.
4 * Author: Joerg Roedel <joerg.roedel@amd.com>
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 #include <linux/scatterlist.h>
21 #include <linux/dma-mapping.h>
22 #include <linux/stacktrace.h>
23 #include <linux/dma-debug.h>
24 #include <linux/spinlock.h>
25 #include <linux/debugfs.h>
26 #include <linux/device.h>
27 #include <linux/types.h>
28 #include <linux/sched.h>
29 #include <linux/list.h>
30 #include <linux/slab.h>
32 #include <asm/sections.h>
34 #define HASH_SIZE 1024ULL
35 #define HASH_FN_SHIFT 13
36 #define HASH_FN_MASK (HASH_SIZE - 1)
45 #define DMA_DEBUG_STACKTRACE_ENTRIES 5
47 struct dma_debug_entry
{
48 struct list_head list
;
57 #ifdef CONFIG_STACKTRACE
58 struct stack_trace stacktrace
;
59 unsigned long st_entries
[DMA_DEBUG_STACKTRACE_ENTRIES
];
64 struct list_head list
;
66 } ____cacheline_aligned_in_smp
;
68 /* Hash list to save the allocated dma addresses */
69 static struct hash_bucket dma_entry_hash
[HASH_SIZE
];
70 /* List of pre-allocated dma_debug_entry's */
71 static LIST_HEAD(free_entries
);
72 /* Lock for the list above */
73 static DEFINE_SPINLOCK(free_entries_lock
);
75 /* Global disable flag - will be set in case of an error */
76 static bool global_disable __read_mostly
;
78 /* Global error count */
79 static u32 error_count
;
81 /* Global error show enable*/
82 static u32 show_all_errors __read_mostly
;
83 /* Number of errors to show */
84 static u32 show_num_errors
= 1;
86 static u32 num_free_entries
;
87 static u32 min_free_entries
;
89 /* number of preallocated entries requested by kernel cmdline */
90 static u32 req_entries
;
92 /* debugfs dentry's for the stuff above */
93 static struct dentry
*dma_debug_dent __read_mostly
;
94 static struct dentry
*global_disable_dent __read_mostly
;
95 static struct dentry
*error_count_dent __read_mostly
;
96 static struct dentry
*show_all_errors_dent __read_mostly
;
97 static struct dentry
*show_num_errors_dent __read_mostly
;
98 static struct dentry
*num_free_entries_dent __read_mostly
;
99 static struct dentry
*min_free_entries_dent __read_mostly
;
101 static const char *type2name
[4] = { "single", "page",
102 "scather-gather", "coherent" };
104 static const char *dir2name
[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE",
105 "DMA_FROM_DEVICE", "DMA_NONE" };
108 * The access to some variables in this macro is racy. We can't use atomic_t
109 * here because all these variables are exported to debugfs. Some of them even
110 * writeable. This is also the reason why a lock won't help much. But anyway,
111 * the races are no big deal. Here is why:
113 * error_count: the addition is racy, but the worst thing that can happen is
114 * that we don't count some errors
115 * show_num_errors: the subtraction is racy. Also no big deal because in
116 * worst case this will result in one warning more in the
117 * system log than the user configured. This variable is
118 * writeable via debugfs.
120 static inline void dump_entry_trace(struct dma_debug_entry
*entry
)
122 #ifdef CONFIG_STACKTRACE
124 printk(KERN_WARNING
"Mapped at:\n");
125 print_stack_trace(&entry
->stacktrace
, 0);
130 #define err_printk(dev, entry, format, arg...) do { \
132 if (show_all_errors || show_num_errors > 0) { \
133 WARN(1, "%s %s: " format, \
134 dev_driver_string(dev), \
135 dev_name(dev) , ## arg); \
136 dump_entry_trace(entry); \
138 if (!show_all_errors && show_num_errors > 0) \
139 show_num_errors -= 1; \
143 * Hash related functions
145 * Every DMA-API request is saved into a struct dma_debug_entry. To
146 * have quick access to these structs they are stored into a hash.
148 static int hash_fn(struct dma_debug_entry
*entry
)
151 * Hash function is based on the dma address.
152 * We use bits 20-27 here as the index into the hash
154 return (entry
->dev_addr
>> HASH_FN_SHIFT
) & HASH_FN_MASK
;
158 * Request exclusive access to a hash bucket for a given dma_debug_entry.
160 static struct hash_bucket
*get_hash_bucket(struct dma_debug_entry
*entry
,
161 unsigned long *flags
)
163 int idx
= hash_fn(entry
);
164 unsigned long __flags
;
166 spin_lock_irqsave(&dma_entry_hash
[idx
].lock
, __flags
);
168 return &dma_entry_hash
[idx
];
172 * Give up exclusive access to the hash bucket
174 static void put_hash_bucket(struct hash_bucket
*bucket
,
175 unsigned long *flags
)
177 unsigned long __flags
= *flags
;
179 spin_unlock_irqrestore(&bucket
->lock
, __flags
);
183 * Search a given entry in the hash bucket list
185 static struct dma_debug_entry
*hash_bucket_find(struct hash_bucket
*bucket
,
186 struct dma_debug_entry
*ref
)
188 struct dma_debug_entry
*entry
, *ret
= NULL
;
189 int matches
= 0, match_lvl
, last_lvl
= 0;
191 list_for_each_entry(entry
, &bucket
->list
, list
) {
192 if ((entry
->dev_addr
!= ref
->dev_addr
) ||
193 (entry
->dev
!= ref
->dev
))
197 * Some drivers map the same physical address multiple
198 * times. Without a hardware IOMMU this results in the
199 * same device addresses being put into the dma-debug
200 * hash multiple times too. This can result in false
201 * positives being reported. Therfore we implement a
202 * best-fit algorithm here which returns the entry from
203 * the hash which fits best to the reference value
204 * instead of the first-fit.
208 entry
->size
== ref
->size
? ++match_lvl
: match_lvl
;
209 entry
->type
== ref
->type
? ++match_lvl
: match_lvl
;
210 entry
->direction
== ref
->direction
? ++match_lvl
: match_lvl
;
212 if (match_lvl
== 3) {
213 /* perfect-fit - return the result */
215 } else if (match_lvl
> last_lvl
) {
217 * We found an entry that fits better then the
220 last_lvl
= match_lvl
;
226 * If we have multiple matches but no perfect-fit, just return
229 ret
= (matches
== 1) ? ret
: NULL
;
235 * Add an entry to a hash bucket
237 static void hash_bucket_add(struct hash_bucket
*bucket
,
238 struct dma_debug_entry
*entry
)
240 list_add_tail(&entry
->list
, &bucket
->list
);
244 * Remove entry from a hash bucket list
246 static void hash_bucket_del(struct dma_debug_entry
*entry
)
248 list_del(&entry
->list
);
252 * Dump mapping entries for debugging purposes
254 void debug_dma_dump_mappings(struct device
*dev
)
258 for (idx
= 0; idx
< HASH_SIZE
; idx
++) {
259 struct hash_bucket
*bucket
= &dma_entry_hash
[idx
];
260 struct dma_debug_entry
*entry
;
263 spin_lock_irqsave(&bucket
->lock
, flags
);
265 list_for_each_entry(entry
, &bucket
->list
, list
) {
266 if (!dev
|| dev
== entry
->dev
) {
268 "%s idx %d P=%Lx D=%Lx L=%Lx %s\n",
269 type2name
[entry
->type
], idx
,
270 (unsigned long long)entry
->paddr
,
271 entry
->dev_addr
, entry
->size
,
272 dir2name
[entry
->direction
]);
276 spin_unlock_irqrestore(&bucket
->lock
, flags
);
279 EXPORT_SYMBOL(debug_dma_dump_mappings
);
282 * Wrapper function for adding an entry to the hash.
283 * This function takes care of locking itself.
285 static void add_dma_entry(struct dma_debug_entry
*entry
)
287 struct hash_bucket
*bucket
;
290 bucket
= get_hash_bucket(entry
, &flags
);
291 hash_bucket_add(bucket
, entry
);
292 put_hash_bucket(bucket
, &flags
);
295 /* struct dma_entry allocator
297 * The next two functions implement the allocator for
298 * struct dma_debug_entries.
300 static struct dma_debug_entry
*dma_entry_alloc(void)
302 struct dma_debug_entry
*entry
= NULL
;
305 spin_lock_irqsave(&free_entries_lock
, flags
);
307 if (list_empty(&free_entries
)) {
308 printk(KERN_ERR
"DMA-API: debugging out of memory "
310 global_disable
= true;
314 entry
= list_entry(free_entries
.next
, struct dma_debug_entry
, list
);
315 list_del(&entry
->list
);
316 memset(entry
, 0, sizeof(*entry
));
318 #ifdef CONFIG_STACKTRACE
319 entry
->stacktrace
.max_entries
= DMA_DEBUG_STACKTRACE_ENTRIES
;
320 entry
->stacktrace
.entries
= entry
->st_entries
;
321 entry
->stacktrace
.skip
= 2;
322 save_stack_trace(&entry
->stacktrace
);
324 num_free_entries
-= 1;
325 if (num_free_entries
< min_free_entries
)
326 min_free_entries
= num_free_entries
;
329 spin_unlock_irqrestore(&free_entries_lock
, flags
);
334 static void dma_entry_free(struct dma_debug_entry
*entry
)
339 * add to beginning of the list - this way the entries are
340 * more likely cache hot when they are reallocated.
342 spin_lock_irqsave(&free_entries_lock
, flags
);
343 list_add(&entry
->list
, &free_entries
);
344 num_free_entries
+= 1;
345 spin_unlock_irqrestore(&free_entries_lock
, flags
);
349 * DMA-API debugging init code
351 * The init code does two things:
352 * 1. Initialize core data structures
353 * 2. Preallocate a given number of dma_debug_entry structs
356 static int prealloc_memory(u32 num_entries
)
358 struct dma_debug_entry
*entry
, *next_entry
;
361 for (i
= 0; i
< num_entries
; ++i
) {
362 entry
= kzalloc(sizeof(*entry
), GFP_KERNEL
);
366 list_add_tail(&entry
->list
, &free_entries
);
369 num_free_entries
= num_entries
;
370 min_free_entries
= num_entries
;
372 printk(KERN_INFO
"DMA-API: preallocated %d debug entries\n",
379 list_for_each_entry_safe(entry
, next_entry
, &free_entries
, list
) {
380 list_del(&entry
->list
);
387 static int dma_debug_fs_init(void)
389 dma_debug_dent
= debugfs_create_dir("dma-api", NULL
);
390 if (!dma_debug_dent
) {
391 printk(KERN_ERR
"DMA-API: can not create debugfs directory\n");
395 global_disable_dent
= debugfs_create_bool("disabled", 0444,
397 (u32
*)&global_disable
);
398 if (!global_disable_dent
)
401 error_count_dent
= debugfs_create_u32("error_count", 0444,
402 dma_debug_dent
, &error_count
);
403 if (!error_count_dent
)
406 show_all_errors_dent
= debugfs_create_u32("all_errors", 0644,
409 if (!show_all_errors_dent
)
412 show_num_errors_dent
= debugfs_create_u32("num_errors", 0644,
415 if (!show_num_errors_dent
)
418 num_free_entries_dent
= debugfs_create_u32("num_free_entries", 0444,
421 if (!num_free_entries_dent
)
424 min_free_entries_dent
= debugfs_create_u32("min_free_entries", 0444,
427 if (!min_free_entries_dent
)
433 debugfs_remove_recursive(dma_debug_dent
);
438 void dma_debug_add_bus(struct bus_type
*bus
)
440 /* FIXME: register notifier */
444 * Let the architectures decide how many entries should be preallocated.
446 void dma_debug_init(u32 num_entries
)
453 for (i
= 0; i
< HASH_SIZE
; ++i
) {
454 INIT_LIST_HEAD(&dma_entry_hash
[i
].list
);
455 dma_entry_hash
[i
].lock
= SPIN_LOCK_UNLOCKED
;
458 if (dma_debug_fs_init() != 0) {
459 printk(KERN_ERR
"DMA-API: error creating debugfs entries "
461 global_disable
= true;
467 num_entries
= req_entries
;
469 if (prealloc_memory(num_entries
) != 0) {
470 printk(KERN_ERR
"DMA-API: debugging out of memory error "
472 global_disable
= true;
477 printk(KERN_INFO
"DMA-API: debugging enabled by kernel config\n");
480 static __init
int dma_debug_cmdline(char *str
)
485 if (strncmp(str
, "off", 3) == 0) {
486 printk(KERN_INFO
"DMA-API: debugging disabled on kernel "
488 global_disable
= true;
494 static __init
int dma_debug_entries_cmdline(char *str
)
501 res
= get_option(&str
, &req_entries
);
509 __setup("dma_debug=", dma_debug_cmdline
);
510 __setup("dma_debug_entries=", dma_debug_entries_cmdline
);
512 static void check_unmap(struct dma_debug_entry
*ref
)
514 struct dma_debug_entry
*entry
;
515 struct hash_bucket
*bucket
;
518 if (dma_mapping_error(ref
->dev
, ref
->dev_addr
)) {
519 err_printk(ref
->dev
, NULL
, "DMA-API: device driver tries "
520 "to free an invalid DMA memory address\n");
524 bucket
= get_hash_bucket(ref
, &flags
);
525 entry
= hash_bucket_find(bucket
, ref
);
528 err_printk(ref
->dev
, NULL
, "DMA-API: device driver tries "
529 "to free DMA memory it has not allocated "
530 "[device address=0x%016llx] [size=%llu bytes]\n",
531 ref
->dev_addr
, ref
->size
);
535 if (ref
->size
!= entry
->size
) {
536 err_printk(ref
->dev
, entry
, "DMA-API: device driver frees "
537 "DMA memory with different size "
538 "[device address=0x%016llx] [map size=%llu bytes] "
539 "[unmap size=%llu bytes]\n",
540 ref
->dev_addr
, entry
->size
, ref
->size
);
543 if (ref
->type
!= entry
->type
) {
544 err_printk(ref
->dev
, entry
, "DMA-API: device driver frees "
545 "DMA memory with wrong function "
546 "[device address=0x%016llx] [size=%llu bytes] "
547 "[mapped as %s] [unmapped as %s]\n",
548 ref
->dev_addr
, ref
->size
,
549 type2name
[entry
->type
], type2name
[ref
->type
]);
550 } else if ((entry
->type
== dma_debug_coherent
) &&
551 (ref
->paddr
!= entry
->paddr
)) {
552 err_printk(ref
->dev
, entry
, "DMA-API: device driver frees "
553 "DMA memory with different CPU address "
554 "[device address=0x%016llx] [size=%llu bytes] "
555 "[cpu alloc address=%p] [cpu free address=%p]",
556 ref
->dev_addr
, ref
->size
,
557 (void *)entry
->paddr
, (void *)ref
->paddr
);
560 if (ref
->sg_call_ents
&& ref
->type
== dma_debug_sg
&&
561 ref
->sg_call_ents
!= entry
->sg_call_ents
) {
562 err_printk(ref
->dev
, entry
, "DMA-API: device driver frees "
563 "DMA sg list with different entry count "
564 "[map count=%d] [unmap count=%d]\n",
565 entry
->sg_call_ents
, ref
->sg_call_ents
);
569 * This may be no bug in reality - but most implementations of the
570 * DMA API don't handle this properly, so check for it here
572 if (ref
->direction
!= entry
->direction
) {
573 err_printk(ref
->dev
, entry
, "DMA-API: device driver frees "
574 "DMA memory with different direction "
575 "[device address=0x%016llx] [size=%llu bytes] "
576 "[mapped with %s] [unmapped with %s]\n",
577 ref
->dev_addr
, ref
->size
,
578 dir2name
[entry
->direction
],
579 dir2name
[ref
->direction
]);
582 hash_bucket_del(entry
);
583 dma_entry_free(entry
);
586 put_hash_bucket(bucket
, &flags
);
589 static void check_for_stack(struct device
*dev
, void *addr
)
591 if (object_is_on_stack(addr
))
592 err_printk(dev
, NULL
, "DMA-API: device driver maps memory from"
593 "stack [addr=%p]\n", addr
);
596 static inline bool overlap(void *addr
, u64 size
, void *start
, void *end
)
598 void *addr2
= (char *)addr
+ size
;
600 return ((addr
>= start
&& addr
< end
) ||
601 (addr2
>= start
&& addr2
< end
) ||
602 ((addr
< start
) && (addr2
> end
)));
605 static void check_for_illegal_area(struct device
*dev
, void *addr
, u64 size
)
607 if (overlap(addr
, size
, _text
, _etext
) ||
608 overlap(addr
, size
, __start_rodata
, __end_rodata
))
609 err_printk(dev
, NULL
, "DMA-API: device driver maps "
610 "memory from kernel text or rodata "
611 "[addr=%p] [size=%llu]\n", addr
, size
);
614 static void check_sync(struct device
*dev
, dma_addr_t addr
,
615 u64 size
, u64 offset
, int direction
, bool to_cpu
)
617 struct dma_debug_entry ref
= {
621 .direction
= direction
,
623 struct dma_debug_entry
*entry
;
624 struct hash_bucket
*bucket
;
627 bucket
= get_hash_bucket(&ref
, &flags
);
629 entry
= hash_bucket_find(bucket
, &ref
);
632 err_printk(dev
, NULL
, "DMA-API: device driver tries "
633 "to sync DMA memory it has not allocated "
634 "[device address=0x%016llx] [size=%llu bytes]\n",
635 (unsigned long long)addr
, size
);
639 if ((offset
+ size
) > entry
->size
) {
640 err_printk(dev
, entry
, "DMA-API: device driver syncs"
641 " DMA memory outside allocated range "
642 "[device address=0x%016llx] "
643 "[allocation size=%llu bytes] [sync offset=%llu] "
644 "[sync size=%llu]\n", entry
->dev_addr
, entry
->size
,
648 if (direction
!= entry
->direction
) {
649 err_printk(dev
, entry
, "DMA-API: device driver syncs "
650 "DMA memory with different direction "
651 "[device address=0x%016llx] [size=%llu bytes] "
652 "[mapped with %s] [synced with %s]\n",
653 (unsigned long long)addr
, entry
->size
,
654 dir2name
[entry
->direction
],
655 dir2name
[direction
]);
658 if (entry
->direction
== DMA_BIDIRECTIONAL
)
661 if (to_cpu
&& !(entry
->direction
== DMA_FROM_DEVICE
) &&
662 !(direction
== DMA_TO_DEVICE
))
663 err_printk(dev
, entry
, "DMA-API: device driver syncs "
664 "device read-only DMA memory for cpu "
665 "[device address=0x%016llx] [size=%llu bytes] "
666 "[mapped with %s] [synced with %s]\n",
667 (unsigned long long)addr
, entry
->size
,
668 dir2name
[entry
->direction
],
669 dir2name
[direction
]);
671 if (!to_cpu
&& !(entry
->direction
== DMA_TO_DEVICE
) &&
672 !(direction
== DMA_FROM_DEVICE
))
673 err_printk(dev
, entry
, "DMA-API: device driver syncs "
674 "device write-only DMA memory to device "
675 "[device address=0x%016llx] [size=%llu bytes] "
676 "[mapped with %s] [synced with %s]\n",
677 (unsigned long long)addr
, entry
->size
,
678 dir2name
[entry
->direction
],
679 dir2name
[direction
]);
682 put_hash_bucket(bucket
, &flags
);
686 void debug_dma_map_page(struct device
*dev
, struct page
*page
, size_t offset
,
687 size_t size
, int direction
, dma_addr_t dma_addr
,
690 struct dma_debug_entry
*entry
;
692 if (unlikely(global_disable
))
695 if (unlikely(dma_mapping_error(dev
, dma_addr
)))
698 entry
= dma_entry_alloc();
703 entry
->type
= dma_debug_page
;
704 entry
->paddr
= page_to_phys(page
) + offset
;
705 entry
->dev_addr
= dma_addr
;
707 entry
->direction
= direction
;
710 entry
->type
= dma_debug_single
;
712 if (!PageHighMem(page
)) {
713 void *addr
= ((char *)page_address(page
)) + offset
;
714 check_for_stack(dev
, addr
);
715 check_for_illegal_area(dev
, addr
, size
);
718 add_dma_entry(entry
);
720 EXPORT_SYMBOL(debug_dma_map_page
);
722 void debug_dma_unmap_page(struct device
*dev
, dma_addr_t addr
,
723 size_t size
, int direction
, bool map_single
)
725 struct dma_debug_entry ref
= {
726 .type
= dma_debug_page
,
730 .direction
= direction
,
733 if (unlikely(global_disable
))
737 ref
.type
= dma_debug_single
;
741 EXPORT_SYMBOL(debug_dma_unmap_page
);
743 void debug_dma_map_sg(struct device
*dev
, struct scatterlist
*sg
,
744 int nents
, int mapped_ents
, int direction
)
746 struct dma_debug_entry
*entry
;
747 struct scatterlist
*s
;
750 if (unlikely(global_disable
))
753 for_each_sg(sg
, s
, mapped_ents
, i
) {
754 entry
= dma_entry_alloc();
758 entry
->type
= dma_debug_sg
;
760 entry
->paddr
= sg_phys(s
);
761 entry
->size
= s
->length
;
762 entry
->dev_addr
= s
->dma_address
;
763 entry
->direction
= direction
;
764 entry
->sg_call_ents
= nents
;
765 entry
->sg_mapped_ents
= mapped_ents
;
767 if (!PageHighMem(sg_page(s
))) {
768 check_for_stack(dev
, sg_virt(s
));
769 check_for_illegal_area(dev
, sg_virt(s
), s
->length
);
772 add_dma_entry(entry
);
775 EXPORT_SYMBOL(debug_dma_map_sg
);
777 void debug_dma_unmap_sg(struct device
*dev
, struct scatterlist
*sglist
,
780 struct dma_debug_entry
*entry
;
781 struct scatterlist
*s
;
782 int mapped_ents
= 0, i
;
785 if (unlikely(global_disable
))
788 for_each_sg(sglist
, s
, nelems
, i
) {
790 struct dma_debug_entry ref
= {
791 .type
= dma_debug_sg
,
794 .dev_addr
= s
->dma_address
,
800 if (mapped_ents
&& i
>= mapped_ents
)
803 if (mapped_ents
== 0) {
804 struct hash_bucket
*bucket
;
805 ref
.sg_call_ents
= nelems
;
806 bucket
= get_hash_bucket(&ref
, &flags
);
807 entry
= hash_bucket_find(bucket
, &ref
);
809 mapped_ents
= entry
->sg_mapped_ents
;
810 put_hash_bucket(bucket
, &flags
);
816 EXPORT_SYMBOL(debug_dma_unmap_sg
);
818 void debug_dma_alloc_coherent(struct device
*dev
, size_t size
,
819 dma_addr_t dma_addr
, void *virt
)
821 struct dma_debug_entry
*entry
;
823 if (unlikely(global_disable
))
826 if (unlikely(virt
== NULL
))
829 entry
= dma_entry_alloc();
833 entry
->type
= dma_debug_coherent
;
835 entry
->paddr
= virt_to_phys(virt
);
837 entry
->dev_addr
= dma_addr
;
838 entry
->direction
= DMA_BIDIRECTIONAL
;
840 add_dma_entry(entry
);
842 EXPORT_SYMBOL(debug_dma_alloc_coherent
);
844 void debug_dma_free_coherent(struct device
*dev
, size_t size
,
845 void *virt
, dma_addr_t addr
)
847 struct dma_debug_entry ref
= {
848 .type
= dma_debug_coherent
,
850 .paddr
= virt_to_phys(virt
),
853 .direction
= DMA_BIDIRECTIONAL
,
856 if (unlikely(global_disable
))
861 EXPORT_SYMBOL(debug_dma_free_coherent
);
863 void debug_dma_sync_single_for_cpu(struct device
*dev
, dma_addr_t dma_handle
,
864 size_t size
, int direction
)
866 if (unlikely(global_disable
))
869 check_sync(dev
, dma_handle
, size
, 0, direction
, true);
871 EXPORT_SYMBOL(debug_dma_sync_single_for_cpu
);
873 void debug_dma_sync_single_for_device(struct device
*dev
,
874 dma_addr_t dma_handle
, size_t size
,
877 if (unlikely(global_disable
))
880 check_sync(dev
, dma_handle
, size
, 0, direction
, false);
882 EXPORT_SYMBOL(debug_dma_sync_single_for_device
);
884 void debug_dma_sync_single_range_for_cpu(struct device
*dev
,
885 dma_addr_t dma_handle
,
886 unsigned long offset
, size_t size
,
889 if (unlikely(global_disable
))
892 check_sync(dev
, dma_handle
, size
, offset
, direction
, true);
894 EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu
);
896 void debug_dma_sync_single_range_for_device(struct device
*dev
,
897 dma_addr_t dma_handle
,
898 unsigned long offset
,
899 size_t size
, int direction
)
901 if (unlikely(global_disable
))
904 check_sync(dev
, dma_handle
, size
, offset
, direction
, false);
906 EXPORT_SYMBOL(debug_dma_sync_single_range_for_device
);
908 void debug_dma_sync_sg_for_cpu(struct device
*dev
, struct scatterlist
*sg
,
909 int nelems
, int direction
)
911 struct scatterlist
*s
;
914 if (unlikely(global_disable
))
917 for_each_sg(sg
, s
, nelems
, i
) {
918 check_sync(dev
, s
->dma_address
, s
->dma_length
, 0,
922 EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu
);
924 void debug_dma_sync_sg_for_device(struct device
*dev
, struct scatterlist
*sg
,
925 int nelems
, int direction
)
927 struct scatterlist
*s
;
930 if (unlikely(global_disable
))
933 for_each_sg(sg
, s
, nelems
, i
) {
934 check_sync(dev
, s
->dma_address
, s
->dma_length
, 0,
938 EXPORT_SYMBOL(debug_dma_sync_sg_for_device
);