2 * SWIOTLB-based DMA API implementation
4 * Copyright (C) 2012 ARM Ltd.
5 * Author: Catalin Marinas <catalin.marinas@arm.com>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
20 #include <linux/gfp.h>
21 #include <linux/acpi.h>
22 #include <linux/bootmem.h>
23 #include <linux/cache.h>
24 #include <linux/export.h>
25 #include <linux/slab.h>
26 #include <linux/genalloc.h>
27 #include <linux/dma-mapping.h>
28 #include <linux/dma-contiguous.h>
29 #include <linux/vmalloc.h>
30 #include <linux/swiotlb.h>
32 #include <asm/cacheflush.h>
34 static int swiotlb __ro_after_init
;
36 static pgprot_t
__get_dma_pgprot(unsigned long attrs
, pgprot_t prot
,
39 if (!coherent
|| (attrs
& DMA_ATTR_WRITE_COMBINE
))
40 return pgprot_writecombine(prot
);
44 static struct gen_pool
*atomic_pool
;
46 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
47 static size_t atomic_pool_size __initdata
= DEFAULT_DMA_COHERENT_POOL_SIZE
;
49 static int __init
early_coherent_pool(char *p
)
51 atomic_pool_size
= memparse(p
, &p
);
54 early_param("coherent_pool", early_coherent_pool
);
56 static void *__alloc_from_pool(size_t size
, struct page
**ret_page
, gfp_t flags
)
62 WARN(1, "coherent pool not initialised!\n");
66 val
= gen_pool_alloc(atomic_pool
, size
);
68 phys_addr_t phys
= gen_pool_virt_to_phys(atomic_pool
, val
);
70 *ret_page
= phys_to_page(phys
);
78 static bool __in_atomic_pool(void *start
, size_t size
)
80 return addr_in_gen_pool(atomic_pool
, (unsigned long)start
, size
);
83 static int __free_from_pool(void *start
, size_t size
)
85 if (!__in_atomic_pool(start
, size
))
88 gen_pool_free(atomic_pool
, (unsigned long)start
, size
);
93 static void *__dma_alloc_coherent(struct device
*dev
, size_t size
,
94 dma_addr_t
*dma_handle
, gfp_t flags
,
98 WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
102 if (IS_ENABLED(CONFIG_ZONE_DMA
) &&
103 dev
->coherent_dma_mask
<= DMA_BIT_MASK(32))
105 if (dev_get_cma_area(dev
) && gfpflags_allow_blocking(flags
)) {
109 page
= dma_alloc_from_contiguous(dev
, size
>> PAGE_SHIFT
,
114 *dma_handle
= phys_to_dma(dev
, page_to_phys(page
));
115 addr
= page_address(page
);
116 memset(addr
, 0, size
);
119 return swiotlb_alloc_coherent(dev
, size
, dma_handle
, flags
);
123 static void __dma_free_coherent(struct device
*dev
, size_t size
,
124 void *vaddr
, dma_addr_t dma_handle
,
128 phys_addr_t paddr
= dma_to_phys(dev
, dma_handle
);
131 WARN_ONCE(1, "Use an actual device structure for DMA allocation\n");
135 freed
= dma_release_from_contiguous(dev
,
139 swiotlb_free_coherent(dev
, size
, vaddr
, dma_handle
);
142 static void *__dma_alloc(struct device
*dev
, size_t size
,
143 dma_addr_t
*dma_handle
, gfp_t flags
,
147 void *ptr
, *coherent_ptr
;
148 bool coherent
= is_device_dma_coherent(dev
);
149 pgprot_t prot
= __get_dma_pgprot(attrs
, PAGE_KERNEL
, false);
151 size
= PAGE_ALIGN(size
);
153 if (!coherent
&& !gfpflags_allow_blocking(flags
)) {
154 struct page
*page
= NULL
;
155 void *addr
= __alloc_from_pool(size
, &page
, flags
);
158 *dma_handle
= phys_to_dma(dev
, page_to_phys(page
));
163 ptr
= __dma_alloc_coherent(dev
, size
, dma_handle
, flags
, attrs
);
167 /* no need for non-cacheable mapping if coherent */
171 /* remove any dirty cache lines on the kernel alias */
172 __dma_flush_area(ptr
, size
);
174 /* create a coherent mapping */
175 page
= virt_to_page(ptr
);
176 coherent_ptr
= dma_common_contiguous_remap(page
, size
, VM_USERMAP
,
184 __dma_free_coherent(dev
, size
, ptr
, *dma_handle
, attrs
);
186 *dma_handle
= DMA_ERROR_CODE
;
190 static void __dma_free(struct device
*dev
, size_t size
,
191 void *vaddr
, dma_addr_t dma_handle
,
194 void *swiotlb_addr
= phys_to_virt(dma_to_phys(dev
, dma_handle
));
196 size
= PAGE_ALIGN(size
);
198 if (!is_device_dma_coherent(dev
)) {
199 if (__free_from_pool(vaddr
, size
))
203 __dma_free_coherent(dev
, size
, swiotlb_addr
, dma_handle
, attrs
);
206 static dma_addr_t
__swiotlb_map_page(struct device
*dev
, struct page
*page
,
207 unsigned long offset
, size_t size
,
208 enum dma_data_direction dir
,
213 dev_addr
= swiotlb_map_page(dev
, page
, offset
, size
, dir
, attrs
);
214 if (!is_device_dma_coherent(dev
))
215 __dma_map_area(phys_to_virt(dma_to_phys(dev
, dev_addr
)), size
, dir
);
221 static void __swiotlb_unmap_page(struct device
*dev
, dma_addr_t dev_addr
,
222 size_t size
, enum dma_data_direction dir
,
225 if (!is_device_dma_coherent(dev
))
226 __dma_unmap_area(phys_to_virt(dma_to_phys(dev
, dev_addr
)), size
, dir
);
227 swiotlb_unmap_page(dev
, dev_addr
, size
, dir
, attrs
);
230 static int __swiotlb_map_sg_attrs(struct device
*dev
, struct scatterlist
*sgl
,
231 int nelems
, enum dma_data_direction dir
,
234 struct scatterlist
*sg
;
237 ret
= swiotlb_map_sg_attrs(dev
, sgl
, nelems
, dir
, attrs
);
238 if (!is_device_dma_coherent(dev
))
239 for_each_sg(sgl
, sg
, ret
, i
)
240 __dma_map_area(phys_to_virt(dma_to_phys(dev
, sg
->dma_address
)),
246 static void __swiotlb_unmap_sg_attrs(struct device
*dev
,
247 struct scatterlist
*sgl
, int nelems
,
248 enum dma_data_direction dir
,
251 struct scatterlist
*sg
;
254 if (!is_device_dma_coherent(dev
))
255 for_each_sg(sgl
, sg
, nelems
, i
)
256 __dma_unmap_area(phys_to_virt(dma_to_phys(dev
, sg
->dma_address
)),
258 swiotlb_unmap_sg_attrs(dev
, sgl
, nelems
, dir
, attrs
);
261 static void __swiotlb_sync_single_for_cpu(struct device
*dev
,
262 dma_addr_t dev_addr
, size_t size
,
263 enum dma_data_direction dir
)
265 if (!is_device_dma_coherent(dev
))
266 __dma_unmap_area(phys_to_virt(dma_to_phys(dev
, dev_addr
)), size
, dir
);
267 swiotlb_sync_single_for_cpu(dev
, dev_addr
, size
, dir
);
270 static void __swiotlb_sync_single_for_device(struct device
*dev
,
271 dma_addr_t dev_addr
, size_t size
,
272 enum dma_data_direction dir
)
274 swiotlb_sync_single_for_device(dev
, dev_addr
, size
, dir
);
275 if (!is_device_dma_coherent(dev
))
276 __dma_map_area(phys_to_virt(dma_to_phys(dev
, dev_addr
)), size
, dir
);
279 static void __swiotlb_sync_sg_for_cpu(struct device
*dev
,
280 struct scatterlist
*sgl
, int nelems
,
281 enum dma_data_direction dir
)
283 struct scatterlist
*sg
;
286 if (!is_device_dma_coherent(dev
))
287 for_each_sg(sgl
, sg
, nelems
, i
)
288 __dma_unmap_area(phys_to_virt(dma_to_phys(dev
, sg
->dma_address
)),
290 swiotlb_sync_sg_for_cpu(dev
, sgl
, nelems
, dir
);
293 static void __swiotlb_sync_sg_for_device(struct device
*dev
,
294 struct scatterlist
*sgl
, int nelems
,
295 enum dma_data_direction dir
)
297 struct scatterlist
*sg
;
300 swiotlb_sync_sg_for_device(dev
, sgl
, nelems
, dir
);
301 if (!is_device_dma_coherent(dev
))
302 for_each_sg(sgl
, sg
, nelems
, i
)
303 __dma_map_area(phys_to_virt(dma_to_phys(dev
, sg
->dma_address
)),
307 static int __swiotlb_mmap(struct device
*dev
,
308 struct vm_area_struct
*vma
,
309 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
,
313 unsigned long nr_vma_pages
= (vma
->vm_end
- vma
->vm_start
) >>
315 unsigned long nr_pages
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
316 unsigned long pfn
= dma_to_phys(dev
, dma_addr
) >> PAGE_SHIFT
;
317 unsigned long off
= vma
->vm_pgoff
;
319 vma
->vm_page_prot
= __get_dma_pgprot(attrs
, vma
->vm_page_prot
,
320 is_device_dma_coherent(dev
));
322 if (dma_mmap_from_coherent(dev
, vma
, cpu_addr
, size
, &ret
))
325 if (off
< nr_pages
&& nr_vma_pages
<= (nr_pages
- off
)) {
326 ret
= remap_pfn_range(vma
, vma
->vm_start
,
328 vma
->vm_end
- vma
->vm_start
,
335 static int __swiotlb_get_sgtable(struct device
*dev
, struct sg_table
*sgt
,
336 void *cpu_addr
, dma_addr_t handle
, size_t size
,
339 int ret
= sg_alloc_table(sgt
, 1, GFP_KERNEL
);
342 sg_set_page(sgt
->sgl
, phys_to_page(dma_to_phys(dev
, handle
)),
343 PAGE_ALIGN(size
), 0);
348 static int __swiotlb_dma_supported(struct device
*hwdev
, u64 mask
)
351 return swiotlb_dma_supported(hwdev
, mask
);
355 static struct dma_map_ops swiotlb_dma_ops
= {
356 .alloc
= __dma_alloc
,
358 .mmap
= __swiotlb_mmap
,
359 .get_sgtable
= __swiotlb_get_sgtable
,
360 .map_page
= __swiotlb_map_page
,
361 .unmap_page
= __swiotlb_unmap_page
,
362 .map_sg
= __swiotlb_map_sg_attrs
,
363 .unmap_sg
= __swiotlb_unmap_sg_attrs
,
364 .sync_single_for_cpu
= __swiotlb_sync_single_for_cpu
,
365 .sync_single_for_device
= __swiotlb_sync_single_for_device
,
366 .sync_sg_for_cpu
= __swiotlb_sync_sg_for_cpu
,
367 .sync_sg_for_device
= __swiotlb_sync_sg_for_device
,
368 .dma_supported
= __swiotlb_dma_supported
,
369 .mapping_error
= swiotlb_dma_mapping_error
,
372 static int __init
atomic_pool_init(void)
374 pgprot_t prot
= __pgprot(PROT_NORMAL_NC
);
375 unsigned long nr_pages
= atomic_pool_size
>> PAGE_SHIFT
;
378 unsigned int pool_size_order
= get_order(atomic_pool_size
);
380 if (dev_get_cma_area(NULL
))
381 page
= dma_alloc_from_contiguous(NULL
, nr_pages
,
384 page
= alloc_pages(GFP_DMA
, pool_size_order
);
388 void *page_addr
= page_address(page
);
390 memset(page_addr
, 0, atomic_pool_size
);
391 __dma_flush_area(page_addr
, atomic_pool_size
);
393 atomic_pool
= gen_pool_create(PAGE_SHIFT
, -1);
397 addr
= dma_common_contiguous_remap(page
, atomic_pool_size
,
398 VM_USERMAP
, prot
, atomic_pool_init
);
401 goto destroy_genpool
;
403 ret
= gen_pool_add_virt(atomic_pool
, (unsigned long)addr
,
405 atomic_pool_size
, -1);
409 gen_pool_set_algo(atomic_pool
,
410 gen_pool_first_fit_order_align
,
413 pr_info("DMA: preallocated %zu KiB pool for atomic allocations\n",
414 atomic_pool_size
/ 1024);
420 dma_common_free_remap(addr
, atomic_pool_size
, VM_USERMAP
);
422 gen_pool_destroy(atomic_pool
);
425 if (!dma_release_from_contiguous(NULL
, page
, nr_pages
))
426 __free_pages(page
, pool_size_order
);
428 pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n",
429 atomic_pool_size
/ 1024);
433 /********************************************
434 * The following APIs are for dummy DMA ops *
435 ********************************************/
437 static void *__dummy_alloc(struct device
*dev
, size_t size
,
438 dma_addr_t
*dma_handle
, gfp_t flags
,
444 static void __dummy_free(struct device
*dev
, size_t size
,
445 void *vaddr
, dma_addr_t dma_handle
,
450 static int __dummy_mmap(struct device
*dev
,
451 struct vm_area_struct
*vma
,
452 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
,
458 static dma_addr_t
__dummy_map_page(struct device
*dev
, struct page
*page
,
459 unsigned long offset
, size_t size
,
460 enum dma_data_direction dir
,
463 return DMA_ERROR_CODE
;
466 static void __dummy_unmap_page(struct device
*dev
, dma_addr_t dev_addr
,
467 size_t size
, enum dma_data_direction dir
,
472 static int __dummy_map_sg(struct device
*dev
, struct scatterlist
*sgl
,
473 int nelems
, enum dma_data_direction dir
,
479 static void __dummy_unmap_sg(struct device
*dev
,
480 struct scatterlist
*sgl
, int nelems
,
481 enum dma_data_direction dir
,
486 static void __dummy_sync_single(struct device
*dev
,
487 dma_addr_t dev_addr
, size_t size
,
488 enum dma_data_direction dir
)
492 static void __dummy_sync_sg(struct device
*dev
,
493 struct scatterlist
*sgl
, int nelems
,
494 enum dma_data_direction dir
)
498 static int __dummy_mapping_error(struct device
*hwdev
, dma_addr_t dma_addr
)
503 static int __dummy_dma_supported(struct device
*hwdev
, u64 mask
)
508 struct dma_map_ops dummy_dma_ops
= {
509 .alloc
= __dummy_alloc
,
510 .free
= __dummy_free
,
511 .mmap
= __dummy_mmap
,
512 .map_page
= __dummy_map_page
,
513 .unmap_page
= __dummy_unmap_page
,
514 .map_sg
= __dummy_map_sg
,
515 .unmap_sg
= __dummy_unmap_sg
,
516 .sync_single_for_cpu
= __dummy_sync_single
,
517 .sync_single_for_device
= __dummy_sync_single
,
518 .sync_sg_for_cpu
= __dummy_sync_sg
,
519 .sync_sg_for_device
= __dummy_sync_sg
,
520 .mapping_error
= __dummy_mapping_error
,
521 .dma_supported
= __dummy_dma_supported
,
523 EXPORT_SYMBOL(dummy_dma_ops
);
525 static int __init
arm64_dma_init(void)
527 if (swiotlb_force
|| max_pfn
> (arm64_dma_phys_limit
>> PAGE_SHIFT
))
530 return atomic_pool_init();
532 arch_initcall(arm64_dma_init
);
534 #define PREALLOC_DMA_DEBUG_ENTRIES 4096
536 static int __init
dma_debug_do_init(void)
538 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES
);
541 fs_initcall(dma_debug_do_init
);
544 #ifdef CONFIG_IOMMU_DMA
545 #include <linux/dma-iommu.h>
546 #include <linux/platform_device.h>
547 #include <linux/amba/bus.h>
549 /* Thankfully, all cache ops are by VA so we can ignore phys here */
550 static void flush_page(struct device
*dev
, const void *virt
, phys_addr_t phys
)
552 __dma_flush_area(virt
, PAGE_SIZE
);
555 static void *__iommu_alloc_attrs(struct device
*dev
, size_t size
,
556 dma_addr_t
*handle
, gfp_t gfp
,
559 bool coherent
= is_device_dma_coherent(dev
);
560 int ioprot
= dma_direction_to_prot(DMA_BIDIRECTIONAL
, coherent
);
561 size_t iosize
= size
;
564 if (WARN(!dev
, "cannot create IOMMU mapping for unknown device\n"))
567 size
= PAGE_ALIGN(size
);
570 * Some drivers rely on this, and we probably don't want the
571 * possibility of stale kernel data being read by devices anyway.
575 if (gfpflags_allow_blocking(gfp
)) {
577 pgprot_t prot
= __get_dma_pgprot(attrs
, PAGE_KERNEL
, coherent
);
579 pages
= iommu_dma_alloc(dev
, iosize
, gfp
, attrs
, ioprot
,
584 addr
= dma_common_pages_remap(pages
, size
, VM_USERMAP
, prot
,
585 __builtin_return_address(0));
587 iommu_dma_free(dev
, pages
, iosize
, handle
);
591 * In atomic context we can't remap anything, so we'll only
592 * get the virtually contiguous buffer we need by way of a
593 * physically contiguous allocation.
596 page
= alloc_pages(gfp
, get_order(size
));
597 addr
= page
? page_address(page
) : NULL
;
599 addr
= __alloc_from_pool(size
, &page
, gfp
);
604 *handle
= iommu_dma_map_page(dev
, page
, 0, iosize
, ioprot
);
605 if (iommu_dma_mapping_error(dev
, *handle
)) {
607 __free_pages(page
, get_order(size
));
609 __free_from_pool(addr
, size
);
616 static void __iommu_free_attrs(struct device
*dev
, size_t size
, void *cpu_addr
,
617 dma_addr_t handle
, unsigned long attrs
)
619 size_t iosize
= size
;
621 size
= PAGE_ALIGN(size
);
623 * @cpu_addr will be one of 3 things depending on how it was allocated:
624 * - A remapped array of pages from iommu_dma_alloc(), for all
625 * non-atomic allocations.
626 * - A non-cacheable alias from the atomic pool, for atomic
627 * allocations by non-coherent devices.
628 * - A normal lowmem address, for atomic allocations by
630 * Hence how dodgy the below logic looks...
632 if (__in_atomic_pool(cpu_addr
, size
)) {
633 iommu_dma_unmap_page(dev
, handle
, iosize
, 0, 0);
634 __free_from_pool(cpu_addr
, size
);
635 } else if (is_vmalloc_addr(cpu_addr
)){
636 struct vm_struct
*area
= find_vm_area(cpu_addr
);
638 if (WARN_ON(!area
|| !area
->pages
))
640 iommu_dma_free(dev
, area
->pages
, iosize
, &handle
);
641 dma_common_free_remap(cpu_addr
, size
, VM_USERMAP
);
643 iommu_dma_unmap_page(dev
, handle
, iosize
, 0, 0);
644 __free_pages(virt_to_page(cpu_addr
), get_order(size
));
648 static int __iommu_mmap_attrs(struct device
*dev
, struct vm_area_struct
*vma
,
649 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
,
652 struct vm_struct
*area
;
655 vma
->vm_page_prot
= __get_dma_pgprot(attrs
, vma
->vm_page_prot
,
656 is_device_dma_coherent(dev
));
658 if (dma_mmap_from_coherent(dev
, vma
, cpu_addr
, size
, &ret
))
661 area
= find_vm_area(cpu_addr
);
662 if (WARN_ON(!area
|| !area
->pages
))
665 return iommu_dma_mmap(area
->pages
, size
, vma
);
668 static int __iommu_get_sgtable(struct device
*dev
, struct sg_table
*sgt
,
669 void *cpu_addr
, dma_addr_t dma_addr
,
670 size_t size
, unsigned long attrs
)
672 unsigned int count
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
673 struct vm_struct
*area
= find_vm_area(cpu_addr
);
675 if (WARN_ON(!area
|| !area
->pages
))
678 return sg_alloc_table_from_pages(sgt
, area
->pages
, count
, 0, size
,
682 static void __iommu_sync_single_for_cpu(struct device
*dev
,
683 dma_addr_t dev_addr
, size_t size
,
684 enum dma_data_direction dir
)
688 if (is_device_dma_coherent(dev
))
691 phys
= iommu_iova_to_phys(iommu_get_domain_for_dev(dev
), dev_addr
);
692 __dma_unmap_area(phys_to_virt(phys
), size
, dir
);
695 static void __iommu_sync_single_for_device(struct device
*dev
,
696 dma_addr_t dev_addr
, size_t size
,
697 enum dma_data_direction dir
)
701 if (is_device_dma_coherent(dev
))
704 phys
= iommu_iova_to_phys(iommu_get_domain_for_dev(dev
), dev_addr
);
705 __dma_map_area(phys_to_virt(phys
), size
, dir
);
708 static dma_addr_t
__iommu_map_page(struct device
*dev
, struct page
*page
,
709 unsigned long offset
, size_t size
,
710 enum dma_data_direction dir
,
713 bool coherent
= is_device_dma_coherent(dev
);
714 int prot
= dma_direction_to_prot(dir
, coherent
);
715 dma_addr_t dev_addr
= iommu_dma_map_page(dev
, page
, offset
, size
, prot
);
717 if (!iommu_dma_mapping_error(dev
, dev_addr
) &&
718 (attrs
& DMA_ATTR_SKIP_CPU_SYNC
) == 0)
719 __iommu_sync_single_for_device(dev
, dev_addr
, size
, dir
);
724 static void __iommu_unmap_page(struct device
*dev
, dma_addr_t dev_addr
,
725 size_t size
, enum dma_data_direction dir
,
728 if ((attrs
& DMA_ATTR_SKIP_CPU_SYNC
) == 0)
729 __iommu_sync_single_for_cpu(dev
, dev_addr
, size
, dir
);
731 iommu_dma_unmap_page(dev
, dev_addr
, size
, dir
, attrs
);
734 static void __iommu_sync_sg_for_cpu(struct device
*dev
,
735 struct scatterlist
*sgl
, int nelems
,
736 enum dma_data_direction dir
)
738 struct scatterlist
*sg
;
741 if (is_device_dma_coherent(dev
))
744 for_each_sg(sgl
, sg
, nelems
, i
)
745 __dma_unmap_area(sg_virt(sg
), sg
->length
, dir
);
748 static void __iommu_sync_sg_for_device(struct device
*dev
,
749 struct scatterlist
*sgl
, int nelems
,
750 enum dma_data_direction dir
)
752 struct scatterlist
*sg
;
755 if (is_device_dma_coherent(dev
))
758 for_each_sg(sgl
, sg
, nelems
, i
)
759 __dma_map_area(sg_virt(sg
), sg
->length
, dir
);
762 static int __iommu_map_sg_attrs(struct device
*dev
, struct scatterlist
*sgl
,
763 int nelems
, enum dma_data_direction dir
,
766 bool coherent
= is_device_dma_coherent(dev
);
768 if ((attrs
& DMA_ATTR_SKIP_CPU_SYNC
) == 0)
769 __iommu_sync_sg_for_device(dev
, sgl
, nelems
, dir
);
771 return iommu_dma_map_sg(dev
, sgl
, nelems
,
772 dma_direction_to_prot(dir
, coherent
));
775 static void __iommu_unmap_sg_attrs(struct device
*dev
,
776 struct scatterlist
*sgl
, int nelems
,
777 enum dma_data_direction dir
,
780 if ((attrs
& DMA_ATTR_SKIP_CPU_SYNC
) == 0)
781 __iommu_sync_sg_for_cpu(dev
, sgl
, nelems
, dir
);
783 iommu_dma_unmap_sg(dev
, sgl
, nelems
, dir
, attrs
);
786 static struct dma_map_ops iommu_dma_ops
= {
787 .alloc
= __iommu_alloc_attrs
,
788 .free
= __iommu_free_attrs
,
789 .mmap
= __iommu_mmap_attrs
,
790 .get_sgtable
= __iommu_get_sgtable
,
791 .map_page
= __iommu_map_page
,
792 .unmap_page
= __iommu_unmap_page
,
793 .map_sg
= __iommu_map_sg_attrs
,
794 .unmap_sg
= __iommu_unmap_sg_attrs
,
795 .sync_single_for_cpu
= __iommu_sync_single_for_cpu
,
796 .sync_single_for_device
= __iommu_sync_single_for_device
,
797 .sync_sg_for_cpu
= __iommu_sync_sg_for_cpu
,
798 .sync_sg_for_device
= __iommu_sync_sg_for_device
,
799 .dma_supported
= iommu_dma_supported
,
800 .mapping_error
= iommu_dma_mapping_error
,
804 * TODO: Right now __iommu_setup_dma_ops() gets called too early to do
805 * everything it needs to - the device is only partially created and the
806 * IOMMU driver hasn't seen it yet, so it can't have a group. Thus we
807 * need this delayed attachment dance. Once IOMMU probe ordering is sorted
808 * to move the arch_setup_dma_ops() call later, all the notifier bits below
809 * become unnecessary, and will go away.
811 struct iommu_dma_notifier_data
{
812 struct list_head list
;
814 const struct iommu_ops
*ops
;
818 static LIST_HEAD(iommu_dma_masters
);
819 static DEFINE_MUTEX(iommu_dma_notifier_lock
);
821 static bool do_iommu_attach(struct device
*dev
, const struct iommu_ops
*ops
,
822 u64 dma_base
, u64 size
)
824 struct iommu_domain
*domain
= iommu_get_domain_for_dev(dev
);
827 * If the IOMMU driver has the DMA domain support that we require,
828 * then the IOMMU core will have already configured a group for this
829 * device, and allocated the default domain for that group.
831 if (!domain
|| iommu_dma_init_domain(domain
, dma_base
, size
, dev
)) {
832 pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n",
837 dev
->archdata
.dma_ops
= &iommu_dma_ops
;
841 static void queue_iommu_attach(struct device
*dev
, const struct iommu_ops
*ops
,
842 u64 dma_base
, u64 size
)
844 struct iommu_dma_notifier_data
*iommudata
;
846 iommudata
= kzalloc(sizeof(*iommudata
), GFP_KERNEL
);
850 iommudata
->dev
= dev
;
851 iommudata
->ops
= ops
;
852 iommudata
->dma_base
= dma_base
;
853 iommudata
->size
= size
;
855 mutex_lock(&iommu_dma_notifier_lock
);
856 list_add(&iommudata
->list
, &iommu_dma_masters
);
857 mutex_unlock(&iommu_dma_notifier_lock
);
860 static int __iommu_attach_notifier(struct notifier_block
*nb
,
861 unsigned long action
, void *data
)
863 struct iommu_dma_notifier_data
*master
, *tmp
;
865 if (action
!= BUS_NOTIFY_BIND_DRIVER
)
868 mutex_lock(&iommu_dma_notifier_lock
);
869 list_for_each_entry_safe(master
, tmp
, &iommu_dma_masters
, list
) {
870 if (data
== master
->dev
&& do_iommu_attach(master
->dev
,
871 master
->ops
, master
->dma_base
, master
->size
)) {
872 list_del(&master
->list
);
877 mutex_unlock(&iommu_dma_notifier_lock
);
881 static int __init
register_iommu_dma_ops_notifier(struct bus_type
*bus
)
883 struct notifier_block
*nb
= kzalloc(sizeof(*nb
), GFP_KERNEL
);
889 nb
->notifier_call
= __iommu_attach_notifier
;
891 ret
= bus_register_notifier(bus
, nb
);
893 pr_warn("Failed to register DMA domain notifier; IOMMU DMA ops unavailable on bus '%s'\n",
900 static int __init
__iommu_dma_init(void)
904 ret
= iommu_dma_init();
906 ret
= register_iommu_dma_ops_notifier(&platform_bus_type
);
908 ret
= register_iommu_dma_ops_notifier(&amba_bustype
);
911 ret
= register_iommu_dma_ops_notifier(&pci_bus_type
);
915 arch_initcall(__iommu_dma_init
);
917 static void __iommu_setup_dma_ops(struct device
*dev
, u64 dma_base
, u64 size
,
918 const struct iommu_ops
*ops
)
920 struct iommu_group
*group
;
925 * TODO: As a concession to the future, we're ready to handle being
926 * called both early and late (i.e. after bus_add_device). Once all
927 * the platform bus code is reworked to call us late and the notifier
928 * junk above goes away, move the body of do_iommu_attach here.
930 group
= iommu_group_get(dev
);
932 do_iommu_attach(dev
, ops
, dma_base
, size
);
933 iommu_group_put(group
);
935 queue_iommu_attach(dev
, ops
, dma_base
, size
);
939 void arch_teardown_dma_ops(struct device
*dev
)
941 struct iommu_domain
*domain
= iommu_get_domain_for_dev(dev
);
944 iommu_detach_device(domain
, dev
);
946 dev
->archdata
.dma_ops
= NULL
;
951 static void __iommu_setup_dma_ops(struct device
*dev
, u64 dma_base
, u64 size
,
952 const struct iommu_ops
*iommu
)
955 #endif /* CONFIG_IOMMU_DMA */
957 void arch_setup_dma_ops(struct device
*dev
, u64 dma_base
, u64 size
,
958 const struct iommu_ops
*iommu
, bool coherent
)
960 if (!dev
->archdata
.dma_ops
)
961 dev
->archdata
.dma_ops
= &swiotlb_dma_ops
;
963 dev
->archdata
.dma_coherent
= coherent
;
964 __iommu_setup_dma_ops(dev
, dma_base
, size
, iommu
);