2 * omap iommu: simple virtual address space management
4 * Copyright (C) 2008-2009 Nokia Corporation
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
13 #include <linux/err.h>
14 #include <linux/slab.h>
15 #include <linux/vmalloc.h>
16 #include <linux/device.h>
17 #include <linux/scatterlist.h>
19 #include <asm/cacheflush.h>
20 #include <asm/mach/map.h>
22 #include <plat/iommu.h>
23 #include <plat/iovmm.h>
25 #include "iopgtable.h"
28 * A device driver needs to create address mappings between:
30 * - iommu/device address
32 * - mpu virtual address
34 * There are 4 possible patterns for them:
36 * |iova/ mapping iommu_ page
37 * | da pa va (d)-(p)-(v) function type
38 * ---------------------------------------------------------------------------
39 * 1 | c c c 1 - 1 - 1 _kmap() / _kunmap() s
40 * 2 | c c,a c 1 - 1 - 1 _kmalloc()/ _kfree() s
41 * 3 | c d c 1 - n - 1 _vmap() / _vunmap() s
42 * 4 | c d,a c 1 - n - 1 _vmalloc()/ _vfree() n*
45 * 'iova': device iommu virtual address
46 * 'da': alias of 'iova'
47 * 'pa': physical address
48 * 'va': mpu virtual address
50 * 'c': contiguous memory area
51 * 'd': discontiguous memory area
52 * 'a': anonymous memory allocation
53 * '()': optional feature
55 * 'n': a normal page(4KB) size is used.
56 * 's': multiple iommu superpage(16MB, 1MB, 64KB, 4KB) size is used.
58 * '*': not yet, but feasible.
61 static struct kmem_cache
*iovm_area_cachep
;
63 /* return total bytes of sg buffers */
64 static size_t sgtable_len(const struct sg_table
*sgt
)
66 unsigned int i
, total
= 0;
67 struct scatterlist
*sg
;
72 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
75 bytes
= sg_dma_len(sg
);
77 if (!iopgsz_ok(bytes
)) {
78 pr_err("%s: sg[%d] not iommu pagesize(%x)\n",
88 #define sgtable_ok(x) (!!sgtable_len(x))
91 * calculate the optimal number sg elements from total bytes based on
94 static unsigned int sgtable_nents(size_t bytes
)
97 unsigned int nr_entries
;
98 const unsigned long pagesize
[] = { SZ_16M
, SZ_1M
, SZ_64K
, SZ_4K
, };
100 if (!IS_ALIGNED(bytes
, PAGE_SIZE
)) {
101 pr_err("%s: wrong size %08x\n", __func__
, bytes
);
106 for (i
= 0; i
< ARRAY_SIZE(pagesize
); i
++) {
107 if (bytes
>= pagesize
[i
]) {
108 nr_entries
+= (bytes
/ pagesize
[i
]);
109 bytes
%= pagesize
[i
];
117 /* allocate and initialize sg_table header(a kind of 'superblock') */
118 static struct sg_table
*sgtable_alloc(const size_t bytes
, u32 flags
)
120 unsigned int nr_entries
;
122 struct sg_table
*sgt
;
125 return ERR_PTR(-EINVAL
);
127 if (!IS_ALIGNED(bytes
, PAGE_SIZE
))
128 return ERR_PTR(-EINVAL
);
130 if ((flags
& IOVMF_LINEAR
) && (flags
& IOVMF_DA_ANON
)) {
131 nr_entries
= sgtable_nents(bytes
);
133 return ERR_PTR(-EINVAL
);
135 nr_entries
= bytes
/ PAGE_SIZE
;
137 sgt
= kzalloc(sizeof(*sgt
), GFP_KERNEL
);
139 return ERR_PTR(-ENOMEM
);
141 err
= sg_alloc_table(sgt
, nr_entries
, GFP_KERNEL
);
147 pr_debug("%s: sgt:%p(%d entries)\n", __func__
, sgt
, nr_entries
);
152 /* free sg_table header(a kind of superblock) */
153 static void sgtable_free(struct sg_table
*sgt
)
161 pr_debug("%s: sgt:%p\n", __func__
, sgt
);
164 /* map 'sglist' to a contiguous mpu virtual area and return 'va' */
165 static void *vmap_sg(const struct sg_table
*sgt
)
170 struct scatterlist
*sg
;
171 struct vm_struct
*new;
172 const struct mem_type
*mtype
;
174 mtype
= get_mem_type(MT_DEVICE
);
176 return ERR_PTR(-EINVAL
);
178 total
= sgtable_len(sgt
);
180 return ERR_PTR(-EINVAL
);
182 new = __get_vm_area(total
, VM_IOREMAP
, VMALLOC_START
, VMALLOC_END
);
184 return ERR_PTR(-ENOMEM
);
187 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
193 bytes
= sg_dma_len(sg
);
195 BUG_ON(bytes
!= PAGE_SIZE
);
197 err
= ioremap_page(va
, pa
, mtype
);
204 flush_cache_vmap((unsigned long)new->addr
,
205 (unsigned long)(new->addr
+ total
));
211 return ERR_PTR(-EAGAIN
);
214 static inline void vunmap_sg(const void *va
)
219 static struct iovm_struct
*__find_iovm_area(struct iommu
*obj
, const u32 da
)
221 struct iovm_struct
*tmp
;
223 list_for_each_entry(tmp
, &obj
->mmap
, list
) {
224 if ((da
>= tmp
->da_start
) && (da
< tmp
->da_end
)) {
227 len
= tmp
->da_end
- tmp
->da_start
;
229 dev_dbg(obj
->dev
, "%s: %08x-%08x-%08x(%x) %08x\n",
230 __func__
, tmp
->da_start
, da
, tmp
->da_end
, len
,
241 * find_iovm_area - find iovma which includes @da
242 * @da: iommu device virtual address
244 * Find the existing iovma starting at @da
246 struct iovm_struct
*find_iovm_area(struct iommu
*obj
, u32 da
)
248 struct iovm_struct
*area
;
250 mutex_lock(&obj
->mmap_lock
);
251 area
= __find_iovm_area(obj
, da
);
252 mutex_unlock(&obj
->mmap_lock
);
256 EXPORT_SYMBOL_GPL(find_iovm_area
);
259 * This finds the hole(area) which fits the requested address and len
260 * in iovmas mmap, and returns the new allocated iovma.
262 static struct iovm_struct
*alloc_iovm_area(struct iommu
*obj
, u32 da
,
263 size_t bytes
, u32 flags
)
265 struct iovm_struct
*new, *tmp
;
266 u32 start
, prev_end
, alignement
;
269 return ERR_PTR(-EINVAL
);
272 alignement
= PAGE_SIZE
;
274 if (flags
& IOVMF_DA_ANON
) {
276 * Reserve the first page for NULL
279 if (flags
& IOVMF_LINEAR
)
280 alignement
= iopgsz_max(bytes
);
281 start
= roundup(start
, alignement
);
285 if (list_empty(&obj
->mmap
))
289 list_for_each_entry(tmp
, &obj
->mmap
, list
) {
291 if (prev_end
>= start
)
294 if (start
+ bytes
< tmp
->da_start
)
297 if (flags
& IOVMF_DA_ANON
)
298 start
= roundup(tmp
->da_end
+ 1, alignement
);
300 prev_end
= tmp
->da_end
;
303 if ((start
> prev_end
) && (ULONG_MAX
- start
>= bytes
))
306 dev_dbg(obj
->dev
, "%s: no space to fit %08x(%x) flags: %08x\n",
307 __func__
, da
, bytes
, flags
);
309 return ERR_PTR(-EINVAL
);
312 new = kmem_cache_zalloc(iovm_area_cachep
, GFP_KERNEL
);
314 return ERR_PTR(-ENOMEM
);
317 new->da_start
= start
;
318 new->da_end
= start
+ bytes
;
322 * keep ascending order of iovmas
325 list_add_tail(&new->list
, &tmp
->list
);
327 list_add(&new->list
, &obj
->mmap
);
329 dev_dbg(obj
->dev
, "%s: found %08x-%08x-%08x(%x) %08x\n",
330 __func__
, new->da_start
, start
, new->da_end
, bytes
, flags
);
335 static void free_iovm_area(struct iommu
*obj
, struct iovm_struct
*area
)
339 BUG_ON(!obj
|| !area
);
341 bytes
= area
->da_end
- area
->da_start
;
343 dev_dbg(obj
->dev
, "%s: %08x-%08x(%x) %08x\n",
344 __func__
, area
->da_start
, area
->da_end
, bytes
, area
->flags
);
346 list_del(&area
->list
);
347 kmem_cache_free(iovm_area_cachep
, area
);
351 * da_to_va - convert (d) to (v)
352 * @obj: objective iommu
353 * @da: iommu device virtual address
354 * @va: mpu virtual address
356 * Returns mpu virtual addr which corresponds to a given device virtual addr
358 void *da_to_va(struct iommu
*obj
, u32 da
)
361 struct iovm_struct
*area
;
363 mutex_lock(&obj
->mmap_lock
);
365 area
= __find_iovm_area(obj
, da
);
367 dev_dbg(obj
->dev
, "%s: no da area(%08x)\n", __func__
, da
);
372 mutex_unlock(&obj
->mmap_lock
);
376 EXPORT_SYMBOL_GPL(da_to_va
);
378 static void sgtable_fill_vmalloc(struct sg_table
*sgt
, void *_va
)
381 struct scatterlist
*sg
;
385 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
387 const size_t bytes
= PAGE_SIZE
;
390 * iommu 'superpage' isn't supported with 'iommu_vmalloc()'
392 pg
= vmalloc_to_page(va
);
394 sg_set_page(sg
, pg
, bytes
, 0);
399 va_end
= _va
+ PAGE_SIZE
* i
;
402 static inline void sgtable_drain_vmalloc(struct sg_table
*sgt
)
405 * Actually this is not necessary at all, just exists for
406 * consistency of the code readability.
411 static void sgtable_fill_kmalloc(struct sg_table
*sgt
, u32 pa
, size_t len
)
414 struct scatterlist
*sg
;
417 va
= phys_to_virt(pa
);
419 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
422 bytes
= iopgsz_max(len
);
424 BUG_ON(!iopgsz_ok(bytes
));
426 sg_set_buf(sg
, phys_to_virt(pa
), bytes
);
428 * 'pa' is cotinuous(linear).
436 static inline void sgtable_drain_kmalloc(struct sg_table
*sgt
)
439 * Actually this is not necessary at all, just exists for
440 * consistency of the code readability
445 /* create 'da' <-> 'pa' mapping from 'sgt' */
446 static int map_iovm_area(struct iommu
*obj
, struct iovm_struct
*new,
447 const struct sg_table
*sgt
, u32 flags
)
451 struct scatterlist
*sg
;
452 u32 da
= new->da_start
;
457 BUG_ON(!sgtable_ok(sgt
));
459 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
463 struct iotlb_entry e
;
466 bytes
= sg_dma_len(sg
);
468 flags
&= ~IOVMF_PGSZ_MASK
;
469 pgsz
= bytes_to_iopgsz(bytes
);
474 pr_debug("%s: [%d] %08x %08x(%x)\n", __func__
,
477 iotlb_init_entry(&e
, da
, pa
, flags
);
478 err
= iopgtable_store_entry(obj
, &e
);
489 for_each_sg(sgt
->sgl
, sg
, i
, j
) {
492 bytes
= iopgtable_clear_entry(obj
, da
);
494 BUG_ON(!iopgsz_ok(bytes
));
501 /* release 'da' <-> 'pa' mapping */
502 static void unmap_iovm_area(struct iommu
*obj
, struct iovm_struct
*area
)
505 size_t total
= area
->da_end
- area
->da_start
;
507 BUG_ON((!total
) || !IS_ALIGNED(total
, PAGE_SIZE
));
509 start
= area
->da_start
;
513 bytes
= iopgtable_clear_entry(obj
, start
);
517 dev_dbg(obj
->dev
, "%s: unmap %08x(%x) %08x\n",
518 __func__
, start
, bytes
, area
->flags
);
520 BUG_ON(!IS_ALIGNED(bytes
, PAGE_SIZE
));
528 /* template function for all unmapping */
529 static struct sg_table
*unmap_vm_area(struct iommu
*obj
, const u32 da
,
530 void (*fn
)(const void *), u32 flags
)
532 struct sg_table
*sgt
= NULL
;
533 struct iovm_struct
*area
;
535 if (!IS_ALIGNED(da
, PAGE_SIZE
)) {
536 dev_err(obj
->dev
, "%s: alignment err(%08x)\n", __func__
, da
);
540 mutex_lock(&obj
->mmap_lock
);
542 area
= __find_iovm_area(obj
, da
);
544 dev_dbg(obj
->dev
, "%s: no da area(%08x)\n", __func__
, da
);
548 if ((area
->flags
& flags
) != flags
) {
549 dev_err(obj
->dev
, "%s: wrong flags(%08x)\n", __func__
,
553 sgt
= (struct sg_table
*)area
->sgt
;
555 unmap_iovm_area(obj
, area
);
559 dev_dbg(obj
->dev
, "%s: %08x-%08x-%08x(%x) %08x\n", __func__
,
560 area
->da_start
, da
, area
->da_end
,
561 area
->da_end
- area
->da_start
, area
->flags
);
563 free_iovm_area(obj
, area
);
565 mutex_unlock(&obj
->mmap_lock
);
570 static u32
map_iommu_region(struct iommu
*obj
, u32 da
,
571 const struct sg_table
*sgt
, void *va
, size_t bytes
, u32 flags
)
574 struct iovm_struct
*new;
576 mutex_lock(&obj
->mmap_lock
);
578 new = alloc_iovm_area(obj
, da
, bytes
, flags
);
581 goto err_alloc_iovma
;
586 if (map_iovm_area(obj
, new, sgt
, new->flags
))
589 mutex_unlock(&obj
->mmap_lock
);
591 dev_dbg(obj
->dev
, "%s: da:%08x(%x) flags:%08x va:%p\n",
592 __func__
, new->da_start
, bytes
, new->flags
, va
);
594 return new->da_start
;
597 free_iovm_area(obj
, new);
599 mutex_unlock(&obj
->mmap_lock
);
603 static inline u32
__iommu_vmap(struct iommu
*obj
, u32 da
,
604 const struct sg_table
*sgt
, void *va
, size_t bytes
, u32 flags
)
606 return map_iommu_region(obj
, da
, sgt
, va
, bytes
, flags
);
610 * iommu_vmap - (d)-(p)-(v) address mapper
611 * @obj: objective iommu
612 * @sgt: address of scatter gather table
613 * @flags: iovma and page property
615 * Creates 1-n-1 mapping with given @sgt and returns @da.
616 * All @sgt element must be io page size aligned.
618 u32
iommu_vmap(struct iommu
*obj
, u32 da
, const struct sg_table
*sgt
,
624 if (!obj
|| !obj
->dev
|| !sgt
)
627 bytes
= sgtable_len(sgt
);
630 bytes
= PAGE_ALIGN(bytes
);
632 if (flags
& IOVMF_MMIO
) {
638 flags
&= IOVMF_HW_MASK
;
639 flags
|= IOVMF_DISCONT
;
641 flags
|= (da
? IOVMF_DA_FIXED
: IOVMF_DA_ANON
);
643 da
= __iommu_vmap(obj
, da
, sgt
, va
, bytes
, flags
);
644 if (IS_ERR_VALUE(da
))
649 EXPORT_SYMBOL_GPL(iommu_vmap
);
652 * iommu_vunmap - release virtual mapping obtained by 'iommu_vmap()'
653 * @obj: objective iommu
654 * @da: iommu device virtual address
656 * Free the iommu virtually contiguous memory area starting at
657 * @da, which was returned by 'iommu_vmap()'.
659 struct sg_table
*iommu_vunmap(struct iommu
*obj
, u32 da
)
661 struct sg_table
*sgt
;
663 * 'sgt' is allocated before 'iommu_vmalloc()' is called.
664 * Just returns 'sgt' to the caller to free
666 sgt
= unmap_vm_area(obj
, da
, vunmap_sg
, IOVMF_DISCONT
| IOVMF_MMIO
);
668 dev_dbg(obj
->dev
, "%s: No sgt\n", __func__
);
671 EXPORT_SYMBOL_GPL(iommu_vunmap
);
674 * iommu_vmalloc - (d)-(p)-(v) address allocator and mapper
675 * @obj: objective iommu
676 * @da: contiguous iommu virtual memory
677 * @bytes: allocation size
678 * @flags: iovma and page property
680 * Allocate @bytes linearly and creates 1-n-1 mapping and returns
681 * @da again, which might be adjusted if 'IOVMF_DA_ANON' is set.
683 u32
iommu_vmalloc(struct iommu
*obj
, u32 da
, size_t bytes
, u32 flags
)
686 struct sg_table
*sgt
;
688 if (!obj
|| !obj
->dev
|| !bytes
)
691 bytes
= PAGE_ALIGN(bytes
);
697 sgt
= sgtable_alloc(bytes
, flags
);
702 sgtable_fill_vmalloc(sgt
, va
);
704 flags
&= IOVMF_HW_MASK
;
705 flags
|= IOVMF_DISCONT
;
706 flags
|= IOVMF_ALLOC
;
707 flags
|= (da
? IOVMF_DA_FIXED
: IOVMF_DA_ANON
);
709 da
= __iommu_vmap(obj
, da
, sgt
, va
, bytes
, flags
);
710 if (IS_ERR_VALUE(da
))
716 sgtable_drain_vmalloc(sgt
);
722 EXPORT_SYMBOL_GPL(iommu_vmalloc
);
725 * iommu_vfree - release memory allocated by 'iommu_vmalloc()'
726 * @obj: objective iommu
727 * @da: iommu device virtual address
729 * Frees the iommu virtually continuous memory area starting at
730 * @da, as obtained from 'iommu_vmalloc()'.
732 void iommu_vfree(struct iommu
*obj
, const u32 da
)
734 struct sg_table
*sgt
;
736 sgt
= unmap_vm_area(obj
, da
, vfree
, IOVMF_DISCONT
| IOVMF_ALLOC
);
738 dev_dbg(obj
->dev
, "%s: No sgt\n", __func__
);
741 EXPORT_SYMBOL_GPL(iommu_vfree
);
743 static u32
__iommu_kmap(struct iommu
*obj
, u32 da
, u32 pa
, void *va
,
744 size_t bytes
, u32 flags
)
746 struct sg_table
*sgt
;
748 sgt
= sgtable_alloc(bytes
, flags
);
752 sgtable_fill_kmalloc(sgt
, pa
, bytes
);
754 da
= map_iommu_region(obj
, da
, sgt
, va
, bytes
, flags
);
755 if (IS_ERR_VALUE(da
)) {
756 sgtable_drain_kmalloc(sgt
);
764 * iommu_kmap - (d)-(p)-(v) address mapper
765 * @obj: objective iommu
766 * @da: contiguous iommu virtual memory
767 * @pa: contiguous physical memory
768 * @flags: iovma and page property
770 * Creates 1-1-1 mapping and returns @da again, which can be
771 * adjusted if 'IOVMF_DA_ANON' is set.
773 u32
iommu_kmap(struct iommu
*obj
, u32 da
, u32 pa
, size_t bytes
,
778 if (!obj
|| !obj
->dev
|| !bytes
)
781 bytes
= PAGE_ALIGN(bytes
);
783 va
= ioremap(pa
, bytes
);
787 flags
&= IOVMF_HW_MASK
;
788 flags
|= IOVMF_LINEAR
;
790 flags
|= (da
? IOVMF_DA_FIXED
: IOVMF_DA_ANON
);
792 da
= __iommu_kmap(obj
, da
, pa
, va
, bytes
, flags
);
793 if (IS_ERR_VALUE(da
))
798 EXPORT_SYMBOL_GPL(iommu_kmap
);
801 * iommu_kunmap - release virtual mapping obtained by 'iommu_kmap()'
802 * @obj: objective iommu
803 * @da: iommu device virtual address
805 * Frees the iommu virtually contiguous memory area starting at
806 * @da, which was passed to and was returned by'iommu_kmap()'.
808 void iommu_kunmap(struct iommu
*obj
, u32 da
)
810 struct sg_table
*sgt
;
811 typedef void (*func_t
)(const void *);
813 sgt
= unmap_vm_area(obj
, da
, (func_t
)__iounmap
,
814 IOVMF_LINEAR
| IOVMF_MMIO
);
816 dev_dbg(obj
->dev
, "%s: No sgt\n", __func__
);
819 EXPORT_SYMBOL_GPL(iommu_kunmap
);
822 * iommu_kmalloc - (d)-(p)-(v) address allocator and mapper
823 * @obj: objective iommu
824 * @da: contiguous iommu virtual memory
825 * @bytes: bytes for allocation
826 * @flags: iovma and page property
828 * Allocate @bytes linearly and creates 1-1-1 mapping and returns
829 * @da again, which might be adjusted if 'IOVMF_DA_ANON' is set.
831 u32
iommu_kmalloc(struct iommu
*obj
, u32 da
, size_t bytes
, u32 flags
)
836 if (!obj
|| !obj
->dev
|| !bytes
)
839 bytes
= PAGE_ALIGN(bytes
);
841 va
= kmalloc(bytes
, GFP_KERNEL
| GFP_DMA
);
844 pa
= virt_to_phys(va
);
846 flags
&= IOVMF_HW_MASK
;
847 flags
|= IOVMF_LINEAR
;
848 flags
|= IOVMF_ALLOC
;
849 flags
|= (da
? IOVMF_DA_FIXED
: IOVMF_DA_ANON
);
851 da
= __iommu_kmap(obj
, da
, pa
, va
, bytes
, flags
);
852 if (IS_ERR_VALUE(da
))
857 EXPORT_SYMBOL_GPL(iommu_kmalloc
);
860 * iommu_kfree - release virtual mapping obtained by 'iommu_kmalloc()'
861 * @obj: objective iommu
862 * @da: iommu device virtual address
864 * Frees the iommu virtually contiguous memory area starting at
865 * @da, which was passed to and was returned by'iommu_kmalloc()'.
867 void iommu_kfree(struct iommu
*obj
, u32 da
)
869 struct sg_table
*sgt
;
871 sgt
= unmap_vm_area(obj
, da
, kfree
, IOVMF_LINEAR
| IOVMF_ALLOC
);
873 dev_dbg(obj
->dev
, "%s: No sgt\n", __func__
);
876 EXPORT_SYMBOL_GPL(iommu_kfree
);
879 static int __init
iovmm_init(void)
881 const unsigned long flags
= SLAB_HWCACHE_ALIGN
;
882 struct kmem_cache
*p
;
884 p
= kmem_cache_create("iovm_area_cache", sizeof(struct iovm_struct
), 0,
888 iovm_area_cachep
= p
;
892 module_init(iovmm_init
);
894 static void __exit
iovmm_exit(void)
896 kmem_cache_destroy(iovm_area_cachep
);
898 module_exit(iovmm_exit
);
900 MODULE_DESCRIPTION("omap iommu: simple virtual address space management");
901 MODULE_AUTHOR("Hiroshi DOYU <Hiroshi.DOYU@nokia.com>");
902 MODULE_LICENSE("GPL v2");