2 * omap iommu: simple virtual address space management
4 * Copyright (C) 2008-2009 Nokia Corporation
6 * Written by Hiroshi DOYU <Hiroshi.DOYU@nokia.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
13 #include <linux/err.h>
14 #include <linux/slab.h>
15 #include <linux/vmalloc.h>
16 #include <linux/device.h>
17 #include <linux/scatterlist.h>
18 #include <linux/iommu.h>
20 #include <asm/cacheflush.h>
21 #include <asm/mach/map.h>
23 #include <plat/iommu.h>
24 #include <plat/iovmm.h>
26 #include <plat/iopgtable.h>
28 static struct kmem_cache
*iovm_area_cachep
;
30 /* return the offset of the first scatterlist entry in a sg table */
31 static unsigned int sgtable_offset(const struct sg_table
*sgt
)
33 if (!sgt
|| !sgt
->nents
)
36 return sgt
->sgl
->offset
;
39 /* return total bytes of sg buffers */
40 static size_t sgtable_len(const struct sg_table
*sgt
)
42 unsigned int i
, total
= 0;
43 struct scatterlist
*sg
;
48 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
51 bytes
= sg
->length
+ sg
->offset
;
53 if (!iopgsz_ok(bytes
)) {
54 pr_err("%s: sg[%d] not iommu pagesize(%u %u)\n",
55 __func__
, i
, bytes
, sg
->offset
);
59 if (i
&& sg
->offset
) {
60 pr_err("%s: sg[%d] offset not allowed in internal "
61 "entries\n", __func__
, i
);
70 #define sgtable_ok(x) (!!sgtable_len(x))
72 static unsigned max_alignment(u32 addr
)
75 unsigned pagesize
[] = { SZ_16M
, SZ_1M
, SZ_64K
, SZ_4K
, };
76 for (i
= 0; i
< ARRAY_SIZE(pagesize
) && addr
& (pagesize
[i
] - 1); i
++)
78 return (i
< ARRAY_SIZE(pagesize
)) ? pagesize
[i
] : 0;
82 * calculate the optimal number sg elements from total bytes based on
85 static unsigned sgtable_nents(size_t bytes
, u32 da
, u32 pa
)
87 unsigned nr_entries
= 0, ent_sz
;
89 if (!IS_ALIGNED(bytes
, PAGE_SIZE
)) {
90 pr_err("%s: wrong size %08x\n", __func__
, bytes
);
95 ent_sz
= max_alignment(da
| pa
);
96 ent_sz
= min_t(unsigned, ent_sz
, iopgsz_max(bytes
));
106 /* allocate and initialize sg_table header(a kind of 'superblock') */
107 static struct sg_table
*sgtable_alloc(const size_t bytes
, u32 flags
,
110 unsigned int nr_entries
;
112 struct sg_table
*sgt
;
115 return ERR_PTR(-EINVAL
);
117 if (!IS_ALIGNED(bytes
, PAGE_SIZE
))
118 return ERR_PTR(-EINVAL
);
120 if (flags
& IOVMF_LINEAR
) {
121 nr_entries
= sgtable_nents(bytes
, da
, pa
);
123 return ERR_PTR(-EINVAL
);
125 nr_entries
= bytes
/ PAGE_SIZE
;
127 sgt
= kzalloc(sizeof(*sgt
), GFP_KERNEL
);
129 return ERR_PTR(-ENOMEM
);
131 err
= sg_alloc_table(sgt
, nr_entries
, GFP_KERNEL
);
137 pr_debug("%s: sgt:%p(%d entries)\n", __func__
, sgt
, nr_entries
);
142 /* free sg_table header(a kind of superblock) */
143 static void sgtable_free(struct sg_table
*sgt
)
151 pr_debug("%s: sgt:%p\n", __func__
, sgt
);
154 /* map 'sglist' to a contiguous mpu virtual area and return 'va' */
155 static void *vmap_sg(const struct sg_table
*sgt
)
160 struct scatterlist
*sg
;
161 struct vm_struct
*new;
162 const struct mem_type
*mtype
;
164 mtype
= get_mem_type(MT_DEVICE
);
166 return ERR_PTR(-EINVAL
);
168 total
= sgtable_len(sgt
);
170 return ERR_PTR(-EINVAL
);
172 new = __get_vm_area(total
, VM_IOREMAP
, VMALLOC_START
, VMALLOC_END
);
174 return ERR_PTR(-ENOMEM
);
177 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
182 pa
= sg_phys(sg
) - sg
->offset
;
183 bytes
= sg
->length
+ sg
->offset
;
185 BUG_ON(bytes
!= PAGE_SIZE
);
187 err
= ioremap_page(va
, pa
, mtype
);
194 flush_cache_vmap((unsigned long)new->addr
,
195 (unsigned long)(new->addr
+ total
));
199 WARN_ON(1); /* FIXME: cleanup some mpu mappings */
201 return ERR_PTR(-EAGAIN
);
204 static inline void vunmap_sg(const void *va
)
209 static struct iovm_struct
*__find_iovm_area(struct omap_iommu
*obj
,
212 struct iovm_struct
*tmp
;
214 list_for_each_entry(tmp
, &obj
->mmap
, list
) {
215 if ((da
>= tmp
->da_start
) && (da
< tmp
->da_end
)) {
218 len
= tmp
->da_end
- tmp
->da_start
;
220 dev_dbg(obj
->dev
, "%s: %08x-%08x-%08x(%x) %08x\n",
221 __func__
, tmp
->da_start
, da
, tmp
->da_end
, len
,
232 * omap_find_iovm_area - find iovma which includes @da
233 * @da: iommu device virtual address
235 * Find the existing iovma starting at @da
237 struct iovm_struct
*omap_find_iovm_area(struct omap_iommu
*obj
, u32 da
)
239 struct iovm_struct
*area
;
241 mutex_lock(&obj
->mmap_lock
);
242 area
= __find_iovm_area(obj
, da
);
243 mutex_unlock(&obj
->mmap_lock
);
247 EXPORT_SYMBOL_GPL(omap_find_iovm_area
);
250 * This finds the hole(area) which fits the requested address and len
251 * in iovmas mmap, and returns the new allocated iovma.
253 static struct iovm_struct
*alloc_iovm_area(struct omap_iommu
*obj
, u32 da
,
254 size_t bytes
, u32 flags
)
256 struct iovm_struct
*new, *tmp
;
257 u32 start
, prev_end
, alignment
;
260 return ERR_PTR(-EINVAL
);
263 alignment
= PAGE_SIZE
;
265 if (~flags
& IOVMF_DA_FIXED
) {
266 /* Don't map address 0 */
267 start
= obj
->da_start
? obj
->da_start
: alignment
;
269 if (flags
& IOVMF_LINEAR
)
270 alignment
= iopgsz_max(bytes
);
271 start
= roundup(start
, alignment
);
272 } else if (start
< obj
->da_start
|| start
> obj
->da_end
||
273 obj
->da_end
- start
< bytes
) {
274 return ERR_PTR(-EINVAL
);
278 if (list_empty(&obj
->mmap
))
282 list_for_each_entry(tmp
, &obj
->mmap
, list
) {
284 if (prev_end
> start
)
287 if (tmp
->da_start
> start
&& (tmp
->da_start
- start
) >= bytes
)
290 if (tmp
->da_end
>= start
&& ~flags
& IOVMF_DA_FIXED
)
291 start
= roundup(tmp
->da_end
+ 1, alignment
);
293 prev_end
= tmp
->da_end
;
296 if ((start
>= prev_end
) && (obj
->da_end
- start
>= bytes
))
299 dev_dbg(obj
->dev
, "%s: no space to fit %08x(%x) flags: %08x\n",
300 __func__
, da
, bytes
, flags
);
302 return ERR_PTR(-EINVAL
);
305 new = kmem_cache_zalloc(iovm_area_cachep
, GFP_KERNEL
);
307 return ERR_PTR(-ENOMEM
);
310 new->da_start
= start
;
311 new->da_end
= start
+ bytes
;
315 * keep ascending order of iovmas
318 list_add_tail(&new->list
, &tmp
->list
);
320 list_add(&new->list
, &obj
->mmap
);
322 dev_dbg(obj
->dev
, "%s: found %08x-%08x-%08x(%x) %08x\n",
323 __func__
, new->da_start
, start
, new->da_end
, bytes
, flags
);
328 static void free_iovm_area(struct omap_iommu
*obj
, struct iovm_struct
*area
)
332 BUG_ON(!obj
|| !area
);
334 bytes
= area
->da_end
- area
->da_start
;
336 dev_dbg(obj
->dev
, "%s: %08x-%08x(%x) %08x\n",
337 __func__
, area
->da_start
, area
->da_end
, bytes
, area
->flags
);
339 list_del(&area
->list
);
340 kmem_cache_free(iovm_area_cachep
, area
);
344 * omap_da_to_va - convert (d) to (v)
345 * @obj: objective iommu
346 * @da: iommu device virtual address
347 * @va: mpu virtual address
349 * Returns mpu virtual addr which corresponds to a given device virtual addr
351 void *omap_da_to_va(struct omap_iommu
*obj
, u32 da
)
354 struct iovm_struct
*area
;
356 mutex_lock(&obj
->mmap_lock
);
358 area
= __find_iovm_area(obj
, da
);
360 dev_dbg(obj
->dev
, "%s: no da area(%08x)\n", __func__
, da
);
365 mutex_unlock(&obj
->mmap_lock
);
369 EXPORT_SYMBOL_GPL(omap_da_to_va
);
371 static void sgtable_fill_vmalloc(struct sg_table
*sgt
, void *_va
)
374 struct scatterlist
*sg
;
378 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
380 const size_t bytes
= PAGE_SIZE
;
383 * iommu 'superpage' isn't supported with 'omap_iommu_vmalloc()'
385 pg
= vmalloc_to_page(va
);
387 sg_set_page(sg
, pg
, bytes
, 0);
392 va_end
= _va
+ PAGE_SIZE
* i
;
395 static inline void sgtable_drain_vmalloc(struct sg_table
*sgt
)
398 * Actually this is not necessary at all, just exists for
399 * consistency of the code readability.
404 /* create 'da' <-> 'pa' mapping from 'sgt' */
405 static int map_iovm_area(struct iommu_domain
*domain
, struct iovm_struct
*new,
406 const struct sg_table
*sgt
, u32 flags
)
410 struct scatterlist
*sg
;
411 u32 da
= new->da_start
;
417 BUG_ON(!sgtable_ok(sgt
));
419 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
423 pa
= sg_phys(sg
) - sg
->offset
;
424 bytes
= sg
->length
+ sg
->offset
;
426 flags
&= ~IOVMF_PGSZ_MASK
;
428 if (bytes_to_iopgsz(bytes
) < 0)
431 order
= get_order(bytes
);
433 pr_debug("%s: [%d] %08x %08x(%x)\n", __func__
,
436 err
= iommu_map(domain
, da
, pa
, order
, flags
);
447 for_each_sg(sgt
->sgl
, sg
, i
, j
) {
450 bytes
= sg
->length
+ sg
->offset
;
451 order
= get_order(bytes
);
453 /* ignore failures.. we're already handling one */
454 iommu_unmap(domain
, da
, order
);
461 /* release 'da' <-> 'pa' mapping */
462 static void unmap_iovm_area(struct iommu_domain
*domain
, struct omap_iommu
*obj
,
463 struct iovm_struct
*area
)
466 size_t total
= area
->da_end
- area
->da_start
;
467 const struct sg_table
*sgt
= area
->sgt
;
468 struct scatterlist
*sg
;
471 BUG_ON(!sgtable_ok(sgt
));
472 BUG_ON((!total
) || !IS_ALIGNED(total
, PAGE_SIZE
));
474 start
= area
->da_start
;
475 for_each_sg(sgt
->sgl
, sg
, sgt
->nents
, i
) {
479 bytes
= sg
->length
+ sg
->offset
;
480 order
= get_order(bytes
);
482 err
= iommu_unmap(domain
, start
, order
);
486 dev_dbg(obj
->dev
, "%s: unmap %08x(%x) %08x\n",
487 __func__
, start
, bytes
, area
->flags
);
489 BUG_ON(!IS_ALIGNED(bytes
, PAGE_SIZE
));
497 /* template function for all unmapping */
498 static struct sg_table
*unmap_vm_area(struct iommu_domain
*domain
,
499 struct omap_iommu
*obj
, const u32 da
,
500 void (*fn
)(const void *), u32 flags
)
502 struct sg_table
*sgt
= NULL
;
503 struct iovm_struct
*area
;
505 if (!IS_ALIGNED(da
, PAGE_SIZE
)) {
506 dev_err(obj
->dev
, "%s: alignment err(%08x)\n", __func__
, da
);
510 mutex_lock(&obj
->mmap_lock
);
512 area
= __find_iovm_area(obj
, da
);
514 dev_dbg(obj
->dev
, "%s: no da area(%08x)\n", __func__
, da
);
518 if ((area
->flags
& flags
) != flags
) {
519 dev_err(obj
->dev
, "%s: wrong flags(%08x)\n", __func__
,
523 sgt
= (struct sg_table
*)area
->sgt
;
525 unmap_iovm_area(domain
, obj
, area
);
529 dev_dbg(obj
->dev
, "%s: %08x-%08x-%08x(%x) %08x\n", __func__
,
530 area
->da_start
, da
, area
->da_end
,
531 area
->da_end
- area
->da_start
, area
->flags
);
533 free_iovm_area(obj
, area
);
535 mutex_unlock(&obj
->mmap_lock
);
540 static u32
map_iommu_region(struct iommu_domain
*domain
, struct omap_iommu
*obj
,
541 u32 da
, const struct sg_table
*sgt
, void *va
,
542 size_t bytes
, u32 flags
)
545 struct iovm_struct
*new;
547 mutex_lock(&obj
->mmap_lock
);
549 new = alloc_iovm_area(obj
, da
, bytes
, flags
);
552 goto err_alloc_iovma
;
557 if (map_iovm_area(domain
, new, sgt
, new->flags
))
560 mutex_unlock(&obj
->mmap_lock
);
562 dev_dbg(obj
->dev
, "%s: da:%08x(%x) flags:%08x va:%p\n",
563 __func__
, new->da_start
, bytes
, new->flags
, va
);
565 return new->da_start
;
568 free_iovm_area(obj
, new);
570 mutex_unlock(&obj
->mmap_lock
);
575 __iommu_vmap(struct iommu_domain
*domain
, struct omap_iommu
*obj
,
576 u32 da
, const struct sg_table
*sgt
,
577 void *va
, size_t bytes
, u32 flags
)
579 return map_iommu_region(domain
, obj
, da
, sgt
, va
, bytes
, flags
);
583 * omap_iommu_vmap - (d)-(p)-(v) address mapper
584 * @obj: objective iommu
585 * @sgt: address of scatter gather table
586 * @flags: iovma and page property
588 * Creates 1-n-1 mapping with given @sgt and returns @da.
589 * All @sgt element must be io page size aligned.
591 u32
omap_iommu_vmap(struct iommu_domain
*domain
, struct omap_iommu
*obj
, u32 da
,
592 const struct sg_table
*sgt
, u32 flags
)
597 if (!obj
|| !obj
->dev
|| !sgt
)
600 bytes
= sgtable_len(sgt
);
603 bytes
= PAGE_ALIGN(bytes
);
605 if (flags
& IOVMF_MMIO
) {
611 flags
|= IOVMF_DISCONT
;
614 da
= __iommu_vmap(domain
, obj
, da
, sgt
, va
, bytes
, flags
);
615 if (IS_ERR_VALUE(da
))
618 return da
+ sgtable_offset(sgt
);
620 EXPORT_SYMBOL_GPL(omap_iommu_vmap
);
623 * omap_iommu_vunmap - release virtual mapping obtained by 'omap_iommu_vmap()'
624 * @obj: objective iommu
625 * @da: iommu device virtual address
627 * Free the iommu virtually contiguous memory area starting at
628 * @da, which was returned by 'omap_iommu_vmap()'.
631 omap_iommu_vunmap(struct iommu_domain
*domain
, struct omap_iommu
*obj
, u32 da
)
633 struct sg_table
*sgt
;
635 * 'sgt' is allocated before 'omap_iommu_vmalloc()' is called.
636 * Just returns 'sgt' to the caller to free
639 sgt
= unmap_vm_area(domain
, obj
, da
, vunmap_sg
,
640 IOVMF_DISCONT
| IOVMF_MMIO
);
642 dev_dbg(obj
->dev
, "%s: No sgt\n", __func__
);
645 EXPORT_SYMBOL_GPL(omap_iommu_vunmap
);
648 * omap_iommu_vmalloc - (d)-(p)-(v) address allocator and mapper
649 * @obj: objective iommu
650 * @da: contiguous iommu virtual memory
651 * @bytes: allocation size
652 * @flags: iovma and page property
654 * Allocate @bytes linearly and creates 1-n-1 mapping and returns
655 * @da again, which might be adjusted if 'IOVMF_DA_FIXED' is not set.
658 omap_iommu_vmalloc(struct iommu_domain
*domain
, struct omap_iommu
*obj
, u32 da
,
659 size_t bytes
, u32 flags
)
662 struct sg_table
*sgt
;
664 if (!obj
|| !obj
->dev
|| !bytes
)
667 bytes
= PAGE_ALIGN(bytes
);
673 flags
|= IOVMF_DISCONT
;
674 flags
|= IOVMF_ALLOC
;
676 sgt
= sgtable_alloc(bytes
, flags
, da
, 0);
681 sgtable_fill_vmalloc(sgt
, va
);
683 da
= __iommu_vmap(domain
, obj
, da
, sgt
, va
, bytes
, flags
);
684 if (IS_ERR_VALUE(da
))
690 sgtable_drain_vmalloc(sgt
);
696 EXPORT_SYMBOL_GPL(omap_iommu_vmalloc
);
699 * omap_iommu_vfree - release memory allocated by 'omap_iommu_vmalloc()'
700 * @obj: objective iommu
701 * @da: iommu device virtual address
703 * Frees the iommu virtually continuous memory area starting at
704 * @da, as obtained from 'omap_iommu_vmalloc()'.
706 void omap_iommu_vfree(struct iommu_domain
*domain
, struct omap_iommu
*obj
,
709 struct sg_table
*sgt
;
711 sgt
= unmap_vm_area(domain
, obj
, da
, vfree
,
712 IOVMF_DISCONT
| IOVMF_ALLOC
);
714 dev_dbg(obj
->dev
, "%s: No sgt\n", __func__
);
717 EXPORT_SYMBOL_GPL(omap_iommu_vfree
);
719 static int __init
iovmm_init(void)
721 const unsigned long flags
= SLAB_HWCACHE_ALIGN
;
722 struct kmem_cache
*p
;
724 p
= kmem_cache_create("iovm_area_cache", sizeof(struct iovm_struct
), 0,
728 iovm_area_cachep
= p
;
732 module_init(iovmm_init
);
734 static void __exit
iovmm_exit(void)
736 kmem_cache_destroy(iovm_area_cachep
);
738 module_exit(iovmm_exit
);
740 MODULE_DESCRIPTION("omap iommu: simple virtual address space management");
741 MODULE_AUTHOR("Hiroshi DOYU <Hiroshi.DOYU@nokia.com>");
742 MODULE_LICENSE("GPL v2");