1 /* linux/drivers/iommu/exynos_iommu.c
3 * Copyright (c) 2011 Samsung Electronics Co., Ltd.
4 * http://www.samsung.com
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
11 #ifdef CONFIG_EXYNOS_IOMMU_DEBUG
16 #include <linux/interrupt.h>
17 #include <linux/platform_device.h>
18 #include <linux/slab.h>
19 #include <linux/pm_runtime.h>
20 #include <linux/clk.h>
21 #include <linux/err.h>
23 #include <linux/iommu.h>
24 #include <linux/errno.h>
25 #include <linux/list.h>
26 #include <linux/memblock.h>
27 #include <linux/export.h>
29 #include <asm/cacheflush.h>
30 #include <asm/pgtable.h>
32 #include <mach/sysmmu.h>
34 /* We does not consider super section mapping (16MB) */
36 #define LPAGE_ORDER 16
37 #define SPAGE_ORDER 12
39 #define SECT_SIZE (1 << SECT_ORDER)
40 #define LPAGE_SIZE (1 << LPAGE_ORDER)
41 #define SPAGE_SIZE (1 << SPAGE_ORDER)
43 #define SECT_MASK (~(SECT_SIZE - 1))
44 #define LPAGE_MASK (~(LPAGE_SIZE - 1))
45 #define SPAGE_MASK (~(SPAGE_SIZE - 1))
47 #define lv1ent_fault(sent) (((*(sent) & 3) == 0) || ((*(sent) & 3) == 3))
48 #define lv1ent_page(sent) ((*(sent) & 3) == 1)
49 #define lv1ent_section(sent) ((*(sent) & 3) == 2)
51 #define lv2ent_fault(pent) ((*(pent) & 3) == 0)
52 #define lv2ent_small(pent) ((*(pent) & 2) == 2)
53 #define lv2ent_large(pent) ((*(pent) & 3) == 1)
55 #define section_phys(sent) (*(sent) & SECT_MASK)
56 #define section_offs(iova) ((iova) & 0xFFFFF)
57 #define lpage_phys(pent) (*(pent) & LPAGE_MASK)
58 #define lpage_offs(iova) ((iova) & 0xFFFF)
59 #define spage_phys(pent) (*(pent) & SPAGE_MASK)
60 #define spage_offs(iova) ((iova) & 0xFFF)
62 #define lv1ent_offset(iova) ((iova) >> SECT_ORDER)
63 #define lv2ent_offset(iova) (((iova) & 0xFF000) >> SPAGE_ORDER)
65 #define NUM_LV1ENTRIES 4096
66 #define NUM_LV2ENTRIES 256
68 #define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(long))
70 #define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE)
72 #define lv2table_base(sent) (*(sent) & 0xFFFFFC00)
74 #define mk_lv1ent_sect(pa) ((pa) | 2)
75 #define mk_lv1ent_page(pa) ((pa) | 1)
76 #define mk_lv2ent_lpage(pa) ((pa) | 1)
77 #define mk_lv2ent_spage(pa) ((pa) | 2)
79 #define CTRL_ENABLE 0x5
80 #define CTRL_BLOCK 0x7
81 #define CTRL_DISABLE 0x0
83 #define REG_MMU_CTRL 0x000
84 #define REG_MMU_CFG 0x004
85 #define REG_MMU_STATUS 0x008
86 #define REG_MMU_FLUSH 0x00C
87 #define REG_MMU_FLUSH_ENTRY 0x010
88 #define REG_PT_BASE_ADDR 0x014
89 #define REG_INT_STATUS 0x018
90 #define REG_INT_CLEAR 0x01C
92 #define REG_PAGE_FAULT_ADDR 0x024
93 #define REG_AW_FAULT_ADDR 0x028
94 #define REG_AR_FAULT_ADDR 0x02C
95 #define REG_DEFAULT_SLAVE_ADDR 0x030
97 #define REG_MMU_VERSION 0x034
99 #define REG_PB0_SADDR 0x04C
100 #define REG_PB0_EADDR 0x050
101 #define REG_PB1_SADDR 0x054
102 #define REG_PB1_EADDR 0x058
104 static unsigned long *section_entry(unsigned long *pgtable
, unsigned long iova
)
106 return pgtable
+ lv1ent_offset(iova
);
109 static unsigned long *page_entry(unsigned long *sent
, unsigned long iova
)
111 return (unsigned long *)__va(lv2table_base(sent
)) + lv2ent_offset(iova
);
114 enum exynos_sysmmu_inttype
{
122 SYSMMU_AW_PROTECTION
, /* 7 */
123 SYSMMU_FAULT_UNKNOWN
,
128 * @itype: type of fault.
129 * @pgtable_base: the physical address of page table base. This is 0 if @itype
130 * is SYSMMU_BUSERROR.
131 * @fault_addr: the device (virtual) address that the System MMU tried to
132 * translated. This is 0 if @itype is SYSMMU_BUSERROR.
134 typedef int (*sysmmu_fault_handler_t
)(enum exynos_sysmmu_inttype itype
,
135 unsigned long pgtable_base
, unsigned long fault_addr
);
137 static unsigned short fault_reg_offset
[SYSMMU_FAULTS_NUM
] = {
141 REG_DEFAULT_SLAVE_ADDR
,
148 static char *sysmmu_fault_name
[SYSMMU_FAULTS_NUM
] = {
150 "AR MULTI-HIT FAULT",
151 "AW MULTI-HIT FAULT",
153 "AR SECURITY PROTECTION FAULT",
154 "AR ACCESS PROTECTION FAULT",
155 "AW SECURITY PROTECTION FAULT",
156 "AW ACCESS PROTECTION FAULT",
160 struct exynos_iommu_domain
{
161 struct list_head clients
; /* list of sysmmu_drvdata.node */
162 unsigned long *pgtable
; /* lv1 page table, 16KB */
163 short *lv2entcnt
; /* free lv2 entry counter for each section */
164 spinlock_t lock
; /* lock for this structure */
165 spinlock_t pgtablelock
; /* lock for modifying page table @ pgtable */
168 struct sysmmu_drvdata
{
169 struct list_head node
; /* entry of exynos_iommu_domain.clients */
170 struct device
*sysmmu
; /* System MMU's device descriptor */
171 struct device
*dev
; /* Owner of system MMU */
174 void __iomem
**sfrbases
;
178 struct iommu_domain
*domain
;
179 sysmmu_fault_handler_t fault_handler
;
180 unsigned long pgtable
;
183 static bool set_sysmmu_active(struct sysmmu_drvdata
*data
)
185 /* return true if the System MMU was not active previously
186 and it needs to be initialized */
187 return ++data
->activations
== 1;
190 static bool set_sysmmu_inactive(struct sysmmu_drvdata
*data
)
192 /* return true if the System MMU is needed to be disabled */
193 BUG_ON(data
->activations
< 1);
194 return --data
->activations
== 0;
197 static bool is_sysmmu_active(struct sysmmu_drvdata
*data
)
199 return data
->activations
> 0;
202 static void sysmmu_unblock(void __iomem
*sfrbase
)
204 __raw_writel(CTRL_ENABLE
, sfrbase
+ REG_MMU_CTRL
);
207 static bool sysmmu_block(void __iomem
*sfrbase
)
211 __raw_writel(CTRL_BLOCK
, sfrbase
+ REG_MMU_CTRL
);
212 while ((i
> 0) && !(__raw_readl(sfrbase
+ REG_MMU_STATUS
) & 1))
215 if (!(__raw_readl(sfrbase
+ REG_MMU_STATUS
) & 1)) {
216 sysmmu_unblock(sfrbase
);
223 static void __sysmmu_tlb_invalidate(void __iomem
*sfrbase
)
225 __raw_writel(0x1, sfrbase
+ REG_MMU_FLUSH
);
228 static void __sysmmu_tlb_invalidate_entry(void __iomem
*sfrbase
,
231 __raw_writel((iova
& SPAGE_MASK
) | 1, sfrbase
+ REG_MMU_FLUSH_ENTRY
);
234 static void __sysmmu_set_ptbase(void __iomem
*sfrbase
,
237 __raw_writel(0x1, sfrbase
+ REG_MMU_CFG
); /* 16KB LV1, LRU */
238 __raw_writel(pgd
, sfrbase
+ REG_PT_BASE_ADDR
);
240 __sysmmu_tlb_invalidate(sfrbase
);
243 static void __sysmmu_set_prefbuf(void __iomem
*sfrbase
, unsigned long base
,
244 unsigned long size
, int idx
)
246 __raw_writel(base
, sfrbase
+ REG_PB0_SADDR
+ idx
* 8);
247 __raw_writel(size
- 1 + base
, sfrbase
+ REG_PB0_EADDR
+ idx
* 8);
250 void exynos_sysmmu_set_prefbuf(struct device
*dev
,
251 unsigned long base0
, unsigned long size0
,
252 unsigned long base1
, unsigned long size1
)
254 struct sysmmu_drvdata
*data
= dev_get_drvdata(dev
->archdata
.iommu
);
258 BUG_ON((base0
+ size0
) <= base0
);
259 BUG_ON((size1
> 0) && ((base1
+ size1
) <= base1
));
261 read_lock_irqsave(&data
->lock
, flags
);
262 if (!is_sysmmu_active(data
))
265 for (i
= 0; i
< data
->nsfrs
; i
++) {
266 if ((readl(data
->sfrbases
[i
] + REG_MMU_VERSION
) >> 28) == 3) {
267 if (!sysmmu_block(data
->sfrbases
[i
]))
271 if (size0
<= SZ_128K
) {
276 ALIGN(size0
/ 2, SZ_64K
);
277 size0
= size0
- size1
;
278 base1
= base0
+ size0
;
282 __sysmmu_set_prefbuf(
283 data
->sfrbases
[i
], base0
, size0
, 0);
284 __sysmmu_set_prefbuf(
285 data
->sfrbases
[i
], base1
, size1
, 1);
287 sysmmu_unblock(data
->sfrbases
[i
]);
291 read_unlock_irqrestore(&data
->lock
, flags
);
294 static void __set_fault_handler(struct sysmmu_drvdata
*data
,
295 sysmmu_fault_handler_t handler
)
299 write_lock_irqsave(&data
->lock
, flags
);
300 data
->fault_handler
= handler
;
301 write_unlock_irqrestore(&data
->lock
, flags
);
304 void exynos_sysmmu_set_fault_handler(struct device
*dev
,
305 sysmmu_fault_handler_t handler
)
307 struct sysmmu_drvdata
*data
= dev_get_drvdata(dev
->archdata
.iommu
);
309 __set_fault_handler(data
, handler
);
312 static int default_fault_handler(enum exynos_sysmmu_inttype itype
,
313 unsigned long pgtable_base
, unsigned long fault_addr
)
317 if ((itype
>= SYSMMU_FAULTS_NUM
) || (itype
< SYSMMU_PAGEFAULT
))
318 itype
= SYSMMU_FAULT_UNKNOWN
;
320 pr_err("%s occurred at 0x%lx(Page table base: 0x%lx)\n",
321 sysmmu_fault_name
[itype
], fault_addr
, pgtable_base
);
323 ent
= section_entry(__va(pgtable_base
), fault_addr
);
324 pr_err("\tLv1 entry: 0x%lx\n", *ent
);
326 if (lv1ent_page(ent
)) {
327 ent
= page_entry(ent
, fault_addr
);
328 pr_err("\t Lv2 entry: 0x%lx\n", *ent
);
331 pr_err("Generating Kernel OOPS... because it is unrecoverable.\n");
338 static irqreturn_t
exynos_sysmmu_irq(int irq
, void *dev_id
)
340 /* SYSMMU is in blocked when interrupt occurred. */
341 struct sysmmu_drvdata
*data
= dev_id
;
342 struct resource
*irqres
;
343 struct platform_device
*pdev
;
344 enum exynos_sysmmu_inttype itype
;
345 unsigned long addr
= -1;
347 int i
, ret
= -ENOSYS
;
349 read_lock(&data
->lock
);
351 WARN_ON(!is_sysmmu_active(data
));
353 pdev
= to_platform_device(data
->sysmmu
);
354 for (i
= 0; i
< (pdev
->num_resources
/ 2); i
++) {
355 irqres
= platform_get_resource(pdev
, IORESOURCE_IRQ
, i
);
356 if (irqres
&& ((int)irqres
->start
== irq
))
360 if (i
== pdev
->num_resources
) {
361 itype
= SYSMMU_FAULT_UNKNOWN
;
363 itype
= (enum exynos_sysmmu_inttype
)
364 __ffs(__raw_readl(data
->sfrbases
[i
] + REG_INT_STATUS
));
365 if (WARN_ON(!((itype
>= 0) && (itype
< SYSMMU_FAULT_UNKNOWN
))))
366 itype
= SYSMMU_FAULT_UNKNOWN
;
369 data
->sfrbases
[i
] + fault_reg_offset
[itype
]);
373 ret
= report_iommu_fault(data
->domain
, data
->dev
,
376 if ((ret
== -ENOSYS
) && data
->fault_handler
) {
377 unsigned long base
= data
->pgtable
;
378 if (itype
!= SYSMMU_FAULT_UNKNOWN
)
380 data
->sfrbases
[i
] + REG_PT_BASE_ADDR
);
381 ret
= data
->fault_handler(itype
, base
, addr
);
384 if (!ret
&& (itype
!= SYSMMU_FAULT_UNKNOWN
))
385 __raw_writel(1 << itype
, data
->sfrbases
[i
] + REG_INT_CLEAR
);
387 dev_dbg(data
->sysmmu
, "(%s) %s is not handled.\n",
388 data
->dbgname
, sysmmu_fault_name
[itype
]);
390 if (itype
!= SYSMMU_FAULT_UNKNOWN
)
391 sysmmu_unblock(data
->sfrbases
[i
]);
393 read_unlock(&data
->lock
);
398 static bool __exynos_sysmmu_disable(struct sysmmu_drvdata
*data
)
401 bool disabled
= false;
404 write_lock_irqsave(&data
->lock
, flags
);
406 if (!set_sysmmu_inactive(data
))
409 for (i
= 0; i
< data
->nsfrs
; i
++)
410 __raw_writel(CTRL_DISABLE
, data
->sfrbases
[i
] + REG_MMU_CTRL
);
413 clk_disable(data
->clk
[1]);
415 clk_disable(data
->clk
[0]);
421 write_unlock_irqrestore(&data
->lock
, flags
);
424 dev_dbg(data
->sysmmu
, "(%s) Disabled\n", data
->dbgname
);
426 dev_dbg(data
->sysmmu
, "(%s) %d times left to be disabled\n",
427 data
->dbgname
, data
->activations
);
432 /* __exynos_sysmmu_enable: Enables System MMU
434 * returns -error if an error occurred and System MMU is not enabled,
435 * 0 if the System MMU has been just enabled and 1 if System MMU was already
438 static int __exynos_sysmmu_enable(struct sysmmu_drvdata
*data
,
439 unsigned long pgtable
, struct iommu_domain
*domain
)
444 write_lock_irqsave(&data
->lock
, flags
);
446 if (!set_sysmmu_active(data
)) {
447 if (WARN_ON(pgtable
!= data
->pgtable
)) {
449 set_sysmmu_inactive(data
);
454 dev_dbg(data
->sysmmu
, "(%s) Already enabled\n", data
->dbgname
);
459 clk_enable(data
->clk
[0]);
461 clk_enable(data
->clk
[1]);
463 data
->pgtable
= pgtable
;
465 for (i
= 0; i
< data
->nsfrs
; i
++) {
466 __sysmmu_set_ptbase(data
->sfrbases
[i
], pgtable
);
468 if ((readl(data
->sfrbases
[i
] + REG_MMU_VERSION
) >> 28) == 3) {
469 /* System MMU version is 3.x */
470 __raw_writel((1 << 12) | (2 << 28),
471 data
->sfrbases
[i
] + REG_MMU_CFG
);
472 __sysmmu_set_prefbuf(data
->sfrbases
[i
], 0, -1, 0);
473 __sysmmu_set_prefbuf(data
->sfrbases
[i
], 0, -1, 1);
476 __raw_writel(CTRL_ENABLE
, data
->sfrbases
[i
] + REG_MMU_CTRL
);
479 data
->domain
= domain
;
481 dev_dbg(data
->sysmmu
, "(%s) Enabled\n", data
->dbgname
);
483 write_unlock_irqrestore(&data
->lock
, flags
);
488 int exynos_sysmmu_enable(struct device
*dev
, unsigned long pgtable
)
490 struct sysmmu_drvdata
*data
= dev_get_drvdata(dev
->archdata
.iommu
);
493 BUG_ON(!memblock_is_memory(pgtable
));
495 ret
= pm_runtime_get_sync(data
->sysmmu
);
497 dev_dbg(data
->sysmmu
, "(%s) Failed to enable\n", data
->dbgname
);
501 ret
= __exynos_sysmmu_enable(data
, pgtable
, NULL
);
502 if (WARN_ON(ret
< 0)) {
503 pm_runtime_put(data
->sysmmu
);
504 dev_err(data
->sysmmu
,
505 "(%s) Already enabled with page table %#lx\n",
506 data
->dbgname
, data
->pgtable
);
514 bool exynos_sysmmu_disable(struct device
*dev
)
516 struct sysmmu_drvdata
*data
= dev_get_drvdata(dev
->archdata
.iommu
);
519 disabled
= __exynos_sysmmu_disable(data
);
520 pm_runtime_put(data
->sysmmu
);
525 static void sysmmu_tlb_invalidate_entry(struct device
*dev
, unsigned long iova
)
528 struct sysmmu_drvdata
*data
= dev_get_drvdata(dev
->archdata
.iommu
);
530 read_lock_irqsave(&data
->lock
, flags
);
532 if (is_sysmmu_active(data
)) {
534 for (i
= 0; i
< data
->nsfrs
; i
++) {
535 if (sysmmu_block(data
->sfrbases
[i
])) {
536 __sysmmu_tlb_invalidate_entry(
537 data
->sfrbases
[i
], iova
);
538 sysmmu_unblock(data
->sfrbases
[i
]);
542 dev_dbg(data
->sysmmu
,
543 "(%s) Disabled. Skipping invalidating TLB.\n",
547 read_unlock_irqrestore(&data
->lock
, flags
);
550 void exynos_sysmmu_tlb_invalidate(struct device
*dev
)
553 struct sysmmu_drvdata
*data
= dev_get_drvdata(dev
->archdata
.iommu
);
555 read_lock_irqsave(&data
->lock
, flags
);
557 if (is_sysmmu_active(data
)) {
559 for (i
= 0; i
< data
->nsfrs
; i
++) {
560 if (sysmmu_block(data
->sfrbases
[i
])) {
561 __sysmmu_tlb_invalidate(data
->sfrbases
[i
]);
562 sysmmu_unblock(data
->sfrbases
[i
]);
566 dev_dbg(data
->sysmmu
,
567 "(%s) Disabled. Skipping invalidating TLB.\n",
571 read_unlock_irqrestore(&data
->lock
, flags
);
574 static int exynos_sysmmu_probe(struct platform_device
*pdev
)
578 struct sysmmu_drvdata
*data
;
582 data
= kzalloc(sizeof(*data
), GFP_KERNEL
);
584 dev_dbg(dev
, "Not enough memory\n");
589 ret
= dev_set_drvdata(dev
, data
);
591 dev_dbg(dev
, "Unabled to initialize driver data\n");
595 data
->nsfrs
= pdev
->num_resources
/ 2;
596 data
->sfrbases
= kmalloc(sizeof(*data
->sfrbases
) * data
->nsfrs
,
598 if (data
->sfrbases
== NULL
) {
599 dev_dbg(dev
, "Not enough memory\n");
604 for (i
= 0; i
< data
->nsfrs
; i
++) {
605 struct resource
*res
;
606 res
= platform_get_resource(pdev
, IORESOURCE_MEM
, i
);
608 dev_dbg(dev
, "Unable to find IOMEM region\n");
613 data
->sfrbases
[i
] = ioremap(res
->start
, resource_size(res
));
614 if (!data
->sfrbases
[i
]) {
615 dev_dbg(dev
, "Unable to map IOMEM @ PA:%#x\n",
622 for (i
= 0; i
< data
->nsfrs
; i
++) {
623 ret
= platform_get_irq(pdev
, i
);
625 dev_dbg(dev
, "Unable to find IRQ resource\n");
629 ret
= request_irq(ret
, exynos_sysmmu_irq
, 0,
630 dev_name(dev
), data
);
632 dev_dbg(dev
, "Unabled to register interrupt handler\n");
637 if (dev_get_platdata(dev
)) {
639 struct sysmmu_platform_data
*platdata
= dev_get_platdata(dev
);
641 beg
= platdata
->clockname
;
643 for (deli
= beg
; (*deli
!= '\0') && (*deli
!= ','); deli
++)
651 data
->clk
[0] = clk_get(dev
, beg
);
652 if (IS_ERR(data
->clk
[0])) {
654 dev_dbg(dev
, "No clock descriptor registered\n");
657 if (data
->clk
[0] && deli
) {
659 data
->clk
[1] = clk_get(dev
, deli
+ 1);
660 if (IS_ERR(data
->clk
[1]))
664 data
->dbgname
= platdata
->dbgname
;
668 rwlock_init(&data
->lock
);
669 INIT_LIST_HEAD(&data
->node
);
671 __set_fault_handler(data
, &default_fault_handler
);
674 pm_runtime_enable(dev
);
676 dev_dbg(dev
, "(%s) Initialized\n", data
->dbgname
);
682 irq
= platform_get_irq(pdev
, i
);
686 while (data
->nsfrs
-- > 0)
687 iounmap(data
->sfrbases
[data
->nsfrs
]);
688 kfree(data
->sfrbases
);
692 dev_err(dev
, "Failed to initialize\n");
696 static struct platform_driver exynos_sysmmu_driver
= {
697 .probe
= exynos_sysmmu_probe
,
699 .owner
= THIS_MODULE
,
700 .name
= "exynos-sysmmu",
704 static inline void pgtable_flush(void *vastart
, void *vaend
)
706 dmac_flush_range(vastart
, vaend
);
707 outer_flush_range(virt_to_phys(vastart
),
708 virt_to_phys(vaend
));
711 static int exynos_iommu_domain_init(struct iommu_domain
*domain
)
713 struct exynos_iommu_domain
*priv
;
715 priv
= kzalloc(sizeof(*priv
), GFP_KERNEL
);
719 priv
->pgtable
= (unsigned long *)__get_free_pages(
720 GFP_KERNEL
| __GFP_ZERO
, 2);
724 priv
->lv2entcnt
= (short *)__get_free_pages(
725 GFP_KERNEL
| __GFP_ZERO
, 1);
726 if (!priv
->lv2entcnt
)
729 pgtable_flush(priv
->pgtable
, priv
->pgtable
+ NUM_LV1ENTRIES
);
731 spin_lock_init(&priv
->lock
);
732 spin_lock_init(&priv
->pgtablelock
);
733 INIT_LIST_HEAD(&priv
->clients
);
735 dom
->geometry
.aperture_start
= 0;
736 dom
->geometry
.aperture_end
= ~0UL;
737 dom
->geometry
.force_aperture
= true;
743 free_pages((unsigned long)priv
->pgtable
, 2);
749 static void exynos_iommu_domain_destroy(struct iommu_domain
*domain
)
751 struct exynos_iommu_domain
*priv
= domain
->priv
;
752 struct sysmmu_drvdata
*data
;
756 WARN_ON(!list_empty(&priv
->clients
));
758 spin_lock_irqsave(&priv
->lock
, flags
);
760 list_for_each_entry(data
, &priv
->clients
, node
) {
761 while (!exynos_sysmmu_disable(data
->dev
))
762 ; /* until System MMU is actually disabled */
765 spin_unlock_irqrestore(&priv
->lock
, flags
);
767 for (i
= 0; i
< NUM_LV1ENTRIES
; i
++)
768 if (lv1ent_page(priv
->pgtable
+ i
))
769 kfree(__va(lv2table_base(priv
->pgtable
+ i
)));
771 free_pages((unsigned long)priv
->pgtable
, 2);
772 free_pages((unsigned long)priv
->lv2entcnt
, 1);
777 static int exynos_iommu_attach_device(struct iommu_domain
*domain
,
780 struct sysmmu_drvdata
*data
= dev_get_drvdata(dev
->archdata
.iommu
);
781 struct exynos_iommu_domain
*priv
= domain
->priv
;
785 ret
= pm_runtime_get_sync(data
->sysmmu
);
791 spin_lock_irqsave(&priv
->lock
, flags
);
793 ret
= __exynos_sysmmu_enable(data
, __pa(priv
->pgtable
), domain
);
796 /* 'data->node' must not be appeared in priv->clients */
797 BUG_ON(!list_empty(&data
->node
));
799 list_add_tail(&data
->node
, &priv
->clients
);
802 spin_unlock_irqrestore(&priv
->lock
, flags
);
805 dev_err(dev
, "%s: Failed to attach IOMMU with pgtable %#lx\n",
806 __func__
, __pa(priv
->pgtable
));
807 pm_runtime_put(data
->sysmmu
);
808 } else if (ret
> 0) {
809 dev_dbg(dev
, "%s: IOMMU with pgtable 0x%lx already attached\n",
810 __func__
, __pa(priv
->pgtable
));
812 dev_dbg(dev
, "%s: Attached new IOMMU with pgtable 0x%lx\n",
813 __func__
, __pa(priv
->pgtable
));
819 static void exynos_iommu_detach_device(struct iommu_domain
*domain
,
822 struct sysmmu_drvdata
*data
= dev_get_drvdata(dev
->archdata
.iommu
);
823 struct exynos_iommu_domain
*priv
= domain
->priv
;
824 struct list_head
*pos
;
828 spin_lock_irqsave(&priv
->lock
, flags
);
830 list_for_each(pos
, &priv
->clients
) {
831 if (list_entry(pos
, struct sysmmu_drvdata
, node
) == data
) {
840 if (__exynos_sysmmu_disable(data
)) {
841 dev_dbg(dev
, "%s: Detached IOMMU with pgtable %#lx\n",
842 __func__
, __pa(priv
->pgtable
));
843 list_del(&data
->node
);
844 INIT_LIST_HEAD(&data
->node
);
847 dev_dbg(dev
, "%s: Detaching IOMMU with pgtable %#lx delayed",
848 __func__
, __pa(priv
->pgtable
));
852 spin_unlock_irqrestore(&priv
->lock
, flags
);
855 pm_runtime_put(data
->sysmmu
);
858 static unsigned long *alloc_lv2entry(unsigned long *sent
, unsigned long iova
,
861 if (lv1ent_fault(sent
)) {
864 pent
= kzalloc(LV2TABLE_SIZE
, GFP_ATOMIC
);
865 BUG_ON((unsigned long)pent
& (LV2TABLE_SIZE
- 1));
869 *sent
= mk_lv1ent_page(__pa(pent
));
870 *pgcounter
= NUM_LV2ENTRIES
;
871 pgtable_flush(pent
, pent
+ NUM_LV2ENTRIES
);
872 pgtable_flush(sent
, sent
+ 1);
875 return page_entry(sent
, iova
);
878 static int lv1set_section(unsigned long *sent
, phys_addr_t paddr
, short *pgcnt
)
880 if (lv1ent_section(sent
))
883 if (lv1ent_page(sent
)) {
884 if (*pgcnt
!= NUM_LV2ENTRIES
)
887 kfree(page_entry(sent
, 0));
892 *sent
= mk_lv1ent_sect(paddr
);
894 pgtable_flush(sent
, sent
+ 1);
899 static int lv2set_page(unsigned long *pent
, phys_addr_t paddr
, size_t size
,
902 if (size
== SPAGE_SIZE
) {
903 if (!lv2ent_fault(pent
))
906 *pent
= mk_lv2ent_spage(paddr
);
907 pgtable_flush(pent
, pent
+ 1);
909 } else { /* size == LPAGE_SIZE */
911 for (i
= 0; i
< SPAGES_PER_LPAGE
; i
++, pent
++) {
912 if (!lv2ent_fault(pent
)) {
913 memset(pent
, 0, sizeof(*pent
) * i
);
917 *pent
= mk_lv2ent_lpage(paddr
);
919 pgtable_flush(pent
- SPAGES_PER_LPAGE
, pent
);
920 *pgcnt
-= SPAGES_PER_LPAGE
;
926 static int exynos_iommu_map(struct iommu_domain
*domain
, unsigned long iova
,
927 phys_addr_t paddr
, size_t size
, int prot
)
929 struct exynos_iommu_domain
*priv
= domain
->priv
;
930 unsigned long *entry
;
934 BUG_ON(priv
->pgtable
== NULL
);
936 spin_lock_irqsave(&priv
->pgtablelock
, flags
);
938 entry
= section_entry(priv
->pgtable
, iova
);
940 if (size
== SECT_SIZE
) {
941 ret
= lv1set_section(entry
, paddr
,
942 &priv
->lv2entcnt
[lv1ent_offset(iova
)]);
946 pent
= alloc_lv2entry(entry
, iova
,
947 &priv
->lv2entcnt
[lv1ent_offset(iova
)]);
952 ret
= lv2set_page(pent
, paddr
, size
,
953 &priv
->lv2entcnt
[lv1ent_offset(iova
)]);
957 pr_debug("%s: Failed to map iova 0x%lx/0x%x bytes\n",
958 __func__
, iova
, size
);
961 spin_unlock_irqrestore(&priv
->pgtablelock
, flags
);
966 static size_t exynos_iommu_unmap(struct iommu_domain
*domain
,
967 unsigned long iova
, size_t size
)
969 struct exynos_iommu_domain
*priv
= domain
->priv
;
970 struct sysmmu_drvdata
*data
;
974 BUG_ON(priv
->pgtable
== NULL
);
976 spin_lock_irqsave(&priv
->pgtablelock
, flags
);
978 ent
= section_entry(priv
->pgtable
, iova
);
980 if (lv1ent_section(ent
)) {
981 BUG_ON(size
< SECT_SIZE
);
984 pgtable_flush(ent
, ent
+ 1);
989 if (unlikely(lv1ent_fault(ent
))) {
990 if (size
> SECT_SIZE
)
995 /* lv1ent_page(sent) == true here */
997 ent
= page_entry(ent
, iova
);
999 if (unlikely(lv2ent_fault(ent
))) {
1004 if (lv2ent_small(ent
)) {
1007 priv
->lv2entcnt
[lv1ent_offset(iova
)] += 1;
1011 /* lv1ent_large(ent) == true here */
1012 BUG_ON(size
< LPAGE_SIZE
);
1014 memset(ent
, 0, sizeof(*ent
) * SPAGES_PER_LPAGE
);
1017 priv
->lv2entcnt
[lv1ent_offset(iova
)] += SPAGES_PER_LPAGE
;
1019 spin_unlock_irqrestore(&priv
->pgtablelock
, flags
);
1021 spin_lock_irqsave(&priv
->lock
, flags
);
1022 list_for_each_entry(data
, &priv
->clients
, node
)
1023 sysmmu_tlb_invalidate_entry(data
->dev
, iova
);
1024 spin_unlock_irqrestore(&priv
->lock
, flags
);
1030 static phys_addr_t
exynos_iommu_iova_to_phys(struct iommu_domain
*domain
,
1033 struct exynos_iommu_domain
*priv
= domain
->priv
;
1034 unsigned long *entry
;
1035 unsigned long flags
;
1036 phys_addr_t phys
= 0;
1038 spin_lock_irqsave(&priv
->pgtablelock
, flags
);
1040 entry
= section_entry(priv
->pgtable
, iova
);
1042 if (lv1ent_section(entry
)) {
1043 phys
= section_phys(entry
) + section_offs(iova
);
1044 } else if (lv1ent_page(entry
)) {
1045 entry
= page_entry(entry
, iova
);
1047 if (lv2ent_large(entry
))
1048 phys
= lpage_phys(entry
) + lpage_offs(iova
);
1049 else if (lv2ent_small(entry
))
1050 phys
= spage_phys(entry
) + spage_offs(iova
);
1053 spin_unlock_irqrestore(&priv
->pgtablelock
, flags
);
1058 static struct iommu_ops exynos_iommu_ops
= {
1059 .domain_init
= &exynos_iommu_domain_init
,
1060 .domain_destroy
= &exynos_iommu_domain_destroy
,
1061 .attach_dev
= &exynos_iommu_attach_device
,
1062 .detach_dev
= &exynos_iommu_detach_device
,
1063 .map
= &exynos_iommu_map
,
1064 .unmap
= &exynos_iommu_unmap
,
1065 .iova_to_phys
= &exynos_iommu_iova_to_phys
,
1066 .pgsize_bitmap
= SECT_SIZE
| LPAGE_SIZE
| SPAGE_SIZE
,
1069 static int __init
exynos_iommu_init(void)
1073 ret
= platform_driver_register(&exynos_sysmmu_driver
);
1076 bus_set_iommu(&platform_bus_type
, &exynos_iommu_ops
);
1080 subsys_initcall(exynos_iommu_init
);