2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/iommu.h>
38 #include <linux/intel-iommu.h>
39 #include <linux/sysdev.h>
40 #include <asm/cacheflush.h>
41 #include <asm/iommu.h>
44 #define ROOT_SIZE VTD_PAGE_SIZE
45 #define CONTEXT_SIZE VTD_PAGE_SIZE
47 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
48 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
50 #define IOAPIC_RANGE_START (0xfee00000)
51 #define IOAPIC_RANGE_END (0xfeefffff)
52 #define IOVA_START_ADDR (0x1000)
54 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
56 #define MAX_AGAW_WIDTH 64
58 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
60 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
61 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
62 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
64 /* global iommu list, set NULL for ignored DMAR units */
65 static struct intel_iommu
**g_iommus
;
67 static int rwbf_quirk
;
72 * 12-63: Context Ptr (12 - (haw-1))
79 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
80 static inline bool root_present(struct root_entry
*root
)
82 return (root
->val
& 1);
84 static inline void set_root_present(struct root_entry
*root
)
88 static inline void set_root_value(struct root_entry
*root
, unsigned long value
)
90 root
->val
|= value
& VTD_PAGE_MASK
;
93 static inline struct context_entry
*
94 get_context_addr_from_root(struct root_entry
*root
)
96 return (struct context_entry
*)
97 (root_present(root
)?phys_to_virt(
98 root
->val
& VTD_PAGE_MASK
) :
105 * 1: fault processing disable
106 * 2-3: translation type
107 * 12-63: address space root
113 struct context_entry
{
118 static inline bool context_present(struct context_entry
*context
)
120 return (context
->lo
& 1);
122 static inline void context_set_present(struct context_entry
*context
)
127 static inline void context_set_fault_enable(struct context_entry
*context
)
129 context
->lo
&= (((u64
)-1) << 2) | 1;
132 static inline void context_set_translation_type(struct context_entry
*context
,
135 context
->lo
&= (((u64
)-1) << 4) | 3;
136 context
->lo
|= (value
& 3) << 2;
139 static inline void context_set_address_root(struct context_entry
*context
,
142 context
->lo
|= value
& VTD_PAGE_MASK
;
145 static inline void context_set_address_width(struct context_entry
*context
,
148 context
->hi
|= value
& 7;
151 static inline void context_set_domain_id(struct context_entry
*context
,
154 context
->hi
|= (value
& ((1 << 16) - 1)) << 8;
157 static inline void context_clear_entry(struct context_entry
*context
)
170 * 12-63: Host physcial address
176 static inline void dma_clear_pte(struct dma_pte
*pte
)
181 static inline void dma_set_pte_readable(struct dma_pte
*pte
)
183 pte
->val
|= DMA_PTE_READ
;
186 static inline void dma_set_pte_writable(struct dma_pte
*pte
)
188 pte
->val
|= DMA_PTE_WRITE
;
191 static inline void dma_set_pte_snp(struct dma_pte
*pte
)
193 pte
->val
|= DMA_PTE_SNP
;
196 static inline void dma_set_pte_prot(struct dma_pte
*pte
, unsigned long prot
)
198 pte
->val
= (pte
->val
& ~3) | (prot
& 3);
201 static inline u64
dma_pte_addr(struct dma_pte
*pte
)
203 return (pte
->val
& VTD_PAGE_MASK
);
206 static inline void dma_set_pte_addr(struct dma_pte
*pte
, u64 addr
)
208 pte
->val
|= (addr
& VTD_PAGE_MASK
);
211 static inline bool dma_pte_present(struct dma_pte
*pte
)
213 return (pte
->val
& 3) != 0;
216 /* devices under the same p2p bridge are owned in one domain */
217 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
219 /* domain represents a virtual machine, more than one devices
220 * across iommus may be owned in one domain, e.g. kvm guest.
222 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
225 int id
; /* domain id */
226 unsigned long iommu_bmp
; /* bitmap of iommus this domain uses*/
228 struct list_head devices
; /* all devices' list */
229 struct iova_domain iovad
; /* iova's that belong to this domain */
231 struct dma_pte
*pgd
; /* virtual address */
232 spinlock_t mapping_lock
; /* page table lock */
233 int gaw
; /* max guest address width */
235 /* adjusted guest address width, 0 is level 2 30-bit */
238 int flags
; /* flags to find out type of domain */
240 int iommu_coherency
;/* indicate coherency of iommu access */
241 int iommu_snooping
; /* indicate snooping control feature*/
242 int iommu_count
; /* reference count of iommu */
243 spinlock_t iommu_lock
; /* protect iommu set in domain */
244 u64 max_addr
; /* maximum mapped address */
247 /* PCI domain-device relationship */
248 struct device_domain_info
{
249 struct list_head link
; /* link to domain siblings */
250 struct list_head global
; /* link to global list */
251 int segment
; /* PCI domain */
252 u8 bus
; /* PCI bus number */
253 u8 devfn
; /* PCI devfn number */
254 struct pci_dev
*dev
; /* it's NULL for PCIE-to-PCI bridge */
255 struct dmar_domain
*domain
; /* pointer to domain */
258 static void flush_unmaps_timeout(unsigned long data
);
260 DEFINE_TIMER(unmap_timer
, flush_unmaps_timeout
, 0, 0);
262 #define HIGH_WATER_MARK 250
263 struct deferred_flush_tables
{
265 struct iova
*iova
[HIGH_WATER_MARK
];
266 struct dmar_domain
*domain
[HIGH_WATER_MARK
];
269 static struct deferred_flush_tables
*deferred_flush
;
271 /* bitmap for indexing intel_iommus */
272 static int g_num_of_iommus
;
274 static DEFINE_SPINLOCK(async_umap_flush_lock
);
275 static LIST_HEAD(unmaps_to_do
);
278 static long list_size
;
280 static void domain_remove_dev_info(struct dmar_domain
*domain
);
282 #ifdef CONFIG_DMAR_DEFAULT_ON
283 int dmar_disabled
= 0;
285 int dmar_disabled
= 1;
286 #endif /*CONFIG_DMAR_DEFAULT_ON*/
288 static int __initdata dmar_map_gfx
= 1;
289 static int dmar_forcedac
;
290 static int intel_iommu_strict
;
292 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
293 static DEFINE_SPINLOCK(device_domain_lock
);
294 static LIST_HEAD(device_domain_list
);
296 static struct iommu_ops intel_iommu_ops
;
298 static int __init
intel_iommu_setup(char *str
)
303 if (!strncmp(str
, "on", 2)) {
305 printk(KERN_INFO
"Intel-IOMMU: enabled\n");
306 } else if (!strncmp(str
, "off", 3)) {
308 printk(KERN_INFO
"Intel-IOMMU: disabled\n");
309 } else if (!strncmp(str
, "igfx_off", 8)) {
312 "Intel-IOMMU: disable GFX device mapping\n");
313 } else if (!strncmp(str
, "forcedac", 8)) {
315 "Intel-IOMMU: Forcing DAC for PCI devices\n");
317 } else if (!strncmp(str
, "strict", 6)) {
319 "Intel-IOMMU: disable batched IOTLB flush\n");
320 intel_iommu_strict
= 1;
323 str
+= strcspn(str
, ",");
329 __setup("intel_iommu=", intel_iommu_setup
);
331 static struct kmem_cache
*iommu_domain_cache
;
332 static struct kmem_cache
*iommu_devinfo_cache
;
333 static struct kmem_cache
*iommu_iova_cache
;
335 static inline void *iommu_kmem_cache_alloc(struct kmem_cache
*cachep
)
340 /* trying to avoid low memory issues */
341 flags
= current
->flags
& PF_MEMALLOC
;
342 current
->flags
|= PF_MEMALLOC
;
343 vaddr
= kmem_cache_alloc(cachep
, GFP_ATOMIC
);
344 current
->flags
&= (~PF_MEMALLOC
| flags
);
349 static inline void *alloc_pgtable_page(void)
354 /* trying to avoid low memory issues */
355 flags
= current
->flags
& PF_MEMALLOC
;
356 current
->flags
|= PF_MEMALLOC
;
357 vaddr
= (void *)get_zeroed_page(GFP_ATOMIC
);
358 current
->flags
&= (~PF_MEMALLOC
| flags
);
362 static inline void free_pgtable_page(void *vaddr
)
364 free_page((unsigned long)vaddr
);
367 static inline void *alloc_domain_mem(void)
369 return iommu_kmem_cache_alloc(iommu_domain_cache
);
372 static void free_domain_mem(void *vaddr
)
374 kmem_cache_free(iommu_domain_cache
, vaddr
);
377 static inline void * alloc_devinfo_mem(void)
379 return iommu_kmem_cache_alloc(iommu_devinfo_cache
);
382 static inline void free_devinfo_mem(void *vaddr
)
384 kmem_cache_free(iommu_devinfo_cache
, vaddr
);
387 struct iova
*alloc_iova_mem(void)
389 return iommu_kmem_cache_alloc(iommu_iova_cache
);
392 void free_iova_mem(struct iova
*iova
)
394 kmem_cache_free(iommu_iova_cache
, iova
);
398 static inline int width_to_agaw(int width
);
400 static int __iommu_calculate_agaw(struct intel_iommu
*iommu
, int max_gaw
)
405 sagaw
= cap_sagaw(iommu
->cap
);
406 for (agaw
= width_to_agaw(max_gaw
);
408 if (test_bit(agaw
, &sagaw
))
416 * Calculate max SAGAW for each iommu.
418 int iommu_calculate_max_sagaw(struct intel_iommu
*iommu
)
420 return __iommu_calculate_agaw(iommu
, MAX_AGAW_WIDTH
);
424 * calculate agaw for each iommu.
425 * "SAGAW" may be different across iommus, use a default agaw, and
426 * get a supported less agaw for iommus that don't support the default agaw.
428 int iommu_calculate_agaw(struct intel_iommu
*iommu
)
430 return __iommu_calculate_agaw(iommu
, DEFAULT_DOMAIN_ADDRESS_WIDTH
);
433 /* in native case, each domain is related to only one iommu */
434 static struct intel_iommu
*domain_get_iommu(struct dmar_domain
*domain
)
438 BUG_ON(domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
);
440 iommu_id
= find_first_bit(&domain
->iommu_bmp
, g_num_of_iommus
);
441 if (iommu_id
< 0 || iommu_id
>= g_num_of_iommus
)
444 return g_iommus
[iommu_id
];
447 static void domain_update_iommu_coherency(struct dmar_domain
*domain
)
451 domain
->iommu_coherency
= 1;
453 i
= find_first_bit(&domain
->iommu_bmp
, g_num_of_iommus
);
454 for (; i
< g_num_of_iommus
; ) {
455 if (!ecap_coherent(g_iommus
[i
]->ecap
)) {
456 domain
->iommu_coherency
= 0;
459 i
= find_next_bit(&domain
->iommu_bmp
, g_num_of_iommus
, i
+1);
463 static void domain_update_iommu_snooping(struct dmar_domain
*domain
)
467 domain
->iommu_snooping
= 1;
469 i
= find_first_bit(&domain
->iommu_bmp
, g_num_of_iommus
);
470 for (; i
< g_num_of_iommus
; ) {
471 if (!ecap_sc_support(g_iommus
[i
]->ecap
)) {
472 domain
->iommu_snooping
= 0;
475 i
= find_next_bit(&domain
->iommu_bmp
, g_num_of_iommus
, i
+1);
479 /* Some capabilities may be different across iommus */
480 static void domain_update_iommu_cap(struct dmar_domain
*domain
)
482 domain_update_iommu_coherency(domain
);
483 domain_update_iommu_snooping(domain
);
486 static struct intel_iommu
*device_to_iommu(int segment
, u8 bus
, u8 devfn
)
488 struct dmar_drhd_unit
*drhd
= NULL
;
491 for_each_drhd_unit(drhd
) {
494 if (segment
!= drhd
->segment
)
497 for (i
= 0; i
< drhd
->devices_cnt
; i
++) {
498 if (drhd
->devices
[i
] &&
499 drhd
->devices
[i
]->bus
->number
== bus
&&
500 drhd
->devices
[i
]->devfn
== devfn
)
502 if (drhd
->devices
[i
] &&
503 drhd
->devices
[i
]->subordinate
&&
504 drhd
->devices
[i
]->subordinate
->number
<= bus
&&
505 drhd
->devices
[i
]->subordinate
->subordinate
>= bus
)
509 if (drhd
->include_all
)
516 static void domain_flush_cache(struct dmar_domain
*domain
,
517 void *addr
, int size
)
519 if (!domain
->iommu_coherency
)
520 clflush_cache_range(addr
, size
);
523 /* Gets context entry for a given bus and devfn */
524 static struct context_entry
* device_to_context_entry(struct intel_iommu
*iommu
,
527 struct root_entry
*root
;
528 struct context_entry
*context
;
529 unsigned long phy_addr
;
532 spin_lock_irqsave(&iommu
->lock
, flags
);
533 root
= &iommu
->root_entry
[bus
];
534 context
= get_context_addr_from_root(root
);
536 context
= (struct context_entry
*)alloc_pgtable_page();
538 spin_unlock_irqrestore(&iommu
->lock
, flags
);
541 __iommu_flush_cache(iommu
, (void *)context
, CONTEXT_SIZE
);
542 phy_addr
= virt_to_phys((void *)context
);
543 set_root_value(root
, phy_addr
);
544 set_root_present(root
);
545 __iommu_flush_cache(iommu
, root
, sizeof(*root
));
547 spin_unlock_irqrestore(&iommu
->lock
, flags
);
548 return &context
[devfn
];
551 static int device_context_mapped(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
553 struct root_entry
*root
;
554 struct context_entry
*context
;
558 spin_lock_irqsave(&iommu
->lock
, flags
);
559 root
= &iommu
->root_entry
[bus
];
560 context
= get_context_addr_from_root(root
);
565 ret
= context_present(&context
[devfn
]);
567 spin_unlock_irqrestore(&iommu
->lock
, flags
);
571 static void clear_context_table(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
573 struct root_entry
*root
;
574 struct context_entry
*context
;
577 spin_lock_irqsave(&iommu
->lock
, flags
);
578 root
= &iommu
->root_entry
[bus
];
579 context
= get_context_addr_from_root(root
);
581 context_clear_entry(&context
[devfn
]);
582 __iommu_flush_cache(iommu
, &context
[devfn
], \
585 spin_unlock_irqrestore(&iommu
->lock
, flags
);
588 static void free_context_table(struct intel_iommu
*iommu
)
590 struct root_entry
*root
;
593 struct context_entry
*context
;
595 spin_lock_irqsave(&iommu
->lock
, flags
);
596 if (!iommu
->root_entry
) {
599 for (i
= 0; i
< ROOT_ENTRY_NR
; i
++) {
600 root
= &iommu
->root_entry
[i
];
601 context
= get_context_addr_from_root(root
);
603 free_pgtable_page(context
);
605 free_pgtable_page(iommu
->root_entry
);
606 iommu
->root_entry
= NULL
;
608 spin_unlock_irqrestore(&iommu
->lock
, flags
);
611 /* page table handling */
612 #define LEVEL_STRIDE (9)
613 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
615 static inline int agaw_to_level(int agaw
)
620 static inline int agaw_to_width(int agaw
)
622 return 30 + agaw
* LEVEL_STRIDE
;
626 static inline int width_to_agaw(int width
)
628 return (width
- 30) / LEVEL_STRIDE
;
631 static inline unsigned int level_to_offset_bits(int level
)
633 return (12 + (level
- 1) * LEVEL_STRIDE
);
636 static inline int address_level_offset(u64 addr
, int level
)
638 return ((addr
>> level_to_offset_bits(level
)) & LEVEL_MASK
);
641 static inline u64
level_mask(int level
)
643 return ((u64
)-1 << level_to_offset_bits(level
));
646 static inline u64
level_size(int level
)
648 return ((u64
)1 << level_to_offset_bits(level
));
651 static inline u64
align_to_level(u64 addr
, int level
)
653 return ((addr
+ level_size(level
) - 1) & level_mask(level
));
656 static struct dma_pte
* addr_to_dma_pte(struct dmar_domain
*domain
, u64 addr
)
658 int addr_width
= agaw_to_width(domain
->agaw
);
659 struct dma_pte
*parent
, *pte
= NULL
;
660 int level
= agaw_to_level(domain
->agaw
);
664 BUG_ON(!domain
->pgd
);
666 addr
&= (((u64
)1) << addr_width
) - 1;
667 parent
= domain
->pgd
;
669 spin_lock_irqsave(&domain
->mapping_lock
, flags
);
673 offset
= address_level_offset(addr
, level
);
674 pte
= &parent
[offset
];
678 if (!dma_pte_present(pte
)) {
679 tmp_page
= alloc_pgtable_page();
682 spin_unlock_irqrestore(&domain
->mapping_lock
,
686 domain_flush_cache(domain
, tmp_page
, PAGE_SIZE
);
687 dma_set_pte_addr(pte
, virt_to_phys(tmp_page
));
689 * high level table always sets r/w, last level page
690 * table control read/write
692 dma_set_pte_readable(pte
);
693 dma_set_pte_writable(pte
);
694 domain_flush_cache(domain
, pte
, sizeof(*pte
));
696 parent
= phys_to_virt(dma_pte_addr(pte
));
700 spin_unlock_irqrestore(&domain
->mapping_lock
, flags
);
704 /* return address's pte at specific level */
705 static struct dma_pte
*dma_addr_level_pte(struct dmar_domain
*domain
, u64 addr
,
708 struct dma_pte
*parent
, *pte
= NULL
;
709 int total
= agaw_to_level(domain
->agaw
);
712 parent
= domain
->pgd
;
713 while (level
<= total
) {
714 offset
= address_level_offset(addr
, total
);
715 pte
= &parent
[offset
];
719 if (!dma_pte_present(pte
))
721 parent
= phys_to_virt(dma_pte_addr(pte
));
727 /* clear one page's page table */
728 static void dma_pte_clear_one(struct dmar_domain
*domain
, u64 addr
)
730 struct dma_pte
*pte
= NULL
;
732 /* get last level pte */
733 pte
= dma_addr_level_pte(domain
, addr
, 1);
737 domain_flush_cache(domain
, pte
, sizeof(*pte
));
741 /* clear last level pte, a tlb flush should be followed */
742 static void dma_pte_clear_range(struct dmar_domain
*domain
, u64 start
, u64 end
)
744 int addr_width
= agaw_to_width(domain
->agaw
);
747 start
&= (((u64
)1) << addr_width
) - 1;
748 end
&= (((u64
)1) << addr_width
) - 1;
749 /* in case it's partial page */
751 end
= PAGE_ALIGN(end
);
752 npages
= (end
- start
) / VTD_PAGE_SIZE
;
754 /* we don't need lock here, nobody else touches the iova range */
756 dma_pte_clear_one(domain
, start
);
757 start
+= VTD_PAGE_SIZE
;
761 /* free page table pages. last level pte should already be cleared */
762 static void dma_pte_free_pagetable(struct dmar_domain
*domain
,
765 int addr_width
= agaw_to_width(domain
->agaw
);
767 int total
= agaw_to_level(domain
->agaw
);
771 start
&= (((u64
)1) << addr_width
) - 1;
772 end
&= (((u64
)1) << addr_width
) - 1;
774 /* we don't need lock here, nobody else touches the iova range */
776 while (level
<= total
) {
777 tmp
= align_to_level(start
, level
);
778 if (tmp
>= end
|| (tmp
+ level_size(level
) > end
))
782 pte
= dma_addr_level_pte(domain
, tmp
, level
);
785 phys_to_virt(dma_pte_addr(pte
)));
787 domain_flush_cache(domain
, pte
, sizeof(*pte
));
789 tmp
+= level_size(level
);
794 if (start
== 0 && end
>= ((((u64
)1) << addr_width
) - 1)) {
795 free_pgtable_page(domain
->pgd
);
801 static int iommu_alloc_root_entry(struct intel_iommu
*iommu
)
803 struct root_entry
*root
;
806 root
= (struct root_entry
*)alloc_pgtable_page();
810 __iommu_flush_cache(iommu
, root
, ROOT_SIZE
);
812 spin_lock_irqsave(&iommu
->lock
, flags
);
813 iommu
->root_entry
= root
;
814 spin_unlock_irqrestore(&iommu
->lock
, flags
);
819 static void iommu_set_root_entry(struct intel_iommu
*iommu
)
825 addr
= iommu
->root_entry
;
827 spin_lock_irqsave(&iommu
->register_lock
, flag
);
828 dmar_writeq(iommu
->reg
+ DMAR_RTADDR_REG
, virt_to_phys(addr
));
830 cmd
= iommu
->gcmd
| DMA_GCMD_SRTP
;
831 writel(cmd
, iommu
->reg
+ DMAR_GCMD_REG
);
833 /* Make sure hardware complete it */
834 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
835 readl
, (sts
& DMA_GSTS_RTPS
), sts
);
837 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
840 static void iommu_flush_write_buffer(struct intel_iommu
*iommu
)
845 if (!rwbf_quirk
&& !cap_rwbf(iommu
->cap
))
847 spin_lock_irqsave(&iommu
->register_lock
, flag
);
848 writel(iommu
->gcmd
| DMA_GCMD_WBF
, iommu
->reg
+ DMAR_GCMD_REG
);
850 /* Make sure hardware complete it */
851 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
852 readl
, (!(val
& DMA_GSTS_WBFS
)), val
);
854 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
857 /* return value determine if we need a write buffer flush */
858 static void __iommu_flush_context(struct intel_iommu
*iommu
,
859 u16 did
, u16 source_id
, u8 function_mask
,
866 case DMA_CCMD_GLOBAL_INVL
:
867 val
= DMA_CCMD_GLOBAL_INVL
;
869 case DMA_CCMD_DOMAIN_INVL
:
870 val
= DMA_CCMD_DOMAIN_INVL
|DMA_CCMD_DID(did
);
872 case DMA_CCMD_DEVICE_INVL
:
873 val
= DMA_CCMD_DEVICE_INVL
|DMA_CCMD_DID(did
)
874 | DMA_CCMD_SID(source_id
) | DMA_CCMD_FM(function_mask
);
881 spin_lock_irqsave(&iommu
->register_lock
, flag
);
882 dmar_writeq(iommu
->reg
+ DMAR_CCMD_REG
, val
);
884 /* Make sure hardware complete it */
885 IOMMU_WAIT_OP(iommu
, DMAR_CCMD_REG
,
886 dmar_readq
, (!(val
& DMA_CCMD_ICC
)), val
);
888 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
891 /* return value determine if we need a write buffer flush */
892 static void __iommu_flush_iotlb(struct intel_iommu
*iommu
, u16 did
,
893 u64 addr
, unsigned int size_order
, u64 type
)
895 int tlb_offset
= ecap_iotlb_offset(iommu
->ecap
);
896 u64 val
= 0, val_iva
= 0;
900 case DMA_TLB_GLOBAL_FLUSH
:
901 /* global flush doesn't need set IVA_REG */
902 val
= DMA_TLB_GLOBAL_FLUSH
|DMA_TLB_IVT
;
904 case DMA_TLB_DSI_FLUSH
:
905 val
= DMA_TLB_DSI_FLUSH
|DMA_TLB_IVT
|DMA_TLB_DID(did
);
907 case DMA_TLB_PSI_FLUSH
:
908 val
= DMA_TLB_PSI_FLUSH
|DMA_TLB_IVT
|DMA_TLB_DID(did
);
909 /* Note: always flush non-leaf currently */
910 val_iva
= size_order
| addr
;
915 /* Note: set drain read/write */
918 * This is probably to be super secure.. Looks like we can
919 * ignore it without any impact.
921 if (cap_read_drain(iommu
->cap
))
922 val
|= DMA_TLB_READ_DRAIN
;
924 if (cap_write_drain(iommu
->cap
))
925 val
|= DMA_TLB_WRITE_DRAIN
;
927 spin_lock_irqsave(&iommu
->register_lock
, flag
);
928 /* Note: Only uses first TLB reg currently */
930 dmar_writeq(iommu
->reg
+ tlb_offset
, val_iva
);
931 dmar_writeq(iommu
->reg
+ tlb_offset
+ 8, val
);
933 /* Make sure hardware complete it */
934 IOMMU_WAIT_OP(iommu
, tlb_offset
+ 8,
935 dmar_readq
, (!(val
& DMA_TLB_IVT
)), val
);
937 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
939 /* check IOTLB invalidation granularity */
940 if (DMA_TLB_IAIG(val
) == 0)
941 printk(KERN_ERR
"IOMMU: flush IOTLB failed\n");
942 if (DMA_TLB_IAIG(val
) != DMA_TLB_IIRG(type
))
943 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
944 (unsigned long long)DMA_TLB_IIRG(type
),
945 (unsigned long long)DMA_TLB_IAIG(val
));
948 static void iommu_flush_iotlb_psi(struct intel_iommu
*iommu
, u16 did
,
949 u64 addr
, unsigned int pages
)
953 BUG_ON(addr
& (~VTD_PAGE_MASK
));
956 /* Fallback to domain selective flush if no PSI support */
957 if (!cap_pgsel_inv(iommu
->cap
))
958 return iommu
->flush
.flush_iotlb(iommu
, did
, 0, 0,
962 * PSI requires page size to be 2 ^ x, and the base address is naturally
963 * aligned to the size
965 mask
= ilog2(__roundup_pow_of_two(pages
));
966 /* Fallback to domain selective flush if size is too big */
967 if (mask
> cap_max_amask_val(iommu
->cap
))
968 return iommu
->flush
.flush_iotlb(iommu
, did
, 0, 0,
971 return iommu
->flush
.flush_iotlb(iommu
, did
, addr
, mask
,
975 static void iommu_disable_protect_mem_regions(struct intel_iommu
*iommu
)
980 spin_lock_irqsave(&iommu
->register_lock
, flags
);
981 pmen
= readl(iommu
->reg
+ DMAR_PMEN_REG
);
982 pmen
&= ~DMA_PMEN_EPM
;
983 writel(pmen
, iommu
->reg
+ DMAR_PMEN_REG
);
985 /* wait for the protected region status bit to clear */
986 IOMMU_WAIT_OP(iommu
, DMAR_PMEN_REG
,
987 readl
, !(pmen
& DMA_PMEN_PRS
), pmen
);
989 spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
992 static int iommu_enable_translation(struct intel_iommu
*iommu
)
997 spin_lock_irqsave(&iommu
->register_lock
, flags
);
998 writel(iommu
->gcmd
|DMA_GCMD_TE
, iommu
->reg
+ DMAR_GCMD_REG
);
1000 /* Make sure hardware complete it */
1001 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1002 readl
, (sts
& DMA_GSTS_TES
), sts
);
1004 iommu
->gcmd
|= DMA_GCMD_TE
;
1005 spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
1009 static int iommu_disable_translation(struct intel_iommu
*iommu
)
1014 spin_lock_irqsave(&iommu
->register_lock
, flag
);
1015 iommu
->gcmd
&= ~DMA_GCMD_TE
;
1016 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
1018 /* Make sure hardware complete it */
1019 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1020 readl
, (!(sts
& DMA_GSTS_TES
)), sts
);
1022 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1027 static int iommu_init_domains(struct intel_iommu
*iommu
)
1029 unsigned long ndomains
;
1030 unsigned long nlongs
;
1032 ndomains
= cap_ndoms(iommu
->cap
);
1033 pr_debug("Number of Domains supportd <%ld>\n", ndomains
);
1034 nlongs
= BITS_TO_LONGS(ndomains
);
1036 /* TBD: there might be 64K domains,
1037 * consider other allocation for future chip
1039 iommu
->domain_ids
= kcalloc(nlongs
, sizeof(unsigned long), GFP_KERNEL
);
1040 if (!iommu
->domain_ids
) {
1041 printk(KERN_ERR
"Allocating domain id array failed\n");
1044 iommu
->domains
= kcalloc(ndomains
, sizeof(struct dmar_domain
*),
1046 if (!iommu
->domains
) {
1047 printk(KERN_ERR
"Allocating domain array failed\n");
1048 kfree(iommu
->domain_ids
);
1052 spin_lock_init(&iommu
->lock
);
1055 * if Caching mode is set, then invalid translations are tagged
1056 * with domainid 0. Hence we need to pre-allocate it.
1058 if (cap_caching_mode(iommu
->cap
))
1059 set_bit(0, iommu
->domain_ids
);
1064 static void domain_exit(struct dmar_domain
*domain
);
1065 static void vm_domain_exit(struct dmar_domain
*domain
);
1067 void free_dmar_iommu(struct intel_iommu
*iommu
)
1069 struct dmar_domain
*domain
;
1071 unsigned long flags
;
1073 i
= find_first_bit(iommu
->domain_ids
, cap_ndoms(iommu
->cap
));
1074 for (; i
< cap_ndoms(iommu
->cap
); ) {
1075 domain
= iommu
->domains
[i
];
1076 clear_bit(i
, iommu
->domain_ids
);
1078 spin_lock_irqsave(&domain
->iommu_lock
, flags
);
1079 if (--domain
->iommu_count
== 0) {
1080 if (domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
)
1081 vm_domain_exit(domain
);
1083 domain_exit(domain
);
1085 spin_unlock_irqrestore(&domain
->iommu_lock
, flags
);
1087 i
= find_next_bit(iommu
->domain_ids
,
1088 cap_ndoms(iommu
->cap
), i
+1);
1091 if (iommu
->gcmd
& DMA_GCMD_TE
)
1092 iommu_disable_translation(iommu
);
1095 set_irq_data(iommu
->irq
, NULL
);
1096 /* This will mask the irq */
1097 free_irq(iommu
->irq
, iommu
);
1098 destroy_irq(iommu
->irq
);
1101 kfree(iommu
->domains
);
1102 kfree(iommu
->domain_ids
);
1104 g_iommus
[iommu
->seq_id
] = NULL
;
1106 /* if all iommus are freed, free g_iommus */
1107 for (i
= 0; i
< g_num_of_iommus
; i
++) {
1112 if (i
== g_num_of_iommus
)
1115 /* free context mapping */
1116 free_context_table(iommu
);
1119 static struct dmar_domain
* iommu_alloc_domain(struct intel_iommu
*iommu
)
1122 unsigned long ndomains
;
1123 struct dmar_domain
*domain
;
1124 unsigned long flags
;
1126 domain
= alloc_domain_mem();
1130 ndomains
= cap_ndoms(iommu
->cap
);
1132 spin_lock_irqsave(&iommu
->lock
, flags
);
1133 num
= find_first_zero_bit(iommu
->domain_ids
, ndomains
);
1134 if (num
>= ndomains
) {
1135 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1136 free_domain_mem(domain
);
1137 printk(KERN_ERR
"IOMMU: no free domain ids\n");
1141 set_bit(num
, iommu
->domain_ids
);
1143 memset(&domain
->iommu_bmp
, 0, sizeof(unsigned long));
1144 set_bit(iommu
->seq_id
, &domain
->iommu_bmp
);
1146 iommu
->domains
[num
] = domain
;
1147 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1152 static void iommu_free_domain(struct dmar_domain
*domain
)
1154 unsigned long flags
;
1155 struct intel_iommu
*iommu
;
1157 iommu
= domain_get_iommu(domain
);
1159 spin_lock_irqsave(&iommu
->lock
, flags
);
1160 clear_bit(domain
->id
, iommu
->domain_ids
);
1161 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1164 static struct iova_domain reserved_iova_list
;
1165 static struct lock_class_key reserved_alloc_key
;
1166 static struct lock_class_key reserved_rbtree_key
;
1168 static void dmar_init_reserved_ranges(void)
1170 struct pci_dev
*pdev
= NULL
;
1175 init_iova_domain(&reserved_iova_list
, DMA_32BIT_PFN
);
1177 lockdep_set_class(&reserved_iova_list
.iova_alloc_lock
,
1178 &reserved_alloc_key
);
1179 lockdep_set_class(&reserved_iova_list
.iova_rbtree_lock
,
1180 &reserved_rbtree_key
);
1182 /* IOAPIC ranges shouldn't be accessed by DMA */
1183 iova
= reserve_iova(&reserved_iova_list
, IOVA_PFN(IOAPIC_RANGE_START
),
1184 IOVA_PFN(IOAPIC_RANGE_END
));
1186 printk(KERN_ERR
"Reserve IOAPIC range failed\n");
1188 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1189 for_each_pci_dev(pdev
) {
1192 for (i
= 0; i
< PCI_NUM_RESOURCES
; i
++) {
1193 r
= &pdev
->resource
[i
];
1194 if (!r
->flags
|| !(r
->flags
& IORESOURCE_MEM
))
1198 size
= r
->end
- addr
;
1199 size
= PAGE_ALIGN(size
);
1200 iova
= reserve_iova(&reserved_iova_list
, IOVA_PFN(addr
),
1201 IOVA_PFN(size
+ addr
) - 1);
1203 printk(KERN_ERR
"Reserve iova failed\n");
1209 static void domain_reserve_special_ranges(struct dmar_domain
*domain
)
1211 copy_reserved_iova(&reserved_iova_list
, &domain
->iovad
);
1214 static inline int guestwidth_to_adjustwidth(int gaw
)
1217 int r
= (gaw
- 12) % 9;
1228 static int domain_init(struct dmar_domain
*domain
, int guest_width
)
1230 struct intel_iommu
*iommu
;
1231 int adjust_width
, agaw
;
1232 unsigned long sagaw
;
1234 init_iova_domain(&domain
->iovad
, DMA_32BIT_PFN
);
1235 spin_lock_init(&domain
->mapping_lock
);
1236 spin_lock_init(&domain
->iommu_lock
);
1238 domain_reserve_special_ranges(domain
);
1240 /* calculate AGAW */
1241 iommu
= domain_get_iommu(domain
);
1242 if (guest_width
> cap_mgaw(iommu
->cap
))
1243 guest_width
= cap_mgaw(iommu
->cap
);
1244 domain
->gaw
= guest_width
;
1245 adjust_width
= guestwidth_to_adjustwidth(guest_width
);
1246 agaw
= width_to_agaw(adjust_width
);
1247 sagaw
= cap_sagaw(iommu
->cap
);
1248 if (!test_bit(agaw
, &sagaw
)) {
1249 /* hardware doesn't support it, choose a bigger one */
1250 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw
);
1251 agaw
= find_next_bit(&sagaw
, 5, agaw
);
1255 domain
->agaw
= agaw
;
1256 INIT_LIST_HEAD(&domain
->devices
);
1258 if (ecap_coherent(iommu
->ecap
))
1259 domain
->iommu_coherency
= 1;
1261 domain
->iommu_coherency
= 0;
1263 if (ecap_sc_support(iommu
->ecap
))
1264 domain
->iommu_snooping
= 1;
1266 domain
->iommu_snooping
= 0;
1268 domain
->iommu_count
= 1;
1270 /* always allocate the top pgd */
1271 domain
->pgd
= (struct dma_pte
*)alloc_pgtable_page();
1274 __iommu_flush_cache(iommu
, domain
->pgd
, PAGE_SIZE
);
1278 static void domain_exit(struct dmar_domain
*domain
)
1282 /* Domain 0 is reserved, so dont process it */
1286 domain_remove_dev_info(domain
);
1288 put_iova_domain(&domain
->iovad
);
1289 end
= DOMAIN_MAX_ADDR(domain
->gaw
);
1290 end
= end
& (~PAGE_MASK
);
1293 dma_pte_clear_range(domain
, 0, end
);
1295 /* free page tables */
1296 dma_pte_free_pagetable(domain
, 0, end
);
1298 iommu_free_domain(domain
);
1299 free_domain_mem(domain
);
1302 static int domain_context_mapping_one(struct dmar_domain
*domain
, int segment
,
1303 u8 bus
, u8 devfn
, int translation
)
1305 struct context_entry
*context
;
1306 unsigned long flags
;
1307 struct intel_iommu
*iommu
;
1308 struct dma_pte
*pgd
;
1310 unsigned long ndomains
;
1314 pr_debug("Set context mapping for %02x:%02x.%d\n",
1315 bus
, PCI_SLOT(devfn
), PCI_FUNC(devfn
));
1317 BUG_ON(!domain
->pgd
);
1318 BUG_ON(translation
!= CONTEXT_TT_PASS_THROUGH
&&
1319 translation
!= CONTEXT_TT_MULTI_LEVEL
);
1321 iommu
= device_to_iommu(segment
, bus
, devfn
);
1325 context
= device_to_context_entry(iommu
, bus
, devfn
);
1328 spin_lock_irqsave(&iommu
->lock
, flags
);
1329 if (context_present(context
)) {
1330 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1337 if (domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
) {
1340 /* find an available domain id for this device in iommu */
1341 ndomains
= cap_ndoms(iommu
->cap
);
1342 num
= find_first_bit(iommu
->domain_ids
, ndomains
);
1343 for (; num
< ndomains
; ) {
1344 if (iommu
->domains
[num
] == domain
) {
1349 num
= find_next_bit(iommu
->domain_ids
,
1350 cap_ndoms(iommu
->cap
), num
+1);
1354 num
= find_first_zero_bit(iommu
->domain_ids
, ndomains
);
1355 if (num
>= ndomains
) {
1356 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1357 printk(KERN_ERR
"IOMMU: no free domain ids\n");
1361 set_bit(num
, iommu
->domain_ids
);
1362 iommu
->domains
[num
] = domain
;
1366 /* Skip top levels of page tables for
1367 * iommu which has less agaw than default.
1369 for (agaw
= domain
->agaw
; agaw
!= iommu
->agaw
; agaw
--) {
1370 pgd
= phys_to_virt(dma_pte_addr(pgd
));
1371 if (!dma_pte_present(pgd
)) {
1372 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1378 context_set_domain_id(context
, id
);
1381 * In pass through mode, AW must be programmed to indicate the largest
1382 * AGAW value supported by hardware. And ASR is ignored by hardware.
1384 if (likely(translation
== CONTEXT_TT_MULTI_LEVEL
)) {
1385 context_set_address_width(context
, iommu
->agaw
);
1386 context_set_address_root(context
, virt_to_phys(pgd
));
1388 context_set_address_width(context
, iommu
->msagaw
);
1390 context_set_translation_type(context
, translation
);
1391 context_set_fault_enable(context
);
1392 context_set_present(context
);
1393 domain_flush_cache(domain
, context
, sizeof(*context
));
1396 * It's a non-present to present mapping. If hardware doesn't cache
1397 * non-present entry we only need to flush the write-buffer. If the
1398 * _does_ cache non-present entries, then it does so in the special
1399 * domain #0, which we have to flush:
1401 if (cap_caching_mode(iommu
->cap
)) {
1402 iommu
->flush
.flush_context(iommu
, 0,
1403 (((u16
)bus
) << 8) | devfn
,
1404 DMA_CCMD_MASK_NOBIT
,
1405 DMA_CCMD_DEVICE_INVL
);
1406 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0, DMA_TLB_DSI_FLUSH
);
1408 iommu_flush_write_buffer(iommu
);
1410 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1412 spin_lock_irqsave(&domain
->iommu_lock
, flags
);
1413 if (!test_and_set_bit(iommu
->seq_id
, &domain
->iommu_bmp
)) {
1414 domain
->iommu_count
++;
1415 domain_update_iommu_cap(domain
);
1417 spin_unlock_irqrestore(&domain
->iommu_lock
, flags
);
1422 domain_context_mapping(struct dmar_domain
*domain
, struct pci_dev
*pdev
,
1426 struct pci_dev
*tmp
, *parent
;
1428 ret
= domain_context_mapping_one(domain
, pci_domain_nr(pdev
->bus
),
1429 pdev
->bus
->number
, pdev
->devfn
,
1434 /* dependent device mapping */
1435 tmp
= pci_find_upstream_pcie_bridge(pdev
);
1438 /* Secondary interface's bus number and devfn 0 */
1439 parent
= pdev
->bus
->self
;
1440 while (parent
!= tmp
) {
1441 ret
= domain_context_mapping_one(domain
,
1442 pci_domain_nr(parent
->bus
),
1443 parent
->bus
->number
,
1444 parent
->devfn
, translation
);
1447 parent
= parent
->bus
->self
;
1449 if (tmp
->is_pcie
) /* this is a PCIE-to-PCI bridge */
1450 return domain_context_mapping_one(domain
,
1451 pci_domain_nr(tmp
->subordinate
),
1452 tmp
->subordinate
->number
, 0,
1454 else /* this is a legacy PCI bridge */
1455 return domain_context_mapping_one(domain
,
1456 pci_domain_nr(tmp
->bus
),
1462 static int domain_context_mapped(struct pci_dev
*pdev
)
1465 struct pci_dev
*tmp
, *parent
;
1466 struct intel_iommu
*iommu
;
1468 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
), pdev
->bus
->number
,
1473 ret
= device_context_mapped(iommu
, pdev
->bus
->number
, pdev
->devfn
);
1476 /* dependent device mapping */
1477 tmp
= pci_find_upstream_pcie_bridge(pdev
);
1480 /* Secondary interface's bus number and devfn 0 */
1481 parent
= pdev
->bus
->self
;
1482 while (parent
!= tmp
) {
1483 ret
= device_context_mapped(iommu
, parent
->bus
->number
,
1487 parent
= parent
->bus
->self
;
1490 return device_context_mapped(iommu
, tmp
->subordinate
->number
,
1493 return device_context_mapped(iommu
, tmp
->bus
->number
,
1498 domain_page_mapping(struct dmar_domain
*domain
, dma_addr_t iova
,
1499 u64 hpa
, size_t size
, int prot
)
1501 u64 start_pfn
, end_pfn
;
1502 struct dma_pte
*pte
;
1504 int addr_width
= agaw_to_width(domain
->agaw
);
1506 hpa
&= (((u64
)1) << addr_width
) - 1;
1508 if ((prot
& (DMA_PTE_READ
|DMA_PTE_WRITE
)) == 0)
1511 start_pfn
= ((u64
)hpa
) >> VTD_PAGE_SHIFT
;
1512 end_pfn
= (VTD_PAGE_ALIGN(((u64
)hpa
) + size
)) >> VTD_PAGE_SHIFT
;
1514 while (start_pfn
< end_pfn
) {
1515 pte
= addr_to_dma_pte(domain
, iova
+ VTD_PAGE_SIZE
* index
);
1518 /* We don't need lock here, nobody else
1519 * touches the iova range
1521 BUG_ON(dma_pte_addr(pte
));
1522 dma_set_pte_addr(pte
, start_pfn
<< VTD_PAGE_SHIFT
);
1523 dma_set_pte_prot(pte
, prot
);
1524 if (prot
& DMA_PTE_SNP
)
1525 dma_set_pte_snp(pte
);
1526 domain_flush_cache(domain
, pte
, sizeof(*pte
));
1533 static void iommu_detach_dev(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
1538 clear_context_table(iommu
, bus
, devfn
);
1539 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
1540 DMA_CCMD_GLOBAL_INVL
);
1541 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH
);
1544 static void domain_remove_dev_info(struct dmar_domain
*domain
)
1546 struct device_domain_info
*info
;
1547 unsigned long flags
;
1548 struct intel_iommu
*iommu
;
1550 spin_lock_irqsave(&device_domain_lock
, flags
);
1551 while (!list_empty(&domain
->devices
)) {
1552 info
= list_entry(domain
->devices
.next
,
1553 struct device_domain_info
, link
);
1554 list_del(&info
->link
);
1555 list_del(&info
->global
);
1557 info
->dev
->dev
.archdata
.iommu
= NULL
;
1558 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1560 iommu
= device_to_iommu(info
->segment
, info
->bus
, info
->devfn
);
1561 iommu_detach_dev(iommu
, info
->bus
, info
->devfn
);
1562 free_devinfo_mem(info
);
1564 spin_lock_irqsave(&device_domain_lock
, flags
);
1566 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1571 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1573 static struct dmar_domain
*
1574 find_domain(struct pci_dev
*pdev
)
1576 struct device_domain_info
*info
;
1578 /* No lock here, assumes no domain exit in normal case */
1579 info
= pdev
->dev
.archdata
.iommu
;
1581 return info
->domain
;
1585 /* domain is initialized */
1586 static struct dmar_domain
*get_domain_for_dev(struct pci_dev
*pdev
, int gaw
)
1588 struct dmar_domain
*domain
, *found
= NULL
;
1589 struct intel_iommu
*iommu
;
1590 struct dmar_drhd_unit
*drhd
;
1591 struct device_domain_info
*info
, *tmp
;
1592 struct pci_dev
*dev_tmp
;
1593 unsigned long flags
;
1594 int bus
= 0, devfn
= 0;
1597 domain
= find_domain(pdev
);
1601 segment
= pci_domain_nr(pdev
->bus
);
1603 dev_tmp
= pci_find_upstream_pcie_bridge(pdev
);
1605 if (dev_tmp
->is_pcie
) {
1606 bus
= dev_tmp
->subordinate
->number
;
1609 bus
= dev_tmp
->bus
->number
;
1610 devfn
= dev_tmp
->devfn
;
1612 spin_lock_irqsave(&device_domain_lock
, flags
);
1613 list_for_each_entry(info
, &device_domain_list
, global
) {
1614 if (info
->segment
== segment
&&
1615 info
->bus
== bus
&& info
->devfn
== devfn
) {
1616 found
= info
->domain
;
1620 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1621 /* pcie-pci bridge already has a domain, uses it */
1628 /* Allocate new domain for the device */
1629 drhd
= dmar_find_matched_drhd_unit(pdev
);
1631 printk(KERN_ERR
"IOMMU: can't find DMAR for device %s\n",
1635 iommu
= drhd
->iommu
;
1637 domain
= iommu_alloc_domain(iommu
);
1641 if (domain_init(domain
, gaw
)) {
1642 domain_exit(domain
);
1646 /* register pcie-to-pci device */
1648 info
= alloc_devinfo_mem();
1650 domain_exit(domain
);
1653 info
->segment
= segment
;
1655 info
->devfn
= devfn
;
1657 info
->domain
= domain
;
1658 /* This domain is shared by devices under p2p bridge */
1659 domain
->flags
|= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES
;
1661 /* pcie-to-pci bridge already has a domain, uses it */
1663 spin_lock_irqsave(&device_domain_lock
, flags
);
1664 list_for_each_entry(tmp
, &device_domain_list
, global
) {
1665 if (tmp
->segment
== segment
&&
1666 tmp
->bus
== bus
&& tmp
->devfn
== devfn
) {
1667 found
= tmp
->domain
;
1672 free_devinfo_mem(info
);
1673 domain_exit(domain
);
1676 list_add(&info
->link
, &domain
->devices
);
1677 list_add(&info
->global
, &device_domain_list
);
1679 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1683 info
= alloc_devinfo_mem();
1686 info
->segment
= segment
;
1687 info
->bus
= pdev
->bus
->number
;
1688 info
->devfn
= pdev
->devfn
;
1690 info
->domain
= domain
;
1691 spin_lock_irqsave(&device_domain_lock
, flags
);
1692 /* somebody is fast */
1693 found
= find_domain(pdev
);
1694 if (found
!= NULL
) {
1695 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1696 if (found
!= domain
) {
1697 domain_exit(domain
);
1700 free_devinfo_mem(info
);
1703 list_add(&info
->link
, &domain
->devices
);
1704 list_add(&info
->global
, &device_domain_list
);
1705 pdev
->dev
.archdata
.iommu
= info
;
1706 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1709 /* recheck it here, maybe others set it */
1710 return find_domain(pdev
);
1713 static int iommu_prepare_identity_map(struct pci_dev
*pdev
,
1714 unsigned long long start
,
1715 unsigned long long end
)
1717 struct dmar_domain
*domain
;
1719 unsigned long long base
;
1723 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1724 pci_name(pdev
), start
, end
);
1725 /* page table init */
1726 domain
= get_domain_for_dev(pdev
, DEFAULT_DOMAIN_ADDRESS_WIDTH
);
1730 /* The address might not be aligned */
1731 base
= start
& PAGE_MASK
;
1733 size
= PAGE_ALIGN(size
);
1734 if (!reserve_iova(&domain
->iovad
, IOVA_PFN(base
),
1735 IOVA_PFN(base
+ size
) - 1)) {
1736 printk(KERN_ERR
"IOMMU: reserve iova failed\n");
1741 pr_debug("Mapping reserved region %lx@%llx for %s\n",
1742 size
, base
, pci_name(pdev
));
1744 * RMRR range might have overlap with physical memory range,
1747 dma_pte_clear_range(domain
, base
, base
+ size
);
1749 ret
= domain_page_mapping(domain
, base
, base
, size
,
1750 DMA_PTE_READ
|DMA_PTE_WRITE
);
1754 /* context entry init */
1755 ret
= domain_context_mapping(domain
, pdev
, CONTEXT_TT_MULTI_LEVEL
);
1759 domain_exit(domain
);
1764 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit
*rmrr
,
1765 struct pci_dev
*pdev
)
1767 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
1769 return iommu_prepare_identity_map(pdev
, rmrr
->base_address
,
1770 rmrr
->end_address
+ 1);
1773 #ifdef CONFIG_DMAR_GFX_WA
1774 struct iommu_prepare_data
{
1775 struct pci_dev
*pdev
;
1779 static int __init
iommu_prepare_work_fn(unsigned long start_pfn
,
1780 unsigned long end_pfn
, void *datax
)
1782 struct iommu_prepare_data
*data
;
1784 data
= (struct iommu_prepare_data
*)datax
;
1786 data
->ret
= iommu_prepare_identity_map(data
->pdev
,
1787 start_pfn
<<PAGE_SHIFT
, end_pfn
<<PAGE_SHIFT
);
1792 static int __init
iommu_prepare_with_active_regions(struct pci_dev
*pdev
)
1795 struct iommu_prepare_data data
;
1800 for_each_online_node(nid
) {
1801 work_with_active_regions(nid
, iommu_prepare_work_fn
, &data
);
1808 static void __init
iommu_prepare_gfx_mapping(void)
1810 struct pci_dev
*pdev
= NULL
;
1813 for_each_pci_dev(pdev
) {
1814 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
||
1815 !IS_GFX_DEVICE(pdev
))
1817 printk(KERN_INFO
"IOMMU: gfx device %s 1-1 mapping\n",
1819 ret
= iommu_prepare_with_active_regions(pdev
);
1821 printk(KERN_ERR
"IOMMU: mapping reserved region failed\n");
1824 #else /* !CONFIG_DMAR_GFX_WA */
1825 static inline void iommu_prepare_gfx_mapping(void)
1831 #ifdef CONFIG_DMAR_FLOPPY_WA
1832 static inline void iommu_prepare_isa(void)
1834 struct pci_dev
*pdev
;
1837 pdev
= pci_get_class(PCI_CLASS_BRIDGE_ISA
<< 8, NULL
);
1841 printk(KERN_INFO
"IOMMU: Prepare 0-16M unity mapping for LPC\n");
1842 ret
= iommu_prepare_identity_map(pdev
, 0, 16*1024*1024);
1845 printk(KERN_ERR
"IOMMU: Failed to create 0-64M identity map, "
1846 "floppy might not work\n");
1850 static inline void iommu_prepare_isa(void)
1854 #endif /* !CONFIG_DMAR_FLPY_WA */
1856 /* Initialize each context entry as pass through.*/
1857 static int __init
init_context_pass_through(void)
1859 struct pci_dev
*pdev
= NULL
;
1860 struct dmar_domain
*domain
;
1863 for_each_pci_dev(pdev
) {
1864 domain
= get_domain_for_dev(pdev
, DEFAULT_DOMAIN_ADDRESS_WIDTH
);
1865 ret
= domain_context_mapping(domain
, pdev
,
1866 CONTEXT_TT_PASS_THROUGH
);
1873 static int __init
init_dmars(void)
1875 struct dmar_drhd_unit
*drhd
;
1876 struct dmar_rmrr_unit
*rmrr
;
1877 struct pci_dev
*pdev
;
1878 struct intel_iommu
*iommu
;
1880 int pass_through
= 1;
1885 * initialize and program root entry to not present
1888 for_each_drhd_unit(drhd
) {
1891 * lock not needed as this is only incremented in the single
1892 * threaded kernel __init code path all other access are read
1897 g_iommus
= kcalloc(g_num_of_iommus
, sizeof(struct intel_iommu
*),
1900 printk(KERN_ERR
"Allocating global iommu array failed\n");
1905 deferred_flush
= kzalloc(g_num_of_iommus
*
1906 sizeof(struct deferred_flush_tables
), GFP_KERNEL
);
1907 if (!deferred_flush
) {
1913 for_each_drhd_unit(drhd
) {
1917 iommu
= drhd
->iommu
;
1918 g_iommus
[iommu
->seq_id
] = iommu
;
1920 ret
= iommu_init_domains(iommu
);
1926 * we could share the same root & context tables
1927 * amoung all IOMMU's. Need to Split it later.
1929 ret
= iommu_alloc_root_entry(iommu
);
1931 printk(KERN_ERR
"IOMMU: allocate root entry failed\n");
1934 if (!ecap_pass_through(iommu
->ecap
))
1937 if (iommu_pass_through
)
1938 if (!pass_through
) {
1940 "Pass Through is not supported by hardware.\n");
1941 iommu_pass_through
= 0;
1945 * Start from the sane iommu hardware state.
1947 for_each_drhd_unit(drhd
) {
1951 iommu
= drhd
->iommu
;
1954 * If the queued invalidation is already initialized by us
1955 * (for example, while enabling interrupt-remapping) then
1956 * we got the things already rolling from a sane state.
1962 * Clear any previous faults.
1964 dmar_fault(-1, iommu
);
1966 * Disable queued invalidation if supported and already enabled
1967 * before OS handover.
1969 dmar_disable_qi(iommu
);
1972 for_each_drhd_unit(drhd
) {
1976 iommu
= drhd
->iommu
;
1978 if (dmar_enable_qi(iommu
)) {
1980 * Queued Invalidate not enabled, use Register Based
1983 iommu
->flush
.flush_context
= __iommu_flush_context
;
1984 iommu
->flush
.flush_iotlb
= __iommu_flush_iotlb
;
1985 printk(KERN_INFO
"IOMMU 0x%Lx: using Register based "
1987 (unsigned long long)drhd
->reg_base_addr
);
1989 iommu
->flush
.flush_context
= qi_flush_context
;
1990 iommu
->flush
.flush_iotlb
= qi_flush_iotlb
;
1991 printk(KERN_INFO
"IOMMU 0x%Lx: using Queued "
1993 (unsigned long long)drhd
->reg_base_addr
);
1997 #ifdef CONFIG_INTR_REMAP
1998 if (!intr_remapping_enabled
) {
1999 ret
= enable_intr_remapping(0);
2002 "IOMMU: enable interrupt remapping failed\n");
2006 * If pass through is set and enabled, context entries of all pci
2007 * devices are intialized by pass through translation type.
2009 if (iommu_pass_through
) {
2010 ret
= init_context_pass_through();
2012 printk(KERN_ERR
"IOMMU: Pass through init failed.\n");
2013 iommu_pass_through
= 0;
2018 * If pass through is not set or not enabled, setup context entries for
2019 * identity mappings for rmrr, gfx, and isa.
2021 if (!iommu_pass_through
) {
2024 * for each dev attached to rmrr
2026 * locate drhd for dev, alloc domain for dev
2027 * allocate free domain
2028 * allocate page table entries for rmrr
2029 * if context not allocated for bus
2030 * allocate and init context
2031 * set present in root table for this bus
2032 * init context with domain, translation etc
2036 for_each_rmrr_units(rmrr
) {
2037 for (i
= 0; i
< rmrr
->devices_cnt
; i
++) {
2038 pdev
= rmrr
->devices
[i
];
2040 * some BIOS lists non-exist devices in DMAR
2045 ret
= iommu_prepare_rmrr_dev(rmrr
, pdev
);
2048 "IOMMU: mapping reserved region failed\n");
2052 iommu_prepare_gfx_mapping();
2054 iommu_prepare_isa();
2060 * global invalidate context cache
2061 * global invalidate iotlb
2062 * enable translation
2064 for_each_drhd_unit(drhd
) {
2067 iommu
= drhd
->iommu
;
2069 iommu_flush_write_buffer(iommu
);
2071 ret
= dmar_set_interrupt(iommu
);
2075 iommu_set_root_entry(iommu
);
2077 iommu
->flush
.flush_context(iommu
, 0, 0, 0, DMA_CCMD_GLOBAL_INVL
);
2078 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH
);
2079 iommu_disable_protect_mem_regions(iommu
);
2081 ret
= iommu_enable_translation(iommu
);
2088 for_each_drhd_unit(drhd
) {
2091 iommu
= drhd
->iommu
;
2098 static inline u64
aligned_size(u64 host_addr
, size_t size
)
2101 addr
= (host_addr
& (~PAGE_MASK
)) + size
;
2102 return PAGE_ALIGN(addr
);
2106 iommu_alloc_iova(struct dmar_domain
*domain
, size_t size
, u64 end
)
2110 /* Make sure it's in range */
2111 end
= min_t(u64
, DOMAIN_MAX_ADDR(domain
->gaw
), end
);
2112 if (!size
|| (IOVA_START_ADDR
+ size
> end
))
2115 piova
= alloc_iova(&domain
->iovad
,
2116 size
>> PAGE_SHIFT
, IOVA_PFN(end
), 1);
2120 static struct iova
*
2121 __intel_alloc_iova(struct device
*dev
, struct dmar_domain
*domain
,
2122 size_t size
, u64 dma_mask
)
2124 struct pci_dev
*pdev
= to_pci_dev(dev
);
2125 struct iova
*iova
= NULL
;
2127 if (dma_mask
<= DMA_BIT_MASK(32) || dmar_forcedac
)
2128 iova
= iommu_alloc_iova(domain
, size
, dma_mask
);
2131 * First try to allocate an io virtual address in
2132 * DMA_BIT_MASK(32) and if that fails then try allocating
2135 iova
= iommu_alloc_iova(domain
, size
, DMA_BIT_MASK(32));
2137 iova
= iommu_alloc_iova(domain
, size
, dma_mask
);
2141 printk(KERN_ERR
"Allocating iova for %s failed", pci_name(pdev
));
2148 static struct dmar_domain
*
2149 get_valid_domain_for_dev(struct pci_dev
*pdev
)
2151 struct dmar_domain
*domain
;
2154 domain
= get_domain_for_dev(pdev
,
2155 DEFAULT_DOMAIN_ADDRESS_WIDTH
);
2158 "Allocating domain for %s failed", pci_name(pdev
));
2162 /* make sure context mapping is ok */
2163 if (unlikely(!domain_context_mapped(pdev
))) {
2164 ret
= domain_context_mapping(domain
, pdev
,
2165 CONTEXT_TT_MULTI_LEVEL
);
2168 "Domain context map for %s failed",
2177 static dma_addr_t
__intel_map_single(struct device
*hwdev
, phys_addr_t paddr
,
2178 size_t size
, int dir
, u64 dma_mask
)
2180 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
2181 struct dmar_domain
*domain
;
2182 phys_addr_t start_paddr
;
2186 struct intel_iommu
*iommu
;
2188 BUG_ON(dir
== DMA_NONE
);
2189 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
2192 domain
= get_valid_domain_for_dev(pdev
);
2196 iommu
= domain_get_iommu(domain
);
2197 size
= aligned_size((u64
)paddr
, size
);
2199 iova
= __intel_alloc_iova(hwdev
, domain
, size
, pdev
->dma_mask
);
2203 start_paddr
= (phys_addr_t
)iova
->pfn_lo
<< PAGE_SHIFT
;
2206 * Check if DMAR supports zero-length reads on write only
2209 if (dir
== DMA_TO_DEVICE
|| dir
== DMA_BIDIRECTIONAL
|| \
2210 !cap_zlr(iommu
->cap
))
2211 prot
|= DMA_PTE_READ
;
2212 if (dir
== DMA_FROM_DEVICE
|| dir
== DMA_BIDIRECTIONAL
)
2213 prot
|= DMA_PTE_WRITE
;
2215 * paddr - (paddr + size) might be partial page, we should map the whole
2216 * page. Note: if two part of one page are separately mapped, we
2217 * might have two guest_addr mapping to the same host paddr, but this
2218 * is not a big problem
2220 ret
= domain_page_mapping(domain
, start_paddr
,
2221 ((u64
)paddr
) & PAGE_MASK
, size
, prot
);
2225 /* it's a non-present to present mapping. Only flush if caching mode */
2226 if (cap_caching_mode(iommu
->cap
))
2227 iommu_flush_iotlb_psi(iommu
, 0, start_paddr
,
2228 size
>> VTD_PAGE_SHIFT
);
2230 iommu_flush_write_buffer(iommu
);
2232 return start_paddr
+ ((u64
)paddr
& (~PAGE_MASK
));
2236 __free_iova(&domain
->iovad
, iova
);
2237 printk(KERN_ERR
"Device %s request: %zx@%llx dir %d --- failed\n",
2238 pci_name(pdev
), size
, (unsigned long long)paddr
, dir
);
2242 static dma_addr_t
intel_map_page(struct device
*dev
, struct page
*page
,
2243 unsigned long offset
, size_t size
,
2244 enum dma_data_direction dir
,
2245 struct dma_attrs
*attrs
)
2247 return __intel_map_single(dev
, page_to_phys(page
) + offset
, size
,
2248 dir
, to_pci_dev(dev
)->dma_mask
);
2251 static void flush_unmaps(void)
2257 /* just flush them all */
2258 for (i
= 0; i
< g_num_of_iommus
; i
++) {
2259 struct intel_iommu
*iommu
= g_iommus
[i
];
2263 if (deferred_flush
[i
].next
) {
2264 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
2265 DMA_TLB_GLOBAL_FLUSH
);
2266 for (j
= 0; j
< deferred_flush
[i
].next
; j
++) {
2267 __free_iova(&deferred_flush
[i
].domain
[j
]->iovad
,
2268 deferred_flush
[i
].iova
[j
]);
2270 deferred_flush
[i
].next
= 0;
2277 static void flush_unmaps_timeout(unsigned long data
)
2279 unsigned long flags
;
2281 spin_lock_irqsave(&async_umap_flush_lock
, flags
);
2283 spin_unlock_irqrestore(&async_umap_flush_lock
, flags
);
2286 static void add_unmap(struct dmar_domain
*dom
, struct iova
*iova
)
2288 unsigned long flags
;
2290 struct intel_iommu
*iommu
;
2292 spin_lock_irqsave(&async_umap_flush_lock
, flags
);
2293 if (list_size
== HIGH_WATER_MARK
)
2296 iommu
= domain_get_iommu(dom
);
2297 iommu_id
= iommu
->seq_id
;
2299 next
= deferred_flush
[iommu_id
].next
;
2300 deferred_flush
[iommu_id
].domain
[next
] = dom
;
2301 deferred_flush
[iommu_id
].iova
[next
] = iova
;
2302 deferred_flush
[iommu_id
].next
++;
2305 mod_timer(&unmap_timer
, jiffies
+ msecs_to_jiffies(10));
2309 spin_unlock_irqrestore(&async_umap_flush_lock
, flags
);
2312 static void intel_unmap_page(struct device
*dev
, dma_addr_t dev_addr
,
2313 size_t size
, enum dma_data_direction dir
,
2314 struct dma_attrs
*attrs
)
2316 struct pci_dev
*pdev
= to_pci_dev(dev
);
2317 struct dmar_domain
*domain
;
2318 unsigned long start_addr
;
2320 struct intel_iommu
*iommu
;
2322 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
2324 domain
= find_domain(pdev
);
2327 iommu
= domain_get_iommu(domain
);
2329 iova
= find_iova(&domain
->iovad
, IOVA_PFN(dev_addr
));
2333 start_addr
= iova
->pfn_lo
<< PAGE_SHIFT
;
2334 size
= aligned_size((u64
)dev_addr
, size
);
2336 pr_debug("Device %s unmapping: %zx@%llx\n",
2337 pci_name(pdev
), size
, (unsigned long long)start_addr
);
2339 /* clear the whole page */
2340 dma_pte_clear_range(domain
, start_addr
, start_addr
+ size
);
2341 /* free page tables */
2342 dma_pte_free_pagetable(domain
, start_addr
, start_addr
+ size
);
2343 if (intel_iommu_strict
) {
2344 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_addr
,
2345 size
>> VTD_PAGE_SHIFT
);
2347 __free_iova(&domain
->iovad
, iova
);
2349 add_unmap(domain
, iova
);
2351 * queue up the release of the unmap to save the 1/6th of the
2352 * cpu used up by the iotlb flush operation...
2357 static void intel_unmap_single(struct device
*dev
, dma_addr_t dev_addr
, size_t size
,
2360 intel_unmap_page(dev
, dev_addr
, size
, dir
, NULL
);
2363 static void *intel_alloc_coherent(struct device
*hwdev
, size_t size
,
2364 dma_addr_t
*dma_handle
, gfp_t flags
)
2369 size
= PAGE_ALIGN(size
);
2370 order
= get_order(size
);
2371 flags
&= ~(GFP_DMA
| GFP_DMA32
);
2373 vaddr
= (void *)__get_free_pages(flags
, order
);
2376 memset(vaddr
, 0, size
);
2378 *dma_handle
= __intel_map_single(hwdev
, virt_to_bus(vaddr
), size
,
2380 hwdev
->coherent_dma_mask
);
2383 free_pages((unsigned long)vaddr
, order
);
2387 static void intel_free_coherent(struct device
*hwdev
, size_t size
, void *vaddr
,
2388 dma_addr_t dma_handle
)
2392 size
= PAGE_ALIGN(size
);
2393 order
= get_order(size
);
2395 intel_unmap_single(hwdev
, dma_handle
, size
, DMA_BIDIRECTIONAL
);
2396 free_pages((unsigned long)vaddr
, order
);
2399 static void intel_unmap_sg(struct device
*hwdev
, struct scatterlist
*sglist
,
2400 int nelems
, enum dma_data_direction dir
,
2401 struct dma_attrs
*attrs
)
2404 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
2405 struct dmar_domain
*domain
;
2406 unsigned long start_addr
;
2410 struct scatterlist
*sg
;
2411 struct intel_iommu
*iommu
;
2413 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
2416 domain
= find_domain(pdev
);
2419 iommu
= domain_get_iommu(domain
);
2421 iova
= find_iova(&domain
->iovad
, IOVA_PFN(sglist
[0].dma_address
));
2424 for_each_sg(sglist
, sg
, nelems
, i
) {
2425 addr
= page_to_phys(sg_page(sg
)) + sg
->offset
;
2426 size
+= aligned_size((u64
)addr
, sg
->length
);
2429 start_addr
= iova
->pfn_lo
<< PAGE_SHIFT
;
2431 /* clear the whole page */
2432 dma_pte_clear_range(domain
, start_addr
, start_addr
+ size
);
2433 /* free page tables */
2434 dma_pte_free_pagetable(domain
, start_addr
, start_addr
+ size
);
2436 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_addr
,
2437 size
>> VTD_PAGE_SHIFT
);
2440 __free_iova(&domain
->iovad
, iova
);
2443 static int intel_nontranslate_map_sg(struct device
*hddev
,
2444 struct scatterlist
*sglist
, int nelems
, int dir
)
2447 struct scatterlist
*sg
;
2449 for_each_sg(sglist
, sg
, nelems
, i
) {
2450 BUG_ON(!sg_page(sg
));
2451 sg
->dma_address
= page_to_phys(sg_page(sg
)) + sg
->offset
;
2452 sg
->dma_length
= sg
->length
;
2457 static int intel_map_sg(struct device
*hwdev
, struct scatterlist
*sglist
, int nelems
,
2458 enum dma_data_direction dir
, struct dma_attrs
*attrs
)
2462 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
2463 struct dmar_domain
*domain
;
2467 struct iova
*iova
= NULL
;
2469 struct scatterlist
*sg
;
2470 unsigned long start_addr
;
2471 struct intel_iommu
*iommu
;
2473 BUG_ON(dir
== DMA_NONE
);
2474 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
2475 return intel_nontranslate_map_sg(hwdev
, sglist
, nelems
, dir
);
2477 domain
= get_valid_domain_for_dev(pdev
);
2481 iommu
= domain_get_iommu(domain
);
2483 for_each_sg(sglist
, sg
, nelems
, i
) {
2484 addr
= page_to_phys(sg_page(sg
)) + sg
->offset
;
2485 size
+= aligned_size((u64
)addr
, sg
->length
);
2488 iova
= __intel_alloc_iova(hwdev
, domain
, size
, pdev
->dma_mask
);
2490 sglist
->dma_length
= 0;
2495 * Check if DMAR supports zero-length reads on write only
2498 if (dir
== DMA_TO_DEVICE
|| dir
== DMA_BIDIRECTIONAL
|| \
2499 !cap_zlr(iommu
->cap
))
2500 prot
|= DMA_PTE_READ
;
2501 if (dir
== DMA_FROM_DEVICE
|| dir
== DMA_BIDIRECTIONAL
)
2502 prot
|= DMA_PTE_WRITE
;
2504 start_addr
= iova
->pfn_lo
<< PAGE_SHIFT
;
2506 for_each_sg(sglist
, sg
, nelems
, i
) {
2507 addr
= page_to_phys(sg_page(sg
)) + sg
->offset
;
2508 size
= aligned_size((u64
)addr
, sg
->length
);
2509 ret
= domain_page_mapping(domain
, start_addr
+ offset
,
2510 ((u64
)addr
) & PAGE_MASK
,
2513 /* clear the page */
2514 dma_pte_clear_range(domain
, start_addr
,
2515 start_addr
+ offset
);
2516 /* free page tables */
2517 dma_pte_free_pagetable(domain
, start_addr
,
2518 start_addr
+ offset
);
2520 __free_iova(&domain
->iovad
, iova
);
2523 sg
->dma_address
= start_addr
+ offset
+
2524 ((u64
)addr
& (~PAGE_MASK
));
2525 sg
->dma_length
= sg
->length
;
2529 /* it's a non-present to present mapping. Only flush if caching mode */
2530 if (cap_caching_mode(iommu
->cap
))
2531 iommu_flush_iotlb_psi(iommu
, 0, start_addr
,
2532 offset
>> VTD_PAGE_SHIFT
);
2534 iommu_flush_write_buffer(iommu
);
2539 static int intel_mapping_error(struct device
*dev
, dma_addr_t dma_addr
)
2544 struct dma_map_ops intel_dma_ops
= {
2545 .alloc_coherent
= intel_alloc_coherent
,
2546 .free_coherent
= intel_free_coherent
,
2547 .map_sg
= intel_map_sg
,
2548 .unmap_sg
= intel_unmap_sg
,
2549 .map_page
= intel_map_page
,
2550 .unmap_page
= intel_unmap_page
,
2551 .mapping_error
= intel_mapping_error
,
2554 static inline int iommu_domain_cache_init(void)
2558 iommu_domain_cache
= kmem_cache_create("iommu_domain",
2559 sizeof(struct dmar_domain
),
2564 if (!iommu_domain_cache
) {
2565 printk(KERN_ERR
"Couldn't create iommu_domain cache\n");
2572 static inline int iommu_devinfo_cache_init(void)
2576 iommu_devinfo_cache
= kmem_cache_create("iommu_devinfo",
2577 sizeof(struct device_domain_info
),
2581 if (!iommu_devinfo_cache
) {
2582 printk(KERN_ERR
"Couldn't create devinfo cache\n");
2589 static inline int iommu_iova_cache_init(void)
2593 iommu_iova_cache
= kmem_cache_create("iommu_iova",
2594 sizeof(struct iova
),
2598 if (!iommu_iova_cache
) {
2599 printk(KERN_ERR
"Couldn't create iova cache\n");
2606 static int __init
iommu_init_mempool(void)
2609 ret
= iommu_iova_cache_init();
2613 ret
= iommu_domain_cache_init();
2617 ret
= iommu_devinfo_cache_init();
2621 kmem_cache_destroy(iommu_domain_cache
);
2623 kmem_cache_destroy(iommu_iova_cache
);
2628 static void __init
iommu_exit_mempool(void)
2630 kmem_cache_destroy(iommu_devinfo_cache
);
2631 kmem_cache_destroy(iommu_domain_cache
);
2632 kmem_cache_destroy(iommu_iova_cache
);
2636 static void __init
init_no_remapping_devices(void)
2638 struct dmar_drhd_unit
*drhd
;
2640 for_each_drhd_unit(drhd
) {
2641 if (!drhd
->include_all
) {
2643 for (i
= 0; i
< drhd
->devices_cnt
; i
++)
2644 if (drhd
->devices
[i
] != NULL
)
2646 /* ignore DMAR unit if no pci devices exist */
2647 if (i
== drhd
->devices_cnt
)
2655 for_each_drhd_unit(drhd
) {
2657 if (drhd
->ignored
|| drhd
->include_all
)
2660 for (i
= 0; i
< drhd
->devices_cnt
; i
++)
2661 if (drhd
->devices
[i
] &&
2662 !IS_GFX_DEVICE(drhd
->devices
[i
]))
2665 if (i
< drhd
->devices_cnt
)
2668 /* bypass IOMMU if it is just for gfx devices */
2670 for (i
= 0; i
< drhd
->devices_cnt
; i
++) {
2671 if (!drhd
->devices
[i
])
2673 drhd
->devices
[i
]->dev
.archdata
.iommu
= DUMMY_DEVICE_DOMAIN_INFO
;
2678 #ifdef CONFIG_SUSPEND
2679 static int init_iommu_hw(void)
2681 struct dmar_drhd_unit
*drhd
;
2682 struct intel_iommu
*iommu
= NULL
;
2684 for_each_active_iommu(iommu
, drhd
)
2686 dmar_reenable_qi(iommu
);
2688 for_each_active_iommu(iommu
, drhd
) {
2689 iommu_flush_write_buffer(iommu
);
2691 iommu_set_root_entry(iommu
);
2693 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
2694 DMA_CCMD_GLOBAL_INVL
);
2695 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
2696 DMA_TLB_GLOBAL_FLUSH
);
2697 iommu_disable_protect_mem_regions(iommu
);
2698 iommu_enable_translation(iommu
);
2704 static void iommu_flush_all(void)
2706 struct dmar_drhd_unit
*drhd
;
2707 struct intel_iommu
*iommu
;
2709 for_each_active_iommu(iommu
, drhd
) {
2710 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
2711 DMA_CCMD_GLOBAL_INVL
);
2712 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
2713 DMA_TLB_GLOBAL_FLUSH
);
2717 static int iommu_suspend(struct sys_device
*dev
, pm_message_t state
)
2719 struct dmar_drhd_unit
*drhd
;
2720 struct intel_iommu
*iommu
= NULL
;
2723 for_each_active_iommu(iommu
, drhd
) {
2724 iommu
->iommu_state
= kzalloc(sizeof(u32
) * MAX_SR_DMAR_REGS
,
2726 if (!iommu
->iommu_state
)
2732 for_each_active_iommu(iommu
, drhd
) {
2733 iommu_disable_translation(iommu
);
2735 spin_lock_irqsave(&iommu
->register_lock
, flag
);
2737 iommu
->iommu_state
[SR_DMAR_FECTL_REG
] =
2738 readl(iommu
->reg
+ DMAR_FECTL_REG
);
2739 iommu
->iommu_state
[SR_DMAR_FEDATA_REG
] =
2740 readl(iommu
->reg
+ DMAR_FEDATA_REG
);
2741 iommu
->iommu_state
[SR_DMAR_FEADDR_REG
] =
2742 readl(iommu
->reg
+ DMAR_FEADDR_REG
);
2743 iommu
->iommu_state
[SR_DMAR_FEUADDR_REG
] =
2744 readl(iommu
->reg
+ DMAR_FEUADDR_REG
);
2746 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
2751 for_each_active_iommu(iommu
, drhd
)
2752 kfree(iommu
->iommu_state
);
2757 static int iommu_resume(struct sys_device
*dev
)
2759 struct dmar_drhd_unit
*drhd
;
2760 struct intel_iommu
*iommu
= NULL
;
2763 if (init_iommu_hw()) {
2764 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
2768 for_each_active_iommu(iommu
, drhd
) {
2770 spin_lock_irqsave(&iommu
->register_lock
, flag
);
2772 writel(iommu
->iommu_state
[SR_DMAR_FECTL_REG
],
2773 iommu
->reg
+ DMAR_FECTL_REG
);
2774 writel(iommu
->iommu_state
[SR_DMAR_FEDATA_REG
],
2775 iommu
->reg
+ DMAR_FEDATA_REG
);
2776 writel(iommu
->iommu_state
[SR_DMAR_FEADDR_REG
],
2777 iommu
->reg
+ DMAR_FEADDR_REG
);
2778 writel(iommu
->iommu_state
[SR_DMAR_FEUADDR_REG
],
2779 iommu
->reg
+ DMAR_FEUADDR_REG
);
2781 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
2784 for_each_active_iommu(iommu
, drhd
)
2785 kfree(iommu
->iommu_state
);
2790 static struct sysdev_class iommu_sysclass
= {
2792 .resume
= iommu_resume
,
2793 .suspend
= iommu_suspend
,
2796 static struct sys_device device_iommu
= {
2797 .cls
= &iommu_sysclass
,
2800 static int __init
init_iommu_sysfs(void)
2804 error
= sysdev_class_register(&iommu_sysclass
);
2808 error
= sysdev_register(&device_iommu
);
2810 sysdev_class_unregister(&iommu_sysclass
);
2816 static int __init
init_iommu_sysfs(void)
2820 #endif /* CONFIG_PM */
2822 int __init
intel_iommu_init(void)
2826 if (dmar_table_init())
2829 if (dmar_dev_scope_init())
2833 * Check the need for DMA-remapping initialization now.
2834 * Above initialization will also be used by Interrupt-remapping.
2836 if (no_iommu
|| (swiotlb
&& !iommu_pass_through
) || dmar_disabled
)
2839 iommu_init_mempool();
2840 dmar_init_reserved_ranges();
2842 init_no_remapping_devices();
2846 printk(KERN_ERR
"IOMMU: dmar init failed\n");
2847 put_iova_domain(&reserved_iova_list
);
2848 iommu_exit_mempool();
2852 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2854 init_timer(&unmap_timer
);
2857 if (!iommu_pass_through
) {
2859 "Multi-level page-table translation for DMAR.\n");
2860 dma_ops
= &intel_dma_ops
;
2863 "DMAR: Pass through translation for DMAR.\n");
2867 register_iommu(&intel_iommu_ops
);
2872 static int vm_domain_add_dev_info(struct dmar_domain
*domain
,
2873 struct pci_dev
*pdev
)
2875 struct device_domain_info
*info
;
2876 unsigned long flags
;
2878 info
= alloc_devinfo_mem();
2882 info
->segment
= pci_domain_nr(pdev
->bus
);
2883 info
->bus
= pdev
->bus
->number
;
2884 info
->devfn
= pdev
->devfn
;
2886 info
->domain
= domain
;
2888 spin_lock_irqsave(&device_domain_lock
, flags
);
2889 list_add(&info
->link
, &domain
->devices
);
2890 list_add(&info
->global
, &device_domain_list
);
2891 pdev
->dev
.archdata
.iommu
= info
;
2892 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2897 static void iommu_detach_dependent_devices(struct intel_iommu
*iommu
,
2898 struct pci_dev
*pdev
)
2900 struct pci_dev
*tmp
, *parent
;
2902 if (!iommu
|| !pdev
)
2905 /* dependent device detach */
2906 tmp
= pci_find_upstream_pcie_bridge(pdev
);
2907 /* Secondary interface's bus number and devfn 0 */
2909 parent
= pdev
->bus
->self
;
2910 while (parent
!= tmp
) {
2911 iommu_detach_dev(iommu
, parent
->bus
->number
,
2913 parent
= parent
->bus
->self
;
2915 if (tmp
->is_pcie
) /* this is a PCIE-to-PCI bridge */
2916 iommu_detach_dev(iommu
,
2917 tmp
->subordinate
->number
, 0);
2918 else /* this is a legacy PCI bridge */
2919 iommu_detach_dev(iommu
, tmp
->bus
->number
,
2924 static void vm_domain_remove_one_dev_info(struct dmar_domain
*domain
,
2925 struct pci_dev
*pdev
)
2927 struct device_domain_info
*info
;
2928 struct intel_iommu
*iommu
;
2929 unsigned long flags
;
2931 struct list_head
*entry
, *tmp
;
2933 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
), pdev
->bus
->number
,
2938 spin_lock_irqsave(&device_domain_lock
, flags
);
2939 list_for_each_safe(entry
, tmp
, &domain
->devices
) {
2940 info
= list_entry(entry
, struct device_domain_info
, link
);
2941 /* No need to compare PCI domain; it has to be the same */
2942 if (info
->bus
== pdev
->bus
->number
&&
2943 info
->devfn
== pdev
->devfn
) {
2944 list_del(&info
->link
);
2945 list_del(&info
->global
);
2947 info
->dev
->dev
.archdata
.iommu
= NULL
;
2948 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2950 iommu_detach_dev(iommu
, info
->bus
, info
->devfn
);
2951 iommu_detach_dependent_devices(iommu
, pdev
);
2952 free_devinfo_mem(info
);
2954 spin_lock_irqsave(&device_domain_lock
, flags
);
2962 /* if there is no other devices under the same iommu
2963 * owned by this domain, clear this iommu in iommu_bmp
2964 * update iommu count and coherency
2966 if (iommu
== device_to_iommu(info
->segment
, info
->bus
,
2972 unsigned long tmp_flags
;
2973 spin_lock_irqsave(&domain
->iommu_lock
, tmp_flags
);
2974 clear_bit(iommu
->seq_id
, &domain
->iommu_bmp
);
2975 domain
->iommu_count
--;
2976 domain_update_iommu_cap(domain
);
2977 spin_unlock_irqrestore(&domain
->iommu_lock
, tmp_flags
);
2980 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2983 static void vm_domain_remove_all_dev_info(struct dmar_domain
*domain
)
2985 struct device_domain_info
*info
;
2986 struct intel_iommu
*iommu
;
2987 unsigned long flags1
, flags2
;
2989 spin_lock_irqsave(&device_domain_lock
, flags1
);
2990 while (!list_empty(&domain
->devices
)) {
2991 info
= list_entry(domain
->devices
.next
,
2992 struct device_domain_info
, link
);
2993 list_del(&info
->link
);
2994 list_del(&info
->global
);
2996 info
->dev
->dev
.archdata
.iommu
= NULL
;
2998 spin_unlock_irqrestore(&device_domain_lock
, flags1
);
3000 iommu
= device_to_iommu(info
->segment
, info
->bus
, info
->devfn
);
3001 iommu_detach_dev(iommu
, info
->bus
, info
->devfn
);
3002 iommu_detach_dependent_devices(iommu
, info
->dev
);
3004 /* clear this iommu in iommu_bmp, update iommu count
3007 spin_lock_irqsave(&domain
->iommu_lock
, flags2
);
3008 if (test_and_clear_bit(iommu
->seq_id
,
3009 &domain
->iommu_bmp
)) {
3010 domain
->iommu_count
--;
3011 domain_update_iommu_cap(domain
);
3013 spin_unlock_irqrestore(&domain
->iommu_lock
, flags2
);
3015 free_devinfo_mem(info
);
3016 spin_lock_irqsave(&device_domain_lock
, flags1
);
3018 spin_unlock_irqrestore(&device_domain_lock
, flags1
);
3021 /* domain id for virtual machine, it won't be set in context */
3022 static unsigned long vm_domid
;
3024 static int vm_domain_min_agaw(struct dmar_domain
*domain
)
3027 int min_agaw
= domain
->agaw
;
3029 i
= find_first_bit(&domain
->iommu_bmp
, g_num_of_iommus
);
3030 for (; i
< g_num_of_iommus
; ) {
3031 if (min_agaw
> g_iommus
[i
]->agaw
)
3032 min_agaw
= g_iommus
[i
]->agaw
;
3034 i
= find_next_bit(&domain
->iommu_bmp
, g_num_of_iommus
, i
+1);
3040 static struct dmar_domain
*iommu_alloc_vm_domain(void)
3042 struct dmar_domain
*domain
;
3044 domain
= alloc_domain_mem();
3048 domain
->id
= vm_domid
++;
3049 memset(&domain
->iommu_bmp
, 0, sizeof(unsigned long));
3050 domain
->flags
= DOMAIN_FLAG_VIRTUAL_MACHINE
;
3055 static int vm_domain_init(struct dmar_domain
*domain
, int guest_width
)
3059 init_iova_domain(&domain
->iovad
, DMA_32BIT_PFN
);
3060 spin_lock_init(&domain
->mapping_lock
);
3061 spin_lock_init(&domain
->iommu_lock
);
3063 domain_reserve_special_ranges(domain
);
3065 /* calculate AGAW */
3066 domain
->gaw
= guest_width
;
3067 adjust_width
= guestwidth_to_adjustwidth(guest_width
);
3068 domain
->agaw
= width_to_agaw(adjust_width
);
3070 INIT_LIST_HEAD(&domain
->devices
);
3072 domain
->iommu_count
= 0;
3073 domain
->iommu_coherency
= 0;
3074 domain
->max_addr
= 0;
3076 /* always allocate the top pgd */
3077 domain
->pgd
= (struct dma_pte
*)alloc_pgtable_page();
3080 domain_flush_cache(domain
, domain
->pgd
, PAGE_SIZE
);
3084 static void iommu_free_vm_domain(struct dmar_domain
*domain
)
3086 unsigned long flags
;
3087 struct dmar_drhd_unit
*drhd
;
3088 struct intel_iommu
*iommu
;
3090 unsigned long ndomains
;
3092 for_each_drhd_unit(drhd
) {
3095 iommu
= drhd
->iommu
;
3097 ndomains
= cap_ndoms(iommu
->cap
);
3098 i
= find_first_bit(iommu
->domain_ids
, ndomains
);
3099 for (; i
< ndomains
; ) {
3100 if (iommu
->domains
[i
] == domain
) {
3101 spin_lock_irqsave(&iommu
->lock
, flags
);
3102 clear_bit(i
, iommu
->domain_ids
);
3103 iommu
->domains
[i
] = NULL
;
3104 spin_unlock_irqrestore(&iommu
->lock
, flags
);
3107 i
= find_next_bit(iommu
->domain_ids
, ndomains
, i
+1);
3112 static void vm_domain_exit(struct dmar_domain
*domain
)
3116 /* Domain 0 is reserved, so dont process it */
3120 vm_domain_remove_all_dev_info(domain
);
3122 put_iova_domain(&domain
->iovad
);
3123 end
= DOMAIN_MAX_ADDR(domain
->gaw
);
3124 end
= end
& (~VTD_PAGE_MASK
);
3127 dma_pte_clear_range(domain
, 0, end
);
3129 /* free page tables */
3130 dma_pte_free_pagetable(domain
, 0, end
);
3132 iommu_free_vm_domain(domain
);
3133 free_domain_mem(domain
);
3136 static int intel_iommu_domain_init(struct iommu_domain
*domain
)
3138 struct dmar_domain
*dmar_domain
;
3140 dmar_domain
= iommu_alloc_vm_domain();
3143 "intel_iommu_domain_init: dmar_domain == NULL\n");
3146 if (vm_domain_init(dmar_domain
, DEFAULT_DOMAIN_ADDRESS_WIDTH
)) {
3148 "intel_iommu_domain_init() failed\n");
3149 vm_domain_exit(dmar_domain
);
3152 domain
->priv
= dmar_domain
;
3157 static void intel_iommu_domain_destroy(struct iommu_domain
*domain
)
3159 struct dmar_domain
*dmar_domain
= domain
->priv
;
3161 domain
->priv
= NULL
;
3162 vm_domain_exit(dmar_domain
);
3165 static int intel_iommu_attach_device(struct iommu_domain
*domain
,
3168 struct dmar_domain
*dmar_domain
= domain
->priv
;
3169 struct pci_dev
*pdev
= to_pci_dev(dev
);
3170 struct intel_iommu
*iommu
;
3175 /* normally pdev is not mapped */
3176 if (unlikely(domain_context_mapped(pdev
))) {
3177 struct dmar_domain
*old_domain
;
3179 old_domain
= find_domain(pdev
);
3181 if (dmar_domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
)
3182 vm_domain_remove_one_dev_info(old_domain
, pdev
);
3184 domain_remove_dev_info(old_domain
);
3188 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
), pdev
->bus
->number
,
3193 /* check if this iommu agaw is sufficient for max mapped address */
3194 addr_width
= agaw_to_width(iommu
->agaw
);
3195 end
= DOMAIN_MAX_ADDR(addr_width
);
3196 end
= end
& VTD_PAGE_MASK
;
3197 if (end
< dmar_domain
->max_addr
) {
3198 printk(KERN_ERR
"%s: iommu agaw (%d) is not "
3199 "sufficient for the mapped address (%llx)\n",
3200 __func__
, iommu
->agaw
, dmar_domain
->max_addr
);
3204 ret
= domain_context_mapping(dmar_domain
, pdev
, CONTEXT_TT_MULTI_LEVEL
);
3208 ret
= vm_domain_add_dev_info(dmar_domain
, pdev
);
3212 static void intel_iommu_detach_device(struct iommu_domain
*domain
,
3215 struct dmar_domain
*dmar_domain
= domain
->priv
;
3216 struct pci_dev
*pdev
= to_pci_dev(dev
);
3218 vm_domain_remove_one_dev_info(dmar_domain
, pdev
);
3221 static int intel_iommu_map_range(struct iommu_domain
*domain
,
3222 unsigned long iova
, phys_addr_t hpa
,
3223 size_t size
, int iommu_prot
)
3225 struct dmar_domain
*dmar_domain
= domain
->priv
;
3231 if (iommu_prot
& IOMMU_READ
)
3232 prot
|= DMA_PTE_READ
;
3233 if (iommu_prot
& IOMMU_WRITE
)
3234 prot
|= DMA_PTE_WRITE
;
3235 if ((iommu_prot
& IOMMU_CACHE
) && dmar_domain
->iommu_snooping
)
3236 prot
|= DMA_PTE_SNP
;
3238 max_addr
= (iova
& VTD_PAGE_MASK
) + VTD_PAGE_ALIGN(size
);
3239 if (dmar_domain
->max_addr
< max_addr
) {
3243 /* check if minimum agaw is sufficient for mapped address */
3244 min_agaw
= vm_domain_min_agaw(dmar_domain
);
3245 addr_width
= agaw_to_width(min_agaw
);
3246 end
= DOMAIN_MAX_ADDR(addr_width
);
3247 end
= end
& VTD_PAGE_MASK
;
3248 if (end
< max_addr
) {
3249 printk(KERN_ERR
"%s: iommu agaw (%d) is not "
3250 "sufficient for the mapped address (%llx)\n",
3251 __func__
, min_agaw
, max_addr
);
3254 dmar_domain
->max_addr
= max_addr
;
3257 ret
= domain_page_mapping(dmar_domain
, iova
, hpa
, size
, prot
);
3261 static void intel_iommu_unmap_range(struct iommu_domain
*domain
,
3262 unsigned long iova
, size_t size
)
3264 struct dmar_domain
*dmar_domain
= domain
->priv
;
3267 /* The address might not be aligned */
3268 base
= iova
& VTD_PAGE_MASK
;
3269 size
= VTD_PAGE_ALIGN(size
);
3270 dma_pte_clear_range(dmar_domain
, base
, base
+ size
);
3272 if (dmar_domain
->max_addr
== base
+ size
)
3273 dmar_domain
->max_addr
= base
;
3276 static phys_addr_t
intel_iommu_iova_to_phys(struct iommu_domain
*domain
,
3279 struct dmar_domain
*dmar_domain
= domain
->priv
;
3280 struct dma_pte
*pte
;
3283 pte
= addr_to_dma_pte(dmar_domain
, iova
);
3285 phys
= dma_pte_addr(pte
);
3290 static int intel_iommu_domain_has_cap(struct iommu_domain
*domain
,
3293 struct dmar_domain
*dmar_domain
= domain
->priv
;
3295 if (cap
== IOMMU_CAP_CACHE_COHERENCY
)
3296 return dmar_domain
->iommu_snooping
;
3301 static struct iommu_ops intel_iommu_ops
= {
3302 .domain_init
= intel_iommu_domain_init
,
3303 .domain_destroy
= intel_iommu_domain_destroy
,
3304 .attach_dev
= intel_iommu_attach_device
,
3305 .detach_dev
= intel_iommu_detach_device
,
3306 .map
= intel_iommu_map_range
,
3307 .unmap
= intel_iommu_unmap_range
,
3308 .iova_to_phys
= intel_iommu_iova_to_phys
,
3309 .domain_has_cap
= intel_iommu_domain_has_cap
,
3312 static void __devinit
quirk_iommu_rwbf(struct pci_dev
*dev
)
3315 * Mobile 4 Series Chipset neglects to set RWBF capability,
3318 printk(KERN_INFO
"DMAR: Forcing write-buffer flush capability\n");
3322 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2a40, quirk_iommu_rwbf
);