2 * Copyright (c) 2006, Intel Corporation.
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/iommu.h>
38 #include <linux/intel-iommu.h>
39 #include <linux/syscore_ops.h>
40 #include <linux/tboot.h>
41 #include <linux/dmi.h>
42 #include <asm/cacheflush.h>
43 #include <asm/iommu.h>
46 #define ROOT_SIZE VTD_PAGE_SIZE
47 #define CONTEXT_SIZE VTD_PAGE_SIZE
49 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
50 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
51 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
53 #define IOAPIC_RANGE_START (0xfee00000)
54 #define IOAPIC_RANGE_END (0xfeefffff)
55 #define IOVA_START_ADDR (0x1000)
57 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
59 #define MAX_AGAW_WIDTH 64
61 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
62 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
64 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
65 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
66 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
67 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
68 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
70 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
71 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
72 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
74 /* page table handling */
75 #define LEVEL_STRIDE (9)
76 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
78 static inline int agaw_to_level(int agaw
)
83 static inline int agaw_to_width(int agaw
)
85 return 30 + agaw
* LEVEL_STRIDE
;
88 static inline int width_to_agaw(int width
)
90 return (width
- 30) / LEVEL_STRIDE
;
93 static inline unsigned int level_to_offset_bits(int level
)
95 return (level
- 1) * LEVEL_STRIDE
;
98 static inline int pfn_level_offset(unsigned long pfn
, int level
)
100 return (pfn
>> level_to_offset_bits(level
)) & LEVEL_MASK
;
103 static inline unsigned long level_mask(int level
)
105 return -1UL << level_to_offset_bits(level
);
108 static inline unsigned long level_size(int level
)
110 return 1UL << level_to_offset_bits(level
);
113 static inline unsigned long align_to_level(unsigned long pfn
, int level
)
115 return (pfn
+ level_size(level
) - 1) & level_mask(level
);
118 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
119 are never going to work. */
120 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn
)
122 return dma_pfn
>> (PAGE_SHIFT
- VTD_PAGE_SHIFT
);
125 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn
)
127 return mm_pfn
<< (PAGE_SHIFT
- VTD_PAGE_SHIFT
);
129 static inline unsigned long page_to_dma_pfn(struct page
*pg
)
131 return mm_to_dma_pfn(page_to_pfn(pg
));
133 static inline unsigned long virt_to_dma_pfn(void *p
)
135 return page_to_dma_pfn(virt_to_page(p
));
138 /* global iommu list, set NULL for ignored DMAR units */
139 static struct intel_iommu
**g_iommus
;
141 static void __init
check_tylersburg_isoch(void);
142 static int rwbf_quirk
;
147 * 12-63: Context Ptr (12 - (haw-1))
154 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
155 static inline bool root_present(struct root_entry
*root
)
157 return (root
->val
& 1);
159 static inline void set_root_present(struct root_entry
*root
)
163 static inline void set_root_value(struct root_entry
*root
, unsigned long value
)
165 root
->val
|= value
& VTD_PAGE_MASK
;
168 static inline struct context_entry
*
169 get_context_addr_from_root(struct root_entry
*root
)
171 return (struct context_entry
*)
172 (root_present(root
)?phys_to_virt(
173 root
->val
& VTD_PAGE_MASK
) :
180 * 1: fault processing disable
181 * 2-3: translation type
182 * 12-63: address space root
188 struct context_entry
{
193 static inline bool context_present(struct context_entry
*context
)
195 return (context
->lo
& 1);
197 static inline void context_set_present(struct context_entry
*context
)
202 static inline void context_set_fault_enable(struct context_entry
*context
)
204 context
->lo
&= (((u64
)-1) << 2) | 1;
207 static inline void context_set_translation_type(struct context_entry
*context
,
210 context
->lo
&= (((u64
)-1) << 4) | 3;
211 context
->lo
|= (value
& 3) << 2;
214 static inline void context_set_address_root(struct context_entry
*context
,
217 context
->lo
|= value
& VTD_PAGE_MASK
;
220 static inline void context_set_address_width(struct context_entry
*context
,
223 context
->hi
|= value
& 7;
226 static inline void context_set_domain_id(struct context_entry
*context
,
229 context
->hi
|= (value
& ((1 << 16) - 1)) << 8;
232 static inline void context_clear_entry(struct context_entry
*context
)
245 * 12-63: Host physcial address
251 static inline void dma_clear_pte(struct dma_pte
*pte
)
256 static inline void dma_set_pte_readable(struct dma_pte
*pte
)
258 pte
->val
|= DMA_PTE_READ
;
261 static inline void dma_set_pte_writable(struct dma_pte
*pte
)
263 pte
->val
|= DMA_PTE_WRITE
;
266 static inline void dma_set_pte_snp(struct dma_pte
*pte
)
268 pte
->val
|= DMA_PTE_SNP
;
271 static inline void dma_set_pte_prot(struct dma_pte
*pte
, unsigned long prot
)
273 pte
->val
= (pte
->val
& ~3) | (prot
& 3);
276 static inline u64
dma_pte_addr(struct dma_pte
*pte
)
279 return pte
->val
& VTD_PAGE_MASK
;
281 /* Must have a full atomic 64-bit read */
282 return __cmpxchg64(&pte
->val
, 0ULL, 0ULL) & VTD_PAGE_MASK
;
286 static inline void dma_set_pte_pfn(struct dma_pte
*pte
, unsigned long pfn
)
288 pte
->val
|= (uint64_t)pfn
<< VTD_PAGE_SHIFT
;
291 static inline bool dma_pte_present(struct dma_pte
*pte
)
293 return (pte
->val
& 3) != 0;
296 static inline int first_pte_in_page(struct dma_pte
*pte
)
298 return !((unsigned long)pte
& ~VTD_PAGE_MASK
);
302 * This domain is a statically identity mapping domain.
303 * 1. This domain creats a static 1:1 mapping to all usable memory.
304 * 2. It maps to each iommu if successful.
305 * 3. Each iommu mapps to this domain if successful.
307 static struct dmar_domain
*si_domain
;
308 static int hw_pass_through
= 1;
310 /* devices under the same p2p bridge are owned in one domain */
311 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
313 /* domain represents a virtual machine, more than one devices
314 * across iommus may be owned in one domain, e.g. kvm guest.
316 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
318 /* si_domain contains mulitple devices */
319 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
322 int id
; /* domain id */
323 int nid
; /* node id */
324 unsigned long iommu_bmp
; /* bitmap of iommus this domain uses*/
326 struct list_head devices
; /* all devices' list */
327 struct iova_domain iovad
; /* iova's that belong to this domain */
329 struct dma_pte
*pgd
; /* virtual address */
330 int gaw
; /* max guest address width */
332 /* adjusted guest address width, 0 is level 2 30-bit */
335 int flags
; /* flags to find out type of domain */
337 int iommu_coherency
;/* indicate coherency of iommu access */
338 int iommu_snooping
; /* indicate snooping control feature*/
339 int iommu_count
; /* reference count of iommu */
340 spinlock_t iommu_lock
; /* protect iommu set in domain */
341 u64 max_addr
; /* maximum mapped address */
344 /* PCI domain-device relationship */
345 struct device_domain_info
{
346 struct list_head link
; /* link to domain siblings */
347 struct list_head global
; /* link to global list */
348 int segment
; /* PCI domain */
349 u8 bus
; /* PCI bus number */
350 u8 devfn
; /* PCI devfn number */
351 struct pci_dev
*dev
; /* it's NULL for PCIe-to-PCI bridge */
352 struct intel_iommu
*iommu
; /* IOMMU used by this device */
353 struct dmar_domain
*domain
; /* pointer to domain */
356 static void flush_unmaps_timeout(unsigned long data
);
358 DEFINE_TIMER(unmap_timer
, flush_unmaps_timeout
, 0, 0);
360 #define HIGH_WATER_MARK 250
361 struct deferred_flush_tables
{
363 struct iova
*iova
[HIGH_WATER_MARK
];
364 struct dmar_domain
*domain
[HIGH_WATER_MARK
];
367 static struct deferred_flush_tables
*deferred_flush
;
369 /* bitmap for indexing intel_iommus */
370 static int g_num_of_iommus
;
372 static DEFINE_SPINLOCK(async_umap_flush_lock
);
373 static LIST_HEAD(unmaps_to_do
);
376 static long list_size
;
378 static void domain_remove_dev_info(struct dmar_domain
*domain
);
380 #ifdef CONFIG_DMAR_DEFAULT_ON
381 int dmar_disabled
= 0;
383 int dmar_disabled
= 1;
384 #endif /*CONFIG_DMAR_DEFAULT_ON*/
386 static int dmar_map_gfx
= 1;
387 static int dmar_forcedac
;
388 static int intel_iommu_strict
;
390 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
391 static DEFINE_SPINLOCK(device_domain_lock
);
392 static LIST_HEAD(device_domain_list
);
394 static struct iommu_ops intel_iommu_ops
;
396 static int __init
intel_iommu_setup(char *str
)
401 if (!strncmp(str
, "on", 2)) {
403 printk(KERN_INFO
"Intel-IOMMU: enabled\n");
404 } else if (!strncmp(str
, "off", 3)) {
406 printk(KERN_INFO
"Intel-IOMMU: disabled\n");
407 } else if (!strncmp(str
, "igfx_off", 8)) {
410 "Intel-IOMMU: disable GFX device mapping\n");
411 } else if (!strncmp(str
, "forcedac", 8)) {
413 "Intel-IOMMU: Forcing DAC for PCI devices\n");
415 } else if (!strncmp(str
, "strict", 6)) {
417 "Intel-IOMMU: disable batched IOTLB flush\n");
418 intel_iommu_strict
= 1;
421 str
+= strcspn(str
, ",");
427 __setup("intel_iommu=", intel_iommu_setup
);
429 static struct kmem_cache
*iommu_domain_cache
;
430 static struct kmem_cache
*iommu_devinfo_cache
;
431 static struct kmem_cache
*iommu_iova_cache
;
433 static inline void *alloc_pgtable_page(int node
)
438 page
= alloc_pages_node(node
, GFP_ATOMIC
| __GFP_ZERO
, 0);
440 vaddr
= page_address(page
);
444 static inline void free_pgtable_page(void *vaddr
)
446 free_page((unsigned long)vaddr
);
449 static inline void *alloc_domain_mem(void)
451 return kmem_cache_alloc(iommu_domain_cache
, GFP_ATOMIC
);
454 static void free_domain_mem(void *vaddr
)
456 kmem_cache_free(iommu_domain_cache
, vaddr
);
459 static inline void * alloc_devinfo_mem(void)
461 return kmem_cache_alloc(iommu_devinfo_cache
, GFP_ATOMIC
);
464 static inline void free_devinfo_mem(void *vaddr
)
466 kmem_cache_free(iommu_devinfo_cache
, vaddr
);
469 struct iova
*alloc_iova_mem(void)
471 return kmem_cache_alloc(iommu_iova_cache
, GFP_ATOMIC
);
474 void free_iova_mem(struct iova
*iova
)
476 kmem_cache_free(iommu_iova_cache
, iova
);
480 static int __iommu_calculate_agaw(struct intel_iommu
*iommu
, int max_gaw
)
485 sagaw
= cap_sagaw(iommu
->cap
);
486 for (agaw
= width_to_agaw(max_gaw
);
488 if (test_bit(agaw
, &sagaw
))
496 * Calculate max SAGAW for each iommu.
498 int iommu_calculate_max_sagaw(struct intel_iommu
*iommu
)
500 return __iommu_calculate_agaw(iommu
, MAX_AGAW_WIDTH
);
504 * calculate agaw for each iommu.
505 * "SAGAW" may be different across iommus, use a default agaw, and
506 * get a supported less agaw for iommus that don't support the default agaw.
508 int iommu_calculate_agaw(struct intel_iommu
*iommu
)
510 return __iommu_calculate_agaw(iommu
, DEFAULT_DOMAIN_ADDRESS_WIDTH
);
513 /* This functionin only returns single iommu in a domain */
514 static struct intel_iommu
*domain_get_iommu(struct dmar_domain
*domain
)
518 /* si_domain and vm domain should not get here. */
519 BUG_ON(domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
);
520 BUG_ON(domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
);
522 iommu_id
= find_first_bit(&domain
->iommu_bmp
, g_num_of_iommus
);
523 if (iommu_id
< 0 || iommu_id
>= g_num_of_iommus
)
526 return g_iommus
[iommu_id
];
529 static void domain_update_iommu_coherency(struct dmar_domain
*domain
)
533 domain
->iommu_coherency
= 1;
535 for_each_set_bit(i
, &domain
->iommu_bmp
, g_num_of_iommus
) {
536 if (!ecap_coherent(g_iommus
[i
]->ecap
)) {
537 domain
->iommu_coherency
= 0;
543 static void domain_update_iommu_snooping(struct dmar_domain
*domain
)
547 domain
->iommu_snooping
= 1;
549 for_each_set_bit(i
, &domain
->iommu_bmp
, g_num_of_iommus
) {
550 if (!ecap_sc_support(g_iommus
[i
]->ecap
)) {
551 domain
->iommu_snooping
= 0;
557 /* Some capabilities may be different across iommus */
558 static void domain_update_iommu_cap(struct dmar_domain
*domain
)
560 domain_update_iommu_coherency(domain
);
561 domain_update_iommu_snooping(domain
);
564 static struct intel_iommu
*device_to_iommu(int segment
, u8 bus
, u8 devfn
)
566 struct dmar_drhd_unit
*drhd
= NULL
;
569 for_each_drhd_unit(drhd
) {
572 if (segment
!= drhd
->segment
)
575 for (i
= 0; i
< drhd
->devices_cnt
; i
++) {
576 if (drhd
->devices
[i
] &&
577 drhd
->devices
[i
]->bus
->number
== bus
&&
578 drhd
->devices
[i
]->devfn
== devfn
)
580 if (drhd
->devices
[i
] &&
581 drhd
->devices
[i
]->subordinate
&&
582 drhd
->devices
[i
]->subordinate
->number
<= bus
&&
583 drhd
->devices
[i
]->subordinate
->subordinate
>= bus
)
587 if (drhd
->include_all
)
594 static void domain_flush_cache(struct dmar_domain
*domain
,
595 void *addr
, int size
)
597 if (!domain
->iommu_coherency
)
598 clflush_cache_range(addr
, size
);
601 /* Gets context entry for a given bus and devfn */
602 static struct context_entry
* device_to_context_entry(struct intel_iommu
*iommu
,
605 struct root_entry
*root
;
606 struct context_entry
*context
;
607 unsigned long phy_addr
;
610 spin_lock_irqsave(&iommu
->lock
, flags
);
611 root
= &iommu
->root_entry
[bus
];
612 context
= get_context_addr_from_root(root
);
614 context
= (struct context_entry
*)
615 alloc_pgtable_page(iommu
->node
);
617 spin_unlock_irqrestore(&iommu
->lock
, flags
);
620 __iommu_flush_cache(iommu
, (void *)context
, CONTEXT_SIZE
);
621 phy_addr
= virt_to_phys((void *)context
);
622 set_root_value(root
, phy_addr
);
623 set_root_present(root
);
624 __iommu_flush_cache(iommu
, root
, sizeof(*root
));
626 spin_unlock_irqrestore(&iommu
->lock
, flags
);
627 return &context
[devfn
];
630 static int device_context_mapped(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
632 struct root_entry
*root
;
633 struct context_entry
*context
;
637 spin_lock_irqsave(&iommu
->lock
, flags
);
638 root
= &iommu
->root_entry
[bus
];
639 context
= get_context_addr_from_root(root
);
644 ret
= context_present(&context
[devfn
]);
646 spin_unlock_irqrestore(&iommu
->lock
, flags
);
650 static void clear_context_table(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
652 struct root_entry
*root
;
653 struct context_entry
*context
;
656 spin_lock_irqsave(&iommu
->lock
, flags
);
657 root
= &iommu
->root_entry
[bus
];
658 context
= get_context_addr_from_root(root
);
660 context_clear_entry(&context
[devfn
]);
661 __iommu_flush_cache(iommu
, &context
[devfn
], \
664 spin_unlock_irqrestore(&iommu
->lock
, flags
);
667 static void free_context_table(struct intel_iommu
*iommu
)
669 struct root_entry
*root
;
672 struct context_entry
*context
;
674 spin_lock_irqsave(&iommu
->lock
, flags
);
675 if (!iommu
->root_entry
) {
678 for (i
= 0; i
< ROOT_ENTRY_NR
; i
++) {
679 root
= &iommu
->root_entry
[i
];
680 context
= get_context_addr_from_root(root
);
682 free_pgtable_page(context
);
684 free_pgtable_page(iommu
->root_entry
);
685 iommu
->root_entry
= NULL
;
687 spin_unlock_irqrestore(&iommu
->lock
, flags
);
690 static struct dma_pte
*pfn_to_dma_pte(struct dmar_domain
*domain
,
693 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
694 struct dma_pte
*parent
, *pte
= NULL
;
695 int level
= agaw_to_level(domain
->agaw
);
698 BUG_ON(!domain
->pgd
);
699 BUG_ON(addr_width
< BITS_PER_LONG
&& pfn
>> addr_width
);
700 parent
= domain
->pgd
;
705 offset
= pfn_level_offset(pfn
, level
);
706 pte
= &parent
[offset
];
710 if (!dma_pte_present(pte
)) {
713 tmp_page
= alloc_pgtable_page(domain
->nid
);
718 domain_flush_cache(domain
, tmp_page
, VTD_PAGE_SIZE
);
719 pteval
= ((uint64_t)virt_to_dma_pfn(tmp_page
) << VTD_PAGE_SHIFT
) | DMA_PTE_READ
| DMA_PTE_WRITE
;
720 if (cmpxchg64(&pte
->val
, 0ULL, pteval
)) {
721 /* Someone else set it while we were thinking; use theirs. */
722 free_pgtable_page(tmp_page
);
725 domain_flush_cache(domain
, pte
, sizeof(*pte
));
728 parent
= phys_to_virt(dma_pte_addr(pte
));
735 /* return address's pte at specific level */
736 static struct dma_pte
*dma_pfn_level_pte(struct dmar_domain
*domain
,
740 struct dma_pte
*parent
, *pte
= NULL
;
741 int total
= agaw_to_level(domain
->agaw
);
744 parent
= domain
->pgd
;
745 while (level
<= total
) {
746 offset
= pfn_level_offset(pfn
, total
);
747 pte
= &parent
[offset
];
751 if (!dma_pte_present(pte
))
753 parent
= phys_to_virt(dma_pte_addr(pte
));
759 /* clear last level pte, a tlb flush should be followed */
760 static void dma_pte_clear_range(struct dmar_domain
*domain
,
761 unsigned long start_pfn
,
762 unsigned long last_pfn
)
764 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
765 struct dma_pte
*first_pte
, *pte
;
767 BUG_ON(addr_width
< BITS_PER_LONG
&& start_pfn
>> addr_width
);
768 BUG_ON(addr_width
< BITS_PER_LONG
&& last_pfn
>> addr_width
);
769 BUG_ON(start_pfn
> last_pfn
);
771 /* we don't need lock here; nobody else touches the iova range */
773 first_pte
= pte
= dma_pfn_level_pte(domain
, start_pfn
, 1);
775 start_pfn
= align_to_level(start_pfn
+ 1, 2);
782 } while (start_pfn
<= last_pfn
&& !first_pte_in_page(pte
));
784 domain_flush_cache(domain
, first_pte
,
785 (void *)pte
- (void *)first_pte
);
787 } while (start_pfn
&& start_pfn
<= last_pfn
);
790 /* free page table pages. last level pte should already be cleared */
791 static void dma_pte_free_pagetable(struct dmar_domain
*domain
,
792 unsigned long start_pfn
,
793 unsigned long last_pfn
)
795 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
796 struct dma_pte
*first_pte
, *pte
;
797 int total
= agaw_to_level(domain
->agaw
);
801 BUG_ON(addr_width
< BITS_PER_LONG
&& start_pfn
>> addr_width
);
802 BUG_ON(addr_width
< BITS_PER_LONG
&& last_pfn
>> addr_width
);
803 BUG_ON(start_pfn
> last_pfn
);
805 /* We don't need lock here; nobody else touches the iova range */
807 while (level
<= total
) {
808 tmp
= align_to_level(start_pfn
, level
);
810 /* If we can't even clear one PTE at this level, we're done */
811 if (tmp
+ level_size(level
) - 1 > last_pfn
)
815 first_pte
= pte
= dma_pfn_level_pte(domain
, tmp
, level
);
817 tmp
= align_to_level(tmp
+ 1, level
+ 1);
821 if (dma_pte_present(pte
)) {
822 free_pgtable_page(phys_to_virt(dma_pte_addr(pte
)));
826 tmp
+= level_size(level
);
827 } while (!first_pte_in_page(pte
) &&
828 tmp
+ level_size(level
) - 1 <= last_pfn
);
830 domain_flush_cache(domain
, first_pte
,
831 (void *)pte
- (void *)first_pte
);
833 } while (tmp
&& tmp
+ level_size(level
) - 1 <= last_pfn
);
837 if (start_pfn
== 0 && last_pfn
== DOMAIN_MAX_PFN(domain
->gaw
)) {
838 free_pgtable_page(domain
->pgd
);
844 static int iommu_alloc_root_entry(struct intel_iommu
*iommu
)
846 struct root_entry
*root
;
849 root
= (struct root_entry
*)alloc_pgtable_page(iommu
->node
);
853 __iommu_flush_cache(iommu
, root
, ROOT_SIZE
);
855 spin_lock_irqsave(&iommu
->lock
, flags
);
856 iommu
->root_entry
= root
;
857 spin_unlock_irqrestore(&iommu
->lock
, flags
);
862 static void iommu_set_root_entry(struct intel_iommu
*iommu
)
868 addr
= iommu
->root_entry
;
870 spin_lock_irqsave(&iommu
->register_lock
, flag
);
871 dmar_writeq(iommu
->reg
+ DMAR_RTADDR_REG
, virt_to_phys(addr
));
873 writel(iommu
->gcmd
| DMA_GCMD_SRTP
, iommu
->reg
+ DMAR_GCMD_REG
);
875 /* Make sure hardware complete it */
876 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
877 readl
, (sts
& DMA_GSTS_RTPS
), sts
);
879 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
882 static void iommu_flush_write_buffer(struct intel_iommu
*iommu
)
887 if (!rwbf_quirk
&& !cap_rwbf(iommu
->cap
))
890 spin_lock_irqsave(&iommu
->register_lock
, flag
);
891 writel(iommu
->gcmd
| DMA_GCMD_WBF
, iommu
->reg
+ DMAR_GCMD_REG
);
893 /* Make sure hardware complete it */
894 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
895 readl
, (!(val
& DMA_GSTS_WBFS
)), val
);
897 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
900 /* return value determine if we need a write buffer flush */
901 static void __iommu_flush_context(struct intel_iommu
*iommu
,
902 u16 did
, u16 source_id
, u8 function_mask
,
909 case DMA_CCMD_GLOBAL_INVL
:
910 val
= DMA_CCMD_GLOBAL_INVL
;
912 case DMA_CCMD_DOMAIN_INVL
:
913 val
= DMA_CCMD_DOMAIN_INVL
|DMA_CCMD_DID(did
);
915 case DMA_CCMD_DEVICE_INVL
:
916 val
= DMA_CCMD_DEVICE_INVL
|DMA_CCMD_DID(did
)
917 | DMA_CCMD_SID(source_id
) | DMA_CCMD_FM(function_mask
);
924 spin_lock_irqsave(&iommu
->register_lock
, flag
);
925 dmar_writeq(iommu
->reg
+ DMAR_CCMD_REG
, val
);
927 /* Make sure hardware complete it */
928 IOMMU_WAIT_OP(iommu
, DMAR_CCMD_REG
,
929 dmar_readq
, (!(val
& DMA_CCMD_ICC
)), val
);
931 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
934 /* return value determine if we need a write buffer flush */
935 static void __iommu_flush_iotlb(struct intel_iommu
*iommu
, u16 did
,
936 u64 addr
, unsigned int size_order
, u64 type
)
938 int tlb_offset
= ecap_iotlb_offset(iommu
->ecap
);
939 u64 val
= 0, val_iva
= 0;
943 case DMA_TLB_GLOBAL_FLUSH
:
944 /* global flush doesn't need set IVA_REG */
945 val
= DMA_TLB_GLOBAL_FLUSH
|DMA_TLB_IVT
;
947 case DMA_TLB_DSI_FLUSH
:
948 val
= DMA_TLB_DSI_FLUSH
|DMA_TLB_IVT
|DMA_TLB_DID(did
);
950 case DMA_TLB_PSI_FLUSH
:
951 val
= DMA_TLB_PSI_FLUSH
|DMA_TLB_IVT
|DMA_TLB_DID(did
);
952 /* Note: always flush non-leaf currently */
953 val_iva
= size_order
| addr
;
958 /* Note: set drain read/write */
961 * This is probably to be super secure.. Looks like we can
962 * ignore it without any impact.
964 if (cap_read_drain(iommu
->cap
))
965 val
|= DMA_TLB_READ_DRAIN
;
967 if (cap_write_drain(iommu
->cap
))
968 val
|= DMA_TLB_WRITE_DRAIN
;
970 spin_lock_irqsave(&iommu
->register_lock
, flag
);
971 /* Note: Only uses first TLB reg currently */
973 dmar_writeq(iommu
->reg
+ tlb_offset
, val_iva
);
974 dmar_writeq(iommu
->reg
+ tlb_offset
+ 8, val
);
976 /* Make sure hardware complete it */
977 IOMMU_WAIT_OP(iommu
, tlb_offset
+ 8,
978 dmar_readq
, (!(val
& DMA_TLB_IVT
)), val
);
980 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
982 /* check IOTLB invalidation granularity */
983 if (DMA_TLB_IAIG(val
) == 0)
984 printk(KERN_ERR
"IOMMU: flush IOTLB failed\n");
985 if (DMA_TLB_IAIG(val
) != DMA_TLB_IIRG(type
))
986 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
987 (unsigned long long)DMA_TLB_IIRG(type
),
988 (unsigned long long)DMA_TLB_IAIG(val
));
991 static struct device_domain_info
*iommu_support_dev_iotlb(
992 struct dmar_domain
*domain
, int segment
, u8 bus
, u8 devfn
)
996 struct device_domain_info
*info
;
997 struct intel_iommu
*iommu
= device_to_iommu(segment
, bus
, devfn
);
999 if (!ecap_dev_iotlb_support(iommu
->ecap
))
1005 spin_lock_irqsave(&device_domain_lock
, flags
);
1006 list_for_each_entry(info
, &domain
->devices
, link
)
1007 if (info
->bus
== bus
&& info
->devfn
== devfn
) {
1011 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1013 if (!found
|| !info
->dev
)
1016 if (!pci_find_ext_capability(info
->dev
, PCI_EXT_CAP_ID_ATS
))
1019 if (!dmar_find_matched_atsr_unit(info
->dev
))
1022 info
->iommu
= iommu
;
1027 static void iommu_enable_dev_iotlb(struct device_domain_info
*info
)
1032 pci_enable_ats(info
->dev
, VTD_PAGE_SHIFT
);
1035 static void iommu_disable_dev_iotlb(struct device_domain_info
*info
)
1037 if (!info
->dev
|| !pci_ats_enabled(info
->dev
))
1040 pci_disable_ats(info
->dev
);
1043 static void iommu_flush_dev_iotlb(struct dmar_domain
*domain
,
1044 u64 addr
, unsigned mask
)
1047 unsigned long flags
;
1048 struct device_domain_info
*info
;
1050 spin_lock_irqsave(&device_domain_lock
, flags
);
1051 list_for_each_entry(info
, &domain
->devices
, link
) {
1052 if (!info
->dev
|| !pci_ats_enabled(info
->dev
))
1055 sid
= info
->bus
<< 8 | info
->devfn
;
1056 qdep
= pci_ats_queue_depth(info
->dev
);
1057 qi_flush_dev_iotlb(info
->iommu
, sid
, qdep
, addr
, mask
);
1059 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1062 static void iommu_flush_iotlb_psi(struct intel_iommu
*iommu
, u16 did
,
1063 unsigned long pfn
, unsigned int pages
, int map
)
1065 unsigned int mask
= ilog2(__roundup_pow_of_two(pages
));
1066 uint64_t addr
= (uint64_t)pfn
<< VTD_PAGE_SHIFT
;
1071 * Fallback to domain selective flush if no PSI support or the size is
1073 * PSI requires page size to be 2 ^ x, and the base address is naturally
1074 * aligned to the size
1076 if (!cap_pgsel_inv(iommu
->cap
) || mask
> cap_max_amask_val(iommu
->cap
))
1077 iommu
->flush
.flush_iotlb(iommu
, did
, 0, 0,
1080 iommu
->flush
.flush_iotlb(iommu
, did
, addr
, mask
,
1084 * In caching mode, changes of pages from non-present to present require
1085 * flush. However, device IOTLB doesn't need to be flushed in this case.
1087 if (!cap_caching_mode(iommu
->cap
) || !map
)
1088 iommu_flush_dev_iotlb(iommu
->domains
[did
], addr
, mask
);
1091 static void iommu_disable_protect_mem_regions(struct intel_iommu
*iommu
)
1094 unsigned long flags
;
1096 spin_lock_irqsave(&iommu
->register_lock
, flags
);
1097 pmen
= readl(iommu
->reg
+ DMAR_PMEN_REG
);
1098 pmen
&= ~DMA_PMEN_EPM
;
1099 writel(pmen
, iommu
->reg
+ DMAR_PMEN_REG
);
1101 /* wait for the protected region status bit to clear */
1102 IOMMU_WAIT_OP(iommu
, DMAR_PMEN_REG
,
1103 readl
, !(pmen
& DMA_PMEN_PRS
), pmen
);
1105 spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
1108 static int iommu_enable_translation(struct intel_iommu
*iommu
)
1111 unsigned long flags
;
1113 spin_lock_irqsave(&iommu
->register_lock
, flags
);
1114 iommu
->gcmd
|= DMA_GCMD_TE
;
1115 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
1117 /* Make sure hardware complete it */
1118 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1119 readl
, (sts
& DMA_GSTS_TES
), sts
);
1121 spin_unlock_irqrestore(&iommu
->register_lock
, flags
);
1125 static int iommu_disable_translation(struct intel_iommu
*iommu
)
1130 spin_lock_irqsave(&iommu
->register_lock
, flag
);
1131 iommu
->gcmd
&= ~DMA_GCMD_TE
;
1132 writel(iommu
->gcmd
, iommu
->reg
+ DMAR_GCMD_REG
);
1134 /* Make sure hardware complete it */
1135 IOMMU_WAIT_OP(iommu
, DMAR_GSTS_REG
,
1136 readl
, (!(sts
& DMA_GSTS_TES
)), sts
);
1138 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
1143 static int iommu_init_domains(struct intel_iommu
*iommu
)
1145 unsigned long ndomains
;
1146 unsigned long nlongs
;
1148 ndomains
= cap_ndoms(iommu
->cap
);
1149 pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu
->seq_id
,
1151 nlongs
= BITS_TO_LONGS(ndomains
);
1153 spin_lock_init(&iommu
->lock
);
1155 /* TBD: there might be 64K domains,
1156 * consider other allocation for future chip
1158 iommu
->domain_ids
= kcalloc(nlongs
, sizeof(unsigned long), GFP_KERNEL
);
1159 if (!iommu
->domain_ids
) {
1160 printk(KERN_ERR
"Allocating domain id array failed\n");
1163 iommu
->domains
= kcalloc(ndomains
, sizeof(struct dmar_domain
*),
1165 if (!iommu
->domains
) {
1166 printk(KERN_ERR
"Allocating domain array failed\n");
1171 * if Caching mode is set, then invalid translations are tagged
1172 * with domainid 0. Hence we need to pre-allocate it.
1174 if (cap_caching_mode(iommu
->cap
))
1175 set_bit(0, iommu
->domain_ids
);
1180 static void domain_exit(struct dmar_domain
*domain
);
1181 static void vm_domain_exit(struct dmar_domain
*domain
);
1183 void free_dmar_iommu(struct intel_iommu
*iommu
)
1185 struct dmar_domain
*domain
;
1187 unsigned long flags
;
1189 if ((iommu
->domains
) && (iommu
->domain_ids
)) {
1190 for_each_set_bit(i
, iommu
->domain_ids
, cap_ndoms(iommu
->cap
)) {
1191 domain
= iommu
->domains
[i
];
1192 clear_bit(i
, iommu
->domain_ids
);
1194 spin_lock_irqsave(&domain
->iommu_lock
, flags
);
1195 if (--domain
->iommu_count
== 0) {
1196 if (domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
)
1197 vm_domain_exit(domain
);
1199 domain_exit(domain
);
1201 spin_unlock_irqrestore(&domain
->iommu_lock
, flags
);
1205 if (iommu
->gcmd
& DMA_GCMD_TE
)
1206 iommu_disable_translation(iommu
);
1209 irq_set_handler_data(iommu
->irq
, NULL
);
1210 /* This will mask the irq */
1211 free_irq(iommu
->irq
, iommu
);
1212 destroy_irq(iommu
->irq
);
1215 kfree(iommu
->domains
);
1216 kfree(iommu
->domain_ids
);
1218 g_iommus
[iommu
->seq_id
] = NULL
;
1220 /* if all iommus are freed, free g_iommus */
1221 for (i
= 0; i
< g_num_of_iommus
; i
++) {
1226 if (i
== g_num_of_iommus
)
1229 /* free context mapping */
1230 free_context_table(iommu
);
1233 static struct dmar_domain
*alloc_domain(void)
1235 struct dmar_domain
*domain
;
1237 domain
= alloc_domain_mem();
1242 memset(&domain
->iommu_bmp
, 0, sizeof(unsigned long));
1248 static int iommu_attach_domain(struct dmar_domain
*domain
,
1249 struct intel_iommu
*iommu
)
1252 unsigned long ndomains
;
1253 unsigned long flags
;
1255 ndomains
= cap_ndoms(iommu
->cap
);
1257 spin_lock_irqsave(&iommu
->lock
, flags
);
1259 num
= find_first_zero_bit(iommu
->domain_ids
, ndomains
);
1260 if (num
>= ndomains
) {
1261 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1262 printk(KERN_ERR
"IOMMU: no free domain ids\n");
1267 set_bit(num
, iommu
->domain_ids
);
1268 set_bit(iommu
->seq_id
, &domain
->iommu_bmp
);
1269 iommu
->domains
[num
] = domain
;
1270 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1275 static void iommu_detach_domain(struct dmar_domain
*domain
,
1276 struct intel_iommu
*iommu
)
1278 unsigned long flags
;
1282 spin_lock_irqsave(&iommu
->lock
, flags
);
1283 ndomains
= cap_ndoms(iommu
->cap
);
1284 for_each_set_bit(num
, iommu
->domain_ids
, ndomains
) {
1285 if (iommu
->domains
[num
] == domain
) {
1292 clear_bit(num
, iommu
->domain_ids
);
1293 clear_bit(iommu
->seq_id
, &domain
->iommu_bmp
);
1294 iommu
->domains
[num
] = NULL
;
1296 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1299 static struct iova_domain reserved_iova_list
;
1300 static struct lock_class_key reserved_rbtree_key
;
1302 static int dmar_init_reserved_ranges(void)
1304 struct pci_dev
*pdev
= NULL
;
1308 init_iova_domain(&reserved_iova_list
, DMA_32BIT_PFN
);
1310 lockdep_set_class(&reserved_iova_list
.iova_rbtree_lock
,
1311 &reserved_rbtree_key
);
1313 /* IOAPIC ranges shouldn't be accessed by DMA */
1314 iova
= reserve_iova(&reserved_iova_list
, IOVA_PFN(IOAPIC_RANGE_START
),
1315 IOVA_PFN(IOAPIC_RANGE_END
));
1317 printk(KERN_ERR
"Reserve IOAPIC range failed\n");
1321 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1322 for_each_pci_dev(pdev
) {
1325 for (i
= 0; i
< PCI_NUM_RESOURCES
; i
++) {
1326 r
= &pdev
->resource
[i
];
1327 if (!r
->flags
|| !(r
->flags
& IORESOURCE_MEM
))
1329 iova
= reserve_iova(&reserved_iova_list
,
1333 printk(KERN_ERR
"Reserve iova failed\n");
1341 static void domain_reserve_special_ranges(struct dmar_domain
*domain
)
1343 copy_reserved_iova(&reserved_iova_list
, &domain
->iovad
);
1346 static inline int guestwidth_to_adjustwidth(int gaw
)
1349 int r
= (gaw
- 12) % 9;
1360 static int domain_init(struct dmar_domain
*domain
, int guest_width
)
1362 struct intel_iommu
*iommu
;
1363 int adjust_width
, agaw
;
1364 unsigned long sagaw
;
1366 init_iova_domain(&domain
->iovad
, DMA_32BIT_PFN
);
1367 spin_lock_init(&domain
->iommu_lock
);
1369 domain_reserve_special_ranges(domain
);
1371 /* calculate AGAW */
1372 iommu
= domain_get_iommu(domain
);
1373 if (guest_width
> cap_mgaw(iommu
->cap
))
1374 guest_width
= cap_mgaw(iommu
->cap
);
1375 domain
->gaw
= guest_width
;
1376 adjust_width
= guestwidth_to_adjustwidth(guest_width
);
1377 agaw
= width_to_agaw(adjust_width
);
1378 sagaw
= cap_sagaw(iommu
->cap
);
1379 if (!test_bit(agaw
, &sagaw
)) {
1380 /* hardware doesn't support it, choose a bigger one */
1381 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw
);
1382 agaw
= find_next_bit(&sagaw
, 5, agaw
);
1386 domain
->agaw
= agaw
;
1387 INIT_LIST_HEAD(&domain
->devices
);
1389 if (ecap_coherent(iommu
->ecap
))
1390 domain
->iommu_coherency
= 1;
1392 domain
->iommu_coherency
= 0;
1394 if (ecap_sc_support(iommu
->ecap
))
1395 domain
->iommu_snooping
= 1;
1397 domain
->iommu_snooping
= 0;
1399 domain
->iommu_count
= 1;
1400 domain
->nid
= iommu
->node
;
1402 /* always allocate the top pgd */
1403 domain
->pgd
= (struct dma_pte
*)alloc_pgtable_page(domain
->nid
);
1406 __iommu_flush_cache(iommu
, domain
->pgd
, PAGE_SIZE
);
1410 static void domain_exit(struct dmar_domain
*domain
)
1412 struct dmar_drhd_unit
*drhd
;
1413 struct intel_iommu
*iommu
;
1415 /* Domain 0 is reserved, so dont process it */
1419 domain_remove_dev_info(domain
);
1421 put_iova_domain(&domain
->iovad
);
1424 dma_pte_clear_range(domain
, 0, DOMAIN_MAX_PFN(domain
->gaw
));
1426 /* free page tables */
1427 dma_pte_free_pagetable(domain
, 0, DOMAIN_MAX_PFN(domain
->gaw
));
1429 for_each_active_iommu(iommu
, drhd
)
1430 if (test_bit(iommu
->seq_id
, &domain
->iommu_bmp
))
1431 iommu_detach_domain(domain
, iommu
);
1433 free_domain_mem(domain
);
1436 static int domain_context_mapping_one(struct dmar_domain
*domain
, int segment
,
1437 u8 bus
, u8 devfn
, int translation
)
1439 struct context_entry
*context
;
1440 unsigned long flags
;
1441 struct intel_iommu
*iommu
;
1442 struct dma_pte
*pgd
;
1444 unsigned long ndomains
;
1447 struct device_domain_info
*info
= NULL
;
1449 pr_debug("Set context mapping for %02x:%02x.%d\n",
1450 bus
, PCI_SLOT(devfn
), PCI_FUNC(devfn
));
1452 BUG_ON(!domain
->pgd
);
1453 BUG_ON(translation
!= CONTEXT_TT_PASS_THROUGH
&&
1454 translation
!= CONTEXT_TT_MULTI_LEVEL
);
1456 iommu
= device_to_iommu(segment
, bus
, devfn
);
1460 context
= device_to_context_entry(iommu
, bus
, devfn
);
1463 spin_lock_irqsave(&iommu
->lock
, flags
);
1464 if (context_present(context
)) {
1465 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1472 if (domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
||
1473 domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
) {
1476 /* find an available domain id for this device in iommu */
1477 ndomains
= cap_ndoms(iommu
->cap
);
1478 for_each_set_bit(num
, iommu
->domain_ids
, ndomains
) {
1479 if (iommu
->domains
[num
] == domain
) {
1487 num
= find_first_zero_bit(iommu
->domain_ids
, ndomains
);
1488 if (num
>= ndomains
) {
1489 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1490 printk(KERN_ERR
"IOMMU: no free domain ids\n");
1494 set_bit(num
, iommu
->domain_ids
);
1495 iommu
->domains
[num
] = domain
;
1499 /* Skip top levels of page tables for
1500 * iommu which has less agaw than default.
1501 * Unnecessary for PT mode.
1503 if (translation
!= CONTEXT_TT_PASS_THROUGH
) {
1504 for (agaw
= domain
->agaw
; agaw
!= iommu
->agaw
; agaw
--) {
1505 pgd
= phys_to_virt(dma_pte_addr(pgd
));
1506 if (!dma_pte_present(pgd
)) {
1507 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1514 context_set_domain_id(context
, id
);
1516 if (translation
!= CONTEXT_TT_PASS_THROUGH
) {
1517 info
= iommu_support_dev_iotlb(domain
, segment
, bus
, devfn
);
1518 translation
= info
? CONTEXT_TT_DEV_IOTLB
:
1519 CONTEXT_TT_MULTI_LEVEL
;
1522 * In pass through mode, AW must be programmed to indicate the largest
1523 * AGAW value supported by hardware. And ASR is ignored by hardware.
1525 if (unlikely(translation
== CONTEXT_TT_PASS_THROUGH
))
1526 context_set_address_width(context
, iommu
->msagaw
);
1528 context_set_address_root(context
, virt_to_phys(pgd
));
1529 context_set_address_width(context
, iommu
->agaw
);
1532 context_set_translation_type(context
, translation
);
1533 context_set_fault_enable(context
);
1534 context_set_present(context
);
1535 domain_flush_cache(domain
, context
, sizeof(*context
));
1538 * It's a non-present to present mapping. If hardware doesn't cache
1539 * non-present entry we only need to flush the write-buffer. If the
1540 * _does_ cache non-present entries, then it does so in the special
1541 * domain #0, which we have to flush:
1543 if (cap_caching_mode(iommu
->cap
)) {
1544 iommu
->flush
.flush_context(iommu
, 0,
1545 (((u16
)bus
) << 8) | devfn
,
1546 DMA_CCMD_MASK_NOBIT
,
1547 DMA_CCMD_DEVICE_INVL
);
1548 iommu
->flush
.flush_iotlb(iommu
, domain
->id
, 0, 0, DMA_TLB_DSI_FLUSH
);
1550 iommu_flush_write_buffer(iommu
);
1552 iommu_enable_dev_iotlb(info
);
1553 spin_unlock_irqrestore(&iommu
->lock
, flags
);
1555 spin_lock_irqsave(&domain
->iommu_lock
, flags
);
1556 if (!test_and_set_bit(iommu
->seq_id
, &domain
->iommu_bmp
)) {
1557 domain
->iommu_count
++;
1558 if (domain
->iommu_count
== 1)
1559 domain
->nid
= iommu
->node
;
1560 domain_update_iommu_cap(domain
);
1562 spin_unlock_irqrestore(&domain
->iommu_lock
, flags
);
1567 domain_context_mapping(struct dmar_domain
*domain
, struct pci_dev
*pdev
,
1571 struct pci_dev
*tmp
, *parent
;
1573 ret
= domain_context_mapping_one(domain
, pci_domain_nr(pdev
->bus
),
1574 pdev
->bus
->number
, pdev
->devfn
,
1579 /* dependent device mapping */
1580 tmp
= pci_find_upstream_pcie_bridge(pdev
);
1583 /* Secondary interface's bus number and devfn 0 */
1584 parent
= pdev
->bus
->self
;
1585 while (parent
!= tmp
) {
1586 ret
= domain_context_mapping_one(domain
,
1587 pci_domain_nr(parent
->bus
),
1588 parent
->bus
->number
,
1589 parent
->devfn
, translation
);
1592 parent
= parent
->bus
->self
;
1594 if (pci_is_pcie(tmp
)) /* this is a PCIe-to-PCI bridge */
1595 return domain_context_mapping_one(domain
,
1596 pci_domain_nr(tmp
->subordinate
),
1597 tmp
->subordinate
->number
, 0,
1599 else /* this is a legacy PCI bridge */
1600 return domain_context_mapping_one(domain
,
1601 pci_domain_nr(tmp
->bus
),
1607 static int domain_context_mapped(struct pci_dev
*pdev
)
1610 struct pci_dev
*tmp
, *parent
;
1611 struct intel_iommu
*iommu
;
1613 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
), pdev
->bus
->number
,
1618 ret
= device_context_mapped(iommu
, pdev
->bus
->number
, pdev
->devfn
);
1621 /* dependent device mapping */
1622 tmp
= pci_find_upstream_pcie_bridge(pdev
);
1625 /* Secondary interface's bus number and devfn 0 */
1626 parent
= pdev
->bus
->self
;
1627 while (parent
!= tmp
) {
1628 ret
= device_context_mapped(iommu
, parent
->bus
->number
,
1632 parent
= parent
->bus
->self
;
1634 if (pci_is_pcie(tmp
))
1635 return device_context_mapped(iommu
, tmp
->subordinate
->number
,
1638 return device_context_mapped(iommu
, tmp
->bus
->number
,
1642 /* Returns a number of VTD pages, but aligned to MM page size */
1643 static inline unsigned long aligned_nrpages(unsigned long host_addr
,
1646 host_addr
&= ~PAGE_MASK
;
1647 return PAGE_ALIGN(host_addr
+ size
) >> VTD_PAGE_SHIFT
;
1650 static int __domain_mapping(struct dmar_domain
*domain
, unsigned long iov_pfn
,
1651 struct scatterlist
*sg
, unsigned long phys_pfn
,
1652 unsigned long nr_pages
, int prot
)
1654 struct dma_pte
*first_pte
= NULL
, *pte
= NULL
;
1655 phys_addr_t
uninitialized_var(pteval
);
1656 int addr_width
= agaw_to_width(domain
->agaw
) - VTD_PAGE_SHIFT
;
1657 unsigned long sg_res
;
1659 BUG_ON(addr_width
< BITS_PER_LONG
&& (iov_pfn
+ nr_pages
- 1) >> addr_width
);
1661 if ((prot
& (DMA_PTE_READ
|DMA_PTE_WRITE
)) == 0)
1664 prot
&= DMA_PTE_READ
| DMA_PTE_WRITE
| DMA_PTE_SNP
;
1669 sg_res
= nr_pages
+ 1;
1670 pteval
= ((phys_addr_t
)phys_pfn
<< VTD_PAGE_SHIFT
) | prot
;
1673 while (nr_pages
--) {
1677 sg_res
= aligned_nrpages(sg
->offset
, sg
->length
);
1678 sg
->dma_address
= ((dma_addr_t
)iov_pfn
<< VTD_PAGE_SHIFT
) + sg
->offset
;
1679 sg
->dma_length
= sg
->length
;
1680 pteval
= page_to_phys(sg_page(sg
)) | prot
;
1683 first_pte
= pte
= pfn_to_dma_pte(domain
, iov_pfn
);
1687 /* We don't need lock here, nobody else
1688 * touches the iova range
1690 tmp
= cmpxchg64_local(&pte
->val
, 0ULL, pteval
);
1692 static int dumps
= 5;
1693 printk(KERN_CRIT
"ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1694 iov_pfn
, tmp
, (unsigned long long)pteval
);
1697 debug_dma_dump_mappings(NULL
);
1702 if (!nr_pages
|| first_pte_in_page(pte
)) {
1703 domain_flush_cache(domain
, first_pte
,
1704 (void *)pte
- (void *)first_pte
);
1708 pteval
+= VTD_PAGE_SIZE
;
1716 static inline int domain_sg_mapping(struct dmar_domain
*domain
, unsigned long iov_pfn
,
1717 struct scatterlist
*sg
, unsigned long nr_pages
,
1720 return __domain_mapping(domain
, iov_pfn
, sg
, 0, nr_pages
, prot
);
1723 static inline int domain_pfn_mapping(struct dmar_domain
*domain
, unsigned long iov_pfn
,
1724 unsigned long phys_pfn
, unsigned long nr_pages
,
1727 return __domain_mapping(domain
, iov_pfn
, NULL
, phys_pfn
, nr_pages
, prot
);
1730 static void iommu_detach_dev(struct intel_iommu
*iommu
, u8 bus
, u8 devfn
)
1735 clear_context_table(iommu
, bus
, devfn
);
1736 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
1737 DMA_CCMD_GLOBAL_INVL
);
1738 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH
);
1741 static void domain_remove_dev_info(struct dmar_domain
*domain
)
1743 struct device_domain_info
*info
;
1744 unsigned long flags
;
1745 struct intel_iommu
*iommu
;
1747 spin_lock_irqsave(&device_domain_lock
, flags
);
1748 while (!list_empty(&domain
->devices
)) {
1749 info
= list_entry(domain
->devices
.next
,
1750 struct device_domain_info
, link
);
1751 list_del(&info
->link
);
1752 list_del(&info
->global
);
1754 info
->dev
->dev
.archdata
.iommu
= NULL
;
1755 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1757 iommu_disable_dev_iotlb(info
);
1758 iommu
= device_to_iommu(info
->segment
, info
->bus
, info
->devfn
);
1759 iommu_detach_dev(iommu
, info
->bus
, info
->devfn
);
1760 free_devinfo_mem(info
);
1762 spin_lock_irqsave(&device_domain_lock
, flags
);
1764 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1769 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1771 static struct dmar_domain
*
1772 find_domain(struct pci_dev
*pdev
)
1774 struct device_domain_info
*info
;
1776 /* No lock here, assumes no domain exit in normal case */
1777 info
= pdev
->dev
.archdata
.iommu
;
1779 return info
->domain
;
1783 /* domain is initialized */
1784 static struct dmar_domain
*get_domain_for_dev(struct pci_dev
*pdev
, int gaw
)
1786 struct dmar_domain
*domain
, *found
= NULL
;
1787 struct intel_iommu
*iommu
;
1788 struct dmar_drhd_unit
*drhd
;
1789 struct device_domain_info
*info
, *tmp
;
1790 struct pci_dev
*dev_tmp
;
1791 unsigned long flags
;
1792 int bus
= 0, devfn
= 0;
1796 domain
= find_domain(pdev
);
1800 segment
= pci_domain_nr(pdev
->bus
);
1802 dev_tmp
= pci_find_upstream_pcie_bridge(pdev
);
1804 if (pci_is_pcie(dev_tmp
)) {
1805 bus
= dev_tmp
->subordinate
->number
;
1808 bus
= dev_tmp
->bus
->number
;
1809 devfn
= dev_tmp
->devfn
;
1811 spin_lock_irqsave(&device_domain_lock
, flags
);
1812 list_for_each_entry(info
, &device_domain_list
, global
) {
1813 if (info
->segment
== segment
&&
1814 info
->bus
== bus
&& info
->devfn
== devfn
) {
1815 found
= info
->domain
;
1819 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1820 /* pcie-pci bridge already has a domain, uses it */
1827 domain
= alloc_domain();
1831 /* Allocate new domain for the device */
1832 drhd
= dmar_find_matched_drhd_unit(pdev
);
1834 printk(KERN_ERR
"IOMMU: can't find DMAR for device %s\n",
1838 iommu
= drhd
->iommu
;
1840 ret
= iommu_attach_domain(domain
, iommu
);
1842 free_domain_mem(domain
);
1846 if (domain_init(domain
, gaw
)) {
1847 domain_exit(domain
);
1851 /* register pcie-to-pci device */
1853 info
= alloc_devinfo_mem();
1855 domain_exit(domain
);
1858 info
->segment
= segment
;
1860 info
->devfn
= devfn
;
1862 info
->domain
= domain
;
1863 /* This domain is shared by devices under p2p bridge */
1864 domain
->flags
|= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES
;
1866 /* pcie-to-pci bridge already has a domain, uses it */
1868 spin_lock_irqsave(&device_domain_lock
, flags
);
1869 list_for_each_entry(tmp
, &device_domain_list
, global
) {
1870 if (tmp
->segment
== segment
&&
1871 tmp
->bus
== bus
&& tmp
->devfn
== devfn
) {
1872 found
= tmp
->domain
;
1877 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1878 free_devinfo_mem(info
);
1879 domain_exit(domain
);
1882 list_add(&info
->link
, &domain
->devices
);
1883 list_add(&info
->global
, &device_domain_list
);
1884 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1889 info
= alloc_devinfo_mem();
1892 info
->segment
= segment
;
1893 info
->bus
= pdev
->bus
->number
;
1894 info
->devfn
= pdev
->devfn
;
1896 info
->domain
= domain
;
1897 spin_lock_irqsave(&device_domain_lock
, flags
);
1898 /* somebody is fast */
1899 found
= find_domain(pdev
);
1900 if (found
!= NULL
) {
1901 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1902 if (found
!= domain
) {
1903 domain_exit(domain
);
1906 free_devinfo_mem(info
);
1909 list_add(&info
->link
, &domain
->devices
);
1910 list_add(&info
->global
, &device_domain_list
);
1911 pdev
->dev
.archdata
.iommu
= info
;
1912 spin_unlock_irqrestore(&device_domain_lock
, flags
);
1915 /* recheck it here, maybe others set it */
1916 return find_domain(pdev
);
1919 static int iommu_identity_mapping
;
1920 #define IDENTMAP_ALL 1
1921 #define IDENTMAP_GFX 2
1922 #define IDENTMAP_AZALIA 4
1924 static int iommu_domain_identity_map(struct dmar_domain
*domain
,
1925 unsigned long long start
,
1926 unsigned long long end
)
1928 unsigned long first_vpfn
= start
>> VTD_PAGE_SHIFT
;
1929 unsigned long last_vpfn
= end
>> VTD_PAGE_SHIFT
;
1931 if (!reserve_iova(&domain
->iovad
, dma_to_mm_pfn(first_vpfn
),
1932 dma_to_mm_pfn(last_vpfn
))) {
1933 printk(KERN_ERR
"IOMMU: reserve iova failed\n");
1937 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
1938 start
, end
, domain
->id
);
1940 * RMRR range might have overlap with physical memory range,
1943 dma_pte_clear_range(domain
, first_vpfn
, last_vpfn
);
1945 return domain_pfn_mapping(domain
, first_vpfn
, first_vpfn
,
1946 last_vpfn
- first_vpfn
+ 1,
1947 DMA_PTE_READ
|DMA_PTE_WRITE
);
1950 static int iommu_prepare_identity_map(struct pci_dev
*pdev
,
1951 unsigned long long start
,
1952 unsigned long long end
)
1954 struct dmar_domain
*domain
;
1957 domain
= get_domain_for_dev(pdev
, DEFAULT_DOMAIN_ADDRESS_WIDTH
);
1961 /* For _hardware_ passthrough, don't bother. But for software
1962 passthrough, we do it anyway -- it may indicate a memory
1963 range which is reserved in E820, so which didn't get set
1964 up to start with in si_domain */
1965 if (domain
== si_domain
&& hw_pass_through
) {
1966 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
1967 pci_name(pdev
), start
, end
);
1972 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1973 pci_name(pdev
), start
, end
);
1976 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
1977 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1978 dmi_get_system_info(DMI_BIOS_VENDOR
),
1979 dmi_get_system_info(DMI_BIOS_VERSION
),
1980 dmi_get_system_info(DMI_PRODUCT_VERSION
));
1985 if (end
>> agaw_to_width(domain
->agaw
)) {
1986 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
1987 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1988 agaw_to_width(domain
->agaw
),
1989 dmi_get_system_info(DMI_BIOS_VENDOR
),
1990 dmi_get_system_info(DMI_BIOS_VERSION
),
1991 dmi_get_system_info(DMI_PRODUCT_VERSION
));
1996 ret
= iommu_domain_identity_map(domain
, start
, end
);
2000 /* context entry init */
2001 ret
= domain_context_mapping(domain
, pdev
, CONTEXT_TT_MULTI_LEVEL
);
2008 domain_exit(domain
);
2012 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit
*rmrr
,
2013 struct pci_dev
*pdev
)
2015 if (pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
)
2017 return iommu_prepare_identity_map(pdev
, rmrr
->base_address
,
2018 rmrr
->end_address
+ 1);
2021 #ifdef CONFIG_DMAR_FLOPPY_WA
2022 static inline void iommu_prepare_isa(void)
2024 struct pci_dev
*pdev
;
2027 pdev
= pci_get_class(PCI_CLASS_BRIDGE_ISA
<< 8, NULL
);
2031 printk(KERN_INFO
"IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2032 ret
= iommu_prepare_identity_map(pdev
, 0, 16*1024*1024);
2035 printk(KERN_ERR
"IOMMU: Failed to create 0-16MiB identity map; "
2036 "floppy might not work\n");
2040 static inline void iommu_prepare_isa(void)
2044 #endif /* !CONFIG_DMAR_FLPY_WA */
2046 static int md_domain_init(struct dmar_domain
*domain
, int guest_width
);
2048 static int __init
si_domain_work_fn(unsigned long start_pfn
,
2049 unsigned long end_pfn
, void *datax
)
2053 *ret
= iommu_domain_identity_map(si_domain
,
2054 (uint64_t)start_pfn
<< PAGE_SHIFT
,
2055 (uint64_t)end_pfn
<< PAGE_SHIFT
);
2060 static int __init
si_domain_init(int hw
)
2062 struct dmar_drhd_unit
*drhd
;
2063 struct intel_iommu
*iommu
;
2066 si_domain
= alloc_domain();
2070 pr_debug("Identity mapping domain is domain %d\n", si_domain
->id
);
2072 for_each_active_iommu(iommu
, drhd
) {
2073 ret
= iommu_attach_domain(si_domain
, iommu
);
2075 domain_exit(si_domain
);
2080 if (md_domain_init(si_domain
, DEFAULT_DOMAIN_ADDRESS_WIDTH
)) {
2081 domain_exit(si_domain
);
2085 si_domain
->flags
= DOMAIN_FLAG_STATIC_IDENTITY
;
2090 for_each_online_node(nid
) {
2091 work_with_active_regions(nid
, si_domain_work_fn
, &ret
);
2099 static void domain_remove_one_dev_info(struct dmar_domain
*domain
,
2100 struct pci_dev
*pdev
);
2101 static int identity_mapping(struct pci_dev
*pdev
)
2103 struct device_domain_info
*info
;
2105 if (likely(!iommu_identity_mapping
))
2109 list_for_each_entry(info
, &si_domain
->devices
, link
)
2110 if (info
->dev
== pdev
)
2115 static int domain_add_dev_info(struct dmar_domain
*domain
,
2116 struct pci_dev
*pdev
,
2119 struct device_domain_info
*info
;
2120 unsigned long flags
;
2123 info
= alloc_devinfo_mem();
2127 ret
= domain_context_mapping(domain
, pdev
, translation
);
2129 free_devinfo_mem(info
);
2133 info
->segment
= pci_domain_nr(pdev
->bus
);
2134 info
->bus
= pdev
->bus
->number
;
2135 info
->devfn
= pdev
->devfn
;
2137 info
->domain
= domain
;
2139 spin_lock_irqsave(&device_domain_lock
, flags
);
2140 list_add(&info
->link
, &domain
->devices
);
2141 list_add(&info
->global
, &device_domain_list
);
2142 pdev
->dev
.archdata
.iommu
= info
;
2143 spin_unlock_irqrestore(&device_domain_lock
, flags
);
2148 static int iommu_should_identity_map(struct pci_dev
*pdev
, int startup
)
2150 if ((iommu_identity_mapping
& IDENTMAP_AZALIA
) && IS_AZALIA(pdev
))
2153 if ((iommu_identity_mapping
& IDENTMAP_GFX
) && IS_GFX_DEVICE(pdev
))
2156 if (!(iommu_identity_mapping
& IDENTMAP_ALL
))
2160 * We want to start off with all devices in the 1:1 domain, and
2161 * take them out later if we find they can't access all of memory.
2163 * However, we can't do this for PCI devices behind bridges,
2164 * because all PCI devices behind the same bridge will end up
2165 * with the same source-id on their transactions.
2167 * Practically speaking, we can't change things around for these
2168 * devices at run-time, because we can't be sure there'll be no
2169 * DMA transactions in flight for any of their siblings.
2171 * So PCI devices (unless they're on the root bus) as well as
2172 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2173 * the 1:1 domain, just in _case_ one of their siblings turns out
2174 * not to be able to map all of memory.
2176 if (!pci_is_pcie(pdev
)) {
2177 if (!pci_is_root_bus(pdev
->bus
))
2179 if (pdev
->class >> 8 == PCI_CLASS_BRIDGE_PCI
)
2181 } else if (pdev
->pcie_type
== PCI_EXP_TYPE_PCI_BRIDGE
)
2185 * At boot time, we don't yet know if devices will be 64-bit capable.
2186 * Assume that they will -- if they turn out not to be, then we can
2187 * take them out of the 1:1 domain later.
2190 return pdev
->dma_mask
> DMA_BIT_MASK(32);
2195 static int __init
iommu_prepare_static_identity_mapping(int hw
)
2197 struct pci_dev
*pdev
= NULL
;
2200 ret
= si_domain_init(hw
);
2204 for_each_pci_dev(pdev
) {
2205 if (iommu_should_identity_map(pdev
, 1)) {
2206 printk(KERN_INFO
"IOMMU: %s identity mapping for device %s\n",
2207 hw
? "hardware" : "software", pci_name(pdev
));
2209 ret
= domain_add_dev_info(si_domain
, pdev
,
2210 hw
? CONTEXT_TT_PASS_THROUGH
:
2211 CONTEXT_TT_MULTI_LEVEL
);
2220 static int __init
init_dmars(int force_on
)
2222 struct dmar_drhd_unit
*drhd
;
2223 struct dmar_rmrr_unit
*rmrr
;
2224 struct pci_dev
*pdev
;
2225 struct intel_iommu
*iommu
;
2231 * initialize and program root entry to not present
2234 for_each_drhd_unit(drhd
) {
2237 * lock not needed as this is only incremented in the single
2238 * threaded kernel __init code path all other access are read
2243 g_iommus
= kcalloc(g_num_of_iommus
, sizeof(struct intel_iommu
*),
2246 printk(KERN_ERR
"Allocating global iommu array failed\n");
2251 deferred_flush
= kzalloc(g_num_of_iommus
*
2252 sizeof(struct deferred_flush_tables
), GFP_KERNEL
);
2253 if (!deferred_flush
) {
2258 for_each_drhd_unit(drhd
) {
2262 iommu
= drhd
->iommu
;
2263 g_iommus
[iommu
->seq_id
] = iommu
;
2265 ret
= iommu_init_domains(iommu
);
2271 * we could share the same root & context tables
2272 * among all IOMMU's. Need to Split it later.
2274 ret
= iommu_alloc_root_entry(iommu
);
2276 printk(KERN_ERR
"IOMMU: allocate root entry failed\n");
2279 if (!ecap_pass_through(iommu
->ecap
))
2280 hw_pass_through
= 0;
2284 * Start from the sane iommu hardware state.
2286 for_each_drhd_unit(drhd
) {
2290 iommu
= drhd
->iommu
;
2293 * If the queued invalidation is already initialized by us
2294 * (for example, while enabling interrupt-remapping) then
2295 * we got the things already rolling from a sane state.
2301 * Clear any previous faults.
2303 dmar_fault(-1, iommu
);
2305 * Disable queued invalidation if supported and already enabled
2306 * before OS handover.
2308 dmar_disable_qi(iommu
);
2311 for_each_drhd_unit(drhd
) {
2315 iommu
= drhd
->iommu
;
2317 if (dmar_enable_qi(iommu
)) {
2319 * Queued Invalidate not enabled, use Register Based
2322 iommu
->flush
.flush_context
= __iommu_flush_context
;
2323 iommu
->flush
.flush_iotlb
= __iommu_flush_iotlb
;
2324 printk(KERN_INFO
"IOMMU %d 0x%Lx: using Register based "
2327 (unsigned long long)drhd
->reg_base_addr
);
2329 iommu
->flush
.flush_context
= qi_flush_context
;
2330 iommu
->flush
.flush_iotlb
= qi_flush_iotlb
;
2331 printk(KERN_INFO
"IOMMU %d 0x%Lx: using Queued "
2334 (unsigned long long)drhd
->reg_base_addr
);
2338 if (iommu_pass_through
)
2339 iommu_identity_mapping
|= IDENTMAP_ALL
;
2341 #ifdef CONFIG_DMAR_BROKEN_GFX_WA
2342 iommu_identity_mapping
|= IDENTMAP_GFX
;
2345 check_tylersburg_isoch();
2348 * If pass through is not set or not enabled, setup context entries for
2349 * identity mappings for rmrr, gfx, and isa and may fall back to static
2350 * identity mapping if iommu_identity_mapping is set.
2352 if (iommu_identity_mapping
) {
2353 ret
= iommu_prepare_static_identity_mapping(hw_pass_through
);
2355 printk(KERN_CRIT
"Failed to setup IOMMU pass-through\n");
2361 * for each dev attached to rmrr
2363 * locate drhd for dev, alloc domain for dev
2364 * allocate free domain
2365 * allocate page table entries for rmrr
2366 * if context not allocated for bus
2367 * allocate and init context
2368 * set present in root table for this bus
2369 * init context with domain, translation etc
2373 printk(KERN_INFO
"IOMMU: Setting RMRR:\n");
2374 for_each_rmrr_units(rmrr
) {
2375 for (i
= 0; i
< rmrr
->devices_cnt
; i
++) {
2376 pdev
= rmrr
->devices
[i
];
2378 * some BIOS lists non-exist devices in DMAR
2383 ret
= iommu_prepare_rmrr_dev(rmrr
, pdev
);
2386 "IOMMU: mapping reserved region failed\n");
2390 iommu_prepare_isa();
2395 * global invalidate context cache
2396 * global invalidate iotlb
2397 * enable translation
2399 for_each_drhd_unit(drhd
) {
2400 if (drhd
->ignored
) {
2402 * we always have to disable PMRs or DMA may fail on
2406 iommu_disable_protect_mem_regions(drhd
->iommu
);
2409 iommu
= drhd
->iommu
;
2411 iommu_flush_write_buffer(iommu
);
2413 ret
= dmar_set_interrupt(iommu
);
2417 iommu_set_root_entry(iommu
);
2419 iommu
->flush
.flush_context(iommu
, 0, 0, 0, DMA_CCMD_GLOBAL_INVL
);
2420 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH
);
2422 ret
= iommu_enable_translation(iommu
);
2426 iommu_disable_protect_mem_regions(iommu
);
2431 for_each_drhd_unit(drhd
) {
2434 iommu
= drhd
->iommu
;
2441 /* This takes a number of _MM_ pages, not VTD pages */
2442 static struct iova
*intel_alloc_iova(struct device
*dev
,
2443 struct dmar_domain
*domain
,
2444 unsigned long nrpages
, uint64_t dma_mask
)
2446 struct pci_dev
*pdev
= to_pci_dev(dev
);
2447 struct iova
*iova
= NULL
;
2449 /* Restrict dma_mask to the width that the iommu can handle */
2450 dma_mask
= min_t(uint64_t, DOMAIN_MAX_ADDR(domain
->gaw
), dma_mask
);
2452 if (!dmar_forcedac
&& dma_mask
> DMA_BIT_MASK(32)) {
2454 * First try to allocate an io virtual address in
2455 * DMA_BIT_MASK(32) and if that fails then try allocating
2458 iova
= alloc_iova(&domain
->iovad
, nrpages
,
2459 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2463 iova
= alloc_iova(&domain
->iovad
, nrpages
, IOVA_PFN(dma_mask
), 1);
2464 if (unlikely(!iova
)) {
2465 printk(KERN_ERR
"Allocating %ld-page iova for %s failed",
2466 nrpages
, pci_name(pdev
));
2473 static struct dmar_domain
*__get_valid_domain_for_dev(struct pci_dev
*pdev
)
2475 struct dmar_domain
*domain
;
2478 domain
= get_domain_for_dev(pdev
,
2479 DEFAULT_DOMAIN_ADDRESS_WIDTH
);
2482 "Allocating domain for %s failed", pci_name(pdev
));
2486 /* make sure context mapping is ok */
2487 if (unlikely(!domain_context_mapped(pdev
))) {
2488 ret
= domain_context_mapping(domain
, pdev
,
2489 CONTEXT_TT_MULTI_LEVEL
);
2492 "Domain context map for %s failed",
2501 static inline struct dmar_domain
*get_valid_domain_for_dev(struct pci_dev
*dev
)
2503 struct device_domain_info
*info
;
2505 /* No lock here, assumes no domain exit in normal case */
2506 info
= dev
->dev
.archdata
.iommu
;
2508 return info
->domain
;
2510 return __get_valid_domain_for_dev(dev
);
2513 static int iommu_dummy(struct pci_dev
*pdev
)
2515 return pdev
->dev
.archdata
.iommu
== DUMMY_DEVICE_DOMAIN_INFO
;
2518 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2519 static int iommu_no_mapping(struct device
*dev
)
2521 struct pci_dev
*pdev
;
2524 if (unlikely(dev
->bus
!= &pci_bus_type
))
2527 pdev
= to_pci_dev(dev
);
2528 if (iommu_dummy(pdev
))
2531 if (!iommu_identity_mapping
)
2534 found
= identity_mapping(pdev
);
2536 if (iommu_should_identity_map(pdev
, 0))
2540 * 32 bit DMA is removed from si_domain and fall back
2541 * to non-identity mapping.
2543 domain_remove_one_dev_info(si_domain
, pdev
);
2544 printk(KERN_INFO
"32bit %s uses non-identity mapping\n",
2550 * In case of a detached 64 bit DMA device from vm, the device
2551 * is put into si_domain for identity mapping.
2553 if (iommu_should_identity_map(pdev
, 0)) {
2555 ret
= domain_add_dev_info(si_domain
, pdev
,
2557 CONTEXT_TT_PASS_THROUGH
:
2558 CONTEXT_TT_MULTI_LEVEL
);
2560 printk(KERN_INFO
"64bit %s uses identity mapping\n",
2570 static dma_addr_t
__intel_map_single(struct device
*hwdev
, phys_addr_t paddr
,
2571 size_t size
, int dir
, u64 dma_mask
)
2573 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
2574 struct dmar_domain
*domain
;
2575 phys_addr_t start_paddr
;
2579 struct intel_iommu
*iommu
;
2580 unsigned long paddr_pfn
= paddr
>> PAGE_SHIFT
;
2582 BUG_ON(dir
== DMA_NONE
);
2584 if (iommu_no_mapping(hwdev
))
2587 domain
= get_valid_domain_for_dev(pdev
);
2591 iommu
= domain_get_iommu(domain
);
2592 size
= aligned_nrpages(paddr
, size
);
2594 iova
= intel_alloc_iova(hwdev
, domain
, dma_to_mm_pfn(size
),
2600 * Check if DMAR supports zero-length reads on write only
2603 if (dir
== DMA_TO_DEVICE
|| dir
== DMA_BIDIRECTIONAL
|| \
2604 !cap_zlr(iommu
->cap
))
2605 prot
|= DMA_PTE_READ
;
2606 if (dir
== DMA_FROM_DEVICE
|| dir
== DMA_BIDIRECTIONAL
)
2607 prot
|= DMA_PTE_WRITE
;
2609 * paddr - (paddr + size) might be partial page, we should map the whole
2610 * page. Note: if two part of one page are separately mapped, we
2611 * might have two guest_addr mapping to the same host paddr, but this
2612 * is not a big problem
2614 ret
= domain_pfn_mapping(domain
, mm_to_dma_pfn(iova
->pfn_lo
),
2615 mm_to_dma_pfn(paddr_pfn
), size
, prot
);
2619 /* it's a non-present to present mapping. Only flush if caching mode */
2620 if (cap_caching_mode(iommu
->cap
))
2621 iommu_flush_iotlb_psi(iommu
, domain
->id
, mm_to_dma_pfn(iova
->pfn_lo
), size
, 1);
2623 iommu_flush_write_buffer(iommu
);
2625 start_paddr
= (phys_addr_t
)iova
->pfn_lo
<< PAGE_SHIFT
;
2626 start_paddr
+= paddr
& ~PAGE_MASK
;
2631 __free_iova(&domain
->iovad
, iova
);
2632 printk(KERN_ERR
"Device %s request: %zx@%llx dir %d --- failed\n",
2633 pci_name(pdev
), size
, (unsigned long long)paddr
, dir
);
2637 static dma_addr_t
intel_map_page(struct device
*dev
, struct page
*page
,
2638 unsigned long offset
, size_t size
,
2639 enum dma_data_direction dir
,
2640 struct dma_attrs
*attrs
)
2642 return __intel_map_single(dev
, page_to_phys(page
) + offset
, size
,
2643 dir
, to_pci_dev(dev
)->dma_mask
);
2646 static void flush_unmaps(void)
2652 /* just flush them all */
2653 for (i
= 0; i
< g_num_of_iommus
; i
++) {
2654 struct intel_iommu
*iommu
= g_iommus
[i
];
2658 if (!deferred_flush
[i
].next
)
2661 /* In caching mode, global flushes turn emulation expensive */
2662 if (!cap_caching_mode(iommu
->cap
))
2663 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
2664 DMA_TLB_GLOBAL_FLUSH
);
2665 for (j
= 0; j
< deferred_flush
[i
].next
; j
++) {
2667 struct iova
*iova
= deferred_flush
[i
].iova
[j
];
2668 struct dmar_domain
*domain
= deferred_flush
[i
].domain
[j
];
2670 /* On real hardware multiple invalidations are expensive */
2671 if (cap_caching_mode(iommu
->cap
))
2672 iommu_flush_iotlb_psi(iommu
, domain
->id
,
2673 iova
->pfn_lo
, iova
->pfn_hi
- iova
->pfn_lo
+ 1, 0);
2675 mask
= ilog2(mm_to_dma_pfn(iova
->pfn_hi
- iova
->pfn_lo
+ 1));
2676 iommu_flush_dev_iotlb(deferred_flush
[i
].domain
[j
],
2677 (uint64_t)iova
->pfn_lo
<< PAGE_SHIFT
, mask
);
2679 __free_iova(&deferred_flush
[i
].domain
[j
]->iovad
, iova
);
2681 deferred_flush
[i
].next
= 0;
2687 static void flush_unmaps_timeout(unsigned long data
)
2689 unsigned long flags
;
2691 spin_lock_irqsave(&async_umap_flush_lock
, flags
);
2693 spin_unlock_irqrestore(&async_umap_flush_lock
, flags
);
2696 static void add_unmap(struct dmar_domain
*dom
, struct iova
*iova
)
2698 unsigned long flags
;
2700 struct intel_iommu
*iommu
;
2702 spin_lock_irqsave(&async_umap_flush_lock
, flags
);
2703 if (list_size
== HIGH_WATER_MARK
)
2706 iommu
= domain_get_iommu(dom
);
2707 iommu_id
= iommu
->seq_id
;
2709 next
= deferred_flush
[iommu_id
].next
;
2710 deferred_flush
[iommu_id
].domain
[next
] = dom
;
2711 deferred_flush
[iommu_id
].iova
[next
] = iova
;
2712 deferred_flush
[iommu_id
].next
++;
2715 mod_timer(&unmap_timer
, jiffies
+ msecs_to_jiffies(10));
2719 spin_unlock_irqrestore(&async_umap_flush_lock
, flags
);
2722 static void intel_unmap_page(struct device
*dev
, dma_addr_t dev_addr
,
2723 size_t size
, enum dma_data_direction dir
,
2724 struct dma_attrs
*attrs
)
2726 struct pci_dev
*pdev
= to_pci_dev(dev
);
2727 struct dmar_domain
*domain
;
2728 unsigned long start_pfn
, last_pfn
;
2730 struct intel_iommu
*iommu
;
2732 if (iommu_no_mapping(dev
))
2735 domain
= find_domain(pdev
);
2738 iommu
= domain_get_iommu(domain
);
2740 iova
= find_iova(&domain
->iovad
, IOVA_PFN(dev_addr
));
2741 if (WARN_ONCE(!iova
, "Driver unmaps unmatched page at PFN %llx\n",
2742 (unsigned long long)dev_addr
))
2745 start_pfn
= mm_to_dma_pfn(iova
->pfn_lo
);
2746 last_pfn
= mm_to_dma_pfn(iova
->pfn_hi
+ 1) - 1;
2748 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2749 pci_name(pdev
), start_pfn
, last_pfn
);
2751 /* clear the whole page */
2752 dma_pte_clear_range(domain
, start_pfn
, last_pfn
);
2754 /* free page tables */
2755 dma_pte_free_pagetable(domain
, start_pfn
, last_pfn
);
2757 if (intel_iommu_strict
) {
2758 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_pfn
,
2759 last_pfn
- start_pfn
+ 1, 0);
2761 __free_iova(&domain
->iovad
, iova
);
2763 add_unmap(domain
, iova
);
2765 * queue up the release of the unmap to save the 1/6th of the
2766 * cpu used up by the iotlb flush operation...
2771 static void *intel_alloc_coherent(struct device
*hwdev
, size_t size
,
2772 dma_addr_t
*dma_handle
, gfp_t flags
)
2777 size
= PAGE_ALIGN(size
);
2778 order
= get_order(size
);
2780 if (!iommu_no_mapping(hwdev
))
2781 flags
&= ~(GFP_DMA
| GFP_DMA32
);
2782 else if (hwdev
->coherent_dma_mask
< dma_get_required_mask(hwdev
)) {
2783 if (hwdev
->coherent_dma_mask
< DMA_BIT_MASK(32))
2789 vaddr
= (void *)__get_free_pages(flags
, order
);
2792 memset(vaddr
, 0, size
);
2794 *dma_handle
= __intel_map_single(hwdev
, virt_to_bus(vaddr
), size
,
2796 hwdev
->coherent_dma_mask
);
2799 free_pages((unsigned long)vaddr
, order
);
2803 static void intel_free_coherent(struct device
*hwdev
, size_t size
, void *vaddr
,
2804 dma_addr_t dma_handle
)
2808 size
= PAGE_ALIGN(size
);
2809 order
= get_order(size
);
2811 intel_unmap_page(hwdev
, dma_handle
, size
, DMA_BIDIRECTIONAL
, NULL
);
2812 free_pages((unsigned long)vaddr
, order
);
2815 static void intel_unmap_sg(struct device
*hwdev
, struct scatterlist
*sglist
,
2816 int nelems
, enum dma_data_direction dir
,
2817 struct dma_attrs
*attrs
)
2819 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
2820 struct dmar_domain
*domain
;
2821 unsigned long start_pfn
, last_pfn
;
2823 struct intel_iommu
*iommu
;
2825 if (iommu_no_mapping(hwdev
))
2828 domain
= find_domain(pdev
);
2831 iommu
= domain_get_iommu(domain
);
2833 iova
= find_iova(&domain
->iovad
, IOVA_PFN(sglist
[0].dma_address
));
2834 if (WARN_ONCE(!iova
, "Driver unmaps unmatched sglist at PFN %llx\n",
2835 (unsigned long long)sglist
[0].dma_address
))
2838 start_pfn
= mm_to_dma_pfn(iova
->pfn_lo
);
2839 last_pfn
= mm_to_dma_pfn(iova
->pfn_hi
+ 1) - 1;
2841 /* clear the whole page */
2842 dma_pte_clear_range(domain
, start_pfn
, last_pfn
);
2844 /* free page tables */
2845 dma_pte_free_pagetable(domain
, start_pfn
, last_pfn
);
2847 if (intel_iommu_strict
) {
2848 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_pfn
,
2849 last_pfn
- start_pfn
+ 1, 0);
2851 __free_iova(&domain
->iovad
, iova
);
2853 add_unmap(domain
, iova
);
2855 * queue up the release of the unmap to save the 1/6th of the
2856 * cpu used up by the iotlb flush operation...
2861 static int intel_nontranslate_map_sg(struct device
*hddev
,
2862 struct scatterlist
*sglist
, int nelems
, int dir
)
2865 struct scatterlist
*sg
;
2867 for_each_sg(sglist
, sg
, nelems
, i
) {
2868 BUG_ON(!sg_page(sg
));
2869 sg
->dma_address
= page_to_phys(sg_page(sg
)) + sg
->offset
;
2870 sg
->dma_length
= sg
->length
;
2875 static int intel_map_sg(struct device
*hwdev
, struct scatterlist
*sglist
, int nelems
,
2876 enum dma_data_direction dir
, struct dma_attrs
*attrs
)
2879 struct pci_dev
*pdev
= to_pci_dev(hwdev
);
2880 struct dmar_domain
*domain
;
2883 struct iova
*iova
= NULL
;
2885 struct scatterlist
*sg
;
2886 unsigned long start_vpfn
;
2887 struct intel_iommu
*iommu
;
2889 BUG_ON(dir
== DMA_NONE
);
2890 if (iommu_no_mapping(hwdev
))
2891 return intel_nontranslate_map_sg(hwdev
, sglist
, nelems
, dir
);
2893 domain
= get_valid_domain_for_dev(pdev
);
2897 iommu
= domain_get_iommu(domain
);
2899 for_each_sg(sglist
, sg
, nelems
, i
)
2900 size
+= aligned_nrpages(sg
->offset
, sg
->length
);
2902 iova
= intel_alloc_iova(hwdev
, domain
, dma_to_mm_pfn(size
),
2905 sglist
->dma_length
= 0;
2910 * Check if DMAR supports zero-length reads on write only
2913 if (dir
== DMA_TO_DEVICE
|| dir
== DMA_BIDIRECTIONAL
|| \
2914 !cap_zlr(iommu
->cap
))
2915 prot
|= DMA_PTE_READ
;
2916 if (dir
== DMA_FROM_DEVICE
|| dir
== DMA_BIDIRECTIONAL
)
2917 prot
|= DMA_PTE_WRITE
;
2919 start_vpfn
= mm_to_dma_pfn(iova
->pfn_lo
);
2921 ret
= domain_sg_mapping(domain
, start_vpfn
, sglist
, size
, prot
);
2922 if (unlikely(ret
)) {
2923 /* clear the page */
2924 dma_pte_clear_range(domain
, start_vpfn
,
2925 start_vpfn
+ size
- 1);
2926 /* free page tables */
2927 dma_pte_free_pagetable(domain
, start_vpfn
,
2928 start_vpfn
+ size
- 1);
2930 __free_iova(&domain
->iovad
, iova
);
2934 /* it's a non-present to present mapping. Only flush if caching mode */
2935 if (cap_caching_mode(iommu
->cap
))
2936 iommu_flush_iotlb_psi(iommu
, domain
->id
, start_vpfn
, size
, 1);
2938 iommu_flush_write_buffer(iommu
);
2943 static int intel_mapping_error(struct device
*dev
, dma_addr_t dma_addr
)
2948 struct dma_map_ops intel_dma_ops
= {
2949 .alloc_coherent
= intel_alloc_coherent
,
2950 .free_coherent
= intel_free_coherent
,
2951 .map_sg
= intel_map_sg
,
2952 .unmap_sg
= intel_unmap_sg
,
2953 .map_page
= intel_map_page
,
2954 .unmap_page
= intel_unmap_page
,
2955 .mapping_error
= intel_mapping_error
,
2958 static inline int iommu_domain_cache_init(void)
2962 iommu_domain_cache
= kmem_cache_create("iommu_domain",
2963 sizeof(struct dmar_domain
),
2968 if (!iommu_domain_cache
) {
2969 printk(KERN_ERR
"Couldn't create iommu_domain cache\n");
2976 static inline int iommu_devinfo_cache_init(void)
2980 iommu_devinfo_cache
= kmem_cache_create("iommu_devinfo",
2981 sizeof(struct device_domain_info
),
2985 if (!iommu_devinfo_cache
) {
2986 printk(KERN_ERR
"Couldn't create devinfo cache\n");
2993 static inline int iommu_iova_cache_init(void)
2997 iommu_iova_cache
= kmem_cache_create("iommu_iova",
2998 sizeof(struct iova
),
3002 if (!iommu_iova_cache
) {
3003 printk(KERN_ERR
"Couldn't create iova cache\n");
3010 static int __init
iommu_init_mempool(void)
3013 ret
= iommu_iova_cache_init();
3017 ret
= iommu_domain_cache_init();
3021 ret
= iommu_devinfo_cache_init();
3025 kmem_cache_destroy(iommu_domain_cache
);
3027 kmem_cache_destroy(iommu_iova_cache
);
3032 static void __init
iommu_exit_mempool(void)
3034 kmem_cache_destroy(iommu_devinfo_cache
);
3035 kmem_cache_destroy(iommu_domain_cache
);
3036 kmem_cache_destroy(iommu_iova_cache
);
3040 static void quirk_ioat_snb_local_iommu(struct pci_dev
*pdev
)
3042 struct dmar_drhd_unit
*drhd
;
3046 /* We know that this device on this chipset has its own IOMMU.
3047 * If we find it under a different IOMMU, then the BIOS is lying
3048 * to us. Hope that the IOMMU for this device is actually
3049 * disabled, and it needs no translation...
3051 rc
= pci_bus_read_config_dword(pdev
->bus
, PCI_DEVFN(0, 0), 0xb0, &vtbar
);
3053 /* "can't" happen */
3054 dev_info(&pdev
->dev
, "failed to run vt-d quirk\n");
3057 vtbar
&= 0xffff0000;
3059 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3060 drhd
= dmar_find_matched_drhd_unit(pdev
);
3061 if (WARN_TAINT_ONCE(!drhd
|| drhd
->reg_base_addr
- vtbar
!= 0xa000,
3062 TAINT_FIRMWARE_WORKAROUND
,
3063 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3064 pdev
->dev
.archdata
.iommu
= DUMMY_DEVICE_DOMAIN_INFO
;
3066 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL
, PCI_DEVICE_ID_INTEL_IOAT_SNB
, quirk_ioat_snb_local_iommu
);
3068 static void __init
init_no_remapping_devices(void)
3070 struct dmar_drhd_unit
*drhd
;
3072 for_each_drhd_unit(drhd
) {
3073 if (!drhd
->include_all
) {
3075 for (i
= 0; i
< drhd
->devices_cnt
; i
++)
3076 if (drhd
->devices
[i
] != NULL
)
3078 /* ignore DMAR unit if no pci devices exist */
3079 if (i
== drhd
->devices_cnt
)
3087 for_each_drhd_unit(drhd
) {
3089 if (drhd
->ignored
|| drhd
->include_all
)
3092 for (i
= 0; i
< drhd
->devices_cnt
; i
++)
3093 if (drhd
->devices
[i
] &&
3094 !IS_GFX_DEVICE(drhd
->devices
[i
]))
3097 if (i
< drhd
->devices_cnt
)
3100 /* bypass IOMMU if it is just for gfx devices */
3102 for (i
= 0; i
< drhd
->devices_cnt
; i
++) {
3103 if (!drhd
->devices
[i
])
3105 drhd
->devices
[i
]->dev
.archdata
.iommu
= DUMMY_DEVICE_DOMAIN_INFO
;
3110 #ifdef CONFIG_SUSPEND
3111 static int init_iommu_hw(void)
3113 struct dmar_drhd_unit
*drhd
;
3114 struct intel_iommu
*iommu
= NULL
;
3116 for_each_active_iommu(iommu
, drhd
)
3118 dmar_reenable_qi(iommu
);
3120 for_each_active_iommu(iommu
, drhd
) {
3121 iommu_flush_write_buffer(iommu
);
3123 iommu_set_root_entry(iommu
);
3125 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
3126 DMA_CCMD_GLOBAL_INVL
);
3127 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
3128 DMA_TLB_GLOBAL_FLUSH
);
3129 iommu_enable_translation(iommu
);
3130 iommu_disable_protect_mem_regions(iommu
);
3136 static void iommu_flush_all(void)
3138 struct dmar_drhd_unit
*drhd
;
3139 struct intel_iommu
*iommu
;
3141 for_each_active_iommu(iommu
, drhd
) {
3142 iommu
->flush
.flush_context(iommu
, 0, 0, 0,
3143 DMA_CCMD_GLOBAL_INVL
);
3144 iommu
->flush
.flush_iotlb(iommu
, 0, 0, 0,
3145 DMA_TLB_GLOBAL_FLUSH
);
3149 static int iommu_suspend(void)
3151 struct dmar_drhd_unit
*drhd
;
3152 struct intel_iommu
*iommu
= NULL
;
3155 for_each_active_iommu(iommu
, drhd
) {
3156 iommu
->iommu_state
= kzalloc(sizeof(u32
) * MAX_SR_DMAR_REGS
,
3158 if (!iommu
->iommu_state
)
3164 for_each_active_iommu(iommu
, drhd
) {
3165 iommu_disable_translation(iommu
);
3167 spin_lock_irqsave(&iommu
->register_lock
, flag
);
3169 iommu
->iommu_state
[SR_DMAR_FECTL_REG
] =
3170 readl(iommu
->reg
+ DMAR_FECTL_REG
);
3171 iommu
->iommu_state
[SR_DMAR_FEDATA_REG
] =
3172 readl(iommu
->reg
+ DMAR_FEDATA_REG
);
3173 iommu
->iommu_state
[SR_DMAR_FEADDR_REG
] =
3174 readl(iommu
->reg
+ DMAR_FEADDR_REG
);
3175 iommu
->iommu_state
[SR_DMAR_FEUADDR_REG
] =
3176 readl(iommu
->reg
+ DMAR_FEUADDR_REG
);
3178 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
3183 for_each_active_iommu(iommu
, drhd
)
3184 kfree(iommu
->iommu_state
);
3189 static void iommu_resume(void)
3191 struct dmar_drhd_unit
*drhd
;
3192 struct intel_iommu
*iommu
= NULL
;
3195 if (init_iommu_hw()) {
3196 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3200 for_each_active_iommu(iommu
, drhd
) {
3202 spin_lock_irqsave(&iommu
->register_lock
, flag
);
3204 writel(iommu
->iommu_state
[SR_DMAR_FECTL_REG
],
3205 iommu
->reg
+ DMAR_FECTL_REG
);
3206 writel(iommu
->iommu_state
[SR_DMAR_FEDATA_REG
],
3207 iommu
->reg
+ DMAR_FEDATA_REG
);
3208 writel(iommu
->iommu_state
[SR_DMAR_FEADDR_REG
],
3209 iommu
->reg
+ DMAR_FEADDR_REG
);
3210 writel(iommu
->iommu_state
[SR_DMAR_FEUADDR_REG
],
3211 iommu
->reg
+ DMAR_FEUADDR_REG
);
3213 spin_unlock_irqrestore(&iommu
->register_lock
, flag
);
3216 for_each_active_iommu(iommu
, drhd
)
3217 kfree(iommu
->iommu_state
);
3220 static struct syscore_ops iommu_syscore_ops
= {
3221 .resume
= iommu_resume
,
3222 .suspend
= iommu_suspend
,
3225 static void __init
init_iommu_pm_ops(void)
3227 register_syscore_ops(&iommu_syscore_ops
);
3231 static inline int init_iommu_pm_ops(void) { }
3232 #endif /* CONFIG_PM */
3235 * Here we only respond to action of unbound device from driver.
3237 * Added device is not attached to its DMAR domain here yet. That will happen
3238 * when mapping the device to iova.
3240 static int device_notifier(struct notifier_block
*nb
,
3241 unsigned long action
, void *data
)
3243 struct device
*dev
= data
;
3244 struct pci_dev
*pdev
= to_pci_dev(dev
);
3245 struct dmar_domain
*domain
;
3247 if (iommu_no_mapping(dev
))
3250 domain
= find_domain(pdev
);
3254 if (action
== BUS_NOTIFY_UNBOUND_DRIVER
&& !iommu_pass_through
) {
3255 domain_remove_one_dev_info(domain
, pdev
);
3257 if (!(domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
) &&
3258 !(domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
) &&
3259 list_empty(&domain
->devices
))
3260 domain_exit(domain
);
3266 static struct notifier_block device_nb
= {
3267 .notifier_call
= device_notifier
,
3270 int __init
intel_iommu_init(void)
3275 /* VT-d is required for a TXT/tboot launch, so enforce that */
3276 force_on
= tboot_force_iommu();
3278 if (dmar_table_init()) {
3280 panic("tboot: Failed to initialize DMAR table\n");
3284 if (dmar_dev_scope_init()) {
3286 panic("tboot: Failed to initialize DMAR device scope\n");
3291 * Check the need for DMA-remapping initialization now.
3292 * Above initialization will also be used by Interrupt-remapping.
3294 if (no_iommu
|| dmar_disabled
)
3297 if (iommu_init_mempool()) {
3299 panic("tboot: Failed to initialize iommu memory\n");
3303 if (dmar_init_reserved_ranges()) {
3305 panic("tboot: Failed to reserve iommu ranges\n");
3309 init_no_remapping_devices();
3311 ret
= init_dmars(force_on
);
3314 panic("tboot: Failed to initialize DMARs\n");
3315 printk(KERN_ERR
"IOMMU: dmar init failed\n");
3316 put_iova_domain(&reserved_iova_list
);
3317 iommu_exit_mempool();
3321 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3323 init_timer(&unmap_timer
);
3324 #ifdef CONFIG_SWIOTLB
3327 dma_ops
= &intel_dma_ops
;
3329 init_iommu_pm_ops();
3331 register_iommu(&intel_iommu_ops
);
3333 bus_register_notifier(&pci_bus_type
, &device_nb
);
3338 static void iommu_detach_dependent_devices(struct intel_iommu
*iommu
,
3339 struct pci_dev
*pdev
)
3341 struct pci_dev
*tmp
, *parent
;
3343 if (!iommu
|| !pdev
)
3346 /* dependent device detach */
3347 tmp
= pci_find_upstream_pcie_bridge(pdev
);
3348 /* Secondary interface's bus number and devfn 0 */
3350 parent
= pdev
->bus
->self
;
3351 while (parent
!= tmp
) {
3352 iommu_detach_dev(iommu
, parent
->bus
->number
,
3354 parent
= parent
->bus
->self
;
3356 if (pci_is_pcie(tmp
)) /* this is a PCIe-to-PCI bridge */
3357 iommu_detach_dev(iommu
,
3358 tmp
->subordinate
->number
, 0);
3359 else /* this is a legacy PCI bridge */
3360 iommu_detach_dev(iommu
, tmp
->bus
->number
,
3365 static void domain_remove_one_dev_info(struct dmar_domain
*domain
,
3366 struct pci_dev
*pdev
)
3368 struct device_domain_info
*info
;
3369 struct intel_iommu
*iommu
;
3370 unsigned long flags
;
3372 struct list_head
*entry
, *tmp
;
3374 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
), pdev
->bus
->number
,
3379 spin_lock_irqsave(&device_domain_lock
, flags
);
3380 list_for_each_safe(entry
, tmp
, &domain
->devices
) {
3381 info
= list_entry(entry
, struct device_domain_info
, link
);
3382 /* No need to compare PCI domain; it has to be the same */
3383 if (info
->bus
== pdev
->bus
->number
&&
3384 info
->devfn
== pdev
->devfn
) {
3385 list_del(&info
->link
);
3386 list_del(&info
->global
);
3388 info
->dev
->dev
.archdata
.iommu
= NULL
;
3389 spin_unlock_irqrestore(&device_domain_lock
, flags
);
3391 iommu_disable_dev_iotlb(info
);
3392 iommu_detach_dev(iommu
, info
->bus
, info
->devfn
);
3393 iommu_detach_dependent_devices(iommu
, pdev
);
3394 free_devinfo_mem(info
);
3396 spin_lock_irqsave(&device_domain_lock
, flags
);
3404 /* if there is no other devices under the same iommu
3405 * owned by this domain, clear this iommu in iommu_bmp
3406 * update iommu count and coherency
3408 if (iommu
== device_to_iommu(info
->segment
, info
->bus
,
3414 unsigned long tmp_flags
;
3415 spin_lock_irqsave(&domain
->iommu_lock
, tmp_flags
);
3416 clear_bit(iommu
->seq_id
, &domain
->iommu_bmp
);
3417 domain
->iommu_count
--;
3418 domain_update_iommu_cap(domain
);
3419 spin_unlock_irqrestore(&domain
->iommu_lock
, tmp_flags
);
3421 spin_lock_irqsave(&iommu
->lock
, tmp_flags
);
3422 clear_bit(domain
->id
, iommu
->domain_ids
);
3423 iommu
->domains
[domain
->id
] = NULL
;
3424 spin_unlock_irqrestore(&iommu
->lock
, tmp_flags
);
3427 spin_unlock_irqrestore(&device_domain_lock
, flags
);
3430 static void vm_domain_remove_all_dev_info(struct dmar_domain
*domain
)
3432 struct device_domain_info
*info
;
3433 struct intel_iommu
*iommu
;
3434 unsigned long flags1
, flags2
;
3436 spin_lock_irqsave(&device_domain_lock
, flags1
);
3437 while (!list_empty(&domain
->devices
)) {
3438 info
= list_entry(domain
->devices
.next
,
3439 struct device_domain_info
, link
);
3440 list_del(&info
->link
);
3441 list_del(&info
->global
);
3443 info
->dev
->dev
.archdata
.iommu
= NULL
;
3445 spin_unlock_irqrestore(&device_domain_lock
, flags1
);
3447 iommu_disable_dev_iotlb(info
);
3448 iommu
= device_to_iommu(info
->segment
, info
->bus
, info
->devfn
);
3449 iommu_detach_dev(iommu
, info
->bus
, info
->devfn
);
3450 iommu_detach_dependent_devices(iommu
, info
->dev
);
3452 /* clear this iommu in iommu_bmp, update iommu count
3455 spin_lock_irqsave(&domain
->iommu_lock
, flags2
);
3456 if (test_and_clear_bit(iommu
->seq_id
,
3457 &domain
->iommu_bmp
)) {
3458 domain
->iommu_count
--;
3459 domain_update_iommu_cap(domain
);
3461 spin_unlock_irqrestore(&domain
->iommu_lock
, flags2
);
3463 free_devinfo_mem(info
);
3464 spin_lock_irqsave(&device_domain_lock
, flags1
);
3466 spin_unlock_irqrestore(&device_domain_lock
, flags1
);
3469 /* domain id for virtual machine, it won't be set in context */
3470 static unsigned long vm_domid
;
3472 static struct dmar_domain
*iommu_alloc_vm_domain(void)
3474 struct dmar_domain
*domain
;
3476 domain
= alloc_domain_mem();
3480 domain
->id
= vm_domid
++;
3482 memset(&domain
->iommu_bmp
, 0, sizeof(unsigned long));
3483 domain
->flags
= DOMAIN_FLAG_VIRTUAL_MACHINE
;
3488 static int md_domain_init(struct dmar_domain
*domain
, int guest_width
)
3492 init_iova_domain(&domain
->iovad
, DMA_32BIT_PFN
);
3493 spin_lock_init(&domain
->iommu_lock
);
3495 domain_reserve_special_ranges(domain
);
3497 /* calculate AGAW */
3498 domain
->gaw
= guest_width
;
3499 adjust_width
= guestwidth_to_adjustwidth(guest_width
);
3500 domain
->agaw
= width_to_agaw(adjust_width
);
3502 INIT_LIST_HEAD(&domain
->devices
);
3504 domain
->iommu_count
= 0;
3505 domain
->iommu_coherency
= 0;
3506 domain
->iommu_snooping
= 0;
3507 domain
->max_addr
= 0;
3510 /* always allocate the top pgd */
3511 domain
->pgd
= (struct dma_pte
*)alloc_pgtable_page(domain
->nid
);
3514 domain_flush_cache(domain
, domain
->pgd
, PAGE_SIZE
);
3518 static void iommu_free_vm_domain(struct dmar_domain
*domain
)
3520 unsigned long flags
;
3521 struct dmar_drhd_unit
*drhd
;
3522 struct intel_iommu
*iommu
;
3524 unsigned long ndomains
;
3526 for_each_drhd_unit(drhd
) {
3529 iommu
= drhd
->iommu
;
3531 ndomains
= cap_ndoms(iommu
->cap
);
3532 for_each_set_bit(i
, iommu
->domain_ids
, ndomains
) {
3533 if (iommu
->domains
[i
] == domain
) {
3534 spin_lock_irqsave(&iommu
->lock
, flags
);
3535 clear_bit(i
, iommu
->domain_ids
);
3536 iommu
->domains
[i
] = NULL
;
3537 spin_unlock_irqrestore(&iommu
->lock
, flags
);
3544 static void vm_domain_exit(struct dmar_domain
*domain
)
3546 /* Domain 0 is reserved, so dont process it */
3550 vm_domain_remove_all_dev_info(domain
);
3552 put_iova_domain(&domain
->iovad
);
3555 dma_pte_clear_range(domain
, 0, DOMAIN_MAX_PFN(domain
->gaw
));
3557 /* free page tables */
3558 dma_pte_free_pagetable(domain
, 0, DOMAIN_MAX_PFN(domain
->gaw
));
3560 iommu_free_vm_domain(domain
);
3561 free_domain_mem(domain
);
3564 static int intel_iommu_domain_init(struct iommu_domain
*domain
)
3566 struct dmar_domain
*dmar_domain
;
3568 dmar_domain
= iommu_alloc_vm_domain();
3571 "intel_iommu_domain_init: dmar_domain == NULL\n");
3574 if (md_domain_init(dmar_domain
, DEFAULT_DOMAIN_ADDRESS_WIDTH
)) {
3576 "intel_iommu_domain_init() failed\n");
3577 vm_domain_exit(dmar_domain
);
3580 domain
->priv
= dmar_domain
;
3585 static void intel_iommu_domain_destroy(struct iommu_domain
*domain
)
3587 struct dmar_domain
*dmar_domain
= domain
->priv
;
3589 domain
->priv
= NULL
;
3590 vm_domain_exit(dmar_domain
);
3593 static int intel_iommu_attach_device(struct iommu_domain
*domain
,
3596 struct dmar_domain
*dmar_domain
= domain
->priv
;
3597 struct pci_dev
*pdev
= to_pci_dev(dev
);
3598 struct intel_iommu
*iommu
;
3601 /* normally pdev is not mapped */
3602 if (unlikely(domain_context_mapped(pdev
))) {
3603 struct dmar_domain
*old_domain
;
3605 old_domain
= find_domain(pdev
);
3607 if (dmar_domain
->flags
& DOMAIN_FLAG_VIRTUAL_MACHINE
||
3608 dmar_domain
->flags
& DOMAIN_FLAG_STATIC_IDENTITY
)
3609 domain_remove_one_dev_info(old_domain
, pdev
);
3611 domain_remove_dev_info(old_domain
);
3615 iommu
= device_to_iommu(pci_domain_nr(pdev
->bus
), pdev
->bus
->number
,
3620 /* check if this iommu agaw is sufficient for max mapped address */
3621 addr_width
= agaw_to_width(iommu
->agaw
);
3622 if (addr_width
> cap_mgaw(iommu
->cap
))
3623 addr_width
= cap_mgaw(iommu
->cap
);
3625 if (dmar_domain
->max_addr
> (1LL << addr_width
)) {
3626 printk(KERN_ERR
"%s: iommu width (%d) is not "
3627 "sufficient for the mapped address (%llx)\n",
3628 __func__
, addr_width
, dmar_domain
->max_addr
);
3631 dmar_domain
->gaw
= addr_width
;
3634 * Knock out extra levels of page tables if necessary
3636 while (iommu
->agaw
< dmar_domain
->agaw
) {
3637 struct dma_pte
*pte
;
3639 pte
= dmar_domain
->pgd
;
3640 if (dma_pte_present(pte
)) {
3641 dmar_domain
->pgd
= (struct dma_pte
*)
3642 phys_to_virt(dma_pte_addr(pte
));
3643 free_pgtable_page(pte
);
3645 dmar_domain
->agaw
--;
3648 return domain_add_dev_info(dmar_domain
, pdev
, CONTEXT_TT_MULTI_LEVEL
);
3651 static void intel_iommu_detach_device(struct iommu_domain
*domain
,
3654 struct dmar_domain
*dmar_domain
= domain
->priv
;
3655 struct pci_dev
*pdev
= to_pci_dev(dev
);
3657 domain_remove_one_dev_info(dmar_domain
, pdev
);
3660 static int intel_iommu_map(struct iommu_domain
*domain
,
3661 unsigned long iova
, phys_addr_t hpa
,
3662 int gfp_order
, int iommu_prot
)
3664 struct dmar_domain
*dmar_domain
= domain
->priv
;
3670 if (iommu_prot
& IOMMU_READ
)
3671 prot
|= DMA_PTE_READ
;
3672 if (iommu_prot
& IOMMU_WRITE
)
3673 prot
|= DMA_PTE_WRITE
;
3674 if ((iommu_prot
& IOMMU_CACHE
) && dmar_domain
->iommu_snooping
)
3675 prot
|= DMA_PTE_SNP
;
3677 size
= PAGE_SIZE
<< gfp_order
;
3678 max_addr
= iova
+ size
;
3679 if (dmar_domain
->max_addr
< max_addr
) {
3682 /* check if minimum agaw is sufficient for mapped address */
3683 end
= __DOMAIN_MAX_ADDR(dmar_domain
->gaw
) + 1;
3684 if (end
< max_addr
) {
3685 printk(KERN_ERR
"%s: iommu width (%d) is not "
3686 "sufficient for the mapped address (%llx)\n",
3687 __func__
, dmar_domain
->gaw
, max_addr
);
3690 dmar_domain
->max_addr
= max_addr
;
3692 /* Round up size to next multiple of PAGE_SIZE, if it and
3693 the low bits of hpa would take us onto the next page */
3694 size
= aligned_nrpages(hpa
, size
);
3695 ret
= domain_pfn_mapping(dmar_domain
, iova
>> VTD_PAGE_SHIFT
,
3696 hpa
>> VTD_PAGE_SHIFT
, size
, prot
);
3700 static int intel_iommu_unmap(struct iommu_domain
*domain
,
3701 unsigned long iova
, int gfp_order
)
3703 struct dmar_domain
*dmar_domain
= domain
->priv
;
3704 size_t size
= PAGE_SIZE
<< gfp_order
;
3706 dma_pte_clear_range(dmar_domain
, iova
>> VTD_PAGE_SHIFT
,
3707 (iova
+ size
- 1) >> VTD_PAGE_SHIFT
);
3709 if (dmar_domain
->max_addr
== iova
+ size
)
3710 dmar_domain
->max_addr
= iova
;
3715 static phys_addr_t
intel_iommu_iova_to_phys(struct iommu_domain
*domain
,
3718 struct dmar_domain
*dmar_domain
= domain
->priv
;
3719 struct dma_pte
*pte
;
3722 pte
= pfn_to_dma_pte(dmar_domain
, iova
>> VTD_PAGE_SHIFT
);
3724 phys
= dma_pte_addr(pte
);
3729 static int intel_iommu_domain_has_cap(struct iommu_domain
*domain
,
3732 struct dmar_domain
*dmar_domain
= domain
->priv
;
3734 if (cap
== IOMMU_CAP_CACHE_COHERENCY
)
3735 return dmar_domain
->iommu_snooping
;
3736 if (cap
== IOMMU_CAP_INTR_REMAP
)
3737 return intr_remapping_enabled
;
3742 static struct iommu_ops intel_iommu_ops
= {
3743 .domain_init
= intel_iommu_domain_init
,
3744 .domain_destroy
= intel_iommu_domain_destroy
,
3745 .attach_dev
= intel_iommu_attach_device
,
3746 .detach_dev
= intel_iommu_detach_device
,
3747 .map
= intel_iommu_map
,
3748 .unmap
= intel_iommu_unmap
,
3749 .iova_to_phys
= intel_iommu_iova_to_phys
,
3750 .domain_has_cap
= intel_iommu_domain_has_cap
,
3753 static void __devinit
quirk_iommu_rwbf(struct pci_dev
*dev
)
3756 * Mobile 4 Series Chipset neglects to set RWBF capability,
3759 printk(KERN_INFO
"DMAR: Forcing write-buffer flush capability\n");
3762 /* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
3763 if (dev
->revision
== 0x07) {
3764 printk(KERN_INFO
"DMAR: Disabling IOMMU for graphics on this chipset\n");
3769 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x2a40, quirk_iommu_rwbf
);
3772 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
3773 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
3774 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
3775 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
3776 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
3777 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
3778 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
3779 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
3781 static void __devinit
quirk_calpella_no_shadow_gtt(struct pci_dev
*dev
)
3785 if (pci_read_config_word(dev
, GGC
, &ggc
))
3788 if (!(ggc
& GGC_MEMORY_VT_ENABLED
)) {
3789 printk(KERN_INFO
"DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
3793 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x0040, quirk_calpella_no_shadow_gtt
);
3794 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x0044, quirk_calpella_no_shadow_gtt
);
3795 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x0062, quirk_calpella_no_shadow_gtt
);
3796 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL
, 0x006a, quirk_calpella_no_shadow_gtt
);
3798 /* On Tylersburg chipsets, some BIOSes have been known to enable the
3799 ISOCH DMAR unit for the Azalia sound device, but not give it any
3800 TLB entries, which causes it to deadlock. Check for that. We do
3801 this in a function called from init_dmars(), instead of in a PCI
3802 quirk, because we don't want to print the obnoxious "BIOS broken"
3803 message if VT-d is actually disabled.
3805 static void __init
check_tylersburg_isoch(void)
3807 struct pci_dev
*pdev
;
3808 uint32_t vtisochctrl
;
3810 /* If there's no Azalia in the system anyway, forget it. */
3811 pdev
= pci_get_device(PCI_VENDOR_ID_INTEL
, 0x3a3e, NULL
);
3816 /* System Management Registers. Might be hidden, in which case
3817 we can't do the sanity check. But that's OK, because the
3818 known-broken BIOSes _don't_ actually hide it, so far. */
3819 pdev
= pci_get_device(PCI_VENDOR_ID_INTEL
, 0x342e, NULL
);
3823 if (pci_read_config_dword(pdev
, 0x188, &vtisochctrl
)) {
3830 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
3831 if (vtisochctrl
& 1)
3834 /* Drop all bits other than the number of TLB entries */
3835 vtisochctrl
&= 0x1c;
3837 /* If we have the recommended number of TLB entries (16), fine. */
3838 if (vtisochctrl
== 0x10)
3841 /* Zero TLB entries? You get to ride the short bus to school. */
3843 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
3844 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3845 dmi_get_system_info(DMI_BIOS_VENDOR
),
3846 dmi_get_system_info(DMI_BIOS_VERSION
),
3847 dmi_get_system_info(DMI_PRODUCT_VERSION
));
3848 iommu_identity_mapping
|= IDENTMAP_AZALIA
;
3852 printk(KERN_WARNING
"DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",