?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
ALSA: hda - Add support for 92HD65 / 92HD66 family of codecs
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / drivers / pci / intel-iommu.c
blobf02c34d26d1b08226599cfa4c183da703b9486ed
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
22 */
23
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/iommu.h>
38 #include <linux/intel-iommu.h>
39 #include <linux/syscore_ops.h>
40 #include <linux/tboot.h>
41 #include <linux/dmi.h>
42 #include <linux/pci-ats.h>
43 #include <asm/cacheflush.h>
44 #include <asm/iommu.h>
45 #include "pci.h"
46
47 #define ROOT_SIZE VTD_PAGE_SIZE
48 #define CONTEXT_SIZE VTD_PAGE_SIZE
49
50 #define IS_BRIDGE_HOST_DEVICE(pdev) \
51 ((pdev->class >> 8) == PCI_CLASS_BRIDGE_HOST)
52 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
54 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
55
56 #define IOAPIC_RANGE_START (0xfee00000)
57 #define IOAPIC_RANGE_END (0xfeefffff)
58 #define IOVA_START_ADDR (0x1000)
59
60 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
61
62 #define MAX_AGAW_WIDTH 64
63
64 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
65 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
66
67 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
68 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
69 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
70 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
71 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
72
73 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
74 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
75 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
76
77 /* page table handling */
78 #define LEVEL_STRIDE (9)
79 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
80
81 static inline int agaw_to_level(int agaw)
82 {
83 return agaw + 2;
84 }
85
86 static inline int agaw_to_width(int agaw)
87 {
88 return 30 + agaw * LEVEL_STRIDE;
89 }
90
91 static inline int width_to_agaw(int width)
92 {
93 return (width - 30) / LEVEL_STRIDE;
94 }
95
96 static inline unsigned int level_to_offset_bits(int level)
97 {
98 return (level - 1) * LEVEL_STRIDE;
99 }
100
101 static inline int pfn_level_offset(unsigned long pfn, int level)
102 {
103 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
104 }
105
106 static inline unsigned long level_mask(int level)
107 {
108 return -1UL << level_to_offset_bits(level);
109 }
110
111 static inline unsigned long level_size(int level)
112 {
113 return 1UL << level_to_offset_bits(level);
114 }
115
116 static inline unsigned long align_to_level(unsigned long pfn, int level)
117 {
118 return (pfn + level_size(level) - 1) & level_mask(level);
119 }
120
121 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
122 {
123 return 1 << ((lvl - 1) * LEVEL_STRIDE);
124 }
125
126 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
127 are never going to work. */
128 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
129 {
130 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
131 }
132
133 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
134 {
135 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
136 }
137 static inline unsigned long page_to_dma_pfn(struct page *pg)
138 {
139 return mm_to_dma_pfn(page_to_pfn(pg));
140 }
141 static inline unsigned long virt_to_dma_pfn(void *p)
142 {
143 return page_to_dma_pfn(virt_to_page(p));
144 }
145
146 /* global iommu list, set NULL for ignored DMAR units */
147 static struct intel_iommu **g_iommus;
148
149 static void __init check_tylersburg_isoch(void);
150 static int rwbf_quirk;
151
152 /*
153 * set to 1 to panic kernel if can't successfully enable VT-d
154 * (used when kernel is launched w/ TXT)
155 */
156 static int force_on = 0;
157
158 /*
159 * 0: Present
160 * 1-11: Reserved
161 * 12-63: Context Ptr (12 - (haw-1))
162 * 64-127: Reserved
163 */
164 struct root_entry {
165 u64 val;
166 u64 rsvd1;
167 };
168 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
169 static inline bool root_present(struct root_entry *root)
170 {
171 return (root->val & 1);
172 }
173 static inline void set_root_present(struct root_entry *root)
174 {
175 root->val |= 1;
176 }
177 static inline void set_root_value(struct root_entry *root, unsigned long value)
178 {
179 root->val |= value & VTD_PAGE_MASK;
180 }
181
182 static inline struct context_entry *
183 get_context_addr_from_root(struct root_entry *root)
184 {
185 return (struct context_entry *)
186 (root_present(root)?phys_to_virt(
187 root->val & VTD_PAGE_MASK) :
188 NULL);
189 }
190
191 /*
192 * low 64 bits:
193 * 0: present
194 * 1: fault processing disable
195 * 2-3: translation type
196 * 12-63: address space root
197 * high 64 bits:
198 * 0-2: address width
199 * 3-6: aval
200 * 8-23: domain id
201 */
202 struct context_entry {
203 u64 lo;
204 u64 hi;
205 };
206
207 static inline bool context_present(struct context_entry *context)
208 {
209 return (context->lo & 1);
210 }
211 static inline void context_set_present(struct context_entry *context)
212 {
213 context->lo |= 1;
214 }
215
216 static inline void context_set_fault_enable(struct context_entry *context)
217 {
218 context->lo &= (((u64)-1) << 2) | 1;
219 }
220
221 static inline void context_set_translation_type(struct context_entry *context,
222 unsigned long value)
223 {
224 context->lo &= (((u64)-1) << 4) | 3;
225 context->lo |= (value & 3) << 2;
226 }
227
228 static inline void context_set_address_root(struct context_entry *context,
229 unsigned long value)
230 {
231 context->lo |= value & VTD_PAGE_MASK;
232 }
233
234 static inline void context_set_address_width(struct context_entry *context,
235 unsigned long value)
236 {
237 context->hi |= value & 7;
238 }
239
240 static inline void context_set_domain_id(struct context_entry *context,
241 unsigned long value)
242 {
243 context->hi |= (value & ((1 << 16) - 1)) << 8;
244 }
245
246 static inline void context_clear_entry(struct context_entry *context)
247 {
248 context->lo = 0;
249 context->hi = 0;
250 }
251
252 /*
253 * 0: readable
254 * 1: writable
255 * 2-6: reserved
256 * 7: super page
257 * 8-10: available
258 * 11: snoop behavior
259 * 12-63: Host physcial address
260 */
261 struct dma_pte {
262 u64 val;
263 };
264
265 static inline void dma_clear_pte(struct dma_pte *pte)
266 {
267 pte->val = 0;
268 }
269
270 static inline void dma_set_pte_readable(struct dma_pte *pte)
271 {
272 pte->val |= DMA_PTE_READ;
273 }
274
275 static inline void dma_set_pte_writable(struct dma_pte *pte)
276 {
277 pte->val |= DMA_PTE_WRITE;
278 }
279
280 static inline void dma_set_pte_snp(struct dma_pte *pte)
281 {
282 pte->val |= DMA_PTE_SNP;
283 }
284
285 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
286 {
287 pte->val = (pte->val & ~3) | (prot & 3);
288 }
289
290 static inline u64 dma_pte_addr(struct dma_pte *pte)
291 {
292 #ifdef CONFIG_64BIT
293 return pte->val & VTD_PAGE_MASK;
294 #else
295 /* Must have a full atomic 64-bit read */
296 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
297 #endif
298 }
299
300 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
301 {
302 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
303 }
304
305 static inline bool dma_pte_present(struct dma_pte *pte)
306 {
307 return (pte->val & 3) != 0;
308 }
309
310 static inline int first_pte_in_page(struct dma_pte *pte)
311 {
312 return !((unsigned long)pte & ~VTD_PAGE_MASK);
313 }
314
315 /*
316 * This domain is a statically identity mapping domain.
317 * 1. This domain creats a static 1:1 mapping to all usable memory.
318 * 2. It maps to each iommu if successful.
319 * 3. Each iommu mapps to this domain if successful.
320 */
321 static struct dmar_domain *si_domain;
322 static int hw_pass_through = 1;
323
324 /* devices under the same p2p bridge are owned in one domain */
325 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
326
327 /* domain represents a virtual machine, more than one devices
328 * across iommus may be owned in one domain, e.g. kvm guest.
329 */
330 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
331
332 /* si_domain contains mulitple devices */
333 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
334
335 struct dmar_domain {
336 int id; /* domain id */
337 int nid; /* node id */
338 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
339
340 struct list_head devices; /* all devices' list */
341 struct iova_domain iovad; /* iova's that belong to this domain */
342
343 struct dma_pte *pgd; /* virtual address */
344 int gaw; /* max guest address width */
345
346 /* adjusted guest address width, 0 is level 2 30-bit */
347 int agaw;
348
349 int flags; /* flags to find out type of domain */
350
351 int iommu_coherency;/* indicate coherency of iommu access */
352 int iommu_snooping; /* indicate snooping control feature*/
353 int iommu_count; /* reference count of iommu */
354 int iommu_superpage;/* Level of superpages supported:
355 0 == 4KiB (no superpages), 1 == 2MiB,
356 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
357 spinlock_t iommu_lock; /* protect iommu set in domain */
358 u64 max_addr; /* maximum mapped address */
359 };
360
361 /* PCI domain-device relationship */
362 struct device_domain_info {
363 struct list_head link; /* link to domain siblings */
364 struct list_head global; /* link to global list */
365 int segment; /* PCI domain */
366 u8 bus; /* PCI bus number */
367 u8 devfn; /* PCI devfn number */
368 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
369 struct intel_iommu *iommu; /* IOMMU used by this device */
370 struct dmar_domain *domain; /* pointer to domain */
371 };
372
373 static void flush_unmaps_timeout(unsigned long data);
374
375 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
376
377 #define HIGH_WATER_MARK 250
378 struct deferred_flush_tables {
379 int next;
380 struct iova *iova[HIGH_WATER_MARK];
381 struct dmar_domain *domain[HIGH_WATER_MARK];
382 };
383
384 static struct deferred_flush_tables *deferred_flush;
385
386 /* bitmap for indexing intel_iommus */
387 static int g_num_of_iommus;
388
389 static DEFINE_SPINLOCK(async_umap_flush_lock);
390 static LIST_HEAD(unmaps_to_do);
391
392 static int timer_on;
393 static long list_size;
394
395 static void domain_remove_dev_info(struct dmar_domain *domain);
396
397 #ifdef CONFIG_DMAR_DEFAULT_ON
398 int dmar_disabled = 0;
399 #else
400 int dmar_disabled = 1;
401 #endif /*CONFIG_DMAR_DEFAULT_ON*/
402
403 static int dmar_map_gfx = 1;
404 static int dmar_forcedac;
405 static int intel_iommu_strict;
406 static int intel_iommu_superpage = 1;
407
408 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
409 static DEFINE_SPINLOCK(device_domain_lock);
410 static LIST_HEAD(device_domain_list);
411
412 static struct iommu_ops intel_iommu_ops;
413
414 static int __init intel_iommu_setup(char *str)
415 {
416 if (!str)
417 return -EINVAL;
418 while (*str) {
419 if (!strncmp(str, "on", 2)) {
420 dmar_disabled = 0;
421 printk(KERN_INFO "Intel-IOMMU: enabled\n");
422 } else if (!strncmp(str, "off", 3)) {
423 dmar_disabled = 1;
424 printk(KERN_INFO "Intel-IOMMU: disabled\n");
425 } else if (!strncmp(str, "igfx_off", 8)) {
426 dmar_map_gfx = 0;
427 printk(KERN_INFO
428 "Intel-IOMMU: disable GFX device mapping\n");
429 } else if (!strncmp(str, "forcedac", 8)) {
430 printk(KERN_INFO
431 "Intel-IOMMU: Forcing DAC for PCI devices\n");
432 dmar_forcedac = 1;
433 } else if (!strncmp(str, "strict", 6)) {
434 printk(KERN_INFO
435 "Intel-IOMMU: disable batched IOTLB flush\n");
436 intel_iommu_strict = 1;
437 } else if (!strncmp(str, "sp_off", 6)) {
438 printk(KERN_INFO
439 "Intel-IOMMU: disable supported super page\n");
440 intel_iommu_superpage = 0;
441 }
442
443 str += strcspn(str, ",");
444 while (*str == ',')
445 str++;
446 }
447 return 0;
448 }
449 __setup("intel_iommu=", intel_iommu_setup);
450
451 static struct kmem_cache *iommu_domain_cache;
452 static struct kmem_cache *iommu_devinfo_cache;
453 static struct kmem_cache *iommu_iova_cache;
454
455 static inline void *alloc_pgtable_page(int node)
456 {
457 struct page *page;
458 void *vaddr = NULL;
459
460 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
461 if (page)
462 vaddr = page_address(page);
463 return vaddr;
464 }
465
466 static inline void free_pgtable_page(void *vaddr)
467 {
468 free_page((unsigned long)vaddr);
469 }
470
471 static inline void *alloc_domain_mem(void)
472 {
473 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
474 }
475
476 static void free_domain_mem(void *vaddr)
477 {
478 kmem_cache_free(iommu_domain_cache, vaddr);
479 }
480
481 static inline void * alloc_devinfo_mem(void)
482 {
483 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
484 }
485
486 static inline void free_devinfo_mem(void *vaddr)
487 {
488 kmem_cache_free(iommu_devinfo_cache, vaddr);
489 }
490
491 struct iova *alloc_iova_mem(void)
492 {
493 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
494 }
495
496 void free_iova_mem(struct iova *iova)
497 {
498 kmem_cache_free(iommu_iova_cache, iova);
499 }
500
501
502 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
503 {
504 unsigned long sagaw;
505 int agaw = -1;
506
507 sagaw = cap_sagaw(iommu->cap);
508 for (agaw = width_to_agaw(max_gaw);
509 agaw >= 0; agaw--) {
510 if (test_bit(agaw, &sagaw))
511 break;
512 }
513
514 return agaw;
515 }
516
517 /*
518 * Calculate max SAGAW for each iommu.
519 */
520 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
521 {
522 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
523 }
524
525 /*
526 * calculate agaw for each iommu.
527 * "SAGAW" may be different across iommus, use a default agaw, and
528 * get a supported less agaw for iommus that don't support the default agaw.
529 */
530 int iommu_calculate_agaw(struct intel_iommu *iommu)
531 {
532 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
533 }
534
535 /* This functionin only returns single iommu in a domain */
536 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
537 {
538 int iommu_id;
539
540 /* si_domain and vm domain should not get here. */
541 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
542 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
543
544 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
545 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
546 return NULL;
547
548 return g_iommus[iommu_id];
549 }
550
551 static void domain_update_iommu_coherency(struct dmar_domain *domain)
552 {
553 int i;
554
555 domain->iommu_coherency = 1;
556
557 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
558 if (!ecap_coherent(g_iommus[i]->ecap)) {
559 domain->iommu_coherency = 0;
560 break;
561 }
562 }
563 }
564
565 static void domain_update_iommu_snooping(struct dmar_domain *domain)
566 {
567 int i;
568
569 domain->iommu_snooping = 1;
570
571 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
572 if (!ecap_sc_support(g_iommus[i]->ecap)) {
573 domain->iommu_snooping = 0;
574 break;
575 }
576 }
577 }
578
579 static void domain_update_iommu_superpage(struct dmar_domain *domain)
580 {
581 int i, mask = 0xf;
582
583 if (!intel_iommu_superpage) {
584 domain->iommu_superpage = 0;
585 return;
586 }
587
588 domain->iommu_superpage = 4; /* 1TiB */
589
590 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
591 mask |= cap_super_page_val(g_iommus[i]->cap);
592 if (!mask) {
593 break;
594 }
595 }
596 domain->iommu_superpage = fls(mask);
597 }
598
599 /* Some capabilities may be different across iommus */
600 static void domain_update_iommu_cap(struct dmar_domain *domain)
601 {
602 domain_update_iommu_coherency(domain);
603 domain_update_iommu_snooping(domain);
604 domain_update_iommu_superpage(domain);
605 }
606
607 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
608 {
609 struct dmar_drhd_unit *drhd = NULL;
610 int i;
611
612 for_each_drhd_unit(drhd) {
613 if (drhd->ignored)
614 continue;
615 if (segment != drhd->segment)
616 continue;
617
618 for (i = 0; i < drhd->devices_cnt; i++) {
619 if (drhd->devices[i] &&
620 drhd->devices[i]->bus->number == bus &&
621 drhd->devices[i]->devfn == devfn)
622 return drhd->iommu;
623 if (drhd->devices[i] &&
624 drhd->devices[i]->subordinate &&
625 drhd->devices[i]->subordinate->number <= bus &&
626 drhd->devices[i]->subordinate->subordinate >= bus)
627 return drhd->iommu;
628 }
629
630 if (drhd->include_all)
631 return drhd->iommu;
632 }
633
634 return NULL;
635 }
636
637 static void domain_flush_cache(struct dmar_domain *domain,
638 void *addr, int size)
639 {
640 if (!domain->iommu_coherency)
641 clflush_cache_range(addr, size);
642 }
643
644 /* Gets context entry for a given bus and devfn */
645 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
646 u8 bus, u8 devfn)
647 {
648 struct root_entry *root;
649 struct context_entry *context;
650 unsigned long phy_addr;
651 unsigned long flags;
652
653 spin_lock_irqsave(&iommu->lock, flags);
654 root = &iommu->root_entry[bus];
655 context = get_context_addr_from_root(root);
656 if (!context) {
657 context = (struct context_entry *)
658 alloc_pgtable_page(iommu->node);
659 if (!context) {
660 spin_unlock_irqrestore(&iommu->lock, flags);
661 return NULL;
662 }
663 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
664 phy_addr = virt_to_phys((void *)context);
665 set_root_value(root, phy_addr);
666 set_root_present(root);
667 __iommu_flush_cache(iommu, root, sizeof(*root));
668 }
669 spin_unlock_irqrestore(&iommu->lock, flags);
670 return &context[devfn];
671 }
672
673 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
674 {
675 struct root_entry *root;
676 struct context_entry *context;
677 int ret;
678 unsigned long flags;
679
680 spin_lock_irqsave(&iommu->lock, flags);
681 root = &iommu->root_entry[bus];
682 context = get_context_addr_from_root(root);
683 if (!context) {
684 ret = 0;
685 goto out;
686 }
687 ret = context_present(&context[devfn]);
688 out:
689 spin_unlock_irqrestore(&iommu->lock, flags);
690 return ret;
691 }
692
693 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
694 {
695 struct root_entry *root;
696 struct context_entry *context;
697 unsigned long flags;
698
699 spin_lock_irqsave(&iommu->lock, flags);
700 root = &iommu->root_entry[bus];
701 context = get_context_addr_from_root(root);
702 if (context) {
703 context_clear_entry(&context[devfn]);
704 __iommu_flush_cache(iommu, &context[devfn], \
705 sizeof(*context));
706 }
707 spin_unlock_irqrestore(&iommu->lock, flags);
708 }
709
710 static void free_context_table(struct intel_iommu *iommu)
711 {
712 struct root_entry *root;
713 int i;
714 unsigned long flags;
715 struct context_entry *context;
716
717 spin_lock_irqsave(&iommu->lock, flags);
718 if (!iommu->root_entry) {
719 goto out;
720 }
721 for (i = 0; i < ROOT_ENTRY_NR; i++) {
722 root = &iommu->root_entry[i];
723 context = get_context_addr_from_root(root);
724 if (context)
725 free_pgtable_page(context);
726 }
727 free_pgtable_page(iommu->root_entry);
728 iommu->root_entry = NULL;
729 out:
730 spin_unlock_irqrestore(&iommu->lock, flags);
731 }
732
733 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
734 unsigned long pfn, int large_level)
735 {
736 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
737 struct dma_pte *parent, *pte = NULL;
738 int level = agaw_to_level(domain->agaw);
739 int offset, target_level;
740
741 BUG_ON(!domain->pgd);
742 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
743 parent = domain->pgd;
744
745 /* Search pte */
746 if (!large_level)
747 target_level = 1;
748 else
749 target_level = large_level;
750
751 while (level > 0) {
752 void *tmp_page;
753
754 offset = pfn_level_offset(pfn, level);
755 pte = &parent[offset];
756 if (!large_level && (pte->val & DMA_PTE_LARGE_PAGE))
757 break;
758 if (level == target_level)
759 break;
760
761 if (!dma_pte_present(pte)) {
762 uint64_t pteval;
763
764 tmp_page = alloc_pgtable_page(domain->nid);
765
766 if (!tmp_page)
767 return NULL;
768
769 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
770 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
771 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
772 /* Someone else set it while we were thinking; use theirs. */
773 free_pgtable_page(tmp_page);
774 } else {
775 dma_pte_addr(pte);
776 domain_flush_cache(domain, pte, sizeof(*pte));
777 }
778 }
779 parent = phys_to_virt(dma_pte_addr(pte));
780 level--;
781 }
782
783 return pte;
784 }
785
786
787 /* return address's pte at specific level */
788 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
789 unsigned long pfn,
790 int level, int *large_page)
791 {
792 struct dma_pte *parent, *pte = NULL;
793 int total = agaw_to_level(domain->agaw);
794 int offset;
795
796 parent = domain->pgd;
797 while (level <= total) {
798 offset = pfn_level_offset(pfn, total);
799 pte = &parent[offset];
800 if (level == total)
801 return pte;
802
803 if (!dma_pte_present(pte)) {
804 *large_page = total;
805 break;
806 }
807
808 if (pte->val & DMA_PTE_LARGE_PAGE) {
809 *large_page = total;
810 return pte;
811 }
812
813 parent = phys_to_virt(dma_pte_addr(pte));
814 total--;
815 }
816 return NULL;
817 }
818
819 /* clear last level pte, a tlb flush should be followed */
820 static void dma_pte_clear_range(struct dmar_domain *domain,
821 unsigned long start_pfn,
822 unsigned long last_pfn)
823 {
824 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
825 unsigned int large_page = 1;
826 struct dma_pte *first_pte, *pte;
827
828 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
829 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
830 BUG_ON(start_pfn > last_pfn);
831
832 /* we don't need lock here; nobody else touches the iova range */
833 do {
834 large_page = 1;
835 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
836 if (!pte) {
837 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
838 continue;
839 }
840 do {
841 dma_clear_pte(pte);
842 start_pfn += lvl_to_nr_pages(large_page);
843 pte++;
844 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
845
846 domain_flush_cache(domain, first_pte,
847 (void *)pte - (void *)first_pte);
848
849 } while (start_pfn && start_pfn <= last_pfn);
850 }
851
852 /* free page table pages. last level pte should already be cleared */
853 static void dma_pte_free_pagetable(struct dmar_domain *domain,
854 unsigned long start_pfn,
855 unsigned long last_pfn)
856 {
857 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
858 struct dma_pte *first_pte, *pte;
859 int total = agaw_to_level(domain->agaw);
860 int level;
861 unsigned long tmp;
862 int large_page = 2;
863
864 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
865 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
866 BUG_ON(start_pfn > last_pfn);
867
868 /* We don't need lock here; nobody else touches the iova range */
869 level = 2;
870 while (level <= total) {
871 tmp = align_to_level(start_pfn, level);
872
873 /* If we can't even clear one PTE at this level, we're done */
874 if (tmp + level_size(level) - 1 > last_pfn)
875 return;
876
877 do {
878 large_page = level;
879 first_pte = pte = dma_pfn_level_pte(domain, tmp, level, &large_page);
880 if (large_page > level)
881 level = large_page + 1;
882 if (!pte) {
883 tmp = align_to_level(tmp + 1, level + 1);
884 continue;
885 }
886 do {
887 if (dma_pte_present(pte)) {
888 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
889 dma_clear_pte(pte);
890 }
891 pte++;
892 tmp += level_size(level);
893 } while (!first_pte_in_page(pte) &&
894 tmp + level_size(level) - 1 <= last_pfn);
895
896 domain_flush_cache(domain, first_pte,
897 (void *)pte - (void *)first_pte);
898
899 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
900 level++;
901 }
902 /* free pgd */
903 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
904 free_pgtable_page(domain->pgd);
905 domain->pgd = NULL;
906 }
907 }
908
909 /* iommu handling */
910 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
911 {
912 struct root_entry *root;
913 unsigned long flags;
914
915 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
916 if (!root)
917 return -ENOMEM;
918
919 __iommu_flush_cache(iommu, root, ROOT_SIZE);
920
921 spin_lock_irqsave(&iommu->lock, flags);
922 iommu->root_entry = root;
923 spin_unlock_irqrestore(&iommu->lock, flags);
924
925 return 0;
926 }
927
928 static void iommu_set_root_entry(struct intel_iommu *iommu)
929 {
930 void *addr;
931 u32 sts;
932 unsigned long flag;
933
934 addr = iommu->root_entry;
935
936 spin_lock_irqsave(&iommu->register_lock, flag);
937 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
938
939 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
940
941 /* Make sure hardware complete it */
942 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
943 readl, (sts & DMA_GSTS_RTPS), sts);
944
945 spin_unlock_irqrestore(&iommu->register_lock, flag);
946 }
947
948 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
949 {
950 u32 val;
951 unsigned long flag;
952
953 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
954 return;
955
956 spin_lock_irqsave(&iommu->register_lock, flag);
957 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
958
959 /* Make sure hardware complete it */
960 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
961 readl, (!(val & DMA_GSTS_WBFS)), val);
962
963 spin_unlock_irqrestore(&iommu->register_lock, flag);
964 }
965
966 /* return value determine if we need a write buffer flush */
967 static void __iommu_flush_context(struct intel_iommu *iommu,
968 u16 did, u16 source_id, u8 function_mask,
969 u64 type)
970 {
971 u64 val = 0;
972 unsigned long flag;
973
974 switch (type) {
975 case DMA_CCMD_GLOBAL_INVL:
976 val = DMA_CCMD_GLOBAL_INVL;
977 break;
978 case DMA_CCMD_DOMAIN_INVL:
979 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
980 break;
981 case DMA_CCMD_DEVICE_INVL:
982 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
983 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
984 break;
985 default:
986 BUG();
987 }
988 val |= DMA_CCMD_ICC;
989
990 spin_lock_irqsave(&iommu->register_lock, flag);
991 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
992
993 /* Make sure hardware complete it */
994 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
995 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
996
997 spin_unlock_irqrestore(&iommu->register_lock, flag);
998 }
999
1000 /* return value determine if we need a write buffer flush */
1001 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1002 u64 addr, unsigned int size_order, u64 type)
1003 {
1004 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1005 u64 val = 0, val_iva = 0;
1006 unsigned long flag;
1007
1008 switch (type) {
1009 case DMA_TLB_GLOBAL_FLUSH:
1010 /* global flush doesn't need set IVA_REG */
1011 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1012 break;
1013 case DMA_TLB_DSI_FLUSH:
1014 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1015 break;
1016 case DMA_TLB_PSI_FLUSH:
1017 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1018 /* Note: always flush non-leaf currently */
1019 val_iva = size_order | addr;
1020 break;
1021 default:
1022 BUG();
1023 }
1024 /* Note: set drain read/write */
1025 #if 0
1026 /*
1027 * This is probably to be super secure.. Looks like we can
1028 * ignore it without any impact.
1029 */
1030 if (cap_read_drain(iommu->cap))
1031 val |= DMA_TLB_READ_DRAIN;
1032 #endif
1033 if (cap_write_drain(iommu->cap))
1034 val |= DMA_TLB_WRITE_DRAIN;
1035
1036 spin_lock_irqsave(&iommu->register_lock, flag);
1037 /* Note: Only uses first TLB reg currently */
1038 if (val_iva)
1039 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1040 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1041
1042 /* Make sure hardware complete it */
1043 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1044 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1045
1046 spin_unlock_irqrestore(&iommu->register_lock, flag);
1047
1048 /* check IOTLB invalidation granularity */
1049 if (DMA_TLB_IAIG(val) == 0)
1050 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1051 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1052 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1053 (unsigned long long)DMA_TLB_IIRG(type),
1054 (unsigned long long)DMA_TLB_IAIG(val));
1055 }
1056
1057 static struct device_domain_info *iommu_support_dev_iotlb(
1058 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1059 {
1060 int found = 0;
1061 unsigned long flags;
1062 struct device_domain_info *info;
1063 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1064
1065 if (!ecap_dev_iotlb_support(iommu->ecap))
1066 return NULL;
1067
1068 if (!iommu->qi)
1069 return NULL;
1070
1071 spin_lock_irqsave(&device_domain_lock, flags);
1072 list_for_each_entry(info, &domain->devices, link)
1073 if (info->bus == bus && info->devfn == devfn) {
1074 found = 1;
1075 break;
1076 }
1077 spin_unlock_irqrestore(&device_domain_lock, flags);
1078
1079 if (!found || !info->dev)
1080 return NULL;
1081
1082 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1083 return NULL;
1084
1085 if (!dmar_find_matched_atsr_unit(info->dev))
1086 return NULL;
1087
1088 info->iommu = iommu;
1089
1090 return info;
1091 }
1092
1093 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1094 {
1095 if (!info)
1096 return;
1097
1098 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1099 }
1100
1101 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1102 {
1103 if (!info->dev || !pci_ats_enabled(info->dev))
1104 return;
1105
1106 pci_disable_ats(info->dev);
1107 }
1108
1109 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1110 u64 addr, unsigned mask)
1111 {
1112 u16 sid, qdep;
1113 unsigned long flags;
1114 struct device_domain_info *info;
1115
1116 spin_lock_irqsave(&device_domain_lock, flags);
1117 list_for_each_entry(info, &domain->devices, link) {
1118 if (!info->dev || !pci_ats_enabled(info->dev))
1119 continue;
1120
1121 sid = info->bus << 8 | info->devfn;
1122 qdep = pci_ats_queue_depth(info->dev);
1123 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1124 }
1125 spin_unlock_irqrestore(&device_domain_lock, flags);
1126 }
1127
1128 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1129 unsigned long pfn, unsigned int pages, int map)
1130 {
1131 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1132 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1133
1134 BUG_ON(pages == 0);
1135
1136 /*
1137 * Fallback to domain selective flush if no PSI support or the size is
1138 * too big.
1139 * PSI requires page size to be 2 ^ x, and the base address is naturally
1140 * aligned to the size
1141 */
1142 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1143 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1144 DMA_TLB_DSI_FLUSH);
1145 else
1146 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1147 DMA_TLB_PSI_FLUSH);
1148
1149 /*
1150 * In caching mode, changes of pages from non-present to present require
1151 * flush. However, device IOTLB doesn't need to be flushed in this case.
1152 */
1153 if (!cap_caching_mode(iommu->cap) || !map)
1154 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1155 }
1156
1157 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1158 {
1159 u32 pmen;
1160 unsigned long flags;
1161
1162 spin_lock_irqsave(&iommu->register_lock, flags);
1163 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1164 pmen &= ~DMA_PMEN_EPM;
1165 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1166
1167 /* wait for the protected region status bit to clear */
1168 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1169 readl, !(pmen & DMA_PMEN_PRS), pmen);
1170
1171 spin_unlock_irqrestore(&iommu->register_lock, flags);
1172 }
1173
1174 static int iommu_enable_translation(struct intel_iommu *iommu)
1175 {
1176 u32 sts;
1177 unsigned long flags;
1178
1179 spin_lock_irqsave(&iommu->register_lock, flags);
1180 iommu->gcmd |= DMA_GCMD_TE;
1181 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1182
1183 /* Make sure hardware complete it */
1184 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1185 readl, (sts & DMA_GSTS_TES), sts);
1186
1187 spin_unlock_irqrestore(&iommu->register_lock, flags);
1188 return 0;
1189 }
1190
1191 static int iommu_disable_translation(struct intel_iommu *iommu)
1192 {
1193 u32 sts;
1194 unsigned long flag;
1195
1196 spin_lock_irqsave(&iommu->register_lock, flag);
1197 iommu->gcmd &= ~DMA_GCMD_TE;
1198 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1199
1200 /* Make sure hardware complete it */
1201 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1202 readl, (!(sts & DMA_GSTS_TES)), sts);
1203
1204 spin_unlock_irqrestore(&iommu->register_lock, flag);
1205 return 0;
1206 }
1207
1208
1209 static int iommu_init_domains(struct intel_iommu *iommu)
1210 {
1211 unsigned long ndomains;
1212 unsigned long nlongs;
1213
1214 ndomains = cap_ndoms(iommu->cap);
1215 pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,
1216 ndomains);
1217 nlongs = BITS_TO_LONGS(ndomains);
1218
1219 spin_lock_init(&iommu->lock);
1220
1221 /* TBD: there might be 64K domains,
1222 * consider other allocation for future chip
1223 */
1224 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1225 if (!iommu->domain_ids) {
1226 printk(KERN_ERR "Allocating domain id array failed\n");
1227 return -ENOMEM;
1228 }
1229 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1230 GFP_KERNEL);
1231 if (!iommu->domains) {
1232 printk(KERN_ERR "Allocating domain array failed\n");
1233 return -ENOMEM;
1234 }
1235
1236 /*
1237 * if Caching mode is set, then invalid translations are tagged
1238 * with domainid 0. Hence we need to pre-allocate it.
1239 */
1240 if (cap_caching_mode(iommu->cap))
1241 set_bit(0, iommu->domain_ids);
1242 return 0;
1243 }
1244
1245
1246 static void domain_exit(struct dmar_domain *domain);
1247 static void vm_domain_exit(struct dmar_domain *domain);
1248
1249 void free_dmar_iommu(struct intel_iommu *iommu)
1250 {
1251 struct dmar_domain *domain;
1252 int i;
1253 unsigned long flags;
1254
1255 if ((iommu->domains) && (iommu->domain_ids)) {
1256 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1257 domain = iommu->domains[i];
1258 clear_bit(i, iommu->domain_ids);
1259
1260 spin_lock_irqsave(&domain->iommu_lock, flags);
1261 if (--domain->iommu_count == 0) {
1262 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1263 vm_domain_exit(domain);
1264 else
1265 domain_exit(domain);
1266 }
1267 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1268 }
1269 }
1270
1271 if (iommu->gcmd & DMA_GCMD_TE)
1272 iommu_disable_translation(iommu);
1273
1274 if (iommu->irq) {
1275 irq_set_handler_data(iommu->irq, NULL);
1276 /* This will mask the irq */
1277 free_irq(iommu->irq, iommu);
1278 destroy_irq(iommu->irq);
1279 }
1280
1281 kfree(iommu->domains);
1282 kfree(iommu->domain_ids);
1283
1284 g_iommus[iommu->seq_id] = NULL;
1285
1286 /* if all iommus are freed, free g_iommus */
1287 for (i = 0; i < g_num_of_iommus; i++) {
1288 if (g_iommus[i])
1289 break;
1290 }
1291
1292 if (i == g_num_of_iommus)
1293 kfree(g_iommus);
1294
1295 /* free context mapping */
1296 free_context_table(iommu);
1297 }
1298
1299 static struct dmar_domain *alloc_domain(void)
1300 {
1301 struct dmar_domain *domain;
1302
1303 domain = alloc_domain_mem();
1304 if (!domain)
1305 return NULL;
1306
1307 domain->nid = -1;
1308 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1309 domain->flags = 0;
1310
1311 return domain;
1312 }
1313
1314 static int iommu_attach_domain(struct dmar_domain *domain,
1315 struct intel_iommu *iommu)
1316 {
1317 int num;
1318 unsigned long ndomains;
1319 unsigned long flags;
1320
1321 ndomains = cap_ndoms(iommu->cap);
1322
1323 spin_lock_irqsave(&iommu->lock, flags);
1324
1325 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1326 if (num >= ndomains) {
1327 spin_unlock_irqrestore(&iommu->lock, flags);
1328 printk(KERN_ERR "IOMMU: no free domain ids\n");
1329 return -ENOMEM;
1330 }
1331
1332 domain->id = num;
1333 set_bit(num, iommu->domain_ids);
1334 set_bit(iommu->seq_id, &domain->iommu_bmp);
1335 iommu->domains[num] = domain;
1336 spin_unlock_irqrestore(&iommu->lock, flags);
1337
1338 return 0;
1339 }
1340
1341 static void iommu_detach_domain(struct dmar_domain *domain,
1342 struct intel_iommu *iommu)
1343 {
1344 unsigned long flags;
1345 int num, ndomains;
1346 int found = 0;
1347
1348 spin_lock_irqsave(&iommu->lock, flags);
1349 ndomains = cap_ndoms(iommu->cap);
1350 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1351 if (iommu->domains[num] == domain) {
1352 found = 1;
1353 break;
1354 }
1355 }
1356
1357 if (found) {
1358 clear_bit(num, iommu->domain_ids);
1359 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1360 iommu->domains[num] = NULL;
1361 }
1362 spin_unlock_irqrestore(&iommu->lock, flags);
1363 }
1364
1365 static struct iova_domain reserved_iova_list;
1366 static struct lock_class_key reserved_rbtree_key;
1367
1368 static int dmar_init_reserved_ranges(void)
1369 {
1370 struct pci_dev *pdev = NULL;
1371 struct iova *iova;
1372 int i;
1373
1374 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1375
1376 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1377 &reserved_rbtree_key);
1378
1379 /* IOAPIC ranges shouldn't be accessed by DMA */
1380 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1381 IOVA_PFN(IOAPIC_RANGE_END));
1382 if (!iova) {
1383 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1384 return -ENODEV;
1385 }
1386
1387 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1388 for_each_pci_dev(pdev) {
1389 struct resource *r;
1390
1391 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1392 r = &pdev->resource[i];
1393 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1394 continue;
1395 iova = reserve_iova(&reserved_iova_list,
1396 IOVA_PFN(r->start),
1397 IOVA_PFN(r->end));
1398 if (!iova) {
1399 printk(KERN_ERR "Reserve iova failed\n");
1400 return -ENODEV;
1401 }
1402 }
1403 }
1404 return 0;
1405 }
1406
1407 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1408 {
1409 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1410 }
1411
1412 static inline int guestwidth_to_adjustwidth(int gaw)
1413 {
1414 int agaw;
1415 int r = (gaw - 12) % 9;
1416
1417 if (r == 0)
1418 agaw = gaw;
1419 else
1420 agaw = gaw + 9 - r;
1421 if (agaw > 64)
1422 agaw = 64;
1423 return agaw;
1424 }
1425
1426 static int domain_init(struct dmar_domain *domain, int guest_width)
1427 {
1428 struct intel_iommu *iommu;
1429 int adjust_width, agaw;
1430 unsigned long sagaw;
1431
1432 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1433 spin_lock_init(&domain->iommu_lock);
1434
1435 domain_reserve_special_ranges(domain);
1436
1437 /* calculate AGAW */
1438 iommu = domain_get_iommu(domain);
1439 if (guest_width > cap_mgaw(iommu->cap))
1440 guest_width = cap_mgaw(iommu->cap);
1441 domain->gaw = guest_width;
1442 adjust_width = guestwidth_to_adjustwidth(guest_width);
1443 agaw = width_to_agaw(adjust_width);
1444 sagaw = cap_sagaw(iommu->cap);
1445 if (!test_bit(agaw, &sagaw)) {
1446 /* hardware doesn't support it, choose a bigger one */
1447 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1448 agaw = find_next_bit(&sagaw, 5, agaw);
1449 if (agaw >= 5)
1450 return -ENODEV;
1451 }
1452 domain->agaw = agaw;
1453 INIT_LIST_HEAD(&domain->devices);
1454
1455 if (ecap_coherent(iommu->ecap))
1456 domain->iommu_coherency = 1;
1457 else
1458 domain->iommu_coherency = 0;
1459
1460 if (ecap_sc_support(iommu->ecap))
1461 domain->iommu_snooping = 1;
1462 else
1463 domain->iommu_snooping = 0;
1464
1465 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1466 domain->iommu_count = 1;
1467 domain->nid = iommu->node;
1468
1469 /* always allocate the top pgd */
1470 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1471 if (!domain->pgd)
1472 return -ENOMEM;
1473 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1474 return 0;
1475 }
1476
1477 static void domain_exit(struct dmar_domain *domain)
1478 {
1479 struct dmar_drhd_unit *drhd;
1480 struct intel_iommu *iommu;
1481
1482 /* Domain 0 is reserved, so dont process it */
1483 if (!domain)
1484 return;
1485
1486 /* Flush any lazy unmaps that may reference this domain */
1487 if (!intel_iommu_strict)
1488 flush_unmaps_timeout(0);
1489
1490 domain_remove_dev_info(domain);
1491 /* destroy iovas */
1492 put_iova_domain(&domain->iovad);
1493
1494 /* clear ptes */
1495 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1496
1497 /* free page tables */
1498 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1499
1500 for_each_active_iommu(iommu, drhd)
1501 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1502 iommu_detach_domain(domain, iommu);
1503
1504 free_domain_mem(domain);
1505 }
1506
1507 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1508 u8 bus, u8 devfn, int translation)
1509 {
1510 struct context_entry *context;
1511 unsigned long flags;
1512 struct intel_iommu *iommu;
1513 struct dma_pte *pgd;
1514 unsigned long num;
1515 unsigned long ndomains;
1516 int id;
1517 int agaw;
1518 struct device_domain_info *info = NULL;
1519
1520 pr_debug("Set context mapping for %02x:%02x.%d\n",
1521 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1522
1523 BUG_ON(!domain->pgd);
1524 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1525 translation != CONTEXT_TT_MULTI_LEVEL);
1526
1527 iommu = device_to_iommu(segment, bus, devfn);
1528 if (!iommu)
1529 return -ENODEV;
1530
1531 context = device_to_context_entry(iommu, bus, devfn);
1532 if (!context)
1533 return -ENOMEM;
1534 spin_lock_irqsave(&iommu->lock, flags);
1535 if (context_present(context)) {
1536 spin_unlock_irqrestore(&iommu->lock, flags);
1537 return 0;
1538 }
1539
1540 id = domain->id;
1541 pgd = domain->pgd;
1542
1543 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1544 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1545 int found = 0;
1546
1547 /* find an available domain id for this device in iommu */
1548 ndomains = cap_ndoms(iommu->cap);
1549 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1550 if (iommu->domains[num] == domain) {
1551 id = num;
1552 found = 1;
1553 break;
1554 }
1555 }
1556
1557 if (found == 0) {
1558 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1559 if (num >= ndomains) {
1560 spin_unlock_irqrestore(&iommu->lock, flags);
1561 printk(KERN_ERR "IOMMU: no free domain ids\n");
1562 return -EFAULT;
1563 }
1564
1565 set_bit(num, iommu->domain_ids);
1566 iommu->domains[num] = domain;
1567 id = num;
1568 }
1569
1570 /* Skip top levels of page tables for
1571 * iommu which has less agaw than default.
1572 * Unnecessary for PT mode.
1573 */
1574 if (translation != CONTEXT_TT_PASS_THROUGH) {
1575 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1576 pgd = phys_to_virt(dma_pte_addr(pgd));
1577 if (!dma_pte_present(pgd)) {
1578 spin_unlock_irqrestore(&iommu->lock, flags);
1579 return -ENOMEM;
1580 }
1581 }
1582 }
1583 }
1584
1585 context_set_domain_id(context, id);
1586
1587 if (translation != CONTEXT_TT_PASS_THROUGH) {
1588 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1589 translation = info ? CONTEXT_TT_DEV_IOTLB :
1590 CONTEXT_TT_MULTI_LEVEL;
1591 }
1592 /*
1593 * In pass through mode, AW must be programmed to indicate the largest
1594 * AGAW value supported by hardware. And ASR is ignored by hardware.
1595 */
1596 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1597 context_set_address_width(context, iommu->msagaw);
1598 else {
1599 context_set_address_root(context, virt_to_phys(pgd));
1600 context_set_address_width(context, iommu->agaw);
1601 }
1602
1603 context_set_translation_type(context, translation);
1604 context_set_fault_enable(context);
1605 context_set_present(context);
1606 domain_flush_cache(domain, context, sizeof(*context));
1607
1608 /*
1609 * It's a non-present to present mapping. If hardware doesn't cache
1610 * non-present entry we only need to flush the write-buffer. If the
1611 * _does_ cache non-present entries, then it does so in the special
1612 * domain #0, which we have to flush:
1613 */
1614 if (cap_caching_mode(iommu->cap)) {
1615 iommu->flush.flush_context(iommu, 0,
1616 (((u16)bus) << 8) | devfn,
1617 DMA_CCMD_MASK_NOBIT,
1618 DMA_CCMD_DEVICE_INVL);
1619 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1620 } else {
1621 iommu_flush_write_buffer(iommu);
1622 }
1623 iommu_enable_dev_iotlb(info);
1624 spin_unlock_irqrestore(&iommu->lock, flags);
1625
1626 spin_lock_irqsave(&domain->iommu_lock, flags);
1627 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1628 domain->iommu_count++;
1629 if (domain->iommu_count == 1)
1630 domain->nid = iommu->node;
1631 domain_update_iommu_cap(domain);
1632 }
1633 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1634 return 0;
1635 }
1636
1637 static int
1638 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1639 int translation)
1640 {
1641 int ret;
1642 struct pci_dev *tmp, *parent;
1643
1644 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1645 pdev->bus->number, pdev->devfn,
1646 translation);
1647 if (ret)
1648 return ret;
1649
1650 /* dependent device mapping */
1651 tmp = pci_find_upstream_pcie_bridge(pdev);
1652 if (!tmp)
1653 return 0;
1654 /* Secondary interface's bus number and devfn 0 */
1655 parent = pdev->bus->self;
1656 while (parent != tmp) {
1657 ret = domain_context_mapping_one(domain,
1658 pci_domain_nr(parent->bus),
1659 parent->bus->number,
1660 parent->devfn, translation);
1661 if (ret)
1662 return ret;
1663 parent = parent->bus->self;
1664 }
1665 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1666 return domain_context_mapping_one(domain,
1667 pci_domain_nr(tmp->subordinate),
1668 tmp->subordinate->number, 0,
1669 translation);
1670 else /* this is a legacy PCI bridge */
1671 return domain_context_mapping_one(domain,
1672 pci_domain_nr(tmp->bus),
1673 tmp->bus->number,
1674 tmp->devfn,
1675 translation);
1676 }
1677
1678 static int domain_context_mapped(struct pci_dev *pdev)
1679 {
1680 int ret;
1681 struct pci_dev *tmp, *parent;
1682 struct intel_iommu *iommu;
1683
1684 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1685 pdev->devfn);
1686 if (!iommu)
1687 return -ENODEV;
1688
1689 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1690 if (!ret)
1691 return ret;
1692 /* dependent device mapping */
1693 tmp = pci_find_upstream_pcie_bridge(pdev);
1694 if (!tmp)
1695 return ret;
1696 /* Secondary interface's bus number and devfn 0 */
1697 parent = pdev->bus->self;
1698 while (parent != tmp) {
1699 ret = device_context_mapped(iommu, parent->bus->number,
1700 parent->devfn);
1701 if (!ret)
1702 return ret;
1703 parent = parent->bus->self;
1704 }
1705 if (pci_is_pcie(tmp))
1706 return device_context_mapped(iommu, tmp->subordinate->number,
1707 0);
1708 else
1709 return device_context_mapped(iommu, tmp->bus->number,
1710 tmp->devfn);
1711 }
1712
1713 /* Returns a number of VTD pages, but aligned to MM page size */
1714 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1715 size_t size)
1716 {
1717 host_addr &= ~PAGE_MASK;
1718 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1719 }
1720
1721 /* Return largest possible superpage level for a given mapping */
1722 static inline int hardware_largepage_caps(struct dmar_domain *domain,
1723 unsigned long iov_pfn,
1724 unsigned long phy_pfn,
1725 unsigned long pages)
1726 {
1727 int support, level = 1;
1728 unsigned long pfnmerge;
1729
1730 support = domain->iommu_superpage;
1731
1732 /* To use a large page, the virtual *and* physical addresses
1733 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1734 of them will mean we have to use smaller pages. So just
1735 merge them and check both at once. */
1736 pfnmerge = iov_pfn | phy_pfn;
1737
1738 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1739 pages >>= VTD_STRIDE_SHIFT;
1740 if (!pages)
1741 break;
1742 pfnmerge >>= VTD_STRIDE_SHIFT;
1743 level++;
1744 support--;
1745 }
1746 return level;
1747 }
1748
1749 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1750 struct scatterlist *sg, unsigned long phys_pfn,
1751 unsigned long nr_pages, int prot)
1752 {
1753 struct dma_pte *first_pte = NULL, *pte = NULL;
1754 phys_addr_t uninitialized_var(pteval);
1755 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1756 unsigned long sg_res;
1757 unsigned int largepage_lvl = 0;
1758 unsigned long lvl_pages = 0;
1759
1760 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1761
1762 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1763 return -EINVAL;
1764
1765 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1766
1767 if (sg)
1768 sg_res = 0;
1769 else {
1770 sg_res = nr_pages + 1;
1771 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1772 }
1773
1774 while (nr_pages > 0) {
1775 uint64_t tmp;
1776
1777 if (!sg_res) {
1778 sg_res = aligned_nrpages(sg->offset, sg->length);
1779 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1780 sg->dma_length = sg->length;
1781 pteval = page_to_phys(sg_page(sg)) | prot;
1782 phys_pfn = pteval >> VTD_PAGE_SHIFT;
1783 }
1784
1785 if (!pte) {
1786 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1787
1788 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
1789 if (!pte)
1790 return -ENOMEM;
1791 /* It is large page*/
1792 if (largepage_lvl > 1)
1793 pteval |= DMA_PTE_LARGE_PAGE;
1794 else
1795 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
1796
1797 }
1798 /* We don't need lock here, nobody else
1799 * touches the iova range
1800 */
1801 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1802 if (tmp) {
1803 static int dumps = 5;
1804 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1805 iov_pfn, tmp, (unsigned long long)pteval);
1806 if (dumps) {
1807 dumps--;
1808 debug_dma_dump_mappings(NULL);
1809 }
1810 WARN_ON(1);
1811 }
1812
1813 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1814
1815 BUG_ON(nr_pages < lvl_pages);
1816 BUG_ON(sg_res < lvl_pages);
1817
1818 nr_pages -= lvl_pages;
1819 iov_pfn += lvl_pages;
1820 phys_pfn += lvl_pages;
1821 pteval += lvl_pages * VTD_PAGE_SIZE;
1822 sg_res -= lvl_pages;
1823
1824 /* If the next PTE would be the first in a new page, then we
1825 need to flush the cache on the entries we've just written.
1826 And then we'll need to recalculate 'pte', so clear it and
1827 let it get set again in the if (!pte) block above.
1828
1829 If we're done (!nr_pages) we need to flush the cache too.
1830
1831 Also if we've been setting superpages, we may need to
1832 recalculate 'pte' and switch back to smaller pages for the
1833 end of the mapping, if the trailing size is not enough to
1834 use another superpage (i.e. sg_res < lvl_pages). */
1835 pte++;
1836 if (!nr_pages || first_pte_in_page(pte) ||
1837 (largepage_lvl > 1 && sg_res < lvl_pages)) {
1838 domain_flush_cache(domain, first_pte,
1839 (void *)pte - (void *)first_pte);
1840 pte = NULL;
1841 }
1842
1843 if (!sg_res && nr_pages)
1844 sg = sg_next(sg);
1845 }
1846 return 0;
1847 }
1848
1849 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1850 struct scatterlist *sg, unsigned long nr_pages,
1851 int prot)
1852 {
1853 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1854 }
1855
1856 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1857 unsigned long phys_pfn, unsigned long nr_pages,
1858 int prot)
1859 {
1860 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1861 }
1862
1863 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1864 {
1865 if (!iommu)
1866 return;
1867
1868 clear_context_table(iommu, bus, devfn);
1869 iommu->flush.flush_context(iommu, 0, 0, 0,
1870 DMA_CCMD_GLOBAL_INVL);
1871 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1872 }
1873
1874 static void domain_remove_dev_info(struct dmar_domain *domain)
1875 {
1876 struct device_domain_info *info;
1877 unsigned long flags;
1878 struct intel_iommu *iommu;
1879
1880 spin_lock_irqsave(&device_domain_lock, flags);
1881 while (!list_empty(&domain->devices)) {
1882 info = list_entry(domain->devices.next,
1883 struct device_domain_info, link);
1884 list_del(&info->link);
1885 list_del(&info->global);
1886 if (info->dev)
1887 info->dev->dev.archdata.iommu = NULL;
1888 spin_unlock_irqrestore(&device_domain_lock, flags);
1889
1890 iommu_disable_dev_iotlb(info);
1891 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1892 iommu_detach_dev(iommu, info->bus, info->devfn);
1893 free_devinfo_mem(info);
1894
1895 spin_lock_irqsave(&device_domain_lock, flags);
1896 }
1897 spin_unlock_irqrestore(&device_domain_lock, flags);
1898 }
1899
1900 /*
1901 * find_domain
1902 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1903 */
1904 static struct dmar_domain *
1905 find_domain(struct pci_dev *pdev)
1906 {
1907 struct device_domain_info *info;
1908
1909 /* No lock here, assumes no domain exit in normal case */
1910 info = pdev->dev.archdata.iommu;
1911 if (info)
1912 return info->domain;
1913 return NULL;
1914 }
1915
1916 /* domain is initialized */
1917 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1918 {
1919 struct dmar_domain *domain, *found = NULL;
1920 struct intel_iommu *iommu;
1921 struct dmar_drhd_unit *drhd;
1922 struct device_domain_info *info, *tmp;
1923 struct pci_dev *dev_tmp;
1924 unsigned long flags;
1925 int bus = 0, devfn = 0;
1926 int segment;
1927 int ret;
1928
1929 domain = find_domain(pdev);
1930 if (domain)
1931 return domain;
1932
1933 segment = pci_domain_nr(pdev->bus);
1934
1935 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1936 if (dev_tmp) {
1937 if (pci_is_pcie(dev_tmp)) {
1938 bus = dev_tmp->subordinate->number;
1939 devfn = 0;
1940 } else {
1941 bus = dev_tmp->bus->number;
1942 devfn = dev_tmp->devfn;
1943 }
1944 spin_lock_irqsave(&device_domain_lock, flags);
1945 list_for_each_entry(info, &device_domain_list, global) {
1946 if (info->segment == segment &&
1947 info->bus == bus && info->devfn == devfn) {
1948 found = info->domain;
1949 break;
1950 }
1951 }
1952 spin_unlock_irqrestore(&device_domain_lock, flags);
1953 /* pcie-pci bridge already has a domain, uses it */
1954 if (found) {
1955 domain = found;
1956 goto found_domain;
1957 }
1958 }
1959
1960 domain = alloc_domain();
1961 if (!domain)
1962 goto error;
1963
1964 /* Allocate new domain for the device */
1965 drhd = dmar_find_matched_drhd_unit(pdev);
1966 if (!drhd) {
1967 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1968 pci_name(pdev));
1969 return NULL;
1970 }
1971 iommu = drhd->iommu;
1972
1973 ret = iommu_attach_domain(domain, iommu);
1974 if (ret) {
1975 free_domain_mem(domain);
1976 goto error;
1977 }
1978
1979 if (domain_init(domain, gaw)) {
1980 domain_exit(domain);
1981 goto error;
1982 }
1983
1984 /* register pcie-to-pci device */
1985 if (dev_tmp) {
1986 info = alloc_devinfo_mem();
1987 if (!info) {
1988 domain_exit(domain);
1989 goto error;
1990 }
1991 info->segment = segment;
1992 info->bus = bus;
1993 info->devfn = devfn;
1994 info->dev = NULL;
1995 info->domain = domain;
1996 /* This domain is shared by devices under p2p bridge */
1997 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1998
1999 /* pcie-to-pci bridge already has a domain, uses it */
2000 found = NULL;
2001 spin_lock_irqsave(&device_domain_lock, flags);
2002 list_for_each_entry(tmp, &device_domain_list, global) {
2003 if (tmp->segment == segment &&
2004 tmp->bus == bus && tmp->devfn == devfn) {
2005 found = tmp->domain;
2006 break;
2007 }
2008 }
2009 if (found) {
2010 spin_unlock_irqrestore(&device_domain_lock, flags);
2011 free_devinfo_mem(info);
2012 domain_exit(domain);
2013 domain = found;
2014 } else {
2015 list_add(&info->link, &domain->devices);
2016 list_add(&info->global, &device_domain_list);
2017 spin_unlock_irqrestore(&device_domain_lock, flags);
2018 }
2019 }
2020
2021 found_domain:
2022 info = alloc_devinfo_mem();
2023 if (!info)
2024 goto error;
2025 info->segment = segment;
2026 info->bus = pdev->bus->number;
2027 info->devfn = pdev->devfn;
2028 info->dev = pdev;
2029 info->domain = domain;
2030 spin_lock_irqsave(&device_domain_lock, flags);
2031 /* somebody is fast */
2032 found = find_domain(pdev);
2033 if (found != NULL) {
2034 spin_unlock_irqrestore(&device_domain_lock, flags);
2035 if (found != domain) {
2036 domain_exit(domain);
2037 domain = found;
2038 }
2039 free_devinfo_mem(info);
2040 return domain;
2041 }
2042 list_add(&info->link, &domain->devices);
2043 list_add(&info->global, &device_domain_list);
2044 pdev->dev.archdata.iommu = info;
2045 spin_unlock_irqrestore(&device_domain_lock, flags);
2046 return domain;
2047 error:
2048 /* recheck it here, maybe others set it */
2049 return find_domain(pdev);
2050 }
2051
2052 static int iommu_identity_mapping;
2053 #define IDENTMAP_ALL 1
2054 #define IDENTMAP_GFX 2
2055 #define IDENTMAP_AZALIA 4
2056
2057 static int iommu_domain_identity_map(struct dmar_domain *domain,
2058 unsigned long long start,
2059 unsigned long long end)
2060 {
2061 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2062 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2063
2064 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2065 dma_to_mm_pfn(last_vpfn))) {
2066 printk(KERN_ERR "IOMMU: reserve iova failed\n");
2067 return -ENOMEM;
2068 }
2069
2070 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2071 start, end, domain->id);
2072 /*
2073 * RMRR range might have overlap with physical memory range,
2074 * clear it first
2075 */
2076 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2077
2078 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2079 last_vpfn - first_vpfn + 1,
2080 DMA_PTE_READ|DMA_PTE_WRITE);
2081 }
2082
2083 static int iommu_prepare_identity_map(struct pci_dev *pdev,
2084 unsigned long long start,
2085 unsigned long long end)
2086 {
2087 struct dmar_domain *domain;
2088 int ret;
2089
2090 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2091 if (!domain)
2092 return -ENOMEM;
2093
2094 /* For _hardware_ passthrough, don't bother. But for software
2095 passthrough, we do it anyway -- it may indicate a memory
2096 range which is reserved in E820, so which didn't get set
2097 up to start with in si_domain */
2098 if (domain == si_domain && hw_pass_through) {
2099 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2100 pci_name(pdev), start, end);
2101 return 0;
2102 }
2103
2104 printk(KERN_INFO
2105 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2106 pci_name(pdev), start, end);
2107
2108 if (end < start) {
2109 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2110 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2111 dmi_get_system_info(DMI_BIOS_VENDOR),
2112 dmi_get_system_info(DMI_BIOS_VERSION),
2113 dmi_get_system_info(DMI_PRODUCT_VERSION));
2114 ret = -EIO;
2115 goto error;
2116 }
2117
2118 if (end >> agaw_to_width(domain->agaw)) {
2119 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2120 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2121 agaw_to_width(domain->agaw),
2122 dmi_get_system_info(DMI_BIOS_VENDOR),
2123 dmi_get_system_info(DMI_BIOS_VERSION),
2124 dmi_get_system_info(DMI_PRODUCT_VERSION));
2125 ret = -EIO;
2126 goto error;
2127 }
2128
2129 ret = iommu_domain_identity_map(domain, start, end);
2130 if (ret)
2131 goto error;
2132
2133 /* context entry init */
2134 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2135 if (ret)
2136 goto error;
2137
2138 return 0;
2139
2140 error:
2141 domain_exit(domain);
2142 return ret;
2143 }
2144
2145 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2146 struct pci_dev *pdev)
2147 {
2148 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2149 return 0;
2150 return iommu_prepare_identity_map(pdev, rmrr->base_address,
2151 rmrr->end_address);
2152 }
2153
2154 #ifdef CONFIG_DMAR_FLOPPY_WA
2155 static inline void iommu_prepare_isa(void)
2156 {
2157 struct pci_dev *pdev;
2158 int ret;
2159
2160 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2161 if (!pdev)
2162 return;
2163
2164 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2165 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
2166
2167 if (ret)
2168 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2169 "floppy might not work\n");
2170
2171 }
2172 #else
2173 static inline void iommu_prepare_isa(void)
2174 {
2175 return;
2176 }
2177 #endif /* !CONFIG_DMAR_FLPY_WA */
2178
2179 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2180
2181 static int __init si_domain_work_fn(unsigned long start_pfn,
2182 unsigned long end_pfn, void *datax)
2183 {
2184 int *ret = datax;
2185
2186 *ret = iommu_domain_identity_map(si_domain,
2187 (uint64_t)start_pfn << PAGE_SHIFT,
2188 (uint64_t)end_pfn << PAGE_SHIFT);
2189 return *ret;
2190
2191 }
2192
2193 static int __init si_domain_init(int hw)
2194 {
2195 struct dmar_drhd_unit *drhd;
2196 struct intel_iommu *iommu;
2197 int nid, ret = 0;
2198
2199 si_domain = alloc_domain();
2200 if (!si_domain)
2201 return -EFAULT;
2202
2203 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2204
2205 for_each_active_iommu(iommu, drhd) {
2206 ret = iommu_attach_domain(si_domain, iommu);
2207 if (ret) {
2208 domain_exit(si_domain);
2209 return -EFAULT;
2210 }
2211 }
2212
2213 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2214 domain_exit(si_domain);
2215 return -EFAULT;
2216 }
2217
2218 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2219
2220 if (hw)
2221 return 0;
2222
2223 for_each_online_node(nid) {
2224 work_with_active_regions(nid, si_domain_work_fn, &ret);
2225 if (ret)
2226 return ret;
2227 }
2228
2229 return 0;
2230 }
2231
2232 static void domain_remove_one_dev_info(struct dmar_domain *domain,
2233 struct pci_dev *pdev);
2234 static int identity_mapping(struct pci_dev *pdev)
2235 {
2236 struct device_domain_info *info;
2237
2238 if (likely(!iommu_identity_mapping))
2239 return 0;
2240
2241 info = pdev->dev.archdata.iommu;
2242 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2243 return (info->domain == si_domain);
2244
2245 return 0;
2246 }
2247
2248 static int domain_add_dev_info(struct dmar_domain *domain,
2249 struct pci_dev *pdev,
2250 int translation)
2251 {
2252 struct device_domain_info *info;
2253 unsigned long flags;
2254 int ret;
2255
2256 info = alloc_devinfo_mem();
2257 if (!info)
2258 return -ENOMEM;
2259
2260 ret = domain_context_mapping(domain, pdev, translation);
2261 if (ret) {
2262 free_devinfo_mem(info);
2263 return ret;
2264 }
2265
2266 info->segment = pci_domain_nr(pdev->bus);
2267 info->bus = pdev->bus->number;
2268 info->devfn = pdev->devfn;
2269 info->dev = pdev;
2270 info->domain = domain;
2271
2272 spin_lock_irqsave(&device_domain_lock, flags);
2273 list_add(&info->link, &domain->devices);
2274 list_add(&info->global, &device_domain_list);
2275 pdev->dev.archdata.iommu = info;
2276 spin_unlock_irqrestore(&device_domain_lock, flags);
2277
2278 return 0;
2279 }
2280
2281 static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2282 {
2283 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2284 return 1;
2285
2286 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2287 return 1;
2288
2289 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2290 return 0;
2291
2292 /*
2293 * We want to start off with all devices in the 1:1 domain, and
2294 * take them out later if we find they can't access all of memory.
2295 *
2296 * However, we can't do this for PCI devices behind bridges,
2297 * because all PCI devices behind the same bridge will end up
2298 * with the same source-id on their transactions.
2299 *
2300 * Practically speaking, we can't change things around for these
2301 * devices at run-time, because we can't be sure there'll be no
2302 * DMA transactions in flight for any of their siblings.
2303 *
2304 * So PCI devices (unless they're on the root bus) as well as
2305 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2306 * the 1:1 domain, just in _case_ one of their siblings turns out
2307 * not to be able to map all of memory.
2308 */
2309 if (!pci_is_pcie(pdev)) {
2310 if (!pci_is_root_bus(pdev->bus))
2311 return 0;
2312 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2313 return 0;
2314 } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2315 return 0;
2316
2317 /*
2318 * At boot time, we don't yet know if devices will be 64-bit capable.
2319 * Assume that they will -- if they turn out not to be, then we can
2320 * take them out of the 1:1 domain later.
2321 */
2322 if (!startup) {
2323 /*
2324 * If the device's dma_mask is less than the system's memory
2325 * size then this is not a candidate for identity mapping.
2326 */
2327 u64 dma_mask = pdev->dma_mask;
2328
2329 if (pdev->dev.coherent_dma_mask &&
2330 pdev->dev.coherent_dma_mask < dma_mask)
2331 dma_mask = pdev->dev.coherent_dma_mask;
2332
2333 return dma_mask >= dma_get_required_mask(&pdev->dev);
2334 }
2335
2336 return 1;
2337 }
2338
2339 static int __init iommu_prepare_static_identity_mapping(int hw)
2340 {
2341 struct pci_dev *pdev = NULL;
2342 int ret;
2343
2344 ret = si_domain_init(hw);
2345 if (ret)
2346 return -EFAULT;
2347
2348 for_each_pci_dev(pdev) {
2349 /* Skip Host/PCI Bridge devices */
2350 if (IS_BRIDGE_HOST_DEVICE(pdev))
2351 continue;
2352 if (iommu_should_identity_map(pdev, 1)) {
2353 printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2354 hw ? "hardware" : "software", pci_name(pdev));
2355
2356 ret = domain_add_dev_info(si_domain, pdev,
2357 hw ? CONTEXT_TT_PASS_THROUGH :
2358 CONTEXT_TT_MULTI_LEVEL);
2359 if (ret)
2360 return ret;
2361 }
2362 }
2363
2364 return 0;
2365 }
2366
2367 static int __init init_dmars(void)
2368 {
2369 struct dmar_drhd_unit *drhd;
2370 struct dmar_rmrr_unit *rmrr;
2371 struct pci_dev *pdev;
2372 struct intel_iommu *iommu;
2373 int i, ret;
2374
2375 /*
2376 * for each drhd
2377 * allocate root
2378 * initialize and program root entry to not present
2379 * endfor
2380 */
2381 for_each_drhd_unit(drhd) {
2382 g_num_of_iommus++;
2383 /*
2384 * lock not needed as this is only incremented in the single
2385 * threaded kernel __init code path all other access are read
2386 * only
2387 */
2388 }
2389
2390 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2391 GFP_KERNEL);
2392 if (!g_iommus) {
2393 printk(KERN_ERR "Allocating global iommu array failed\n");
2394 ret = -ENOMEM;
2395 goto error;
2396 }
2397
2398 deferred_flush = kzalloc(g_num_of_iommus *
2399 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2400 if (!deferred_flush) {
2401 ret = -ENOMEM;
2402 goto error;
2403 }
2404
2405 for_each_drhd_unit(drhd) {
2406 if (drhd->ignored)
2407 continue;
2408
2409 iommu = drhd->iommu;
2410 g_iommus[iommu->seq_id] = iommu;
2411
2412 ret = iommu_init_domains(iommu);
2413 if (ret)
2414 goto error;
2415
2416 /*
2417 * TBD:
2418 * we could share the same root & context tables
2419 * among all IOMMU's. Need to Split it later.
2420 */
2421 ret = iommu_alloc_root_entry(iommu);
2422 if (ret) {
2423 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2424 goto error;
2425 }
2426 if (!ecap_pass_through(iommu->ecap))
2427 hw_pass_through = 0;
2428 }
2429
2430 /*
2431 * Start from the sane iommu hardware state.
2432 */
2433 for_each_drhd_unit(drhd) {
2434 if (drhd->ignored)
2435 continue;
2436
2437 iommu = drhd->iommu;
2438
2439 /*
2440 * If the queued invalidation is already initialized by us
2441 * (for example, while enabling interrupt-remapping) then
2442 * we got the things already rolling from a sane state.
2443 */
2444 if (iommu->qi)
2445 continue;
2446
2447 /*
2448 * Clear any previous faults.
2449 */
2450 dmar_fault(-1, iommu);
2451 /*
2452 * Disable queued invalidation if supported and already enabled
2453 * before OS handover.
2454 */
2455 dmar_disable_qi(iommu);
2456 }
2457
2458 for_each_drhd_unit(drhd) {
2459 if (drhd->ignored)
2460 continue;
2461
2462 iommu = drhd->iommu;
2463
2464 if (dmar_enable_qi(iommu)) {
2465 /*
2466 * Queued Invalidate not enabled, use Register Based
2467 * Invalidate
2468 */
2469 iommu->flush.flush_context = __iommu_flush_context;
2470 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2471 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2472 "invalidation\n",
2473 iommu->seq_id,
2474 (unsigned long long)drhd->reg_base_addr);
2475 } else {
2476 iommu->flush.flush_context = qi_flush_context;
2477 iommu->flush.flush_iotlb = qi_flush_iotlb;
2478 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2479 "invalidation\n",
2480 iommu->seq_id,
2481 (unsigned long long)drhd->reg_base_addr);
2482 }
2483 }
2484
2485 if (iommu_pass_through)
2486 iommu_identity_mapping |= IDENTMAP_ALL;
2487
2488 #ifdef CONFIG_DMAR_BROKEN_GFX_WA
2489 iommu_identity_mapping |= IDENTMAP_GFX;
2490 #endif
2491
2492 check_tylersburg_isoch();
2493
2494 /*
2495 * If pass through is not set or not enabled, setup context entries for
2496 * identity mappings for rmrr, gfx, and isa and may fall back to static
2497 * identity mapping if iommu_identity_mapping is set.
2498 */
2499 if (iommu_identity_mapping) {
2500 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2501 if (ret) {
2502 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2503 goto error;
2504 }
2505 }
2506 /*
2507 * For each rmrr
2508 * for each dev attached to rmrr
2509 * do
2510 * locate drhd for dev, alloc domain for dev
2511 * allocate free domain
2512 * allocate page table entries for rmrr
2513 * if context not allocated for bus
2514 * allocate and init context
2515 * set present in root table for this bus
2516 * init context with domain, translation etc
2517 * endfor
2518 * endfor
2519 */
2520 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2521 for_each_rmrr_units(rmrr) {
2522 for (i = 0; i < rmrr->devices_cnt; i++) {
2523 pdev = rmrr->devices[i];
2524 /*
2525 * some BIOS lists non-exist devices in DMAR
2526 * table.
2527 */
2528 if (!pdev)
2529 continue;
2530 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2531 if (ret)
2532 printk(KERN_ERR
2533 "IOMMU: mapping reserved region failed\n");
2534 }
2535 }
2536
2537 iommu_prepare_isa();
2538
2539 /*
2540 * for each drhd
2541 * enable fault log
2542 * global invalidate context cache
2543 * global invalidate iotlb
2544 * enable translation
2545 */
2546 for_each_drhd_unit(drhd) {
2547 if (drhd->ignored) {
2548 /*
2549 * we always have to disable PMRs or DMA may fail on
2550 * this device
2551 */
2552 if (force_on)
2553 iommu_disable_protect_mem_regions(drhd->iommu);
2554 continue;
2555 }
2556 iommu = drhd->iommu;
2557
2558 iommu_flush_write_buffer(iommu);
2559
2560 ret = dmar_set_interrupt(iommu);
2561 if (ret)
2562 goto error;
2563
2564 iommu_set_root_entry(iommu);
2565
2566 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2567 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2568
2569 ret = iommu_enable_translation(iommu);
2570 if (ret)
2571 goto error;
2572
2573 iommu_disable_protect_mem_regions(iommu);
2574 }
2575
2576 return 0;
2577 error:
2578 for_each_drhd_unit(drhd) {
2579 if (drhd->ignored)
2580 continue;
2581 iommu = drhd->iommu;
2582 free_iommu(iommu);
2583 }
2584 kfree(g_iommus);
2585 return ret;
2586 }
2587
2588 /* This takes a number of _MM_ pages, not VTD pages */
2589 static struct iova *intel_alloc_iova(struct device *dev,
2590 struct dmar_domain *domain,
2591 unsigned long nrpages, uint64_t dma_mask)
2592 {
2593 struct pci_dev *pdev = to_pci_dev(dev);
2594 struct iova *iova = NULL;
2595
2596 /* Restrict dma_mask to the width that the iommu can handle */
2597 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2598
2599 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2600 /*
2601 * First try to allocate an io virtual address in
2602 * DMA_BIT_MASK(32) and if that fails then try allocating
2603 * from higher range
2604 */
2605 iova = alloc_iova(&domain->iovad, nrpages,
2606 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2607 if (iova)
2608 return iova;
2609 }
2610 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2611 if (unlikely(!iova)) {
2612 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2613 nrpages, pci_name(pdev));
2614 return NULL;
2615 }
2616
2617 return iova;
2618 }
2619
2620 static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2621 {
2622 struct dmar_domain *domain;
2623 int ret;
2624
2625 domain = get_domain_for_dev(pdev,
2626 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2627 if (!domain) {
2628 printk(KERN_ERR
2629 "Allocating domain for %s failed", pci_name(pdev));
2630 return NULL;
2631 }
2632
2633 /* make sure context mapping is ok */
2634 if (unlikely(!domain_context_mapped(pdev))) {
2635 ret = domain_context_mapping(domain, pdev,
2636 CONTEXT_TT_MULTI_LEVEL);
2637 if (ret) {
2638 printk(KERN_ERR
2639 "Domain context map for %s failed",
2640 pci_name(pdev));
2641 return NULL;
2642 }
2643 }
2644
2645 return domain;
2646 }
2647
2648 static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2649 {
2650 struct device_domain_info *info;
2651
2652 /* No lock here, assumes no domain exit in normal case */
2653 info = dev->dev.archdata.iommu;
2654 if (likely(info))
2655 return info->domain;
2656
2657 return __get_valid_domain_for_dev(dev);
2658 }
2659
2660 static int iommu_dummy(struct pci_dev *pdev)
2661 {
2662 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2663 }
2664
2665 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2666 static int iommu_no_mapping(struct device *dev)
2667 {
2668 struct pci_dev *pdev;
2669 int found;
2670
2671 if (unlikely(dev->bus != &pci_bus_type))
2672 return 1;
2673
2674 pdev = to_pci_dev(dev);
2675 if (iommu_dummy(pdev))
2676 return 1;
2677
2678 if (!iommu_identity_mapping)
2679 return 0;
2680
2681 found = identity_mapping(pdev);
2682 if (found) {
2683 if (iommu_should_identity_map(pdev, 0))
2684 return 1;
2685 else {
2686 /*
2687 * 32 bit DMA is removed from si_domain and fall back
2688 * to non-identity mapping.
2689 */
2690 domain_remove_one_dev_info(si_domain, pdev);
2691 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2692 pci_name(pdev));
2693 return 0;
2694 }
2695 } else {
2696 /*
2697 * In case of a detached 64 bit DMA device from vm, the device
2698 * is put into si_domain for identity mapping.
2699 */
2700 if (iommu_should_identity_map(pdev, 0)) {
2701 int ret;
2702 ret = domain_add_dev_info(si_domain, pdev,
2703 hw_pass_through ?
2704 CONTEXT_TT_PASS_THROUGH :
2705 CONTEXT_TT_MULTI_LEVEL);
2706 if (!ret) {
2707 printk(KERN_INFO "64bit %s uses identity mapping\n",
2708 pci_name(pdev));
2709 return 1;
2710 }
2711 }
2712 }
2713
2714 return 0;
2715 }
2716
2717 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2718 size_t size, int dir, u64 dma_mask)
2719 {
2720 struct pci_dev *pdev = to_pci_dev(hwdev);
2721 struct dmar_domain *domain;
2722 phys_addr_t start_paddr;
2723 struct iova *iova;
2724 int prot = 0;
2725 int ret;
2726 struct intel_iommu *iommu;
2727 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2728
2729 BUG_ON(dir == DMA_NONE);
2730
2731 if (iommu_no_mapping(hwdev))
2732 return paddr;
2733
2734 domain = get_valid_domain_for_dev(pdev);
2735 if (!domain)
2736 return 0;
2737
2738 iommu = domain_get_iommu(domain);
2739 size = aligned_nrpages(paddr, size);
2740
2741 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
2742 if (!iova)
2743 goto error;
2744
2745 /*
2746 * Check if DMAR supports zero-length reads on write only
2747 * mappings..
2748 */
2749 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2750 !cap_zlr(iommu->cap))
2751 prot |= DMA_PTE_READ;
2752 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2753 prot |= DMA_PTE_WRITE;
2754 /*
2755 * paddr - (paddr + size) might be partial page, we should map the whole
2756 * page. Note: if two part of one page are separately mapped, we
2757 * might have two guest_addr mapping to the same host paddr, but this
2758 * is not a big problem
2759 */
2760 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2761 mm_to_dma_pfn(paddr_pfn), size, prot);
2762 if (ret)
2763 goto error;
2764
2765 /* it's a non-present to present mapping. Only flush if caching mode */
2766 if (cap_caching_mode(iommu->cap))
2767 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
2768 else
2769 iommu_flush_write_buffer(iommu);
2770
2771 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2772 start_paddr += paddr & ~PAGE_MASK;
2773 return start_paddr;
2774
2775 error:
2776 if (iova)
2777 __free_iova(&domain->iovad, iova);
2778 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2779 pci_name(pdev), size, (unsigned long long)paddr, dir);
2780 return 0;
2781 }
2782
2783 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2784 unsigned long offset, size_t size,
2785 enum dma_data_direction dir,
2786 struct dma_attrs *attrs)
2787 {
2788 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2789 dir, to_pci_dev(dev)->dma_mask);
2790 }
2791
2792 static void flush_unmaps(void)
2793 {
2794 int i, j;
2795
2796 timer_on = 0;
2797
2798 /* just flush them all */
2799 for (i = 0; i < g_num_of_iommus; i++) {
2800 struct intel_iommu *iommu = g_iommus[i];
2801 if (!iommu)
2802 continue;
2803
2804 if (!deferred_flush[i].next)
2805 continue;
2806
2807 /* In caching mode, global flushes turn emulation expensive */
2808 if (!cap_caching_mode(iommu->cap))
2809 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2810 DMA_TLB_GLOBAL_FLUSH);
2811 for (j = 0; j < deferred_flush[i].next; j++) {
2812 unsigned long mask;
2813 struct iova *iova = deferred_flush[i].iova[j];
2814 struct dmar_domain *domain = deferred_flush[i].domain[j];
2815
2816 /* On real hardware multiple invalidations are expensive */
2817 if (cap_caching_mode(iommu->cap))
2818 iommu_flush_iotlb_psi(iommu, domain->id,
2819 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2820 else {
2821 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2822 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2823 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2824 }
2825 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2826 }
2827 deferred_flush[i].next = 0;
2828 }
2829
2830 list_size = 0;
2831 }
2832
2833 static void flush_unmaps_timeout(unsigned long data)
2834 {
2835 unsigned long flags;
2836
2837 spin_lock_irqsave(&async_umap_flush_lock, flags);
2838 flush_unmaps();
2839 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2840 }
2841
2842 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2843 {
2844 unsigned long flags;
2845 int next, iommu_id;
2846 struct intel_iommu *iommu;
2847
2848 spin_lock_irqsave(&async_umap_flush_lock, flags);
2849 if (list_size == HIGH_WATER_MARK)
2850 flush_unmaps();
2851
2852 iommu = domain_get_iommu(dom);
2853 iommu_id = iommu->seq_id;
2854
2855 next = deferred_flush[iommu_id].next;
2856 deferred_flush[iommu_id].domain[next] = dom;
2857 deferred_flush[iommu_id].iova[next] = iova;
2858 deferred_flush[iommu_id].next++;
2859
2860 if (!timer_on) {
2861 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2862 timer_on = 1;
2863 }
2864 list_size++;
2865 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2866 }
2867
2868 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2869 size_t size, enum dma_data_direction dir,
2870 struct dma_attrs *attrs)
2871 {
2872 struct pci_dev *pdev = to_pci_dev(dev);
2873 struct dmar_domain *domain;
2874 unsigned long start_pfn, last_pfn;
2875 struct iova *iova;
2876 struct intel_iommu *iommu;
2877
2878 if (iommu_no_mapping(dev))
2879 return;
2880
2881 domain = find_domain(pdev);
2882 BUG_ON(!domain);
2883
2884 iommu = domain_get_iommu(domain);
2885
2886 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2887 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2888 (unsigned long long)dev_addr))
2889 return;
2890
2891 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2892 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2893
2894 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2895 pci_name(pdev), start_pfn, last_pfn);
2896
2897 /* clear the whole page */
2898 dma_pte_clear_range(domain, start_pfn, last_pfn);
2899
2900 /* free page tables */
2901 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2902
2903 if (intel_iommu_strict) {
2904 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2905 last_pfn - start_pfn + 1, 0);
2906 /* free iova */
2907 __free_iova(&domain->iovad, iova);
2908 } else {
2909 add_unmap(domain, iova);
2910 /*
2911 * queue up the release of the unmap to save the 1/6th of the
2912 * cpu used up by the iotlb flush operation...
2913 */
2914 }
2915 }
2916
2917 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2918 dma_addr_t *dma_handle, gfp_t flags)
2919 {
2920 void *vaddr;
2921 int order;
2922
2923 size = PAGE_ALIGN(size);
2924 order = get_order(size);
2925
2926 if (!iommu_no_mapping(hwdev))
2927 flags &= ~(GFP_DMA | GFP_DMA32);
2928 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2929 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2930 flags |= GFP_DMA;
2931 else
2932 flags |= GFP_DMA32;
2933 }
2934
2935 vaddr = (void *)__get_free_pages(flags, order);
2936 if (!vaddr)
2937 return NULL;
2938 memset(vaddr, 0, size);
2939
2940 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2941 DMA_BIDIRECTIONAL,
2942 hwdev->coherent_dma_mask);
2943 if (*dma_handle)
2944 return vaddr;
2945 free_pages((unsigned long)vaddr, order);
2946 return NULL;
2947 }
2948
2949 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2950 dma_addr_t dma_handle)
2951 {
2952 int order;
2953
2954 size = PAGE_ALIGN(size);
2955 order = get_order(size);
2956
2957 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
2958 free_pages((unsigned long)vaddr, order);
2959 }
2960
2961 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2962 int nelems, enum dma_data_direction dir,
2963 struct dma_attrs *attrs)
2964 {
2965 struct pci_dev *pdev = to_pci_dev(hwdev);
2966 struct dmar_domain *domain;
2967 unsigned long start_pfn, last_pfn;
2968 struct iova *iova;
2969 struct intel_iommu *iommu;
2970
2971 if (iommu_no_mapping(hwdev))
2972 return;
2973
2974 domain = find_domain(pdev);
2975 BUG_ON(!domain);
2976
2977 iommu = domain_get_iommu(domain);
2978
2979 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2980 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2981 (unsigned long long)sglist[0].dma_address))
2982 return;
2983
2984 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2985 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2986
2987 /* clear the whole page */
2988 dma_pte_clear_range(domain, start_pfn, last_pfn);
2989
2990 /* free page tables */
2991 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2992
2993 if (intel_iommu_strict) {
2994 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2995 last_pfn - start_pfn + 1, 0);
2996 /* free iova */
2997 __free_iova(&domain->iovad, iova);
2998 } else {
2999 add_unmap(domain, iova);
3000 /*
3001 * queue up the release of the unmap to save the 1/6th of the
3002 * cpu used up by the iotlb flush operation...
3003 */
3004 }
3005 }
3006
3007 static int intel_nontranslate_map_sg(struct device *hddev,
3008 struct scatterlist *sglist, int nelems, int dir)
3009 {
3010 int i;
3011 struct scatterlist *sg;
3012
3013 for_each_sg(sglist, sg, nelems, i) {
3014 BUG_ON(!sg_page(sg));
3015 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3016 sg->dma_length = sg->length;
3017 }
3018 return nelems;
3019 }
3020
3021 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3022 enum dma_data_direction dir, struct dma_attrs *attrs)
3023 {
3024 int i;
3025 struct pci_dev *pdev = to_pci_dev(hwdev);
3026 struct dmar_domain *domain;
3027 size_t size = 0;
3028 int prot = 0;
3029 struct iova *iova = NULL;
3030 int ret;
3031 struct scatterlist *sg;
3032 unsigned long start_vpfn;
3033 struct intel_iommu *iommu;
3034
3035 BUG_ON(dir == DMA_NONE);
3036 if (iommu_no_mapping(hwdev))
3037 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
3038
3039 domain = get_valid_domain_for_dev(pdev);
3040 if (!domain)
3041 return 0;
3042
3043 iommu = domain_get_iommu(domain);
3044
3045 for_each_sg(sglist, sg, nelems, i)
3046 size += aligned_nrpages(sg->offset, sg->length);
3047
3048 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3049 pdev->dma_mask);
3050 if (!iova) {
3051 sglist->dma_length = 0;
3052 return 0;
3053 }
3054
3055 /*
3056 * Check if DMAR supports zero-length reads on write only
3057 * mappings..
3058 */
3059 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3060 !cap_zlr(iommu->cap))
3061 prot |= DMA_PTE_READ;
3062 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3063 prot |= DMA_PTE_WRITE;
3064
3065 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3066
3067 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3068 if (unlikely(ret)) {
3069 /* clear the page */
3070 dma_pte_clear_range(domain, start_vpfn,
3071 start_vpfn + size - 1);
3072 /* free page tables */
3073 dma_pte_free_pagetable(domain, start_vpfn,
3074 start_vpfn + size - 1);
3075 /* free iova */
3076 __free_iova(&domain->iovad, iova);
3077 return 0;
3078 }
3079
3080 /* it's a non-present to present mapping. Only flush if caching mode */
3081 if (cap_caching_mode(iommu->cap))
3082 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
3083 else
3084 iommu_flush_write_buffer(iommu);
3085
3086 return nelems;
3087 }
3088
3089 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3090 {
3091 return !dma_addr;
3092 }
3093
3094 struct dma_map_ops intel_dma_ops = {
3095 .alloc_coherent = intel_alloc_coherent,
3096 .free_coherent = intel_free_coherent,
3097 .map_sg = intel_map_sg,
3098 .unmap_sg = intel_unmap_sg,
3099 .map_page = intel_map_page,
3100 .unmap_page = intel_unmap_page,
3101 .mapping_error = intel_mapping_error,
3102 };
3103
3104 static inline int iommu_domain_cache_init(void)
3105 {
3106 int ret = 0;
3107
3108 iommu_domain_cache = kmem_cache_create("iommu_domain",
3109 sizeof(struct dmar_domain),
3110 0,
3111 SLAB_HWCACHE_ALIGN,
3112
3113 NULL);
3114 if (!iommu_domain_cache) {
3115 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3116 ret = -ENOMEM;
3117 }
3118
3119 return ret;
3120 }
3121
3122 static inline int iommu_devinfo_cache_init(void)
3123 {
3124 int ret = 0;
3125
3126 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3127 sizeof(struct device_domain_info),
3128 0,
3129 SLAB_HWCACHE_ALIGN,
3130 NULL);
3131 if (!iommu_devinfo_cache) {
3132 printk(KERN_ERR "Couldn't create devinfo cache\n");
3133 ret = -ENOMEM;
3134 }
3135
3136 return ret;
3137 }
3138
3139 static inline int iommu_iova_cache_init(void)
3140 {
3141 int ret = 0;
3142
3143 iommu_iova_cache = kmem_cache_create("iommu_iova",
3144 sizeof(struct iova),
3145 0,
3146 SLAB_HWCACHE_ALIGN,
3147 NULL);
3148 if (!iommu_iova_cache) {
3149 printk(KERN_ERR "Couldn't create iova cache\n");
3150 ret = -ENOMEM;
3151 }
3152
3153 return ret;
3154 }
3155
3156 static int __init iommu_init_mempool(void)
3157 {
3158 int ret;
3159 ret = iommu_iova_cache_init();
3160 if (ret)
3161 return ret;
3162
3163 ret = iommu_domain_cache_init();
3164 if (ret)
3165 goto domain_error;
3166
3167 ret = iommu_devinfo_cache_init();
3168 if (!ret)
3169 return ret;
3170
3171 kmem_cache_destroy(iommu_domain_cache);
3172 domain_error:
3173 kmem_cache_destroy(iommu_iova_cache);
3174
3175 return -ENOMEM;
3176 }
3177
3178 static void __init iommu_exit_mempool(void)
3179 {
3180 kmem_cache_destroy(iommu_devinfo_cache);
3181 kmem_cache_destroy(iommu_domain_cache);
3182 kmem_cache_destroy(iommu_iova_cache);
3183
3184 }
3185
3186 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3187 {
3188 struct dmar_drhd_unit *drhd;
3189 u32 vtbar;
3190 int rc;
3191
3192 /* We know that this device on this chipset has its own IOMMU.
3193 * If we find it under a different IOMMU, then the BIOS is lying
3194 * to us. Hope that the IOMMU for this device is actually
3195 * disabled, and it needs no translation...
3196 */
3197 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3198 if (rc) {
3199 /* "can't" happen */
3200 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3201 return;
3202 }
3203 vtbar &= 0xffff0000;
3204
3205 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3206 drhd = dmar_find_matched_drhd_unit(pdev);
3207 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3208 TAINT_FIRMWARE_WORKAROUND,
3209 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3210 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3211 }
3212 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3213
3214 static void __init init_no_remapping_devices(void)
3215 {
3216 struct dmar_drhd_unit *drhd;
3217
3218 for_each_drhd_unit(drhd) {
3219 if (!drhd->include_all) {
3220 int i;
3221 for (i = 0; i < drhd->devices_cnt; i++)
3222 if (drhd->devices[i] != NULL)
3223 break;
3224 /* ignore DMAR unit if no pci devices exist */
3225 if (i == drhd->devices_cnt)
3226 drhd->ignored = 1;
3227 }
3228 }
3229
3230 if (dmar_map_gfx)
3231 return;
3232
3233 for_each_drhd_unit(drhd) {
3234 int i;
3235 if (drhd->ignored || drhd->include_all)
3236 continue;
3237
3238 for (i = 0; i < drhd->devices_cnt; i++)
3239 if (drhd->devices[i] &&
3240 !IS_GFX_DEVICE(drhd->devices[i]))
3241 break;
3242
3243 if (i < drhd->devices_cnt)
3244 continue;
3245
3246 /* bypass IOMMU if it is just for gfx devices */
3247 drhd->ignored = 1;
3248 for (i = 0; i < drhd->devices_cnt; i++) {
3249 if (!drhd->devices[i])
3250 continue;
3251 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3252 }
3253 }
3254 }
3255
3256 #ifdef CONFIG_SUSPEND
3257 static int init_iommu_hw(void)
3258 {
3259 struct dmar_drhd_unit *drhd;
3260 struct intel_iommu *iommu = NULL;
3261
3262 for_each_active_iommu(iommu, drhd)
3263 if (iommu->qi)
3264 dmar_reenable_qi(iommu);
3265
3266 for_each_iommu(iommu, drhd) {
3267 if (drhd->ignored) {
3268 /*
3269 * we always have to disable PMRs or DMA may fail on
3270 * this device
3271 */
3272 if (force_on)
3273 iommu_disable_protect_mem_regions(iommu);
3274 continue;
3275 }
3276
3277 iommu_flush_write_buffer(iommu);
3278
3279 iommu_set_root_entry(iommu);
3280
3281 iommu->flush.flush_context(iommu, 0, 0, 0,
3282 DMA_CCMD_GLOBAL_INVL);
3283 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3284 DMA_TLB_GLOBAL_FLUSH);
3285 if (iommu_enable_translation(iommu))
3286 return 1;
3287 iommu_disable_protect_mem_regions(iommu);
3288 }
3289
3290 return 0;
3291 }
3292
3293 static void iommu_flush_all(void)
3294 {
3295 struct dmar_drhd_unit *drhd;
3296 struct intel_iommu *iommu;
3297
3298 for_each_active_iommu(iommu, drhd) {
3299 iommu->flush.flush_context(iommu, 0, 0, 0,
3300 DMA_CCMD_GLOBAL_INVL);
3301 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3302 DMA_TLB_GLOBAL_FLUSH);
3303 }
3304 }
3305
3306 static int iommu_suspend(void)
3307 {
3308 struct dmar_drhd_unit *drhd;
3309 struct intel_iommu *iommu = NULL;
3310 unsigned long flag;
3311
3312 for_each_active_iommu(iommu, drhd) {
3313 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3314 GFP_ATOMIC);
3315 if (!iommu->iommu_state)
3316 goto nomem;
3317 }
3318
3319 iommu_flush_all();
3320
3321 for_each_active_iommu(iommu, drhd) {
3322 iommu_disable_translation(iommu);
3323
3324 spin_lock_irqsave(&iommu->register_lock, flag);
3325
3326 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3327 readl(iommu->reg + DMAR_FECTL_REG);
3328 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3329 readl(iommu->reg + DMAR_FEDATA_REG);
3330 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3331 readl(iommu->reg + DMAR_FEADDR_REG);
3332 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3333 readl(iommu->reg + DMAR_FEUADDR_REG);
3334
3335 spin_unlock_irqrestore(&iommu->register_lock, flag);
3336 }
3337 return 0;
3338
3339 nomem:
3340 for_each_active_iommu(iommu, drhd)
3341 kfree(iommu->iommu_state);
3342
3343 return -ENOMEM;
3344 }
3345
3346 static void iommu_resume(void)
3347 {
3348 struct dmar_drhd_unit *drhd;
3349 struct intel_iommu *iommu = NULL;
3350 unsigned long flag;
3351
3352 if (init_iommu_hw()) {
3353 if (force_on)
3354 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3355 else
3356 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3357 return;
3358 }
3359
3360 for_each_active_iommu(iommu, drhd) {
3361
3362 spin_lock_irqsave(&iommu->register_lock, flag);
3363
3364 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3365 iommu->reg + DMAR_FECTL_REG);
3366 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3367 iommu->reg + DMAR_FEDATA_REG);
3368 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3369 iommu->reg + DMAR_FEADDR_REG);
3370 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3371 iommu->reg + DMAR_FEUADDR_REG);
3372
3373 spin_unlock_irqrestore(&iommu->register_lock, flag);
3374 }
3375
3376 for_each_active_iommu(iommu, drhd)
3377 kfree(iommu->iommu_state);
3378 }
3379
3380 static struct syscore_ops iommu_syscore_ops = {
3381 .resume = iommu_resume,
3382 .suspend = iommu_suspend,
3383 };
3384
3385 static void __init init_iommu_pm_ops(void)
3386 {
3387 register_syscore_ops(&iommu_syscore_ops);
3388 }
3389
3390 #else
3391 static inline void init_iommu_pm_ops(void) {}
3392 #endif /* CONFIG_PM */
3393
3394 /*
3395 * Here we only respond to action of unbound device from driver.
3396 *
3397 * Added device is not attached to its DMAR domain here yet. That will happen
3398 * when mapping the device to iova.
3399 */
3400 static int device_notifier(struct notifier_block *nb,
3401 unsigned long action, void *data)
3402 {
3403 struct device *dev = data;
3404 struct pci_dev *pdev = to_pci_dev(dev);
3405 struct dmar_domain *domain;
3406
3407 if (iommu_no_mapping(dev))
3408 return 0;
3409
3410 domain = find_domain(pdev);
3411 if (!domain)
3412 return 0;
3413
3414 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
3415 domain_remove_one_dev_info(domain, pdev);
3416
3417 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3418 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3419 list_empty(&domain->devices))
3420 domain_exit(domain);
3421 }
3422
3423 return 0;
3424 }
3425
3426 static struct notifier_block device_nb = {
3427 .notifier_call = device_notifier,
3428 };
3429
3430 int __init intel_iommu_init(void)
3431 {
3432 int ret = 0;
3433
3434 /* VT-d is required for a TXT/tboot launch, so enforce that */
3435 force_on = tboot_force_iommu();
3436
3437 if (dmar_table_init()) {
3438 if (force_on)
3439 panic("tboot: Failed to initialize DMAR table\n");
3440 return -ENODEV;
3441 }
3442
3443 if (dmar_dev_scope_init()) {
3444 if (force_on)
3445 panic("tboot: Failed to initialize DMAR device scope\n");
3446 return -ENODEV;
3447 }
3448
3449 /*
3450 * Check the need for DMA-remapping initialization now.
3451 * Above initialization will also be used by Interrupt-remapping.
3452 */
3453 if (no_iommu || dmar_disabled)
3454 return -ENODEV;
3455
3456 if (iommu_init_mempool()) {
3457 if (force_on)
3458 panic("tboot: Failed to initialize iommu memory\n");
3459 return -ENODEV;
3460 }
3461
3462 if (dmar_init_reserved_ranges()) {
3463 if (force_on)
3464 panic("tboot: Failed to reserve iommu ranges\n");
3465 return -ENODEV;
3466 }
3467
3468 init_no_remapping_devices();
3469
3470 ret = init_dmars();
3471 if (ret) {
3472 if (force_on)
3473 panic("tboot: Failed to initialize DMARs\n");
3474 printk(KERN_ERR "IOMMU: dmar init failed\n");
3475 put_iova_domain(&reserved_iova_list);
3476 iommu_exit_mempool();
3477 return ret;
3478 }
3479 printk(KERN_INFO
3480 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3481
3482 init_timer(&unmap_timer);
3483 #ifdef CONFIG_SWIOTLB
3484 swiotlb = 0;
3485 #endif
3486 dma_ops = &intel_dma_ops;
3487
3488 init_iommu_pm_ops();
3489
3490 register_iommu(&intel_iommu_ops);
3491
3492 bus_register_notifier(&pci_bus_type, &device_nb);
3493
3494 return 0;
3495 }
3496
3497 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3498 struct pci_dev *pdev)
3499 {
3500 struct pci_dev *tmp, *parent;
3501
3502 if (!iommu || !pdev)
3503 return;
3504
3505 /* dependent device detach */
3506 tmp = pci_find_upstream_pcie_bridge(pdev);
3507 /* Secondary interface's bus number and devfn 0 */
3508 if (tmp) {
3509 parent = pdev->bus->self;
3510 while (parent != tmp) {
3511 iommu_detach_dev(iommu, parent->bus->number,
3512 parent->devfn);
3513 parent = parent->bus->self;
3514 }
3515 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3516 iommu_detach_dev(iommu,
3517 tmp->subordinate->number, 0);
3518 else /* this is a legacy PCI bridge */
3519 iommu_detach_dev(iommu, tmp->bus->number,
3520 tmp->devfn);
3521 }
3522 }
3523
3524 static void domain_remove_one_dev_info(struct dmar_domain *domain,
3525 struct pci_dev *pdev)
3526 {
3527 struct device_domain_info *info;
3528 struct intel_iommu *iommu;
3529 unsigned long flags;
3530 int found = 0;
3531 struct list_head *entry, *tmp;
3532
3533 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3534 pdev->devfn);
3535 if (!iommu)
3536 return;
3537
3538 spin_lock_irqsave(&device_domain_lock, flags);
3539 list_for_each_safe(entry, tmp, &domain->devices) {
3540 info = list_entry(entry, struct device_domain_info, link);
3541 if (info->segment == pci_domain_nr(pdev->bus) &&
3542 info->bus == pdev->bus->number &&
3543 info->devfn == pdev->devfn) {
3544 list_del(&info->link);
3545 list_del(&info->global);
3546 if (info->dev)
3547 info->dev->dev.archdata.iommu = NULL;
3548 spin_unlock_irqrestore(&device_domain_lock, flags);
3549
3550 iommu_disable_dev_iotlb(info);
3551 iommu_detach_dev(iommu, info->bus, info->devfn);
3552 iommu_detach_dependent_devices(iommu, pdev);
3553 free_devinfo_mem(info);
3554
3555 spin_lock_irqsave(&device_domain_lock, flags);
3556
3557 if (found)
3558 break;
3559 else
3560 continue;
3561 }
3562
3563 /* if there is no other devices under the same iommu
3564 * owned by this domain, clear this iommu in iommu_bmp
3565 * update iommu count and coherency
3566 */
3567 if (iommu == device_to_iommu(info->segment, info->bus,
3568 info->devfn))
3569 found = 1;
3570 }
3571
3572 if (found == 0) {
3573 unsigned long tmp_flags;
3574 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3575 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3576 domain->iommu_count--;
3577 domain_update_iommu_cap(domain);
3578 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3579
3580 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3581 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3582 spin_lock_irqsave(&iommu->lock, tmp_flags);
3583 clear_bit(domain->id, iommu->domain_ids);
3584 iommu->domains[domain->id] = NULL;
3585 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3586 }
3587 }
3588
3589 spin_unlock_irqrestore(&device_domain_lock, flags);
3590 }
3591
3592 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3593 {
3594 struct device_domain_info *info;
3595 struct intel_iommu *iommu;
3596 unsigned long flags1, flags2;
3597
3598 spin_lock_irqsave(&device_domain_lock, flags1);
3599 while (!list_empty(&domain->devices)) {
3600 info = list_entry(domain->devices.next,
3601 struct device_domain_info, link);
3602 list_del(&info->link);
3603 list_del(&info->global);
3604 if (info->dev)
3605 info->dev->dev.archdata.iommu = NULL;
3606
3607 spin_unlock_irqrestore(&device_domain_lock, flags1);
3608
3609 iommu_disable_dev_iotlb(info);
3610 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3611 iommu_detach_dev(iommu, info->bus, info->devfn);
3612 iommu_detach_dependent_devices(iommu, info->dev);
3613
3614 /* clear this iommu in iommu_bmp, update iommu count
3615 * and capabilities
3616 */
3617 spin_lock_irqsave(&domain->iommu_lock, flags2);
3618 if (test_and_clear_bit(iommu->seq_id,
3619 &domain->iommu_bmp)) {
3620 domain->iommu_count--;
3621 domain_update_iommu_cap(domain);
3622 }
3623 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3624
3625 free_devinfo_mem(info);
3626 spin_lock_irqsave(&device_domain_lock, flags1);
3627 }
3628 spin_unlock_irqrestore(&device_domain_lock, flags1);
3629 }
3630
3631 /* domain id for virtual machine, it won't be set in context */
3632 static unsigned long vm_domid;
3633
3634 static struct dmar_domain *iommu_alloc_vm_domain(void)
3635 {
3636 struct dmar_domain *domain;
3637
3638 domain = alloc_domain_mem();
3639 if (!domain)
3640 return NULL;
3641
3642 domain->id = vm_domid++;
3643 domain->nid = -1;
3644 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3645 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3646
3647 return domain;
3648 }
3649
3650 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3651 {
3652 int adjust_width;
3653
3654 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3655 spin_lock_init(&domain->iommu_lock);
3656
3657 domain_reserve_special_ranges(domain);
3658
3659 /* calculate AGAW */
3660 domain->gaw = guest_width;
3661 adjust_width = guestwidth_to_adjustwidth(guest_width);
3662 domain->agaw = width_to_agaw(adjust_width);
3663
3664 INIT_LIST_HEAD(&domain->devices);
3665
3666 domain->iommu_count = 0;
3667 domain->iommu_coherency = 0;
3668 domain->iommu_snooping = 0;
3669 domain->iommu_superpage = 0;
3670 domain->max_addr = 0;
3671 domain->nid = -1;
3672
3673 /* always allocate the top pgd */
3674 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
3675 if (!domain->pgd)
3676 return -ENOMEM;
3677 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3678 return 0;
3679 }
3680
3681 static void iommu_free_vm_domain(struct dmar_domain *domain)
3682 {
3683 unsigned long flags;
3684 struct dmar_drhd_unit *drhd;
3685 struct intel_iommu *iommu;
3686 unsigned long i;
3687 unsigned long ndomains;
3688
3689 for_each_drhd_unit(drhd) {
3690 if (drhd->ignored)
3691 continue;
3692 iommu = drhd->iommu;
3693
3694 ndomains = cap_ndoms(iommu->cap);
3695 for_each_set_bit(i, iommu->domain_ids, ndomains) {
3696 if (iommu->domains[i] == domain) {
3697 spin_lock_irqsave(&iommu->lock, flags);
3698 clear_bit(i, iommu->domain_ids);
3699 iommu->domains[i] = NULL;
3700 spin_unlock_irqrestore(&iommu->lock, flags);
3701 break;
3702 }
3703 }
3704 }
3705 }
3706
3707 static void vm_domain_exit(struct dmar_domain *domain)
3708 {
3709 /* Domain 0 is reserved, so dont process it */
3710 if (!domain)
3711 return;
3712
3713 vm_domain_remove_all_dev_info(domain);
3714 /* destroy iovas */
3715 put_iova_domain(&domain->iovad);
3716
3717 /* clear ptes */
3718 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3719
3720 /* free page tables */
3721 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3722
3723 iommu_free_vm_domain(domain);
3724 free_domain_mem(domain);
3725 }
3726
3727 static int intel_iommu_domain_init(struct iommu_domain *domain)
3728 {
3729 struct dmar_domain *dmar_domain;
3730
3731 dmar_domain = iommu_alloc_vm_domain();
3732 if (!dmar_domain) {
3733 printk(KERN_ERR
3734 "intel_iommu_domain_init: dmar_domain == NULL\n");
3735 return -ENOMEM;
3736 }
3737 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3738 printk(KERN_ERR
3739 "intel_iommu_domain_init() failed\n");
3740 vm_domain_exit(dmar_domain);
3741 return -ENOMEM;
3742 }
3743 domain->priv = dmar_domain;
3744
3745 return 0;
3746 }
3747
3748 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3749 {
3750 struct dmar_domain *dmar_domain = domain->priv;
3751
3752 domain->priv = NULL;
3753 vm_domain_exit(dmar_domain);
3754 }
3755
3756 static int intel_iommu_attach_device(struct iommu_domain *domain,
3757 struct device *dev)
3758 {
3759 struct dmar_domain *dmar_domain = domain->priv;
3760 struct pci_dev *pdev = to_pci_dev(dev);
3761 struct intel_iommu *iommu;
3762 int addr_width;
3763
3764 /* normally pdev is not mapped */
3765 if (unlikely(domain_context_mapped(pdev))) {
3766 struct dmar_domain *old_domain;
3767
3768 old_domain = find_domain(pdev);
3769 if (old_domain) {
3770 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3771 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3772 domain_remove_one_dev_info(old_domain, pdev);
3773 else
3774 domain_remove_dev_info(old_domain);
3775 }
3776 }
3777
3778 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3779 pdev->devfn);
3780 if (!iommu)
3781 return -ENODEV;
3782
3783 /* check if this iommu agaw is sufficient for max mapped address */
3784 addr_width = agaw_to_width(iommu->agaw);
3785 if (addr_width > cap_mgaw(iommu->cap))
3786 addr_width = cap_mgaw(iommu->cap);
3787
3788 if (dmar_domain->max_addr > (1LL << addr_width)) {
3789 printk(KERN_ERR "%s: iommu width (%d) is not "
3790 "sufficient for the mapped address (%llx)\n",
3791 __func__, addr_width, dmar_domain->max_addr);
3792 return -EFAULT;
3793 }
3794 dmar_domain->gaw = addr_width;
3795
3796 /*
3797 * Knock out extra levels of page tables if necessary
3798 */
3799 while (iommu->agaw < dmar_domain->agaw) {
3800 struct dma_pte *pte;
3801
3802 pte = dmar_domain->pgd;
3803 if (dma_pte_present(pte)) {
3804 dmar_domain->pgd = (struct dma_pte *)
3805 phys_to_virt(dma_pte_addr(pte));
3806 free_pgtable_page(pte);
3807 }
3808 dmar_domain->agaw--;
3809 }
3810
3811 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
3812 }
3813
3814 static void intel_iommu_detach_device(struct iommu_domain *domain,
3815 struct device *dev)
3816 {
3817 struct dmar_domain *dmar_domain = domain->priv;
3818 struct pci_dev *pdev = to_pci_dev(dev);
3819
3820 domain_remove_one_dev_info(dmar_domain, pdev);
3821 }
3822
3823 static int intel_iommu_map(struct iommu_domain *domain,
3824 unsigned long iova, phys_addr_t hpa,
3825 int gfp_order, int iommu_prot)
3826 {
3827 struct dmar_domain *dmar_domain = domain->priv;
3828 u64 max_addr;
3829 int prot = 0;
3830 size_t size;
3831 int ret;
3832
3833 if (iommu_prot & IOMMU_READ)
3834 prot |= DMA_PTE_READ;
3835 if (iommu_prot & IOMMU_WRITE)
3836 prot |= DMA_PTE_WRITE;
3837 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3838 prot |= DMA_PTE_SNP;
3839
3840 size = PAGE_SIZE << gfp_order;
3841 max_addr = iova + size;
3842 if (dmar_domain->max_addr < max_addr) {
3843 u64 end;
3844
3845 /* check if minimum agaw is sufficient for mapped address */
3846 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
3847 if (end < max_addr) {
3848 printk(KERN_ERR "%s: iommu width (%d) is not "
3849 "sufficient for the mapped address (%llx)\n",
3850 __func__, dmar_domain->gaw, max_addr);
3851 return -EFAULT;
3852 }
3853 dmar_domain->max_addr = max_addr;
3854 }
3855 /* Round up size to next multiple of PAGE_SIZE, if it and
3856 the low bits of hpa would take us onto the next page */
3857 size = aligned_nrpages(hpa, size);
3858 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3859 hpa >> VTD_PAGE_SHIFT, size, prot);
3860 return ret;
3861 }
3862
3863 static int intel_iommu_unmap(struct iommu_domain *domain,
3864 unsigned long iova, int gfp_order)
3865 {
3866 struct dmar_domain *dmar_domain = domain->priv;
3867 size_t size = PAGE_SIZE << gfp_order;
3868
3869 dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3870 (iova + size - 1) >> VTD_PAGE_SHIFT);
3871
3872 if (dmar_domain->max_addr == iova + size)
3873 dmar_domain->max_addr = iova;
3874
3875 return gfp_order;
3876 }
3877
3878 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3879 unsigned long iova)
3880 {
3881 struct dmar_domain *dmar_domain = domain->priv;
3882 struct dma_pte *pte;
3883 u64 phys = 0;
3884
3885 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
3886 if (pte)
3887 phys = dma_pte_addr(pte);
3888
3889 return phys;
3890 }
3891
3892 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3893 unsigned long cap)
3894 {
3895 struct dmar_domain *dmar_domain = domain->priv;
3896
3897 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3898 return dmar_domain->iommu_snooping;
3899 if (cap == IOMMU_CAP_INTR_REMAP)
3900 return intr_remapping_enabled;
3901
3902 return 0;
3903 }
3904
3905 static struct iommu_ops intel_iommu_ops = {
3906 .domain_init = intel_iommu_domain_init,
3907 .domain_destroy = intel_iommu_domain_destroy,
3908 .attach_dev = intel_iommu_attach_device,
3909 .detach_dev = intel_iommu_detach_device,
3910 .map = intel_iommu_map,
3911 .unmap = intel_iommu_unmap,
3912 .iova_to_phys = intel_iommu_iova_to_phys,
3913 .domain_has_cap = intel_iommu_domain_has_cap,
3914 };
3915
3916 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3917 {
3918 /*
3919 * Mobile 4 Series Chipset neglects to set RWBF capability,
3920 * but needs it:
3921 */
3922 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3923 rwbf_quirk = 1;
3924
3925 /* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
3926 if (dev->revision == 0x07) {
3927 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
3928 dmar_map_gfx = 0;
3929 }
3930 }
3931
3932 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
3933
3934 #define GGC 0x52
3935 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
3936 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
3937 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
3938 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
3939 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
3940 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
3941 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
3942 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
3943
3944 static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
3945 {
3946 unsigned short ggc;
3947
3948 if (pci_read_config_word(dev, GGC, &ggc))
3949 return;
3950
3951 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
3952 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
3953 dmar_map_gfx = 0;
3954 }
3955 }
3956 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
3957 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
3958 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
3959 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
3960
3961 /* On Tylersburg chipsets, some BIOSes have been known to enable the
3962 ISOCH DMAR unit for the Azalia sound device, but not give it any
3963 TLB entries, which causes it to deadlock. Check for that. We do
3964 this in a function called from init_dmars(), instead of in a PCI
3965 quirk, because we don't want to print the obnoxious "BIOS broken"
3966 message if VT-d is actually disabled.
3967 */
3968 static void __init check_tylersburg_isoch(void)
3969 {
3970 struct pci_dev *pdev;
3971 uint32_t vtisochctrl;
3972
3973 /* If there's no Azalia in the system anyway, forget it. */
3974 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3975 if (!pdev)
3976 return;
3977 pci_dev_put(pdev);
3978
3979 /* System Management Registers. Might be hidden, in which case
3980 we can't do the sanity check. But that's OK, because the
3981 known-broken BIOSes _don't_ actually hide it, so far. */
3982 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3983 if (!pdev)
3984 return;
3985
3986 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
3987 pci_dev_put(pdev);
3988 return;
3989 }
3990
3991 pci_dev_put(pdev);
3992
3993 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
3994 if (vtisochctrl & 1)
3995 return;
3996
3997 /* Drop all bits other than the number of TLB entries */
3998 vtisochctrl &= 0x1c;
3999
4000 /* If we have the recommended number of TLB entries (16), fine. */
4001 if (vtisochctrl == 0x10)
4002 return;
4003
4004 /* Zero TLB entries? You get to ride the short bus to school. */
4005 if (!vtisochctrl) {
4006 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4007 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4008 dmi_get_system_info(DMI_BIOS_VENDOR),
4009 dmi_get_system_info(DMI_BIOS_VERSION),
4010 dmi_get_system_info(DMI_PRODUCT_VERSION));
4011 iommu_identity_mapping |= IDENTMAP_AZALIA;
4012 return;
4013 }
4014
4015 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4016 vtisochctrl);
4017 }
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree