?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'master' of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / drivers / pci / intel-iommu.c
blobcb5cae3e0205c5479e37585aa2a6cfd12e20f735
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
22 */
23
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/iommu.h>
38 #include <linux/intel-iommu.h>
39 #include <linux/sysdev.h>
40 #include <linux/tboot.h>
41 #include <linux/dmi.h>
42 #include <asm/cacheflush.h>
43 #include <asm/iommu.h>
44 #include "pci.h"
45
46 #define ROOT_SIZE VTD_PAGE_SIZE
47 #define CONTEXT_SIZE VTD_PAGE_SIZE
48
49 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
50 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
51 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
52
53 #define IOAPIC_RANGE_START (0xfee00000)
54 #define IOAPIC_RANGE_END (0xfeefffff)
55 #define IOVA_START_ADDR (0x1000)
56
57 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
58
59 #define MAX_AGAW_WIDTH 64
60
61 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
62 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
63
64 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
65 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
66 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
67 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
68 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
69
70 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
71 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
72 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
73
74
75 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
76 are never going to work. */
77 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
78 {
79 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
80 }
81
82 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
83 {
84 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
85 }
86 static inline unsigned long page_to_dma_pfn(struct page *pg)
87 {
88 return mm_to_dma_pfn(page_to_pfn(pg));
89 }
90 static inline unsigned long virt_to_dma_pfn(void *p)
91 {
92 return page_to_dma_pfn(virt_to_page(p));
93 }
94
95 /* global iommu list, set NULL for ignored DMAR units */
96 static struct intel_iommu **g_iommus;
97
98 static void __init check_tylersburg_isoch(void);
99 static int rwbf_quirk;
100
101 /*
102 * 0: Present
103 * 1-11: Reserved
104 * 12-63: Context Ptr (12 - (haw-1))
105 * 64-127: Reserved
106 */
107 struct root_entry {
108 u64 val;
109 u64 rsvd1;
110 };
111 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
112 static inline bool root_present(struct root_entry *root)
113 {
114 return (root->val & 1);
115 }
116 static inline void set_root_present(struct root_entry *root)
117 {
118 root->val |= 1;
119 }
120 static inline void set_root_value(struct root_entry *root, unsigned long value)
121 {
122 root->val |= value & VTD_PAGE_MASK;
123 }
124
125 static inline struct context_entry *
126 get_context_addr_from_root(struct root_entry *root)
127 {
128 return (struct context_entry *)
129 (root_present(root)?phys_to_virt(
130 root->val & VTD_PAGE_MASK) :
131 NULL);
132 }
133
134 /*
135 * low 64 bits:
136 * 0: present
137 * 1: fault processing disable
138 * 2-3: translation type
139 * 12-63: address space root
140 * high 64 bits:
141 * 0-2: address width
142 * 3-6: aval
143 * 8-23: domain id
144 */
145 struct context_entry {
146 u64 lo;
147 u64 hi;
148 };
149
150 static inline bool context_present(struct context_entry *context)
151 {
152 return (context->lo & 1);
153 }
154 static inline void context_set_present(struct context_entry *context)
155 {
156 context->lo |= 1;
157 }
158
159 static inline void context_set_fault_enable(struct context_entry *context)
160 {
161 context->lo &= (((u64)-1) << 2) | 1;
162 }
163
164 static inline void context_set_translation_type(struct context_entry *context,
165 unsigned long value)
166 {
167 context->lo &= (((u64)-1) << 4) | 3;
168 context->lo |= (value & 3) << 2;
169 }
170
171 static inline void context_set_address_root(struct context_entry *context,
172 unsigned long value)
173 {
174 context->lo |= value & VTD_PAGE_MASK;
175 }
176
177 static inline void context_set_address_width(struct context_entry *context,
178 unsigned long value)
179 {
180 context->hi |= value & 7;
181 }
182
183 static inline void context_set_domain_id(struct context_entry *context,
184 unsigned long value)
185 {
186 context->hi |= (value & ((1 << 16) - 1)) << 8;
187 }
188
189 static inline void context_clear_entry(struct context_entry *context)
190 {
191 context->lo = 0;
192 context->hi = 0;
193 }
194
195 /*
196 * 0: readable
197 * 1: writable
198 * 2-6: reserved
199 * 7: super page
200 * 8-10: available
201 * 11: snoop behavior
202 * 12-63: Host physcial address
203 */
204 struct dma_pte {
205 u64 val;
206 };
207
208 static inline void dma_clear_pte(struct dma_pte *pte)
209 {
210 pte->val = 0;
211 }
212
213 static inline void dma_set_pte_readable(struct dma_pte *pte)
214 {
215 pte->val |= DMA_PTE_READ;
216 }
217
218 static inline void dma_set_pte_writable(struct dma_pte *pte)
219 {
220 pte->val |= DMA_PTE_WRITE;
221 }
222
223 static inline void dma_set_pte_snp(struct dma_pte *pte)
224 {
225 pte->val |= DMA_PTE_SNP;
226 }
227
228 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
229 {
230 pte->val = (pte->val & ~3) | (prot & 3);
231 }
232
233 static inline u64 dma_pte_addr(struct dma_pte *pte)
234 {
235 #ifdef CONFIG_64BIT
236 return pte->val & VTD_PAGE_MASK;
237 #else
238 /* Must have a full atomic 64-bit read */
239 return __cmpxchg64(pte, 0ULL, 0ULL) & VTD_PAGE_MASK;
240 #endif
241 }
242
243 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
244 {
245 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
246 }
247
248 static inline bool dma_pte_present(struct dma_pte *pte)
249 {
250 return (pte->val & 3) != 0;
251 }
252
253 static inline int first_pte_in_page(struct dma_pte *pte)
254 {
255 return !((unsigned long)pte & ~VTD_PAGE_MASK);
256 }
257
258 /*
259 * This domain is a statically identity mapping domain.
260 * 1. This domain creats a static 1:1 mapping to all usable memory.
261 * 2. It maps to each iommu if successful.
262 * 3. Each iommu mapps to this domain if successful.
263 */
264 static struct dmar_domain *si_domain;
265 static int hw_pass_through = 1;
266
267 /* devices under the same p2p bridge are owned in one domain */
268 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
269
270 /* domain represents a virtual machine, more than one devices
271 * across iommus may be owned in one domain, e.g. kvm guest.
272 */
273 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
274
275 /* si_domain contains mulitple devices */
276 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
277
278 struct dmar_domain {
279 int id; /* domain id */
280 int nid; /* node id */
281 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
282
283 struct list_head devices; /* all devices' list */
284 struct iova_domain iovad; /* iova's that belong to this domain */
285
286 struct dma_pte *pgd; /* virtual address */
287 int gaw; /* max guest address width */
288
289 /* adjusted guest address width, 0 is level 2 30-bit */
290 int agaw;
291
292 int flags; /* flags to find out type of domain */
293
294 int iommu_coherency;/* indicate coherency of iommu access */
295 int iommu_snooping; /* indicate snooping control feature*/
296 int iommu_count; /* reference count of iommu */
297 spinlock_t iommu_lock; /* protect iommu set in domain */
298 u64 max_addr; /* maximum mapped address */
299 };
300
301 /* PCI domain-device relationship */
302 struct device_domain_info {
303 struct list_head link; /* link to domain siblings */
304 struct list_head global; /* link to global list */
305 int segment; /* PCI domain */
306 u8 bus; /* PCI bus number */
307 u8 devfn; /* PCI devfn number */
308 struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
309 struct intel_iommu *iommu; /* IOMMU used by this device */
310 struct dmar_domain *domain; /* pointer to domain */
311 };
312
313 static void flush_unmaps_timeout(unsigned long data);
314
315 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
316
317 #define HIGH_WATER_MARK 250
318 struct deferred_flush_tables {
319 int next;
320 struct iova *iova[HIGH_WATER_MARK];
321 struct dmar_domain *domain[HIGH_WATER_MARK];
322 };
323
324 static struct deferred_flush_tables *deferred_flush;
325
326 /* bitmap for indexing intel_iommus */
327 static int g_num_of_iommus;
328
329 static DEFINE_SPINLOCK(async_umap_flush_lock);
330 static LIST_HEAD(unmaps_to_do);
331
332 static int timer_on;
333 static long list_size;
334
335 static void domain_remove_dev_info(struct dmar_domain *domain);
336
337 #ifdef CONFIG_DMAR_DEFAULT_ON
338 int dmar_disabled = 0;
339 #else
340 int dmar_disabled = 1;
341 #endif /*CONFIG_DMAR_DEFAULT_ON*/
342
343 static int __initdata dmar_map_gfx = 1;
344 static int dmar_forcedac;
345 static int intel_iommu_strict;
346
347 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
348 static DEFINE_SPINLOCK(device_domain_lock);
349 static LIST_HEAD(device_domain_list);
350
351 static struct iommu_ops intel_iommu_ops;
352
353 static int __init intel_iommu_setup(char *str)
354 {
355 if (!str)
356 return -EINVAL;
357 while (*str) {
358 if (!strncmp(str, "on", 2)) {
359 dmar_disabled = 0;
360 printk(KERN_INFO "Intel-IOMMU: enabled\n");
361 } else if (!strncmp(str, "off", 3)) {
362 dmar_disabled = 1;
363 printk(KERN_INFO "Intel-IOMMU: disabled\n");
364 } else if (!strncmp(str, "igfx_off", 8)) {
365 dmar_map_gfx = 0;
366 printk(KERN_INFO
367 "Intel-IOMMU: disable GFX device mapping\n");
368 } else if (!strncmp(str, "forcedac", 8)) {
369 printk(KERN_INFO
370 "Intel-IOMMU: Forcing DAC for PCI devices\n");
371 dmar_forcedac = 1;
372 } else if (!strncmp(str, "strict", 6)) {
373 printk(KERN_INFO
374 "Intel-IOMMU: disable batched IOTLB flush\n");
375 intel_iommu_strict = 1;
376 }
377
378 str += strcspn(str, ",");
379 while (*str == ',')
380 str++;
381 }
382 return 0;
383 }
384 __setup("intel_iommu=", intel_iommu_setup);
385
386 static struct kmem_cache *iommu_domain_cache;
387 static struct kmem_cache *iommu_devinfo_cache;
388 static struct kmem_cache *iommu_iova_cache;
389
390 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
391 {
392 unsigned int flags;
393 void *vaddr;
394
395 /* trying to avoid low memory issues */
396 flags = current->flags & PF_MEMALLOC;
397 current->flags |= PF_MEMALLOC;
398 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
399 current->flags &= (~PF_MEMALLOC | flags);
400 return vaddr;
401 }
402
403
404 static inline void *alloc_pgtable_page(int node)
405 {
406 unsigned int flags;
407 struct page *page;
408 void *vaddr = NULL;
409
410 /* trying to avoid low memory issues */
411 flags = current->flags & PF_MEMALLOC;
412 current->flags |= PF_MEMALLOC;
413 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
414 if (page)
415 vaddr = page_address(page);
416 current->flags &= (~PF_MEMALLOC | flags);
417 return vaddr;
418 }
419
420 static inline void free_pgtable_page(void *vaddr)
421 {
422 free_page((unsigned long)vaddr);
423 }
424
425 static inline void *alloc_domain_mem(void)
426 {
427 return iommu_kmem_cache_alloc(iommu_domain_cache);
428 }
429
430 static void free_domain_mem(void *vaddr)
431 {
432 kmem_cache_free(iommu_domain_cache, vaddr);
433 }
434
435 static inline void * alloc_devinfo_mem(void)
436 {
437 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
438 }
439
440 static inline void free_devinfo_mem(void *vaddr)
441 {
442 kmem_cache_free(iommu_devinfo_cache, vaddr);
443 }
444
445 struct iova *alloc_iova_mem(void)
446 {
447 return iommu_kmem_cache_alloc(iommu_iova_cache);
448 }
449
450 void free_iova_mem(struct iova *iova)
451 {
452 kmem_cache_free(iommu_iova_cache, iova);
453 }
454
455
456 static inline int width_to_agaw(int width);
457
458 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
459 {
460 unsigned long sagaw;
461 int agaw = -1;
462
463 sagaw = cap_sagaw(iommu->cap);
464 for (agaw = width_to_agaw(max_gaw);
465 agaw >= 0; agaw--) {
466 if (test_bit(agaw, &sagaw))
467 break;
468 }
469
470 return agaw;
471 }
472
473 /*
474 * Calculate max SAGAW for each iommu.
475 */
476 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
477 {
478 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
479 }
480
481 /*
482 * calculate agaw for each iommu.
483 * "SAGAW" may be different across iommus, use a default agaw, and
484 * get a supported less agaw for iommus that don't support the default agaw.
485 */
486 int iommu_calculate_agaw(struct intel_iommu *iommu)
487 {
488 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
489 }
490
491 /* This functionin only returns single iommu in a domain */
492 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
493 {
494 int iommu_id;
495
496 /* si_domain and vm domain should not get here. */
497 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
498 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
499
500 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
501 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
502 return NULL;
503
504 return g_iommus[iommu_id];
505 }
506
507 static void domain_update_iommu_coherency(struct dmar_domain *domain)
508 {
509 int i;
510
511 domain->iommu_coherency = 1;
512
513 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
514 for (; i < g_num_of_iommus; ) {
515 if (!ecap_coherent(g_iommus[i]->ecap)) {
516 domain->iommu_coherency = 0;
517 break;
518 }
519 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
520 }
521 }
522
523 static void domain_update_iommu_snooping(struct dmar_domain *domain)
524 {
525 int i;
526
527 domain->iommu_snooping = 1;
528
529 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
530 for (; i < g_num_of_iommus; ) {
531 if (!ecap_sc_support(g_iommus[i]->ecap)) {
532 domain->iommu_snooping = 0;
533 break;
534 }
535 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
536 }
537 }
538
539 /* Some capabilities may be different across iommus */
540 static void domain_update_iommu_cap(struct dmar_domain *domain)
541 {
542 domain_update_iommu_coherency(domain);
543 domain_update_iommu_snooping(domain);
544 }
545
546 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
547 {
548 struct dmar_drhd_unit *drhd = NULL;
549 int i;
550
551 for_each_drhd_unit(drhd) {
552 if (drhd->ignored)
553 continue;
554 if (segment != drhd->segment)
555 continue;
556
557 for (i = 0; i < drhd->devices_cnt; i++) {
558 if (drhd->devices[i] &&
559 drhd->devices[i]->bus->number == bus &&
560 drhd->devices[i]->devfn == devfn)
561 return drhd->iommu;
562 if (drhd->devices[i] &&
563 drhd->devices[i]->subordinate &&
564 drhd->devices[i]->subordinate->number <= bus &&
565 drhd->devices[i]->subordinate->subordinate >= bus)
566 return drhd->iommu;
567 }
568
569 if (drhd->include_all)
570 return drhd->iommu;
571 }
572
573 return NULL;
574 }
575
576 static void domain_flush_cache(struct dmar_domain *domain,
577 void *addr, int size)
578 {
579 if (!domain->iommu_coherency)
580 clflush_cache_range(addr, size);
581 }
582
583 /* Gets context entry for a given bus and devfn */
584 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
585 u8 bus, u8 devfn)
586 {
587 struct root_entry *root;
588 struct context_entry *context;
589 unsigned long phy_addr;
590 unsigned long flags;
591
592 spin_lock_irqsave(&iommu->lock, flags);
593 root = &iommu->root_entry[bus];
594 context = get_context_addr_from_root(root);
595 if (!context) {
596 context = (struct context_entry *)
597 alloc_pgtable_page(iommu->node);
598 if (!context) {
599 spin_unlock_irqrestore(&iommu->lock, flags);
600 return NULL;
601 }
602 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
603 phy_addr = virt_to_phys((void *)context);
604 set_root_value(root, phy_addr);
605 set_root_present(root);
606 __iommu_flush_cache(iommu, root, sizeof(*root));
607 }
608 spin_unlock_irqrestore(&iommu->lock, flags);
609 return &context[devfn];
610 }
611
612 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
613 {
614 struct root_entry *root;
615 struct context_entry *context;
616 int ret;
617 unsigned long flags;
618
619 spin_lock_irqsave(&iommu->lock, flags);
620 root = &iommu->root_entry[bus];
621 context = get_context_addr_from_root(root);
622 if (!context) {
623 ret = 0;
624 goto out;
625 }
626 ret = context_present(&context[devfn]);
627 out:
628 spin_unlock_irqrestore(&iommu->lock, flags);
629 return ret;
630 }
631
632 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
633 {
634 struct root_entry *root;
635 struct context_entry *context;
636 unsigned long flags;
637
638 spin_lock_irqsave(&iommu->lock, flags);
639 root = &iommu->root_entry[bus];
640 context = get_context_addr_from_root(root);
641 if (context) {
642 context_clear_entry(&context[devfn]);
643 __iommu_flush_cache(iommu, &context[devfn], \
644 sizeof(*context));
645 }
646 spin_unlock_irqrestore(&iommu->lock, flags);
647 }
648
649 static void free_context_table(struct intel_iommu *iommu)
650 {
651 struct root_entry *root;
652 int i;
653 unsigned long flags;
654 struct context_entry *context;
655
656 spin_lock_irqsave(&iommu->lock, flags);
657 if (!iommu->root_entry) {
658 goto out;
659 }
660 for (i = 0; i < ROOT_ENTRY_NR; i++) {
661 root = &iommu->root_entry[i];
662 context = get_context_addr_from_root(root);
663 if (context)
664 free_pgtable_page(context);
665 }
666 free_pgtable_page(iommu->root_entry);
667 iommu->root_entry = NULL;
668 out:
669 spin_unlock_irqrestore(&iommu->lock, flags);
670 }
671
672 /* page table handling */
673 #define LEVEL_STRIDE (9)
674 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
675
676 static inline int agaw_to_level(int agaw)
677 {
678 return agaw + 2;
679 }
680
681 static inline int agaw_to_width(int agaw)
682 {
683 return 30 + agaw * LEVEL_STRIDE;
684
685 }
686
687 static inline int width_to_agaw(int width)
688 {
689 return (width - 30) / LEVEL_STRIDE;
690 }
691
692 static inline unsigned int level_to_offset_bits(int level)
693 {
694 return (level - 1) * LEVEL_STRIDE;
695 }
696
697 static inline int pfn_level_offset(unsigned long pfn, int level)
698 {
699 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
700 }
701
702 static inline unsigned long level_mask(int level)
703 {
704 return -1UL << level_to_offset_bits(level);
705 }
706
707 static inline unsigned long level_size(int level)
708 {
709 return 1UL << level_to_offset_bits(level);
710 }
711
712 static inline unsigned long align_to_level(unsigned long pfn, int level)
713 {
714 return (pfn + level_size(level) - 1) & level_mask(level);
715 }
716
717 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
718 unsigned long pfn)
719 {
720 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
721 struct dma_pte *parent, *pte = NULL;
722 int level = agaw_to_level(domain->agaw);
723 int offset;
724
725 BUG_ON(!domain->pgd);
726 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
727 parent = domain->pgd;
728
729 while (level > 0) {
730 void *tmp_page;
731
732 offset = pfn_level_offset(pfn, level);
733 pte = &parent[offset];
734 if (level == 1)
735 break;
736
737 if (!dma_pte_present(pte)) {
738 uint64_t pteval;
739
740 tmp_page = alloc_pgtable_page(domain->nid);
741
742 if (!tmp_page)
743 return NULL;
744
745 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
746 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
747 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
748 /* Someone else set it while we were thinking; use theirs. */
749 free_pgtable_page(tmp_page);
750 } else {
751 dma_pte_addr(pte);
752 domain_flush_cache(domain, pte, sizeof(*pte));
753 }
754 }
755 parent = phys_to_virt(dma_pte_addr(pte));
756 level--;
757 }
758
759 return pte;
760 }
761
762 /* return address's pte at specific level */
763 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
764 unsigned long pfn,
765 int level)
766 {
767 struct dma_pte *parent, *pte = NULL;
768 int total = agaw_to_level(domain->agaw);
769 int offset;
770
771 parent = domain->pgd;
772 while (level <= total) {
773 offset = pfn_level_offset(pfn, total);
774 pte = &parent[offset];
775 if (level == total)
776 return pte;
777
778 if (!dma_pte_present(pte))
779 break;
780 parent = phys_to_virt(dma_pte_addr(pte));
781 total--;
782 }
783 return NULL;
784 }
785
786 /* clear last level pte, a tlb flush should be followed */
787 static void dma_pte_clear_range(struct dmar_domain *domain,
788 unsigned long start_pfn,
789 unsigned long last_pfn)
790 {
791 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
792 struct dma_pte *first_pte, *pte;
793
794 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
795 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
796 BUG_ON(start_pfn > last_pfn);
797
798 /* we don't need lock here; nobody else touches the iova range */
799 do {
800 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1);
801 if (!pte) {
802 start_pfn = align_to_level(start_pfn + 1, 2);
803 continue;
804 }
805 do {
806 dma_clear_pte(pte);
807 start_pfn++;
808 pte++;
809 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
810
811 domain_flush_cache(domain, first_pte,
812 (void *)pte - (void *)first_pte);
813
814 } while (start_pfn && start_pfn <= last_pfn);
815 }
816
817 /* free page table pages. last level pte should already be cleared */
818 static void dma_pte_free_pagetable(struct dmar_domain *domain,
819 unsigned long start_pfn,
820 unsigned long last_pfn)
821 {
822 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
823 struct dma_pte *first_pte, *pte;
824 int total = agaw_to_level(domain->agaw);
825 int level;
826 unsigned long tmp;
827
828 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
829 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
830 BUG_ON(start_pfn > last_pfn);
831
832 /* We don't need lock here; nobody else touches the iova range */
833 level = 2;
834 while (level <= total) {
835 tmp = align_to_level(start_pfn, level);
836
837 /* If we can't even clear one PTE at this level, we're done */
838 if (tmp + level_size(level) - 1 > last_pfn)
839 return;
840
841 do {
842 first_pte = pte = dma_pfn_level_pte(domain, tmp, level);
843 if (!pte) {
844 tmp = align_to_level(tmp + 1, level + 1);
845 continue;
846 }
847 do {
848 if (dma_pte_present(pte)) {
849 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
850 dma_clear_pte(pte);
851 }
852 pte++;
853 tmp += level_size(level);
854 } while (!first_pte_in_page(pte) &&
855 tmp + level_size(level) - 1 <= last_pfn);
856
857 domain_flush_cache(domain, first_pte,
858 (void *)pte - (void *)first_pte);
859
860 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
861 level++;
862 }
863 /* free pgd */
864 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
865 free_pgtable_page(domain->pgd);
866 domain->pgd = NULL;
867 }
868 }
869
870 /* iommu handling */
871 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
872 {
873 struct root_entry *root;
874 unsigned long flags;
875
876 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
877 if (!root)
878 return -ENOMEM;
879
880 __iommu_flush_cache(iommu, root, ROOT_SIZE);
881
882 spin_lock_irqsave(&iommu->lock, flags);
883 iommu->root_entry = root;
884 spin_unlock_irqrestore(&iommu->lock, flags);
885
886 return 0;
887 }
888
889 static void iommu_set_root_entry(struct intel_iommu *iommu)
890 {
891 void *addr;
892 u32 sts;
893 unsigned long flag;
894
895 addr = iommu->root_entry;
896
897 spin_lock_irqsave(&iommu->register_lock, flag);
898 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
899
900 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
901
902 /* Make sure hardware complete it */
903 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
904 readl, (sts & DMA_GSTS_RTPS), sts);
905
906 spin_unlock_irqrestore(&iommu->register_lock, flag);
907 }
908
909 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
910 {
911 u32 val;
912 unsigned long flag;
913
914 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
915 return;
916
917 spin_lock_irqsave(&iommu->register_lock, flag);
918 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
919
920 /* Make sure hardware complete it */
921 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
922 readl, (!(val & DMA_GSTS_WBFS)), val);
923
924 spin_unlock_irqrestore(&iommu->register_lock, flag);
925 }
926
927 /* return value determine if we need a write buffer flush */
928 static void __iommu_flush_context(struct intel_iommu *iommu,
929 u16 did, u16 source_id, u8 function_mask,
930 u64 type)
931 {
932 u64 val = 0;
933 unsigned long flag;
934
935 switch (type) {
936 case DMA_CCMD_GLOBAL_INVL:
937 val = DMA_CCMD_GLOBAL_INVL;
938 break;
939 case DMA_CCMD_DOMAIN_INVL:
940 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
941 break;
942 case DMA_CCMD_DEVICE_INVL:
943 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
944 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
945 break;
946 default:
947 BUG();
948 }
949 val |= DMA_CCMD_ICC;
950
951 spin_lock_irqsave(&iommu->register_lock, flag);
952 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
953
954 /* Make sure hardware complete it */
955 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
956 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
957
958 spin_unlock_irqrestore(&iommu->register_lock, flag);
959 }
960
961 /* return value determine if we need a write buffer flush */
962 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
963 u64 addr, unsigned int size_order, u64 type)
964 {
965 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
966 u64 val = 0, val_iva = 0;
967 unsigned long flag;
968
969 switch (type) {
970 case DMA_TLB_GLOBAL_FLUSH:
971 /* global flush doesn't need set IVA_REG */
972 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
973 break;
974 case DMA_TLB_DSI_FLUSH:
975 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
976 break;
977 case DMA_TLB_PSI_FLUSH:
978 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
979 /* Note: always flush non-leaf currently */
980 val_iva = size_order | addr;
981 break;
982 default:
983 BUG();
984 }
985 /* Note: set drain read/write */
986 #if 0
987 /*
988 * This is probably to be super secure.. Looks like we can
989 * ignore it without any impact.
990 */
991 if (cap_read_drain(iommu->cap))
992 val |= DMA_TLB_READ_DRAIN;
993 #endif
994 if (cap_write_drain(iommu->cap))
995 val |= DMA_TLB_WRITE_DRAIN;
996
997 spin_lock_irqsave(&iommu->register_lock, flag);
998 /* Note: Only uses first TLB reg currently */
999 if (val_iva)
1000 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1001 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1002
1003 /* Make sure hardware complete it */
1004 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1005 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1006
1007 spin_unlock_irqrestore(&iommu->register_lock, flag);
1008
1009 /* check IOTLB invalidation granularity */
1010 if (DMA_TLB_IAIG(val) == 0)
1011 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1012 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1013 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1014 (unsigned long long)DMA_TLB_IIRG(type),
1015 (unsigned long long)DMA_TLB_IAIG(val));
1016 }
1017
1018 static struct device_domain_info *iommu_support_dev_iotlb(
1019 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1020 {
1021 int found = 0;
1022 unsigned long flags;
1023 struct device_domain_info *info;
1024 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1025
1026 if (!ecap_dev_iotlb_support(iommu->ecap))
1027 return NULL;
1028
1029 if (!iommu->qi)
1030 return NULL;
1031
1032 spin_lock_irqsave(&device_domain_lock, flags);
1033 list_for_each_entry(info, &domain->devices, link)
1034 if (info->bus == bus && info->devfn == devfn) {
1035 found = 1;
1036 break;
1037 }
1038 spin_unlock_irqrestore(&device_domain_lock, flags);
1039
1040 if (!found || !info->dev)
1041 return NULL;
1042
1043 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1044 return NULL;
1045
1046 if (!dmar_find_matched_atsr_unit(info->dev))
1047 return NULL;
1048
1049 info->iommu = iommu;
1050
1051 return info;
1052 }
1053
1054 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1055 {
1056 if (!info)
1057 return;
1058
1059 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1060 }
1061
1062 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1063 {
1064 if (!info->dev || !pci_ats_enabled(info->dev))
1065 return;
1066
1067 pci_disable_ats(info->dev);
1068 }
1069
1070 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1071 u64 addr, unsigned mask)
1072 {
1073 u16 sid, qdep;
1074 unsigned long flags;
1075 struct device_domain_info *info;
1076
1077 spin_lock_irqsave(&device_domain_lock, flags);
1078 list_for_each_entry(info, &domain->devices, link) {
1079 if (!info->dev || !pci_ats_enabled(info->dev))
1080 continue;
1081
1082 sid = info->bus << 8 | info->devfn;
1083 qdep = pci_ats_queue_depth(info->dev);
1084 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1085 }
1086 spin_unlock_irqrestore(&device_domain_lock, flags);
1087 }
1088
1089 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1090 unsigned long pfn, unsigned int pages)
1091 {
1092 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1093 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1094
1095 BUG_ON(pages == 0);
1096
1097 /*
1098 * Fallback to domain selective flush if no PSI support or the size is
1099 * too big.
1100 * PSI requires page size to be 2 ^ x, and the base address is naturally
1101 * aligned to the size
1102 */
1103 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1104 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1105 DMA_TLB_DSI_FLUSH);
1106 else
1107 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1108 DMA_TLB_PSI_FLUSH);
1109
1110 /*
1111 * In caching mode, domain ID 0 is reserved for non-present to present
1112 * mapping flush. Device IOTLB doesn't need to be flushed in this case.
1113 */
1114 if (!cap_caching_mode(iommu->cap) || did)
1115 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1116 }
1117
1118 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1119 {
1120 u32 pmen;
1121 unsigned long flags;
1122
1123 spin_lock_irqsave(&iommu->register_lock, flags);
1124 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1125 pmen &= ~DMA_PMEN_EPM;
1126 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1127
1128 /* wait for the protected region status bit to clear */
1129 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1130 readl, !(pmen & DMA_PMEN_PRS), pmen);
1131
1132 spin_unlock_irqrestore(&iommu->register_lock, flags);
1133 }
1134
1135 static int iommu_enable_translation(struct intel_iommu *iommu)
1136 {
1137 u32 sts;
1138 unsigned long flags;
1139
1140 spin_lock_irqsave(&iommu->register_lock, flags);
1141 iommu->gcmd |= DMA_GCMD_TE;
1142 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1143
1144 /* Make sure hardware complete it */
1145 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1146 readl, (sts & DMA_GSTS_TES), sts);
1147
1148 spin_unlock_irqrestore(&iommu->register_lock, flags);
1149 return 0;
1150 }
1151
1152 static int iommu_disable_translation(struct intel_iommu *iommu)
1153 {
1154 u32 sts;
1155 unsigned long flag;
1156
1157 spin_lock_irqsave(&iommu->register_lock, flag);
1158 iommu->gcmd &= ~DMA_GCMD_TE;
1159 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1160
1161 /* Make sure hardware complete it */
1162 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1163 readl, (!(sts & DMA_GSTS_TES)), sts);
1164
1165 spin_unlock_irqrestore(&iommu->register_lock, flag);
1166 return 0;
1167 }
1168
1169
1170 static int iommu_init_domains(struct intel_iommu *iommu)
1171 {
1172 unsigned long ndomains;
1173 unsigned long nlongs;
1174
1175 ndomains = cap_ndoms(iommu->cap);
1176 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1177 nlongs = BITS_TO_LONGS(ndomains);
1178
1179 spin_lock_init(&iommu->lock);
1180
1181 /* TBD: there might be 64K domains,
1182 * consider other allocation for future chip
1183 */
1184 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1185 if (!iommu->domain_ids) {
1186 printk(KERN_ERR "Allocating domain id array failed\n");
1187 return -ENOMEM;
1188 }
1189 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1190 GFP_KERNEL);
1191 if (!iommu->domains) {
1192 printk(KERN_ERR "Allocating domain array failed\n");
1193 return -ENOMEM;
1194 }
1195
1196 /*
1197 * if Caching mode is set, then invalid translations are tagged
1198 * with domainid 0. Hence we need to pre-allocate it.
1199 */
1200 if (cap_caching_mode(iommu->cap))
1201 set_bit(0, iommu->domain_ids);
1202 return 0;
1203 }
1204
1205
1206 static void domain_exit(struct dmar_domain *domain);
1207 static void vm_domain_exit(struct dmar_domain *domain);
1208
1209 void free_dmar_iommu(struct intel_iommu *iommu)
1210 {
1211 struct dmar_domain *domain;
1212 int i;
1213 unsigned long flags;
1214
1215 if ((iommu->domains) && (iommu->domain_ids)) {
1216 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1217 for (; i < cap_ndoms(iommu->cap); ) {
1218 domain = iommu->domains[i];
1219 clear_bit(i, iommu->domain_ids);
1220
1221 spin_lock_irqsave(&domain->iommu_lock, flags);
1222 if (--domain->iommu_count == 0) {
1223 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1224 vm_domain_exit(domain);
1225 else
1226 domain_exit(domain);
1227 }
1228 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1229
1230 i = find_next_bit(iommu->domain_ids,
1231 cap_ndoms(iommu->cap), i+1);
1232 }
1233 }
1234
1235 if (iommu->gcmd & DMA_GCMD_TE)
1236 iommu_disable_translation(iommu);
1237
1238 if (iommu->irq) {
1239 set_irq_data(iommu->irq, NULL);
1240 /* This will mask the irq */
1241 free_irq(iommu->irq, iommu);
1242 destroy_irq(iommu->irq);
1243 }
1244
1245 kfree(iommu->domains);
1246 kfree(iommu->domain_ids);
1247
1248 g_iommus[iommu->seq_id] = NULL;
1249
1250 /* if all iommus are freed, free g_iommus */
1251 for (i = 0; i < g_num_of_iommus; i++) {
1252 if (g_iommus[i])
1253 break;
1254 }
1255
1256 if (i == g_num_of_iommus)
1257 kfree(g_iommus);
1258
1259 /* free context mapping */
1260 free_context_table(iommu);
1261 }
1262
1263 static struct dmar_domain *alloc_domain(void)
1264 {
1265 struct dmar_domain *domain;
1266
1267 domain = alloc_domain_mem();
1268 if (!domain)
1269 return NULL;
1270
1271 domain->nid = -1;
1272 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1273 domain->flags = 0;
1274
1275 return domain;
1276 }
1277
1278 static int iommu_attach_domain(struct dmar_domain *domain,
1279 struct intel_iommu *iommu)
1280 {
1281 int num;
1282 unsigned long ndomains;
1283 unsigned long flags;
1284
1285 ndomains = cap_ndoms(iommu->cap);
1286
1287 spin_lock_irqsave(&iommu->lock, flags);
1288
1289 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1290 if (num >= ndomains) {
1291 spin_unlock_irqrestore(&iommu->lock, flags);
1292 printk(KERN_ERR "IOMMU: no free domain ids\n");
1293 return -ENOMEM;
1294 }
1295
1296 domain->id = num;
1297 set_bit(num, iommu->domain_ids);
1298 set_bit(iommu->seq_id, &domain->iommu_bmp);
1299 iommu->domains[num] = domain;
1300 spin_unlock_irqrestore(&iommu->lock, flags);
1301
1302 return 0;
1303 }
1304
1305 static void iommu_detach_domain(struct dmar_domain *domain,
1306 struct intel_iommu *iommu)
1307 {
1308 unsigned long flags;
1309 int num, ndomains;
1310 int found = 0;
1311
1312 spin_lock_irqsave(&iommu->lock, flags);
1313 ndomains = cap_ndoms(iommu->cap);
1314 num = find_first_bit(iommu->domain_ids, ndomains);
1315 for (; num < ndomains; ) {
1316 if (iommu->domains[num] == domain) {
1317 found = 1;
1318 break;
1319 }
1320 num = find_next_bit(iommu->domain_ids,
1321 cap_ndoms(iommu->cap), num+1);
1322 }
1323
1324 if (found) {
1325 clear_bit(num, iommu->domain_ids);
1326 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1327 iommu->domains[num] = NULL;
1328 }
1329 spin_unlock_irqrestore(&iommu->lock, flags);
1330 }
1331
1332 static struct iova_domain reserved_iova_list;
1333 static struct lock_class_key reserved_rbtree_key;
1334
1335 static void dmar_init_reserved_ranges(void)
1336 {
1337 struct pci_dev *pdev = NULL;
1338 struct iova *iova;
1339 int i;
1340
1341 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1342
1343 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1344 &reserved_rbtree_key);
1345
1346 /* IOAPIC ranges shouldn't be accessed by DMA */
1347 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1348 IOVA_PFN(IOAPIC_RANGE_END));
1349 if (!iova)
1350 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1351
1352 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1353 for_each_pci_dev(pdev) {
1354 struct resource *r;
1355
1356 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1357 r = &pdev->resource[i];
1358 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1359 continue;
1360 iova = reserve_iova(&reserved_iova_list,
1361 IOVA_PFN(r->start),
1362 IOVA_PFN(r->end));
1363 if (!iova)
1364 printk(KERN_ERR "Reserve iova failed\n");
1365 }
1366 }
1367
1368 }
1369
1370 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1371 {
1372 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1373 }
1374
1375 static inline int guestwidth_to_adjustwidth(int gaw)
1376 {
1377 int agaw;
1378 int r = (gaw - 12) % 9;
1379
1380 if (r == 0)
1381 agaw = gaw;
1382 else
1383 agaw = gaw + 9 - r;
1384 if (agaw > 64)
1385 agaw = 64;
1386 return agaw;
1387 }
1388
1389 static int domain_init(struct dmar_domain *domain, int guest_width)
1390 {
1391 struct intel_iommu *iommu;
1392 int adjust_width, agaw;
1393 unsigned long sagaw;
1394
1395 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1396 spin_lock_init(&domain->iommu_lock);
1397
1398 domain_reserve_special_ranges(domain);
1399
1400 /* calculate AGAW */
1401 iommu = domain_get_iommu(domain);
1402 if (guest_width > cap_mgaw(iommu->cap))
1403 guest_width = cap_mgaw(iommu->cap);
1404 domain->gaw = guest_width;
1405 adjust_width = guestwidth_to_adjustwidth(guest_width);
1406 agaw = width_to_agaw(adjust_width);
1407 sagaw = cap_sagaw(iommu->cap);
1408 if (!test_bit(agaw, &sagaw)) {
1409 /* hardware doesn't support it, choose a bigger one */
1410 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1411 agaw = find_next_bit(&sagaw, 5, agaw);
1412 if (agaw >= 5)
1413 return -ENODEV;
1414 }
1415 domain->agaw = agaw;
1416 INIT_LIST_HEAD(&domain->devices);
1417
1418 if (ecap_coherent(iommu->ecap))
1419 domain->iommu_coherency = 1;
1420 else
1421 domain->iommu_coherency = 0;
1422
1423 if (ecap_sc_support(iommu->ecap))
1424 domain->iommu_snooping = 1;
1425 else
1426 domain->iommu_snooping = 0;
1427
1428 domain->iommu_count = 1;
1429 domain->nid = iommu->node;
1430
1431 /* always allocate the top pgd */
1432 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1433 if (!domain->pgd)
1434 return -ENOMEM;
1435 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1436 return 0;
1437 }
1438
1439 static void domain_exit(struct dmar_domain *domain)
1440 {
1441 struct dmar_drhd_unit *drhd;
1442 struct intel_iommu *iommu;
1443
1444 /* Domain 0 is reserved, so dont process it */
1445 if (!domain)
1446 return;
1447
1448 domain_remove_dev_info(domain);
1449 /* destroy iovas */
1450 put_iova_domain(&domain->iovad);
1451
1452 /* clear ptes */
1453 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1454
1455 /* free page tables */
1456 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1457
1458 for_each_active_iommu(iommu, drhd)
1459 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1460 iommu_detach_domain(domain, iommu);
1461
1462 free_domain_mem(domain);
1463 }
1464
1465 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1466 u8 bus, u8 devfn, int translation)
1467 {
1468 struct context_entry *context;
1469 unsigned long flags;
1470 struct intel_iommu *iommu;
1471 struct dma_pte *pgd;
1472 unsigned long num;
1473 unsigned long ndomains;
1474 int id;
1475 int agaw;
1476 struct device_domain_info *info = NULL;
1477
1478 pr_debug("Set context mapping for %02x:%02x.%d\n",
1479 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1480
1481 BUG_ON(!domain->pgd);
1482 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1483 translation != CONTEXT_TT_MULTI_LEVEL);
1484
1485 iommu = device_to_iommu(segment, bus, devfn);
1486 if (!iommu)
1487 return -ENODEV;
1488
1489 context = device_to_context_entry(iommu, bus, devfn);
1490 if (!context)
1491 return -ENOMEM;
1492 spin_lock_irqsave(&iommu->lock, flags);
1493 if (context_present(context)) {
1494 spin_unlock_irqrestore(&iommu->lock, flags);
1495 return 0;
1496 }
1497
1498 id = domain->id;
1499 pgd = domain->pgd;
1500
1501 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1502 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1503 int found = 0;
1504
1505 /* find an available domain id for this device in iommu */
1506 ndomains = cap_ndoms(iommu->cap);
1507 num = find_first_bit(iommu->domain_ids, ndomains);
1508 for (; num < ndomains; ) {
1509 if (iommu->domains[num] == domain) {
1510 id = num;
1511 found = 1;
1512 break;
1513 }
1514 num = find_next_bit(iommu->domain_ids,
1515 cap_ndoms(iommu->cap), num+1);
1516 }
1517
1518 if (found == 0) {
1519 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1520 if (num >= ndomains) {
1521 spin_unlock_irqrestore(&iommu->lock, flags);
1522 printk(KERN_ERR "IOMMU: no free domain ids\n");
1523 return -EFAULT;
1524 }
1525
1526 set_bit(num, iommu->domain_ids);
1527 iommu->domains[num] = domain;
1528 id = num;
1529 }
1530
1531 /* Skip top levels of page tables for
1532 * iommu which has less agaw than default.
1533 */
1534 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1535 pgd = phys_to_virt(dma_pte_addr(pgd));
1536 if (!dma_pte_present(pgd)) {
1537 spin_unlock_irqrestore(&iommu->lock, flags);
1538 return -ENOMEM;
1539 }
1540 }
1541 }
1542
1543 context_set_domain_id(context, id);
1544
1545 if (translation != CONTEXT_TT_PASS_THROUGH) {
1546 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1547 translation = info ? CONTEXT_TT_DEV_IOTLB :
1548 CONTEXT_TT_MULTI_LEVEL;
1549 }
1550 /*
1551 * In pass through mode, AW must be programmed to indicate the largest
1552 * AGAW value supported by hardware. And ASR is ignored by hardware.
1553 */
1554 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1555 context_set_address_width(context, iommu->msagaw);
1556 else {
1557 context_set_address_root(context, virt_to_phys(pgd));
1558 context_set_address_width(context, iommu->agaw);
1559 }
1560
1561 context_set_translation_type(context, translation);
1562 context_set_fault_enable(context);
1563 context_set_present(context);
1564 domain_flush_cache(domain, context, sizeof(*context));
1565
1566 /*
1567 * It's a non-present to present mapping. If hardware doesn't cache
1568 * non-present entry we only need to flush the write-buffer. If the
1569 * _does_ cache non-present entries, then it does so in the special
1570 * domain #0, which we have to flush:
1571 */
1572 if (cap_caching_mode(iommu->cap)) {
1573 iommu->flush.flush_context(iommu, 0,
1574 (((u16)bus) << 8) | devfn,
1575 DMA_CCMD_MASK_NOBIT,
1576 DMA_CCMD_DEVICE_INVL);
1577 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH);
1578 } else {
1579 iommu_flush_write_buffer(iommu);
1580 }
1581 iommu_enable_dev_iotlb(info);
1582 spin_unlock_irqrestore(&iommu->lock, flags);
1583
1584 spin_lock_irqsave(&domain->iommu_lock, flags);
1585 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1586 domain->iommu_count++;
1587 if (domain->iommu_count == 1)
1588 domain->nid = iommu->node;
1589 domain_update_iommu_cap(domain);
1590 }
1591 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1592 return 0;
1593 }
1594
1595 static int
1596 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1597 int translation)
1598 {
1599 int ret;
1600 struct pci_dev *tmp, *parent;
1601
1602 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1603 pdev->bus->number, pdev->devfn,
1604 translation);
1605 if (ret)
1606 return ret;
1607
1608 /* dependent device mapping */
1609 tmp = pci_find_upstream_pcie_bridge(pdev);
1610 if (!tmp)
1611 return 0;
1612 /* Secondary interface's bus number and devfn 0 */
1613 parent = pdev->bus->self;
1614 while (parent != tmp) {
1615 ret = domain_context_mapping_one(domain,
1616 pci_domain_nr(parent->bus),
1617 parent->bus->number,
1618 parent->devfn, translation);
1619 if (ret)
1620 return ret;
1621 parent = parent->bus->self;
1622 }
1623 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1624 return domain_context_mapping_one(domain,
1625 pci_domain_nr(tmp->subordinate),
1626 tmp->subordinate->number, 0,
1627 translation);
1628 else /* this is a legacy PCI bridge */
1629 return domain_context_mapping_one(domain,
1630 pci_domain_nr(tmp->bus),
1631 tmp->bus->number,
1632 tmp->devfn,
1633 translation);
1634 }
1635
1636 static int domain_context_mapped(struct pci_dev *pdev)
1637 {
1638 int ret;
1639 struct pci_dev *tmp, *parent;
1640 struct intel_iommu *iommu;
1641
1642 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1643 pdev->devfn);
1644 if (!iommu)
1645 return -ENODEV;
1646
1647 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1648 if (!ret)
1649 return ret;
1650 /* dependent device mapping */
1651 tmp = pci_find_upstream_pcie_bridge(pdev);
1652 if (!tmp)
1653 return ret;
1654 /* Secondary interface's bus number and devfn 0 */
1655 parent = pdev->bus->self;
1656 while (parent != tmp) {
1657 ret = device_context_mapped(iommu, parent->bus->number,
1658 parent->devfn);
1659 if (!ret)
1660 return ret;
1661 parent = parent->bus->self;
1662 }
1663 if (tmp->is_pcie)
1664 return device_context_mapped(iommu, tmp->subordinate->number,
1665 0);
1666 else
1667 return device_context_mapped(iommu, tmp->bus->number,
1668 tmp->devfn);
1669 }
1670
1671 /* Returns a number of VTD pages, but aligned to MM page size */
1672 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1673 size_t size)
1674 {
1675 host_addr &= ~PAGE_MASK;
1676 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1677 }
1678
1679 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1680 struct scatterlist *sg, unsigned long phys_pfn,
1681 unsigned long nr_pages, int prot)
1682 {
1683 struct dma_pte *first_pte = NULL, *pte = NULL;
1684 phys_addr_t uninitialized_var(pteval);
1685 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1686 unsigned long sg_res;
1687
1688 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1689
1690 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1691 return -EINVAL;
1692
1693 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1694
1695 if (sg)
1696 sg_res = 0;
1697 else {
1698 sg_res = nr_pages + 1;
1699 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1700 }
1701
1702 while (nr_pages--) {
1703 uint64_t tmp;
1704
1705 if (!sg_res) {
1706 sg_res = aligned_nrpages(sg->offset, sg->length);
1707 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1708 sg->dma_length = sg->length;
1709 pteval = page_to_phys(sg_page(sg)) | prot;
1710 }
1711 if (!pte) {
1712 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn);
1713 if (!pte)
1714 return -ENOMEM;
1715 }
1716 /* We don't need lock here, nobody else
1717 * touches the iova range
1718 */
1719 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1720 if (tmp) {
1721 static int dumps = 5;
1722 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1723 iov_pfn, tmp, (unsigned long long)pteval);
1724 if (dumps) {
1725 dumps--;
1726 debug_dma_dump_mappings(NULL);
1727 }
1728 WARN_ON(1);
1729 }
1730 pte++;
1731 if (!nr_pages || first_pte_in_page(pte)) {
1732 domain_flush_cache(domain, first_pte,
1733 (void *)pte - (void *)first_pte);
1734 pte = NULL;
1735 }
1736 iov_pfn++;
1737 pteval += VTD_PAGE_SIZE;
1738 sg_res--;
1739 if (!sg_res)
1740 sg = sg_next(sg);
1741 }
1742 return 0;
1743 }
1744
1745 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1746 struct scatterlist *sg, unsigned long nr_pages,
1747 int prot)
1748 {
1749 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1750 }
1751
1752 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1753 unsigned long phys_pfn, unsigned long nr_pages,
1754 int prot)
1755 {
1756 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1757 }
1758
1759 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1760 {
1761 if (!iommu)
1762 return;
1763
1764 clear_context_table(iommu, bus, devfn);
1765 iommu->flush.flush_context(iommu, 0, 0, 0,
1766 DMA_CCMD_GLOBAL_INVL);
1767 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1768 }
1769
1770 static void domain_remove_dev_info(struct dmar_domain *domain)
1771 {
1772 struct device_domain_info *info;
1773 unsigned long flags;
1774 struct intel_iommu *iommu;
1775
1776 spin_lock_irqsave(&device_domain_lock, flags);
1777 while (!list_empty(&domain->devices)) {
1778 info = list_entry(domain->devices.next,
1779 struct device_domain_info, link);
1780 list_del(&info->link);
1781 list_del(&info->global);
1782 if (info->dev)
1783 info->dev->dev.archdata.iommu = NULL;
1784 spin_unlock_irqrestore(&device_domain_lock, flags);
1785
1786 iommu_disable_dev_iotlb(info);
1787 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1788 iommu_detach_dev(iommu, info->bus, info->devfn);
1789 free_devinfo_mem(info);
1790
1791 spin_lock_irqsave(&device_domain_lock, flags);
1792 }
1793 spin_unlock_irqrestore(&device_domain_lock, flags);
1794 }
1795
1796 /*
1797 * find_domain
1798 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1799 */
1800 static struct dmar_domain *
1801 find_domain(struct pci_dev *pdev)
1802 {
1803 struct device_domain_info *info;
1804
1805 /* No lock here, assumes no domain exit in normal case */
1806 info = pdev->dev.archdata.iommu;
1807 if (info)
1808 return info->domain;
1809 return NULL;
1810 }
1811
1812 /* domain is initialized */
1813 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1814 {
1815 struct dmar_domain *domain, *found = NULL;
1816 struct intel_iommu *iommu;
1817 struct dmar_drhd_unit *drhd;
1818 struct device_domain_info *info, *tmp;
1819 struct pci_dev *dev_tmp;
1820 unsigned long flags;
1821 int bus = 0, devfn = 0;
1822 int segment;
1823 int ret;
1824
1825 domain = find_domain(pdev);
1826 if (domain)
1827 return domain;
1828
1829 segment = pci_domain_nr(pdev->bus);
1830
1831 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1832 if (dev_tmp) {
1833 if (dev_tmp->is_pcie) {
1834 bus = dev_tmp->subordinate->number;
1835 devfn = 0;
1836 } else {
1837 bus = dev_tmp->bus->number;
1838 devfn = dev_tmp->devfn;
1839 }
1840 spin_lock_irqsave(&device_domain_lock, flags);
1841 list_for_each_entry(info, &device_domain_list, global) {
1842 if (info->segment == segment &&
1843 info->bus == bus && info->devfn == devfn) {
1844 found = info->domain;
1845 break;
1846 }
1847 }
1848 spin_unlock_irqrestore(&device_domain_lock, flags);
1849 /* pcie-pci bridge already has a domain, uses it */
1850 if (found) {
1851 domain = found;
1852 goto found_domain;
1853 }
1854 }
1855
1856 domain = alloc_domain();
1857 if (!domain)
1858 goto error;
1859
1860 /* Allocate new domain for the device */
1861 drhd = dmar_find_matched_drhd_unit(pdev);
1862 if (!drhd) {
1863 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1864 pci_name(pdev));
1865 return NULL;
1866 }
1867 iommu = drhd->iommu;
1868
1869 ret = iommu_attach_domain(domain, iommu);
1870 if (ret) {
1871 domain_exit(domain);
1872 goto error;
1873 }
1874
1875 if (domain_init(domain, gaw)) {
1876 domain_exit(domain);
1877 goto error;
1878 }
1879
1880 /* register pcie-to-pci device */
1881 if (dev_tmp) {
1882 info = alloc_devinfo_mem();
1883 if (!info) {
1884 domain_exit(domain);
1885 goto error;
1886 }
1887 info->segment = segment;
1888 info->bus = bus;
1889 info->devfn = devfn;
1890 info->dev = NULL;
1891 info->domain = domain;
1892 /* This domain is shared by devices under p2p bridge */
1893 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1894
1895 /* pcie-to-pci bridge already has a domain, uses it */
1896 found = NULL;
1897 spin_lock_irqsave(&device_domain_lock, flags);
1898 list_for_each_entry(tmp, &device_domain_list, global) {
1899 if (tmp->segment == segment &&
1900 tmp->bus == bus && tmp->devfn == devfn) {
1901 found = tmp->domain;
1902 break;
1903 }
1904 }
1905 if (found) {
1906 free_devinfo_mem(info);
1907 domain_exit(domain);
1908 domain = found;
1909 } else {
1910 list_add(&info->link, &domain->devices);
1911 list_add(&info->global, &device_domain_list);
1912 }
1913 spin_unlock_irqrestore(&device_domain_lock, flags);
1914 }
1915
1916 found_domain:
1917 info = alloc_devinfo_mem();
1918 if (!info)
1919 goto error;
1920 info->segment = segment;
1921 info->bus = pdev->bus->number;
1922 info->devfn = pdev->devfn;
1923 info->dev = pdev;
1924 info->domain = domain;
1925 spin_lock_irqsave(&device_domain_lock, flags);
1926 /* somebody is fast */
1927 found = find_domain(pdev);
1928 if (found != NULL) {
1929 spin_unlock_irqrestore(&device_domain_lock, flags);
1930 if (found != domain) {
1931 domain_exit(domain);
1932 domain = found;
1933 }
1934 free_devinfo_mem(info);
1935 return domain;
1936 }
1937 list_add(&info->link, &domain->devices);
1938 list_add(&info->global, &device_domain_list);
1939 pdev->dev.archdata.iommu = info;
1940 spin_unlock_irqrestore(&device_domain_lock, flags);
1941 return domain;
1942 error:
1943 /* recheck it here, maybe others set it */
1944 return find_domain(pdev);
1945 }
1946
1947 static int iommu_identity_mapping;
1948 #define IDENTMAP_ALL 1
1949 #define IDENTMAP_GFX 2
1950 #define IDENTMAP_AZALIA 4
1951
1952 static int iommu_domain_identity_map(struct dmar_domain *domain,
1953 unsigned long long start,
1954 unsigned long long end)
1955 {
1956 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
1957 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
1958
1959 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
1960 dma_to_mm_pfn(last_vpfn))) {
1961 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1962 return -ENOMEM;
1963 }
1964
1965 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
1966 start, end, domain->id);
1967 /*
1968 * RMRR range might have overlap with physical memory range,
1969 * clear it first
1970 */
1971 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
1972
1973 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
1974 last_vpfn - first_vpfn + 1,
1975 DMA_PTE_READ|DMA_PTE_WRITE);
1976 }
1977
1978 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1979 unsigned long long start,
1980 unsigned long long end)
1981 {
1982 struct dmar_domain *domain;
1983 int ret;
1984
1985 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1986 if (!domain)
1987 return -ENOMEM;
1988
1989 /* For _hardware_ passthrough, don't bother. But for software
1990 passthrough, we do it anyway -- it may indicate a memory
1991 range which is reserved in E820, so which didn't get set
1992 up to start with in si_domain */
1993 if (domain == si_domain && hw_pass_through) {
1994 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
1995 pci_name(pdev), start, end);
1996 return 0;
1997 }
1998
1999 printk(KERN_INFO
2000 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2001 pci_name(pdev), start, end);
2002
2003 if (end >> agaw_to_width(domain->agaw)) {
2004 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2005 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2006 agaw_to_width(domain->agaw),
2007 dmi_get_system_info(DMI_BIOS_VENDOR),
2008 dmi_get_system_info(DMI_BIOS_VERSION),
2009 dmi_get_system_info(DMI_PRODUCT_VERSION));
2010 ret = -EIO;
2011 goto error;
2012 }
2013
2014 ret = iommu_domain_identity_map(domain, start, end);
2015 if (ret)
2016 goto error;
2017
2018 /* context entry init */
2019 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2020 if (ret)
2021 goto error;
2022
2023 return 0;
2024
2025 error:
2026 domain_exit(domain);
2027 return ret;
2028 }
2029
2030 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2031 struct pci_dev *pdev)
2032 {
2033 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2034 return 0;
2035 return iommu_prepare_identity_map(pdev, rmrr->base_address,
2036 rmrr->end_address + 1);
2037 }
2038
2039 #ifdef CONFIG_DMAR_FLOPPY_WA
2040 static inline void iommu_prepare_isa(void)
2041 {
2042 struct pci_dev *pdev;
2043 int ret;
2044
2045 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2046 if (!pdev)
2047 return;
2048
2049 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2050 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
2051
2052 if (ret)
2053 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2054 "floppy might not work\n");
2055
2056 }
2057 #else
2058 static inline void iommu_prepare_isa(void)
2059 {
2060 return;
2061 }
2062 #endif /* !CONFIG_DMAR_FLPY_WA */
2063
2064 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2065
2066 static int __init si_domain_work_fn(unsigned long start_pfn,
2067 unsigned long end_pfn, void *datax)
2068 {
2069 int *ret = datax;
2070
2071 *ret = iommu_domain_identity_map(si_domain,
2072 (uint64_t)start_pfn << PAGE_SHIFT,
2073 (uint64_t)end_pfn << PAGE_SHIFT);
2074 return *ret;
2075
2076 }
2077
2078 static int __init si_domain_init(int hw)
2079 {
2080 struct dmar_drhd_unit *drhd;
2081 struct intel_iommu *iommu;
2082 int nid, ret = 0;
2083
2084 si_domain = alloc_domain();
2085 if (!si_domain)
2086 return -EFAULT;
2087
2088 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2089
2090 for_each_active_iommu(iommu, drhd) {
2091 ret = iommu_attach_domain(si_domain, iommu);
2092 if (ret) {
2093 domain_exit(si_domain);
2094 return -EFAULT;
2095 }
2096 }
2097
2098 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2099 domain_exit(si_domain);
2100 return -EFAULT;
2101 }
2102
2103 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2104
2105 if (hw)
2106 return 0;
2107
2108 for_each_online_node(nid) {
2109 work_with_active_regions(nid, si_domain_work_fn, &ret);
2110 if (ret)
2111 return ret;
2112 }
2113
2114 return 0;
2115 }
2116
2117 static void domain_remove_one_dev_info(struct dmar_domain *domain,
2118 struct pci_dev *pdev);
2119 static int identity_mapping(struct pci_dev *pdev)
2120 {
2121 struct device_domain_info *info;
2122
2123 if (likely(!iommu_identity_mapping))
2124 return 0;
2125
2126
2127 list_for_each_entry(info, &si_domain->devices, link)
2128 if (info->dev == pdev)
2129 return 1;
2130 return 0;
2131 }
2132
2133 static int domain_add_dev_info(struct dmar_domain *domain,
2134 struct pci_dev *pdev,
2135 int translation)
2136 {
2137 struct device_domain_info *info;
2138 unsigned long flags;
2139 int ret;
2140
2141 info = alloc_devinfo_mem();
2142 if (!info)
2143 return -ENOMEM;
2144
2145 ret = domain_context_mapping(domain, pdev, translation);
2146 if (ret) {
2147 free_devinfo_mem(info);
2148 return ret;
2149 }
2150
2151 info->segment = pci_domain_nr(pdev->bus);
2152 info->bus = pdev->bus->number;
2153 info->devfn = pdev->devfn;
2154 info->dev = pdev;
2155 info->domain = domain;
2156
2157 spin_lock_irqsave(&device_domain_lock, flags);
2158 list_add(&info->link, &domain->devices);
2159 list_add(&info->global, &device_domain_list);
2160 pdev->dev.archdata.iommu = info;
2161 spin_unlock_irqrestore(&device_domain_lock, flags);
2162
2163 return 0;
2164 }
2165
2166 static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2167 {
2168 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2169 return 1;
2170
2171 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2172 return 1;
2173
2174 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2175 return 0;
2176
2177 /*
2178 * We want to start off with all devices in the 1:1 domain, and
2179 * take them out later if we find they can't access all of memory.
2180 *
2181 * However, we can't do this for PCI devices behind bridges,
2182 * because all PCI devices behind the same bridge will end up
2183 * with the same source-id on their transactions.
2184 *
2185 * Practically speaking, we can't change things around for these
2186 * devices at run-time, because we can't be sure there'll be no
2187 * DMA transactions in flight for any of their siblings.
2188 *
2189 * So PCI devices (unless they're on the root bus) as well as
2190 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2191 * the 1:1 domain, just in _case_ one of their siblings turns out
2192 * not to be able to map all of memory.
2193 */
2194 if (!pdev->is_pcie) {
2195 if (!pci_is_root_bus(pdev->bus))
2196 return 0;
2197 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2198 return 0;
2199 } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2200 return 0;
2201
2202 /*
2203 * At boot time, we don't yet know if devices will be 64-bit capable.
2204 * Assume that they will -- if they turn out not to be, then we can
2205 * take them out of the 1:1 domain later.
2206 */
2207 if (!startup)
2208 return pdev->dma_mask > DMA_BIT_MASK(32);
2209
2210 return 1;
2211 }
2212
2213 static int __init iommu_prepare_static_identity_mapping(int hw)
2214 {
2215 struct pci_dev *pdev = NULL;
2216 int ret;
2217
2218 ret = si_domain_init(hw);
2219 if (ret)
2220 return -EFAULT;
2221
2222 for_each_pci_dev(pdev) {
2223 if (iommu_should_identity_map(pdev, 1)) {
2224 printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2225 hw ? "hardware" : "software", pci_name(pdev));
2226
2227 ret = domain_add_dev_info(si_domain, pdev,
2228 hw ? CONTEXT_TT_PASS_THROUGH :
2229 CONTEXT_TT_MULTI_LEVEL);
2230 if (ret)
2231 return ret;
2232 }
2233 }
2234
2235 return 0;
2236 }
2237
2238 int __init init_dmars(void)
2239 {
2240 struct dmar_drhd_unit *drhd;
2241 struct dmar_rmrr_unit *rmrr;
2242 struct pci_dev *pdev;
2243 struct intel_iommu *iommu;
2244 int i, ret;
2245
2246 /*
2247 * for each drhd
2248 * allocate root
2249 * initialize and program root entry to not present
2250 * endfor
2251 */
2252 for_each_drhd_unit(drhd) {
2253 g_num_of_iommus++;
2254 /*
2255 * lock not needed as this is only incremented in the single
2256 * threaded kernel __init code path all other access are read
2257 * only
2258 */
2259 }
2260
2261 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2262 GFP_KERNEL);
2263 if (!g_iommus) {
2264 printk(KERN_ERR "Allocating global iommu array failed\n");
2265 ret = -ENOMEM;
2266 goto error;
2267 }
2268
2269 deferred_flush = kzalloc(g_num_of_iommus *
2270 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2271 if (!deferred_flush) {
2272 ret = -ENOMEM;
2273 goto error;
2274 }
2275
2276 for_each_drhd_unit(drhd) {
2277 if (drhd->ignored)
2278 continue;
2279
2280 iommu = drhd->iommu;
2281 g_iommus[iommu->seq_id] = iommu;
2282
2283 ret = iommu_init_domains(iommu);
2284 if (ret)
2285 goto error;
2286
2287 /*
2288 * TBD:
2289 * we could share the same root & context tables
2290 * amoung all IOMMU's. Need to Split it later.
2291 */
2292 ret = iommu_alloc_root_entry(iommu);
2293 if (ret) {
2294 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2295 goto error;
2296 }
2297 if (!ecap_pass_through(iommu->ecap))
2298 hw_pass_through = 0;
2299 }
2300
2301 /*
2302 * Start from the sane iommu hardware state.
2303 */
2304 for_each_drhd_unit(drhd) {
2305 if (drhd->ignored)
2306 continue;
2307
2308 iommu = drhd->iommu;
2309
2310 /*
2311 * If the queued invalidation is already initialized by us
2312 * (for example, while enabling interrupt-remapping) then
2313 * we got the things already rolling from a sane state.
2314 */
2315 if (iommu->qi)
2316 continue;
2317
2318 /*
2319 * Clear any previous faults.
2320 */
2321 dmar_fault(-1, iommu);
2322 /*
2323 * Disable queued invalidation if supported and already enabled
2324 * before OS handover.
2325 */
2326 dmar_disable_qi(iommu);
2327 }
2328
2329 for_each_drhd_unit(drhd) {
2330 if (drhd->ignored)
2331 continue;
2332
2333 iommu = drhd->iommu;
2334
2335 if (dmar_enable_qi(iommu)) {
2336 /*
2337 * Queued Invalidate not enabled, use Register Based
2338 * Invalidate
2339 */
2340 iommu->flush.flush_context = __iommu_flush_context;
2341 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2342 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
2343 "invalidation\n",
2344 (unsigned long long)drhd->reg_base_addr);
2345 } else {
2346 iommu->flush.flush_context = qi_flush_context;
2347 iommu->flush.flush_iotlb = qi_flush_iotlb;
2348 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
2349 "invalidation\n",
2350 (unsigned long long)drhd->reg_base_addr);
2351 }
2352 }
2353
2354 if (iommu_pass_through)
2355 iommu_identity_mapping |= IDENTMAP_ALL;
2356
2357 #ifdef CONFIG_DMAR_BROKEN_GFX_WA
2358 iommu_identity_mapping |= IDENTMAP_GFX;
2359 #endif
2360
2361 check_tylersburg_isoch();
2362
2363 /*
2364 * If pass through is not set or not enabled, setup context entries for
2365 * identity mappings for rmrr, gfx, and isa and may fall back to static
2366 * identity mapping if iommu_identity_mapping is set.
2367 */
2368 if (iommu_identity_mapping) {
2369 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2370 if (ret) {
2371 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2372 goto error;
2373 }
2374 }
2375 /*
2376 * For each rmrr
2377 * for each dev attached to rmrr
2378 * do
2379 * locate drhd for dev, alloc domain for dev
2380 * allocate free domain
2381 * allocate page table entries for rmrr
2382 * if context not allocated for bus
2383 * allocate and init context
2384 * set present in root table for this bus
2385 * init context with domain, translation etc
2386 * endfor
2387 * endfor
2388 */
2389 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2390 for_each_rmrr_units(rmrr) {
2391 for (i = 0; i < rmrr->devices_cnt; i++) {
2392 pdev = rmrr->devices[i];
2393 /*
2394 * some BIOS lists non-exist devices in DMAR
2395 * table.
2396 */
2397 if (!pdev)
2398 continue;
2399 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2400 if (ret)
2401 printk(KERN_ERR
2402 "IOMMU: mapping reserved region failed\n");
2403 }
2404 }
2405
2406 iommu_prepare_isa();
2407
2408 /*
2409 * for each drhd
2410 * enable fault log
2411 * global invalidate context cache
2412 * global invalidate iotlb
2413 * enable translation
2414 */
2415 for_each_drhd_unit(drhd) {
2416 if (drhd->ignored)
2417 continue;
2418 iommu = drhd->iommu;
2419
2420 iommu_flush_write_buffer(iommu);
2421
2422 ret = dmar_set_interrupt(iommu);
2423 if (ret)
2424 goto error;
2425
2426 iommu_set_root_entry(iommu);
2427
2428 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2429 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2430
2431 ret = iommu_enable_translation(iommu);
2432 if (ret)
2433 goto error;
2434
2435 iommu_disable_protect_mem_regions(iommu);
2436 }
2437
2438 return 0;
2439 error:
2440 for_each_drhd_unit(drhd) {
2441 if (drhd->ignored)
2442 continue;
2443 iommu = drhd->iommu;
2444 free_iommu(iommu);
2445 }
2446 kfree(g_iommus);
2447 return ret;
2448 }
2449
2450 /* This takes a number of _MM_ pages, not VTD pages */
2451 static struct iova *intel_alloc_iova(struct device *dev,
2452 struct dmar_domain *domain,
2453 unsigned long nrpages, uint64_t dma_mask)
2454 {
2455 struct pci_dev *pdev = to_pci_dev(dev);
2456 struct iova *iova = NULL;
2457
2458 /* Restrict dma_mask to the width that the iommu can handle */
2459 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2460
2461 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2462 /*
2463 * First try to allocate an io virtual address in
2464 * DMA_BIT_MASK(32) and if that fails then try allocating
2465 * from higher range
2466 */
2467 iova = alloc_iova(&domain->iovad, nrpages,
2468 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2469 if (iova)
2470 return iova;
2471 }
2472 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2473 if (unlikely(!iova)) {
2474 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2475 nrpages, pci_name(pdev));
2476 return NULL;
2477 }
2478
2479 return iova;
2480 }
2481
2482 static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2483 {
2484 struct dmar_domain *domain;
2485 int ret;
2486
2487 domain = get_domain_for_dev(pdev,
2488 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2489 if (!domain) {
2490 printk(KERN_ERR
2491 "Allocating domain for %s failed", pci_name(pdev));
2492 return NULL;
2493 }
2494
2495 /* make sure context mapping is ok */
2496 if (unlikely(!domain_context_mapped(pdev))) {
2497 ret = domain_context_mapping(domain, pdev,
2498 CONTEXT_TT_MULTI_LEVEL);
2499 if (ret) {
2500 printk(KERN_ERR
2501 "Domain context map for %s failed",
2502 pci_name(pdev));
2503 return NULL;
2504 }
2505 }
2506
2507 return domain;
2508 }
2509
2510 static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2511 {
2512 struct device_domain_info *info;
2513
2514 /* No lock here, assumes no domain exit in normal case */
2515 info = dev->dev.archdata.iommu;
2516 if (likely(info))
2517 return info->domain;
2518
2519 return __get_valid_domain_for_dev(dev);
2520 }
2521
2522 static int iommu_dummy(struct pci_dev *pdev)
2523 {
2524 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2525 }
2526
2527 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2528 static int iommu_no_mapping(struct device *dev)
2529 {
2530 struct pci_dev *pdev;
2531 int found;
2532
2533 if (unlikely(dev->bus != &pci_bus_type))
2534 return 1;
2535
2536 pdev = to_pci_dev(dev);
2537 if (iommu_dummy(pdev))
2538 return 1;
2539
2540 if (!iommu_identity_mapping)
2541 return 0;
2542
2543 found = identity_mapping(pdev);
2544 if (found) {
2545 if (iommu_should_identity_map(pdev, 0))
2546 return 1;
2547 else {
2548 /*
2549 * 32 bit DMA is removed from si_domain and fall back
2550 * to non-identity mapping.
2551 */
2552 domain_remove_one_dev_info(si_domain, pdev);
2553 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2554 pci_name(pdev));
2555 return 0;
2556 }
2557 } else {
2558 /*
2559 * In case of a detached 64 bit DMA device from vm, the device
2560 * is put into si_domain for identity mapping.
2561 */
2562 if (iommu_should_identity_map(pdev, 0)) {
2563 int ret;
2564 ret = domain_add_dev_info(si_domain, pdev,
2565 hw_pass_through ?
2566 CONTEXT_TT_PASS_THROUGH :
2567 CONTEXT_TT_MULTI_LEVEL);
2568 if (!ret) {
2569 printk(KERN_INFO "64bit %s uses identity mapping\n",
2570 pci_name(pdev));
2571 return 1;
2572 }
2573 }
2574 }
2575
2576 return 0;
2577 }
2578
2579 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2580 size_t size, int dir, u64 dma_mask)
2581 {
2582 struct pci_dev *pdev = to_pci_dev(hwdev);
2583 struct dmar_domain *domain;
2584 phys_addr_t start_paddr;
2585 struct iova *iova;
2586 int prot = 0;
2587 int ret;
2588 struct intel_iommu *iommu;
2589 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2590
2591 BUG_ON(dir == DMA_NONE);
2592
2593 if (iommu_no_mapping(hwdev))
2594 return paddr;
2595
2596 domain = get_valid_domain_for_dev(pdev);
2597 if (!domain)
2598 return 0;
2599
2600 iommu = domain_get_iommu(domain);
2601 size = aligned_nrpages(paddr, size);
2602
2603 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2604 pdev->dma_mask);
2605 if (!iova)
2606 goto error;
2607
2608 /*
2609 * Check if DMAR supports zero-length reads on write only
2610 * mappings..
2611 */
2612 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2613 !cap_zlr(iommu->cap))
2614 prot |= DMA_PTE_READ;
2615 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2616 prot |= DMA_PTE_WRITE;
2617 /*
2618 * paddr - (paddr + size) might be partial page, we should map the whole
2619 * page. Note: if two part of one page are separately mapped, we
2620 * might have two guest_addr mapping to the same host paddr, but this
2621 * is not a big problem
2622 */
2623 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2624 mm_to_dma_pfn(paddr_pfn), size, prot);
2625 if (ret)
2626 goto error;
2627
2628 /* it's a non-present to present mapping. Only flush if caching mode */
2629 if (cap_caching_mode(iommu->cap))
2630 iommu_flush_iotlb_psi(iommu, 0, mm_to_dma_pfn(iova->pfn_lo), size);
2631 else
2632 iommu_flush_write_buffer(iommu);
2633
2634 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2635 start_paddr += paddr & ~PAGE_MASK;
2636 return start_paddr;
2637
2638 error:
2639 if (iova)
2640 __free_iova(&domain->iovad, iova);
2641 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2642 pci_name(pdev), size, (unsigned long long)paddr, dir);
2643 return 0;
2644 }
2645
2646 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2647 unsigned long offset, size_t size,
2648 enum dma_data_direction dir,
2649 struct dma_attrs *attrs)
2650 {
2651 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2652 dir, to_pci_dev(dev)->dma_mask);
2653 }
2654
2655 static void flush_unmaps(void)
2656 {
2657 int i, j;
2658
2659 timer_on = 0;
2660
2661 /* just flush them all */
2662 for (i = 0; i < g_num_of_iommus; i++) {
2663 struct intel_iommu *iommu = g_iommus[i];
2664 if (!iommu)
2665 continue;
2666
2667 if (!deferred_flush[i].next)
2668 continue;
2669
2670 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2671 DMA_TLB_GLOBAL_FLUSH);
2672 for (j = 0; j < deferred_flush[i].next; j++) {
2673 unsigned long mask;
2674 struct iova *iova = deferred_flush[i].iova[j];
2675
2676 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2677 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2678 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2679 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2680 }
2681 deferred_flush[i].next = 0;
2682 }
2683
2684 list_size = 0;
2685 }
2686
2687 static void flush_unmaps_timeout(unsigned long data)
2688 {
2689 unsigned long flags;
2690
2691 spin_lock_irqsave(&async_umap_flush_lock, flags);
2692 flush_unmaps();
2693 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2694 }
2695
2696 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2697 {
2698 unsigned long flags;
2699 int next, iommu_id;
2700 struct intel_iommu *iommu;
2701
2702 spin_lock_irqsave(&async_umap_flush_lock, flags);
2703 if (list_size == HIGH_WATER_MARK)
2704 flush_unmaps();
2705
2706 iommu = domain_get_iommu(dom);
2707 iommu_id = iommu->seq_id;
2708
2709 next = deferred_flush[iommu_id].next;
2710 deferred_flush[iommu_id].domain[next] = dom;
2711 deferred_flush[iommu_id].iova[next] = iova;
2712 deferred_flush[iommu_id].next++;
2713
2714 if (!timer_on) {
2715 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2716 timer_on = 1;
2717 }
2718 list_size++;
2719 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2720 }
2721
2722 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2723 size_t size, enum dma_data_direction dir,
2724 struct dma_attrs *attrs)
2725 {
2726 struct pci_dev *pdev = to_pci_dev(dev);
2727 struct dmar_domain *domain;
2728 unsigned long start_pfn, last_pfn;
2729 struct iova *iova;
2730 struct intel_iommu *iommu;
2731
2732 if (iommu_no_mapping(dev))
2733 return;
2734
2735 domain = find_domain(pdev);
2736 BUG_ON(!domain);
2737
2738 iommu = domain_get_iommu(domain);
2739
2740 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2741 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2742 (unsigned long long)dev_addr))
2743 return;
2744
2745 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2746 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2747
2748 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2749 pci_name(pdev), start_pfn, last_pfn);
2750
2751 /* clear the whole page */
2752 dma_pte_clear_range(domain, start_pfn, last_pfn);
2753
2754 /* free page tables */
2755 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2756
2757 if (intel_iommu_strict) {
2758 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2759 last_pfn - start_pfn + 1);
2760 /* free iova */
2761 __free_iova(&domain->iovad, iova);
2762 } else {
2763 add_unmap(domain, iova);
2764 /*
2765 * queue up the release of the unmap to save the 1/6th of the
2766 * cpu used up by the iotlb flush operation...
2767 */
2768 }
2769 }
2770
2771 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2772 dma_addr_t *dma_handle, gfp_t flags)
2773 {
2774 void *vaddr;
2775 int order;
2776
2777 size = PAGE_ALIGN(size);
2778 order = get_order(size);
2779
2780 if (!iommu_no_mapping(hwdev))
2781 flags &= ~(GFP_DMA | GFP_DMA32);
2782 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2783 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2784 flags |= GFP_DMA;
2785 else
2786 flags |= GFP_DMA32;
2787 }
2788
2789 vaddr = (void *)__get_free_pages(flags, order);
2790 if (!vaddr)
2791 return NULL;
2792 memset(vaddr, 0, size);
2793
2794 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2795 DMA_BIDIRECTIONAL,
2796 hwdev->coherent_dma_mask);
2797 if (*dma_handle)
2798 return vaddr;
2799 free_pages((unsigned long)vaddr, order);
2800 return NULL;
2801 }
2802
2803 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2804 dma_addr_t dma_handle)
2805 {
2806 int order;
2807
2808 size = PAGE_ALIGN(size);
2809 order = get_order(size);
2810
2811 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
2812 free_pages((unsigned long)vaddr, order);
2813 }
2814
2815 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2816 int nelems, enum dma_data_direction dir,
2817 struct dma_attrs *attrs)
2818 {
2819 struct pci_dev *pdev = to_pci_dev(hwdev);
2820 struct dmar_domain *domain;
2821 unsigned long start_pfn, last_pfn;
2822 struct iova *iova;
2823 struct intel_iommu *iommu;
2824
2825 if (iommu_no_mapping(hwdev))
2826 return;
2827
2828 domain = find_domain(pdev);
2829 BUG_ON(!domain);
2830
2831 iommu = domain_get_iommu(domain);
2832
2833 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2834 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2835 (unsigned long long)sglist[0].dma_address))
2836 return;
2837
2838 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2839 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2840
2841 /* clear the whole page */
2842 dma_pte_clear_range(domain, start_pfn, last_pfn);
2843
2844 /* free page tables */
2845 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2846
2847 if (intel_iommu_strict) {
2848 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2849 last_pfn - start_pfn + 1);
2850 /* free iova */
2851 __free_iova(&domain->iovad, iova);
2852 } else {
2853 add_unmap(domain, iova);
2854 /*
2855 * queue up the release of the unmap to save the 1/6th of the
2856 * cpu used up by the iotlb flush operation...
2857 */
2858 }
2859 }
2860
2861 static int intel_nontranslate_map_sg(struct device *hddev,
2862 struct scatterlist *sglist, int nelems, int dir)
2863 {
2864 int i;
2865 struct scatterlist *sg;
2866
2867 for_each_sg(sglist, sg, nelems, i) {
2868 BUG_ON(!sg_page(sg));
2869 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
2870 sg->dma_length = sg->length;
2871 }
2872 return nelems;
2873 }
2874
2875 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2876 enum dma_data_direction dir, struct dma_attrs *attrs)
2877 {
2878 int i;
2879 struct pci_dev *pdev = to_pci_dev(hwdev);
2880 struct dmar_domain *domain;
2881 size_t size = 0;
2882 int prot = 0;
2883 size_t offset_pfn = 0;
2884 struct iova *iova = NULL;
2885 int ret;
2886 struct scatterlist *sg;
2887 unsigned long start_vpfn;
2888 struct intel_iommu *iommu;
2889
2890 BUG_ON(dir == DMA_NONE);
2891 if (iommu_no_mapping(hwdev))
2892 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2893
2894 domain = get_valid_domain_for_dev(pdev);
2895 if (!domain)
2896 return 0;
2897
2898 iommu = domain_get_iommu(domain);
2899
2900 for_each_sg(sglist, sg, nelems, i)
2901 size += aligned_nrpages(sg->offset, sg->length);
2902
2903 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2904 pdev->dma_mask);
2905 if (!iova) {
2906 sglist->dma_length = 0;
2907 return 0;
2908 }
2909
2910 /*
2911 * Check if DMAR supports zero-length reads on write only
2912 * mappings..
2913 */
2914 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2915 !cap_zlr(iommu->cap))
2916 prot |= DMA_PTE_READ;
2917 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2918 prot |= DMA_PTE_WRITE;
2919
2920 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
2921
2922 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
2923 if (unlikely(ret)) {
2924 /* clear the page */
2925 dma_pte_clear_range(domain, start_vpfn,
2926 start_vpfn + size - 1);
2927 /* free page tables */
2928 dma_pte_free_pagetable(domain, start_vpfn,
2929 start_vpfn + size - 1);
2930 /* free iova */
2931 __free_iova(&domain->iovad, iova);
2932 return 0;
2933 }
2934
2935 /* it's a non-present to present mapping. Only flush if caching mode */
2936 if (cap_caching_mode(iommu->cap))
2937 iommu_flush_iotlb_psi(iommu, 0, start_vpfn, offset_pfn);
2938 else
2939 iommu_flush_write_buffer(iommu);
2940
2941 return nelems;
2942 }
2943
2944 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2945 {
2946 return !dma_addr;
2947 }
2948
2949 struct dma_map_ops intel_dma_ops = {
2950 .alloc_coherent = intel_alloc_coherent,
2951 .free_coherent = intel_free_coherent,
2952 .map_sg = intel_map_sg,
2953 .unmap_sg = intel_unmap_sg,
2954 .map_page = intel_map_page,
2955 .unmap_page = intel_unmap_page,
2956 .mapping_error = intel_mapping_error,
2957 };
2958
2959 static inline int iommu_domain_cache_init(void)
2960 {
2961 int ret = 0;
2962
2963 iommu_domain_cache = kmem_cache_create("iommu_domain",
2964 sizeof(struct dmar_domain),
2965 0,
2966 SLAB_HWCACHE_ALIGN,
2967
2968 NULL);
2969 if (!iommu_domain_cache) {
2970 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2971 ret = -ENOMEM;
2972 }
2973
2974 return ret;
2975 }
2976
2977 static inline int iommu_devinfo_cache_init(void)
2978 {
2979 int ret = 0;
2980
2981 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2982 sizeof(struct device_domain_info),
2983 0,
2984 SLAB_HWCACHE_ALIGN,
2985 NULL);
2986 if (!iommu_devinfo_cache) {
2987 printk(KERN_ERR "Couldn't create devinfo cache\n");
2988 ret = -ENOMEM;
2989 }
2990
2991 return ret;
2992 }
2993
2994 static inline int iommu_iova_cache_init(void)
2995 {
2996 int ret = 0;
2997
2998 iommu_iova_cache = kmem_cache_create("iommu_iova",
2999 sizeof(struct iova),
3000 0,
3001 SLAB_HWCACHE_ALIGN,
3002 NULL);
3003 if (!iommu_iova_cache) {
3004 printk(KERN_ERR "Couldn't create iova cache\n");
3005 ret = -ENOMEM;
3006 }
3007
3008 return ret;
3009 }
3010
3011 static int __init iommu_init_mempool(void)
3012 {
3013 int ret;
3014 ret = iommu_iova_cache_init();
3015 if (ret)
3016 return ret;
3017
3018 ret = iommu_domain_cache_init();
3019 if (ret)
3020 goto domain_error;
3021
3022 ret = iommu_devinfo_cache_init();
3023 if (!ret)
3024 return ret;
3025
3026 kmem_cache_destroy(iommu_domain_cache);
3027 domain_error:
3028 kmem_cache_destroy(iommu_iova_cache);
3029
3030 return -ENOMEM;
3031 }
3032
3033 static void __init iommu_exit_mempool(void)
3034 {
3035 kmem_cache_destroy(iommu_devinfo_cache);
3036 kmem_cache_destroy(iommu_domain_cache);
3037 kmem_cache_destroy(iommu_iova_cache);
3038
3039 }
3040
3041 static void __init init_no_remapping_devices(void)
3042 {
3043 struct dmar_drhd_unit *drhd;
3044
3045 for_each_drhd_unit(drhd) {
3046 if (!drhd->include_all) {
3047 int i;
3048 for (i = 0; i < drhd->devices_cnt; i++)
3049 if (drhd->devices[i] != NULL)
3050 break;
3051 /* ignore DMAR unit if no pci devices exist */
3052 if (i == drhd->devices_cnt)
3053 drhd->ignored = 1;
3054 }
3055 }
3056
3057 if (dmar_map_gfx)
3058 return;
3059
3060 for_each_drhd_unit(drhd) {
3061 int i;
3062 if (drhd->ignored || drhd->include_all)
3063 continue;
3064
3065 for (i = 0; i < drhd->devices_cnt; i++)
3066 if (drhd->devices[i] &&
3067 !IS_GFX_DEVICE(drhd->devices[i]))
3068 break;
3069
3070 if (i < drhd->devices_cnt)
3071 continue;
3072
3073 /* bypass IOMMU if it is just for gfx devices */
3074 drhd->ignored = 1;
3075 for (i = 0; i < drhd->devices_cnt; i++) {
3076 if (!drhd->devices[i])
3077 continue;
3078 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3079 }
3080 }
3081 }
3082
3083 #ifdef CONFIG_SUSPEND
3084 static int init_iommu_hw(void)
3085 {
3086 struct dmar_drhd_unit *drhd;
3087 struct intel_iommu *iommu = NULL;
3088
3089 for_each_active_iommu(iommu, drhd)
3090 if (iommu->qi)
3091 dmar_reenable_qi(iommu);
3092
3093 for_each_active_iommu(iommu, drhd) {
3094 iommu_flush_write_buffer(iommu);
3095
3096 iommu_set_root_entry(iommu);
3097
3098 iommu->flush.flush_context(iommu, 0, 0, 0,
3099 DMA_CCMD_GLOBAL_INVL);
3100 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3101 DMA_TLB_GLOBAL_FLUSH);
3102 iommu_enable_translation(iommu);
3103 iommu_disable_protect_mem_regions(iommu);
3104 }
3105
3106 return 0;
3107 }
3108
3109 static void iommu_flush_all(void)
3110 {
3111 struct dmar_drhd_unit *drhd;
3112 struct intel_iommu *iommu;
3113
3114 for_each_active_iommu(iommu, drhd) {
3115 iommu->flush.flush_context(iommu, 0, 0, 0,
3116 DMA_CCMD_GLOBAL_INVL);
3117 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3118 DMA_TLB_GLOBAL_FLUSH);
3119 }
3120 }
3121
3122 static int iommu_suspend(struct sys_device *dev, pm_message_t state)
3123 {
3124 struct dmar_drhd_unit *drhd;
3125 struct intel_iommu *iommu = NULL;
3126 unsigned long flag;
3127
3128 for_each_active_iommu(iommu, drhd) {
3129 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3130 GFP_ATOMIC);
3131 if (!iommu->iommu_state)
3132 goto nomem;
3133 }
3134
3135 iommu_flush_all();
3136
3137 for_each_active_iommu(iommu, drhd) {
3138 iommu_disable_translation(iommu);
3139
3140 spin_lock_irqsave(&iommu->register_lock, flag);
3141
3142 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3143 readl(iommu->reg + DMAR_FECTL_REG);
3144 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3145 readl(iommu->reg + DMAR_FEDATA_REG);
3146 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3147 readl(iommu->reg + DMAR_FEADDR_REG);
3148 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3149 readl(iommu->reg + DMAR_FEUADDR_REG);
3150
3151 spin_unlock_irqrestore(&iommu->register_lock, flag);
3152 }
3153 return 0;
3154
3155 nomem:
3156 for_each_active_iommu(iommu, drhd)
3157 kfree(iommu->iommu_state);
3158
3159 return -ENOMEM;
3160 }
3161
3162 static int iommu_resume(struct sys_device *dev)
3163 {
3164 struct dmar_drhd_unit *drhd;
3165 struct intel_iommu *iommu = NULL;
3166 unsigned long flag;
3167
3168 if (init_iommu_hw()) {
3169 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3170 return -EIO;
3171 }
3172
3173 for_each_active_iommu(iommu, drhd) {
3174
3175 spin_lock_irqsave(&iommu->register_lock, flag);
3176
3177 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3178 iommu->reg + DMAR_FECTL_REG);
3179 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3180 iommu->reg + DMAR_FEDATA_REG);
3181 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3182 iommu->reg + DMAR_FEADDR_REG);
3183 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3184 iommu->reg + DMAR_FEUADDR_REG);
3185
3186 spin_unlock_irqrestore(&iommu->register_lock, flag);
3187 }
3188
3189 for_each_active_iommu(iommu, drhd)
3190 kfree(iommu->iommu_state);
3191
3192 return 0;
3193 }
3194
3195 static struct sysdev_class iommu_sysclass = {
3196 .name = "iommu",
3197 .resume = iommu_resume,
3198 .suspend = iommu_suspend,
3199 };
3200
3201 static struct sys_device device_iommu = {
3202 .cls = &iommu_sysclass,
3203 };
3204
3205 static int __init init_iommu_sysfs(void)
3206 {
3207 int error;
3208
3209 error = sysdev_class_register(&iommu_sysclass);
3210 if (error)
3211 return error;
3212
3213 error = sysdev_register(&device_iommu);
3214 if (error)
3215 sysdev_class_unregister(&iommu_sysclass);
3216
3217 return error;
3218 }
3219
3220 #else
3221 static int __init init_iommu_sysfs(void)
3222 {
3223 return 0;
3224 }
3225 #endif /* CONFIG_PM */
3226
3227 /*
3228 * Here we only respond to action of unbound device from driver.
3229 *
3230 * Added device is not attached to its DMAR domain here yet. That will happen
3231 * when mapping the device to iova.
3232 */
3233 static int device_notifier(struct notifier_block *nb,
3234 unsigned long action, void *data)
3235 {
3236 struct device *dev = data;
3237 struct pci_dev *pdev = to_pci_dev(dev);
3238 struct dmar_domain *domain;
3239
3240 domain = find_domain(pdev);
3241 if (!domain)
3242 return 0;
3243
3244 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through)
3245 domain_remove_one_dev_info(domain, pdev);
3246
3247 return 0;
3248 }
3249
3250 static struct notifier_block device_nb = {
3251 .notifier_call = device_notifier,
3252 };
3253
3254 int __init intel_iommu_init(void)
3255 {
3256 int ret = 0;
3257 int force_on = 0;
3258
3259 /* VT-d is required for a TXT/tboot launch, so enforce that */
3260 force_on = tboot_force_iommu();
3261
3262 if (dmar_table_init()) {
3263 if (force_on)
3264 panic("tboot: Failed to initialize DMAR table\n");
3265 return -ENODEV;
3266 }
3267
3268 if (dmar_dev_scope_init()) {
3269 if (force_on)
3270 panic("tboot: Failed to initialize DMAR device scope\n");
3271 return -ENODEV;
3272 }
3273
3274 /*
3275 * Check the need for DMA-remapping initialization now.
3276 * Above initialization will also be used by Interrupt-remapping.
3277 */
3278 if (no_iommu || dmar_disabled)
3279 return -ENODEV;
3280
3281 iommu_init_mempool();
3282 dmar_init_reserved_ranges();
3283
3284 init_no_remapping_devices();
3285
3286 ret = init_dmars();
3287 if (ret) {
3288 if (force_on)
3289 panic("tboot: Failed to initialize DMARs\n");
3290 printk(KERN_ERR "IOMMU: dmar init failed\n");
3291 put_iova_domain(&reserved_iova_list);
3292 iommu_exit_mempool();
3293 return ret;
3294 }
3295 printk(KERN_INFO
3296 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3297
3298 init_timer(&unmap_timer);
3299 #ifdef CONFIG_SWIOTLB
3300 swiotlb = 0;
3301 #endif
3302 dma_ops = &intel_dma_ops;
3303
3304 init_iommu_sysfs();
3305
3306 register_iommu(&intel_iommu_ops);
3307
3308 bus_register_notifier(&pci_bus_type, &device_nb);
3309
3310 return 0;
3311 }
3312
3313 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3314 struct pci_dev *pdev)
3315 {
3316 struct pci_dev *tmp, *parent;
3317
3318 if (!iommu || !pdev)
3319 return;
3320
3321 /* dependent device detach */
3322 tmp = pci_find_upstream_pcie_bridge(pdev);
3323 /* Secondary interface's bus number and devfn 0 */
3324 if (tmp) {
3325 parent = pdev->bus->self;
3326 while (parent != tmp) {
3327 iommu_detach_dev(iommu, parent->bus->number,
3328 parent->devfn);
3329 parent = parent->bus->self;
3330 }
3331 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
3332 iommu_detach_dev(iommu,
3333 tmp->subordinate->number, 0);
3334 else /* this is a legacy PCI bridge */
3335 iommu_detach_dev(iommu, tmp->bus->number,
3336 tmp->devfn);
3337 }
3338 }
3339
3340 static void domain_remove_one_dev_info(struct dmar_domain *domain,
3341 struct pci_dev *pdev)
3342 {
3343 struct device_domain_info *info;
3344 struct intel_iommu *iommu;
3345 unsigned long flags;
3346 int found = 0;
3347 struct list_head *entry, *tmp;
3348
3349 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3350 pdev->devfn);
3351 if (!iommu)
3352 return;
3353
3354 spin_lock_irqsave(&device_domain_lock, flags);
3355 list_for_each_safe(entry, tmp, &domain->devices) {
3356 info = list_entry(entry, struct device_domain_info, link);
3357 /* No need to compare PCI domain; it has to be the same */
3358 if (info->bus == pdev->bus->number &&
3359 info->devfn == pdev->devfn) {
3360 list_del(&info->link);
3361 list_del(&info->global);
3362 if (info->dev)
3363 info->dev->dev.archdata.iommu = NULL;
3364 spin_unlock_irqrestore(&device_domain_lock, flags);
3365
3366 iommu_disable_dev_iotlb(info);
3367 iommu_detach_dev(iommu, info->bus, info->devfn);
3368 iommu_detach_dependent_devices(iommu, pdev);
3369 free_devinfo_mem(info);
3370
3371 spin_lock_irqsave(&device_domain_lock, flags);
3372
3373 if (found)
3374 break;
3375 else
3376 continue;
3377 }
3378
3379 /* if there is no other devices under the same iommu
3380 * owned by this domain, clear this iommu in iommu_bmp
3381 * update iommu count and coherency
3382 */
3383 if (iommu == device_to_iommu(info->segment, info->bus,
3384 info->devfn))
3385 found = 1;
3386 }
3387
3388 if (found == 0) {
3389 unsigned long tmp_flags;
3390 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3391 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3392 domain->iommu_count--;
3393 domain_update_iommu_cap(domain);
3394 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3395 }
3396
3397 spin_unlock_irqrestore(&device_domain_lock, flags);
3398 }
3399
3400 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3401 {
3402 struct device_domain_info *info;
3403 struct intel_iommu *iommu;
3404 unsigned long flags1, flags2;
3405
3406 spin_lock_irqsave(&device_domain_lock, flags1);
3407 while (!list_empty(&domain->devices)) {
3408 info = list_entry(domain->devices.next,
3409 struct device_domain_info, link);
3410 list_del(&info->link);
3411 list_del(&info->global);
3412 if (info->dev)
3413 info->dev->dev.archdata.iommu = NULL;
3414
3415 spin_unlock_irqrestore(&device_domain_lock, flags1);
3416
3417 iommu_disable_dev_iotlb(info);
3418 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3419 iommu_detach_dev(iommu, info->bus, info->devfn);
3420 iommu_detach_dependent_devices(iommu, info->dev);
3421
3422 /* clear this iommu in iommu_bmp, update iommu count
3423 * and capabilities
3424 */
3425 spin_lock_irqsave(&domain->iommu_lock, flags2);
3426 if (test_and_clear_bit(iommu->seq_id,
3427 &domain->iommu_bmp)) {
3428 domain->iommu_count--;
3429 domain_update_iommu_cap(domain);
3430 }
3431 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3432
3433 free_devinfo_mem(info);
3434 spin_lock_irqsave(&device_domain_lock, flags1);
3435 }
3436 spin_unlock_irqrestore(&device_domain_lock, flags1);
3437 }
3438
3439 /* domain id for virtual machine, it won't be set in context */
3440 static unsigned long vm_domid;
3441
3442 static int vm_domain_min_agaw(struct dmar_domain *domain)
3443 {
3444 int i;
3445 int min_agaw = domain->agaw;
3446
3447 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
3448 for (; i < g_num_of_iommus; ) {
3449 if (min_agaw > g_iommus[i]->agaw)
3450 min_agaw = g_iommus[i]->agaw;
3451
3452 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
3453 }
3454
3455 return min_agaw;
3456 }
3457
3458 static struct dmar_domain *iommu_alloc_vm_domain(void)
3459 {
3460 struct dmar_domain *domain;
3461
3462 domain = alloc_domain_mem();
3463 if (!domain)
3464 return NULL;
3465
3466 domain->id = vm_domid++;
3467 domain->nid = -1;
3468 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3469 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3470
3471 return domain;
3472 }
3473
3474 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3475 {
3476 int adjust_width;
3477
3478 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3479 spin_lock_init(&domain->iommu_lock);
3480
3481 domain_reserve_special_ranges(domain);
3482
3483 /* calculate AGAW */
3484 domain->gaw = guest_width;
3485 adjust_width = guestwidth_to_adjustwidth(guest_width);
3486 domain->agaw = width_to_agaw(adjust_width);
3487
3488 INIT_LIST_HEAD(&domain->devices);
3489
3490 domain->iommu_count = 0;
3491 domain->iommu_coherency = 0;
3492 domain->iommu_snooping = 0;
3493 domain->max_addr = 0;
3494 domain->nid = -1;
3495
3496 /* always allocate the top pgd */
3497 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
3498 if (!domain->pgd)
3499 return -ENOMEM;
3500 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3501 return 0;
3502 }
3503
3504 static void iommu_free_vm_domain(struct dmar_domain *domain)
3505 {
3506 unsigned long flags;
3507 struct dmar_drhd_unit *drhd;
3508 struct intel_iommu *iommu;
3509 unsigned long i;
3510 unsigned long ndomains;
3511
3512 for_each_drhd_unit(drhd) {
3513 if (drhd->ignored)
3514 continue;
3515 iommu = drhd->iommu;
3516
3517 ndomains = cap_ndoms(iommu->cap);
3518 i = find_first_bit(iommu->domain_ids, ndomains);
3519 for (; i < ndomains; ) {
3520 if (iommu->domains[i] == domain) {
3521 spin_lock_irqsave(&iommu->lock, flags);
3522 clear_bit(i, iommu->domain_ids);
3523 iommu->domains[i] = NULL;
3524 spin_unlock_irqrestore(&iommu->lock, flags);
3525 break;
3526 }
3527 i = find_next_bit(iommu->domain_ids, ndomains, i+1);
3528 }
3529 }
3530 }
3531
3532 static void vm_domain_exit(struct dmar_domain *domain)
3533 {
3534 /* Domain 0 is reserved, so dont process it */
3535 if (!domain)
3536 return;
3537
3538 vm_domain_remove_all_dev_info(domain);
3539 /* destroy iovas */
3540 put_iova_domain(&domain->iovad);
3541
3542 /* clear ptes */
3543 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3544
3545 /* free page tables */
3546 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3547
3548 iommu_free_vm_domain(domain);
3549 free_domain_mem(domain);
3550 }
3551
3552 static int intel_iommu_domain_init(struct iommu_domain *domain)
3553 {
3554 struct dmar_domain *dmar_domain;
3555
3556 dmar_domain = iommu_alloc_vm_domain();
3557 if (!dmar_domain) {
3558 printk(KERN_ERR
3559 "intel_iommu_domain_init: dmar_domain == NULL\n");
3560 return -ENOMEM;
3561 }
3562 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3563 printk(KERN_ERR
3564 "intel_iommu_domain_init() failed\n");
3565 vm_domain_exit(dmar_domain);
3566 return -ENOMEM;
3567 }
3568 domain->priv = dmar_domain;
3569
3570 return 0;
3571 }
3572
3573 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3574 {
3575 struct dmar_domain *dmar_domain = domain->priv;
3576
3577 domain->priv = NULL;
3578 vm_domain_exit(dmar_domain);
3579 }
3580
3581 static int intel_iommu_attach_device(struct iommu_domain *domain,
3582 struct device *dev)
3583 {
3584 struct dmar_domain *dmar_domain = domain->priv;
3585 struct pci_dev *pdev = to_pci_dev(dev);
3586 struct intel_iommu *iommu;
3587 int addr_width;
3588 u64 end;
3589
3590 /* normally pdev is not mapped */
3591 if (unlikely(domain_context_mapped(pdev))) {
3592 struct dmar_domain *old_domain;
3593
3594 old_domain = find_domain(pdev);
3595 if (old_domain) {
3596 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3597 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3598 domain_remove_one_dev_info(old_domain, pdev);
3599 else
3600 domain_remove_dev_info(old_domain);
3601 }
3602 }
3603
3604 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3605 pdev->devfn);
3606 if (!iommu)
3607 return -ENODEV;
3608
3609 /* check if this iommu agaw is sufficient for max mapped address */
3610 addr_width = agaw_to_width(iommu->agaw);
3611 end = DOMAIN_MAX_ADDR(addr_width);
3612 end = end & VTD_PAGE_MASK;
3613 if (end < dmar_domain->max_addr) {
3614 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3615 "sufficient for the mapped address (%llx)\n",
3616 __func__, iommu->agaw, dmar_domain->max_addr);
3617 return -EFAULT;
3618 }
3619
3620 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
3621 }
3622
3623 static void intel_iommu_detach_device(struct iommu_domain *domain,
3624 struct device *dev)
3625 {
3626 struct dmar_domain *dmar_domain = domain->priv;
3627 struct pci_dev *pdev = to_pci_dev(dev);
3628
3629 domain_remove_one_dev_info(dmar_domain, pdev);
3630 }
3631
3632 static int intel_iommu_map_range(struct iommu_domain *domain,
3633 unsigned long iova, phys_addr_t hpa,
3634 size_t size, int iommu_prot)
3635 {
3636 struct dmar_domain *dmar_domain = domain->priv;
3637 u64 max_addr;
3638 int addr_width;
3639 int prot = 0;
3640 int ret;
3641
3642 if (iommu_prot & IOMMU_READ)
3643 prot |= DMA_PTE_READ;
3644 if (iommu_prot & IOMMU_WRITE)
3645 prot |= DMA_PTE_WRITE;
3646 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3647 prot |= DMA_PTE_SNP;
3648
3649 max_addr = iova + size;
3650 if (dmar_domain->max_addr < max_addr) {
3651 int min_agaw;
3652 u64 end;
3653
3654 /* check if minimum agaw is sufficient for mapped address */
3655 min_agaw = vm_domain_min_agaw(dmar_domain);
3656 addr_width = agaw_to_width(min_agaw);
3657 end = DOMAIN_MAX_ADDR(addr_width);
3658 end = end & VTD_PAGE_MASK;
3659 if (end < max_addr) {
3660 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3661 "sufficient for the mapped address (%llx)\n",
3662 __func__, min_agaw, max_addr);
3663 return -EFAULT;
3664 }
3665 dmar_domain->max_addr = max_addr;
3666 }
3667 /* Round up size to next multiple of PAGE_SIZE, if it and
3668 the low bits of hpa would take us onto the next page */
3669 size = aligned_nrpages(hpa, size);
3670 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3671 hpa >> VTD_PAGE_SHIFT, size, prot);
3672 return ret;
3673 }
3674
3675 static void intel_iommu_unmap_range(struct iommu_domain *domain,
3676 unsigned long iova, size_t size)
3677 {
3678 struct dmar_domain *dmar_domain = domain->priv;
3679
3680 if (!size)
3681 return;
3682
3683 dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3684 (iova + size - 1) >> VTD_PAGE_SHIFT);
3685
3686 if (dmar_domain->max_addr == iova + size)
3687 dmar_domain->max_addr = iova;
3688 }
3689
3690 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3691 unsigned long iova)
3692 {
3693 struct dmar_domain *dmar_domain = domain->priv;
3694 struct dma_pte *pte;
3695 u64 phys = 0;
3696
3697 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT);
3698 if (pte)
3699 phys = dma_pte_addr(pte);
3700
3701 return phys;
3702 }
3703
3704 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3705 unsigned long cap)
3706 {
3707 struct dmar_domain *dmar_domain = domain->priv;
3708
3709 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3710 return dmar_domain->iommu_snooping;
3711
3712 return 0;
3713 }
3714
3715 static struct iommu_ops intel_iommu_ops = {
3716 .domain_init = intel_iommu_domain_init,
3717 .domain_destroy = intel_iommu_domain_destroy,
3718 .attach_dev = intel_iommu_attach_device,
3719 .detach_dev = intel_iommu_detach_device,
3720 .map = intel_iommu_map_range,
3721 .unmap = intel_iommu_unmap_range,
3722 .iova_to_phys = intel_iommu_iova_to_phys,
3723 .domain_has_cap = intel_iommu_domain_has_cap,
3724 };
3725
3726 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3727 {
3728 /*
3729 * Mobile 4 Series Chipset neglects to set RWBF capability,
3730 * but needs it:
3731 */
3732 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3733 rwbf_quirk = 1;
3734 }
3735
3736 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
3737
3738 /* On Tylersburg chipsets, some BIOSes have been known to enable the
3739 ISOCH DMAR unit for the Azalia sound device, but not give it any
3740 TLB entries, which causes it to deadlock. Check for that. We do
3741 this in a function called from init_dmars(), instead of in a PCI
3742 quirk, because we don't want to print the obnoxious "BIOS broken"
3743 message if VT-d is actually disabled.
3744 */
3745 static void __init check_tylersburg_isoch(void)
3746 {
3747 struct pci_dev *pdev;
3748 uint32_t vtisochctrl;
3749
3750 /* If there's no Azalia in the system anyway, forget it. */
3751 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3752 if (!pdev)
3753 return;
3754 pci_dev_put(pdev);
3755
3756 /* System Management Registers. Might be hidden, in which case
3757 we can't do the sanity check. But that's OK, because the
3758 known-broken BIOSes _don't_ actually hide it, so far. */
3759 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3760 if (!pdev)
3761 return;
3762
3763 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
3764 pci_dev_put(pdev);
3765 return;
3766 }
3767
3768 pci_dev_put(pdev);
3769
3770 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
3771 if (vtisochctrl & 1)
3772 return;
3773
3774 /* Drop all bits other than the number of TLB entries */
3775 vtisochctrl &= 0x1c;
3776
3777 /* If we have the recommended number of TLB entries (16), fine. */
3778 if (vtisochctrl == 0x10)
3779 return;
3780
3781 /* Zero TLB entries? You get to ride the short bus to school. */
3782 if (!vtisochctrl) {
3783 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
3784 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3785 dmi_get_system_info(DMI_BIOS_VENDOR),
3786 dmi_get_system_info(DMI_BIOS_VERSION),
3787 dmi_get_system_info(DMI_PRODUCT_VERSION));
3788 iommu_identity_mapping |= IDENTMAP_AZALIA;
3789 return;
3790 }
3791
3792 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
3793 vtisochctrl);
3794 }
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree