?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge tag 'iommu-updates-v3.8' of git://git.kernel.org/pub/scm/linux/kernel/git/joro...
[linux-2.6.git] / drivers / iommu / intel-iommu.c
blobc2c07a4a7f21c5f57d9eca2884933dd99088b60a
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
22 */
23
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/export.h>
28 #include <linux/slab.h>
29 #include <linux/irq.h>
30 #include <linux/interrupt.h>
31 #include <linux/spinlock.h>
32 #include <linux/pci.h>
33 #include <linux/dmar.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/mempool.h>
36 #include <linux/timer.h>
37 #include <linux/iova.h>
38 #include <linux/iommu.h>
39 #include <linux/intel-iommu.h>
40 #include <linux/syscore_ops.h>
41 #include <linux/tboot.h>
42 #include <linux/dmi.h>
43 #include <linux/pci-ats.h>
44 #include <linux/memblock.h>
45 #include <asm/irq_remapping.h>
46 #include <asm/cacheflush.h>
47 #include <asm/iommu.h>
48
49 #define ROOT_SIZE VTD_PAGE_SIZE
50 #define CONTEXT_SIZE VTD_PAGE_SIZE
51
52 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
53 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
54 #define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
55
56 #define IOAPIC_RANGE_START (0xfee00000)
57 #define IOAPIC_RANGE_END (0xfeefffff)
58 #define IOVA_START_ADDR (0x1000)
59
60 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
61
62 #define MAX_AGAW_WIDTH 64
63
64 #define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
65 #define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
66
67 /* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
68 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
69 #define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
70 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
71 #define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
72
73 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
74 #define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
75 #define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
76
77 /* page table handling */
78 #define LEVEL_STRIDE (9)
79 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
80
81 /*
82 * This bitmap is used to advertise the page sizes our hardware support
83 * to the IOMMU core, which will then use this information to split
84 * physically contiguous memory regions it is mapping into page sizes
85 * that we support.
86 *
87 * Traditionally the IOMMU core just handed us the mappings directly,
88 * after making sure the size is an order of a 4KiB page and that the
89 * mapping has natural alignment.
90 *
91 * To retain this behavior, we currently advertise that we support
92 * all page sizes that are an order of 4KiB.
93 *
94 * If at some point we'd like to utilize the IOMMU core's new behavior,
95 * we could change this to advertise the real page sizes we support.
96 */
97 #define INTEL_IOMMU_PGSIZES (~0xFFFUL)
98
99 static inline int agaw_to_level(int agaw)
100 {
101 return agaw + 2;
102 }
103
104 static inline int agaw_to_width(int agaw)
105 {
106 return 30 + agaw * LEVEL_STRIDE;
107 }
108
109 static inline int width_to_agaw(int width)
110 {
111 return (width - 30) / LEVEL_STRIDE;
112 }
113
114 static inline unsigned int level_to_offset_bits(int level)
115 {
116 return (level - 1) * LEVEL_STRIDE;
117 }
118
119 static inline int pfn_level_offset(unsigned long pfn, int level)
120 {
121 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
122 }
123
124 static inline unsigned long level_mask(int level)
125 {
126 return -1UL << level_to_offset_bits(level);
127 }
128
129 static inline unsigned long level_size(int level)
130 {
131 return 1UL << level_to_offset_bits(level);
132 }
133
134 static inline unsigned long align_to_level(unsigned long pfn, int level)
135 {
136 return (pfn + level_size(level) - 1) & level_mask(level);
137 }
138
139 static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
140 {
141 return 1 << ((lvl - 1) * LEVEL_STRIDE);
142 }
143
144 /* VT-d pages must always be _smaller_ than MM pages. Otherwise things
145 are never going to work. */
146 static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
147 {
148 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
149 }
150
151 static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
152 {
153 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
154 }
155 static inline unsigned long page_to_dma_pfn(struct page *pg)
156 {
157 return mm_to_dma_pfn(page_to_pfn(pg));
158 }
159 static inline unsigned long virt_to_dma_pfn(void *p)
160 {
161 return page_to_dma_pfn(virt_to_page(p));
162 }
163
164 /* global iommu list, set NULL for ignored DMAR units */
165 static struct intel_iommu **g_iommus;
166
167 static void __init check_tylersburg_isoch(void);
168 static int rwbf_quirk;
169
170 /*
171 * set to 1 to panic kernel if can't successfully enable VT-d
172 * (used when kernel is launched w/ TXT)
173 */
174 static int force_on = 0;
175
176 /*
177 * 0: Present
178 * 1-11: Reserved
179 * 12-63: Context Ptr (12 - (haw-1))
180 * 64-127: Reserved
181 */
182 struct root_entry {
183 u64 val;
184 u64 rsvd1;
185 };
186 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
187 static inline bool root_present(struct root_entry *root)
188 {
189 return (root->val & 1);
190 }
191 static inline void set_root_present(struct root_entry *root)
192 {
193 root->val |= 1;
194 }
195 static inline void set_root_value(struct root_entry *root, unsigned long value)
196 {
197 root->val |= value & VTD_PAGE_MASK;
198 }
199
200 static inline struct context_entry *
201 get_context_addr_from_root(struct root_entry *root)
202 {
203 return (struct context_entry *)
204 (root_present(root)?phys_to_virt(
205 root->val & VTD_PAGE_MASK) :
206 NULL);
207 }
208
209 /*
210 * low 64 bits:
211 * 0: present
212 * 1: fault processing disable
213 * 2-3: translation type
214 * 12-63: address space root
215 * high 64 bits:
216 * 0-2: address width
217 * 3-6: aval
218 * 8-23: domain id
219 */
220 struct context_entry {
221 u64 lo;
222 u64 hi;
223 };
224
225 static inline bool context_present(struct context_entry *context)
226 {
227 return (context->lo & 1);
228 }
229 static inline void context_set_present(struct context_entry *context)
230 {
231 context->lo |= 1;
232 }
233
234 static inline void context_set_fault_enable(struct context_entry *context)
235 {
236 context->lo &= (((u64)-1) << 2) | 1;
237 }
238
239 static inline void context_set_translation_type(struct context_entry *context,
240 unsigned long value)
241 {
242 context->lo &= (((u64)-1) << 4) | 3;
243 context->lo |= (value & 3) << 2;
244 }
245
246 static inline void context_set_address_root(struct context_entry *context,
247 unsigned long value)
248 {
249 context->lo |= value & VTD_PAGE_MASK;
250 }
251
252 static inline void context_set_address_width(struct context_entry *context,
253 unsigned long value)
254 {
255 context->hi |= value & 7;
256 }
257
258 static inline void context_set_domain_id(struct context_entry *context,
259 unsigned long value)
260 {
261 context->hi |= (value & ((1 << 16) - 1)) << 8;
262 }
263
264 static inline void context_clear_entry(struct context_entry *context)
265 {
266 context->lo = 0;
267 context->hi = 0;
268 }
269
270 /*
271 * 0: readable
272 * 1: writable
273 * 2-6: reserved
274 * 7: super page
275 * 8-10: available
276 * 11: snoop behavior
277 * 12-63: Host physcial address
278 */
279 struct dma_pte {
280 u64 val;
281 };
282
283 static inline void dma_clear_pte(struct dma_pte *pte)
284 {
285 pte->val = 0;
286 }
287
288 static inline void dma_set_pte_readable(struct dma_pte *pte)
289 {
290 pte->val |= DMA_PTE_READ;
291 }
292
293 static inline void dma_set_pte_writable(struct dma_pte *pte)
294 {
295 pte->val |= DMA_PTE_WRITE;
296 }
297
298 static inline void dma_set_pte_snp(struct dma_pte *pte)
299 {
300 pte->val |= DMA_PTE_SNP;
301 }
302
303 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
304 {
305 pte->val = (pte->val & ~3) | (prot & 3);
306 }
307
308 static inline u64 dma_pte_addr(struct dma_pte *pte)
309 {
310 #ifdef CONFIG_64BIT
311 return pte->val & VTD_PAGE_MASK;
312 #else
313 /* Must have a full atomic 64-bit read */
314 return __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
315 #endif
316 }
317
318 static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
319 {
320 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
321 }
322
323 static inline bool dma_pte_present(struct dma_pte *pte)
324 {
325 return (pte->val & 3) != 0;
326 }
327
328 static inline bool dma_pte_superpage(struct dma_pte *pte)
329 {
330 return (pte->val & (1 << 7));
331 }
332
333 static inline int first_pte_in_page(struct dma_pte *pte)
334 {
335 return !((unsigned long)pte & ~VTD_PAGE_MASK);
336 }
337
338 /*
339 * This domain is a statically identity mapping domain.
340 * 1. This domain creats a static 1:1 mapping to all usable memory.
341 * 2. It maps to each iommu if successful.
342 * 3. Each iommu mapps to this domain if successful.
343 */
344 static struct dmar_domain *si_domain;
345 static int hw_pass_through = 1;
346
347 /* devices under the same p2p bridge are owned in one domain */
348 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
349
350 /* domain represents a virtual machine, more than one devices
351 * across iommus may be owned in one domain, e.g. kvm guest.
352 */
353 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
354
355 /* si_domain contains mulitple devices */
356 #define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
357
358 /* define the limit of IOMMUs supported in each domain */
359 #ifdef CONFIG_X86
360 # define IOMMU_UNITS_SUPPORTED MAX_IO_APICS
361 #else
362 # define IOMMU_UNITS_SUPPORTED 64
363 #endif
364
365 struct dmar_domain {
366 int id; /* domain id */
367 int nid; /* node id */
368 DECLARE_BITMAP(iommu_bmp, IOMMU_UNITS_SUPPORTED);
369 /* bitmap of iommus this domain uses*/
370
371 struct list_head devices; /* all devices' list */
372 struct iova_domain iovad; /* iova's that belong to this domain */
373
374 struct dma_pte *pgd; /* virtual address */
375 int gaw; /* max guest address width */
376
377 /* adjusted guest address width, 0 is level 2 30-bit */
378 int agaw;
379
380 int flags; /* flags to find out type of domain */
381
382 int iommu_coherency;/* indicate coherency of iommu access */
383 int iommu_snooping; /* indicate snooping control feature*/
384 int iommu_count; /* reference count of iommu */
385 int iommu_superpage;/* Level of superpages supported:
386 0 == 4KiB (no superpages), 1 == 2MiB,
387 2 == 1GiB, 3 == 512GiB, 4 == 1TiB */
388 spinlock_t iommu_lock; /* protect iommu set in domain */
389 u64 max_addr; /* maximum mapped address */
390 };
391
392 /* PCI domain-device relationship */
393 struct device_domain_info {
394 struct list_head link; /* link to domain siblings */
395 struct list_head global; /* link to global list */
396 int segment; /* PCI domain */
397 u8 bus; /* PCI bus number */
398 u8 devfn; /* PCI devfn number */
399 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
400 struct intel_iommu *iommu; /* IOMMU used by this device */
401 struct dmar_domain *domain; /* pointer to domain */
402 };
403
404 static void flush_unmaps_timeout(unsigned long data);
405
406 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
407
408 #define HIGH_WATER_MARK 250
409 struct deferred_flush_tables {
410 int next;
411 struct iova *iova[HIGH_WATER_MARK];
412 struct dmar_domain *domain[HIGH_WATER_MARK];
413 };
414
415 static struct deferred_flush_tables *deferred_flush;
416
417 /* bitmap for indexing intel_iommus */
418 static int g_num_of_iommus;
419
420 static DEFINE_SPINLOCK(async_umap_flush_lock);
421 static LIST_HEAD(unmaps_to_do);
422
423 static int timer_on;
424 static long list_size;
425
426 static void domain_remove_dev_info(struct dmar_domain *domain);
427
428 #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON
429 int dmar_disabled = 0;
430 #else
431 int dmar_disabled = 1;
432 #endif /*CONFIG_INTEL_IOMMU_DEFAULT_ON*/
433
434 int intel_iommu_enabled = 0;
435 EXPORT_SYMBOL_GPL(intel_iommu_enabled);
436
437 static int dmar_map_gfx = 1;
438 static int dmar_forcedac;
439 static int intel_iommu_strict;
440 static int intel_iommu_superpage = 1;
441
442 int intel_iommu_gfx_mapped;
443 EXPORT_SYMBOL_GPL(intel_iommu_gfx_mapped);
444
445 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
446 static DEFINE_SPINLOCK(device_domain_lock);
447 static LIST_HEAD(device_domain_list);
448
449 static struct iommu_ops intel_iommu_ops;
450
451 static int __init intel_iommu_setup(char *str)
452 {
453 if (!str)
454 return -EINVAL;
455 while (*str) {
456 if (!strncmp(str, "on", 2)) {
457 dmar_disabled = 0;
458 printk(KERN_INFO "Intel-IOMMU: enabled\n");
459 } else if (!strncmp(str, "off", 3)) {
460 dmar_disabled = 1;
461 printk(KERN_INFO "Intel-IOMMU: disabled\n");
462 } else if (!strncmp(str, "igfx_off", 8)) {
463 dmar_map_gfx = 0;
464 printk(KERN_INFO
465 "Intel-IOMMU: disable GFX device mapping\n");
466 } else if (!strncmp(str, "forcedac", 8)) {
467 printk(KERN_INFO
468 "Intel-IOMMU: Forcing DAC for PCI devices\n");
469 dmar_forcedac = 1;
470 } else if (!strncmp(str, "strict", 6)) {
471 printk(KERN_INFO
472 "Intel-IOMMU: disable batched IOTLB flush\n");
473 intel_iommu_strict = 1;
474 } else if (!strncmp(str, "sp_off", 6)) {
475 printk(KERN_INFO
476 "Intel-IOMMU: disable supported super page\n");
477 intel_iommu_superpage = 0;
478 }
479
480 str += strcspn(str, ",");
481 while (*str == ',')
482 str++;
483 }
484 return 0;
485 }
486 __setup("intel_iommu=", intel_iommu_setup);
487
488 static struct kmem_cache *iommu_domain_cache;
489 static struct kmem_cache *iommu_devinfo_cache;
490 static struct kmem_cache *iommu_iova_cache;
491
492 static inline void *alloc_pgtable_page(int node)
493 {
494 struct page *page;
495 void *vaddr = NULL;
496
497 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
498 if (page)
499 vaddr = page_address(page);
500 return vaddr;
501 }
502
503 static inline void free_pgtable_page(void *vaddr)
504 {
505 free_page((unsigned long)vaddr);
506 }
507
508 static inline void *alloc_domain_mem(void)
509 {
510 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
511 }
512
513 static void free_domain_mem(void *vaddr)
514 {
515 kmem_cache_free(iommu_domain_cache, vaddr);
516 }
517
518 static inline void * alloc_devinfo_mem(void)
519 {
520 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
521 }
522
523 static inline void free_devinfo_mem(void *vaddr)
524 {
525 kmem_cache_free(iommu_devinfo_cache, vaddr);
526 }
527
528 struct iova *alloc_iova_mem(void)
529 {
530 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
531 }
532
533 void free_iova_mem(struct iova *iova)
534 {
535 kmem_cache_free(iommu_iova_cache, iova);
536 }
537
538
539 static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
540 {
541 unsigned long sagaw;
542 int agaw = -1;
543
544 sagaw = cap_sagaw(iommu->cap);
545 for (agaw = width_to_agaw(max_gaw);
546 agaw >= 0; agaw--) {
547 if (test_bit(agaw, &sagaw))
548 break;
549 }
550
551 return agaw;
552 }
553
554 /*
555 * Calculate max SAGAW for each iommu.
556 */
557 int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
558 {
559 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
560 }
561
562 /*
563 * calculate agaw for each iommu.
564 * "SAGAW" may be different across iommus, use a default agaw, and
565 * get a supported less agaw for iommus that don't support the default agaw.
566 */
567 int iommu_calculate_agaw(struct intel_iommu *iommu)
568 {
569 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
570 }
571
572 /* This functionin only returns single iommu in a domain */
573 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
574 {
575 int iommu_id;
576
577 /* si_domain and vm domain should not get here. */
578 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
579 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
580
581 iommu_id = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
582 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
583 return NULL;
584
585 return g_iommus[iommu_id];
586 }
587
588 static void domain_update_iommu_coherency(struct dmar_domain *domain)
589 {
590 int i;
591
592 i = find_first_bit(domain->iommu_bmp, g_num_of_iommus);
593
594 domain->iommu_coherency = i < g_num_of_iommus ? 1 : 0;
595
596 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
597 if (!ecap_coherent(g_iommus[i]->ecap)) {
598 domain->iommu_coherency = 0;
599 break;
600 }
601 }
602 }
603
604 static void domain_update_iommu_snooping(struct dmar_domain *domain)
605 {
606 int i;
607
608 domain->iommu_snooping = 1;
609
610 for_each_set_bit(i, domain->iommu_bmp, g_num_of_iommus) {
611 if (!ecap_sc_support(g_iommus[i]->ecap)) {
612 domain->iommu_snooping = 0;
613 break;
614 }
615 }
616 }
617
618 static void domain_update_iommu_superpage(struct dmar_domain *domain)
619 {
620 struct dmar_drhd_unit *drhd;
621 struct intel_iommu *iommu = NULL;
622 int mask = 0xf;
623
624 if (!intel_iommu_superpage) {
625 domain->iommu_superpage = 0;
626 return;
627 }
628
629 /* set iommu_superpage to the smallest common denominator */
630 for_each_active_iommu(iommu, drhd) {
631 mask &= cap_super_page_val(iommu->cap);
632 if (!mask) {
633 break;
634 }
635 }
636 domain->iommu_superpage = fls(mask);
637 }
638
639 /* Some capabilities may be different across iommus */
640 static void domain_update_iommu_cap(struct dmar_domain *domain)
641 {
642 domain_update_iommu_coherency(domain);
643 domain_update_iommu_snooping(domain);
644 domain_update_iommu_superpage(domain);
645 }
646
647 static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
648 {
649 struct dmar_drhd_unit *drhd = NULL;
650 int i;
651
652 for_each_drhd_unit(drhd) {
653 if (drhd->ignored)
654 continue;
655 if (segment != drhd->segment)
656 continue;
657
658 for (i = 0; i < drhd->devices_cnt; i++) {
659 if (drhd->devices[i] &&
660 drhd->devices[i]->bus->number == bus &&
661 drhd->devices[i]->devfn == devfn)
662 return drhd->iommu;
663 if (drhd->devices[i] &&
664 drhd->devices[i]->subordinate &&
665 drhd->devices[i]->subordinate->number <= bus &&
666 drhd->devices[i]->subordinate->busn_res.end >= bus)
667 return drhd->iommu;
668 }
669
670 if (drhd->include_all)
671 return drhd->iommu;
672 }
673
674 return NULL;
675 }
676
677 static void domain_flush_cache(struct dmar_domain *domain,
678 void *addr, int size)
679 {
680 if (!domain->iommu_coherency)
681 clflush_cache_range(addr, size);
682 }
683
684 /* Gets context entry for a given bus and devfn */
685 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
686 u8 bus, u8 devfn)
687 {
688 struct root_entry *root;
689 struct context_entry *context;
690 unsigned long phy_addr;
691 unsigned long flags;
692
693 spin_lock_irqsave(&iommu->lock, flags);
694 root = &iommu->root_entry[bus];
695 context = get_context_addr_from_root(root);
696 if (!context) {
697 context = (struct context_entry *)
698 alloc_pgtable_page(iommu->node);
699 if (!context) {
700 spin_unlock_irqrestore(&iommu->lock, flags);
701 return NULL;
702 }
703 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
704 phy_addr = virt_to_phys((void *)context);
705 set_root_value(root, phy_addr);
706 set_root_present(root);
707 __iommu_flush_cache(iommu, root, sizeof(*root));
708 }
709 spin_unlock_irqrestore(&iommu->lock, flags);
710 return &context[devfn];
711 }
712
713 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
714 {
715 struct root_entry *root;
716 struct context_entry *context;
717 int ret;
718 unsigned long flags;
719
720 spin_lock_irqsave(&iommu->lock, flags);
721 root = &iommu->root_entry[bus];
722 context = get_context_addr_from_root(root);
723 if (!context) {
724 ret = 0;
725 goto out;
726 }
727 ret = context_present(&context[devfn]);
728 out:
729 spin_unlock_irqrestore(&iommu->lock, flags);
730 return ret;
731 }
732
733 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
734 {
735 struct root_entry *root;
736 struct context_entry *context;
737 unsigned long flags;
738
739 spin_lock_irqsave(&iommu->lock, flags);
740 root = &iommu->root_entry[bus];
741 context = get_context_addr_from_root(root);
742 if (context) {
743 context_clear_entry(&context[devfn]);
744 __iommu_flush_cache(iommu, &context[devfn], \
745 sizeof(*context));
746 }
747 spin_unlock_irqrestore(&iommu->lock, flags);
748 }
749
750 static void free_context_table(struct intel_iommu *iommu)
751 {
752 struct root_entry *root;
753 int i;
754 unsigned long flags;
755 struct context_entry *context;
756
757 spin_lock_irqsave(&iommu->lock, flags);
758 if (!iommu->root_entry) {
759 goto out;
760 }
761 for (i = 0; i < ROOT_ENTRY_NR; i++) {
762 root = &iommu->root_entry[i];
763 context = get_context_addr_from_root(root);
764 if (context)
765 free_pgtable_page(context);
766 }
767 free_pgtable_page(iommu->root_entry);
768 iommu->root_entry = NULL;
769 out:
770 spin_unlock_irqrestore(&iommu->lock, flags);
771 }
772
773 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
774 unsigned long pfn, int target_level)
775 {
776 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
777 struct dma_pte *parent, *pte = NULL;
778 int level = agaw_to_level(domain->agaw);
779 int offset;
780
781 BUG_ON(!domain->pgd);
782 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
783 parent = domain->pgd;
784
785 while (level > 0) {
786 void *tmp_page;
787
788 offset = pfn_level_offset(pfn, level);
789 pte = &parent[offset];
790 if (!target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
791 break;
792 if (level == target_level)
793 break;
794
795 if (!dma_pte_present(pte)) {
796 uint64_t pteval;
797
798 tmp_page = alloc_pgtable_page(domain->nid);
799
800 if (!tmp_page)
801 return NULL;
802
803 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
804 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
805 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
806 /* Someone else set it while we were thinking; use theirs. */
807 free_pgtable_page(tmp_page);
808 } else {
809 dma_pte_addr(pte);
810 domain_flush_cache(domain, pte, sizeof(*pte));
811 }
812 }
813 parent = phys_to_virt(dma_pte_addr(pte));
814 level--;
815 }
816
817 return pte;
818 }
819
820
821 /* return address's pte at specific level */
822 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
823 unsigned long pfn,
824 int level, int *large_page)
825 {
826 struct dma_pte *parent, *pte = NULL;
827 int total = agaw_to_level(domain->agaw);
828 int offset;
829
830 parent = domain->pgd;
831 while (level <= total) {
832 offset = pfn_level_offset(pfn, total);
833 pte = &parent[offset];
834 if (level == total)
835 return pte;
836
837 if (!dma_pte_present(pte)) {
838 *large_page = total;
839 break;
840 }
841
842 if (pte->val & DMA_PTE_LARGE_PAGE) {
843 *large_page = total;
844 return pte;
845 }
846
847 parent = phys_to_virt(dma_pte_addr(pte));
848 total--;
849 }
850 return NULL;
851 }
852
853 /* clear last level pte, a tlb flush should be followed */
854 static int dma_pte_clear_range(struct dmar_domain *domain,
855 unsigned long start_pfn,
856 unsigned long last_pfn)
857 {
858 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
859 unsigned int large_page = 1;
860 struct dma_pte *first_pte, *pte;
861 int order;
862
863 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
864 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
865 BUG_ON(start_pfn > last_pfn);
866
867 /* we don't need lock here; nobody else touches the iova range */
868 do {
869 large_page = 1;
870 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
871 if (!pte) {
872 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
873 continue;
874 }
875 do {
876 dma_clear_pte(pte);
877 start_pfn += lvl_to_nr_pages(large_page);
878 pte++;
879 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
880
881 domain_flush_cache(domain, first_pte,
882 (void *)pte - (void *)first_pte);
883
884 } while (start_pfn && start_pfn <= last_pfn);
885
886 order = (large_page - 1) * 9;
887 return order;
888 }
889
890 /* free page table pages. last level pte should already be cleared */
891 static void dma_pte_free_pagetable(struct dmar_domain *domain,
892 unsigned long start_pfn,
893 unsigned long last_pfn)
894 {
895 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
896 struct dma_pte *first_pte, *pte;
897 int total = agaw_to_level(domain->agaw);
898 int level;
899 unsigned long tmp;
900 int large_page = 2;
901
902 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
903 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
904 BUG_ON(start_pfn > last_pfn);
905
906 /* We don't need lock here; nobody else touches the iova range */
907 level = 2;
908 while (level <= total) {
909 tmp = align_to_level(start_pfn, level);
910
911 /* If we can't even clear one PTE at this level, we're done */
912 if (tmp + level_size(level) - 1 > last_pfn)
913 return;
914
915 do {
916 large_page = level;
917 first_pte = pte = dma_pfn_level_pte(domain, tmp, level, &large_page);
918 if (large_page > level)
919 level = large_page + 1;
920 if (!pte) {
921 tmp = align_to_level(tmp + 1, level + 1);
922 continue;
923 }
924 do {
925 if (dma_pte_present(pte)) {
926 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
927 dma_clear_pte(pte);
928 }
929 pte++;
930 tmp += level_size(level);
931 } while (!first_pte_in_page(pte) &&
932 tmp + level_size(level) - 1 <= last_pfn);
933
934 domain_flush_cache(domain, first_pte,
935 (void *)pte - (void *)first_pte);
936
937 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
938 level++;
939 }
940 /* free pgd */
941 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
942 free_pgtable_page(domain->pgd);
943 domain->pgd = NULL;
944 }
945 }
946
947 /* iommu handling */
948 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
949 {
950 struct root_entry *root;
951 unsigned long flags;
952
953 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
954 if (!root)
955 return -ENOMEM;
956
957 __iommu_flush_cache(iommu, root, ROOT_SIZE);
958
959 spin_lock_irqsave(&iommu->lock, flags);
960 iommu->root_entry = root;
961 spin_unlock_irqrestore(&iommu->lock, flags);
962
963 return 0;
964 }
965
966 static void iommu_set_root_entry(struct intel_iommu *iommu)
967 {
968 void *addr;
969 u32 sts;
970 unsigned long flag;
971
972 addr = iommu->root_entry;
973
974 raw_spin_lock_irqsave(&iommu->register_lock, flag);
975 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
976
977 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
978
979 /* Make sure hardware complete it */
980 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
981 readl, (sts & DMA_GSTS_RTPS), sts);
982
983 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
984 }
985
986 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
987 {
988 u32 val;
989 unsigned long flag;
990
991 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
992 return;
993
994 raw_spin_lock_irqsave(&iommu->register_lock, flag);
995 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
996
997 /* Make sure hardware complete it */
998 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
999 readl, (!(val & DMA_GSTS_WBFS)), val);
1000
1001 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1002 }
1003
1004 /* return value determine if we need a write buffer flush */
1005 static void __iommu_flush_context(struct intel_iommu *iommu,
1006 u16 did, u16 source_id, u8 function_mask,
1007 u64 type)
1008 {
1009 u64 val = 0;
1010 unsigned long flag;
1011
1012 switch (type) {
1013 case DMA_CCMD_GLOBAL_INVL:
1014 val = DMA_CCMD_GLOBAL_INVL;
1015 break;
1016 case DMA_CCMD_DOMAIN_INVL:
1017 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1018 break;
1019 case DMA_CCMD_DEVICE_INVL:
1020 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1021 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1022 break;
1023 default:
1024 BUG();
1025 }
1026 val |= DMA_CCMD_ICC;
1027
1028 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1029 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1030
1031 /* Make sure hardware complete it */
1032 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1033 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1034
1035 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1036 }
1037
1038 /* return value determine if we need a write buffer flush */
1039 static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1040 u64 addr, unsigned int size_order, u64 type)
1041 {
1042 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1043 u64 val = 0, val_iva = 0;
1044 unsigned long flag;
1045
1046 switch (type) {
1047 case DMA_TLB_GLOBAL_FLUSH:
1048 /* global flush doesn't need set IVA_REG */
1049 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1050 break;
1051 case DMA_TLB_DSI_FLUSH:
1052 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1053 break;
1054 case DMA_TLB_PSI_FLUSH:
1055 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1056 /* Note: always flush non-leaf currently */
1057 val_iva = size_order | addr;
1058 break;
1059 default:
1060 BUG();
1061 }
1062 /* Note: set drain read/write */
1063 #if 0
1064 /*
1065 * This is probably to be super secure.. Looks like we can
1066 * ignore it without any impact.
1067 */
1068 if (cap_read_drain(iommu->cap))
1069 val |= DMA_TLB_READ_DRAIN;
1070 #endif
1071 if (cap_write_drain(iommu->cap))
1072 val |= DMA_TLB_WRITE_DRAIN;
1073
1074 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1075 /* Note: Only uses first TLB reg currently */
1076 if (val_iva)
1077 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1078 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1079
1080 /* Make sure hardware complete it */
1081 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1082 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1083
1084 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1085
1086 /* check IOTLB invalidation granularity */
1087 if (DMA_TLB_IAIG(val) == 0)
1088 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
1089 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1090 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
1091 (unsigned long long)DMA_TLB_IIRG(type),
1092 (unsigned long long)DMA_TLB_IAIG(val));
1093 }
1094
1095 static struct device_domain_info *iommu_support_dev_iotlb(
1096 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
1097 {
1098 int found = 0;
1099 unsigned long flags;
1100 struct device_domain_info *info;
1101 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1102
1103 if (!ecap_dev_iotlb_support(iommu->ecap))
1104 return NULL;
1105
1106 if (!iommu->qi)
1107 return NULL;
1108
1109 spin_lock_irqsave(&device_domain_lock, flags);
1110 list_for_each_entry(info, &domain->devices, link)
1111 if (info->bus == bus && info->devfn == devfn) {
1112 found = 1;
1113 break;
1114 }
1115 spin_unlock_irqrestore(&device_domain_lock, flags);
1116
1117 if (!found || !info->dev)
1118 return NULL;
1119
1120 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1121 return NULL;
1122
1123 if (!dmar_find_matched_atsr_unit(info->dev))
1124 return NULL;
1125
1126 info->iommu = iommu;
1127
1128 return info;
1129 }
1130
1131 static void iommu_enable_dev_iotlb(struct device_domain_info *info)
1132 {
1133 if (!info)
1134 return;
1135
1136 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1137 }
1138
1139 static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1140 {
1141 if (!info->dev || !pci_ats_enabled(info->dev))
1142 return;
1143
1144 pci_disable_ats(info->dev);
1145 }
1146
1147 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1148 u64 addr, unsigned mask)
1149 {
1150 u16 sid, qdep;
1151 unsigned long flags;
1152 struct device_domain_info *info;
1153
1154 spin_lock_irqsave(&device_domain_lock, flags);
1155 list_for_each_entry(info, &domain->devices, link) {
1156 if (!info->dev || !pci_ats_enabled(info->dev))
1157 continue;
1158
1159 sid = info->bus << 8 | info->devfn;
1160 qdep = pci_ats_queue_depth(info->dev);
1161 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1162 }
1163 spin_unlock_irqrestore(&device_domain_lock, flags);
1164 }
1165
1166 static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
1167 unsigned long pfn, unsigned int pages, int map)
1168 {
1169 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
1170 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1171
1172 BUG_ON(pages == 0);
1173
1174 /*
1175 * Fallback to domain selective flush if no PSI support or the size is
1176 * too big.
1177 * PSI requires page size to be 2 ^ x, and the base address is naturally
1178 * aligned to the size
1179 */
1180 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1181 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1182 DMA_TLB_DSI_FLUSH);
1183 else
1184 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1185 DMA_TLB_PSI_FLUSH);
1186
1187 /*
1188 * In caching mode, changes of pages from non-present to present require
1189 * flush. However, device IOTLB doesn't need to be flushed in this case.
1190 */
1191 if (!cap_caching_mode(iommu->cap) || !map)
1192 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
1193 }
1194
1195 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1196 {
1197 u32 pmen;
1198 unsigned long flags;
1199
1200 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1201 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1202 pmen &= ~DMA_PMEN_EPM;
1203 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1204
1205 /* wait for the protected region status bit to clear */
1206 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1207 readl, !(pmen & DMA_PMEN_PRS), pmen);
1208
1209 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1210 }
1211
1212 static int iommu_enable_translation(struct intel_iommu *iommu)
1213 {
1214 u32 sts;
1215 unsigned long flags;
1216
1217 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1218 iommu->gcmd |= DMA_GCMD_TE;
1219 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1220
1221 /* Make sure hardware complete it */
1222 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1223 readl, (sts & DMA_GSTS_TES), sts);
1224
1225 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1226 return 0;
1227 }
1228
1229 static int iommu_disable_translation(struct intel_iommu *iommu)
1230 {
1231 u32 sts;
1232 unsigned long flag;
1233
1234 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1235 iommu->gcmd &= ~DMA_GCMD_TE;
1236 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1237
1238 /* Make sure hardware complete it */
1239 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1240 readl, (!(sts & DMA_GSTS_TES)), sts);
1241
1242 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1243 return 0;
1244 }
1245
1246
1247 static int iommu_init_domains(struct intel_iommu *iommu)
1248 {
1249 unsigned long ndomains;
1250 unsigned long nlongs;
1251
1252 ndomains = cap_ndoms(iommu->cap);
1253 pr_debug("IOMMU %d: Number of Domains supported <%ld>\n", iommu->seq_id,
1254 ndomains);
1255 nlongs = BITS_TO_LONGS(ndomains);
1256
1257 spin_lock_init(&iommu->lock);
1258
1259 /* TBD: there might be 64K domains,
1260 * consider other allocation for future chip
1261 */
1262 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1263 if (!iommu->domain_ids) {
1264 printk(KERN_ERR "Allocating domain id array failed\n");
1265 return -ENOMEM;
1266 }
1267 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1268 GFP_KERNEL);
1269 if (!iommu->domains) {
1270 printk(KERN_ERR "Allocating domain array failed\n");
1271 return -ENOMEM;
1272 }
1273
1274 /*
1275 * if Caching mode is set, then invalid translations are tagged
1276 * with domainid 0. Hence we need to pre-allocate it.
1277 */
1278 if (cap_caching_mode(iommu->cap))
1279 set_bit(0, iommu->domain_ids);
1280 return 0;
1281 }
1282
1283
1284 static void domain_exit(struct dmar_domain *domain);
1285 static void vm_domain_exit(struct dmar_domain *domain);
1286
1287 void free_dmar_iommu(struct intel_iommu *iommu)
1288 {
1289 struct dmar_domain *domain;
1290 int i;
1291 unsigned long flags;
1292
1293 if ((iommu->domains) && (iommu->domain_ids)) {
1294 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
1295 domain = iommu->domains[i];
1296 clear_bit(i, iommu->domain_ids);
1297
1298 spin_lock_irqsave(&domain->iommu_lock, flags);
1299 if (--domain->iommu_count == 0) {
1300 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1301 vm_domain_exit(domain);
1302 else
1303 domain_exit(domain);
1304 }
1305 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1306 }
1307 }
1308
1309 if (iommu->gcmd & DMA_GCMD_TE)
1310 iommu_disable_translation(iommu);
1311
1312 if (iommu->irq) {
1313 irq_set_handler_data(iommu->irq, NULL);
1314 /* This will mask the irq */
1315 free_irq(iommu->irq, iommu);
1316 destroy_irq(iommu->irq);
1317 }
1318
1319 kfree(iommu->domains);
1320 kfree(iommu->domain_ids);
1321
1322 g_iommus[iommu->seq_id] = NULL;
1323
1324 /* if all iommus are freed, free g_iommus */
1325 for (i = 0; i < g_num_of_iommus; i++) {
1326 if (g_iommus[i])
1327 break;
1328 }
1329
1330 if (i == g_num_of_iommus)
1331 kfree(g_iommus);
1332
1333 /* free context mapping */
1334 free_context_table(iommu);
1335 }
1336
1337 static struct dmar_domain *alloc_domain(void)
1338 {
1339 struct dmar_domain *domain;
1340
1341 domain = alloc_domain_mem();
1342 if (!domain)
1343 return NULL;
1344
1345 domain->nid = -1;
1346 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
1347 domain->flags = 0;
1348
1349 return domain;
1350 }
1351
1352 static int iommu_attach_domain(struct dmar_domain *domain,
1353 struct intel_iommu *iommu)
1354 {
1355 int num;
1356 unsigned long ndomains;
1357 unsigned long flags;
1358
1359 ndomains = cap_ndoms(iommu->cap);
1360
1361 spin_lock_irqsave(&iommu->lock, flags);
1362
1363 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1364 if (num >= ndomains) {
1365 spin_unlock_irqrestore(&iommu->lock, flags);
1366 printk(KERN_ERR "IOMMU: no free domain ids\n");
1367 return -ENOMEM;
1368 }
1369
1370 domain->id = num;
1371 set_bit(num, iommu->domain_ids);
1372 set_bit(iommu->seq_id, domain->iommu_bmp);
1373 iommu->domains[num] = domain;
1374 spin_unlock_irqrestore(&iommu->lock, flags);
1375
1376 return 0;
1377 }
1378
1379 static void iommu_detach_domain(struct dmar_domain *domain,
1380 struct intel_iommu *iommu)
1381 {
1382 unsigned long flags;
1383 int num, ndomains;
1384 int found = 0;
1385
1386 spin_lock_irqsave(&iommu->lock, flags);
1387 ndomains = cap_ndoms(iommu->cap);
1388 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1389 if (iommu->domains[num] == domain) {
1390 found = 1;
1391 break;
1392 }
1393 }
1394
1395 if (found) {
1396 clear_bit(num, iommu->domain_ids);
1397 clear_bit(iommu->seq_id, domain->iommu_bmp);
1398 iommu->domains[num] = NULL;
1399 }
1400 spin_unlock_irqrestore(&iommu->lock, flags);
1401 }
1402
1403 static struct iova_domain reserved_iova_list;
1404 static struct lock_class_key reserved_rbtree_key;
1405
1406 static int dmar_init_reserved_ranges(void)
1407 {
1408 struct pci_dev *pdev = NULL;
1409 struct iova *iova;
1410 int i;
1411
1412 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1413
1414 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1415 &reserved_rbtree_key);
1416
1417 /* IOAPIC ranges shouldn't be accessed by DMA */
1418 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1419 IOVA_PFN(IOAPIC_RANGE_END));
1420 if (!iova) {
1421 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1422 return -ENODEV;
1423 }
1424
1425 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1426 for_each_pci_dev(pdev) {
1427 struct resource *r;
1428
1429 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1430 r = &pdev->resource[i];
1431 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1432 continue;
1433 iova = reserve_iova(&reserved_iova_list,
1434 IOVA_PFN(r->start),
1435 IOVA_PFN(r->end));
1436 if (!iova) {
1437 printk(KERN_ERR "Reserve iova failed\n");
1438 return -ENODEV;
1439 }
1440 }
1441 }
1442 return 0;
1443 }
1444
1445 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1446 {
1447 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1448 }
1449
1450 static inline int guestwidth_to_adjustwidth(int gaw)
1451 {
1452 int agaw;
1453 int r = (gaw - 12) % 9;
1454
1455 if (r == 0)
1456 agaw = gaw;
1457 else
1458 agaw = gaw + 9 - r;
1459 if (agaw > 64)
1460 agaw = 64;
1461 return agaw;
1462 }
1463
1464 static int domain_init(struct dmar_domain *domain, int guest_width)
1465 {
1466 struct intel_iommu *iommu;
1467 int adjust_width, agaw;
1468 unsigned long sagaw;
1469
1470 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1471 spin_lock_init(&domain->iommu_lock);
1472
1473 domain_reserve_special_ranges(domain);
1474
1475 /* calculate AGAW */
1476 iommu = domain_get_iommu(domain);
1477 if (guest_width > cap_mgaw(iommu->cap))
1478 guest_width = cap_mgaw(iommu->cap);
1479 domain->gaw = guest_width;
1480 adjust_width = guestwidth_to_adjustwidth(guest_width);
1481 agaw = width_to_agaw(adjust_width);
1482 sagaw = cap_sagaw(iommu->cap);
1483 if (!test_bit(agaw, &sagaw)) {
1484 /* hardware doesn't support it, choose a bigger one */
1485 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1486 agaw = find_next_bit(&sagaw, 5, agaw);
1487 if (agaw >= 5)
1488 return -ENODEV;
1489 }
1490 domain->agaw = agaw;
1491 INIT_LIST_HEAD(&domain->devices);
1492
1493 if (ecap_coherent(iommu->ecap))
1494 domain->iommu_coherency = 1;
1495 else
1496 domain->iommu_coherency = 0;
1497
1498 if (ecap_sc_support(iommu->ecap))
1499 domain->iommu_snooping = 1;
1500 else
1501 domain->iommu_snooping = 0;
1502
1503 domain->iommu_superpage = fls(cap_super_page_val(iommu->cap));
1504 domain->iommu_count = 1;
1505 domain->nid = iommu->node;
1506
1507 /* always allocate the top pgd */
1508 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
1509 if (!domain->pgd)
1510 return -ENOMEM;
1511 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1512 return 0;
1513 }
1514
1515 static void domain_exit(struct dmar_domain *domain)
1516 {
1517 struct dmar_drhd_unit *drhd;
1518 struct intel_iommu *iommu;
1519
1520 /* Domain 0 is reserved, so dont process it */
1521 if (!domain)
1522 return;
1523
1524 /* Flush any lazy unmaps that may reference this domain */
1525 if (!intel_iommu_strict)
1526 flush_unmaps_timeout(0);
1527
1528 domain_remove_dev_info(domain);
1529 /* destroy iovas */
1530 put_iova_domain(&domain->iovad);
1531
1532 /* clear ptes */
1533 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1534
1535 /* free page tables */
1536 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
1537
1538 for_each_active_iommu(iommu, drhd)
1539 if (test_bit(iommu->seq_id, domain->iommu_bmp))
1540 iommu_detach_domain(domain, iommu);
1541
1542 free_domain_mem(domain);
1543 }
1544
1545 static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1546 u8 bus, u8 devfn, int translation)
1547 {
1548 struct context_entry *context;
1549 unsigned long flags;
1550 struct intel_iommu *iommu;
1551 struct dma_pte *pgd;
1552 unsigned long num;
1553 unsigned long ndomains;
1554 int id;
1555 int agaw;
1556 struct device_domain_info *info = NULL;
1557
1558 pr_debug("Set context mapping for %02x:%02x.%d\n",
1559 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1560
1561 BUG_ON(!domain->pgd);
1562 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1563 translation != CONTEXT_TT_MULTI_LEVEL);
1564
1565 iommu = device_to_iommu(segment, bus, devfn);
1566 if (!iommu)
1567 return -ENODEV;
1568
1569 context = device_to_context_entry(iommu, bus, devfn);
1570 if (!context)
1571 return -ENOMEM;
1572 spin_lock_irqsave(&iommu->lock, flags);
1573 if (context_present(context)) {
1574 spin_unlock_irqrestore(&iommu->lock, flags);
1575 return 0;
1576 }
1577
1578 id = domain->id;
1579 pgd = domain->pgd;
1580
1581 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1582 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
1583 int found = 0;
1584
1585 /* find an available domain id for this device in iommu */
1586 ndomains = cap_ndoms(iommu->cap);
1587 for_each_set_bit(num, iommu->domain_ids, ndomains) {
1588 if (iommu->domains[num] == domain) {
1589 id = num;
1590 found = 1;
1591 break;
1592 }
1593 }
1594
1595 if (found == 0) {
1596 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1597 if (num >= ndomains) {
1598 spin_unlock_irqrestore(&iommu->lock, flags);
1599 printk(KERN_ERR "IOMMU: no free domain ids\n");
1600 return -EFAULT;
1601 }
1602
1603 set_bit(num, iommu->domain_ids);
1604 iommu->domains[num] = domain;
1605 id = num;
1606 }
1607
1608 /* Skip top levels of page tables for
1609 * iommu which has less agaw than default.
1610 * Unnecessary for PT mode.
1611 */
1612 if (translation != CONTEXT_TT_PASS_THROUGH) {
1613 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1614 pgd = phys_to_virt(dma_pte_addr(pgd));
1615 if (!dma_pte_present(pgd)) {
1616 spin_unlock_irqrestore(&iommu->lock, flags);
1617 return -ENOMEM;
1618 }
1619 }
1620 }
1621 }
1622
1623 context_set_domain_id(context, id);
1624
1625 if (translation != CONTEXT_TT_PASS_THROUGH) {
1626 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1627 translation = info ? CONTEXT_TT_DEV_IOTLB :
1628 CONTEXT_TT_MULTI_LEVEL;
1629 }
1630 /*
1631 * In pass through mode, AW must be programmed to indicate the largest
1632 * AGAW value supported by hardware. And ASR is ignored by hardware.
1633 */
1634 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
1635 context_set_address_width(context, iommu->msagaw);
1636 else {
1637 context_set_address_root(context, virt_to_phys(pgd));
1638 context_set_address_width(context, iommu->agaw);
1639 }
1640
1641 context_set_translation_type(context, translation);
1642 context_set_fault_enable(context);
1643 context_set_present(context);
1644 domain_flush_cache(domain, context, sizeof(*context));
1645
1646 /*
1647 * It's a non-present to present mapping. If hardware doesn't cache
1648 * non-present entry we only need to flush the write-buffer. If the
1649 * _does_ cache non-present entries, then it does so in the special
1650 * domain #0, which we have to flush:
1651 */
1652 if (cap_caching_mode(iommu->cap)) {
1653 iommu->flush.flush_context(iommu, 0,
1654 (((u16)bus) << 8) | devfn,
1655 DMA_CCMD_MASK_NOBIT,
1656 DMA_CCMD_DEVICE_INVL);
1657 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
1658 } else {
1659 iommu_flush_write_buffer(iommu);
1660 }
1661 iommu_enable_dev_iotlb(info);
1662 spin_unlock_irqrestore(&iommu->lock, flags);
1663
1664 spin_lock_irqsave(&domain->iommu_lock, flags);
1665 if (!test_and_set_bit(iommu->seq_id, domain->iommu_bmp)) {
1666 domain->iommu_count++;
1667 if (domain->iommu_count == 1)
1668 domain->nid = iommu->node;
1669 domain_update_iommu_cap(domain);
1670 }
1671 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1672 return 0;
1673 }
1674
1675 static int
1676 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1677 int translation)
1678 {
1679 int ret;
1680 struct pci_dev *tmp, *parent;
1681
1682 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
1683 pdev->bus->number, pdev->devfn,
1684 translation);
1685 if (ret)
1686 return ret;
1687
1688 /* dependent device mapping */
1689 tmp = pci_find_upstream_pcie_bridge(pdev);
1690 if (!tmp)
1691 return 0;
1692 /* Secondary interface's bus number and devfn 0 */
1693 parent = pdev->bus->self;
1694 while (parent != tmp) {
1695 ret = domain_context_mapping_one(domain,
1696 pci_domain_nr(parent->bus),
1697 parent->bus->number,
1698 parent->devfn, translation);
1699 if (ret)
1700 return ret;
1701 parent = parent->bus->self;
1702 }
1703 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
1704 return domain_context_mapping_one(domain,
1705 pci_domain_nr(tmp->subordinate),
1706 tmp->subordinate->number, 0,
1707 translation);
1708 else /* this is a legacy PCI bridge */
1709 return domain_context_mapping_one(domain,
1710 pci_domain_nr(tmp->bus),
1711 tmp->bus->number,
1712 tmp->devfn,
1713 translation);
1714 }
1715
1716 static int domain_context_mapped(struct pci_dev *pdev)
1717 {
1718 int ret;
1719 struct pci_dev *tmp, *parent;
1720 struct intel_iommu *iommu;
1721
1722 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1723 pdev->devfn);
1724 if (!iommu)
1725 return -ENODEV;
1726
1727 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
1728 if (!ret)
1729 return ret;
1730 /* dependent device mapping */
1731 tmp = pci_find_upstream_pcie_bridge(pdev);
1732 if (!tmp)
1733 return ret;
1734 /* Secondary interface's bus number and devfn 0 */
1735 parent = pdev->bus->self;
1736 while (parent != tmp) {
1737 ret = device_context_mapped(iommu, parent->bus->number,
1738 parent->devfn);
1739 if (!ret)
1740 return ret;
1741 parent = parent->bus->self;
1742 }
1743 if (pci_is_pcie(tmp))
1744 return device_context_mapped(iommu, tmp->subordinate->number,
1745 0);
1746 else
1747 return device_context_mapped(iommu, tmp->bus->number,
1748 tmp->devfn);
1749 }
1750
1751 /* Returns a number of VTD pages, but aligned to MM page size */
1752 static inline unsigned long aligned_nrpages(unsigned long host_addr,
1753 size_t size)
1754 {
1755 host_addr &= ~PAGE_MASK;
1756 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1757 }
1758
1759 /* Return largest possible superpage level for a given mapping */
1760 static inline int hardware_largepage_caps(struct dmar_domain *domain,
1761 unsigned long iov_pfn,
1762 unsigned long phy_pfn,
1763 unsigned long pages)
1764 {
1765 int support, level = 1;
1766 unsigned long pfnmerge;
1767
1768 support = domain->iommu_superpage;
1769
1770 /* To use a large page, the virtual *and* physical addresses
1771 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1772 of them will mean we have to use smaller pages. So just
1773 merge them and check both at once. */
1774 pfnmerge = iov_pfn | phy_pfn;
1775
1776 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1777 pages >>= VTD_STRIDE_SHIFT;
1778 if (!pages)
1779 break;
1780 pfnmerge >>= VTD_STRIDE_SHIFT;
1781 level++;
1782 support--;
1783 }
1784 return level;
1785 }
1786
1787 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1788 struct scatterlist *sg, unsigned long phys_pfn,
1789 unsigned long nr_pages, int prot)
1790 {
1791 struct dma_pte *first_pte = NULL, *pte = NULL;
1792 phys_addr_t uninitialized_var(pteval);
1793 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
1794 unsigned long sg_res;
1795 unsigned int largepage_lvl = 0;
1796 unsigned long lvl_pages = 0;
1797
1798 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1799
1800 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1801 return -EINVAL;
1802
1803 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1804
1805 if (sg)
1806 sg_res = 0;
1807 else {
1808 sg_res = nr_pages + 1;
1809 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1810 }
1811
1812 while (nr_pages > 0) {
1813 uint64_t tmp;
1814
1815 if (!sg_res) {
1816 sg_res = aligned_nrpages(sg->offset, sg->length);
1817 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1818 sg->dma_length = sg->length;
1819 pteval = page_to_phys(sg_page(sg)) | prot;
1820 phys_pfn = pteval >> VTD_PAGE_SHIFT;
1821 }
1822
1823 if (!pte) {
1824 largepage_lvl = hardware_largepage_caps(domain, iov_pfn, phys_pfn, sg_res);
1825
1826 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn, largepage_lvl);
1827 if (!pte)
1828 return -ENOMEM;
1829 /* It is large page*/
1830 if (largepage_lvl > 1) {
1831 pteval |= DMA_PTE_LARGE_PAGE;
1832 /* Ensure that old small page tables are removed to make room
1833 for superpage, if they exist. */
1834 dma_pte_clear_range(domain, iov_pfn,
1835 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1836 dma_pte_free_pagetable(domain, iov_pfn,
1837 iov_pfn + lvl_to_nr_pages(largepage_lvl) - 1);
1838 } else {
1839 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
1840 }
1841
1842 }
1843 /* We don't need lock here, nobody else
1844 * touches the iova range
1845 */
1846 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
1847 if (tmp) {
1848 static int dumps = 5;
1849 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1850 iov_pfn, tmp, (unsigned long long)pteval);
1851 if (dumps) {
1852 dumps--;
1853 debug_dma_dump_mappings(NULL);
1854 }
1855 WARN_ON(1);
1856 }
1857
1858 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1859
1860 BUG_ON(nr_pages < lvl_pages);
1861 BUG_ON(sg_res < lvl_pages);
1862
1863 nr_pages -= lvl_pages;
1864 iov_pfn += lvl_pages;
1865 phys_pfn += lvl_pages;
1866 pteval += lvl_pages * VTD_PAGE_SIZE;
1867 sg_res -= lvl_pages;
1868
1869 /* If the next PTE would be the first in a new page, then we
1870 need to flush the cache on the entries we've just written.
1871 And then we'll need to recalculate 'pte', so clear it and
1872 let it get set again in the if (!pte) block above.
1873
1874 If we're done (!nr_pages) we need to flush the cache too.
1875
1876 Also if we've been setting superpages, we may need to
1877 recalculate 'pte' and switch back to smaller pages for the
1878 end of the mapping, if the trailing size is not enough to
1879 use another superpage (i.e. sg_res < lvl_pages). */
1880 pte++;
1881 if (!nr_pages || first_pte_in_page(pte) ||
1882 (largepage_lvl > 1 && sg_res < lvl_pages)) {
1883 domain_flush_cache(domain, first_pte,
1884 (void *)pte - (void *)first_pte);
1885 pte = NULL;
1886 }
1887
1888 if (!sg_res && nr_pages)
1889 sg = sg_next(sg);
1890 }
1891 return 0;
1892 }
1893
1894 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1895 struct scatterlist *sg, unsigned long nr_pages,
1896 int prot)
1897 {
1898 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1899 }
1900
1901 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1902 unsigned long phys_pfn, unsigned long nr_pages,
1903 int prot)
1904 {
1905 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
1906 }
1907
1908 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1909 {
1910 if (!iommu)
1911 return;
1912
1913 clear_context_table(iommu, bus, devfn);
1914 iommu->flush.flush_context(iommu, 0, 0, 0,
1915 DMA_CCMD_GLOBAL_INVL);
1916 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1917 }
1918
1919 static inline void unlink_domain_info(struct device_domain_info *info)
1920 {
1921 assert_spin_locked(&device_domain_lock);
1922 list_del(&info->link);
1923 list_del(&info->global);
1924 if (info->dev)
1925 info->dev->dev.archdata.iommu = NULL;
1926 }
1927
1928 static void domain_remove_dev_info(struct dmar_domain *domain)
1929 {
1930 struct device_domain_info *info;
1931 unsigned long flags;
1932 struct intel_iommu *iommu;
1933
1934 spin_lock_irqsave(&device_domain_lock, flags);
1935 while (!list_empty(&domain->devices)) {
1936 info = list_entry(domain->devices.next,
1937 struct device_domain_info, link);
1938 unlink_domain_info(info);
1939 spin_unlock_irqrestore(&device_domain_lock, flags);
1940
1941 iommu_disable_dev_iotlb(info);
1942 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
1943 iommu_detach_dev(iommu, info->bus, info->devfn);
1944 free_devinfo_mem(info);
1945
1946 spin_lock_irqsave(&device_domain_lock, flags);
1947 }
1948 spin_unlock_irqrestore(&device_domain_lock, flags);
1949 }
1950
1951 /*
1952 * find_domain
1953 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1954 */
1955 static struct dmar_domain *
1956 find_domain(struct pci_dev *pdev)
1957 {
1958 struct device_domain_info *info;
1959
1960 /* No lock here, assumes no domain exit in normal case */
1961 info = pdev->dev.archdata.iommu;
1962 if (info)
1963 return info->domain;
1964 return NULL;
1965 }
1966
1967 /* domain is initialized */
1968 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1969 {
1970 struct dmar_domain *domain, *found = NULL;
1971 struct intel_iommu *iommu;
1972 struct dmar_drhd_unit *drhd;
1973 struct device_domain_info *info, *tmp;
1974 struct pci_dev *dev_tmp;
1975 unsigned long flags;
1976 int bus = 0, devfn = 0;
1977 int segment;
1978 int ret;
1979
1980 domain = find_domain(pdev);
1981 if (domain)
1982 return domain;
1983
1984 segment = pci_domain_nr(pdev->bus);
1985
1986 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1987 if (dev_tmp) {
1988 if (pci_is_pcie(dev_tmp)) {
1989 bus = dev_tmp->subordinate->number;
1990 devfn = 0;
1991 } else {
1992 bus = dev_tmp->bus->number;
1993 devfn = dev_tmp->devfn;
1994 }
1995 spin_lock_irqsave(&device_domain_lock, flags);
1996 list_for_each_entry(info, &device_domain_list, global) {
1997 if (info->segment == segment &&
1998 info->bus == bus && info->devfn == devfn) {
1999 found = info->domain;
2000 break;
2001 }
2002 }
2003 spin_unlock_irqrestore(&device_domain_lock, flags);
2004 /* pcie-pci bridge already has a domain, uses it */
2005 if (found) {
2006 domain = found;
2007 goto found_domain;
2008 }
2009 }
2010
2011 domain = alloc_domain();
2012 if (!domain)
2013 goto error;
2014
2015 /* Allocate new domain for the device */
2016 drhd = dmar_find_matched_drhd_unit(pdev);
2017 if (!drhd) {
2018 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
2019 pci_name(pdev));
2020 free_domain_mem(domain);
2021 return NULL;
2022 }
2023 iommu = drhd->iommu;
2024
2025 ret = iommu_attach_domain(domain, iommu);
2026 if (ret) {
2027 free_domain_mem(domain);
2028 goto error;
2029 }
2030
2031 if (domain_init(domain, gaw)) {
2032 domain_exit(domain);
2033 goto error;
2034 }
2035
2036 /* register pcie-to-pci device */
2037 if (dev_tmp) {
2038 info = alloc_devinfo_mem();
2039 if (!info) {
2040 domain_exit(domain);
2041 goto error;
2042 }
2043 info->segment = segment;
2044 info->bus = bus;
2045 info->devfn = devfn;
2046 info->dev = NULL;
2047 info->domain = domain;
2048 /* This domain is shared by devices under p2p bridge */
2049 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
2050
2051 /* pcie-to-pci bridge already has a domain, uses it */
2052 found = NULL;
2053 spin_lock_irqsave(&device_domain_lock, flags);
2054 list_for_each_entry(tmp, &device_domain_list, global) {
2055 if (tmp->segment == segment &&
2056 tmp->bus == bus && tmp->devfn == devfn) {
2057 found = tmp->domain;
2058 break;
2059 }
2060 }
2061 if (found) {
2062 spin_unlock_irqrestore(&device_domain_lock, flags);
2063 free_devinfo_mem(info);
2064 domain_exit(domain);
2065 domain = found;
2066 } else {
2067 list_add(&info->link, &domain->devices);
2068 list_add(&info->global, &device_domain_list);
2069 spin_unlock_irqrestore(&device_domain_lock, flags);
2070 }
2071 }
2072
2073 found_domain:
2074 info = alloc_devinfo_mem();
2075 if (!info)
2076 goto error;
2077 info->segment = segment;
2078 info->bus = pdev->bus->number;
2079 info->devfn = pdev->devfn;
2080 info->dev = pdev;
2081 info->domain = domain;
2082 spin_lock_irqsave(&device_domain_lock, flags);
2083 /* somebody is fast */
2084 found = find_domain(pdev);
2085 if (found != NULL) {
2086 spin_unlock_irqrestore(&device_domain_lock, flags);
2087 if (found != domain) {
2088 domain_exit(domain);
2089 domain = found;
2090 }
2091 free_devinfo_mem(info);
2092 return domain;
2093 }
2094 list_add(&info->link, &domain->devices);
2095 list_add(&info->global, &device_domain_list);
2096 pdev->dev.archdata.iommu = info;
2097 spin_unlock_irqrestore(&device_domain_lock, flags);
2098 return domain;
2099 error:
2100 /* recheck it here, maybe others set it */
2101 return find_domain(pdev);
2102 }
2103
2104 static int iommu_identity_mapping;
2105 #define IDENTMAP_ALL 1
2106 #define IDENTMAP_GFX 2
2107 #define IDENTMAP_AZALIA 4
2108
2109 static int iommu_domain_identity_map(struct dmar_domain *domain,
2110 unsigned long long start,
2111 unsigned long long end)
2112 {
2113 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
2114 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
2115
2116 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
2117 dma_to_mm_pfn(last_vpfn))) {
2118 printk(KERN_ERR "IOMMU: reserve iova failed\n");
2119 return -ENOMEM;
2120 }
2121
2122 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
2123 start, end, domain->id);
2124 /*
2125 * RMRR range might have overlap with physical memory range,
2126 * clear it first
2127 */
2128 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2129
2130 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
2131 last_vpfn - first_vpfn + 1,
2132 DMA_PTE_READ|DMA_PTE_WRITE);
2133 }
2134
2135 static int iommu_prepare_identity_map(struct pci_dev *pdev,
2136 unsigned long long start,
2137 unsigned long long end)
2138 {
2139 struct dmar_domain *domain;
2140 int ret;
2141
2142 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
2143 if (!domain)
2144 return -ENOMEM;
2145
2146 /* For _hardware_ passthrough, don't bother. But for software
2147 passthrough, we do it anyway -- it may indicate a memory
2148 range which is reserved in E820, so which didn't get set
2149 up to start with in si_domain */
2150 if (domain == si_domain && hw_pass_through) {
2151 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
2152 pci_name(pdev), start, end);
2153 return 0;
2154 }
2155
2156 printk(KERN_INFO
2157 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
2158 pci_name(pdev), start, end);
2159
2160 if (end < start) {
2161 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
2162 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2163 dmi_get_system_info(DMI_BIOS_VENDOR),
2164 dmi_get_system_info(DMI_BIOS_VERSION),
2165 dmi_get_system_info(DMI_PRODUCT_VERSION));
2166 ret = -EIO;
2167 goto error;
2168 }
2169
2170 if (end >> agaw_to_width(domain->agaw)) {
2171 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
2172 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2173 agaw_to_width(domain->agaw),
2174 dmi_get_system_info(DMI_BIOS_VENDOR),
2175 dmi_get_system_info(DMI_BIOS_VERSION),
2176 dmi_get_system_info(DMI_PRODUCT_VERSION));
2177 ret = -EIO;
2178 goto error;
2179 }
2180
2181 ret = iommu_domain_identity_map(domain, start, end);
2182 if (ret)
2183 goto error;
2184
2185 /* context entry init */
2186 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
2187 if (ret)
2188 goto error;
2189
2190 return 0;
2191
2192 error:
2193 domain_exit(domain);
2194 return ret;
2195 }
2196
2197 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2198 struct pci_dev *pdev)
2199 {
2200 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2201 return 0;
2202 return iommu_prepare_identity_map(pdev, rmrr->base_address,
2203 rmrr->end_address);
2204 }
2205
2206 #ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
2207 static inline void iommu_prepare_isa(void)
2208 {
2209 struct pci_dev *pdev;
2210 int ret;
2211
2212 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2213 if (!pdev)
2214 return;
2215
2216 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
2217 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024 - 1);
2218
2219 if (ret)
2220 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2221 "floppy might not work\n");
2222
2223 }
2224 #else
2225 static inline void iommu_prepare_isa(void)
2226 {
2227 return;
2228 }
2229 #endif /* !CONFIG_INTEL_IOMMU_FLPY_WA */
2230
2231 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2232
2233 static int __init si_domain_init(int hw)
2234 {
2235 struct dmar_drhd_unit *drhd;
2236 struct intel_iommu *iommu;
2237 int nid, ret = 0;
2238
2239 si_domain = alloc_domain();
2240 if (!si_domain)
2241 return -EFAULT;
2242
2243 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2244
2245 for_each_active_iommu(iommu, drhd) {
2246 ret = iommu_attach_domain(si_domain, iommu);
2247 if (ret) {
2248 domain_exit(si_domain);
2249 return -EFAULT;
2250 }
2251 }
2252
2253 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2254 domain_exit(si_domain);
2255 return -EFAULT;
2256 }
2257
2258 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2259
2260 if (hw)
2261 return 0;
2262
2263 for_each_online_node(nid) {
2264 unsigned long start_pfn, end_pfn;
2265 int i;
2266
2267 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2268 ret = iommu_domain_identity_map(si_domain,
2269 PFN_PHYS(start_pfn), PFN_PHYS(end_pfn));
2270 if (ret)
2271 return ret;
2272 }
2273 }
2274
2275 return 0;
2276 }
2277
2278 static void domain_remove_one_dev_info(struct dmar_domain *domain,
2279 struct pci_dev *pdev);
2280 static int identity_mapping(struct pci_dev *pdev)
2281 {
2282 struct device_domain_info *info;
2283
2284 if (likely(!iommu_identity_mapping))
2285 return 0;
2286
2287 info = pdev->dev.archdata.iommu;
2288 if (info && info != DUMMY_DEVICE_DOMAIN_INFO)
2289 return (info->domain == si_domain);
2290
2291 return 0;
2292 }
2293
2294 static int domain_add_dev_info(struct dmar_domain *domain,
2295 struct pci_dev *pdev,
2296 int translation)
2297 {
2298 struct device_domain_info *info;
2299 unsigned long flags;
2300 int ret;
2301
2302 info = alloc_devinfo_mem();
2303 if (!info)
2304 return -ENOMEM;
2305
2306 info->segment = pci_domain_nr(pdev->bus);
2307 info->bus = pdev->bus->number;
2308 info->devfn = pdev->devfn;
2309 info->dev = pdev;
2310 info->domain = domain;
2311
2312 spin_lock_irqsave(&device_domain_lock, flags);
2313 list_add(&info->link, &domain->devices);
2314 list_add(&info->global, &device_domain_list);
2315 pdev->dev.archdata.iommu = info;
2316 spin_unlock_irqrestore(&device_domain_lock, flags);
2317
2318 ret = domain_context_mapping(domain, pdev, translation);
2319 if (ret) {
2320 spin_lock_irqsave(&device_domain_lock, flags);
2321 unlink_domain_info(info);
2322 spin_unlock_irqrestore(&device_domain_lock, flags);
2323 free_devinfo_mem(info);
2324 return ret;
2325 }
2326
2327 return 0;
2328 }
2329
2330 static bool device_has_rmrr(struct pci_dev *dev)
2331 {
2332 struct dmar_rmrr_unit *rmrr;
2333 int i;
2334
2335 for_each_rmrr_units(rmrr) {
2336 for (i = 0; i < rmrr->devices_cnt; i++) {
2337 /*
2338 * Return TRUE if this RMRR contains the device that
2339 * is passed in.
2340 */
2341 if (rmrr->devices[i] == dev)
2342 return true;
2343 }
2344 }
2345 return false;
2346 }
2347
2348 static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2349 {
2350
2351 /*
2352 * We want to prevent any device associated with an RMRR from
2353 * getting placed into the SI Domain. This is done because
2354 * problems exist when devices are moved in and out of domains
2355 * and their respective RMRR info is lost. We exempt USB devices
2356 * from this process due to their usage of RMRRs that are known
2357 * to not be needed after BIOS hand-off to OS.
2358 */
2359 if (device_has_rmrr(pdev) &&
2360 (pdev->class >> 8) != PCI_CLASS_SERIAL_USB)
2361 return 0;
2362
2363 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2364 return 1;
2365
2366 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2367 return 1;
2368
2369 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2370 return 0;
2371
2372 /*
2373 * We want to start off with all devices in the 1:1 domain, and
2374 * take them out later if we find they can't access all of memory.
2375 *
2376 * However, we can't do this for PCI devices behind bridges,
2377 * because all PCI devices behind the same bridge will end up
2378 * with the same source-id on their transactions.
2379 *
2380 * Practically speaking, we can't change things around for these
2381 * devices at run-time, because we can't be sure there'll be no
2382 * DMA transactions in flight for any of their siblings.
2383 *
2384 * So PCI devices (unless they're on the root bus) as well as
2385 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2386 * the 1:1 domain, just in _case_ one of their siblings turns out
2387 * not to be able to map all of memory.
2388 */
2389 if (!pci_is_pcie(pdev)) {
2390 if (!pci_is_root_bus(pdev->bus))
2391 return 0;
2392 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2393 return 0;
2394 } else if (pci_pcie_type(pdev) == PCI_EXP_TYPE_PCI_BRIDGE)
2395 return 0;
2396
2397 /*
2398 * At boot time, we don't yet know if devices will be 64-bit capable.
2399 * Assume that they will -- if they turn out not to be, then we can
2400 * take them out of the 1:1 domain later.
2401 */
2402 if (!startup) {
2403 /*
2404 * If the device's dma_mask is less than the system's memory
2405 * size then this is not a candidate for identity mapping.
2406 */
2407 u64 dma_mask = pdev->dma_mask;
2408
2409 if (pdev->dev.coherent_dma_mask &&
2410 pdev->dev.coherent_dma_mask < dma_mask)
2411 dma_mask = pdev->dev.coherent_dma_mask;
2412
2413 return dma_mask >= dma_get_required_mask(&pdev->dev);
2414 }
2415
2416 return 1;
2417 }
2418
2419 static int __init iommu_prepare_static_identity_mapping(int hw)
2420 {
2421 struct pci_dev *pdev = NULL;
2422 int ret;
2423
2424 ret = si_domain_init(hw);
2425 if (ret)
2426 return -EFAULT;
2427
2428 for_each_pci_dev(pdev) {
2429 if (iommu_should_identity_map(pdev, 1)) {
2430 ret = domain_add_dev_info(si_domain, pdev,
2431 hw ? CONTEXT_TT_PASS_THROUGH :
2432 CONTEXT_TT_MULTI_LEVEL);
2433 if (ret) {
2434 /* device not associated with an iommu */
2435 if (ret == -ENODEV)
2436 continue;
2437 return ret;
2438 }
2439 pr_info("IOMMU: %s identity mapping for device %s\n",
2440 hw ? "hardware" : "software", pci_name(pdev));
2441 }
2442 }
2443
2444 return 0;
2445 }
2446
2447 static int __init init_dmars(void)
2448 {
2449 struct dmar_drhd_unit *drhd;
2450 struct dmar_rmrr_unit *rmrr;
2451 struct pci_dev *pdev;
2452 struct intel_iommu *iommu;
2453 int i, ret;
2454
2455 /*
2456 * for each drhd
2457 * allocate root
2458 * initialize and program root entry to not present
2459 * endfor
2460 */
2461 for_each_drhd_unit(drhd) {
2462 /*
2463 * lock not needed as this is only incremented in the single
2464 * threaded kernel __init code path all other access are read
2465 * only
2466 */
2467 if (g_num_of_iommus < IOMMU_UNITS_SUPPORTED) {
2468 g_num_of_iommus++;
2469 continue;
2470 }
2471 printk_once(KERN_ERR "intel-iommu: exceeded %d IOMMUs\n",
2472 IOMMU_UNITS_SUPPORTED);
2473 }
2474
2475 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2476 GFP_KERNEL);
2477 if (!g_iommus) {
2478 printk(KERN_ERR "Allocating global iommu array failed\n");
2479 ret = -ENOMEM;
2480 goto error;
2481 }
2482
2483 deferred_flush = kzalloc(g_num_of_iommus *
2484 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2485 if (!deferred_flush) {
2486 ret = -ENOMEM;
2487 goto error;
2488 }
2489
2490 for_each_drhd_unit(drhd) {
2491 if (drhd->ignored)
2492 continue;
2493
2494 iommu = drhd->iommu;
2495 g_iommus[iommu->seq_id] = iommu;
2496
2497 ret = iommu_init_domains(iommu);
2498 if (ret)
2499 goto error;
2500
2501 /*
2502 * TBD:
2503 * we could share the same root & context tables
2504 * among all IOMMU's. Need to Split it later.
2505 */
2506 ret = iommu_alloc_root_entry(iommu);
2507 if (ret) {
2508 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2509 goto error;
2510 }
2511 if (!ecap_pass_through(iommu->ecap))
2512 hw_pass_through = 0;
2513 }
2514
2515 /*
2516 * Start from the sane iommu hardware state.
2517 */
2518 for_each_drhd_unit(drhd) {
2519 if (drhd->ignored)
2520 continue;
2521
2522 iommu = drhd->iommu;
2523
2524 /*
2525 * If the queued invalidation is already initialized by us
2526 * (for example, while enabling interrupt-remapping) then
2527 * we got the things already rolling from a sane state.
2528 */
2529 if (iommu->qi)
2530 continue;
2531
2532 /*
2533 * Clear any previous faults.
2534 */
2535 dmar_fault(-1, iommu);
2536 /*
2537 * Disable queued invalidation if supported and already enabled
2538 * before OS handover.
2539 */
2540 dmar_disable_qi(iommu);
2541 }
2542
2543 for_each_drhd_unit(drhd) {
2544 if (drhd->ignored)
2545 continue;
2546
2547 iommu = drhd->iommu;
2548
2549 if (dmar_enable_qi(iommu)) {
2550 /*
2551 * Queued Invalidate not enabled, use Register Based
2552 * Invalidate
2553 */
2554 iommu->flush.flush_context = __iommu_flush_context;
2555 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2556 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
2557 "invalidation\n",
2558 iommu->seq_id,
2559 (unsigned long long)drhd->reg_base_addr);
2560 } else {
2561 iommu->flush.flush_context = qi_flush_context;
2562 iommu->flush.flush_iotlb = qi_flush_iotlb;
2563 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
2564 "invalidation\n",
2565 iommu->seq_id,
2566 (unsigned long long)drhd->reg_base_addr);
2567 }
2568 }
2569
2570 if (iommu_pass_through)
2571 iommu_identity_mapping |= IDENTMAP_ALL;
2572
2573 #ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2574 iommu_identity_mapping |= IDENTMAP_GFX;
2575 #endif
2576
2577 check_tylersburg_isoch();
2578
2579 /*
2580 * If pass through is not set or not enabled, setup context entries for
2581 * identity mappings for rmrr, gfx, and isa and may fall back to static
2582 * identity mapping if iommu_identity_mapping is set.
2583 */
2584 if (iommu_identity_mapping) {
2585 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
2586 if (ret) {
2587 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2588 goto error;
2589 }
2590 }
2591 /*
2592 * For each rmrr
2593 * for each dev attached to rmrr
2594 * do
2595 * locate drhd for dev, alloc domain for dev
2596 * allocate free domain
2597 * allocate page table entries for rmrr
2598 * if context not allocated for bus
2599 * allocate and init context
2600 * set present in root table for this bus
2601 * init context with domain, translation etc
2602 * endfor
2603 * endfor
2604 */
2605 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2606 for_each_rmrr_units(rmrr) {
2607 for (i = 0; i < rmrr->devices_cnt; i++) {
2608 pdev = rmrr->devices[i];
2609 /*
2610 * some BIOS lists non-exist devices in DMAR
2611 * table.
2612 */
2613 if (!pdev)
2614 continue;
2615 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2616 if (ret)
2617 printk(KERN_ERR
2618 "IOMMU: mapping reserved region failed\n");
2619 }
2620 }
2621
2622 iommu_prepare_isa();
2623
2624 /*
2625 * for each drhd
2626 * enable fault log
2627 * global invalidate context cache
2628 * global invalidate iotlb
2629 * enable translation
2630 */
2631 for_each_drhd_unit(drhd) {
2632 if (drhd->ignored) {
2633 /*
2634 * we always have to disable PMRs or DMA may fail on
2635 * this device
2636 */
2637 if (force_on)
2638 iommu_disable_protect_mem_regions(drhd->iommu);
2639 continue;
2640 }
2641 iommu = drhd->iommu;
2642
2643 iommu_flush_write_buffer(iommu);
2644
2645 ret = dmar_set_interrupt(iommu);
2646 if (ret)
2647 goto error;
2648
2649 iommu_set_root_entry(iommu);
2650
2651 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
2652 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
2653
2654 ret = iommu_enable_translation(iommu);
2655 if (ret)
2656 goto error;
2657
2658 iommu_disable_protect_mem_regions(iommu);
2659 }
2660
2661 return 0;
2662 error:
2663 for_each_drhd_unit(drhd) {
2664 if (drhd->ignored)
2665 continue;
2666 iommu = drhd->iommu;
2667 free_iommu(iommu);
2668 }
2669 kfree(g_iommus);
2670 return ret;
2671 }
2672
2673 /* This takes a number of _MM_ pages, not VTD pages */
2674 static struct iova *intel_alloc_iova(struct device *dev,
2675 struct dmar_domain *domain,
2676 unsigned long nrpages, uint64_t dma_mask)
2677 {
2678 struct pci_dev *pdev = to_pci_dev(dev);
2679 struct iova *iova = NULL;
2680
2681 /* Restrict dma_mask to the width that the iommu can handle */
2682 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2683
2684 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
2685 /*
2686 * First try to allocate an io virtual address in
2687 * DMA_BIT_MASK(32) and if that fails then try allocating
2688 * from higher range
2689 */
2690 iova = alloc_iova(&domain->iovad, nrpages,
2691 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2692 if (iova)
2693 return iova;
2694 }
2695 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2696 if (unlikely(!iova)) {
2697 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2698 nrpages, pci_name(pdev));
2699 return NULL;
2700 }
2701
2702 return iova;
2703 }
2704
2705 static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
2706 {
2707 struct dmar_domain *domain;
2708 int ret;
2709
2710 domain = get_domain_for_dev(pdev,
2711 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2712 if (!domain) {
2713 printk(KERN_ERR
2714 "Allocating domain for %s failed", pci_name(pdev));
2715 return NULL;
2716 }
2717
2718 /* make sure context mapping is ok */
2719 if (unlikely(!domain_context_mapped(pdev))) {
2720 ret = domain_context_mapping(domain, pdev,
2721 CONTEXT_TT_MULTI_LEVEL);
2722 if (ret) {
2723 printk(KERN_ERR
2724 "Domain context map for %s failed",
2725 pci_name(pdev));
2726 return NULL;
2727 }
2728 }
2729
2730 return domain;
2731 }
2732
2733 static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2734 {
2735 struct device_domain_info *info;
2736
2737 /* No lock here, assumes no domain exit in normal case */
2738 info = dev->dev.archdata.iommu;
2739 if (likely(info))
2740 return info->domain;
2741
2742 return __get_valid_domain_for_dev(dev);
2743 }
2744
2745 static int iommu_dummy(struct pci_dev *pdev)
2746 {
2747 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2748 }
2749
2750 /* Check if the pdev needs to go through non-identity map and unmap process.*/
2751 static int iommu_no_mapping(struct device *dev)
2752 {
2753 struct pci_dev *pdev;
2754 int found;
2755
2756 if (unlikely(dev->bus != &pci_bus_type))
2757 return 1;
2758
2759 pdev = to_pci_dev(dev);
2760 if (iommu_dummy(pdev))
2761 return 1;
2762
2763 if (!iommu_identity_mapping)
2764 return 0;
2765
2766 found = identity_mapping(pdev);
2767 if (found) {
2768 if (iommu_should_identity_map(pdev, 0))
2769 return 1;
2770 else {
2771 /*
2772 * 32 bit DMA is removed from si_domain and fall back
2773 * to non-identity mapping.
2774 */
2775 domain_remove_one_dev_info(si_domain, pdev);
2776 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2777 pci_name(pdev));
2778 return 0;
2779 }
2780 } else {
2781 /*
2782 * In case of a detached 64 bit DMA device from vm, the device
2783 * is put into si_domain for identity mapping.
2784 */
2785 if (iommu_should_identity_map(pdev, 0)) {
2786 int ret;
2787 ret = domain_add_dev_info(si_domain, pdev,
2788 hw_pass_through ?
2789 CONTEXT_TT_PASS_THROUGH :
2790 CONTEXT_TT_MULTI_LEVEL);
2791 if (!ret) {
2792 printk(KERN_INFO "64bit %s uses identity mapping\n",
2793 pci_name(pdev));
2794 return 1;
2795 }
2796 }
2797 }
2798
2799 return 0;
2800 }
2801
2802 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2803 size_t size, int dir, u64 dma_mask)
2804 {
2805 struct pci_dev *pdev = to_pci_dev(hwdev);
2806 struct dmar_domain *domain;
2807 phys_addr_t start_paddr;
2808 struct iova *iova;
2809 int prot = 0;
2810 int ret;
2811 struct intel_iommu *iommu;
2812 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
2813
2814 BUG_ON(dir == DMA_NONE);
2815
2816 if (iommu_no_mapping(hwdev))
2817 return paddr;
2818
2819 domain = get_valid_domain_for_dev(pdev);
2820 if (!domain)
2821 return 0;
2822
2823 iommu = domain_get_iommu(domain);
2824 size = aligned_nrpages(paddr, size);
2825
2826 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size), dma_mask);
2827 if (!iova)
2828 goto error;
2829
2830 /*
2831 * Check if DMAR supports zero-length reads on write only
2832 * mappings..
2833 */
2834 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2835 !cap_zlr(iommu->cap))
2836 prot |= DMA_PTE_READ;
2837 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2838 prot |= DMA_PTE_WRITE;
2839 /*
2840 * paddr - (paddr + size) might be partial page, we should map the whole
2841 * page. Note: if two part of one page are separately mapped, we
2842 * might have two guest_addr mapping to the same host paddr, but this
2843 * is not a big problem
2844 */
2845 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
2846 mm_to_dma_pfn(paddr_pfn), size, prot);
2847 if (ret)
2848 goto error;
2849
2850 /* it's a non-present to present mapping. Only flush if caching mode */
2851 if (cap_caching_mode(iommu->cap))
2852 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
2853 else
2854 iommu_flush_write_buffer(iommu);
2855
2856 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2857 start_paddr += paddr & ~PAGE_MASK;
2858 return start_paddr;
2859
2860 error:
2861 if (iova)
2862 __free_iova(&domain->iovad, iova);
2863 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
2864 pci_name(pdev), size, (unsigned long long)paddr, dir);
2865 return 0;
2866 }
2867
2868 static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2869 unsigned long offset, size_t size,
2870 enum dma_data_direction dir,
2871 struct dma_attrs *attrs)
2872 {
2873 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2874 dir, to_pci_dev(dev)->dma_mask);
2875 }
2876
2877 static void flush_unmaps(void)
2878 {
2879 int i, j;
2880
2881 timer_on = 0;
2882
2883 /* just flush them all */
2884 for (i = 0; i < g_num_of_iommus; i++) {
2885 struct intel_iommu *iommu = g_iommus[i];
2886 if (!iommu)
2887 continue;
2888
2889 if (!deferred_flush[i].next)
2890 continue;
2891
2892 /* In caching mode, global flushes turn emulation expensive */
2893 if (!cap_caching_mode(iommu->cap))
2894 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2895 DMA_TLB_GLOBAL_FLUSH);
2896 for (j = 0; j < deferred_flush[i].next; j++) {
2897 unsigned long mask;
2898 struct iova *iova = deferred_flush[i].iova[j];
2899 struct dmar_domain *domain = deferred_flush[i].domain[j];
2900
2901 /* On real hardware multiple invalidations are expensive */
2902 if (cap_caching_mode(iommu->cap))
2903 iommu_flush_iotlb_psi(iommu, domain->id,
2904 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2905 else {
2906 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2907 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2908 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2909 }
2910 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
2911 }
2912 deferred_flush[i].next = 0;
2913 }
2914
2915 list_size = 0;
2916 }
2917
2918 static void flush_unmaps_timeout(unsigned long data)
2919 {
2920 unsigned long flags;
2921
2922 spin_lock_irqsave(&async_umap_flush_lock, flags);
2923 flush_unmaps();
2924 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2925 }
2926
2927 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2928 {
2929 unsigned long flags;
2930 int next, iommu_id;
2931 struct intel_iommu *iommu;
2932
2933 spin_lock_irqsave(&async_umap_flush_lock, flags);
2934 if (list_size == HIGH_WATER_MARK)
2935 flush_unmaps();
2936
2937 iommu = domain_get_iommu(dom);
2938 iommu_id = iommu->seq_id;
2939
2940 next = deferred_flush[iommu_id].next;
2941 deferred_flush[iommu_id].domain[next] = dom;
2942 deferred_flush[iommu_id].iova[next] = iova;
2943 deferred_flush[iommu_id].next++;
2944
2945 if (!timer_on) {
2946 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2947 timer_on = 1;
2948 }
2949 list_size++;
2950 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2951 }
2952
2953 static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2954 size_t size, enum dma_data_direction dir,
2955 struct dma_attrs *attrs)
2956 {
2957 struct pci_dev *pdev = to_pci_dev(dev);
2958 struct dmar_domain *domain;
2959 unsigned long start_pfn, last_pfn;
2960 struct iova *iova;
2961 struct intel_iommu *iommu;
2962
2963 if (iommu_no_mapping(dev))
2964 return;
2965
2966 domain = find_domain(pdev);
2967 BUG_ON(!domain);
2968
2969 iommu = domain_get_iommu(domain);
2970
2971 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2972 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2973 (unsigned long long)dev_addr))
2974 return;
2975
2976 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2977 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
2978
2979 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2980 pci_name(pdev), start_pfn, last_pfn);
2981
2982 /* clear the whole page */
2983 dma_pte_clear_range(domain, start_pfn, last_pfn);
2984
2985 /* free page tables */
2986 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2987
2988 if (intel_iommu_strict) {
2989 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
2990 last_pfn - start_pfn + 1, 0);
2991 /* free iova */
2992 __free_iova(&domain->iovad, iova);
2993 } else {
2994 add_unmap(domain, iova);
2995 /*
2996 * queue up the release of the unmap to save the 1/6th of the
2997 * cpu used up by the iotlb flush operation...
2998 */
2999 }
3000 }
3001
3002 static void *intel_alloc_coherent(struct device *hwdev, size_t size,
3003 dma_addr_t *dma_handle, gfp_t flags,
3004 struct dma_attrs *attrs)
3005 {
3006 void *vaddr;
3007 int order;
3008
3009 size = PAGE_ALIGN(size);
3010 order = get_order(size);
3011
3012 if (!iommu_no_mapping(hwdev))
3013 flags &= ~(GFP_DMA | GFP_DMA32);
3014 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
3015 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
3016 flags |= GFP_DMA;
3017 else
3018 flags |= GFP_DMA32;
3019 }
3020
3021 vaddr = (void *)__get_free_pages(flags, order);
3022 if (!vaddr)
3023 return NULL;
3024 memset(vaddr, 0, size);
3025
3026 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
3027 DMA_BIDIRECTIONAL,
3028 hwdev->coherent_dma_mask);
3029 if (*dma_handle)
3030 return vaddr;
3031 free_pages((unsigned long)vaddr, order);
3032 return NULL;
3033 }
3034
3035 static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
3036 dma_addr_t dma_handle, struct dma_attrs *attrs)
3037 {
3038 int order;
3039
3040 size = PAGE_ALIGN(size);
3041 order = get_order(size);
3042
3043 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
3044 free_pages((unsigned long)vaddr, order);
3045 }
3046
3047 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
3048 int nelems, enum dma_data_direction dir,
3049 struct dma_attrs *attrs)
3050 {
3051 struct pci_dev *pdev = to_pci_dev(hwdev);
3052 struct dmar_domain *domain;
3053 unsigned long start_pfn, last_pfn;
3054 struct iova *iova;
3055 struct intel_iommu *iommu;
3056
3057 if (iommu_no_mapping(hwdev))
3058 return;
3059
3060 domain = find_domain(pdev);
3061 BUG_ON(!domain);
3062
3063 iommu = domain_get_iommu(domain);
3064
3065 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
3066 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
3067 (unsigned long long)sglist[0].dma_address))
3068 return;
3069
3070 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
3071 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
3072
3073 /* clear the whole page */
3074 dma_pte_clear_range(domain, start_pfn, last_pfn);
3075
3076 /* free page tables */
3077 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
3078
3079 if (intel_iommu_strict) {
3080 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
3081 last_pfn - start_pfn + 1, 0);
3082 /* free iova */
3083 __free_iova(&domain->iovad, iova);
3084 } else {
3085 add_unmap(domain, iova);
3086 /*
3087 * queue up the release of the unmap to save the 1/6th of the
3088 * cpu used up by the iotlb flush operation...
3089 */
3090 }
3091 }
3092
3093 static int intel_nontranslate_map_sg(struct device *hddev,
3094 struct scatterlist *sglist, int nelems, int dir)
3095 {
3096 int i;
3097 struct scatterlist *sg;
3098
3099 for_each_sg(sglist, sg, nelems, i) {
3100 BUG_ON(!sg_page(sg));
3101 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
3102 sg->dma_length = sg->length;
3103 }
3104 return nelems;
3105 }
3106
3107 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
3108 enum dma_data_direction dir, struct dma_attrs *attrs)
3109 {
3110 int i;
3111 struct pci_dev *pdev = to_pci_dev(hwdev);
3112 struct dmar_domain *domain;
3113 size_t size = 0;
3114 int prot = 0;
3115 struct iova *iova = NULL;
3116 int ret;
3117 struct scatterlist *sg;
3118 unsigned long start_vpfn;
3119 struct intel_iommu *iommu;
3120
3121 BUG_ON(dir == DMA_NONE);
3122 if (iommu_no_mapping(hwdev))
3123 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
3124
3125 domain = get_valid_domain_for_dev(pdev);
3126 if (!domain)
3127 return 0;
3128
3129 iommu = domain_get_iommu(domain);
3130
3131 for_each_sg(sglist, sg, nelems, i)
3132 size += aligned_nrpages(sg->offset, sg->length);
3133
3134 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
3135 pdev->dma_mask);
3136 if (!iova) {
3137 sglist->dma_length = 0;
3138 return 0;
3139 }
3140
3141 /*
3142 * Check if DMAR supports zero-length reads on write only
3143 * mappings..
3144 */
3145 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
3146 !cap_zlr(iommu->cap))
3147 prot |= DMA_PTE_READ;
3148 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
3149 prot |= DMA_PTE_WRITE;
3150
3151 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
3152
3153 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
3154 if (unlikely(ret)) {
3155 /* clear the page */
3156 dma_pte_clear_range(domain, start_vpfn,
3157 start_vpfn + size - 1);
3158 /* free page tables */
3159 dma_pte_free_pagetable(domain, start_vpfn,
3160 start_vpfn + size - 1);
3161 /* free iova */
3162 __free_iova(&domain->iovad, iova);
3163 return 0;
3164 }
3165
3166 /* it's a non-present to present mapping. Only flush if caching mode */
3167 if (cap_caching_mode(iommu->cap))
3168 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
3169 else
3170 iommu_flush_write_buffer(iommu);
3171
3172 return nelems;
3173 }
3174
3175 static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
3176 {
3177 return !dma_addr;
3178 }
3179
3180 struct dma_map_ops intel_dma_ops = {
3181 .alloc = intel_alloc_coherent,
3182 .free = intel_free_coherent,
3183 .map_sg = intel_map_sg,
3184 .unmap_sg = intel_unmap_sg,
3185 .map_page = intel_map_page,
3186 .unmap_page = intel_unmap_page,
3187 .mapping_error = intel_mapping_error,
3188 };
3189
3190 static inline int iommu_domain_cache_init(void)
3191 {
3192 int ret = 0;
3193
3194 iommu_domain_cache = kmem_cache_create("iommu_domain",
3195 sizeof(struct dmar_domain),
3196 0,
3197 SLAB_HWCACHE_ALIGN,
3198
3199 NULL);
3200 if (!iommu_domain_cache) {
3201 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
3202 ret = -ENOMEM;
3203 }
3204
3205 return ret;
3206 }
3207
3208 static inline int iommu_devinfo_cache_init(void)
3209 {
3210 int ret = 0;
3211
3212 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
3213 sizeof(struct device_domain_info),
3214 0,
3215 SLAB_HWCACHE_ALIGN,
3216 NULL);
3217 if (!iommu_devinfo_cache) {
3218 printk(KERN_ERR "Couldn't create devinfo cache\n");
3219 ret = -ENOMEM;
3220 }
3221
3222 return ret;
3223 }
3224
3225 static inline int iommu_iova_cache_init(void)
3226 {
3227 int ret = 0;
3228
3229 iommu_iova_cache = kmem_cache_create("iommu_iova",
3230 sizeof(struct iova),
3231 0,
3232 SLAB_HWCACHE_ALIGN,
3233 NULL);
3234 if (!iommu_iova_cache) {
3235 printk(KERN_ERR "Couldn't create iova cache\n");
3236 ret = -ENOMEM;
3237 }
3238
3239 return ret;
3240 }
3241
3242 static int __init iommu_init_mempool(void)
3243 {
3244 int ret;
3245 ret = iommu_iova_cache_init();
3246 if (ret)
3247 return ret;
3248
3249 ret = iommu_domain_cache_init();
3250 if (ret)
3251 goto domain_error;
3252
3253 ret = iommu_devinfo_cache_init();
3254 if (!ret)
3255 return ret;
3256
3257 kmem_cache_destroy(iommu_domain_cache);
3258 domain_error:
3259 kmem_cache_destroy(iommu_iova_cache);
3260
3261 return -ENOMEM;
3262 }
3263
3264 static void __init iommu_exit_mempool(void)
3265 {
3266 kmem_cache_destroy(iommu_devinfo_cache);
3267 kmem_cache_destroy(iommu_domain_cache);
3268 kmem_cache_destroy(iommu_iova_cache);
3269
3270 }
3271
3272 static void quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
3273 {
3274 struct dmar_drhd_unit *drhd;
3275 u32 vtbar;
3276 int rc;
3277
3278 /* We know that this device on this chipset has its own IOMMU.
3279 * If we find it under a different IOMMU, then the BIOS is lying
3280 * to us. Hope that the IOMMU for this device is actually
3281 * disabled, and it needs no translation...
3282 */
3283 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
3284 if (rc) {
3285 /* "can't" happen */
3286 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
3287 return;
3288 }
3289 vtbar &= 0xffff0000;
3290
3291 /* we know that the this iommu should be at offset 0xa000 from vtbar */
3292 drhd = dmar_find_matched_drhd_unit(pdev);
3293 if (WARN_TAINT_ONCE(!drhd || drhd->reg_base_addr - vtbar != 0xa000,
3294 TAINT_FIRMWARE_WORKAROUND,
3295 "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n"))
3296 pdev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3297 }
3298 DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quirk_ioat_snb_local_iommu);
3299
3300 static void __init init_no_remapping_devices(void)
3301 {
3302 struct dmar_drhd_unit *drhd;
3303
3304 for_each_drhd_unit(drhd) {
3305 if (!drhd->include_all) {
3306 int i;
3307 for (i = 0; i < drhd->devices_cnt; i++)
3308 if (drhd->devices[i] != NULL)
3309 break;
3310 /* ignore DMAR unit if no pci devices exist */
3311 if (i == drhd->devices_cnt)
3312 drhd->ignored = 1;
3313 }
3314 }
3315
3316 for_each_drhd_unit(drhd) {
3317 int i;
3318 if (drhd->ignored || drhd->include_all)
3319 continue;
3320
3321 for (i = 0; i < drhd->devices_cnt; i++)
3322 if (drhd->devices[i] &&
3323 !IS_GFX_DEVICE(drhd->devices[i]))
3324 break;
3325
3326 if (i < drhd->devices_cnt)
3327 continue;
3328
3329 /* This IOMMU has *only* gfx devices. Either bypass it or
3330 set the gfx_mapped flag, as appropriate */
3331 if (dmar_map_gfx) {
3332 intel_iommu_gfx_mapped = 1;
3333 } else {
3334 drhd->ignored = 1;
3335 for (i = 0; i < drhd->devices_cnt; i++) {
3336 if (!drhd->devices[i])
3337 continue;
3338 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
3339 }
3340 }
3341 }
3342 }
3343
3344 #ifdef CONFIG_SUSPEND
3345 static int init_iommu_hw(void)
3346 {
3347 struct dmar_drhd_unit *drhd;
3348 struct intel_iommu *iommu = NULL;
3349
3350 for_each_active_iommu(iommu, drhd)
3351 if (iommu->qi)
3352 dmar_reenable_qi(iommu);
3353
3354 for_each_iommu(iommu, drhd) {
3355 if (drhd->ignored) {
3356 /*
3357 * we always have to disable PMRs or DMA may fail on
3358 * this device
3359 */
3360 if (force_on)
3361 iommu_disable_protect_mem_regions(iommu);
3362 continue;
3363 }
3364
3365 iommu_flush_write_buffer(iommu);
3366
3367 iommu_set_root_entry(iommu);
3368
3369 iommu->flush.flush_context(iommu, 0, 0, 0,
3370 DMA_CCMD_GLOBAL_INVL);
3371 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3372 DMA_TLB_GLOBAL_FLUSH);
3373 if (iommu_enable_translation(iommu))
3374 return 1;
3375 iommu_disable_protect_mem_regions(iommu);
3376 }
3377
3378 return 0;
3379 }
3380
3381 static void iommu_flush_all(void)
3382 {
3383 struct dmar_drhd_unit *drhd;
3384 struct intel_iommu *iommu;
3385
3386 for_each_active_iommu(iommu, drhd) {
3387 iommu->flush.flush_context(iommu, 0, 0, 0,
3388 DMA_CCMD_GLOBAL_INVL);
3389 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3390 DMA_TLB_GLOBAL_FLUSH);
3391 }
3392 }
3393
3394 static int iommu_suspend(void)
3395 {
3396 struct dmar_drhd_unit *drhd;
3397 struct intel_iommu *iommu = NULL;
3398 unsigned long flag;
3399
3400 for_each_active_iommu(iommu, drhd) {
3401 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3402 GFP_ATOMIC);
3403 if (!iommu->iommu_state)
3404 goto nomem;
3405 }
3406
3407 iommu_flush_all();
3408
3409 for_each_active_iommu(iommu, drhd) {
3410 iommu_disable_translation(iommu);
3411
3412 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3413
3414 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3415 readl(iommu->reg + DMAR_FECTL_REG);
3416 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3417 readl(iommu->reg + DMAR_FEDATA_REG);
3418 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3419 readl(iommu->reg + DMAR_FEADDR_REG);
3420 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3421 readl(iommu->reg + DMAR_FEUADDR_REG);
3422
3423 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3424 }
3425 return 0;
3426
3427 nomem:
3428 for_each_active_iommu(iommu, drhd)
3429 kfree(iommu->iommu_state);
3430
3431 return -ENOMEM;
3432 }
3433
3434 static void iommu_resume(void)
3435 {
3436 struct dmar_drhd_unit *drhd;
3437 struct intel_iommu *iommu = NULL;
3438 unsigned long flag;
3439
3440 if (init_iommu_hw()) {
3441 if (force_on)
3442 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3443 else
3444 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3445 return;
3446 }
3447
3448 for_each_active_iommu(iommu, drhd) {
3449
3450 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3451
3452 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3453 iommu->reg + DMAR_FECTL_REG);
3454 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3455 iommu->reg + DMAR_FEDATA_REG);
3456 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3457 iommu->reg + DMAR_FEADDR_REG);
3458 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3459 iommu->reg + DMAR_FEUADDR_REG);
3460
3461 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3462 }
3463
3464 for_each_active_iommu(iommu, drhd)
3465 kfree(iommu->iommu_state);
3466 }
3467
3468 static struct syscore_ops iommu_syscore_ops = {
3469 .resume = iommu_resume,
3470 .suspend = iommu_suspend,
3471 };
3472
3473 static void __init init_iommu_pm_ops(void)
3474 {
3475 register_syscore_ops(&iommu_syscore_ops);
3476 }
3477
3478 #else
3479 static inline void init_iommu_pm_ops(void) {}
3480 #endif /* CONFIG_PM */
3481
3482 LIST_HEAD(dmar_rmrr_units);
3483
3484 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
3485 {
3486 list_add(&rmrr->list, &dmar_rmrr_units);
3487 }
3488
3489
3490 int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header)
3491 {
3492 struct acpi_dmar_reserved_memory *rmrr;
3493 struct dmar_rmrr_unit *rmrru;
3494
3495 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3496 if (!rmrru)
3497 return -ENOMEM;
3498
3499 rmrru->hdr = header;
3500 rmrr = (struct acpi_dmar_reserved_memory *)header;
3501 rmrru->base_address = rmrr->base_address;
3502 rmrru->end_address = rmrr->end_address;
3503
3504 dmar_register_rmrr_unit(rmrru);
3505 return 0;
3506 }
3507
3508 static int __init
3509 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
3510 {
3511 struct acpi_dmar_reserved_memory *rmrr;
3512 int ret;
3513
3514 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
3515 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
3516 ((void *)rmrr) + rmrr->header.length,
3517 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
3518
3519 if (ret || (rmrru->devices_cnt == 0)) {
3520 list_del(&rmrru->list);
3521 kfree(rmrru);
3522 }
3523 return ret;
3524 }
3525
3526 static LIST_HEAD(dmar_atsr_units);
3527
3528 int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
3529 {
3530 struct acpi_dmar_atsr *atsr;
3531 struct dmar_atsr_unit *atsru;
3532
3533 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3534 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
3535 if (!atsru)
3536 return -ENOMEM;
3537
3538 atsru->hdr = hdr;
3539 atsru->include_all = atsr->flags & 0x1;
3540
3541 list_add(&atsru->list, &dmar_atsr_units);
3542
3543 return 0;
3544 }
3545
3546 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
3547 {
3548 int rc;
3549 struct acpi_dmar_atsr *atsr;
3550
3551 if (atsru->include_all)
3552 return 0;
3553
3554 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3555 rc = dmar_parse_dev_scope((void *)(atsr + 1),
3556 (void *)atsr + atsr->header.length,
3557 &atsru->devices_cnt, &atsru->devices,
3558 atsr->segment);
3559 if (rc || !atsru->devices_cnt) {
3560 list_del(&atsru->list);
3561 kfree(atsru);
3562 }
3563
3564 return rc;
3565 }
3566
3567 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
3568 {
3569 int i;
3570 struct pci_bus *bus;
3571 struct acpi_dmar_atsr *atsr;
3572 struct dmar_atsr_unit *atsru;
3573
3574 dev = pci_physfn(dev);
3575
3576 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3577 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3578 if (atsr->segment == pci_domain_nr(dev->bus))
3579 goto found;
3580 }
3581
3582 return 0;
3583
3584 found:
3585 for (bus = dev->bus; bus; bus = bus->parent) {
3586 struct pci_dev *bridge = bus->self;
3587
3588 if (!bridge || !pci_is_pcie(bridge) ||
3589 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
3590 return 0;
3591
3592 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT) {
3593 for (i = 0; i < atsru->devices_cnt; i++)
3594 if (atsru->devices[i] == bridge)
3595 return 1;
3596 break;
3597 }
3598 }
3599
3600 if (atsru->include_all)
3601 return 1;
3602
3603 return 0;
3604 }
3605
3606 int __init dmar_parse_rmrr_atsr_dev(void)
3607 {
3608 struct dmar_rmrr_unit *rmrr, *rmrr_n;
3609 struct dmar_atsr_unit *atsr, *atsr_n;
3610 int ret = 0;
3611
3612 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
3613 ret = rmrr_parse_dev(rmrr);
3614 if (ret)
3615 return ret;
3616 }
3617
3618 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
3619 ret = atsr_parse_dev(atsr);
3620 if (ret)
3621 return ret;
3622 }
3623
3624 return ret;
3625 }
3626
3627 /*
3628 * Here we only respond to action of unbound device from driver.
3629 *
3630 * Added device is not attached to its DMAR domain here yet. That will happen
3631 * when mapping the device to iova.
3632 */
3633 static int device_notifier(struct notifier_block *nb,
3634 unsigned long action, void *data)
3635 {
3636 struct device *dev = data;
3637 struct pci_dev *pdev = to_pci_dev(dev);
3638 struct dmar_domain *domain;
3639
3640 if (iommu_no_mapping(dev))
3641 return 0;
3642
3643 domain = find_domain(pdev);
3644 if (!domain)
3645 return 0;
3646
3647 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through) {
3648 domain_remove_one_dev_info(domain, pdev);
3649
3650 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3651 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) &&
3652 list_empty(&domain->devices))
3653 domain_exit(domain);
3654 }
3655
3656 return 0;
3657 }
3658
3659 static struct notifier_block device_nb = {
3660 .notifier_call = device_notifier,
3661 };
3662
3663 int __init intel_iommu_init(void)
3664 {
3665 int ret = 0;
3666
3667 /* VT-d is required for a TXT/tboot launch, so enforce that */
3668 force_on = tboot_force_iommu();
3669
3670 if (dmar_table_init()) {
3671 if (force_on)
3672 panic("tboot: Failed to initialize DMAR table\n");
3673 return -ENODEV;
3674 }
3675
3676 if (dmar_dev_scope_init() < 0) {
3677 if (force_on)
3678 panic("tboot: Failed to initialize DMAR device scope\n");
3679 return -ENODEV;
3680 }
3681
3682 if (no_iommu || dmar_disabled)
3683 return -ENODEV;
3684
3685 if (iommu_init_mempool()) {
3686 if (force_on)
3687 panic("tboot: Failed to initialize iommu memory\n");
3688 return -ENODEV;
3689 }
3690
3691 if (list_empty(&dmar_rmrr_units))
3692 printk(KERN_INFO "DMAR: No RMRR found\n");
3693
3694 if (list_empty(&dmar_atsr_units))
3695 printk(KERN_INFO "DMAR: No ATSR found\n");
3696
3697 if (dmar_init_reserved_ranges()) {
3698 if (force_on)
3699 panic("tboot: Failed to reserve iommu ranges\n");
3700 return -ENODEV;
3701 }
3702
3703 init_no_remapping_devices();
3704
3705 ret = init_dmars();
3706 if (ret) {
3707 if (force_on)
3708 panic("tboot: Failed to initialize DMARs\n");
3709 printk(KERN_ERR "IOMMU: dmar init failed\n");
3710 put_iova_domain(&reserved_iova_list);
3711 iommu_exit_mempool();
3712 return ret;
3713 }
3714 printk(KERN_INFO
3715 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3716
3717 init_timer(&unmap_timer);
3718 #ifdef CONFIG_SWIOTLB
3719 swiotlb = 0;
3720 #endif
3721 dma_ops = &intel_dma_ops;
3722
3723 init_iommu_pm_ops();
3724
3725 bus_set_iommu(&pci_bus_type, &intel_iommu_ops);
3726
3727 bus_register_notifier(&pci_bus_type, &device_nb);
3728
3729 intel_iommu_enabled = 1;
3730
3731 return 0;
3732 }
3733
3734 static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3735 struct pci_dev *pdev)
3736 {
3737 struct pci_dev *tmp, *parent;
3738
3739 if (!iommu || !pdev)
3740 return;
3741
3742 /* dependent device detach */
3743 tmp = pci_find_upstream_pcie_bridge(pdev);
3744 /* Secondary interface's bus number and devfn 0 */
3745 if (tmp) {
3746 parent = pdev->bus->self;
3747 while (parent != tmp) {
3748 iommu_detach_dev(iommu, parent->bus->number,
3749 parent->devfn);
3750 parent = parent->bus->self;
3751 }
3752 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3753 iommu_detach_dev(iommu,
3754 tmp->subordinate->number, 0);
3755 else /* this is a legacy PCI bridge */
3756 iommu_detach_dev(iommu, tmp->bus->number,
3757 tmp->devfn);
3758 }
3759 }
3760
3761 static void domain_remove_one_dev_info(struct dmar_domain *domain,
3762 struct pci_dev *pdev)
3763 {
3764 struct device_domain_info *info;
3765 struct intel_iommu *iommu;
3766 unsigned long flags;
3767 int found = 0;
3768 struct list_head *entry, *tmp;
3769
3770 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3771 pdev->devfn);
3772 if (!iommu)
3773 return;
3774
3775 spin_lock_irqsave(&device_domain_lock, flags);
3776 list_for_each_safe(entry, tmp, &domain->devices) {
3777 info = list_entry(entry, struct device_domain_info, link);
3778 if (info->segment == pci_domain_nr(pdev->bus) &&
3779 info->bus == pdev->bus->number &&
3780 info->devfn == pdev->devfn) {
3781 unlink_domain_info(info);
3782 spin_unlock_irqrestore(&device_domain_lock, flags);
3783
3784 iommu_disable_dev_iotlb(info);
3785 iommu_detach_dev(iommu, info->bus, info->devfn);
3786 iommu_detach_dependent_devices(iommu, pdev);
3787 free_devinfo_mem(info);
3788
3789 spin_lock_irqsave(&device_domain_lock, flags);
3790
3791 if (found)
3792 break;
3793 else
3794 continue;
3795 }
3796
3797 /* if there is no other devices under the same iommu
3798 * owned by this domain, clear this iommu in iommu_bmp
3799 * update iommu count and coherency
3800 */
3801 if (iommu == device_to_iommu(info->segment, info->bus,
3802 info->devfn))
3803 found = 1;
3804 }
3805
3806 spin_unlock_irqrestore(&device_domain_lock, flags);
3807
3808 if (found == 0) {
3809 unsigned long tmp_flags;
3810 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3811 clear_bit(iommu->seq_id, domain->iommu_bmp);
3812 domain->iommu_count--;
3813 domain_update_iommu_cap(domain);
3814 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3815
3816 if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) &&
3817 !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)) {
3818 spin_lock_irqsave(&iommu->lock, tmp_flags);
3819 clear_bit(domain->id, iommu->domain_ids);
3820 iommu->domains[domain->id] = NULL;
3821 spin_unlock_irqrestore(&iommu->lock, tmp_flags);
3822 }
3823 }
3824 }
3825
3826 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3827 {
3828 struct device_domain_info *info;
3829 struct intel_iommu *iommu;
3830 unsigned long flags1, flags2;
3831
3832 spin_lock_irqsave(&device_domain_lock, flags1);
3833 while (!list_empty(&domain->devices)) {
3834 info = list_entry(domain->devices.next,
3835 struct device_domain_info, link);
3836 unlink_domain_info(info);
3837 spin_unlock_irqrestore(&device_domain_lock, flags1);
3838
3839 iommu_disable_dev_iotlb(info);
3840 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
3841 iommu_detach_dev(iommu, info->bus, info->devfn);
3842 iommu_detach_dependent_devices(iommu, info->dev);
3843
3844 /* clear this iommu in iommu_bmp, update iommu count
3845 * and capabilities
3846 */
3847 spin_lock_irqsave(&domain->iommu_lock, flags2);
3848 if (test_and_clear_bit(iommu->seq_id,
3849 domain->iommu_bmp)) {
3850 domain->iommu_count--;
3851 domain_update_iommu_cap(domain);
3852 }
3853 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3854
3855 free_devinfo_mem(info);
3856 spin_lock_irqsave(&device_domain_lock, flags1);
3857 }
3858 spin_unlock_irqrestore(&device_domain_lock, flags1);
3859 }
3860
3861 /* domain id for virtual machine, it won't be set in context */
3862 static unsigned long vm_domid;
3863
3864 static struct dmar_domain *iommu_alloc_vm_domain(void)
3865 {
3866 struct dmar_domain *domain;
3867
3868 domain = alloc_domain_mem();
3869 if (!domain)
3870 return NULL;
3871
3872 domain->id = vm_domid++;
3873 domain->nid = -1;
3874 memset(domain->iommu_bmp, 0, sizeof(domain->iommu_bmp));
3875 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3876
3877 return domain;
3878 }
3879
3880 static int md_domain_init(struct dmar_domain *domain, int guest_width)
3881 {
3882 int adjust_width;
3883
3884 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
3885 spin_lock_init(&domain->iommu_lock);
3886
3887 domain_reserve_special_ranges(domain);
3888
3889 /* calculate AGAW */
3890 domain->gaw = guest_width;
3891 adjust_width = guestwidth_to_adjustwidth(guest_width);
3892 domain->agaw = width_to_agaw(adjust_width);
3893
3894 INIT_LIST_HEAD(&domain->devices);
3895
3896 domain->iommu_count = 0;
3897 domain->iommu_coherency = 0;
3898 domain->iommu_snooping = 0;
3899 domain->iommu_superpage = 0;
3900 domain->max_addr = 0;
3901 domain->nid = -1;
3902
3903 /* always allocate the top pgd */
3904 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
3905 if (!domain->pgd)
3906 return -ENOMEM;
3907 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3908 return 0;
3909 }
3910
3911 static void iommu_free_vm_domain(struct dmar_domain *domain)
3912 {
3913 unsigned long flags;
3914 struct dmar_drhd_unit *drhd;
3915 struct intel_iommu *iommu;
3916 unsigned long i;
3917 unsigned long ndomains;
3918
3919 for_each_drhd_unit(drhd) {
3920 if (drhd->ignored)
3921 continue;
3922 iommu = drhd->iommu;
3923
3924 ndomains = cap_ndoms(iommu->cap);
3925 for_each_set_bit(i, iommu->domain_ids, ndomains) {
3926 if (iommu->domains[i] == domain) {
3927 spin_lock_irqsave(&iommu->lock, flags);
3928 clear_bit(i, iommu->domain_ids);
3929 iommu->domains[i] = NULL;
3930 spin_unlock_irqrestore(&iommu->lock, flags);
3931 break;
3932 }
3933 }
3934 }
3935 }
3936
3937 static void vm_domain_exit(struct dmar_domain *domain)
3938 {
3939 /* Domain 0 is reserved, so dont process it */
3940 if (!domain)
3941 return;
3942
3943 vm_domain_remove_all_dev_info(domain);
3944 /* destroy iovas */
3945 put_iova_domain(&domain->iovad);
3946
3947 /* clear ptes */
3948 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3949
3950 /* free page tables */
3951 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
3952
3953 iommu_free_vm_domain(domain);
3954 free_domain_mem(domain);
3955 }
3956
3957 static int intel_iommu_domain_init(struct iommu_domain *domain)
3958 {
3959 struct dmar_domain *dmar_domain;
3960
3961 dmar_domain = iommu_alloc_vm_domain();
3962 if (!dmar_domain) {
3963 printk(KERN_ERR
3964 "intel_iommu_domain_init: dmar_domain == NULL\n");
3965 return -ENOMEM;
3966 }
3967 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
3968 printk(KERN_ERR
3969 "intel_iommu_domain_init() failed\n");
3970 vm_domain_exit(dmar_domain);
3971 return -ENOMEM;
3972 }
3973 domain_update_iommu_cap(dmar_domain);
3974 domain->priv = dmar_domain;
3975
3976 domain->geometry.aperture_start = 0;
3977 domain->geometry.aperture_end = __DOMAIN_MAX_ADDR(dmar_domain->gaw);
3978 domain->geometry.force_aperture = true;
3979
3980 return 0;
3981 }
3982
3983 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3984 {
3985 struct dmar_domain *dmar_domain = domain->priv;
3986
3987 domain->priv = NULL;
3988 vm_domain_exit(dmar_domain);
3989 }
3990
3991 static int intel_iommu_attach_device(struct iommu_domain *domain,
3992 struct device *dev)
3993 {
3994 struct dmar_domain *dmar_domain = domain->priv;
3995 struct pci_dev *pdev = to_pci_dev(dev);
3996 struct intel_iommu *iommu;
3997 int addr_width;
3998
3999 /* normally pdev is not mapped */
4000 if (unlikely(domain_context_mapped(pdev))) {
4001 struct dmar_domain *old_domain;
4002
4003 old_domain = find_domain(pdev);
4004 if (old_domain) {
4005 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
4006 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
4007 domain_remove_one_dev_info(old_domain, pdev);
4008 else
4009 domain_remove_dev_info(old_domain);
4010 }
4011 }
4012
4013 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
4014 pdev->devfn);
4015 if (!iommu)
4016 return -ENODEV;
4017
4018 /* check if this iommu agaw is sufficient for max mapped address */
4019 addr_width = agaw_to_width(iommu->agaw);
4020 if (addr_width > cap_mgaw(iommu->cap))
4021 addr_width = cap_mgaw(iommu->cap);
4022
4023 if (dmar_domain->max_addr > (1LL << addr_width)) {
4024 printk(KERN_ERR "%s: iommu width (%d) is not "
4025 "sufficient for the mapped address (%llx)\n",
4026 __func__, addr_width, dmar_domain->max_addr);
4027 return -EFAULT;
4028 }
4029 dmar_domain->gaw = addr_width;
4030
4031 /*
4032 * Knock out extra levels of page tables if necessary
4033 */
4034 while (iommu->agaw < dmar_domain->agaw) {
4035 struct dma_pte *pte;
4036
4037 pte = dmar_domain->pgd;
4038 if (dma_pte_present(pte)) {
4039 dmar_domain->pgd = (struct dma_pte *)
4040 phys_to_virt(dma_pte_addr(pte));
4041 free_pgtable_page(pte);
4042 }
4043 dmar_domain->agaw--;
4044 }
4045
4046 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
4047 }
4048
4049 static void intel_iommu_detach_device(struct iommu_domain *domain,
4050 struct device *dev)
4051 {
4052 struct dmar_domain *dmar_domain = domain->priv;
4053 struct pci_dev *pdev = to_pci_dev(dev);
4054
4055 domain_remove_one_dev_info(dmar_domain, pdev);
4056 }
4057
4058 static int intel_iommu_map(struct iommu_domain *domain,
4059 unsigned long iova, phys_addr_t hpa,
4060 size_t size, int iommu_prot)
4061 {
4062 struct dmar_domain *dmar_domain = domain->priv;
4063 u64 max_addr;
4064 int prot = 0;
4065 int ret;
4066
4067 if (iommu_prot & IOMMU_READ)
4068 prot |= DMA_PTE_READ;
4069 if (iommu_prot & IOMMU_WRITE)
4070 prot |= DMA_PTE_WRITE;
4071 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
4072 prot |= DMA_PTE_SNP;
4073
4074 max_addr = iova + size;
4075 if (dmar_domain->max_addr < max_addr) {
4076 u64 end;
4077
4078 /* check if minimum agaw is sufficient for mapped address */
4079 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4080 if (end < max_addr) {
4081 printk(KERN_ERR "%s: iommu width (%d) is not "
4082 "sufficient for the mapped address (%llx)\n",
4083 __func__, dmar_domain->gaw, max_addr);
4084 return -EFAULT;
4085 }
4086 dmar_domain->max_addr = max_addr;
4087 }
4088 /* Round up size to next multiple of PAGE_SIZE, if it and
4089 the low bits of hpa would take us onto the next page */
4090 size = aligned_nrpages(hpa, size);
4091 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4092 hpa >> VTD_PAGE_SHIFT, size, prot);
4093 return ret;
4094 }
4095
4096 static size_t intel_iommu_unmap(struct iommu_domain *domain,
4097 unsigned long iova, size_t size)
4098 {
4099 struct dmar_domain *dmar_domain = domain->priv;
4100 int order;
4101
4102 order = dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
4103 (iova + size - 1) >> VTD_PAGE_SHIFT);
4104
4105 if (dmar_domain->max_addr == iova + size)
4106 dmar_domain->max_addr = iova;
4107
4108 return PAGE_SIZE << order;
4109 }
4110
4111 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4112 unsigned long iova)
4113 {
4114 struct dmar_domain *dmar_domain = domain->priv;
4115 struct dma_pte *pte;
4116 u64 phys = 0;
4117
4118 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, 0);
4119 if (pte)
4120 phys = dma_pte_addr(pte);
4121
4122 return phys;
4123 }
4124
4125 static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
4126 unsigned long cap)
4127 {
4128 struct dmar_domain *dmar_domain = domain->priv;
4129
4130 if (cap == IOMMU_CAP_CACHE_COHERENCY)
4131 return dmar_domain->iommu_snooping;
4132 if (cap == IOMMU_CAP_INTR_REMAP)
4133 return irq_remapping_enabled;
4134
4135 return 0;
4136 }
4137
4138 static void swap_pci_ref(struct pci_dev **from, struct pci_dev *to)
4139 {
4140 pci_dev_put(*from);
4141 *from = to;
4142 }
4143
4144 #define REQ_ACS_FLAGS (PCI_ACS_SV | PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_UF)
4145
4146 static int intel_iommu_add_device(struct device *dev)
4147 {
4148 struct pci_dev *pdev = to_pci_dev(dev);
4149 struct pci_dev *bridge, *dma_pdev = NULL;
4150 struct iommu_group *group;
4151 int ret;
4152
4153 if (!device_to_iommu(pci_domain_nr(pdev->bus),
4154 pdev->bus->number, pdev->devfn))
4155 return -ENODEV;
4156
4157 bridge = pci_find_upstream_pcie_bridge(pdev);
4158 if (bridge) {
4159 if (pci_is_pcie(bridge))
4160 dma_pdev = pci_get_domain_bus_and_slot(
4161 pci_domain_nr(pdev->bus),
4162 bridge->subordinate->number, 0);
4163 if (!dma_pdev)
4164 dma_pdev = pci_dev_get(bridge);
4165 } else
4166 dma_pdev = pci_dev_get(pdev);
4167
4168 /* Account for quirked devices */
4169 swap_pci_ref(&dma_pdev, pci_get_dma_source(dma_pdev));
4170
4171 /*
4172 * If it's a multifunction device that does not support our
4173 * required ACS flags, add to the same group as function 0.
4174 */
4175 if (dma_pdev->multifunction &&
4176 !pci_acs_enabled(dma_pdev, REQ_ACS_FLAGS))
4177 swap_pci_ref(&dma_pdev,
4178 pci_get_slot(dma_pdev->bus,
4179 PCI_DEVFN(PCI_SLOT(dma_pdev->devfn),
4180 0)));
4181
4182 /*
4183 * Devices on the root bus go through the iommu. If that's not us,
4184 * find the next upstream device and test ACS up to the root bus.
4185 * Finding the next device may require skipping virtual buses.
4186 */
4187 while (!pci_is_root_bus(dma_pdev->bus)) {
4188 struct pci_bus *bus = dma_pdev->bus;
4189
4190 while (!bus->self) {
4191 if (!pci_is_root_bus(bus))
4192 bus = bus->parent;
4193 else
4194 goto root_bus;
4195 }
4196
4197 if (pci_acs_path_enabled(bus->self, NULL, REQ_ACS_FLAGS))
4198 break;
4199
4200 swap_pci_ref(&dma_pdev, pci_dev_get(bus->self));
4201 }
4202
4203 root_bus:
4204 group = iommu_group_get(&dma_pdev->dev);
4205 pci_dev_put(dma_pdev);
4206 if (!group) {
4207 group = iommu_group_alloc();
4208 if (IS_ERR(group))
4209 return PTR_ERR(group);
4210 }
4211
4212 ret = iommu_group_add_device(group, dev);
4213
4214 iommu_group_put(group);
4215 return ret;
4216 }
4217
4218 static void intel_iommu_remove_device(struct device *dev)
4219 {
4220 iommu_group_remove_device(dev);
4221 }
4222
4223 static struct iommu_ops intel_iommu_ops = {
4224 .domain_init = intel_iommu_domain_init,
4225 .domain_destroy = intel_iommu_domain_destroy,
4226 .attach_dev = intel_iommu_attach_device,
4227 .detach_dev = intel_iommu_detach_device,
4228 .map = intel_iommu_map,
4229 .unmap = intel_iommu_unmap,
4230 .iova_to_phys = intel_iommu_iova_to_phys,
4231 .domain_has_cap = intel_iommu_domain_has_cap,
4232 .add_device = intel_iommu_add_device,
4233 .remove_device = intel_iommu_remove_device,
4234 .pgsize_bitmap = INTEL_IOMMU_PGSIZES,
4235 };
4236
4237 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
4238 {
4239 /*
4240 * Mobile 4 Series Chipset neglects to set RWBF capability,
4241 * but needs it:
4242 */
4243 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
4244 rwbf_quirk = 1;
4245
4246 /* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
4247 if (dev->revision == 0x07) {
4248 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
4249 dmar_map_gfx = 0;
4250 }
4251 }
4252
4253 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4254
4255 #define GGC 0x52
4256 #define GGC_MEMORY_SIZE_MASK (0xf << 8)
4257 #define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4258 #define GGC_MEMORY_SIZE_1M (0x1 << 8)
4259 #define GGC_MEMORY_SIZE_2M (0x3 << 8)
4260 #define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4261 #define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4262 #define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4263 #define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4264
4265 static void __devinit quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4266 {
4267 unsigned short ggc;
4268
4269 if (pci_read_config_word(dev, GGC, &ggc))
4270 return;
4271
4272 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4273 printk(KERN_INFO "DMAR: BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4274 dmar_map_gfx = 0;
4275 } else if (dmar_map_gfx) {
4276 /* we have to ensure the gfx device is idle before we flush */
4277 printk(KERN_INFO "DMAR: Disabling batched IOTLB flush on Ironlake\n");
4278 intel_iommu_strict = 1;
4279 }
4280 }
4281 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4282 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4283 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4284 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4285
4286 /* On Tylersburg chipsets, some BIOSes have been known to enable the
4287 ISOCH DMAR unit for the Azalia sound device, but not give it any
4288 TLB entries, which causes it to deadlock. Check for that. We do
4289 this in a function called from init_dmars(), instead of in a PCI
4290 quirk, because we don't want to print the obnoxious "BIOS broken"
4291 message if VT-d is actually disabled.
4292 */
4293 static void __init check_tylersburg_isoch(void)
4294 {
4295 struct pci_dev *pdev;
4296 uint32_t vtisochctrl;
4297
4298 /* If there's no Azalia in the system anyway, forget it. */
4299 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4300 if (!pdev)
4301 return;
4302 pci_dev_put(pdev);
4303
4304 /* System Management Registers. Might be hidden, in which case
4305 we can't do the sanity check. But that's OK, because the
4306 known-broken BIOSes _don't_ actually hide it, so far. */
4307 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4308 if (!pdev)
4309 return;
4310
4311 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4312 pci_dev_put(pdev);
4313 return;
4314 }
4315
4316 pci_dev_put(pdev);
4317
4318 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4319 if (vtisochctrl & 1)
4320 return;
4321
4322 /* Drop all bits other than the number of TLB entries */
4323 vtisochctrl &= 0x1c;
4324
4325 /* If we have the recommended number of TLB entries (16), fine. */
4326 if (vtisochctrl == 0x10)
4327 return;
4328
4329 /* Zero TLB entries? You get to ride the short bus to school. */
4330 if (!vtisochctrl) {
4331 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4332 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4333 dmi_get_system_info(DMI_BIOS_VENDOR),
4334 dmi_get_system_info(DMI_BIOS_VERSION),
4335 dmi_get_system_info(DMI_PRODUCT_VERSION));
4336 iommu_identity_mapping |= IDENTMAP_AZALIA;
4337 return;
4338 }
4339
4340 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4341 vtisochctrl);
4342 }
Linus' kernel tree