?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Linux 2.6.28-rc7
[linux-2.6/sactl.git] / drivers / pci / intel-iommu.c
blob5c8baa43ac9c5ed86bde043702af795d37dcf2b8
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
22 */
23
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/sysdev.h>
31 #include <linux/spinlock.h>
32 #include <linux/pci.h>
33 #include <linux/dmar.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/mempool.h>
36 #include <linux/timer.h>
37 #include <linux/iova.h>
38 #include <linux/intel-iommu.h>
39 #include <asm/cacheflush.h>
40 #include <asm/iommu.h>
41 #include "pci.h"
42
43 #define ROOT_SIZE VTD_PAGE_SIZE
44 #define CONTEXT_SIZE VTD_PAGE_SIZE
45
46 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
47 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
48
49 #define IOAPIC_RANGE_START (0xfee00000)
50 #define IOAPIC_RANGE_END (0xfeefffff)
51 #define IOVA_START_ADDR (0x1000)
52
53 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
54
55 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
56
57
58 static void flush_unmaps_timeout(unsigned long data);
59
60 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
61
62 #define HIGH_WATER_MARK 250
63 struct deferred_flush_tables {
64 int next;
65 struct iova *iova[HIGH_WATER_MARK];
66 struct dmar_domain *domain[HIGH_WATER_MARK];
67 };
68
69 static struct deferred_flush_tables *deferred_flush;
70
71 /* bitmap for indexing intel_iommus */
72 static int g_num_of_iommus;
73
74 static DEFINE_SPINLOCK(async_umap_flush_lock);
75 static LIST_HEAD(unmaps_to_do);
76
77 static int timer_on;
78 static long list_size;
79
80 static void domain_remove_dev_info(struct dmar_domain *domain);
81
82 int dmar_disabled;
83 static int __initdata dmar_map_gfx = 1;
84 static int dmar_forcedac;
85 static int intel_iommu_strict;
86
87 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
88 static DEFINE_SPINLOCK(device_domain_lock);
89 static LIST_HEAD(device_domain_list);
90
91 static int __init intel_iommu_setup(char *str)
92 {
93 if (!str)
94 return -EINVAL;
95 while (*str) {
96 if (!strncmp(str, "off", 3)) {
97 dmar_disabled = 1;
98 printk(KERN_INFO"Intel-IOMMU: disabled\n");
99 } else if (!strncmp(str, "igfx_off", 8)) {
100 dmar_map_gfx = 0;
101 printk(KERN_INFO
102 "Intel-IOMMU: disable GFX device mapping\n");
103 } else if (!strncmp(str, "forcedac", 8)) {
104 printk(KERN_INFO
105 "Intel-IOMMU: Forcing DAC for PCI devices\n");
106 dmar_forcedac = 1;
107 } else if (!strncmp(str, "strict", 6)) {
108 printk(KERN_INFO
109 "Intel-IOMMU: disable batched IOTLB flush\n");
110 intel_iommu_strict = 1;
111 }
112
113 str += strcspn(str, ",");
114 while (*str == ',')
115 str++;
116 }
117 return 0;
118 }
119 __setup("intel_iommu=", intel_iommu_setup);
120
121 static struct kmem_cache *iommu_domain_cache;
122 static struct kmem_cache *iommu_devinfo_cache;
123 static struct kmem_cache *iommu_iova_cache;
124
125 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
126 {
127 unsigned int flags;
128 void *vaddr;
129
130 /* trying to avoid low memory issues */
131 flags = current->flags & PF_MEMALLOC;
132 current->flags |= PF_MEMALLOC;
133 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
134 current->flags &= (~PF_MEMALLOC | flags);
135 return vaddr;
136 }
137
138
139 static inline void *alloc_pgtable_page(void)
140 {
141 unsigned int flags;
142 void *vaddr;
143
144 /* trying to avoid low memory issues */
145 flags = current->flags & PF_MEMALLOC;
146 current->flags |= PF_MEMALLOC;
147 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
148 current->flags &= (~PF_MEMALLOC | flags);
149 return vaddr;
150 }
151
152 static inline void free_pgtable_page(void *vaddr)
153 {
154 free_page((unsigned long)vaddr);
155 }
156
157 static inline void *alloc_domain_mem(void)
158 {
159 return iommu_kmem_cache_alloc(iommu_domain_cache);
160 }
161
162 static void free_domain_mem(void *vaddr)
163 {
164 kmem_cache_free(iommu_domain_cache, vaddr);
165 }
166
167 static inline void * alloc_devinfo_mem(void)
168 {
169 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
170 }
171
172 static inline void free_devinfo_mem(void *vaddr)
173 {
174 kmem_cache_free(iommu_devinfo_cache, vaddr);
175 }
176
177 struct iova *alloc_iova_mem(void)
178 {
179 return iommu_kmem_cache_alloc(iommu_iova_cache);
180 }
181
182 void free_iova_mem(struct iova *iova)
183 {
184 kmem_cache_free(iommu_iova_cache, iova);
185 }
186
187 /* Gets context entry for a given bus and devfn */
188 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
189 u8 bus, u8 devfn)
190 {
191 struct root_entry *root;
192 struct context_entry *context;
193 unsigned long phy_addr;
194 unsigned long flags;
195
196 spin_lock_irqsave(&iommu->lock, flags);
197 root = &iommu->root_entry[bus];
198 context = get_context_addr_from_root(root);
199 if (!context) {
200 context = (struct context_entry *)alloc_pgtable_page();
201 if (!context) {
202 spin_unlock_irqrestore(&iommu->lock, flags);
203 return NULL;
204 }
205 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
206 phy_addr = virt_to_phys((void *)context);
207 set_root_value(root, phy_addr);
208 set_root_present(root);
209 __iommu_flush_cache(iommu, root, sizeof(*root));
210 }
211 spin_unlock_irqrestore(&iommu->lock, flags);
212 return &context[devfn];
213 }
214
215 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
216 {
217 struct root_entry *root;
218 struct context_entry *context;
219 int ret;
220 unsigned long flags;
221
222 spin_lock_irqsave(&iommu->lock, flags);
223 root = &iommu->root_entry[bus];
224 context = get_context_addr_from_root(root);
225 if (!context) {
226 ret = 0;
227 goto out;
228 }
229 ret = context_present(context[devfn]);
230 out:
231 spin_unlock_irqrestore(&iommu->lock, flags);
232 return ret;
233 }
234
235 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
236 {
237 struct root_entry *root;
238 struct context_entry *context;
239 unsigned long flags;
240
241 spin_lock_irqsave(&iommu->lock, flags);
242 root = &iommu->root_entry[bus];
243 context = get_context_addr_from_root(root);
244 if (context) {
245 context_clear_entry(context[devfn]);
246 __iommu_flush_cache(iommu, &context[devfn], \
247 sizeof(*context));
248 }
249 spin_unlock_irqrestore(&iommu->lock, flags);
250 }
251
252 static void free_context_table(struct intel_iommu *iommu)
253 {
254 struct root_entry *root;
255 int i;
256 unsigned long flags;
257 struct context_entry *context;
258
259 spin_lock_irqsave(&iommu->lock, flags);
260 if (!iommu->root_entry) {
261 goto out;
262 }
263 for (i = 0; i < ROOT_ENTRY_NR; i++) {
264 root = &iommu->root_entry[i];
265 context = get_context_addr_from_root(root);
266 if (context)
267 free_pgtable_page(context);
268 }
269 free_pgtable_page(iommu->root_entry);
270 iommu->root_entry = NULL;
271 out:
272 spin_unlock_irqrestore(&iommu->lock, flags);
273 }
274
275 /* page table handling */
276 #define LEVEL_STRIDE (9)
277 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
278
279 static inline int agaw_to_level(int agaw)
280 {
281 return agaw + 2;
282 }
283
284 static inline int agaw_to_width(int agaw)
285 {
286 return 30 + agaw * LEVEL_STRIDE;
287
288 }
289
290 static inline int width_to_agaw(int width)
291 {
292 return (width - 30) / LEVEL_STRIDE;
293 }
294
295 static inline unsigned int level_to_offset_bits(int level)
296 {
297 return (12 + (level - 1) * LEVEL_STRIDE);
298 }
299
300 static inline int address_level_offset(u64 addr, int level)
301 {
302 return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
303 }
304
305 static inline u64 level_mask(int level)
306 {
307 return ((u64)-1 << level_to_offset_bits(level));
308 }
309
310 static inline u64 level_size(int level)
311 {
312 return ((u64)1 << level_to_offset_bits(level));
313 }
314
315 static inline u64 align_to_level(u64 addr, int level)
316 {
317 return ((addr + level_size(level) - 1) & level_mask(level));
318 }
319
320 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
321 {
322 int addr_width = agaw_to_width(domain->agaw);
323 struct dma_pte *parent, *pte = NULL;
324 int level = agaw_to_level(domain->agaw);
325 int offset;
326 unsigned long flags;
327
328 BUG_ON(!domain->pgd);
329
330 addr &= (((u64)1) << addr_width) - 1;
331 parent = domain->pgd;
332
333 spin_lock_irqsave(&domain->mapping_lock, flags);
334 while (level > 0) {
335 void *tmp_page;
336
337 offset = address_level_offset(addr, level);
338 pte = &parent[offset];
339 if (level == 1)
340 break;
341
342 if (!dma_pte_present(*pte)) {
343 tmp_page = alloc_pgtable_page();
344
345 if (!tmp_page) {
346 spin_unlock_irqrestore(&domain->mapping_lock,
347 flags);
348 return NULL;
349 }
350 __iommu_flush_cache(domain->iommu, tmp_page,
351 PAGE_SIZE);
352 dma_set_pte_addr(*pte, virt_to_phys(tmp_page));
353 /*
354 * high level table always sets r/w, last level page
355 * table control read/write
356 */
357 dma_set_pte_readable(*pte);
358 dma_set_pte_writable(*pte);
359 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
360 }
361 parent = phys_to_virt(dma_pte_addr(*pte));
362 level--;
363 }
364
365 spin_unlock_irqrestore(&domain->mapping_lock, flags);
366 return pte;
367 }
368
369 /* return address's pte at specific level */
370 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
371 int level)
372 {
373 struct dma_pte *parent, *pte = NULL;
374 int total = agaw_to_level(domain->agaw);
375 int offset;
376
377 parent = domain->pgd;
378 while (level <= total) {
379 offset = address_level_offset(addr, total);
380 pte = &parent[offset];
381 if (level == total)
382 return pte;
383
384 if (!dma_pte_present(*pte))
385 break;
386 parent = phys_to_virt(dma_pte_addr(*pte));
387 total--;
388 }
389 return NULL;
390 }
391
392 /* clear one page's page table */
393 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
394 {
395 struct dma_pte *pte = NULL;
396
397 /* get last level pte */
398 pte = dma_addr_level_pte(domain, addr, 1);
399
400 if (pte) {
401 dma_clear_pte(*pte);
402 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
403 }
404 }
405
406 /* clear last level pte, a tlb flush should be followed */
407 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
408 {
409 int addr_width = agaw_to_width(domain->agaw);
410
411 start &= (((u64)1) << addr_width) - 1;
412 end &= (((u64)1) << addr_width) - 1;
413 /* in case it's partial page */
414 start = PAGE_ALIGN(start);
415 end &= PAGE_MASK;
416
417 /* we don't need lock here, nobody else touches the iova range */
418 while (start < end) {
419 dma_pte_clear_one(domain, start);
420 start += VTD_PAGE_SIZE;
421 }
422 }
423
424 /* free page table pages. last level pte should already be cleared */
425 static void dma_pte_free_pagetable(struct dmar_domain *domain,
426 u64 start, u64 end)
427 {
428 int addr_width = agaw_to_width(domain->agaw);
429 struct dma_pte *pte;
430 int total = agaw_to_level(domain->agaw);
431 int level;
432 u64 tmp;
433
434 start &= (((u64)1) << addr_width) - 1;
435 end &= (((u64)1) << addr_width) - 1;
436
437 /* we don't need lock here, nobody else touches the iova range */
438 level = 2;
439 while (level <= total) {
440 tmp = align_to_level(start, level);
441 if (tmp >= end || (tmp + level_size(level) > end))
442 return;
443
444 while (tmp < end) {
445 pte = dma_addr_level_pte(domain, tmp, level);
446 if (pte) {
447 free_pgtable_page(
448 phys_to_virt(dma_pte_addr(*pte)));
449 dma_clear_pte(*pte);
450 __iommu_flush_cache(domain->iommu,
451 pte, sizeof(*pte));
452 }
453 tmp += level_size(level);
454 }
455 level++;
456 }
457 /* free pgd */
458 if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
459 free_pgtable_page(domain->pgd);
460 domain->pgd = NULL;
461 }
462 }
463
464 /* iommu handling */
465 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
466 {
467 struct root_entry *root;
468 unsigned long flags;
469
470 root = (struct root_entry *)alloc_pgtable_page();
471 if (!root)
472 return -ENOMEM;
473
474 __iommu_flush_cache(iommu, root, ROOT_SIZE);
475
476 spin_lock_irqsave(&iommu->lock, flags);
477 iommu->root_entry = root;
478 spin_unlock_irqrestore(&iommu->lock, flags);
479
480 return 0;
481 }
482
483 static void iommu_set_root_entry(struct intel_iommu *iommu)
484 {
485 void *addr;
486 u32 cmd, sts;
487 unsigned long flag;
488
489 addr = iommu->root_entry;
490
491 spin_lock_irqsave(&iommu->register_lock, flag);
492 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
493
494 cmd = iommu->gcmd | DMA_GCMD_SRTP;
495 writel(cmd, iommu->reg + DMAR_GCMD_REG);
496
497 /* Make sure hardware complete it */
498 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
499 readl, (sts & DMA_GSTS_RTPS), sts);
500
501 spin_unlock_irqrestore(&iommu->register_lock, flag);
502 }
503
504 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
505 {
506 u32 val;
507 unsigned long flag;
508
509 if (!cap_rwbf(iommu->cap))
510 return;
511 val = iommu->gcmd | DMA_GCMD_WBF;
512
513 spin_lock_irqsave(&iommu->register_lock, flag);
514 writel(val, iommu->reg + DMAR_GCMD_REG);
515
516 /* Make sure hardware complete it */
517 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
518 readl, (!(val & DMA_GSTS_WBFS)), val);
519
520 spin_unlock_irqrestore(&iommu->register_lock, flag);
521 }
522
523 /* return value determine if we need a write buffer flush */
524 static int __iommu_flush_context(struct intel_iommu *iommu,
525 u16 did, u16 source_id, u8 function_mask, u64 type,
526 int non_present_entry_flush)
527 {
528 u64 val = 0;
529 unsigned long flag;
530
531 /*
532 * In the non-present entry flush case, if hardware doesn't cache
533 * non-present entry we do nothing and if hardware cache non-present
534 * entry, we flush entries of domain 0 (the domain id is used to cache
535 * any non-present entries)
536 */
537 if (non_present_entry_flush) {
538 if (!cap_caching_mode(iommu->cap))
539 return 1;
540 else
541 did = 0;
542 }
543
544 switch (type) {
545 case DMA_CCMD_GLOBAL_INVL:
546 val = DMA_CCMD_GLOBAL_INVL;
547 break;
548 case DMA_CCMD_DOMAIN_INVL:
549 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
550 break;
551 case DMA_CCMD_DEVICE_INVL:
552 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
553 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
554 break;
555 default:
556 BUG();
557 }
558 val |= DMA_CCMD_ICC;
559
560 spin_lock_irqsave(&iommu->register_lock, flag);
561 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
562
563 /* Make sure hardware complete it */
564 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
565 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
566
567 spin_unlock_irqrestore(&iommu->register_lock, flag);
568
569 /* flush context entry will implicitly flush write buffer */
570 return 0;
571 }
572
573 /* return value determine if we need a write buffer flush */
574 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
575 u64 addr, unsigned int size_order, u64 type,
576 int non_present_entry_flush)
577 {
578 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
579 u64 val = 0, val_iva = 0;
580 unsigned long flag;
581
582 /*
583 * In the non-present entry flush case, if hardware doesn't cache
584 * non-present entry we do nothing and if hardware cache non-present
585 * entry, we flush entries of domain 0 (the domain id is used to cache
586 * any non-present entries)
587 */
588 if (non_present_entry_flush) {
589 if (!cap_caching_mode(iommu->cap))
590 return 1;
591 else
592 did = 0;
593 }
594
595 switch (type) {
596 case DMA_TLB_GLOBAL_FLUSH:
597 /* global flush doesn't need set IVA_REG */
598 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
599 break;
600 case DMA_TLB_DSI_FLUSH:
601 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
602 break;
603 case DMA_TLB_PSI_FLUSH:
604 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
605 /* Note: always flush non-leaf currently */
606 val_iva = size_order | addr;
607 break;
608 default:
609 BUG();
610 }
611 /* Note: set drain read/write */
612 #if 0
613 /*
614 * This is probably to be super secure.. Looks like we can
615 * ignore it without any impact.
616 */
617 if (cap_read_drain(iommu->cap))
618 val |= DMA_TLB_READ_DRAIN;
619 #endif
620 if (cap_write_drain(iommu->cap))
621 val |= DMA_TLB_WRITE_DRAIN;
622
623 spin_lock_irqsave(&iommu->register_lock, flag);
624 /* Note: Only uses first TLB reg currently */
625 if (val_iva)
626 dmar_writeq(iommu->reg + tlb_offset, val_iva);
627 dmar_writeq(iommu->reg + tlb_offset + 8, val);
628
629 /* Make sure hardware complete it */
630 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
631 dmar_readq, (!(val & DMA_TLB_IVT)), val);
632
633 spin_unlock_irqrestore(&iommu->register_lock, flag);
634
635 /* check IOTLB invalidation granularity */
636 if (DMA_TLB_IAIG(val) == 0)
637 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
638 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
639 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
640 (unsigned long long)DMA_TLB_IIRG(type),
641 (unsigned long long)DMA_TLB_IAIG(val));
642 /* flush iotlb entry will implicitly flush write buffer */
643 return 0;
644 }
645
646 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
647 u64 addr, unsigned int pages, int non_present_entry_flush)
648 {
649 unsigned int mask;
650
651 BUG_ON(addr & (~VTD_PAGE_MASK));
652 BUG_ON(pages == 0);
653
654 /* Fallback to domain selective flush if no PSI support */
655 if (!cap_pgsel_inv(iommu->cap))
656 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
657 DMA_TLB_DSI_FLUSH,
658 non_present_entry_flush);
659
660 /*
661 * PSI requires page size to be 2 ^ x, and the base address is naturally
662 * aligned to the size
663 */
664 mask = ilog2(__roundup_pow_of_two(pages));
665 /* Fallback to domain selective flush if size is too big */
666 if (mask > cap_max_amask_val(iommu->cap))
667 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
668 DMA_TLB_DSI_FLUSH, non_present_entry_flush);
669
670 return iommu->flush.flush_iotlb(iommu, did, addr, mask,
671 DMA_TLB_PSI_FLUSH,
672 non_present_entry_flush);
673 }
674
675 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
676 {
677 u32 pmen;
678 unsigned long flags;
679
680 spin_lock_irqsave(&iommu->register_lock, flags);
681 pmen = readl(iommu->reg + DMAR_PMEN_REG);
682 pmen &= ~DMA_PMEN_EPM;
683 writel(pmen, iommu->reg + DMAR_PMEN_REG);
684
685 /* wait for the protected region status bit to clear */
686 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
687 readl, !(pmen & DMA_PMEN_PRS), pmen);
688
689 spin_unlock_irqrestore(&iommu->register_lock, flags);
690 }
691
692 static int iommu_enable_translation(struct intel_iommu *iommu)
693 {
694 u32 sts;
695 unsigned long flags;
696
697 spin_lock_irqsave(&iommu->register_lock, flags);
698 writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
699
700 /* Make sure hardware complete it */
701 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
702 readl, (sts & DMA_GSTS_TES), sts);
703
704 iommu->gcmd |= DMA_GCMD_TE;
705 spin_unlock_irqrestore(&iommu->register_lock, flags);
706 return 0;
707 }
708
709 static int iommu_disable_translation(struct intel_iommu *iommu)
710 {
711 u32 sts;
712 unsigned long flag;
713
714 spin_lock_irqsave(&iommu->register_lock, flag);
715 iommu->gcmd &= ~DMA_GCMD_TE;
716 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
717
718 /* Make sure hardware complete it */
719 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
720 readl, (!(sts & DMA_GSTS_TES)), sts);
721
722 spin_unlock_irqrestore(&iommu->register_lock, flag);
723 return 0;
724 }
725
726 /* iommu interrupt handling. Most stuff are MSI-like. */
727
728 static const char *fault_reason_strings[] =
729 {
730 "Software",
731 "Present bit in root entry is clear",
732 "Present bit in context entry is clear",
733 "Invalid context entry",
734 "Access beyond MGAW",
735 "PTE Write access is not set",
736 "PTE Read access is not set",
737 "Next page table ptr is invalid",
738 "Root table address invalid",
739 "Context table ptr is invalid",
740 "non-zero reserved fields in RTP",
741 "non-zero reserved fields in CTP",
742 "non-zero reserved fields in PTE",
743 };
744 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
745
746 const char *dmar_get_fault_reason(u8 fault_reason)
747 {
748 if (fault_reason > MAX_FAULT_REASON_IDX)
749 return "Unknown";
750 else
751 return fault_reason_strings[fault_reason];
752 }
753
754 void dmar_msi_unmask(unsigned int irq)
755 {
756 struct intel_iommu *iommu = get_irq_data(irq);
757 unsigned long flag;
758
759 /* unmask it */
760 spin_lock_irqsave(&iommu->register_lock, flag);
761 writel(0, iommu->reg + DMAR_FECTL_REG);
762 /* Read a reg to force flush the post write */
763 readl(iommu->reg + DMAR_FECTL_REG);
764 spin_unlock_irqrestore(&iommu->register_lock, flag);
765 }
766
767 void dmar_msi_mask(unsigned int irq)
768 {
769 unsigned long flag;
770 struct intel_iommu *iommu = get_irq_data(irq);
771
772 /* mask it */
773 spin_lock_irqsave(&iommu->register_lock, flag);
774 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
775 /* Read a reg to force flush the post write */
776 readl(iommu->reg + DMAR_FECTL_REG);
777 spin_unlock_irqrestore(&iommu->register_lock, flag);
778 }
779
780 void dmar_msi_write(int irq, struct msi_msg *msg)
781 {
782 struct intel_iommu *iommu = get_irq_data(irq);
783 unsigned long flag;
784
785 spin_lock_irqsave(&iommu->register_lock, flag);
786 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
787 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
788 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
789 spin_unlock_irqrestore(&iommu->register_lock, flag);
790 }
791
792 void dmar_msi_read(int irq, struct msi_msg *msg)
793 {
794 struct intel_iommu *iommu = get_irq_data(irq);
795 unsigned long flag;
796
797 spin_lock_irqsave(&iommu->register_lock, flag);
798 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
799 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
800 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
801 spin_unlock_irqrestore(&iommu->register_lock, flag);
802 }
803
804 static int iommu_page_fault_do_one(struct intel_iommu *iommu, int type,
805 u8 fault_reason, u16 source_id, unsigned long long addr)
806 {
807 const char *reason;
808
809 reason = dmar_get_fault_reason(fault_reason);
810
811 printk(KERN_ERR
812 "DMAR:[%s] Request device [%02x:%02x.%d] "
813 "fault addr %llx \n"
814 "DMAR:[fault reason %02d] %s\n",
815 (type ? "DMA Read" : "DMA Write"),
816 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
817 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
818 return 0;
819 }
820
821 #define PRIMARY_FAULT_REG_LEN (16)
822 static irqreturn_t iommu_page_fault(int irq, void *dev_id)
823 {
824 struct intel_iommu *iommu = dev_id;
825 int reg, fault_index;
826 u32 fault_status;
827 unsigned long flag;
828
829 spin_lock_irqsave(&iommu->register_lock, flag);
830 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
831
832 /* TBD: ignore advanced fault log currently */
833 if (!(fault_status & DMA_FSTS_PPF))
834 goto clear_overflow;
835
836 fault_index = dma_fsts_fault_record_index(fault_status);
837 reg = cap_fault_reg_offset(iommu->cap);
838 while (1) {
839 u8 fault_reason;
840 u16 source_id;
841 u64 guest_addr;
842 int type;
843 u32 data;
844
845 /* highest 32 bits */
846 data = readl(iommu->reg + reg +
847 fault_index * PRIMARY_FAULT_REG_LEN + 12);
848 if (!(data & DMA_FRCD_F))
849 break;
850
851 fault_reason = dma_frcd_fault_reason(data);
852 type = dma_frcd_type(data);
853
854 data = readl(iommu->reg + reg +
855 fault_index * PRIMARY_FAULT_REG_LEN + 8);
856 source_id = dma_frcd_source_id(data);
857
858 guest_addr = dmar_readq(iommu->reg + reg +
859 fault_index * PRIMARY_FAULT_REG_LEN);
860 guest_addr = dma_frcd_page_addr(guest_addr);
861 /* clear the fault */
862 writel(DMA_FRCD_F, iommu->reg + reg +
863 fault_index * PRIMARY_FAULT_REG_LEN + 12);
864
865 spin_unlock_irqrestore(&iommu->register_lock, flag);
866
867 iommu_page_fault_do_one(iommu, type, fault_reason,
868 source_id, guest_addr);
869
870 fault_index++;
871 if (fault_index > cap_num_fault_regs(iommu->cap))
872 fault_index = 0;
873 spin_lock_irqsave(&iommu->register_lock, flag);
874 }
875 clear_overflow:
876 /* clear primary fault overflow */
877 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
878 if (fault_status & DMA_FSTS_PFO)
879 writel(DMA_FSTS_PFO, iommu->reg + DMAR_FSTS_REG);
880
881 spin_unlock_irqrestore(&iommu->register_lock, flag);
882 return IRQ_HANDLED;
883 }
884
885 int dmar_set_interrupt(struct intel_iommu *iommu)
886 {
887 int irq, ret;
888
889 irq = create_irq();
890 if (!irq) {
891 printk(KERN_ERR "IOMMU: no free vectors\n");
892 return -EINVAL;
893 }
894
895 set_irq_data(irq, iommu);
896 iommu->irq = irq;
897
898 ret = arch_setup_dmar_msi(irq);
899 if (ret) {
900 set_irq_data(irq, NULL);
901 iommu->irq = 0;
902 destroy_irq(irq);
903 return 0;
904 }
905
906 /* Force fault register is cleared */
907 iommu_page_fault(irq, iommu);
908
909 ret = request_irq(irq, iommu_page_fault, 0, iommu->name, iommu);
910 if (ret)
911 printk(KERN_ERR "IOMMU: can't request irq\n");
912 return ret;
913 }
914
915 static int iommu_init_domains(struct intel_iommu *iommu)
916 {
917 unsigned long ndomains;
918 unsigned long nlongs;
919
920 ndomains = cap_ndoms(iommu->cap);
921 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
922 nlongs = BITS_TO_LONGS(ndomains);
923
924 /* TBD: there might be 64K domains,
925 * consider other allocation for future chip
926 */
927 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
928 if (!iommu->domain_ids) {
929 printk(KERN_ERR "Allocating domain id array failed\n");
930 return -ENOMEM;
931 }
932 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
933 GFP_KERNEL);
934 if (!iommu->domains) {
935 printk(KERN_ERR "Allocating domain array failed\n");
936 kfree(iommu->domain_ids);
937 return -ENOMEM;
938 }
939
940 spin_lock_init(&iommu->lock);
941
942 /*
943 * if Caching mode is set, then invalid translations are tagged
944 * with domainid 0. Hence we need to pre-allocate it.
945 */
946 if (cap_caching_mode(iommu->cap))
947 set_bit(0, iommu->domain_ids);
948 return 0;
949 }
950
951
952 static void domain_exit(struct dmar_domain *domain);
953
954 void free_dmar_iommu(struct intel_iommu *iommu)
955 {
956 struct dmar_domain *domain;
957 int i;
958
959 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
960 for (; i < cap_ndoms(iommu->cap); ) {
961 domain = iommu->domains[i];
962 clear_bit(i, iommu->domain_ids);
963 domain_exit(domain);
964 i = find_next_bit(iommu->domain_ids,
965 cap_ndoms(iommu->cap), i+1);
966 }
967
968 if (iommu->gcmd & DMA_GCMD_TE)
969 iommu_disable_translation(iommu);
970
971 if (iommu->irq) {
972 set_irq_data(iommu->irq, NULL);
973 /* This will mask the irq */
974 free_irq(iommu->irq, iommu);
975 destroy_irq(iommu->irq);
976 }
977
978 kfree(iommu->domains);
979 kfree(iommu->domain_ids);
980
981 /* free context mapping */
982 free_context_table(iommu);
983 }
984
985 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
986 {
987 unsigned long num;
988 unsigned long ndomains;
989 struct dmar_domain *domain;
990 unsigned long flags;
991
992 domain = alloc_domain_mem();
993 if (!domain)
994 return NULL;
995
996 ndomains = cap_ndoms(iommu->cap);
997
998 spin_lock_irqsave(&iommu->lock, flags);
999 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1000 if (num >= ndomains) {
1001 spin_unlock_irqrestore(&iommu->lock, flags);
1002 free_domain_mem(domain);
1003 printk(KERN_ERR "IOMMU: no free domain ids\n");
1004 return NULL;
1005 }
1006
1007 set_bit(num, iommu->domain_ids);
1008 domain->id = num;
1009 domain->iommu = iommu;
1010 iommu->domains[num] = domain;
1011 spin_unlock_irqrestore(&iommu->lock, flags);
1012
1013 return domain;
1014 }
1015
1016 static void iommu_free_domain(struct dmar_domain *domain)
1017 {
1018 unsigned long flags;
1019
1020 spin_lock_irqsave(&domain->iommu->lock, flags);
1021 clear_bit(domain->id, domain->iommu->domain_ids);
1022 spin_unlock_irqrestore(&domain->iommu->lock, flags);
1023 }
1024
1025 static struct iova_domain reserved_iova_list;
1026 static struct lock_class_key reserved_alloc_key;
1027 static struct lock_class_key reserved_rbtree_key;
1028
1029 static void dmar_init_reserved_ranges(void)
1030 {
1031 struct pci_dev *pdev = NULL;
1032 struct iova *iova;
1033 int i;
1034 u64 addr, size;
1035
1036 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1037
1038 lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1039 &reserved_alloc_key);
1040 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1041 &reserved_rbtree_key);
1042
1043 /* IOAPIC ranges shouldn't be accessed by DMA */
1044 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1045 IOVA_PFN(IOAPIC_RANGE_END));
1046 if (!iova)
1047 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1048
1049 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1050 for_each_pci_dev(pdev) {
1051 struct resource *r;
1052
1053 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1054 r = &pdev->resource[i];
1055 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1056 continue;
1057 addr = r->start;
1058 addr &= PAGE_MASK;
1059 size = r->end - addr;
1060 size = PAGE_ALIGN(size);
1061 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1062 IOVA_PFN(size + addr) - 1);
1063 if (!iova)
1064 printk(KERN_ERR "Reserve iova failed\n");
1065 }
1066 }
1067
1068 }
1069
1070 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1071 {
1072 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1073 }
1074
1075 static inline int guestwidth_to_adjustwidth(int gaw)
1076 {
1077 int agaw;
1078 int r = (gaw - 12) % 9;
1079
1080 if (r == 0)
1081 agaw = gaw;
1082 else
1083 agaw = gaw + 9 - r;
1084 if (agaw > 64)
1085 agaw = 64;
1086 return agaw;
1087 }
1088
1089 static int domain_init(struct dmar_domain *domain, int guest_width)
1090 {
1091 struct intel_iommu *iommu;
1092 int adjust_width, agaw;
1093 unsigned long sagaw;
1094
1095 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1096 spin_lock_init(&domain->mapping_lock);
1097
1098 domain_reserve_special_ranges(domain);
1099
1100 /* calculate AGAW */
1101 iommu = domain->iommu;
1102 if (guest_width > cap_mgaw(iommu->cap))
1103 guest_width = cap_mgaw(iommu->cap);
1104 domain->gaw = guest_width;
1105 adjust_width = guestwidth_to_adjustwidth(guest_width);
1106 agaw = width_to_agaw(adjust_width);
1107 sagaw = cap_sagaw(iommu->cap);
1108 if (!test_bit(agaw, &sagaw)) {
1109 /* hardware doesn't support it, choose a bigger one */
1110 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1111 agaw = find_next_bit(&sagaw, 5, agaw);
1112 if (agaw >= 5)
1113 return -ENODEV;
1114 }
1115 domain->agaw = agaw;
1116 INIT_LIST_HEAD(&domain->devices);
1117
1118 /* always allocate the top pgd */
1119 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1120 if (!domain->pgd)
1121 return -ENOMEM;
1122 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1123 return 0;
1124 }
1125
1126 static void domain_exit(struct dmar_domain *domain)
1127 {
1128 u64 end;
1129
1130 /* Domain 0 is reserved, so dont process it */
1131 if (!domain)
1132 return;
1133
1134 domain_remove_dev_info(domain);
1135 /* destroy iovas */
1136 put_iova_domain(&domain->iovad);
1137 end = DOMAIN_MAX_ADDR(domain->gaw);
1138 end = end & (~PAGE_MASK);
1139
1140 /* clear ptes */
1141 dma_pte_clear_range(domain, 0, end);
1142
1143 /* free page tables */
1144 dma_pte_free_pagetable(domain, 0, end);
1145
1146 iommu_free_domain(domain);
1147 free_domain_mem(domain);
1148 }
1149
1150 static int domain_context_mapping_one(struct dmar_domain *domain,
1151 u8 bus, u8 devfn)
1152 {
1153 struct context_entry *context;
1154 struct intel_iommu *iommu = domain->iommu;
1155 unsigned long flags;
1156
1157 pr_debug("Set context mapping for %02x:%02x.%d\n",
1158 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1159 BUG_ON(!domain->pgd);
1160 context = device_to_context_entry(iommu, bus, devfn);
1161 if (!context)
1162 return -ENOMEM;
1163 spin_lock_irqsave(&iommu->lock, flags);
1164 if (context_present(*context)) {
1165 spin_unlock_irqrestore(&iommu->lock, flags);
1166 return 0;
1167 }
1168
1169 context_set_domain_id(*context, domain->id);
1170 context_set_address_width(*context, domain->agaw);
1171 context_set_address_root(*context, virt_to_phys(domain->pgd));
1172 context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1173 context_set_fault_enable(*context);
1174 context_set_present(*context);
1175 __iommu_flush_cache(iommu, context, sizeof(*context));
1176
1177 /* it's a non-present to present mapping */
1178 if (iommu->flush.flush_context(iommu, domain->id,
1179 (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT,
1180 DMA_CCMD_DEVICE_INVL, 1))
1181 iommu_flush_write_buffer(iommu);
1182 else
1183 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH, 0);
1184
1185 spin_unlock_irqrestore(&iommu->lock, flags);
1186 return 0;
1187 }
1188
1189 static int
1190 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1191 {
1192 int ret;
1193 struct pci_dev *tmp, *parent;
1194
1195 ret = domain_context_mapping_one(domain, pdev->bus->number,
1196 pdev->devfn);
1197 if (ret)
1198 return ret;
1199
1200 /* dependent device mapping */
1201 tmp = pci_find_upstream_pcie_bridge(pdev);
1202 if (!tmp)
1203 return 0;
1204 /* Secondary interface's bus number and devfn 0 */
1205 parent = pdev->bus->self;
1206 while (parent != tmp) {
1207 ret = domain_context_mapping_one(domain, parent->bus->number,
1208 parent->devfn);
1209 if (ret)
1210 return ret;
1211 parent = parent->bus->self;
1212 }
1213 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1214 return domain_context_mapping_one(domain,
1215 tmp->subordinate->number, 0);
1216 else /* this is a legacy PCI bridge */
1217 return domain_context_mapping_one(domain,
1218 tmp->bus->number, tmp->devfn);
1219 }
1220
1221 static int domain_context_mapped(struct dmar_domain *domain,
1222 struct pci_dev *pdev)
1223 {
1224 int ret;
1225 struct pci_dev *tmp, *parent;
1226
1227 ret = device_context_mapped(domain->iommu,
1228 pdev->bus->number, pdev->devfn);
1229 if (!ret)
1230 return ret;
1231 /* dependent device mapping */
1232 tmp = pci_find_upstream_pcie_bridge(pdev);
1233 if (!tmp)
1234 return ret;
1235 /* Secondary interface's bus number and devfn 0 */
1236 parent = pdev->bus->self;
1237 while (parent != tmp) {
1238 ret = device_context_mapped(domain->iommu, parent->bus->number,
1239 parent->devfn);
1240 if (!ret)
1241 return ret;
1242 parent = parent->bus->self;
1243 }
1244 if (tmp->is_pcie)
1245 return device_context_mapped(domain->iommu,
1246 tmp->subordinate->number, 0);
1247 else
1248 return device_context_mapped(domain->iommu,
1249 tmp->bus->number, tmp->devfn);
1250 }
1251
1252 static int
1253 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1254 u64 hpa, size_t size, int prot)
1255 {
1256 u64 start_pfn, end_pfn;
1257 struct dma_pte *pte;
1258 int index;
1259 int addr_width = agaw_to_width(domain->agaw);
1260
1261 hpa &= (((u64)1) << addr_width) - 1;
1262
1263 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1264 return -EINVAL;
1265 iova &= PAGE_MASK;
1266 start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT;
1267 end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT;
1268 index = 0;
1269 while (start_pfn < end_pfn) {
1270 pte = addr_to_dma_pte(domain, iova + VTD_PAGE_SIZE * index);
1271 if (!pte)
1272 return -ENOMEM;
1273 /* We don't need lock here, nobody else
1274 * touches the iova range
1275 */
1276 BUG_ON(dma_pte_addr(*pte));
1277 dma_set_pte_addr(*pte, start_pfn << VTD_PAGE_SHIFT);
1278 dma_set_pte_prot(*pte, prot);
1279 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
1280 start_pfn++;
1281 index++;
1282 }
1283 return 0;
1284 }
1285
1286 static void detach_domain_for_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
1287 {
1288 clear_context_table(domain->iommu, bus, devfn);
1289 domain->iommu->flush.flush_context(domain->iommu, 0, 0, 0,
1290 DMA_CCMD_GLOBAL_INVL, 0);
1291 domain->iommu->flush.flush_iotlb(domain->iommu, 0, 0, 0,
1292 DMA_TLB_GLOBAL_FLUSH, 0);
1293 }
1294
1295 static void domain_remove_dev_info(struct dmar_domain *domain)
1296 {
1297 struct device_domain_info *info;
1298 unsigned long flags;
1299
1300 spin_lock_irqsave(&device_domain_lock, flags);
1301 while (!list_empty(&domain->devices)) {
1302 info = list_entry(domain->devices.next,
1303 struct device_domain_info, link);
1304 list_del(&info->link);
1305 list_del(&info->global);
1306 if (info->dev)
1307 info->dev->dev.archdata.iommu = NULL;
1308 spin_unlock_irqrestore(&device_domain_lock, flags);
1309
1310 detach_domain_for_dev(info->domain, info->bus, info->devfn);
1311 free_devinfo_mem(info);
1312
1313 spin_lock_irqsave(&device_domain_lock, flags);
1314 }
1315 spin_unlock_irqrestore(&device_domain_lock, flags);
1316 }
1317
1318 /*
1319 * find_domain
1320 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1321 */
1322 static struct dmar_domain *
1323 find_domain(struct pci_dev *pdev)
1324 {
1325 struct device_domain_info *info;
1326
1327 /* No lock here, assumes no domain exit in normal case */
1328 info = pdev->dev.archdata.iommu;
1329 if (info)
1330 return info->domain;
1331 return NULL;
1332 }
1333
1334 /* domain is initialized */
1335 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1336 {
1337 struct dmar_domain *domain, *found = NULL;
1338 struct intel_iommu *iommu;
1339 struct dmar_drhd_unit *drhd;
1340 struct device_domain_info *info, *tmp;
1341 struct pci_dev *dev_tmp;
1342 unsigned long flags;
1343 int bus = 0, devfn = 0;
1344
1345 domain = find_domain(pdev);
1346 if (domain)
1347 return domain;
1348
1349 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1350 if (dev_tmp) {
1351 if (dev_tmp->is_pcie) {
1352 bus = dev_tmp->subordinate->number;
1353 devfn = 0;
1354 } else {
1355 bus = dev_tmp->bus->number;
1356 devfn = dev_tmp->devfn;
1357 }
1358 spin_lock_irqsave(&device_domain_lock, flags);
1359 list_for_each_entry(info, &device_domain_list, global) {
1360 if (info->bus == bus && info->devfn == devfn) {
1361 found = info->domain;
1362 break;
1363 }
1364 }
1365 spin_unlock_irqrestore(&device_domain_lock, flags);
1366 /* pcie-pci bridge already has a domain, uses it */
1367 if (found) {
1368 domain = found;
1369 goto found_domain;
1370 }
1371 }
1372
1373 /* Allocate new domain for the device */
1374 drhd = dmar_find_matched_drhd_unit(pdev);
1375 if (!drhd) {
1376 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1377 pci_name(pdev));
1378 return NULL;
1379 }
1380 iommu = drhd->iommu;
1381
1382 domain = iommu_alloc_domain(iommu);
1383 if (!domain)
1384 goto error;
1385
1386 if (domain_init(domain, gaw)) {
1387 domain_exit(domain);
1388 goto error;
1389 }
1390
1391 /* register pcie-to-pci device */
1392 if (dev_tmp) {
1393 info = alloc_devinfo_mem();
1394 if (!info) {
1395 domain_exit(domain);
1396 goto error;
1397 }
1398 info->bus = bus;
1399 info->devfn = devfn;
1400 info->dev = NULL;
1401 info->domain = domain;
1402 /* This domain is shared by devices under p2p bridge */
1403 domain->flags |= DOMAIN_FLAG_MULTIPLE_DEVICES;
1404
1405 /* pcie-to-pci bridge already has a domain, uses it */
1406 found = NULL;
1407 spin_lock_irqsave(&device_domain_lock, flags);
1408 list_for_each_entry(tmp, &device_domain_list, global) {
1409 if (tmp->bus == bus && tmp->devfn == devfn) {
1410 found = tmp->domain;
1411 break;
1412 }
1413 }
1414 if (found) {
1415 free_devinfo_mem(info);
1416 domain_exit(domain);
1417 domain = found;
1418 } else {
1419 list_add(&info->link, &domain->devices);
1420 list_add(&info->global, &device_domain_list);
1421 }
1422 spin_unlock_irqrestore(&device_domain_lock, flags);
1423 }
1424
1425 found_domain:
1426 info = alloc_devinfo_mem();
1427 if (!info)
1428 goto error;
1429 info->bus = pdev->bus->number;
1430 info->devfn = pdev->devfn;
1431 info->dev = pdev;
1432 info->domain = domain;
1433 spin_lock_irqsave(&device_domain_lock, flags);
1434 /* somebody is fast */
1435 found = find_domain(pdev);
1436 if (found != NULL) {
1437 spin_unlock_irqrestore(&device_domain_lock, flags);
1438 if (found != domain) {
1439 domain_exit(domain);
1440 domain = found;
1441 }
1442 free_devinfo_mem(info);
1443 return domain;
1444 }
1445 list_add(&info->link, &domain->devices);
1446 list_add(&info->global, &device_domain_list);
1447 pdev->dev.archdata.iommu = info;
1448 spin_unlock_irqrestore(&device_domain_lock, flags);
1449 return domain;
1450 error:
1451 /* recheck it here, maybe others set it */
1452 return find_domain(pdev);
1453 }
1454
1455 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1456 unsigned long long start,
1457 unsigned long long end)
1458 {
1459 struct dmar_domain *domain;
1460 unsigned long size;
1461 unsigned long long base;
1462 int ret;
1463
1464 printk(KERN_INFO
1465 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1466 pci_name(pdev), start, end);
1467 /* page table init */
1468 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1469 if (!domain)
1470 return -ENOMEM;
1471
1472 /* The address might not be aligned */
1473 base = start & PAGE_MASK;
1474 size = end - base;
1475 size = PAGE_ALIGN(size);
1476 if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1477 IOVA_PFN(base + size) - 1)) {
1478 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1479 ret = -ENOMEM;
1480 goto error;
1481 }
1482
1483 pr_debug("Mapping reserved region %lx@%llx for %s\n",
1484 size, base, pci_name(pdev));
1485 /*
1486 * RMRR range might have overlap with physical memory range,
1487 * clear it first
1488 */
1489 dma_pte_clear_range(domain, base, base + size);
1490
1491 ret = domain_page_mapping(domain, base, base, size,
1492 DMA_PTE_READ|DMA_PTE_WRITE);
1493 if (ret)
1494 goto error;
1495
1496 /* context entry init */
1497 ret = domain_context_mapping(domain, pdev);
1498 if (!ret)
1499 return 0;
1500 error:
1501 domain_exit(domain);
1502 return ret;
1503
1504 }
1505
1506 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1507 struct pci_dev *pdev)
1508 {
1509 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1510 return 0;
1511 return iommu_prepare_identity_map(pdev, rmrr->base_address,
1512 rmrr->end_address + 1);
1513 }
1514
1515 #ifdef CONFIG_DMAR_GFX_WA
1516 struct iommu_prepare_data {
1517 struct pci_dev *pdev;
1518 int ret;
1519 };
1520
1521 static int __init iommu_prepare_work_fn(unsigned long start_pfn,
1522 unsigned long end_pfn, void *datax)
1523 {
1524 struct iommu_prepare_data *data;
1525
1526 data = (struct iommu_prepare_data *)datax;
1527
1528 data->ret = iommu_prepare_identity_map(data->pdev,
1529 start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
1530 return data->ret;
1531
1532 }
1533
1534 static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
1535 {
1536 int nid;
1537 struct iommu_prepare_data data;
1538
1539 data.pdev = pdev;
1540 data.ret = 0;
1541
1542 for_each_online_node(nid) {
1543 work_with_active_regions(nid, iommu_prepare_work_fn, &data);
1544 if (data.ret)
1545 return data.ret;
1546 }
1547 return data.ret;
1548 }
1549
1550 static void __init iommu_prepare_gfx_mapping(void)
1551 {
1552 struct pci_dev *pdev = NULL;
1553 int ret;
1554
1555 for_each_pci_dev(pdev) {
1556 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
1557 !IS_GFX_DEVICE(pdev))
1558 continue;
1559 printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
1560 pci_name(pdev));
1561 ret = iommu_prepare_with_active_regions(pdev);
1562 if (ret)
1563 printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
1564 }
1565 }
1566 #endif
1567
1568 #ifdef CONFIG_DMAR_FLOPPY_WA
1569 static inline void iommu_prepare_isa(void)
1570 {
1571 struct pci_dev *pdev;
1572 int ret;
1573
1574 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1575 if (!pdev)
1576 return;
1577
1578 printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n");
1579 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1580
1581 if (ret)
1582 printk("IOMMU: Failed to create 0-64M identity map, "
1583 "floppy might not work\n");
1584
1585 }
1586 #else
1587 static inline void iommu_prepare_isa(void)
1588 {
1589 return;
1590 }
1591 #endif /* !CONFIG_DMAR_FLPY_WA */
1592
1593 int __init init_dmars(void)
1594 {
1595 struct dmar_drhd_unit *drhd;
1596 struct dmar_rmrr_unit *rmrr;
1597 struct pci_dev *pdev;
1598 struct intel_iommu *iommu;
1599 int i, ret, unit = 0;
1600
1601 /*
1602 * for each drhd
1603 * allocate root
1604 * initialize and program root entry to not present
1605 * endfor
1606 */
1607 for_each_drhd_unit(drhd) {
1608 g_num_of_iommus++;
1609 /*
1610 * lock not needed as this is only incremented in the single
1611 * threaded kernel __init code path all other access are read
1612 * only
1613 */
1614 }
1615
1616 deferred_flush = kzalloc(g_num_of_iommus *
1617 sizeof(struct deferred_flush_tables), GFP_KERNEL);
1618 if (!deferred_flush) {
1619 ret = -ENOMEM;
1620 goto error;
1621 }
1622
1623 for_each_drhd_unit(drhd) {
1624 if (drhd->ignored)
1625 continue;
1626
1627 iommu = drhd->iommu;
1628
1629 ret = iommu_init_domains(iommu);
1630 if (ret)
1631 goto error;
1632
1633 /*
1634 * TBD:
1635 * we could share the same root & context tables
1636 * amoung all IOMMU's. Need to Split it later.
1637 */
1638 ret = iommu_alloc_root_entry(iommu);
1639 if (ret) {
1640 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1641 goto error;
1642 }
1643 }
1644
1645 for_each_drhd_unit(drhd) {
1646 if (drhd->ignored)
1647 continue;
1648
1649 iommu = drhd->iommu;
1650 if (dmar_enable_qi(iommu)) {
1651 /*
1652 * Queued Invalidate not enabled, use Register Based
1653 * Invalidate
1654 */
1655 iommu->flush.flush_context = __iommu_flush_context;
1656 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
1657 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
1658 "invalidation\n",
1659 (unsigned long long)drhd->reg_base_addr);
1660 } else {
1661 iommu->flush.flush_context = qi_flush_context;
1662 iommu->flush.flush_iotlb = qi_flush_iotlb;
1663 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
1664 "invalidation\n",
1665 (unsigned long long)drhd->reg_base_addr);
1666 }
1667 }
1668
1669 /*
1670 * For each rmrr
1671 * for each dev attached to rmrr
1672 * do
1673 * locate drhd for dev, alloc domain for dev
1674 * allocate free domain
1675 * allocate page table entries for rmrr
1676 * if context not allocated for bus
1677 * allocate and init context
1678 * set present in root table for this bus
1679 * init context with domain, translation etc
1680 * endfor
1681 * endfor
1682 */
1683 for_each_rmrr_units(rmrr) {
1684 for (i = 0; i < rmrr->devices_cnt; i++) {
1685 pdev = rmrr->devices[i];
1686 /* some BIOS lists non-exist devices in DMAR table */
1687 if (!pdev)
1688 continue;
1689 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1690 if (ret)
1691 printk(KERN_ERR
1692 "IOMMU: mapping reserved region failed\n");
1693 }
1694 }
1695
1696 iommu_prepare_gfx_mapping();
1697
1698 iommu_prepare_isa();
1699
1700 /*
1701 * for each drhd
1702 * enable fault log
1703 * global invalidate context cache
1704 * global invalidate iotlb
1705 * enable translation
1706 */
1707 for_each_drhd_unit(drhd) {
1708 if (drhd->ignored)
1709 continue;
1710 iommu = drhd->iommu;
1711 sprintf (iommu->name, "dmar%d", unit++);
1712
1713 iommu_flush_write_buffer(iommu);
1714
1715 ret = dmar_set_interrupt(iommu);
1716 if (ret)
1717 goto error;
1718
1719 iommu_set_root_entry(iommu);
1720
1721 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
1722 0);
1723 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
1724 0);
1725 iommu_disable_protect_mem_regions(iommu);
1726
1727 ret = iommu_enable_translation(iommu);
1728 if (ret)
1729 goto error;
1730 }
1731
1732 return 0;
1733 error:
1734 for_each_drhd_unit(drhd) {
1735 if (drhd->ignored)
1736 continue;
1737 iommu = drhd->iommu;
1738 free_iommu(iommu);
1739 }
1740 return ret;
1741 }
1742
1743 static inline u64 aligned_size(u64 host_addr, size_t size)
1744 {
1745 u64 addr;
1746 addr = (host_addr & (~PAGE_MASK)) + size;
1747 return PAGE_ALIGN(addr);
1748 }
1749
1750 struct iova *
1751 iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
1752 {
1753 struct iova *piova;
1754
1755 /* Make sure it's in range */
1756 end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
1757 if (!size || (IOVA_START_ADDR + size > end))
1758 return NULL;
1759
1760 piova = alloc_iova(&domain->iovad,
1761 size >> PAGE_SHIFT, IOVA_PFN(end), 1);
1762 return piova;
1763 }
1764
1765 static struct iova *
1766 __intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
1767 size_t size, u64 dma_mask)
1768 {
1769 struct pci_dev *pdev = to_pci_dev(dev);
1770 struct iova *iova = NULL;
1771
1772 if (dma_mask <= DMA_32BIT_MASK || dmar_forcedac)
1773 iova = iommu_alloc_iova(domain, size, dma_mask);
1774 else {
1775 /*
1776 * First try to allocate an io virtual address in
1777 * DMA_32BIT_MASK and if that fails then try allocating
1778 * from higher range
1779 */
1780 iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
1781 if (!iova)
1782 iova = iommu_alloc_iova(domain, size, dma_mask);
1783 }
1784
1785 if (!iova) {
1786 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
1787 return NULL;
1788 }
1789
1790 return iova;
1791 }
1792
1793 static struct dmar_domain *
1794 get_valid_domain_for_dev(struct pci_dev *pdev)
1795 {
1796 struct dmar_domain *domain;
1797 int ret;
1798
1799 domain = get_domain_for_dev(pdev,
1800 DEFAULT_DOMAIN_ADDRESS_WIDTH);
1801 if (!domain) {
1802 printk(KERN_ERR
1803 "Allocating domain for %s failed", pci_name(pdev));
1804 return NULL;
1805 }
1806
1807 /* make sure context mapping is ok */
1808 if (unlikely(!domain_context_mapped(domain, pdev))) {
1809 ret = domain_context_mapping(domain, pdev);
1810 if (ret) {
1811 printk(KERN_ERR
1812 "Domain context map for %s failed",
1813 pci_name(pdev));
1814 return NULL;
1815 }
1816 }
1817
1818 return domain;
1819 }
1820
1821 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
1822 size_t size, int dir, u64 dma_mask)
1823 {
1824 struct pci_dev *pdev = to_pci_dev(hwdev);
1825 struct dmar_domain *domain;
1826 phys_addr_t start_paddr;
1827 struct iova *iova;
1828 int prot = 0;
1829 int ret;
1830
1831 BUG_ON(dir == DMA_NONE);
1832 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1833 return paddr;
1834
1835 domain = get_valid_domain_for_dev(pdev);
1836 if (!domain)
1837 return 0;
1838
1839 size = aligned_size((u64)paddr, size);
1840
1841 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
1842 if (!iova)
1843 goto error;
1844
1845 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
1846
1847 /*
1848 * Check if DMAR supports zero-length reads on write only
1849 * mappings..
1850 */
1851 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
1852 !cap_zlr(domain->iommu->cap))
1853 prot |= DMA_PTE_READ;
1854 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1855 prot |= DMA_PTE_WRITE;
1856 /*
1857 * paddr - (paddr + size) might be partial page, we should map the whole
1858 * page. Note: if two part of one page are separately mapped, we
1859 * might have two guest_addr mapping to the same host paddr, but this
1860 * is not a big problem
1861 */
1862 ret = domain_page_mapping(domain, start_paddr,
1863 ((u64)paddr) & PAGE_MASK, size, prot);
1864 if (ret)
1865 goto error;
1866
1867 /* it's a non-present to present mapping */
1868 ret = iommu_flush_iotlb_psi(domain->iommu, domain->id,
1869 start_paddr, size >> VTD_PAGE_SHIFT, 1);
1870 if (ret)
1871 iommu_flush_write_buffer(domain->iommu);
1872
1873 return start_paddr + ((u64)paddr & (~PAGE_MASK));
1874
1875 error:
1876 if (iova)
1877 __free_iova(&domain->iovad, iova);
1878 printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
1879 pci_name(pdev), size, (unsigned long long)paddr, dir);
1880 return 0;
1881 }
1882
1883 dma_addr_t intel_map_single(struct device *hwdev, phys_addr_t paddr,
1884 size_t size, int dir)
1885 {
1886 return __intel_map_single(hwdev, paddr, size, dir,
1887 to_pci_dev(hwdev)->dma_mask);
1888 }
1889
1890 static void flush_unmaps(void)
1891 {
1892 int i, j;
1893
1894 timer_on = 0;
1895
1896 /* just flush them all */
1897 for (i = 0; i < g_num_of_iommus; i++) {
1898 if (deferred_flush[i].next) {
1899 struct intel_iommu *iommu =
1900 deferred_flush[i].domain[0]->iommu;
1901
1902 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1903 DMA_TLB_GLOBAL_FLUSH, 0);
1904 for (j = 0; j < deferred_flush[i].next; j++) {
1905 __free_iova(&deferred_flush[i].domain[j]->iovad,
1906 deferred_flush[i].iova[j]);
1907 }
1908 deferred_flush[i].next = 0;
1909 }
1910 }
1911
1912 list_size = 0;
1913 }
1914
1915 static void flush_unmaps_timeout(unsigned long data)
1916 {
1917 unsigned long flags;
1918
1919 spin_lock_irqsave(&async_umap_flush_lock, flags);
1920 flush_unmaps();
1921 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
1922 }
1923
1924 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
1925 {
1926 unsigned long flags;
1927 int next, iommu_id;
1928
1929 spin_lock_irqsave(&async_umap_flush_lock, flags);
1930 if (list_size == HIGH_WATER_MARK)
1931 flush_unmaps();
1932
1933 iommu_id = dom->iommu->seq_id;
1934
1935 next = deferred_flush[iommu_id].next;
1936 deferred_flush[iommu_id].domain[next] = dom;
1937 deferred_flush[iommu_id].iova[next] = iova;
1938 deferred_flush[iommu_id].next++;
1939
1940 if (!timer_on) {
1941 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
1942 timer_on = 1;
1943 }
1944 list_size++;
1945 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
1946 }
1947
1948 void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
1949 int dir)
1950 {
1951 struct pci_dev *pdev = to_pci_dev(dev);
1952 struct dmar_domain *domain;
1953 unsigned long start_addr;
1954 struct iova *iova;
1955
1956 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1957 return;
1958 domain = find_domain(pdev);
1959 BUG_ON(!domain);
1960
1961 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
1962 if (!iova)
1963 return;
1964
1965 start_addr = iova->pfn_lo << PAGE_SHIFT;
1966 size = aligned_size((u64)dev_addr, size);
1967
1968 pr_debug("Device %s unmapping: %lx@%llx\n",
1969 pci_name(pdev), size, (unsigned long long)start_addr);
1970
1971 /* clear the whole page */
1972 dma_pte_clear_range(domain, start_addr, start_addr + size);
1973 /* free page tables */
1974 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
1975 if (intel_iommu_strict) {
1976 if (iommu_flush_iotlb_psi(domain->iommu,
1977 domain->id, start_addr, size >> VTD_PAGE_SHIFT, 0))
1978 iommu_flush_write_buffer(domain->iommu);
1979 /* free iova */
1980 __free_iova(&domain->iovad, iova);
1981 } else {
1982 add_unmap(domain, iova);
1983 /*
1984 * queue up the release of the unmap to save the 1/6th of the
1985 * cpu used up by the iotlb flush operation...
1986 */
1987 }
1988 }
1989
1990 void *intel_alloc_coherent(struct device *hwdev, size_t size,
1991 dma_addr_t *dma_handle, gfp_t flags)
1992 {
1993 void *vaddr;
1994 int order;
1995
1996 size = PAGE_ALIGN(size);
1997 order = get_order(size);
1998 flags &= ~(GFP_DMA | GFP_DMA32);
1999
2000 vaddr = (void *)__get_free_pages(flags, order);
2001 if (!vaddr)
2002 return NULL;
2003 memset(vaddr, 0, size);
2004
2005 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2006 DMA_BIDIRECTIONAL,
2007 hwdev->coherent_dma_mask);
2008 if (*dma_handle)
2009 return vaddr;
2010 free_pages((unsigned long)vaddr, order);
2011 return NULL;
2012 }
2013
2014 void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2015 dma_addr_t dma_handle)
2016 {
2017 int order;
2018
2019 size = PAGE_ALIGN(size);
2020 order = get_order(size);
2021
2022 intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2023 free_pages((unsigned long)vaddr, order);
2024 }
2025
2026 #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
2027
2028 void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2029 int nelems, int dir)
2030 {
2031 int i;
2032 struct pci_dev *pdev = to_pci_dev(hwdev);
2033 struct dmar_domain *domain;
2034 unsigned long start_addr;
2035 struct iova *iova;
2036 size_t size = 0;
2037 void *addr;
2038 struct scatterlist *sg;
2039
2040 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2041 return;
2042
2043 domain = find_domain(pdev);
2044
2045 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2046 if (!iova)
2047 return;
2048 for_each_sg(sglist, sg, nelems, i) {
2049 addr = SG_ENT_VIRT_ADDRESS(sg);
2050 size += aligned_size((u64)addr, sg->length);
2051 }
2052
2053 start_addr = iova->pfn_lo << PAGE_SHIFT;
2054
2055 /* clear the whole page */
2056 dma_pte_clear_range(domain, start_addr, start_addr + size);
2057 /* free page tables */
2058 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2059
2060 if (iommu_flush_iotlb_psi(domain->iommu, domain->id, start_addr,
2061 size >> VTD_PAGE_SHIFT, 0))
2062 iommu_flush_write_buffer(domain->iommu);
2063
2064 /* free iova */
2065 __free_iova(&domain->iovad, iova);
2066 }
2067
2068 static int intel_nontranslate_map_sg(struct device *hddev,
2069 struct scatterlist *sglist, int nelems, int dir)
2070 {
2071 int i;
2072 struct scatterlist *sg;
2073
2074 for_each_sg(sglist, sg, nelems, i) {
2075 BUG_ON(!sg_page(sg));
2076 sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg));
2077 sg->dma_length = sg->length;
2078 }
2079 return nelems;
2080 }
2081
2082 int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2083 int dir)
2084 {
2085 void *addr;
2086 int i;
2087 struct pci_dev *pdev = to_pci_dev(hwdev);
2088 struct dmar_domain *domain;
2089 size_t size = 0;
2090 int prot = 0;
2091 size_t offset = 0;
2092 struct iova *iova = NULL;
2093 int ret;
2094 struct scatterlist *sg;
2095 unsigned long start_addr;
2096
2097 BUG_ON(dir == DMA_NONE);
2098 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2099 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2100
2101 domain = get_valid_domain_for_dev(pdev);
2102 if (!domain)
2103 return 0;
2104
2105 for_each_sg(sglist, sg, nelems, i) {
2106 addr = SG_ENT_VIRT_ADDRESS(sg);
2107 addr = (void *)virt_to_phys(addr);
2108 size += aligned_size((u64)addr, sg->length);
2109 }
2110
2111 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2112 if (!iova) {
2113 sglist->dma_length = 0;
2114 return 0;
2115 }
2116
2117 /*
2118 * Check if DMAR supports zero-length reads on write only
2119 * mappings..
2120 */
2121 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2122 !cap_zlr(domain->iommu->cap))
2123 prot |= DMA_PTE_READ;
2124 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2125 prot |= DMA_PTE_WRITE;
2126
2127 start_addr = iova->pfn_lo << PAGE_SHIFT;
2128 offset = 0;
2129 for_each_sg(sglist, sg, nelems, i) {
2130 addr = SG_ENT_VIRT_ADDRESS(sg);
2131 addr = (void *)virt_to_phys(addr);
2132 size = aligned_size((u64)addr, sg->length);
2133 ret = domain_page_mapping(domain, start_addr + offset,
2134 ((u64)addr) & PAGE_MASK,
2135 size, prot);
2136 if (ret) {
2137 /* clear the page */
2138 dma_pte_clear_range(domain, start_addr,
2139 start_addr + offset);
2140 /* free page tables */
2141 dma_pte_free_pagetable(domain, start_addr,
2142 start_addr + offset);
2143 /* free iova */
2144 __free_iova(&domain->iovad, iova);
2145 return 0;
2146 }
2147 sg->dma_address = start_addr + offset +
2148 ((u64)addr & (~PAGE_MASK));
2149 sg->dma_length = sg->length;
2150 offset += size;
2151 }
2152
2153 /* it's a non-present to present mapping */
2154 if (iommu_flush_iotlb_psi(domain->iommu, domain->id,
2155 start_addr, offset >> VTD_PAGE_SHIFT, 1))
2156 iommu_flush_write_buffer(domain->iommu);
2157 return nelems;
2158 }
2159
2160 static struct dma_mapping_ops intel_dma_ops = {
2161 .alloc_coherent = intel_alloc_coherent,
2162 .free_coherent = intel_free_coherent,
2163 .map_single = intel_map_single,
2164 .unmap_single = intel_unmap_single,
2165 .map_sg = intel_map_sg,
2166 .unmap_sg = intel_unmap_sg,
2167 };
2168
2169 static inline int iommu_domain_cache_init(void)
2170 {
2171 int ret = 0;
2172
2173 iommu_domain_cache = kmem_cache_create("iommu_domain",
2174 sizeof(struct dmar_domain),
2175 0,
2176 SLAB_HWCACHE_ALIGN,
2177
2178 NULL);
2179 if (!iommu_domain_cache) {
2180 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2181 ret = -ENOMEM;
2182 }
2183
2184 return ret;
2185 }
2186
2187 static inline int iommu_devinfo_cache_init(void)
2188 {
2189 int ret = 0;
2190
2191 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2192 sizeof(struct device_domain_info),
2193 0,
2194 SLAB_HWCACHE_ALIGN,
2195 NULL);
2196 if (!iommu_devinfo_cache) {
2197 printk(KERN_ERR "Couldn't create devinfo cache\n");
2198 ret = -ENOMEM;
2199 }
2200
2201 return ret;
2202 }
2203
2204 static inline int iommu_iova_cache_init(void)
2205 {
2206 int ret = 0;
2207
2208 iommu_iova_cache = kmem_cache_create("iommu_iova",
2209 sizeof(struct iova),
2210 0,
2211 SLAB_HWCACHE_ALIGN,
2212 NULL);
2213 if (!iommu_iova_cache) {
2214 printk(KERN_ERR "Couldn't create iova cache\n");
2215 ret = -ENOMEM;
2216 }
2217
2218 return ret;
2219 }
2220
2221 static int __init iommu_init_mempool(void)
2222 {
2223 int ret;
2224 ret = iommu_iova_cache_init();
2225 if (ret)
2226 return ret;
2227
2228 ret = iommu_domain_cache_init();
2229 if (ret)
2230 goto domain_error;
2231
2232 ret = iommu_devinfo_cache_init();
2233 if (!ret)
2234 return ret;
2235
2236 kmem_cache_destroy(iommu_domain_cache);
2237 domain_error:
2238 kmem_cache_destroy(iommu_iova_cache);
2239
2240 return -ENOMEM;
2241 }
2242
2243 static void __init iommu_exit_mempool(void)
2244 {
2245 kmem_cache_destroy(iommu_devinfo_cache);
2246 kmem_cache_destroy(iommu_domain_cache);
2247 kmem_cache_destroy(iommu_iova_cache);
2248
2249 }
2250
2251 static void __init init_no_remapping_devices(void)
2252 {
2253 struct dmar_drhd_unit *drhd;
2254
2255 for_each_drhd_unit(drhd) {
2256 if (!drhd->include_all) {
2257 int i;
2258 for (i = 0; i < drhd->devices_cnt; i++)
2259 if (drhd->devices[i] != NULL)
2260 break;
2261 /* ignore DMAR unit if no pci devices exist */
2262 if (i == drhd->devices_cnt)
2263 drhd->ignored = 1;
2264 }
2265 }
2266
2267 if (dmar_map_gfx)
2268 return;
2269
2270 for_each_drhd_unit(drhd) {
2271 int i;
2272 if (drhd->ignored || drhd->include_all)
2273 continue;
2274
2275 for (i = 0; i < drhd->devices_cnt; i++)
2276 if (drhd->devices[i] &&
2277 !IS_GFX_DEVICE(drhd->devices[i]))
2278 break;
2279
2280 if (i < drhd->devices_cnt)
2281 continue;
2282
2283 /* bypass IOMMU if it is just for gfx devices */
2284 drhd->ignored = 1;
2285 for (i = 0; i < drhd->devices_cnt; i++) {
2286 if (!drhd->devices[i])
2287 continue;
2288 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
2289 }
2290 }
2291 }
2292
2293 int __init intel_iommu_init(void)
2294 {
2295 int ret = 0;
2296
2297 if (dmar_table_init())
2298 return -ENODEV;
2299
2300 if (dmar_dev_scope_init())
2301 return -ENODEV;
2302
2303 /*
2304 * Check the need for DMA-remapping initialization now.
2305 * Above initialization will also be used by Interrupt-remapping.
2306 */
2307 if (no_iommu || swiotlb || dmar_disabled)
2308 return -ENODEV;
2309
2310 iommu_init_mempool();
2311 dmar_init_reserved_ranges();
2312
2313 init_no_remapping_devices();
2314
2315 ret = init_dmars();
2316 if (ret) {
2317 printk(KERN_ERR "IOMMU: dmar init failed\n");
2318 put_iova_domain(&reserved_iova_list);
2319 iommu_exit_mempool();
2320 return ret;
2321 }
2322 printk(KERN_INFO
2323 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2324
2325 init_timer(&unmap_timer);
2326 force_iommu = 1;
2327 dma_ops = &intel_dma_ops;
2328 return 0;
2329 }
2330
2331 void intel_iommu_domain_exit(struct dmar_domain *domain)
2332 {
2333 u64 end;
2334
2335 /* Domain 0 is reserved, so dont process it */
2336 if (!domain)
2337 return;
2338
2339 end = DOMAIN_MAX_ADDR(domain->gaw);
2340 end = end & (~VTD_PAGE_MASK);
2341
2342 /* clear ptes */
2343 dma_pte_clear_range(domain, 0, end);
2344
2345 /* free page tables */
2346 dma_pte_free_pagetable(domain, 0, end);
2347
2348 iommu_free_domain(domain);
2349 free_domain_mem(domain);
2350 }
2351 EXPORT_SYMBOL_GPL(intel_iommu_domain_exit);
2352
2353 struct dmar_domain *intel_iommu_domain_alloc(struct pci_dev *pdev)
2354 {
2355 struct dmar_drhd_unit *drhd;
2356 struct dmar_domain *domain;
2357 struct intel_iommu *iommu;
2358
2359 drhd = dmar_find_matched_drhd_unit(pdev);
2360 if (!drhd) {
2361 printk(KERN_ERR "intel_iommu_domain_alloc: drhd == NULL\n");
2362 return NULL;
2363 }
2364
2365 iommu = drhd->iommu;
2366 if (!iommu) {
2367 printk(KERN_ERR
2368 "intel_iommu_domain_alloc: iommu == NULL\n");
2369 return NULL;
2370 }
2371 domain = iommu_alloc_domain(iommu);
2372 if (!domain) {
2373 printk(KERN_ERR
2374 "intel_iommu_domain_alloc: domain == NULL\n");
2375 return NULL;
2376 }
2377 if (domain_init(domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2378 printk(KERN_ERR
2379 "intel_iommu_domain_alloc: domain_init() failed\n");
2380 intel_iommu_domain_exit(domain);
2381 return NULL;
2382 }
2383 return domain;
2384 }
2385 EXPORT_SYMBOL_GPL(intel_iommu_domain_alloc);
2386
2387 int intel_iommu_context_mapping(
2388 struct dmar_domain *domain, struct pci_dev *pdev)
2389 {
2390 int rc;
2391 rc = domain_context_mapping(domain, pdev);
2392 return rc;
2393 }
2394 EXPORT_SYMBOL_GPL(intel_iommu_context_mapping);
2395
2396 int intel_iommu_page_mapping(
2397 struct dmar_domain *domain, dma_addr_t iova,
2398 u64 hpa, size_t size, int prot)
2399 {
2400 int rc;
2401 rc = domain_page_mapping(domain, iova, hpa, size, prot);
2402 return rc;
2403 }
2404 EXPORT_SYMBOL_GPL(intel_iommu_page_mapping);
2405
2406 void intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
2407 {
2408 detach_domain_for_dev(domain, bus, devfn);
2409 }
2410 EXPORT_SYMBOL_GPL(intel_iommu_detach_dev);
2411
2412 struct dmar_domain *
2413 intel_iommu_find_domain(struct pci_dev *pdev)
2414 {
2415 return find_domain(pdev);
2416 }
2417 EXPORT_SYMBOL_GPL(intel_iommu_find_domain);
2418
2419 int intel_iommu_found(void)
2420 {
2421 return g_num_of_iommus;
2422 }
2423 EXPORT_SYMBOL_GPL(intel_iommu_found);
2424
2425 u64 intel_iommu_iova_to_pfn(struct dmar_domain *domain, u64 iova)
2426 {
2427 struct dma_pte *pte;
2428 u64 pfn;
2429
2430 pfn = 0;
2431 pte = addr_to_dma_pte(domain, iova);
2432
2433 if (pte)
2434 pfn = dma_pte_addr(*pte);
2435
2436 return pfn >> VTD_PAGE_SHIFT;
2437 }
2438 EXPORT_SYMBOL_GPL(intel_iommu_iova_to_pfn);
SocketAccessConTroL development