?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Merge branch 'master' of master.kernel.org:/pub/scm/linux/kernel/git/torvalds/linux-2.6
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / drivers / pci / intel-iommu.c
bloba2692724b68ffe04382bdd8a2d15c4774511c47d
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
22 */
23
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/sysdev.h>
31 #include <linux/spinlock.h>
32 #include <linux/pci.h>
33 #include <linux/dmar.h>
34 #include <linux/dma-mapping.h>
35 #include <linux/mempool.h>
36 #include <linux/timer.h>
37 #include <linux/iova.h>
38 #include <linux/intel-iommu.h>
39 #include <asm/cacheflush.h>
40 #include <asm/iommu.h>
41 #include "pci.h"
42
43 #define ROOT_SIZE VTD_PAGE_SIZE
44 #define CONTEXT_SIZE VTD_PAGE_SIZE
45
46 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
47 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
48
49 #define IOAPIC_RANGE_START (0xfee00000)
50 #define IOAPIC_RANGE_END (0xfeefffff)
51 #define IOVA_START_ADDR (0x1000)
52
53 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
54
55 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
56
57
58 static void flush_unmaps_timeout(unsigned long data);
59
60 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
61
62 #define HIGH_WATER_MARK 250
63 struct deferred_flush_tables {
64 int next;
65 struct iova *iova[HIGH_WATER_MARK];
66 struct dmar_domain *domain[HIGH_WATER_MARK];
67 };
68
69 static struct deferred_flush_tables *deferred_flush;
70
71 /* bitmap for indexing intel_iommus */
72 static int g_num_of_iommus;
73
74 static DEFINE_SPINLOCK(async_umap_flush_lock);
75 static LIST_HEAD(unmaps_to_do);
76
77 static int timer_on;
78 static long list_size;
79
80 static void domain_remove_dev_info(struct dmar_domain *domain);
81
82 int dmar_disabled;
83 static int __initdata dmar_map_gfx = 1;
84 static int dmar_forcedac;
85 static int intel_iommu_strict;
86
87 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
88 static DEFINE_SPINLOCK(device_domain_lock);
89 static LIST_HEAD(device_domain_list);
90
91 static int __init intel_iommu_setup(char *str)
92 {
93 if (!str)
94 return -EINVAL;
95 while (*str) {
96 if (!strncmp(str, "off", 3)) {
97 dmar_disabled = 1;
98 printk(KERN_INFO"Intel-IOMMU: disabled\n");
99 } else if (!strncmp(str, "igfx_off", 8)) {
100 dmar_map_gfx = 0;
101 printk(KERN_INFO
102 "Intel-IOMMU: disable GFX device mapping\n");
103 } else if (!strncmp(str, "forcedac", 8)) {
104 printk(KERN_INFO
105 "Intel-IOMMU: Forcing DAC for PCI devices\n");
106 dmar_forcedac = 1;
107 } else if (!strncmp(str, "strict", 6)) {
108 printk(KERN_INFO
109 "Intel-IOMMU: disable batched IOTLB flush\n");
110 intel_iommu_strict = 1;
111 }
112
113 str += strcspn(str, ",");
114 while (*str == ',')
115 str++;
116 }
117 return 0;
118 }
119 __setup("intel_iommu=", intel_iommu_setup);
120
121 static struct kmem_cache *iommu_domain_cache;
122 static struct kmem_cache *iommu_devinfo_cache;
123 static struct kmem_cache *iommu_iova_cache;
124
125 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
126 {
127 unsigned int flags;
128 void *vaddr;
129
130 /* trying to avoid low memory issues */
131 flags = current->flags & PF_MEMALLOC;
132 current->flags |= PF_MEMALLOC;
133 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
134 current->flags &= (~PF_MEMALLOC | flags);
135 return vaddr;
136 }
137
138
139 static inline void *alloc_pgtable_page(void)
140 {
141 unsigned int flags;
142 void *vaddr;
143
144 /* trying to avoid low memory issues */
145 flags = current->flags & PF_MEMALLOC;
146 current->flags |= PF_MEMALLOC;
147 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
148 current->flags &= (~PF_MEMALLOC | flags);
149 return vaddr;
150 }
151
152 static inline void free_pgtable_page(void *vaddr)
153 {
154 free_page((unsigned long)vaddr);
155 }
156
157 static inline void *alloc_domain_mem(void)
158 {
159 return iommu_kmem_cache_alloc(iommu_domain_cache);
160 }
161
162 static void free_domain_mem(void *vaddr)
163 {
164 kmem_cache_free(iommu_domain_cache, vaddr);
165 }
166
167 static inline void * alloc_devinfo_mem(void)
168 {
169 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
170 }
171
172 static inline void free_devinfo_mem(void *vaddr)
173 {
174 kmem_cache_free(iommu_devinfo_cache, vaddr);
175 }
176
177 struct iova *alloc_iova_mem(void)
178 {
179 return iommu_kmem_cache_alloc(iommu_iova_cache);
180 }
181
182 void free_iova_mem(struct iova *iova)
183 {
184 kmem_cache_free(iommu_iova_cache, iova);
185 }
186
187 /* Gets context entry for a given bus and devfn */
188 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
189 u8 bus, u8 devfn)
190 {
191 struct root_entry *root;
192 struct context_entry *context;
193 unsigned long phy_addr;
194 unsigned long flags;
195
196 spin_lock_irqsave(&iommu->lock, flags);
197 root = &iommu->root_entry[bus];
198 context = get_context_addr_from_root(root);
199 if (!context) {
200 context = (struct context_entry *)alloc_pgtable_page();
201 if (!context) {
202 spin_unlock_irqrestore(&iommu->lock, flags);
203 return NULL;
204 }
205 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
206 phy_addr = virt_to_phys((void *)context);
207 set_root_value(root, phy_addr);
208 set_root_present(root);
209 __iommu_flush_cache(iommu, root, sizeof(*root));
210 }
211 spin_unlock_irqrestore(&iommu->lock, flags);
212 return &context[devfn];
213 }
214
215 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
216 {
217 struct root_entry *root;
218 struct context_entry *context;
219 int ret;
220 unsigned long flags;
221
222 spin_lock_irqsave(&iommu->lock, flags);
223 root = &iommu->root_entry[bus];
224 context = get_context_addr_from_root(root);
225 if (!context) {
226 ret = 0;
227 goto out;
228 }
229 ret = context_present(context[devfn]);
230 out:
231 spin_unlock_irqrestore(&iommu->lock, flags);
232 return ret;
233 }
234
235 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
236 {
237 struct root_entry *root;
238 struct context_entry *context;
239 unsigned long flags;
240
241 spin_lock_irqsave(&iommu->lock, flags);
242 root = &iommu->root_entry[bus];
243 context = get_context_addr_from_root(root);
244 if (context) {
245 context_clear_entry(context[devfn]);
246 __iommu_flush_cache(iommu, &context[devfn], \
247 sizeof(*context));
248 }
249 spin_unlock_irqrestore(&iommu->lock, flags);
250 }
251
252 static void free_context_table(struct intel_iommu *iommu)
253 {
254 struct root_entry *root;
255 int i;
256 unsigned long flags;
257 struct context_entry *context;
258
259 spin_lock_irqsave(&iommu->lock, flags);
260 if (!iommu->root_entry) {
261 goto out;
262 }
263 for (i = 0; i < ROOT_ENTRY_NR; i++) {
264 root = &iommu->root_entry[i];
265 context = get_context_addr_from_root(root);
266 if (context)
267 free_pgtable_page(context);
268 }
269 free_pgtable_page(iommu->root_entry);
270 iommu->root_entry = NULL;
271 out:
272 spin_unlock_irqrestore(&iommu->lock, flags);
273 }
274
275 /* page table handling */
276 #define LEVEL_STRIDE (9)
277 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
278
279 static inline int agaw_to_level(int agaw)
280 {
281 return agaw + 2;
282 }
283
284 static inline int agaw_to_width(int agaw)
285 {
286 return 30 + agaw * LEVEL_STRIDE;
287
288 }
289
290 static inline int width_to_agaw(int width)
291 {
292 return (width - 30) / LEVEL_STRIDE;
293 }
294
295 static inline unsigned int level_to_offset_bits(int level)
296 {
297 return (12 + (level - 1) * LEVEL_STRIDE);
298 }
299
300 static inline int address_level_offset(u64 addr, int level)
301 {
302 return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
303 }
304
305 static inline u64 level_mask(int level)
306 {
307 return ((u64)-1 << level_to_offset_bits(level));
308 }
309
310 static inline u64 level_size(int level)
311 {
312 return ((u64)1 << level_to_offset_bits(level));
313 }
314
315 static inline u64 align_to_level(u64 addr, int level)
316 {
317 return ((addr + level_size(level) - 1) & level_mask(level));
318 }
319
320 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
321 {
322 int addr_width = agaw_to_width(domain->agaw);
323 struct dma_pte *parent, *pte = NULL;
324 int level = agaw_to_level(domain->agaw);
325 int offset;
326 unsigned long flags;
327
328 BUG_ON(!domain->pgd);
329
330 addr &= (((u64)1) << addr_width) - 1;
331 parent = domain->pgd;
332
333 spin_lock_irqsave(&domain->mapping_lock, flags);
334 while (level > 0) {
335 void *tmp_page;
336
337 offset = address_level_offset(addr, level);
338 pte = &parent[offset];
339 if (level == 1)
340 break;
341
342 if (!dma_pte_present(*pte)) {
343 tmp_page = alloc_pgtable_page();
344
345 if (!tmp_page) {
346 spin_unlock_irqrestore(&domain->mapping_lock,
347 flags);
348 return NULL;
349 }
350 __iommu_flush_cache(domain->iommu, tmp_page,
351 PAGE_SIZE);
352 dma_set_pte_addr(*pte, virt_to_phys(tmp_page));
353 /*
354 * high level table always sets r/w, last level page
355 * table control read/write
356 */
357 dma_set_pte_readable(*pte);
358 dma_set_pte_writable(*pte);
359 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
360 }
361 parent = phys_to_virt(dma_pte_addr(*pte));
362 level--;
363 }
364
365 spin_unlock_irqrestore(&domain->mapping_lock, flags);
366 return pte;
367 }
368
369 /* return address's pte at specific level */
370 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
371 int level)
372 {
373 struct dma_pte *parent, *pte = NULL;
374 int total = agaw_to_level(domain->agaw);
375 int offset;
376
377 parent = domain->pgd;
378 while (level <= total) {
379 offset = address_level_offset(addr, total);
380 pte = &parent[offset];
381 if (level == total)
382 return pte;
383
384 if (!dma_pte_present(*pte))
385 break;
386 parent = phys_to_virt(dma_pte_addr(*pte));
387 total--;
388 }
389 return NULL;
390 }
391
392 /* clear one page's page table */
393 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
394 {
395 struct dma_pte *pte = NULL;
396
397 /* get last level pte */
398 pte = dma_addr_level_pte(domain, addr, 1);
399
400 if (pte) {
401 dma_clear_pte(*pte);
402 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
403 }
404 }
405
406 /* clear last level pte, a tlb flush should be followed */
407 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
408 {
409 int addr_width = agaw_to_width(domain->agaw);
410
411 start &= (((u64)1) << addr_width) - 1;
412 end &= (((u64)1) << addr_width) - 1;
413 /* in case it's partial page */
414 start = PAGE_ALIGN(start);
415 end &= PAGE_MASK;
416
417 /* we don't need lock here, nobody else touches the iova range */
418 while (start < end) {
419 dma_pte_clear_one(domain, start);
420 start += VTD_PAGE_SIZE;
421 }
422 }
423
424 /* free page table pages. last level pte should already be cleared */
425 static void dma_pte_free_pagetable(struct dmar_domain *domain,
426 u64 start, u64 end)
427 {
428 int addr_width = agaw_to_width(domain->agaw);
429 struct dma_pte *pte;
430 int total = agaw_to_level(domain->agaw);
431 int level;
432 u64 tmp;
433
434 start &= (((u64)1) << addr_width) - 1;
435 end &= (((u64)1) << addr_width) - 1;
436
437 /* we don't need lock here, nobody else touches the iova range */
438 level = 2;
439 while (level <= total) {
440 tmp = align_to_level(start, level);
441 if (tmp >= end || (tmp + level_size(level) > end))
442 return;
443
444 while (tmp < end) {
445 pte = dma_addr_level_pte(domain, tmp, level);
446 if (pte) {
447 free_pgtable_page(
448 phys_to_virt(dma_pte_addr(*pte)));
449 dma_clear_pte(*pte);
450 __iommu_flush_cache(domain->iommu,
451 pte, sizeof(*pte));
452 }
453 tmp += level_size(level);
454 }
455 level++;
456 }
457 /* free pgd */
458 if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
459 free_pgtable_page(domain->pgd);
460 domain->pgd = NULL;
461 }
462 }
463
464 /* iommu handling */
465 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
466 {
467 struct root_entry *root;
468 unsigned long flags;
469
470 root = (struct root_entry *)alloc_pgtable_page();
471 if (!root)
472 return -ENOMEM;
473
474 __iommu_flush_cache(iommu, root, ROOT_SIZE);
475
476 spin_lock_irqsave(&iommu->lock, flags);
477 iommu->root_entry = root;
478 spin_unlock_irqrestore(&iommu->lock, flags);
479
480 return 0;
481 }
482
483 static void iommu_set_root_entry(struct intel_iommu *iommu)
484 {
485 void *addr;
486 u32 cmd, sts;
487 unsigned long flag;
488
489 addr = iommu->root_entry;
490
491 spin_lock_irqsave(&iommu->register_lock, flag);
492 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
493
494 cmd = iommu->gcmd | DMA_GCMD_SRTP;
495 writel(cmd, iommu->reg + DMAR_GCMD_REG);
496
497 /* Make sure hardware complete it */
498 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
499 readl, (sts & DMA_GSTS_RTPS), sts);
500
501 spin_unlock_irqrestore(&iommu->register_lock, flag);
502 }
503
504 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
505 {
506 u32 val;
507 unsigned long flag;
508
509 if (!cap_rwbf(iommu->cap))
510 return;
511 val = iommu->gcmd | DMA_GCMD_WBF;
512
513 spin_lock_irqsave(&iommu->register_lock, flag);
514 writel(val, iommu->reg + DMAR_GCMD_REG);
515
516 /* Make sure hardware complete it */
517 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
518 readl, (!(val & DMA_GSTS_WBFS)), val);
519
520 spin_unlock_irqrestore(&iommu->register_lock, flag);
521 }
522
523 /* return value determine if we need a write buffer flush */
524 static int __iommu_flush_context(struct intel_iommu *iommu,
525 u16 did, u16 source_id, u8 function_mask, u64 type,
526 int non_present_entry_flush)
527 {
528 u64 val = 0;
529 unsigned long flag;
530
531 /*
532 * In the non-present entry flush case, if hardware doesn't cache
533 * non-present entry we do nothing and if hardware cache non-present
534 * entry, we flush entries of domain 0 (the domain id is used to cache
535 * any non-present entries)
536 */
537 if (non_present_entry_flush) {
538 if (!cap_caching_mode(iommu->cap))
539 return 1;
540 else
541 did = 0;
542 }
543
544 switch (type) {
545 case DMA_CCMD_GLOBAL_INVL:
546 val = DMA_CCMD_GLOBAL_INVL;
547 break;
548 case DMA_CCMD_DOMAIN_INVL:
549 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
550 break;
551 case DMA_CCMD_DEVICE_INVL:
552 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
553 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
554 break;
555 default:
556 BUG();
557 }
558 val |= DMA_CCMD_ICC;
559
560 spin_lock_irqsave(&iommu->register_lock, flag);
561 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
562
563 /* Make sure hardware complete it */
564 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
565 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
566
567 spin_unlock_irqrestore(&iommu->register_lock, flag);
568
569 /* flush context entry will implicitly flush write buffer */
570 return 0;
571 }
572
573 /* return value determine if we need a write buffer flush */
574 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
575 u64 addr, unsigned int size_order, u64 type,
576 int non_present_entry_flush)
577 {
578 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
579 u64 val = 0, val_iva = 0;
580 unsigned long flag;
581
582 /*
583 * In the non-present entry flush case, if hardware doesn't cache
584 * non-present entry we do nothing and if hardware cache non-present
585 * entry, we flush entries of domain 0 (the domain id is used to cache
586 * any non-present entries)
587 */
588 if (non_present_entry_flush) {
589 if (!cap_caching_mode(iommu->cap))
590 return 1;
591 else
592 did = 0;
593 }
594
595 switch (type) {
596 case DMA_TLB_GLOBAL_FLUSH:
597 /* global flush doesn't need set IVA_REG */
598 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
599 break;
600 case DMA_TLB_DSI_FLUSH:
601 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
602 break;
603 case DMA_TLB_PSI_FLUSH:
604 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
605 /* Note: always flush non-leaf currently */
606 val_iva = size_order | addr;
607 break;
608 default:
609 BUG();
610 }
611 /* Note: set drain read/write */
612 #if 0
613 /*
614 * This is probably to be super secure.. Looks like we can
615 * ignore it without any impact.
616 */
617 if (cap_read_drain(iommu->cap))
618 val |= DMA_TLB_READ_DRAIN;
619 #endif
620 if (cap_write_drain(iommu->cap))
621 val |= DMA_TLB_WRITE_DRAIN;
622
623 spin_lock_irqsave(&iommu->register_lock, flag);
624 /* Note: Only uses first TLB reg currently */
625 if (val_iva)
626 dmar_writeq(iommu->reg + tlb_offset, val_iva);
627 dmar_writeq(iommu->reg + tlb_offset + 8, val);
628
629 /* Make sure hardware complete it */
630 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
631 dmar_readq, (!(val & DMA_TLB_IVT)), val);
632
633 spin_unlock_irqrestore(&iommu->register_lock, flag);
634
635 /* check IOTLB invalidation granularity */
636 if (DMA_TLB_IAIG(val) == 0)
637 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
638 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
639 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
640 (unsigned long long)DMA_TLB_IIRG(type),
641 (unsigned long long)DMA_TLB_IAIG(val));
642 /* flush iotlb entry will implicitly flush write buffer */
643 return 0;
644 }
645
646 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
647 u64 addr, unsigned int pages, int non_present_entry_flush)
648 {
649 unsigned int mask;
650
651 BUG_ON(addr & (~VTD_PAGE_MASK));
652 BUG_ON(pages == 0);
653
654 /* Fallback to domain selective flush if no PSI support */
655 if (!cap_pgsel_inv(iommu->cap))
656 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
657 DMA_TLB_DSI_FLUSH,
658 non_present_entry_flush);
659
660 /*
661 * PSI requires page size to be 2 ^ x, and the base address is naturally
662 * aligned to the size
663 */
664 mask = ilog2(__roundup_pow_of_two(pages));
665 /* Fallback to domain selective flush if size is too big */
666 if (mask > cap_max_amask_val(iommu->cap))
667 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
668 DMA_TLB_DSI_FLUSH, non_present_entry_flush);
669
670 return iommu->flush.flush_iotlb(iommu, did, addr, mask,
671 DMA_TLB_PSI_FLUSH,
672 non_present_entry_flush);
673 }
674
675 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
676 {
677 u32 pmen;
678 unsigned long flags;
679
680 spin_lock_irqsave(&iommu->register_lock, flags);
681 pmen = readl(iommu->reg + DMAR_PMEN_REG);
682 pmen &= ~DMA_PMEN_EPM;
683 writel(pmen, iommu->reg + DMAR_PMEN_REG);
684
685 /* wait for the protected region status bit to clear */
686 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
687 readl, !(pmen & DMA_PMEN_PRS), pmen);
688
689 spin_unlock_irqrestore(&iommu->register_lock, flags);
690 }
691
692 static int iommu_enable_translation(struct intel_iommu *iommu)
693 {
694 u32 sts;
695 unsigned long flags;
696
697 spin_lock_irqsave(&iommu->register_lock, flags);
698 writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
699
700 /* Make sure hardware complete it */
701 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
702 readl, (sts & DMA_GSTS_TES), sts);
703
704 iommu->gcmd |= DMA_GCMD_TE;
705 spin_unlock_irqrestore(&iommu->register_lock, flags);
706 return 0;
707 }
708
709 static int iommu_disable_translation(struct intel_iommu *iommu)
710 {
711 u32 sts;
712 unsigned long flag;
713
714 spin_lock_irqsave(&iommu->register_lock, flag);
715 iommu->gcmd &= ~DMA_GCMD_TE;
716 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
717
718 /* Make sure hardware complete it */
719 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
720 readl, (!(sts & DMA_GSTS_TES)), sts);
721
722 spin_unlock_irqrestore(&iommu->register_lock, flag);
723 return 0;
724 }
725
726 /* iommu interrupt handling. Most stuff are MSI-like. */
727
728 static const char *fault_reason_strings[] =
729 {
730 "Software",
731 "Present bit in root entry is clear",
732 "Present bit in context entry is clear",
733 "Invalid context entry",
734 "Access beyond MGAW",
735 "PTE Write access is not set",
736 "PTE Read access is not set",
737 "Next page table ptr is invalid",
738 "Root table address invalid",
739 "Context table ptr is invalid",
740 "non-zero reserved fields in RTP",
741 "non-zero reserved fields in CTP",
742 "non-zero reserved fields in PTE",
743 };
744 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
745
746 const char *dmar_get_fault_reason(u8 fault_reason)
747 {
748 if (fault_reason > MAX_FAULT_REASON_IDX)
749 return "Unknown";
750 else
751 return fault_reason_strings[fault_reason];
752 }
753
754 void dmar_msi_unmask(unsigned int irq)
755 {
756 struct intel_iommu *iommu = get_irq_data(irq);
757 unsigned long flag;
758
759 /* unmask it */
760 spin_lock_irqsave(&iommu->register_lock, flag);
761 writel(0, iommu->reg + DMAR_FECTL_REG);
762 /* Read a reg to force flush the post write */
763 readl(iommu->reg + DMAR_FECTL_REG);
764 spin_unlock_irqrestore(&iommu->register_lock, flag);
765 }
766
767 void dmar_msi_mask(unsigned int irq)
768 {
769 unsigned long flag;
770 struct intel_iommu *iommu = get_irq_data(irq);
771
772 /* mask it */
773 spin_lock_irqsave(&iommu->register_lock, flag);
774 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
775 /* Read a reg to force flush the post write */
776 readl(iommu->reg + DMAR_FECTL_REG);
777 spin_unlock_irqrestore(&iommu->register_lock, flag);
778 }
779
780 void dmar_msi_write(int irq, struct msi_msg *msg)
781 {
782 struct intel_iommu *iommu = get_irq_data(irq);
783 unsigned long flag;
784
785 spin_lock_irqsave(&iommu->register_lock, flag);
786 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
787 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
788 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
789 spin_unlock_irqrestore(&iommu->register_lock, flag);
790 }
791
792 void dmar_msi_read(int irq, struct msi_msg *msg)
793 {
794 struct intel_iommu *iommu = get_irq_data(irq);
795 unsigned long flag;
796
797 spin_lock_irqsave(&iommu->register_lock, flag);
798 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
799 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
800 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
801 spin_unlock_irqrestore(&iommu->register_lock, flag);
802 }
803
804 static int iommu_page_fault_do_one(struct intel_iommu *iommu, int type,
805 u8 fault_reason, u16 source_id, unsigned long long addr)
806 {
807 const char *reason;
808
809 reason = dmar_get_fault_reason(fault_reason);
810
811 printk(KERN_ERR
812 "DMAR:[%s] Request device [%02x:%02x.%d] "
813 "fault addr %llx \n"
814 "DMAR:[fault reason %02d] %s\n",
815 (type ? "DMA Read" : "DMA Write"),
816 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
817 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
818 return 0;
819 }
820
821 #define PRIMARY_FAULT_REG_LEN (16)
822 static irqreturn_t iommu_page_fault(int irq, void *dev_id)
823 {
824 struct intel_iommu *iommu = dev_id;
825 int reg, fault_index;
826 u32 fault_status;
827 unsigned long flag;
828
829 spin_lock_irqsave(&iommu->register_lock, flag);
830 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
831
832 /* TBD: ignore advanced fault log currently */
833 if (!(fault_status & DMA_FSTS_PPF))
834 goto clear_overflow;
835
836 fault_index = dma_fsts_fault_record_index(fault_status);
837 reg = cap_fault_reg_offset(iommu->cap);
838 while (1) {
839 u8 fault_reason;
840 u16 source_id;
841 u64 guest_addr;
842 int type;
843 u32 data;
844
845 /* highest 32 bits */
846 data = readl(iommu->reg + reg +
847 fault_index * PRIMARY_FAULT_REG_LEN + 12);
848 if (!(data & DMA_FRCD_F))
849 break;
850
851 fault_reason = dma_frcd_fault_reason(data);
852 type = dma_frcd_type(data);
853
854 data = readl(iommu->reg + reg +
855 fault_index * PRIMARY_FAULT_REG_LEN + 8);
856 source_id = dma_frcd_source_id(data);
857
858 guest_addr = dmar_readq(iommu->reg + reg +
859 fault_index * PRIMARY_FAULT_REG_LEN);
860 guest_addr = dma_frcd_page_addr(guest_addr);
861 /* clear the fault */
862 writel(DMA_FRCD_F, iommu->reg + reg +
863 fault_index * PRIMARY_FAULT_REG_LEN + 12);
864
865 spin_unlock_irqrestore(&iommu->register_lock, flag);
866
867 iommu_page_fault_do_one(iommu, type, fault_reason,
868 source_id, guest_addr);
869
870 fault_index++;
871 if (fault_index > cap_num_fault_regs(iommu->cap))
872 fault_index = 0;
873 spin_lock_irqsave(&iommu->register_lock, flag);
874 }
875 clear_overflow:
876 /* clear primary fault overflow */
877 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
878 if (fault_status & DMA_FSTS_PFO)
879 writel(DMA_FSTS_PFO, iommu->reg + DMAR_FSTS_REG);
880
881 spin_unlock_irqrestore(&iommu->register_lock, flag);
882 return IRQ_HANDLED;
883 }
884
885 int dmar_set_interrupt(struct intel_iommu *iommu)
886 {
887 int irq, ret;
888
889 irq = create_irq();
890 if (!irq) {
891 printk(KERN_ERR "IOMMU: no free vectors\n");
892 return -EINVAL;
893 }
894
895 set_irq_data(irq, iommu);
896 iommu->irq = irq;
897
898 ret = arch_setup_dmar_msi(irq);
899 if (ret) {
900 set_irq_data(irq, NULL);
901 iommu->irq = 0;
902 destroy_irq(irq);
903 return 0;
904 }
905
906 /* Force fault register is cleared */
907 iommu_page_fault(irq, iommu);
908
909 ret = request_irq(irq, iommu_page_fault, 0, iommu->name, iommu);
910 if (ret)
911 printk(KERN_ERR "IOMMU: can't request irq\n");
912 return ret;
913 }
914
915 static int iommu_init_domains(struct intel_iommu *iommu)
916 {
917 unsigned long ndomains;
918 unsigned long nlongs;
919
920 ndomains = cap_ndoms(iommu->cap);
921 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
922 nlongs = BITS_TO_LONGS(ndomains);
923
924 /* TBD: there might be 64K domains,
925 * consider other allocation for future chip
926 */
927 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
928 if (!iommu->domain_ids) {
929 printk(KERN_ERR "Allocating domain id array failed\n");
930 return -ENOMEM;
931 }
932 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
933 GFP_KERNEL);
934 if (!iommu->domains) {
935 printk(KERN_ERR "Allocating domain array failed\n");
936 kfree(iommu->domain_ids);
937 return -ENOMEM;
938 }
939
940 spin_lock_init(&iommu->lock);
941
942 /*
943 * if Caching mode is set, then invalid translations are tagged
944 * with domainid 0. Hence we need to pre-allocate it.
945 */
946 if (cap_caching_mode(iommu->cap))
947 set_bit(0, iommu->domain_ids);
948 return 0;
949 }
950
951
952 static void domain_exit(struct dmar_domain *domain);
953
954 void free_dmar_iommu(struct intel_iommu *iommu)
955 {
956 struct dmar_domain *domain;
957 int i;
958
959 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
960 for (; i < cap_ndoms(iommu->cap); ) {
961 domain = iommu->domains[i];
962 clear_bit(i, iommu->domain_ids);
963 domain_exit(domain);
964 i = find_next_bit(iommu->domain_ids,
965 cap_ndoms(iommu->cap), i+1);
966 }
967
968 if (iommu->gcmd & DMA_GCMD_TE)
969 iommu_disable_translation(iommu);
970
971 if (iommu->irq) {
972 set_irq_data(iommu->irq, NULL);
973 /* This will mask the irq */
974 free_irq(iommu->irq, iommu);
975 destroy_irq(iommu->irq);
976 }
977
978 kfree(iommu->domains);
979 kfree(iommu->domain_ids);
980
981 /* free context mapping */
982 free_context_table(iommu);
983 }
984
985 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
986 {
987 unsigned long num;
988 unsigned long ndomains;
989 struct dmar_domain *domain;
990 unsigned long flags;
991
992 domain = alloc_domain_mem();
993 if (!domain)
994 return NULL;
995
996 ndomains = cap_ndoms(iommu->cap);
997
998 spin_lock_irqsave(&iommu->lock, flags);
999 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1000 if (num >= ndomains) {
1001 spin_unlock_irqrestore(&iommu->lock, flags);
1002 free_domain_mem(domain);
1003 printk(KERN_ERR "IOMMU: no free domain ids\n");
1004 return NULL;
1005 }
1006
1007 set_bit(num, iommu->domain_ids);
1008 domain->id = num;
1009 domain->iommu = iommu;
1010 iommu->domains[num] = domain;
1011 spin_unlock_irqrestore(&iommu->lock, flags);
1012
1013 return domain;
1014 }
1015
1016 static void iommu_free_domain(struct dmar_domain *domain)
1017 {
1018 unsigned long flags;
1019
1020 spin_lock_irqsave(&domain->iommu->lock, flags);
1021 clear_bit(domain->id, domain->iommu->domain_ids);
1022 spin_unlock_irqrestore(&domain->iommu->lock, flags);
1023 }
1024
1025 static struct iova_domain reserved_iova_list;
1026 static struct lock_class_key reserved_alloc_key;
1027 static struct lock_class_key reserved_rbtree_key;
1028
1029 static void dmar_init_reserved_ranges(void)
1030 {
1031 struct pci_dev *pdev = NULL;
1032 struct iova *iova;
1033 int i;
1034 u64 addr, size;
1035
1036 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1037
1038 lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1039 &reserved_alloc_key);
1040 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1041 &reserved_rbtree_key);
1042
1043 /* IOAPIC ranges shouldn't be accessed by DMA */
1044 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1045 IOVA_PFN(IOAPIC_RANGE_END));
1046 if (!iova)
1047 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1048
1049 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1050 for_each_pci_dev(pdev) {
1051 struct resource *r;
1052
1053 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1054 r = &pdev->resource[i];
1055 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1056 continue;
1057 addr = r->start;
1058 addr &= PAGE_MASK;
1059 size = r->end - addr;
1060 size = PAGE_ALIGN(size);
1061 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1062 IOVA_PFN(size + addr) - 1);
1063 if (!iova)
1064 printk(KERN_ERR "Reserve iova failed\n");
1065 }
1066 }
1067
1068 }
1069
1070 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1071 {
1072 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1073 }
1074
1075 static inline int guestwidth_to_adjustwidth(int gaw)
1076 {
1077 int agaw;
1078 int r = (gaw - 12) % 9;
1079
1080 if (r == 0)
1081 agaw = gaw;
1082 else
1083 agaw = gaw + 9 - r;
1084 if (agaw > 64)
1085 agaw = 64;
1086 return agaw;
1087 }
1088
1089 static int domain_init(struct dmar_domain *domain, int guest_width)
1090 {
1091 struct intel_iommu *iommu;
1092 int adjust_width, agaw;
1093 unsigned long sagaw;
1094
1095 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1096 spin_lock_init(&domain->mapping_lock);
1097
1098 domain_reserve_special_ranges(domain);
1099
1100 /* calculate AGAW */
1101 iommu = domain->iommu;
1102 if (guest_width > cap_mgaw(iommu->cap))
1103 guest_width = cap_mgaw(iommu->cap);
1104 domain->gaw = guest_width;
1105 adjust_width = guestwidth_to_adjustwidth(guest_width);
1106 agaw = width_to_agaw(adjust_width);
1107 sagaw = cap_sagaw(iommu->cap);
1108 if (!test_bit(agaw, &sagaw)) {
1109 /* hardware doesn't support it, choose a bigger one */
1110 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1111 agaw = find_next_bit(&sagaw, 5, agaw);
1112 if (agaw >= 5)
1113 return -ENODEV;
1114 }
1115 domain->agaw = agaw;
1116 INIT_LIST_HEAD(&domain->devices);
1117
1118 /* always allocate the top pgd */
1119 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1120 if (!domain->pgd)
1121 return -ENOMEM;
1122 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1123 return 0;
1124 }
1125
1126 static void domain_exit(struct dmar_domain *domain)
1127 {
1128 u64 end;
1129
1130 /* Domain 0 is reserved, so dont process it */
1131 if (!domain)
1132 return;
1133
1134 domain_remove_dev_info(domain);
1135 /* destroy iovas */
1136 put_iova_domain(&domain->iovad);
1137 end = DOMAIN_MAX_ADDR(domain->gaw);
1138 end = end & (~PAGE_MASK);
1139
1140 /* clear ptes */
1141 dma_pte_clear_range(domain, 0, end);
1142
1143 /* free page tables */
1144 dma_pte_free_pagetable(domain, 0, end);
1145
1146 iommu_free_domain(domain);
1147 free_domain_mem(domain);
1148 }
1149
1150 static int domain_context_mapping_one(struct dmar_domain *domain,
1151 u8 bus, u8 devfn)
1152 {
1153 struct context_entry *context;
1154 struct intel_iommu *iommu = domain->iommu;
1155 unsigned long flags;
1156
1157 pr_debug("Set context mapping for %02x:%02x.%d\n",
1158 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1159 BUG_ON(!domain->pgd);
1160 context = device_to_context_entry(iommu, bus, devfn);
1161 if (!context)
1162 return -ENOMEM;
1163 spin_lock_irqsave(&iommu->lock, flags);
1164 if (context_present(*context)) {
1165 spin_unlock_irqrestore(&iommu->lock, flags);
1166 return 0;
1167 }
1168
1169 context_set_domain_id(*context, domain->id);
1170 context_set_address_width(*context, domain->agaw);
1171 context_set_address_root(*context, virt_to_phys(domain->pgd));
1172 context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1173 context_set_fault_enable(*context);
1174 context_set_present(*context);
1175 __iommu_flush_cache(iommu, context, sizeof(*context));
1176
1177 /* it's a non-present to present mapping */
1178 if (iommu->flush.flush_context(iommu, domain->id,
1179 (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT,
1180 DMA_CCMD_DEVICE_INVL, 1))
1181 iommu_flush_write_buffer(iommu);
1182 else
1183 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH, 0);
1184
1185 spin_unlock_irqrestore(&iommu->lock, flags);
1186 return 0;
1187 }
1188
1189 static int
1190 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1191 {
1192 int ret;
1193 struct pci_dev *tmp, *parent;
1194
1195 ret = domain_context_mapping_one(domain, pdev->bus->number,
1196 pdev->devfn);
1197 if (ret)
1198 return ret;
1199
1200 /* dependent device mapping */
1201 tmp = pci_find_upstream_pcie_bridge(pdev);
1202 if (!tmp)
1203 return 0;
1204 /* Secondary interface's bus number and devfn 0 */
1205 parent = pdev->bus->self;
1206 while (parent != tmp) {
1207 ret = domain_context_mapping_one(domain, parent->bus->number,
1208 parent->devfn);
1209 if (ret)
1210 return ret;
1211 parent = parent->bus->self;
1212 }
1213 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1214 return domain_context_mapping_one(domain,
1215 tmp->subordinate->number, 0);
1216 else /* this is a legacy PCI bridge */
1217 return domain_context_mapping_one(domain,
1218 tmp->bus->number, tmp->devfn);
1219 }
1220
1221 static int domain_context_mapped(struct dmar_domain *domain,
1222 struct pci_dev *pdev)
1223 {
1224 int ret;
1225 struct pci_dev *tmp, *parent;
1226
1227 ret = device_context_mapped(domain->iommu,
1228 pdev->bus->number, pdev->devfn);
1229 if (!ret)
1230 return ret;
1231 /* dependent device mapping */
1232 tmp = pci_find_upstream_pcie_bridge(pdev);
1233 if (!tmp)
1234 return ret;
1235 /* Secondary interface's bus number and devfn 0 */
1236 parent = pdev->bus->self;
1237 while (parent != tmp) {
1238 ret = device_context_mapped(domain->iommu, parent->bus->number,
1239 parent->devfn);
1240 if (!ret)
1241 return ret;
1242 parent = parent->bus->self;
1243 }
1244 if (tmp->is_pcie)
1245 return device_context_mapped(domain->iommu,
1246 tmp->subordinate->number, 0);
1247 else
1248 return device_context_mapped(domain->iommu,
1249 tmp->bus->number, tmp->devfn);
1250 }
1251
1252 static int
1253 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1254 u64 hpa, size_t size, int prot)
1255 {
1256 u64 start_pfn, end_pfn;
1257 struct dma_pte *pte;
1258 int index;
1259 int addr_width = agaw_to_width(domain->agaw);
1260
1261 hpa &= (((u64)1) << addr_width) - 1;
1262
1263 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1264 return -EINVAL;
1265 iova &= PAGE_MASK;
1266 start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT;
1267 end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT;
1268 index = 0;
1269 while (start_pfn < end_pfn) {
1270 pte = addr_to_dma_pte(domain, iova + VTD_PAGE_SIZE * index);
1271 if (!pte)
1272 return -ENOMEM;
1273 /* We don't need lock here, nobody else
1274 * touches the iova range
1275 */
1276 BUG_ON(dma_pte_addr(*pte));
1277 dma_set_pte_addr(*pte, start_pfn << VTD_PAGE_SHIFT);
1278 dma_set_pte_prot(*pte, prot);
1279 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
1280 start_pfn++;
1281 index++;
1282 }
1283 return 0;
1284 }
1285
1286 static void detach_domain_for_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
1287 {
1288 clear_context_table(domain->iommu, bus, devfn);
1289 domain->iommu->flush.flush_context(domain->iommu, 0, 0, 0,
1290 DMA_CCMD_GLOBAL_INVL, 0);
1291 domain->iommu->flush.flush_iotlb(domain->iommu, 0, 0, 0,
1292 DMA_TLB_GLOBAL_FLUSH, 0);
1293 }
1294
1295 static void domain_remove_dev_info(struct dmar_domain *domain)
1296 {
1297 struct device_domain_info *info;
1298 unsigned long flags;
1299
1300 spin_lock_irqsave(&device_domain_lock, flags);
1301 while (!list_empty(&domain->devices)) {
1302 info = list_entry(domain->devices.next,
1303 struct device_domain_info, link);
1304 list_del(&info->link);
1305 list_del(&info->global);
1306 if (info->dev)
1307 info->dev->dev.archdata.iommu = NULL;
1308 spin_unlock_irqrestore(&device_domain_lock, flags);
1309
1310 detach_domain_for_dev(info->domain, info->bus, info->devfn);
1311 free_devinfo_mem(info);
1312
1313 spin_lock_irqsave(&device_domain_lock, flags);
1314 }
1315 spin_unlock_irqrestore(&device_domain_lock, flags);
1316 }
1317
1318 /*
1319 * find_domain
1320 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1321 */
1322 static struct dmar_domain *
1323 find_domain(struct pci_dev *pdev)
1324 {
1325 struct device_domain_info *info;
1326
1327 /* No lock here, assumes no domain exit in normal case */
1328 info = pdev->dev.archdata.iommu;
1329 if (info)
1330 return info->domain;
1331 return NULL;
1332 }
1333
1334 /* domain is initialized */
1335 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1336 {
1337 struct dmar_domain *domain, *found = NULL;
1338 struct intel_iommu *iommu;
1339 struct dmar_drhd_unit *drhd;
1340 struct device_domain_info *info, *tmp;
1341 struct pci_dev *dev_tmp;
1342 unsigned long flags;
1343 int bus = 0, devfn = 0;
1344
1345 domain = find_domain(pdev);
1346 if (domain)
1347 return domain;
1348
1349 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1350 if (dev_tmp) {
1351 if (dev_tmp->is_pcie) {
1352 bus = dev_tmp->subordinate->number;
1353 devfn = 0;
1354 } else {
1355 bus = dev_tmp->bus->number;
1356 devfn = dev_tmp->devfn;
1357 }
1358 spin_lock_irqsave(&device_domain_lock, flags);
1359 list_for_each_entry(info, &device_domain_list, global) {
1360 if (info->bus == bus && info->devfn == devfn) {
1361 found = info->domain;
1362 break;
1363 }
1364 }
1365 spin_unlock_irqrestore(&device_domain_lock, flags);
1366 /* pcie-pci bridge already has a domain, uses it */
1367 if (found) {
1368 domain = found;
1369 goto found_domain;
1370 }
1371 }
1372
1373 /* Allocate new domain for the device */
1374 drhd = dmar_find_matched_drhd_unit(pdev);
1375 if (!drhd) {
1376 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1377 pci_name(pdev));
1378 return NULL;
1379 }
1380 iommu = drhd->iommu;
1381
1382 domain = iommu_alloc_domain(iommu);
1383 if (!domain)
1384 goto error;
1385
1386 if (domain_init(domain, gaw)) {
1387 domain_exit(domain);
1388 goto error;
1389 }
1390
1391 /* register pcie-to-pci device */
1392 if (dev_tmp) {
1393 info = alloc_devinfo_mem();
1394 if (!info) {
1395 domain_exit(domain);
1396 goto error;
1397 }
1398 info->bus = bus;
1399 info->devfn = devfn;
1400 info->dev = NULL;
1401 info->domain = domain;
1402 /* This domain is shared by devices under p2p bridge */
1403 domain->flags |= DOMAIN_FLAG_MULTIPLE_DEVICES;
1404
1405 /* pcie-to-pci bridge already has a domain, uses it */
1406 found = NULL;
1407 spin_lock_irqsave(&device_domain_lock, flags);
1408 list_for_each_entry(tmp, &device_domain_list, global) {
1409 if (tmp->bus == bus && tmp->devfn == devfn) {
1410 found = tmp->domain;
1411 break;
1412 }
1413 }
1414 if (found) {
1415 free_devinfo_mem(info);
1416 domain_exit(domain);
1417 domain = found;
1418 } else {
1419 list_add(&info->link, &domain->devices);
1420 list_add(&info->global, &device_domain_list);
1421 }
1422 spin_unlock_irqrestore(&device_domain_lock, flags);
1423 }
1424
1425 found_domain:
1426 info = alloc_devinfo_mem();
1427 if (!info)
1428 goto error;
1429 info->bus = pdev->bus->number;
1430 info->devfn = pdev->devfn;
1431 info->dev = pdev;
1432 info->domain = domain;
1433 spin_lock_irqsave(&device_domain_lock, flags);
1434 /* somebody is fast */
1435 found = find_domain(pdev);
1436 if (found != NULL) {
1437 spin_unlock_irqrestore(&device_domain_lock, flags);
1438 if (found != domain) {
1439 domain_exit(domain);
1440 domain = found;
1441 }
1442 free_devinfo_mem(info);
1443 return domain;
1444 }
1445 list_add(&info->link, &domain->devices);
1446 list_add(&info->global, &device_domain_list);
1447 pdev->dev.archdata.iommu = info;
1448 spin_unlock_irqrestore(&device_domain_lock, flags);
1449 return domain;
1450 error:
1451 /* recheck it here, maybe others set it */
1452 return find_domain(pdev);
1453 }
1454
1455 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1456 unsigned long long start,
1457 unsigned long long end)
1458 {
1459 struct dmar_domain *domain;
1460 unsigned long size;
1461 unsigned long long base;
1462 int ret;
1463
1464 printk(KERN_INFO
1465 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1466 pci_name(pdev), start, end);
1467 /* page table init */
1468 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1469 if (!domain)
1470 return -ENOMEM;
1471
1472 /* The address might not be aligned */
1473 base = start & PAGE_MASK;
1474 size = end - base;
1475 size = PAGE_ALIGN(size);
1476 if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1477 IOVA_PFN(base + size) - 1)) {
1478 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1479 ret = -ENOMEM;
1480 goto error;
1481 }
1482
1483 pr_debug("Mapping reserved region %lx@%llx for %s\n",
1484 size, base, pci_name(pdev));
1485 /*
1486 * RMRR range might have overlap with physical memory range,
1487 * clear it first
1488 */
1489 dma_pte_clear_range(domain, base, base + size);
1490
1491 ret = domain_page_mapping(domain, base, base, size,
1492 DMA_PTE_READ|DMA_PTE_WRITE);
1493 if (ret)
1494 goto error;
1495
1496 /* context entry init */
1497 ret = domain_context_mapping(domain, pdev);
1498 if (!ret)
1499 return 0;
1500 error:
1501 domain_exit(domain);
1502 return ret;
1503
1504 }
1505
1506 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1507 struct pci_dev *pdev)
1508 {
1509 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1510 return 0;
1511 return iommu_prepare_identity_map(pdev, rmrr->base_address,
1512 rmrr->end_address + 1);
1513 }
1514
1515 #ifdef CONFIG_DMAR_GFX_WA
1516 struct iommu_prepare_data {
1517 struct pci_dev *pdev;
1518 int ret;
1519 };
1520
1521 static int __init iommu_prepare_work_fn(unsigned long start_pfn,
1522 unsigned long end_pfn, void *datax)
1523 {
1524 struct iommu_prepare_data *data;
1525
1526 data = (struct iommu_prepare_data *)datax;
1527
1528 data->ret = iommu_prepare_identity_map(data->pdev,
1529 start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
1530 return data->ret;
1531
1532 }
1533
1534 static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
1535 {
1536 int nid;
1537 struct iommu_prepare_data data;
1538
1539 data.pdev = pdev;
1540 data.ret = 0;
1541
1542 for_each_online_node(nid) {
1543 work_with_active_regions(nid, iommu_prepare_work_fn, &data);
1544 if (data.ret)
1545 return data.ret;
1546 }
1547 return data.ret;
1548 }
1549
1550 static void __init iommu_prepare_gfx_mapping(void)
1551 {
1552 struct pci_dev *pdev = NULL;
1553 int ret;
1554
1555 for_each_pci_dev(pdev) {
1556 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
1557 !IS_GFX_DEVICE(pdev))
1558 continue;
1559 printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
1560 pci_name(pdev));
1561 ret = iommu_prepare_with_active_regions(pdev);
1562 if (ret)
1563 printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
1564 }
1565 }
1566 #endif
1567
1568 #ifdef CONFIG_DMAR_FLOPPY_WA
1569 static inline void iommu_prepare_isa(void)
1570 {
1571 struct pci_dev *pdev;
1572 int ret;
1573
1574 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1575 if (!pdev)
1576 return;
1577
1578 printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n");
1579 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1580
1581 if (ret)
1582 printk("IOMMU: Failed to create 0-64M identity map, "
1583 "floppy might not work\n");
1584
1585 }
1586 #else
1587 static inline void iommu_prepare_isa(void)
1588 {
1589 return;
1590 }
1591 #endif /* !CONFIG_DMAR_FLPY_WA */
1592
1593 int __init init_dmars(void)
1594 {
1595 struct dmar_drhd_unit *drhd;
1596 struct dmar_rmrr_unit *rmrr;
1597 struct pci_dev *pdev;
1598 struct intel_iommu *iommu;
1599 int i, ret, unit = 0;
1600
1601 /*
1602 * for each drhd
1603 * allocate root
1604 * initialize and program root entry to not present
1605 * endfor
1606 */
1607 for_each_drhd_unit(drhd) {
1608 g_num_of_iommus++;
1609 /*
1610 * lock not needed as this is only incremented in the single
1611 * threaded kernel __init code path all other access are read
1612 * only
1613 */
1614 }
1615
1616 deferred_flush = kzalloc(g_num_of_iommus *
1617 sizeof(struct deferred_flush_tables), GFP_KERNEL);
1618 if (!deferred_flush) {
1619 ret = -ENOMEM;
1620 goto error;
1621 }
1622
1623 for_each_drhd_unit(drhd) {
1624 if (drhd->ignored)
1625 continue;
1626
1627 iommu = drhd->iommu;
1628
1629 ret = iommu_init_domains(iommu);
1630 if (ret)
1631 goto error;
1632
1633 /*
1634 * TBD:
1635 * we could share the same root & context tables
1636 * amoung all IOMMU's. Need to Split it later.
1637 */
1638 ret = iommu_alloc_root_entry(iommu);
1639 if (ret) {
1640 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1641 goto error;
1642 }
1643 }
1644
1645 for_each_drhd_unit(drhd) {
1646 if (drhd->ignored)
1647 continue;
1648
1649 iommu = drhd->iommu;
1650 if (dmar_enable_qi(iommu)) {
1651 /*
1652 * Queued Invalidate not enabled, use Register Based
1653 * Invalidate
1654 */
1655 iommu->flush.flush_context = __iommu_flush_context;
1656 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
1657 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
1658 "invalidation\n", drhd->reg_base_addr);
1659 } else {
1660 iommu->flush.flush_context = qi_flush_context;
1661 iommu->flush.flush_iotlb = qi_flush_iotlb;
1662 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
1663 "invalidation\n", drhd->reg_base_addr);
1664 }
1665 }
1666
1667 /*
1668 * For each rmrr
1669 * for each dev attached to rmrr
1670 * do
1671 * locate drhd for dev, alloc domain for dev
1672 * allocate free domain
1673 * allocate page table entries for rmrr
1674 * if context not allocated for bus
1675 * allocate and init context
1676 * set present in root table for this bus
1677 * init context with domain, translation etc
1678 * endfor
1679 * endfor
1680 */
1681 for_each_rmrr_units(rmrr) {
1682 for (i = 0; i < rmrr->devices_cnt; i++) {
1683 pdev = rmrr->devices[i];
1684 /* some BIOS lists non-exist devices in DMAR table */
1685 if (!pdev)
1686 continue;
1687 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1688 if (ret)
1689 printk(KERN_ERR
1690 "IOMMU: mapping reserved region failed\n");
1691 }
1692 }
1693
1694 iommu_prepare_gfx_mapping();
1695
1696 iommu_prepare_isa();
1697
1698 /*
1699 * for each drhd
1700 * enable fault log
1701 * global invalidate context cache
1702 * global invalidate iotlb
1703 * enable translation
1704 */
1705 for_each_drhd_unit(drhd) {
1706 if (drhd->ignored)
1707 continue;
1708 iommu = drhd->iommu;
1709 sprintf (iommu->name, "dmar%d", unit++);
1710
1711 iommu_flush_write_buffer(iommu);
1712
1713 ret = dmar_set_interrupt(iommu);
1714 if (ret)
1715 goto error;
1716
1717 iommu_set_root_entry(iommu);
1718
1719 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
1720 0);
1721 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
1722 0);
1723 iommu_disable_protect_mem_regions(iommu);
1724
1725 ret = iommu_enable_translation(iommu);
1726 if (ret)
1727 goto error;
1728 }
1729
1730 return 0;
1731 error:
1732 for_each_drhd_unit(drhd) {
1733 if (drhd->ignored)
1734 continue;
1735 iommu = drhd->iommu;
1736 free_iommu(iommu);
1737 }
1738 return ret;
1739 }
1740
1741 static inline u64 aligned_size(u64 host_addr, size_t size)
1742 {
1743 u64 addr;
1744 addr = (host_addr & (~PAGE_MASK)) + size;
1745 return PAGE_ALIGN(addr);
1746 }
1747
1748 struct iova *
1749 iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
1750 {
1751 struct iova *piova;
1752
1753 /* Make sure it's in range */
1754 end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
1755 if (!size || (IOVA_START_ADDR + size > end))
1756 return NULL;
1757
1758 piova = alloc_iova(&domain->iovad,
1759 size >> PAGE_SHIFT, IOVA_PFN(end), 1);
1760 return piova;
1761 }
1762
1763 static struct iova *
1764 __intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
1765 size_t size, u64 dma_mask)
1766 {
1767 struct pci_dev *pdev = to_pci_dev(dev);
1768 struct iova *iova = NULL;
1769
1770 if (dma_mask <= DMA_32BIT_MASK || dmar_forcedac)
1771 iova = iommu_alloc_iova(domain, size, dma_mask);
1772 else {
1773 /*
1774 * First try to allocate an io virtual address in
1775 * DMA_32BIT_MASK and if that fails then try allocating
1776 * from higher range
1777 */
1778 iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
1779 if (!iova)
1780 iova = iommu_alloc_iova(domain, size, dma_mask);
1781 }
1782
1783 if (!iova) {
1784 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
1785 return NULL;
1786 }
1787
1788 return iova;
1789 }
1790
1791 static struct dmar_domain *
1792 get_valid_domain_for_dev(struct pci_dev *pdev)
1793 {
1794 struct dmar_domain *domain;
1795 int ret;
1796
1797 domain = get_domain_for_dev(pdev,
1798 DEFAULT_DOMAIN_ADDRESS_WIDTH);
1799 if (!domain) {
1800 printk(KERN_ERR
1801 "Allocating domain for %s failed", pci_name(pdev));
1802 return NULL;
1803 }
1804
1805 /* make sure context mapping is ok */
1806 if (unlikely(!domain_context_mapped(domain, pdev))) {
1807 ret = domain_context_mapping(domain, pdev);
1808 if (ret) {
1809 printk(KERN_ERR
1810 "Domain context map for %s failed",
1811 pci_name(pdev));
1812 return NULL;
1813 }
1814 }
1815
1816 return domain;
1817 }
1818
1819 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
1820 size_t size, int dir, u64 dma_mask)
1821 {
1822 struct pci_dev *pdev = to_pci_dev(hwdev);
1823 struct dmar_domain *domain;
1824 phys_addr_t start_paddr;
1825 struct iova *iova;
1826 int prot = 0;
1827 int ret;
1828
1829 BUG_ON(dir == DMA_NONE);
1830 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1831 return paddr;
1832
1833 domain = get_valid_domain_for_dev(pdev);
1834 if (!domain)
1835 return 0;
1836
1837 size = aligned_size((u64)paddr, size);
1838
1839 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
1840 if (!iova)
1841 goto error;
1842
1843 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
1844
1845 /*
1846 * Check if DMAR supports zero-length reads on write only
1847 * mappings..
1848 */
1849 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
1850 !cap_zlr(domain->iommu->cap))
1851 prot |= DMA_PTE_READ;
1852 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1853 prot |= DMA_PTE_WRITE;
1854 /*
1855 * paddr - (paddr + size) might be partial page, we should map the whole
1856 * page. Note: if two part of one page are separately mapped, we
1857 * might have two guest_addr mapping to the same host paddr, but this
1858 * is not a big problem
1859 */
1860 ret = domain_page_mapping(domain, start_paddr,
1861 ((u64)paddr) & PAGE_MASK, size, prot);
1862 if (ret)
1863 goto error;
1864
1865 /* it's a non-present to present mapping */
1866 ret = iommu_flush_iotlb_psi(domain->iommu, domain->id,
1867 start_paddr, size >> VTD_PAGE_SHIFT, 1);
1868 if (ret)
1869 iommu_flush_write_buffer(domain->iommu);
1870
1871 return start_paddr + ((u64)paddr & (~PAGE_MASK));
1872
1873 error:
1874 if (iova)
1875 __free_iova(&domain->iovad, iova);
1876 printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
1877 pci_name(pdev), size, (unsigned long long)paddr, dir);
1878 return 0;
1879 }
1880
1881 dma_addr_t intel_map_single(struct device *hwdev, phys_addr_t paddr,
1882 size_t size, int dir)
1883 {
1884 return __intel_map_single(hwdev, paddr, size, dir,
1885 to_pci_dev(hwdev)->dma_mask);
1886 }
1887
1888 static void flush_unmaps(void)
1889 {
1890 int i, j;
1891
1892 timer_on = 0;
1893
1894 /* just flush them all */
1895 for (i = 0; i < g_num_of_iommus; i++) {
1896 if (deferred_flush[i].next) {
1897 struct intel_iommu *iommu =
1898 deferred_flush[i].domain[0]->iommu;
1899
1900 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1901 DMA_TLB_GLOBAL_FLUSH, 0);
1902 for (j = 0; j < deferred_flush[i].next; j++) {
1903 __free_iova(&deferred_flush[i].domain[j]->iovad,
1904 deferred_flush[i].iova[j]);
1905 }
1906 deferred_flush[i].next = 0;
1907 }
1908 }
1909
1910 list_size = 0;
1911 }
1912
1913 static void flush_unmaps_timeout(unsigned long data)
1914 {
1915 unsigned long flags;
1916
1917 spin_lock_irqsave(&async_umap_flush_lock, flags);
1918 flush_unmaps();
1919 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
1920 }
1921
1922 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
1923 {
1924 unsigned long flags;
1925 int next, iommu_id;
1926
1927 spin_lock_irqsave(&async_umap_flush_lock, flags);
1928 if (list_size == HIGH_WATER_MARK)
1929 flush_unmaps();
1930
1931 iommu_id = dom->iommu->seq_id;
1932
1933 next = deferred_flush[iommu_id].next;
1934 deferred_flush[iommu_id].domain[next] = dom;
1935 deferred_flush[iommu_id].iova[next] = iova;
1936 deferred_flush[iommu_id].next++;
1937
1938 if (!timer_on) {
1939 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
1940 timer_on = 1;
1941 }
1942 list_size++;
1943 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
1944 }
1945
1946 void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
1947 int dir)
1948 {
1949 struct pci_dev *pdev = to_pci_dev(dev);
1950 struct dmar_domain *domain;
1951 unsigned long start_addr;
1952 struct iova *iova;
1953
1954 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1955 return;
1956 domain = find_domain(pdev);
1957 BUG_ON(!domain);
1958
1959 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
1960 if (!iova)
1961 return;
1962
1963 start_addr = iova->pfn_lo << PAGE_SHIFT;
1964 size = aligned_size((u64)dev_addr, size);
1965
1966 pr_debug("Device %s unmapping: %lx@%llx\n",
1967 pci_name(pdev), size, (unsigned long long)start_addr);
1968
1969 /* clear the whole page */
1970 dma_pte_clear_range(domain, start_addr, start_addr + size);
1971 /* free page tables */
1972 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
1973 if (intel_iommu_strict) {
1974 if (iommu_flush_iotlb_psi(domain->iommu,
1975 domain->id, start_addr, size >> VTD_PAGE_SHIFT, 0))
1976 iommu_flush_write_buffer(domain->iommu);
1977 /* free iova */
1978 __free_iova(&domain->iovad, iova);
1979 } else {
1980 add_unmap(domain, iova);
1981 /*
1982 * queue up the release of the unmap to save the 1/6th of the
1983 * cpu used up by the iotlb flush operation...
1984 */
1985 }
1986 }
1987
1988 void *intel_alloc_coherent(struct device *hwdev, size_t size,
1989 dma_addr_t *dma_handle, gfp_t flags)
1990 {
1991 void *vaddr;
1992 int order;
1993
1994 size = PAGE_ALIGN(size);
1995 order = get_order(size);
1996 flags &= ~(GFP_DMA | GFP_DMA32);
1997
1998 vaddr = (void *)__get_free_pages(flags, order);
1999 if (!vaddr)
2000 return NULL;
2001 memset(vaddr, 0, size);
2002
2003 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2004 DMA_BIDIRECTIONAL,
2005 hwdev->coherent_dma_mask);
2006 if (*dma_handle)
2007 return vaddr;
2008 free_pages((unsigned long)vaddr, order);
2009 return NULL;
2010 }
2011
2012 void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2013 dma_addr_t dma_handle)
2014 {
2015 int order;
2016
2017 size = PAGE_ALIGN(size);
2018 order = get_order(size);
2019
2020 intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2021 free_pages((unsigned long)vaddr, order);
2022 }
2023
2024 #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
2025
2026 void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2027 int nelems, int dir)
2028 {
2029 int i;
2030 struct pci_dev *pdev = to_pci_dev(hwdev);
2031 struct dmar_domain *domain;
2032 unsigned long start_addr;
2033 struct iova *iova;
2034 size_t size = 0;
2035 void *addr;
2036 struct scatterlist *sg;
2037
2038 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2039 return;
2040
2041 domain = find_domain(pdev);
2042
2043 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2044 if (!iova)
2045 return;
2046 for_each_sg(sglist, sg, nelems, i) {
2047 addr = SG_ENT_VIRT_ADDRESS(sg);
2048 size += aligned_size((u64)addr, sg->length);
2049 }
2050
2051 start_addr = iova->pfn_lo << PAGE_SHIFT;
2052
2053 /* clear the whole page */
2054 dma_pte_clear_range(domain, start_addr, start_addr + size);
2055 /* free page tables */
2056 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2057
2058 if (iommu_flush_iotlb_psi(domain->iommu, domain->id, start_addr,
2059 size >> VTD_PAGE_SHIFT, 0))
2060 iommu_flush_write_buffer(domain->iommu);
2061
2062 /* free iova */
2063 __free_iova(&domain->iovad, iova);
2064 }
2065
2066 static int intel_nontranslate_map_sg(struct device *hddev,
2067 struct scatterlist *sglist, int nelems, int dir)
2068 {
2069 int i;
2070 struct scatterlist *sg;
2071
2072 for_each_sg(sglist, sg, nelems, i) {
2073 BUG_ON(!sg_page(sg));
2074 sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg));
2075 sg->dma_length = sg->length;
2076 }
2077 return nelems;
2078 }
2079
2080 int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2081 int dir)
2082 {
2083 void *addr;
2084 int i;
2085 struct pci_dev *pdev = to_pci_dev(hwdev);
2086 struct dmar_domain *domain;
2087 size_t size = 0;
2088 int prot = 0;
2089 size_t offset = 0;
2090 struct iova *iova = NULL;
2091 int ret;
2092 struct scatterlist *sg;
2093 unsigned long start_addr;
2094
2095 BUG_ON(dir == DMA_NONE);
2096 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2097 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2098
2099 domain = get_valid_domain_for_dev(pdev);
2100 if (!domain)
2101 return 0;
2102
2103 for_each_sg(sglist, sg, nelems, i) {
2104 addr = SG_ENT_VIRT_ADDRESS(sg);
2105 addr = (void *)virt_to_phys(addr);
2106 size += aligned_size((u64)addr, sg->length);
2107 }
2108
2109 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2110 if (!iova) {
2111 sglist->dma_length = 0;
2112 return 0;
2113 }
2114
2115 /*
2116 * Check if DMAR supports zero-length reads on write only
2117 * mappings..
2118 */
2119 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2120 !cap_zlr(domain->iommu->cap))
2121 prot |= DMA_PTE_READ;
2122 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2123 prot |= DMA_PTE_WRITE;
2124
2125 start_addr = iova->pfn_lo << PAGE_SHIFT;
2126 offset = 0;
2127 for_each_sg(sglist, sg, nelems, i) {
2128 addr = SG_ENT_VIRT_ADDRESS(sg);
2129 addr = (void *)virt_to_phys(addr);
2130 size = aligned_size((u64)addr, sg->length);
2131 ret = domain_page_mapping(domain, start_addr + offset,
2132 ((u64)addr) & PAGE_MASK,
2133 size, prot);
2134 if (ret) {
2135 /* clear the page */
2136 dma_pte_clear_range(domain, start_addr,
2137 start_addr + offset);
2138 /* free page tables */
2139 dma_pte_free_pagetable(domain, start_addr,
2140 start_addr + offset);
2141 /* free iova */
2142 __free_iova(&domain->iovad, iova);
2143 return 0;
2144 }
2145 sg->dma_address = start_addr + offset +
2146 ((u64)addr & (~PAGE_MASK));
2147 sg->dma_length = sg->length;
2148 offset += size;
2149 }
2150
2151 /* it's a non-present to present mapping */
2152 if (iommu_flush_iotlb_psi(domain->iommu, domain->id,
2153 start_addr, offset >> VTD_PAGE_SHIFT, 1))
2154 iommu_flush_write_buffer(domain->iommu);
2155 return nelems;
2156 }
2157
2158 static struct dma_mapping_ops intel_dma_ops = {
2159 .alloc_coherent = intel_alloc_coherent,
2160 .free_coherent = intel_free_coherent,
2161 .map_single = intel_map_single,
2162 .unmap_single = intel_unmap_single,
2163 .map_sg = intel_map_sg,
2164 .unmap_sg = intel_unmap_sg,
2165 };
2166
2167 static inline int iommu_domain_cache_init(void)
2168 {
2169 int ret = 0;
2170
2171 iommu_domain_cache = kmem_cache_create("iommu_domain",
2172 sizeof(struct dmar_domain),
2173 0,
2174 SLAB_HWCACHE_ALIGN,
2175
2176 NULL);
2177 if (!iommu_domain_cache) {
2178 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2179 ret = -ENOMEM;
2180 }
2181
2182 return ret;
2183 }
2184
2185 static inline int iommu_devinfo_cache_init(void)
2186 {
2187 int ret = 0;
2188
2189 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2190 sizeof(struct device_domain_info),
2191 0,
2192 SLAB_HWCACHE_ALIGN,
2193 NULL);
2194 if (!iommu_devinfo_cache) {
2195 printk(KERN_ERR "Couldn't create devinfo cache\n");
2196 ret = -ENOMEM;
2197 }
2198
2199 return ret;
2200 }
2201
2202 static inline int iommu_iova_cache_init(void)
2203 {
2204 int ret = 0;
2205
2206 iommu_iova_cache = kmem_cache_create("iommu_iova",
2207 sizeof(struct iova),
2208 0,
2209 SLAB_HWCACHE_ALIGN,
2210 NULL);
2211 if (!iommu_iova_cache) {
2212 printk(KERN_ERR "Couldn't create iova cache\n");
2213 ret = -ENOMEM;
2214 }
2215
2216 return ret;
2217 }
2218
2219 static int __init iommu_init_mempool(void)
2220 {
2221 int ret;
2222 ret = iommu_iova_cache_init();
2223 if (ret)
2224 return ret;
2225
2226 ret = iommu_domain_cache_init();
2227 if (ret)
2228 goto domain_error;
2229
2230 ret = iommu_devinfo_cache_init();
2231 if (!ret)
2232 return ret;
2233
2234 kmem_cache_destroy(iommu_domain_cache);
2235 domain_error:
2236 kmem_cache_destroy(iommu_iova_cache);
2237
2238 return -ENOMEM;
2239 }
2240
2241 static void __init iommu_exit_mempool(void)
2242 {
2243 kmem_cache_destroy(iommu_devinfo_cache);
2244 kmem_cache_destroy(iommu_domain_cache);
2245 kmem_cache_destroy(iommu_iova_cache);
2246
2247 }
2248
2249 static void __init init_no_remapping_devices(void)
2250 {
2251 struct dmar_drhd_unit *drhd;
2252
2253 for_each_drhd_unit(drhd) {
2254 if (!drhd->include_all) {
2255 int i;
2256 for (i = 0; i < drhd->devices_cnt; i++)
2257 if (drhd->devices[i] != NULL)
2258 break;
2259 /* ignore DMAR unit if no pci devices exist */
2260 if (i == drhd->devices_cnt)
2261 drhd->ignored = 1;
2262 }
2263 }
2264
2265 if (dmar_map_gfx)
2266 return;
2267
2268 for_each_drhd_unit(drhd) {
2269 int i;
2270 if (drhd->ignored || drhd->include_all)
2271 continue;
2272
2273 for (i = 0; i < drhd->devices_cnt; i++)
2274 if (drhd->devices[i] &&
2275 !IS_GFX_DEVICE(drhd->devices[i]))
2276 break;
2277
2278 if (i < drhd->devices_cnt)
2279 continue;
2280
2281 /* bypass IOMMU if it is just for gfx devices */
2282 drhd->ignored = 1;
2283 for (i = 0; i < drhd->devices_cnt; i++) {
2284 if (!drhd->devices[i])
2285 continue;
2286 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
2287 }
2288 }
2289 }
2290
2291 int __init intel_iommu_init(void)
2292 {
2293 int ret = 0;
2294
2295 if (dmar_table_init())
2296 return -ENODEV;
2297
2298 if (dmar_dev_scope_init())
2299 return -ENODEV;
2300
2301 /*
2302 * Check the need for DMA-remapping initialization now.
2303 * Above initialization will also be used by Interrupt-remapping.
2304 */
2305 if (no_iommu || swiotlb || dmar_disabled)
2306 return -ENODEV;
2307
2308 iommu_init_mempool();
2309 dmar_init_reserved_ranges();
2310
2311 init_no_remapping_devices();
2312
2313 ret = init_dmars();
2314 if (ret) {
2315 printk(KERN_ERR "IOMMU: dmar init failed\n");
2316 put_iova_domain(&reserved_iova_list);
2317 iommu_exit_mempool();
2318 return ret;
2319 }
2320 printk(KERN_INFO
2321 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2322
2323 init_timer(&unmap_timer);
2324 force_iommu = 1;
2325 dma_ops = &intel_dma_ops;
2326 return 0;
2327 }
2328
2329 void intel_iommu_domain_exit(struct dmar_domain *domain)
2330 {
2331 u64 end;
2332
2333 /* Domain 0 is reserved, so dont process it */
2334 if (!domain)
2335 return;
2336
2337 end = DOMAIN_MAX_ADDR(domain->gaw);
2338 end = end & (~VTD_PAGE_MASK);
2339
2340 /* clear ptes */
2341 dma_pte_clear_range(domain, 0, end);
2342
2343 /* free page tables */
2344 dma_pte_free_pagetable(domain, 0, end);
2345
2346 iommu_free_domain(domain);
2347 free_domain_mem(domain);
2348 }
2349 EXPORT_SYMBOL_GPL(intel_iommu_domain_exit);
2350
2351 struct dmar_domain *intel_iommu_domain_alloc(struct pci_dev *pdev)
2352 {
2353 struct dmar_drhd_unit *drhd;
2354 struct dmar_domain *domain;
2355 struct intel_iommu *iommu;
2356
2357 drhd = dmar_find_matched_drhd_unit(pdev);
2358 if (!drhd) {
2359 printk(KERN_ERR "intel_iommu_domain_alloc: drhd == NULL\n");
2360 return NULL;
2361 }
2362
2363 iommu = drhd->iommu;
2364 if (!iommu) {
2365 printk(KERN_ERR
2366 "intel_iommu_domain_alloc: iommu == NULL\n");
2367 return NULL;
2368 }
2369 domain = iommu_alloc_domain(iommu);
2370 if (!domain) {
2371 printk(KERN_ERR
2372 "intel_iommu_domain_alloc: domain == NULL\n");
2373 return NULL;
2374 }
2375 if (domain_init(domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2376 printk(KERN_ERR
2377 "intel_iommu_domain_alloc: domain_init() failed\n");
2378 intel_iommu_domain_exit(domain);
2379 return NULL;
2380 }
2381 return domain;
2382 }
2383 EXPORT_SYMBOL_GPL(intel_iommu_domain_alloc);
2384
2385 int intel_iommu_context_mapping(
2386 struct dmar_domain *domain, struct pci_dev *pdev)
2387 {
2388 int rc;
2389 rc = domain_context_mapping(domain, pdev);
2390 return rc;
2391 }
2392 EXPORT_SYMBOL_GPL(intel_iommu_context_mapping);
2393
2394 int intel_iommu_page_mapping(
2395 struct dmar_domain *domain, dma_addr_t iova,
2396 u64 hpa, size_t size, int prot)
2397 {
2398 int rc;
2399 rc = domain_page_mapping(domain, iova, hpa, size, prot);
2400 return rc;
2401 }
2402 EXPORT_SYMBOL_GPL(intel_iommu_page_mapping);
2403
2404 void intel_iommu_detach_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
2405 {
2406 detach_domain_for_dev(domain, bus, devfn);
2407 }
2408 EXPORT_SYMBOL_GPL(intel_iommu_detach_dev);
2409
2410 struct dmar_domain *
2411 intel_iommu_find_domain(struct pci_dev *pdev)
2412 {
2413 return find_domain(pdev);
2414 }
2415 EXPORT_SYMBOL_GPL(intel_iommu_find_domain);
2416
2417 int intel_iommu_found(void)
2418 {
2419 return g_num_of_iommus;
2420 }
2421 EXPORT_SYMBOL_GPL(intel_iommu_found);
2422
2423 u64 intel_iommu_iova_to_pfn(struct dmar_domain *domain, u64 iova)
2424 {
2425 struct dma_pte *pte;
2426 u64 pfn;
2427
2428 pfn = 0;
2429 pte = addr_to_dma_pte(domain, iova);
2430
2431 if (pte)
2432 pfn = dma_pte_addr(*pte);
2433
2434 return pfn >> VTD_PAGE_SHIFT;
2435 }
2436 EXPORT_SYMBOL_GPL(intel_iommu_iova_to_pfn);
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree