?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
sched: don't allow setuid to succeed if the user does not have rt bandwidth
[linux-2.6/mini2440.git] / drivers / pci / intel-iommu.c
blobf3f686581a9026685dc3e6c994b00e88ed88486c
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 * Author: Fenghua Yu <fenghua.yu@intel.com>
22 */
23
24 #include <linux/init.h>
25 #include <linux/bitmap.h>
26 #include <linux/debugfs.h>
27 #include <linux/slab.h>
28 #include <linux/irq.h>
29 #include <linux/interrupt.h>
30 #include <linux/spinlock.h>
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/dma-mapping.h>
34 #include <linux/mempool.h>
35 #include <linux/timer.h>
36 #include <linux/iova.h>
37 #include <linux/iommu.h>
38 #include <linux/intel-iommu.h>
39 #include <asm/cacheflush.h>
40 #include <asm/iommu.h>
41 #include "pci.h"
42
43 #define ROOT_SIZE VTD_PAGE_SIZE
44 #define CONTEXT_SIZE VTD_PAGE_SIZE
45
46 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
47 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
48
49 #define IOAPIC_RANGE_START (0xfee00000)
50 #define IOAPIC_RANGE_END (0xfeefffff)
51 #define IOVA_START_ADDR (0x1000)
52
53 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
54
55 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
56
57 #define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
58 #define DMA_32BIT_PFN IOVA_PFN(DMA_32BIT_MASK)
59 #define DMA_64BIT_PFN IOVA_PFN(DMA_64BIT_MASK)
60
61 /* global iommu list, set NULL for ignored DMAR units */
62 static struct intel_iommu **g_iommus;
63
64 static int rwbf_quirk;
65
66 /*
67 * 0: Present
68 * 1-11: Reserved
69 * 12-63: Context Ptr (12 - (haw-1))
70 * 64-127: Reserved
71 */
72 struct root_entry {
73 u64 val;
74 u64 rsvd1;
75 };
76 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
77 static inline bool root_present(struct root_entry *root)
78 {
79 return (root->val & 1);
80 }
81 static inline void set_root_present(struct root_entry *root)
82 {
83 root->val |= 1;
84 }
85 static inline void set_root_value(struct root_entry *root, unsigned long value)
86 {
87 root->val |= value & VTD_PAGE_MASK;
88 }
89
90 static inline struct context_entry *
91 get_context_addr_from_root(struct root_entry *root)
92 {
93 return (struct context_entry *)
94 (root_present(root)?phys_to_virt(
95 root->val & VTD_PAGE_MASK) :
96 NULL);
97 }
98
99 /*
100 * low 64 bits:
101 * 0: present
102 * 1: fault processing disable
103 * 2-3: translation type
104 * 12-63: address space root
105 * high 64 bits:
106 * 0-2: address width
107 * 3-6: aval
108 * 8-23: domain id
109 */
110 struct context_entry {
111 u64 lo;
112 u64 hi;
113 };
114
115 static inline bool context_present(struct context_entry *context)
116 {
117 return (context->lo & 1);
118 }
119 static inline void context_set_present(struct context_entry *context)
120 {
121 context->lo |= 1;
122 }
123
124 static inline void context_set_fault_enable(struct context_entry *context)
125 {
126 context->lo &= (((u64)-1) << 2) | 1;
127 }
128
129 #define CONTEXT_TT_MULTI_LEVEL 0
130
131 static inline void context_set_translation_type(struct context_entry *context,
132 unsigned long value)
133 {
134 context->lo &= (((u64)-1) << 4) | 3;
135 context->lo |= (value & 3) << 2;
136 }
137
138 static inline void context_set_address_root(struct context_entry *context,
139 unsigned long value)
140 {
141 context->lo |= value & VTD_PAGE_MASK;
142 }
143
144 static inline void context_set_address_width(struct context_entry *context,
145 unsigned long value)
146 {
147 context->hi |= value & 7;
148 }
149
150 static inline void context_set_domain_id(struct context_entry *context,
151 unsigned long value)
152 {
153 context->hi |= (value & ((1 << 16) - 1)) << 8;
154 }
155
156 static inline void context_clear_entry(struct context_entry *context)
157 {
158 context->lo = 0;
159 context->hi = 0;
160 }
161
162 /*
163 * 0: readable
164 * 1: writable
165 * 2-6: reserved
166 * 7: super page
167 * 8-11: available
168 * 12-63: Host physcial address
169 */
170 struct dma_pte {
171 u64 val;
172 };
173
174 static inline void dma_clear_pte(struct dma_pte *pte)
175 {
176 pte->val = 0;
177 }
178
179 static inline void dma_set_pte_readable(struct dma_pte *pte)
180 {
181 pte->val |= DMA_PTE_READ;
182 }
183
184 static inline void dma_set_pte_writable(struct dma_pte *pte)
185 {
186 pte->val |= DMA_PTE_WRITE;
187 }
188
189 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
190 {
191 pte->val = (pte->val & ~3) | (prot & 3);
192 }
193
194 static inline u64 dma_pte_addr(struct dma_pte *pte)
195 {
196 return (pte->val & VTD_PAGE_MASK);
197 }
198
199 static inline void dma_set_pte_addr(struct dma_pte *pte, u64 addr)
200 {
201 pte->val |= (addr & VTD_PAGE_MASK);
202 }
203
204 static inline bool dma_pte_present(struct dma_pte *pte)
205 {
206 return (pte->val & 3) != 0;
207 }
208
209 /* devices under the same p2p bridge are owned in one domain */
210 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
211
212 /* domain represents a virtual machine, more than one devices
213 * across iommus may be owned in one domain, e.g. kvm guest.
214 */
215 #define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
216
217 struct dmar_domain {
218 int id; /* domain id */
219 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
220
221 struct list_head devices; /* all devices' list */
222 struct iova_domain iovad; /* iova's that belong to this domain */
223
224 struct dma_pte *pgd; /* virtual address */
225 spinlock_t mapping_lock; /* page table lock */
226 int gaw; /* max guest address width */
227
228 /* adjusted guest address width, 0 is level 2 30-bit */
229 int agaw;
230
231 int flags; /* flags to find out type of domain */
232
233 int iommu_coherency;/* indicate coherency of iommu access */
234 int iommu_count; /* reference count of iommu */
235 spinlock_t iommu_lock; /* protect iommu set in domain */
236 u64 max_addr; /* maximum mapped address */
237 };
238
239 /* PCI domain-device relationship */
240 struct device_domain_info {
241 struct list_head link; /* link to domain siblings */
242 struct list_head global; /* link to global list */
243 u8 bus; /* PCI bus numer */
244 u8 devfn; /* PCI devfn number */
245 struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
246 struct dmar_domain *domain; /* pointer to domain */
247 };
248
249 static void flush_unmaps_timeout(unsigned long data);
250
251 DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
252
253 #define HIGH_WATER_MARK 250
254 struct deferred_flush_tables {
255 int next;
256 struct iova *iova[HIGH_WATER_MARK];
257 struct dmar_domain *domain[HIGH_WATER_MARK];
258 };
259
260 static struct deferred_flush_tables *deferred_flush;
261
262 /* bitmap for indexing intel_iommus */
263 static int g_num_of_iommus;
264
265 static DEFINE_SPINLOCK(async_umap_flush_lock);
266 static LIST_HEAD(unmaps_to_do);
267
268 static int timer_on;
269 static long list_size;
270
271 static void domain_remove_dev_info(struct dmar_domain *domain);
272
273 #ifdef CONFIG_DMAR_DEFAULT_ON
274 int dmar_disabled = 0;
275 #else
276 int dmar_disabled = 1;
277 #endif /*CONFIG_DMAR_DEFAULT_ON*/
278
279 static int __initdata dmar_map_gfx = 1;
280 static int dmar_forcedac;
281 static int intel_iommu_strict;
282
283 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
284 static DEFINE_SPINLOCK(device_domain_lock);
285 static LIST_HEAD(device_domain_list);
286
287 static struct iommu_ops intel_iommu_ops;
288
289 static int __init intel_iommu_setup(char *str)
290 {
291 if (!str)
292 return -EINVAL;
293 while (*str) {
294 if (!strncmp(str, "on", 2)) {
295 dmar_disabled = 0;
296 printk(KERN_INFO "Intel-IOMMU: enabled\n");
297 } else if (!strncmp(str, "off", 3)) {
298 dmar_disabled = 1;
299 printk(KERN_INFO "Intel-IOMMU: disabled\n");
300 } else if (!strncmp(str, "igfx_off", 8)) {
301 dmar_map_gfx = 0;
302 printk(KERN_INFO
303 "Intel-IOMMU: disable GFX device mapping\n");
304 } else if (!strncmp(str, "forcedac", 8)) {
305 printk(KERN_INFO
306 "Intel-IOMMU: Forcing DAC for PCI devices\n");
307 dmar_forcedac = 1;
308 } else if (!strncmp(str, "strict", 6)) {
309 printk(KERN_INFO
310 "Intel-IOMMU: disable batched IOTLB flush\n");
311 intel_iommu_strict = 1;
312 }
313
314 str += strcspn(str, ",");
315 while (*str == ',')
316 str++;
317 }
318 return 0;
319 }
320 __setup("intel_iommu=", intel_iommu_setup);
321
322 static struct kmem_cache *iommu_domain_cache;
323 static struct kmem_cache *iommu_devinfo_cache;
324 static struct kmem_cache *iommu_iova_cache;
325
326 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
327 {
328 unsigned int flags;
329 void *vaddr;
330
331 /* trying to avoid low memory issues */
332 flags = current->flags & PF_MEMALLOC;
333 current->flags |= PF_MEMALLOC;
334 vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
335 current->flags &= (~PF_MEMALLOC | flags);
336 return vaddr;
337 }
338
339
340 static inline void *alloc_pgtable_page(void)
341 {
342 unsigned int flags;
343 void *vaddr;
344
345 /* trying to avoid low memory issues */
346 flags = current->flags & PF_MEMALLOC;
347 current->flags |= PF_MEMALLOC;
348 vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
349 current->flags &= (~PF_MEMALLOC | flags);
350 return vaddr;
351 }
352
353 static inline void free_pgtable_page(void *vaddr)
354 {
355 free_page((unsigned long)vaddr);
356 }
357
358 static inline void *alloc_domain_mem(void)
359 {
360 return iommu_kmem_cache_alloc(iommu_domain_cache);
361 }
362
363 static void free_domain_mem(void *vaddr)
364 {
365 kmem_cache_free(iommu_domain_cache, vaddr);
366 }
367
368 static inline void * alloc_devinfo_mem(void)
369 {
370 return iommu_kmem_cache_alloc(iommu_devinfo_cache);
371 }
372
373 static inline void free_devinfo_mem(void *vaddr)
374 {
375 kmem_cache_free(iommu_devinfo_cache, vaddr);
376 }
377
378 struct iova *alloc_iova_mem(void)
379 {
380 return iommu_kmem_cache_alloc(iommu_iova_cache);
381 }
382
383 void free_iova_mem(struct iova *iova)
384 {
385 kmem_cache_free(iommu_iova_cache, iova);
386 }
387
388
389 static inline int width_to_agaw(int width);
390
391 /* calculate agaw for each iommu.
392 * "SAGAW" may be different across iommus, use a default agaw, and
393 * get a supported less agaw for iommus that don't support the default agaw.
394 */
395 int iommu_calculate_agaw(struct intel_iommu *iommu)
396 {
397 unsigned long sagaw;
398 int agaw = -1;
399
400 sagaw = cap_sagaw(iommu->cap);
401 for (agaw = width_to_agaw(DEFAULT_DOMAIN_ADDRESS_WIDTH);
402 agaw >= 0; agaw--) {
403 if (test_bit(agaw, &sagaw))
404 break;
405 }
406
407 return agaw;
408 }
409
410 /* in native case, each domain is related to only one iommu */
411 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
412 {
413 int iommu_id;
414
415 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
416
417 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
418 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
419 return NULL;
420
421 return g_iommus[iommu_id];
422 }
423
424 /* "Coherency" capability may be different across iommus */
425 static void domain_update_iommu_coherency(struct dmar_domain *domain)
426 {
427 int i;
428
429 domain->iommu_coherency = 1;
430
431 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
432 for (; i < g_num_of_iommus; ) {
433 if (!ecap_coherent(g_iommus[i]->ecap)) {
434 domain->iommu_coherency = 0;
435 break;
436 }
437 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
438 }
439 }
440
441 static struct intel_iommu *device_to_iommu(u8 bus, u8 devfn)
442 {
443 struct dmar_drhd_unit *drhd = NULL;
444 int i;
445
446 for_each_drhd_unit(drhd) {
447 if (drhd->ignored)
448 continue;
449
450 for (i = 0; i < drhd->devices_cnt; i++)
451 if (drhd->devices[i] &&
452 drhd->devices[i]->bus->number == bus &&
453 drhd->devices[i]->devfn == devfn)
454 return drhd->iommu;
455
456 if (drhd->include_all)
457 return drhd->iommu;
458 }
459
460 return NULL;
461 }
462
463 static void domain_flush_cache(struct dmar_domain *domain,
464 void *addr, int size)
465 {
466 if (!domain->iommu_coherency)
467 clflush_cache_range(addr, size);
468 }
469
470 /* Gets context entry for a given bus and devfn */
471 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
472 u8 bus, u8 devfn)
473 {
474 struct root_entry *root;
475 struct context_entry *context;
476 unsigned long phy_addr;
477 unsigned long flags;
478
479 spin_lock_irqsave(&iommu->lock, flags);
480 root = &iommu->root_entry[bus];
481 context = get_context_addr_from_root(root);
482 if (!context) {
483 context = (struct context_entry *)alloc_pgtable_page();
484 if (!context) {
485 spin_unlock_irqrestore(&iommu->lock, flags);
486 return NULL;
487 }
488 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
489 phy_addr = virt_to_phys((void *)context);
490 set_root_value(root, phy_addr);
491 set_root_present(root);
492 __iommu_flush_cache(iommu, root, sizeof(*root));
493 }
494 spin_unlock_irqrestore(&iommu->lock, flags);
495 return &context[devfn];
496 }
497
498 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
499 {
500 struct root_entry *root;
501 struct context_entry *context;
502 int ret;
503 unsigned long flags;
504
505 spin_lock_irqsave(&iommu->lock, flags);
506 root = &iommu->root_entry[bus];
507 context = get_context_addr_from_root(root);
508 if (!context) {
509 ret = 0;
510 goto out;
511 }
512 ret = context_present(&context[devfn]);
513 out:
514 spin_unlock_irqrestore(&iommu->lock, flags);
515 return ret;
516 }
517
518 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
519 {
520 struct root_entry *root;
521 struct context_entry *context;
522 unsigned long flags;
523
524 spin_lock_irqsave(&iommu->lock, flags);
525 root = &iommu->root_entry[bus];
526 context = get_context_addr_from_root(root);
527 if (context) {
528 context_clear_entry(&context[devfn]);
529 __iommu_flush_cache(iommu, &context[devfn], \
530 sizeof(*context));
531 }
532 spin_unlock_irqrestore(&iommu->lock, flags);
533 }
534
535 static void free_context_table(struct intel_iommu *iommu)
536 {
537 struct root_entry *root;
538 int i;
539 unsigned long flags;
540 struct context_entry *context;
541
542 spin_lock_irqsave(&iommu->lock, flags);
543 if (!iommu->root_entry) {
544 goto out;
545 }
546 for (i = 0; i < ROOT_ENTRY_NR; i++) {
547 root = &iommu->root_entry[i];
548 context = get_context_addr_from_root(root);
549 if (context)
550 free_pgtable_page(context);
551 }
552 free_pgtable_page(iommu->root_entry);
553 iommu->root_entry = NULL;
554 out:
555 spin_unlock_irqrestore(&iommu->lock, flags);
556 }
557
558 /* page table handling */
559 #define LEVEL_STRIDE (9)
560 #define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
561
562 static inline int agaw_to_level(int agaw)
563 {
564 return agaw + 2;
565 }
566
567 static inline int agaw_to_width(int agaw)
568 {
569 return 30 + agaw * LEVEL_STRIDE;
570
571 }
572
573 static inline int width_to_agaw(int width)
574 {
575 return (width - 30) / LEVEL_STRIDE;
576 }
577
578 static inline unsigned int level_to_offset_bits(int level)
579 {
580 return (12 + (level - 1) * LEVEL_STRIDE);
581 }
582
583 static inline int address_level_offset(u64 addr, int level)
584 {
585 return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
586 }
587
588 static inline u64 level_mask(int level)
589 {
590 return ((u64)-1 << level_to_offset_bits(level));
591 }
592
593 static inline u64 level_size(int level)
594 {
595 return ((u64)1 << level_to_offset_bits(level));
596 }
597
598 static inline u64 align_to_level(u64 addr, int level)
599 {
600 return ((addr + level_size(level) - 1) & level_mask(level));
601 }
602
603 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
604 {
605 int addr_width = agaw_to_width(domain->agaw);
606 struct dma_pte *parent, *pte = NULL;
607 int level = agaw_to_level(domain->agaw);
608 int offset;
609 unsigned long flags;
610
611 BUG_ON(!domain->pgd);
612
613 addr &= (((u64)1) << addr_width) - 1;
614 parent = domain->pgd;
615
616 spin_lock_irqsave(&domain->mapping_lock, flags);
617 while (level > 0) {
618 void *tmp_page;
619
620 offset = address_level_offset(addr, level);
621 pte = &parent[offset];
622 if (level == 1)
623 break;
624
625 if (!dma_pte_present(pte)) {
626 tmp_page = alloc_pgtable_page();
627
628 if (!tmp_page) {
629 spin_unlock_irqrestore(&domain->mapping_lock,
630 flags);
631 return NULL;
632 }
633 domain_flush_cache(domain, tmp_page, PAGE_SIZE);
634 dma_set_pte_addr(pte, virt_to_phys(tmp_page));
635 /*
636 * high level table always sets r/w, last level page
637 * table control read/write
638 */
639 dma_set_pte_readable(pte);
640 dma_set_pte_writable(pte);
641 domain_flush_cache(domain, pte, sizeof(*pte));
642 }
643 parent = phys_to_virt(dma_pte_addr(pte));
644 level--;
645 }
646
647 spin_unlock_irqrestore(&domain->mapping_lock, flags);
648 return pte;
649 }
650
651 /* return address's pte at specific level */
652 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
653 int level)
654 {
655 struct dma_pte *parent, *pte = NULL;
656 int total = agaw_to_level(domain->agaw);
657 int offset;
658
659 parent = domain->pgd;
660 while (level <= total) {
661 offset = address_level_offset(addr, total);
662 pte = &parent[offset];
663 if (level == total)
664 return pte;
665
666 if (!dma_pte_present(pte))
667 break;
668 parent = phys_to_virt(dma_pte_addr(pte));
669 total--;
670 }
671 return NULL;
672 }
673
674 /* clear one page's page table */
675 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
676 {
677 struct dma_pte *pte = NULL;
678
679 /* get last level pte */
680 pte = dma_addr_level_pte(domain, addr, 1);
681
682 if (pte) {
683 dma_clear_pte(pte);
684 domain_flush_cache(domain, pte, sizeof(*pte));
685 }
686 }
687
688 /* clear last level pte, a tlb flush should be followed */
689 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
690 {
691 int addr_width = agaw_to_width(domain->agaw);
692
693 start &= (((u64)1) << addr_width) - 1;
694 end &= (((u64)1) << addr_width) - 1;
695 /* in case it's partial page */
696 start = PAGE_ALIGN(start);
697 end &= PAGE_MASK;
698
699 /* we don't need lock here, nobody else touches the iova range */
700 while (start < end) {
701 dma_pte_clear_one(domain, start);
702 start += VTD_PAGE_SIZE;
703 }
704 }
705
706 /* free page table pages. last level pte should already be cleared */
707 static void dma_pte_free_pagetable(struct dmar_domain *domain,
708 u64 start, u64 end)
709 {
710 int addr_width = agaw_to_width(domain->agaw);
711 struct dma_pte *pte;
712 int total = agaw_to_level(domain->agaw);
713 int level;
714 u64 tmp;
715
716 start &= (((u64)1) << addr_width) - 1;
717 end &= (((u64)1) << addr_width) - 1;
718
719 /* we don't need lock here, nobody else touches the iova range */
720 level = 2;
721 while (level <= total) {
722 tmp = align_to_level(start, level);
723 if (tmp >= end || (tmp + level_size(level) > end))
724 return;
725
726 while (tmp < end) {
727 pte = dma_addr_level_pte(domain, tmp, level);
728 if (pte) {
729 free_pgtable_page(
730 phys_to_virt(dma_pte_addr(pte)));
731 dma_clear_pte(pte);
732 domain_flush_cache(domain, pte, sizeof(*pte));
733 }
734 tmp += level_size(level);
735 }
736 level++;
737 }
738 /* free pgd */
739 if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
740 free_pgtable_page(domain->pgd);
741 domain->pgd = NULL;
742 }
743 }
744
745 /* iommu handling */
746 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
747 {
748 struct root_entry *root;
749 unsigned long flags;
750
751 root = (struct root_entry *)alloc_pgtable_page();
752 if (!root)
753 return -ENOMEM;
754
755 __iommu_flush_cache(iommu, root, ROOT_SIZE);
756
757 spin_lock_irqsave(&iommu->lock, flags);
758 iommu->root_entry = root;
759 spin_unlock_irqrestore(&iommu->lock, flags);
760
761 return 0;
762 }
763
764 static void iommu_set_root_entry(struct intel_iommu *iommu)
765 {
766 void *addr;
767 u32 cmd, sts;
768 unsigned long flag;
769
770 addr = iommu->root_entry;
771
772 spin_lock_irqsave(&iommu->register_lock, flag);
773 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
774
775 cmd = iommu->gcmd | DMA_GCMD_SRTP;
776 writel(cmd, iommu->reg + DMAR_GCMD_REG);
777
778 /* Make sure hardware complete it */
779 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
780 readl, (sts & DMA_GSTS_RTPS), sts);
781
782 spin_unlock_irqrestore(&iommu->register_lock, flag);
783 }
784
785 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
786 {
787 u32 val;
788 unsigned long flag;
789
790 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
791 return;
792 val = iommu->gcmd | DMA_GCMD_WBF;
793
794 spin_lock_irqsave(&iommu->register_lock, flag);
795 writel(val, iommu->reg + DMAR_GCMD_REG);
796
797 /* Make sure hardware complete it */
798 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
799 readl, (!(val & DMA_GSTS_WBFS)), val);
800
801 spin_unlock_irqrestore(&iommu->register_lock, flag);
802 }
803
804 /* return value determine if we need a write buffer flush */
805 static int __iommu_flush_context(struct intel_iommu *iommu,
806 u16 did, u16 source_id, u8 function_mask, u64 type,
807 int non_present_entry_flush)
808 {
809 u64 val = 0;
810 unsigned long flag;
811
812 /*
813 * In the non-present entry flush case, if hardware doesn't cache
814 * non-present entry we do nothing and if hardware cache non-present
815 * entry, we flush entries of domain 0 (the domain id is used to cache
816 * any non-present entries)
817 */
818 if (non_present_entry_flush) {
819 if (!cap_caching_mode(iommu->cap))
820 return 1;
821 else
822 did = 0;
823 }
824
825 switch (type) {
826 case DMA_CCMD_GLOBAL_INVL:
827 val = DMA_CCMD_GLOBAL_INVL;
828 break;
829 case DMA_CCMD_DOMAIN_INVL:
830 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
831 break;
832 case DMA_CCMD_DEVICE_INVL:
833 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
834 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
835 break;
836 default:
837 BUG();
838 }
839 val |= DMA_CCMD_ICC;
840
841 spin_lock_irqsave(&iommu->register_lock, flag);
842 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
843
844 /* Make sure hardware complete it */
845 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
846 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
847
848 spin_unlock_irqrestore(&iommu->register_lock, flag);
849
850 /* flush context entry will implicitly flush write buffer */
851 return 0;
852 }
853
854 /* return value determine if we need a write buffer flush */
855 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
856 u64 addr, unsigned int size_order, u64 type,
857 int non_present_entry_flush)
858 {
859 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
860 u64 val = 0, val_iva = 0;
861 unsigned long flag;
862
863 /*
864 * In the non-present entry flush case, if hardware doesn't cache
865 * non-present entry we do nothing and if hardware cache non-present
866 * entry, we flush entries of domain 0 (the domain id is used to cache
867 * any non-present entries)
868 */
869 if (non_present_entry_flush) {
870 if (!cap_caching_mode(iommu->cap))
871 return 1;
872 else
873 did = 0;
874 }
875
876 switch (type) {
877 case DMA_TLB_GLOBAL_FLUSH:
878 /* global flush doesn't need set IVA_REG */
879 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
880 break;
881 case DMA_TLB_DSI_FLUSH:
882 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
883 break;
884 case DMA_TLB_PSI_FLUSH:
885 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
886 /* Note: always flush non-leaf currently */
887 val_iva = size_order | addr;
888 break;
889 default:
890 BUG();
891 }
892 /* Note: set drain read/write */
893 #if 0
894 /*
895 * This is probably to be super secure.. Looks like we can
896 * ignore it without any impact.
897 */
898 if (cap_read_drain(iommu->cap))
899 val |= DMA_TLB_READ_DRAIN;
900 #endif
901 if (cap_write_drain(iommu->cap))
902 val |= DMA_TLB_WRITE_DRAIN;
903
904 spin_lock_irqsave(&iommu->register_lock, flag);
905 /* Note: Only uses first TLB reg currently */
906 if (val_iva)
907 dmar_writeq(iommu->reg + tlb_offset, val_iva);
908 dmar_writeq(iommu->reg + tlb_offset + 8, val);
909
910 /* Make sure hardware complete it */
911 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
912 dmar_readq, (!(val & DMA_TLB_IVT)), val);
913
914 spin_unlock_irqrestore(&iommu->register_lock, flag);
915
916 /* check IOTLB invalidation granularity */
917 if (DMA_TLB_IAIG(val) == 0)
918 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
919 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
920 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
921 (unsigned long long)DMA_TLB_IIRG(type),
922 (unsigned long long)DMA_TLB_IAIG(val));
923 /* flush iotlb entry will implicitly flush write buffer */
924 return 0;
925 }
926
927 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
928 u64 addr, unsigned int pages, int non_present_entry_flush)
929 {
930 unsigned int mask;
931
932 BUG_ON(addr & (~VTD_PAGE_MASK));
933 BUG_ON(pages == 0);
934
935 /* Fallback to domain selective flush if no PSI support */
936 if (!cap_pgsel_inv(iommu->cap))
937 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
938 DMA_TLB_DSI_FLUSH,
939 non_present_entry_flush);
940
941 /*
942 * PSI requires page size to be 2 ^ x, and the base address is naturally
943 * aligned to the size
944 */
945 mask = ilog2(__roundup_pow_of_two(pages));
946 /* Fallback to domain selective flush if size is too big */
947 if (mask > cap_max_amask_val(iommu->cap))
948 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
949 DMA_TLB_DSI_FLUSH, non_present_entry_flush);
950
951 return iommu->flush.flush_iotlb(iommu, did, addr, mask,
952 DMA_TLB_PSI_FLUSH,
953 non_present_entry_flush);
954 }
955
956 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
957 {
958 u32 pmen;
959 unsigned long flags;
960
961 spin_lock_irqsave(&iommu->register_lock, flags);
962 pmen = readl(iommu->reg + DMAR_PMEN_REG);
963 pmen &= ~DMA_PMEN_EPM;
964 writel(pmen, iommu->reg + DMAR_PMEN_REG);
965
966 /* wait for the protected region status bit to clear */
967 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
968 readl, !(pmen & DMA_PMEN_PRS), pmen);
969
970 spin_unlock_irqrestore(&iommu->register_lock, flags);
971 }
972
973 static int iommu_enable_translation(struct intel_iommu *iommu)
974 {
975 u32 sts;
976 unsigned long flags;
977
978 spin_lock_irqsave(&iommu->register_lock, flags);
979 writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
980
981 /* Make sure hardware complete it */
982 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
983 readl, (sts & DMA_GSTS_TES), sts);
984
985 iommu->gcmd |= DMA_GCMD_TE;
986 spin_unlock_irqrestore(&iommu->register_lock, flags);
987 return 0;
988 }
989
990 static int iommu_disable_translation(struct intel_iommu *iommu)
991 {
992 u32 sts;
993 unsigned long flag;
994
995 spin_lock_irqsave(&iommu->register_lock, flag);
996 iommu->gcmd &= ~DMA_GCMD_TE;
997 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
998
999 /* Make sure hardware complete it */
1000 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1001 readl, (!(sts & DMA_GSTS_TES)), sts);
1002
1003 spin_unlock_irqrestore(&iommu->register_lock, flag);
1004 return 0;
1005 }
1006
1007 /* iommu interrupt handling. Most stuff are MSI-like. */
1008
1009 static const char *fault_reason_strings[] =
1010 {
1011 "Software",
1012 "Present bit in root entry is clear",
1013 "Present bit in context entry is clear",
1014 "Invalid context entry",
1015 "Access beyond MGAW",
1016 "PTE Write access is not set",
1017 "PTE Read access is not set",
1018 "Next page table ptr is invalid",
1019 "Root table address invalid",
1020 "Context table ptr is invalid",
1021 "non-zero reserved fields in RTP",
1022 "non-zero reserved fields in CTP",
1023 "non-zero reserved fields in PTE",
1024 };
1025 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
1026
1027 const char *dmar_get_fault_reason(u8 fault_reason)
1028 {
1029 if (fault_reason > MAX_FAULT_REASON_IDX)
1030 return "Unknown";
1031 else
1032 return fault_reason_strings[fault_reason];
1033 }
1034
1035 void dmar_msi_unmask(unsigned int irq)
1036 {
1037 struct intel_iommu *iommu = get_irq_data(irq);
1038 unsigned long flag;
1039
1040 /* unmask it */
1041 spin_lock_irqsave(&iommu->register_lock, flag);
1042 writel(0, iommu->reg + DMAR_FECTL_REG);
1043 /* Read a reg to force flush the post write */
1044 readl(iommu->reg + DMAR_FECTL_REG);
1045 spin_unlock_irqrestore(&iommu->register_lock, flag);
1046 }
1047
1048 void dmar_msi_mask(unsigned int irq)
1049 {
1050 unsigned long flag;
1051 struct intel_iommu *iommu = get_irq_data(irq);
1052
1053 /* mask it */
1054 spin_lock_irqsave(&iommu->register_lock, flag);
1055 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1056 /* Read a reg to force flush the post write */
1057 readl(iommu->reg + DMAR_FECTL_REG);
1058 spin_unlock_irqrestore(&iommu->register_lock, flag);
1059 }
1060
1061 void dmar_msi_write(int irq, struct msi_msg *msg)
1062 {
1063 struct intel_iommu *iommu = get_irq_data(irq);
1064 unsigned long flag;
1065
1066 spin_lock_irqsave(&iommu->register_lock, flag);
1067 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1068 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1069 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1070 spin_unlock_irqrestore(&iommu->register_lock, flag);
1071 }
1072
1073 void dmar_msi_read(int irq, struct msi_msg *msg)
1074 {
1075 struct intel_iommu *iommu = get_irq_data(irq);
1076 unsigned long flag;
1077
1078 spin_lock_irqsave(&iommu->register_lock, flag);
1079 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1080 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1081 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1082 spin_unlock_irqrestore(&iommu->register_lock, flag);
1083 }
1084
1085 static int iommu_page_fault_do_one(struct intel_iommu *iommu, int type,
1086 u8 fault_reason, u16 source_id, unsigned long long addr)
1087 {
1088 const char *reason;
1089
1090 reason = dmar_get_fault_reason(fault_reason);
1091
1092 printk(KERN_ERR
1093 "DMAR:[%s] Request device [%02x:%02x.%d] "
1094 "fault addr %llx \n"
1095 "DMAR:[fault reason %02d] %s\n",
1096 (type ? "DMA Read" : "DMA Write"),
1097 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1098 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1099 return 0;
1100 }
1101
1102 #define PRIMARY_FAULT_REG_LEN (16)
1103 static irqreturn_t iommu_page_fault(int irq, void *dev_id)
1104 {
1105 struct intel_iommu *iommu = dev_id;
1106 int reg, fault_index;
1107 u32 fault_status;
1108 unsigned long flag;
1109
1110 spin_lock_irqsave(&iommu->register_lock, flag);
1111 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1112
1113 /* TBD: ignore advanced fault log currently */
1114 if (!(fault_status & DMA_FSTS_PPF))
1115 goto clear_overflow;
1116
1117 fault_index = dma_fsts_fault_record_index(fault_status);
1118 reg = cap_fault_reg_offset(iommu->cap);
1119 while (1) {
1120 u8 fault_reason;
1121 u16 source_id;
1122 u64 guest_addr;
1123 int type;
1124 u32 data;
1125
1126 /* highest 32 bits */
1127 data = readl(iommu->reg + reg +
1128 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1129 if (!(data & DMA_FRCD_F))
1130 break;
1131
1132 fault_reason = dma_frcd_fault_reason(data);
1133 type = dma_frcd_type(data);
1134
1135 data = readl(iommu->reg + reg +
1136 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1137 source_id = dma_frcd_source_id(data);
1138
1139 guest_addr = dmar_readq(iommu->reg + reg +
1140 fault_index * PRIMARY_FAULT_REG_LEN);
1141 guest_addr = dma_frcd_page_addr(guest_addr);
1142 /* clear the fault */
1143 writel(DMA_FRCD_F, iommu->reg + reg +
1144 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1145
1146 spin_unlock_irqrestore(&iommu->register_lock, flag);
1147
1148 iommu_page_fault_do_one(iommu, type, fault_reason,
1149 source_id, guest_addr);
1150
1151 fault_index++;
1152 if (fault_index > cap_num_fault_regs(iommu->cap))
1153 fault_index = 0;
1154 spin_lock_irqsave(&iommu->register_lock, flag);
1155 }
1156 clear_overflow:
1157 /* clear primary fault overflow */
1158 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1159 if (fault_status & DMA_FSTS_PFO)
1160 writel(DMA_FSTS_PFO, iommu->reg + DMAR_FSTS_REG);
1161
1162 spin_unlock_irqrestore(&iommu->register_lock, flag);
1163 return IRQ_HANDLED;
1164 }
1165
1166 int dmar_set_interrupt(struct intel_iommu *iommu)
1167 {
1168 int irq, ret;
1169
1170 irq = create_irq();
1171 if (!irq) {
1172 printk(KERN_ERR "IOMMU: no free vectors\n");
1173 return -EINVAL;
1174 }
1175
1176 set_irq_data(irq, iommu);
1177 iommu->irq = irq;
1178
1179 ret = arch_setup_dmar_msi(irq);
1180 if (ret) {
1181 set_irq_data(irq, NULL);
1182 iommu->irq = 0;
1183 destroy_irq(irq);
1184 return 0;
1185 }
1186
1187 /* Force fault register is cleared */
1188 iommu_page_fault(irq, iommu);
1189
1190 ret = request_irq(irq, iommu_page_fault, 0, iommu->name, iommu);
1191 if (ret)
1192 printk(KERN_ERR "IOMMU: can't request irq\n");
1193 return ret;
1194 }
1195
1196 static int iommu_init_domains(struct intel_iommu *iommu)
1197 {
1198 unsigned long ndomains;
1199 unsigned long nlongs;
1200
1201 ndomains = cap_ndoms(iommu->cap);
1202 pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1203 nlongs = BITS_TO_LONGS(ndomains);
1204
1205 /* TBD: there might be 64K domains,
1206 * consider other allocation for future chip
1207 */
1208 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1209 if (!iommu->domain_ids) {
1210 printk(KERN_ERR "Allocating domain id array failed\n");
1211 return -ENOMEM;
1212 }
1213 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1214 GFP_KERNEL);
1215 if (!iommu->domains) {
1216 printk(KERN_ERR "Allocating domain array failed\n");
1217 kfree(iommu->domain_ids);
1218 return -ENOMEM;
1219 }
1220
1221 spin_lock_init(&iommu->lock);
1222
1223 /*
1224 * if Caching mode is set, then invalid translations are tagged
1225 * with domainid 0. Hence we need to pre-allocate it.
1226 */
1227 if (cap_caching_mode(iommu->cap))
1228 set_bit(0, iommu->domain_ids);
1229 return 0;
1230 }
1231
1232
1233 static void domain_exit(struct dmar_domain *domain);
1234 static void vm_domain_exit(struct dmar_domain *domain);
1235
1236 void free_dmar_iommu(struct intel_iommu *iommu)
1237 {
1238 struct dmar_domain *domain;
1239 int i;
1240 unsigned long flags;
1241
1242 i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1243 for (; i < cap_ndoms(iommu->cap); ) {
1244 domain = iommu->domains[i];
1245 clear_bit(i, iommu->domain_ids);
1246
1247 spin_lock_irqsave(&domain->iommu_lock, flags);
1248 if (--domain->iommu_count == 0) {
1249 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1250 vm_domain_exit(domain);
1251 else
1252 domain_exit(domain);
1253 }
1254 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1255
1256 i = find_next_bit(iommu->domain_ids,
1257 cap_ndoms(iommu->cap), i+1);
1258 }
1259
1260 if (iommu->gcmd & DMA_GCMD_TE)
1261 iommu_disable_translation(iommu);
1262
1263 if (iommu->irq) {
1264 set_irq_data(iommu->irq, NULL);
1265 /* This will mask the irq */
1266 free_irq(iommu->irq, iommu);
1267 destroy_irq(iommu->irq);
1268 }
1269
1270 kfree(iommu->domains);
1271 kfree(iommu->domain_ids);
1272
1273 g_iommus[iommu->seq_id] = NULL;
1274
1275 /* if all iommus are freed, free g_iommus */
1276 for (i = 0; i < g_num_of_iommus; i++) {
1277 if (g_iommus[i])
1278 break;
1279 }
1280
1281 if (i == g_num_of_iommus)
1282 kfree(g_iommus);
1283
1284 /* free context mapping */
1285 free_context_table(iommu);
1286 }
1287
1288 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
1289 {
1290 unsigned long num;
1291 unsigned long ndomains;
1292 struct dmar_domain *domain;
1293 unsigned long flags;
1294
1295 domain = alloc_domain_mem();
1296 if (!domain)
1297 return NULL;
1298
1299 ndomains = cap_ndoms(iommu->cap);
1300
1301 spin_lock_irqsave(&iommu->lock, flags);
1302 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1303 if (num >= ndomains) {
1304 spin_unlock_irqrestore(&iommu->lock, flags);
1305 free_domain_mem(domain);
1306 printk(KERN_ERR "IOMMU: no free domain ids\n");
1307 return NULL;
1308 }
1309
1310 set_bit(num, iommu->domain_ids);
1311 domain->id = num;
1312 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1313 set_bit(iommu->seq_id, &domain->iommu_bmp);
1314 domain->flags = 0;
1315 iommu->domains[num] = domain;
1316 spin_unlock_irqrestore(&iommu->lock, flags);
1317
1318 return domain;
1319 }
1320
1321 static void iommu_free_domain(struct dmar_domain *domain)
1322 {
1323 unsigned long flags;
1324 struct intel_iommu *iommu;
1325
1326 iommu = domain_get_iommu(domain);
1327
1328 spin_lock_irqsave(&iommu->lock, flags);
1329 clear_bit(domain->id, iommu->domain_ids);
1330 spin_unlock_irqrestore(&iommu->lock, flags);
1331 }
1332
1333 static struct iova_domain reserved_iova_list;
1334 static struct lock_class_key reserved_alloc_key;
1335 static struct lock_class_key reserved_rbtree_key;
1336
1337 static void dmar_init_reserved_ranges(void)
1338 {
1339 struct pci_dev *pdev = NULL;
1340 struct iova *iova;
1341 int i;
1342 u64 addr, size;
1343
1344 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1345
1346 lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1347 &reserved_alloc_key);
1348 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1349 &reserved_rbtree_key);
1350
1351 /* IOAPIC ranges shouldn't be accessed by DMA */
1352 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1353 IOVA_PFN(IOAPIC_RANGE_END));
1354 if (!iova)
1355 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1356
1357 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1358 for_each_pci_dev(pdev) {
1359 struct resource *r;
1360
1361 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1362 r = &pdev->resource[i];
1363 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1364 continue;
1365 addr = r->start;
1366 addr &= PAGE_MASK;
1367 size = r->end - addr;
1368 size = PAGE_ALIGN(size);
1369 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1370 IOVA_PFN(size + addr) - 1);
1371 if (!iova)
1372 printk(KERN_ERR "Reserve iova failed\n");
1373 }
1374 }
1375
1376 }
1377
1378 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1379 {
1380 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1381 }
1382
1383 static inline int guestwidth_to_adjustwidth(int gaw)
1384 {
1385 int agaw;
1386 int r = (gaw - 12) % 9;
1387
1388 if (r == 0)
1389 agaw = gaw;
1390 else
1391 agaw = gaw + 9 - r;
1392 if (agaw > 64)
1393 agaw = 64;
1394 return agaw;
1395 }
1396
1397 static int domain_init(struct dmar_domain *domain, int guest_width)
1398 {
1399 struct intel_iommu *iommu;
1400 int adjust_width, agaw;
1401 unsigned long sagaw;
1402
1403 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1404 spin_lock_init(&domain->mapping_lock);
1405 spin_lock_init(&domain->iommu_lock);
1406
1407 domain_reserve_special_ranges(domain);
1408
1409 /* calculate AGAW */
1410 iommu = domain_get_iommu(domain);
1411 if (guest_width > cap_mgaw(iommu->cap))
1412 guest_width = cap_mgaw(iommu->cap);
1413 domain->gaw = guest_width;
1414 adjust_width = guestwidth_to_adjustwidth(guest_width);
1415 agaw = width_to_agaw(adjust_width);
1416 sagaw = cap_sagaw(iommu->cap);
1417 if (!test_bit(agaw, &sagaw)) {
1418 /* hardware doesn't support it, choose a bigger one */
1419 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1420 agaw = find_next_bit(&sagaw, 5, agaw);
1421 if (agaw >= 5)
1422 return -ENODEV;
1423 }
1424 domain->agaw = agaw;
1425 INIT_LIST_HEAD(&domain->devices);
1426
1427 if (ecap_coherent(iommu->ecap))
1428 domain->iommu_coherency = 1;
1429 else
1430 domain->iommu_coherency = 0;
1431
1432 domain->iommu_count = 1;
1433
1434 /* always allocate the top pgd */
1435 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1436 if (!domain->pgd)
1437 return -ENOMEM;
1438 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1439 return 0;
1440 }
1441
1442 static void domain_exit(struct dmar_domain *domain)
1443 {
1444 u64 end;
1445
1446 /* Domain 0 is reserved, so dont process it */
1447 if (!domain)
1448 return;
1449
1450 domain_remove_dev_info(domain);
1451 /* destroy iovas */
1452 put_iova_domain(&domain->iovad);
1453 end = DOMAIN_MAX_ADDR(domain->gaw);
1454 end = end & (~PAGE_MASK);
1455
1456 /* clear ptes */
1457 dma_pte_clear_range(domain, 0, end);
1458
1459 /* free page tables */
1460 dma_pte_free_pagetable(domain, 0, end);
1461
1462 iommu_free_domain(domain);
1463 free_domain_mem(domain);
1464 }
1465
1466 static int domain_context_mapping_one(struct dmar_domain *domain,
1467 u8 bus, u8 devfn)
1468 {
1469 struct context_entry *context;
1470 unsigned long flags;
1471 struct intel_iommu *iommu;
1472 struct dma_pte *pgd;
1473 unsigned long num;
1474 unsigned long ndomains;
1475 int id;
1476 int agaw;
1477
1478 pr_debug("Set context mapping for %02x:%02x.%d\n",
1479 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1480 BUG_ON(!domain->pgd);
1481
1482 iommu = device_to_iommu(bus, devfn);
1483 if (!iommu)
1484 return -ENODEV;
1485
1486 context = device_to_context_entry(iommu, bus, devfn);
1487 if (!context)
1488 return -ENOMEM;
1489 spin_lock_irqsave(&iommu->lock, flags);
1490 if (context_present(context)) {
1491 spin_unlock_irqrestore(&iommu->lock, flags);
1492 return 0;
1493 }
1494
1495 id = domain->id;
1496 pgd = domain->pgd;
1497
1498 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
1499 int found = 0;
1500
1501 /* find an available domain id for this device in iommu */
1502 ndomains = cap_ndoms(iommu->cap);
1503 num = find_first_bit(iommu->domain_ids, ndomains);
1504 for (; num < ndomains; ) {
1505 if (iommu->domains[num] == domain) {
1506 id = num;
1507 found = 1;
1508 break;
1509 }
1510 num = find_next_bit(iommu->domain_ids,
1511 cap_ndoms(iommu->cap), num+1);
1512 }
1513
1514 if (found == 0) {
1515 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1516 if (num >= ndomains) {
1517 spin_unlock_irqrestore(&iommu->lock, flags);
1518 printk(KERN_ERR "IOMMU: no free domain ids\n");
1519 return -EFAULT;
1520 }
1521
1522 set_bit(num, iommu->domain_ids);
1523 iommu->domains[num] = domain;
1524 id = num;
1525 }
1526
1527 /* Skip top levels of page tables for
1528 * iommu which has less agaw than default.
1529 */
1530 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1531 pgd = phys_to_virt(dma_pte_addr(pgd));
1532 if (!dma_pte_present(pgd)) {
1533 spin_unlock_irqrestore(&iommu->lock, flags);
1534 return -ENOMEM;
1535 }
1536 }
1537 }
1538
1539 context_set_domain_id(context, id);
1540 context_set_address_width(context, iommu->agaw);
1541 context_set_address_root(context, virt_to_phys(pgd));
1542 context_set_translation_type(context, CONTEXT_TT_MULTI_LEVEL);
1543 context_set_fault_enable(context);
1544 context_set_present(context);
1545 domain_flush_cache(domain, context, sizeof(*context));
1546
1547 /* it's a non-present to present mapping */
1548 if (iommu->flush.flush_context(iommu, domain->id,
1549 (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT,
1550 DMA_CCMD_DEVICE_INVL, 1))
1551 iommu_flush_write_buffer(iommu);
1552 else
1553 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH, 0);
1554
1555 spin_unlock_irqrestore(&iommu->lock, flags);
1556
1557 spin_lock_irqsave(&domain->iommu_lock, flags);
1558 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1559 domain->iommu_count++;
1560 domain_update_iommu_coherency(domain);
1561 }
1562 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1563 return 0;
1564 }
1565
1566 static int
1567 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1568 {
1569 int ret;
1570 struct pci_dev *tmp, *parent;
1571
1572 ret = domain_context_mapping_one(domain, pdev->bus->number,
1573 pdev->devfn);
1574 if (ret)
1575 return ret;
1576
1577 /* dependent device mapping */
1578 tmp = pci_find_upstream_pcie_bridge(pdev);
1579 if (!tmp)
1580 return 0;
1581 /* Secondary interface's bus number and devfn 0 */
1582 parent = pdev->bus->self;
1583 while (parent != tmp) {
1584 ret = domain_context_mapping_one(domain, parent->bus->number,
1585 parent->devfn);
1586 if (ret)
1587 return ret;
1588 parent = parent->bus->self;
1589 }
1590 if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1591 return domain_context_mapping_one(domain,
1592 tmp->subordinate->number, 0);
1593 else /* this is a legacy PCI bridge */
1594 return domain_context_mapping_one(domain,
1595 tmp->bus->number, tmp->devfn);
1596 }
1597
1598 static int domain_context_mapped(struct pci_dev *pdev)
1599 {
1600 int ret;
1601 struct pci_dev *tmp, *parent;
1602 struct intel_iommu *iommu;
1603
1604 iommu = device_to_iommu(pdev->bus->number, pdev->devfn);
1605 if (!iommu)
1606 return -ENODEV;
1607
1608 ret = device_context_mapped(iommu,
1609 pdev->bus->number, pdev->devfn);
1610 if (!ret)
1611 return ret;
1612 /* dependent device mapping */
1613 tmp = pci_find_upstream_pcie_bridge(pdev);
1614 if (!tmp)
1615 return ret;
1616 /* Secondary interface's bus number and devfn 0 */
1617 parent = pdev->bus->self;
1618 while (parent != tmp) {
1619 ret = device_context_mapped(iommu, parent->bus->number,
1620 parent->devfn);
1621 if (!ret)
1622 return ret;
1623 parent = parent->bus->self;
1624 }
1625 if (tmp->is_pcie)
1626 return device_context_mapped(iommu,
1627 tmp->subordinate->number, 0);
1628 else
1629 return device_context_mapped(iommu,
1630 tmp->bus->number, tmp->devfn);
1631 }
1632
1633 static int
1634 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1635 u64 hpa, size_t size, int prot)
1636 {
1637 u64 start_pfn, end_pfn;
1638 struct dma_pte *pte;
1639 int index;
1640 int addr_width = agaw_to_width(domain->agaw);
1641
1642 hpa &= (((u64)1) << addr_width) - 1;
1643
1644 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1645 return -EINVAL;
1646 iova &= PAGE_MASK;
1647 start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT;
1648 end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT;
1649 index = 0;
1650 while (start_pfn < end_pfn) {
1651 pte = addr_to_dma_pte(domain, iova + VTD_PAGE_SIZE * index);
1652 if (!pte)
1653 return -ENOMEM;
1654 /* We don't need lock here, nobody else
1655 * touches the iova range
1656 */
1657 BUG_ON(dma_pte_addr(pte));
1658 dma_set_pte_addr(pte, start_pfn << VTD_PAGE_SHIFT);
1659 dma_set_pte_prot(pte, prot);
1660 domain_flush_cache(domain, pte, sizeof(*pte));
1661 start_pfn++;
1662 index++;
1663 }
1664 return 0;
1665 }
1666
1667 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1668 {
1669 if (!iommu)
1670 return;
1671
1672 clear_context_table(iommu, bus, devfn);
1673 iommu->flush.flush_context(iommu, 0, 0, 0,
1674 DMA_CCMD_GLOBAL_INVL, 0);
1675 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1676 DMA_TLB_GLOBAL_FLUSH, 0);
1677 }
1678
1679 static void domain_remove_dev_info(struct dmar_domain *domain)
1680 {
1681 struct device_domain_info *info;
1682 unsigned long flags;
1683 struct intel_iommu *iommu;
1684
1685 spin_lock_irqsave(&device_domain_lock, flags);
1686 while (!list_empty(&domain->devices)) {
1687 info = list_entry(domain->devices.next,
1688 struct device_domain_info, link);
1689 list_del(&info->link);
1690 list_del(&info->global);
1691 if (info->dev)
1692 info->dev->dev.archdata.iommu = NULL;
1693 spin_unlock_irqrestore(&device_domain_lock, flags);
1694
1695 iommu = device_to_iommu(info->bus, info->devfn);
1696 iommu_detach_dev(iommu, info->bus, info->devfn);
1697 free_devinfo_mem(info);
1698
1699 spin_lock_irqsave(&device_domain_lock, flags);
1700 }
1701 spin_unlock_irqrestore(&device_domain_lock, flags);
1702 }
1703
1704 /*
1705 * find_domain
1706 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1707 */
1708 static struct dmar_domain *
1709 find_domain(struct pci_dev *pdev)
1710 {
1711 struct device_domain_info *info;
1712
1713 /* No lock here, assumes no domain exit in normal case */
1714 info = pdev->dev.archdata.iommu;
1715 if (info)
1716 return info->domain;
1717 return NULL;
1718 }
1719
1720 /* domain is initialized */
1721 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1722 {
1723 struct dmar_domain *domain, *found = NULL;
1724 struct intel_iommu *iommu;
1725 struct dmar_drhd_unit *drhd;
1726 struct device_domain_info *info, *tmp;
1727 struct pci_dev *dev_tmp;
1728 unsigned long flags;
1729 int bus = 0, devfn = 0;
1730
1731 domain = find_domain(pdev);
1732 if (domain)
1733 return domain;
1734
1735 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1736 if (dev_tmp) {
1737 if (dev_tmp->is_pcie) {
1738 bus = dev_tmp->subordinate->number;
1739 devfn = 0;
1740 } else {
1741 bus = dev_tmp->bus->number;
1742 devfn = dev_tmp->devfn;
1743 }
1744 spin_lock_irqsave(&device_domain_lock, flags);
1745 list_for_each_entry(info, &device_domain_list, global) {
1746 if (info->bus == bus && info->devfn == devfn) {
1747 found = info->domain;
1748 break;
1749 }
1750 }
1751 spin_unlock_irqrestore(&device_domain_lock, flags);
1752 /* pcie-pci bridge already has a domain, uses it */
1753 if (found) {
1754 domain = found;
1755 goto found_domain;
1756 }
1757 }
1758
1759 /* Allocate new domain for the device */
1760 drhd = dmar_find_matched_drhd_unit(pdev);
1761 if (!drhd) {
1762 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1763 pci_name(pdev));
1764 return NULL;
1765 }
1766 iommu = drhd->iommu;
1767
1768 domain = iommu_alloc_domain(iommu);
1769 if (!domain)
1770 goto error;
1771
1772 if (domain_init(domain, gaw)) {
1773 domain_exit(domain);
1774 goto error;
1775 }
1776
1777 /* register pcie-to-pci device */
1778 if (dev_tmp) {
1779 info = alloc_devinfo_mem();
1780 if (!info) {
1781 domain_exit(domain);
1782 goto error;
1783 }
1784 info->bus = bus;
1785 info->devfn = devfn;
1786 info->dev = NULL;
1787 info->domain = domain;
1788 /* This domain is shared by devices under p2p bridge */
1789 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1790
1791 /* pcie-to-pci bridge already has a domain, uses it */
1792 found = NULL;
1793 spin_lock_irqsave(&device_domain_lock, flags);
1794 list_for_each_entry(tmp, &device_domain_list, global) {
1795 if (tmp->bus == bus && tmp->devfn == devfn) {
1796 found = tmp->domain;
1797 break;
1798 }
1799 }
1800 if (found) {
1801 free_devinfo_mem(info);
1802 domain_exit(domain);
1803 domain = found;
1804 } else {
1805 list_add(&info->link, &domain->devices);
1806 list_add(&info->global, &device_domain_list);
1807 }
1808 spin_unlock_irqrestore(&device_domain_lock, flags);
1809 }
1810
1811 found_domain:
1812 info = alloc_devinfo_mem();
1813 if (!info)
1814 goto error;
1815 info->bus = pdev->bus->number;
1816 info->devfn = pdev->devfn;
1817 info->dev = pdev;
1818 info->domain = domain;
1819 spin_lock_irqsave(&device_domain_lock, flags);
1820 /* somebody is fast */
1821 found = find_domain(pdev);
1822 if (found != NULL) {
1823 spin_unlock_irqrestore(&device_domain_lock, flags);
1824 if (found != domain) {
1825 domain_exit(domain);
1826 domain = found;
1827 }
1828 free_devinfo_mem(info);
1829 return domain;
1830 }
1831 list_add(&info->link, &domain->devices);
1832 list_add(&info->global, &device_domain_list);
1833 pdev->dev.archdata.iommu = info;
1834 spin_unlock_irqrestore(&device_domain_lock, flags);
1835 return domain;
1836 error:
1837 /* recheck it here, maybe others set it */
1838 return find_domain(pdev);
1839 }
1840
1841 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1842 unsigned long long start,
1843 unsigned long long end)
1844 {
1845 struct dmar_domain *domain;
1846 unsigned long size;
1847 unsigned long long base;
1848 int ret;
1849
1850 printk(KERN_INFO
1851 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1852 pci_name(pdev), start, end);
1853 /* page table init */
1854 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1855 if (!domain)
1856 return -ENOMEM;
1857
1858 /* The address might not be aligned */
1859 base = start & PAGE_MASK;
1860 size = end - base;
1861 size = PAGE_ALIGN(size);
1862 if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1863 IOVA_PFN(base + size) - 1)) {
1864 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1865 ret = -ENOMEM;
1866 goto error;
1867 }
1868
1869 pr_debug("Mapping reserved region %lx@%llx for %s\n",
1870 size, base, pci_name(pdev));
1871 /*
1872 * RMRR range might have overlap with physical memory range,
1873 * clear it first
1874 */
1875 dma_pte_clear_range(domain, base, base + size);
1876
1877 ret = domain_page_mapping(domain, base, base, size,
1878 DMA_PTE_READ|DMA_PTE_WRITE);
1879 if (ret)
1880 goto error;
1881
1882 /* context entry init */
1883 ret = domain_context_mapping(domain, pdev);
1884 if (!ret)
1885 return 0;
1886 error:
1887 domain_exit(domain);
1888 return ret;
1889
1890 }
1891
1892 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1893 struct pci_dev *pdev)
1894 {
1895 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1896 return 0;
1897 return iommu_prepare_identity_map(pdev, rmrr->base_address,
1898 rmrr->end_address + 1);
1899 }
1900
1901 #ifdef CONFIG_DMAR_GFX_WA
1902 struct iommu_prepare_data {
1903 struct pci_dev *pdev;
1904 int ret;
1905 };
1906
1907 static int __init iommu_prepare_work_fn(unsigned long start_pfn,
1908 unsigned long end_pfn, void *datax)
1909 {
1910 struct iommu_prepare_data *data;
1911
1912 data = (struct iommu_prepare_data *)datax;
1913
1914 data->ret = iommu_prepare_identity_map(data->pdev,
1915 start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
1916 return data->ret;
1917
1918 }
1919
1920 static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
1921 {
1922 int nid;
1923 struct iommu_prepare_data data;
1924
1925 data.pdev = pdev;
1926 data.ret = 0;
1927
1928 for_each_online_node(nid) {
1929 work_with_active_regions(nid, iommu_prepare_work_fn, &data);
1930 if (data.ret)
1931 return data.ret;
1932 }
1933 return data.ret;
1934 }
1935
1936 static void __init iommu_prepare_gfx_mapping(void)
1937 {
1938 struct pci_dev *pdev = NULL;
1939 int ret;
1940
1941 for_each_pci_dev(pdev) {
1942 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
1943 !IS_GFX_DEVICE(pdev))
1944 continue;
1945 printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
1946 pci_name(pdev));
1947 ret = iommu_prepare_with_active_regions(pdev);
1948 if (ret)
1949 printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
1950 }
1951 }
1952 #else /* !CONFIG_DMAR_GFX_WA */
1953 static inline void iommu_prepare_gfx_mapping(void)
1954 {
1955 return;
1956 }
1957 #endif
1958
1959 #ifdef CONFIG_DMAR_FLOPPY_WA
1960 static inline void iommu_prepare_isa(void)
1961 {
1962 struct pci_dev *pdev;
1963 int ret;
1964
1965 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1966 if (!pdev)
1967 return;
1968
1969 printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n");
1970 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1971
1972 if (ret)
1973 printk("IOMMU: Failed to create 0-64M identity map, "
1974 "floppy might not work\n");
1975
1976 }
1977 #else
1978 static inline void iommu_prepare_isa(void)
1979 {
1980 return;
1981 }
1982 #endif /* !CONFIG_DMAR_FLPY_WA */
1983
1984 static int __init init_dmars(void)
1985 {
1986 struct dmar_drhd_unit *drhd;
1987 struct dmar_rmrr_unit *rmrr;
1988 struct pci_dev *pdev;
1989 struct intel_iommu *iommu;
1990 int i, ret, unit = 0;
1991
1992 /*
1993 * for each drhd
1994 * allocate root
1995 * initialize and program root entry to not present
1996 * endfor
1997 */
1998 for_each_drhd_unit(drhd) {
1999 g_num_of_iommus++;
2000 /*
2001 * lock not needed as this is only incremented in the single
2002 * threaded kernel __init code path all other access are read
2003 * only
2004 */
2005 }
2006
2007 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2008 GFP_KERNEL);
2009 if (!g_iommus) {
2010 printk(KERN_ERR "Allocating global iommu array failed\n");
2011 ret = -ENOMEM;
2012 goto error;
2013 }
2014
2015 deferred_flush = kzalloc(g_num_of_iommus *
2016 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2017 if (!deferred_flush) {
2018 kfree(g_iommus);
2019 ret = -ENOMEM;
2020 goto error;
2021 }
2022
2023 for_each_drhd_unit(drhd) {
2024 if (drhd->ignored)
2025 continue;
2026
2027 iommu = drhd->iommu;
2028 g_iommus[iommu->seq_id] = iommu;
2029
2030 ret = iommu_init_domains(iommu);
2031 if (ret)
2032 goto error;
2033
2034 /*
2035 * TBD:
2036 * we could share the same root & context tables
2037 * amoung all IOMMU's. Need to Split it later.
2038 */
2039 ret = iommu_alloc_root_entry(iommu);
2040 if (ret) {
2041 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2042 goto error;
2043 }
2044 }
2045
2046 for_each_drhd_unit(drhd) {
2047 if (drhd->ignored)
2048 continue;
2049
2050 iommu = drhd->iommu;
2051 if (dmar_enable_qi(iommu)) {
2052 /*
2053 * Queued Invalidate not enabled, use Register Based
2054 * Invalidate
2055 */
2056 iommu->flush.flush_context = __iommu_flush_context;
2057 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2058 printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
2059 "invalidation\n",
2060 (unsigned long long)drhd->reg_base_addr);
2061 } else {
2062 iommu->flush.flush_context = qi_flush_context;
2063 iommu->flush.flush_iotlb = qi_flush_iotlb;
2064 printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
2065 "invalidation\n",
2066 (unsigned long long)drhd->reg_base_addr);
2067 }
2068 }
2069
2070 /*
2071 * For each rmrr
2072 * for each dev attached to rmrr
2073 * do
2074 * locate drhd for dev, alloc domain for dev
2075 * allocate free domain
2076 * allocate page table entries for rmrr
2077 * if context not allocated for bus
2078 * allocate and init context
2079 * set present in root table for this bus
2080 * init context with domain, translation etc
2081 * endfor
2082 * endfor
2083 */
2084 for_each_rmrr_units(rmrr) {
2085 for (i = 0; i < rmrr->devices_cnt; i++) {
2086 pdev = rmrr->devices[i];
2087 /* some BIOS lists non-exist devices in DMAR table */
2088 if (!pdev)
2089 continue;
2090 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2091 if (ret)
2092 printk(KERN_ERR
2093 "IOMMU: mapping reserved region failed\n");
2094 }
2095 }
2096
2097 iommu_prepare_gfx_mapping();
2098
2099 iommu_prepare_isa();
2100
2101 /*
2102 * for each drhd
2103 * enable fault log
2104 * global invalidate context cache
2105 * global invalidate iotlb
2106 * enable translation
2107 */
2108 for_each_drhd_unit(drhd) {
2109 if (drhd->ignored)
2110 continue;
2111 iommu = drhd->iommu;
2112 sprintf (iommu->name, "dmar%d", unit++);
2113
2114 iommu_flush_write_buffer(iommu);
2115
2116 ret = dmar_set_interrupt(iommu);
2117 if (ret)
2118 goto error;
2119
2120 iommu_set_root_entry(iommu);
2121
2122 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
2123 0);
2124 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
2125 0);
2126 iommu_disable_protect_mem_regions(iommu);
2127
2128 ret = iommu_enable_translation(iommu);
2129 if (ret)
2130 goto error;
2131 }
2132
2133 return 0;
2134 error:
2135 for_each_drhd_unit(drhd) {
2136 if (drhd->ignored)
2137 continue;
2138 iommu = drhd->iommu;
2139 free_iommu(iommu);
2140 }
2141 kfree(g_iommus);
2142 return ret;
2143 }
2144
2145 static inline u64 aligned_size(u64 host_addr, size_t size)
2146 {
2147 u64 addr;
2148 addr = (host_addr & (~PAGE_MASK)) + size;
2149 return PAGE_ALIGN(addr);
2150 }
2151
2152 struct iova *
2153 iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
2154 {
2155 struct iova *piova;
2156
2157 /* Make sure it's in range */
2158 end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
2159 if (!size || (IOVA_START_ADDR + size > end))
2160 return NULL;
2161
2162 piova = alloc_iova(&domain->iovad,
2163 size >> PAGE_SHIFT, IOVA_PFN(end), 1);
2164 return piova;
2165 }
2166
2167 static struct iova *
2168 __intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
2169 size_t size, u64 dma_mask)
2170 {
2171 struct pci_dev *pdev = to_pci_dev(dev);
2172 struct iova *iova = NULL;
2173
2174 if (dma_mask <= DMA_32BIT_MASK || dmar_forcedac)
2175 iova = iommu_alloc_iova(domain, size, dma_mask);
2176 else {
2177 /*
2178 * First try to allocate an io virtual address in
2179 * DMA_32BIT_MASK and if that fails then try allocating
2180 * from higher range
2181 */
2182 iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
2183 if (!iova)
2184 iova = iommu_alloc_iova(domain, size, dma_mask);
2185 }
2186
2187 if (!iova) {
2188 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
2189 return NULL;
2190 }
2191
2192 return iova;
2193 }
2194
2195 static struct dmar_domain *
2196 get_valid_domain_for_dev(struct pci_dev *pdev)
2197 {
2198 struct dmar_domain *domain;
2199 int ret;
2200
2201 domain = get_domain_for_dev(pdev,
2202 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2203 if (!domain) {
2204 printk(KERN_ERR
2205 "Allocating domain for %s failed", pci_name(pdev));
2206 return NULL;
2207 }
2208
2209 /* make sure context mapping is ok */
2210 if (unlikely(!domain_context_mapped(pdev))) {
2211 ret = domain_context_mapping(domain, pdev);
2212 if (ret) {
2213 printk(KERN_ERR
2214 "Domain context map for %s failed",
2215 pci_name(pdev));
2216 return NULL;
2217 }
2218 }
2219
2220 return domain;
2221 }
2222
2223 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2224 size_t size, int dir, u64 dma_mask)
2225 {
2226 struct pci_dev *pdev = to_pci_dev(hwdev);
2227 struct dmar_domain *domain;
2228 phys_addr_t start_paddr;
2229 struct iova *iova;
2230 int prot = 0;
2231 int ret;
2232 struct intel_iommu *iommu;
2233
2234 BUG_ON(dir == DMA_NONE);
2235 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2236 return paddr;
2237
2238 domain = get_valid_domain_for_dev(pdev);
2239 if (!domain)
2240 return 0;
2241
2242 iommu = domain_get_iommu(domain);
2243 size = aligned_size((u64)paddr, size);
2244
2245 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2246 if (!iova)
2247 goto error;
2248
2249 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2250
2251 /*
2252 * Check if DMAR supports zero-length reads on write only
2253 * mappings..
2254 */
2255 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2256 !cap_zlr(iommu->cap))
2257 prot |= DMA_PTE_READ;
2258 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2259 prot |= DMA_PTE_WRITE;
2260 /*
2261 * paddr - (paddr + size) might be partial page, we should map the whole
2262 * page. Note: if two part of one page are separately mapped, we
2263 * might have two guest_addr mapping to the same host paddr, but this
2264 * is not a big problem
2265 */
2266 ret = domain_page_mapping(domain, start_paddr,
2267 ((u64)paddr) & PAGE_MASK, size, prot);
2268 if (ret)
2269 goto error;
2270
2271 /* it's a non-present to present mapping */
2272 ret = iommu_flush_iotlb_psi(iommu, domain->id,
2273 start_paddr, size >> VTD_PAGE_SHIFT, 1);
2274 if (ret)
2275 iommu_flush_write_buffer(iommu);
2276
2277 return start_paddr + ((u64)paddr & (~PAGE_MASK));
2278
2279 error:
2280 if (iova)
2281 __free_iova(&domain->iovad, iova);
2282 printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
2283 pci_name(pdev), size, (unsigned long long)paddr, dir);
2284 return 0;
2285 }
2286
2287 dma_addr_t intel_map_single(struct device *hwdev, phys_addr_t paddr,
2288 size_t size, int dir)
2289 {
2290 return __intel_map_single(hwdev, paddr, size, dir,
2291 to_pci_dev(hwdev)->dma_mask);
2292 }
2293
2294 static void flush_unmaps(void)
2295 {
2296 int i, j;
2297
2298 timer_on = 0;
2299
2300 /* just flush them all */
2301 for (i = 0; i < g_num_of_iommus; i++) {
2302 struct intel_iommu *iommu = g_iommus[i];
2303 if (!iommu)
2304 continue;
2305
2306 if (deferred_flush[i].next) {
2307 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2308 DMA_TLB_GLOBAL_FLUSH, 0);
2309 for (j = 0; j < deferred_flush[i].next; j++) {
2310 __free_iova(&deferred_flush[i].domain[j]->iovad,
2311 deferred_flush[i].iova[j]);
2312 }
2313 deferred_flush[i].next = 0;
2314 }
2315 }
2316
2317 list_size = 0;
2318 }
2319
2320 static void flush_unmaps_timeout(unsigned long data)
2321 {
2322 unsigned long flags;
2323
2324 spin_lock_irqsave(&async_umap_flush_lock, flags);
2325 flush_unmaps();
2326 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2327 }
2328
2329 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2330 {
2331 unsigned long flags;
2332 int next, iommu_id;
2333 struct intel_iommu *iommu;
2334
2335 spin_lock_irqsave(&async_umap_flush_lock, flags);
2336 if (list_size == HIGH_WATER_MARK)
2337 flush_unmaps();
2338
2339 iommu = domain_get_iommu(dom);
2340 iommu_id = iommu->seq_id;
2341
2342 next = deferred_flush[iommu_id].next;
2343 deferred_flush[iommu_id].domain[next] = dom;
2344 deferred_flush[iommu_id].iova[next] = iova;
2345 deferred_flush[iommu_id].next++;
2346
2347 if (!timer_on) {
2348 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2349 timer_on = 1;
2350 }
2351 list_size++;
2352 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2353 }
2354
2355 void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
2356 int dir)
2357 {
2358 struct pci_dev *pdev = to_pci_dev(dev);
2359 struct dmar_domain *domain;
2360 unsigned long start_addr;
2361 struct iova *iova;
2362 struct intel_iommu *iommu;
2363
2364 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2365 return;
2366 domain = find_domain(pdev);
2367 BUG_ON(!domain);
2368
2369 iommu = domain_get_iommu(domain);
2370
2371 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2372 if (!iova)
2373 return;
2374
2375 start_addr = iova->pfn_lo << PAGE_SHIFT;
2376 size = aligned_size((u64)dev_addr, size);
2377
2378 pr_debug("Device %s unmapping: %lx@%llx\n",
2379 pci_name(pdev), size, (unsigned long long)start_addr);
2380
2381 /* clear the whole page */
2382 dma_pte_clear_range(domain, start_addr, start_addr + size);
2383 /* free page tables */
2384 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2385 if (intel_iommu_strict) {
2386 if (iommu_flush_iotlb_psi(iommu,
2387 domain->id, start_addr, size >> VTD_PAGE_SHIFT, 0))
2388 iommu_flush_write_buffer(iommu);
2389 /* free iova */
2390 __free_iova(&domain->iovad, iova);
2391 } else {
2392 add_unmap(domain, iova);
2393 /*
2394 * queue up the release of the unmap to save the 1/6th of the
2395 * cpu used up by the iotlb flush operation...
2396 */
2397 }
2398 }
2399
2400 void *intel_alloc_coherent(struct device *hwdev, size_t size,
2401 dma_addr_t *dma_handle, gfp_t flags)
2402 {
2403 void *vaddr;
2404 int order;
2405
2406 size = PAGE_ALIGN(size);
2407 order = get_order(size);
2408 flags &= ~(GFP_DMA | GFP_DMA32);
2409
2410 vaddr = (void *)__get_free_pages(flags, order);
2411 if (!vaddr)
2412 return NULL;
2413 memset(vaddr, 0, size);
2414
2415 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2416 DMA_BIDIRECTIONAL,
2417 hwdev->coherent_dma_mask);
2418 if (*dma_handle)
2419 return vaddr;
2420 free_pages((unsigned long)vaddr, order);
2421 return NULL;
2422 }
2423
2424 void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2425 dma_addr_t dma_handle)
2426 {
2427 int order;
2428
2429 size = PAGE_ALIGN(size);
2430 order = get_order(size);
2431
2432 intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2433 free_pages((unsigned long)vaddr, order);
2434 }
2435
2436 #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
2437
2438 void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2439 int nelems, int dir)
2440 {
2441 int i;
2442 struct pci_dev *pdev = to_pci_dev(hwdev);
2443 struct dmar_domain *domain;
2444 unsigned long start_addr;
2445 struct iova *iova;
2446 size_t size = 0;
2447 void *addr;
2448 struct scatterlist *sg;
2449 struct intel_iommu *iommu;
2450
2451 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2452 return;
2453
2454 domain = find_domain(pdev);
2455 BUG_ON(!domain);
2456
2457 iommu = domain_get_iommu(domain);
2458
2459 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2460 if (!iova)
2461 return;
2462 for_each_sg(sglist, sg, nelems, i) {
2463 addr = SG_ENT_VIRT_ADDRESS(sg);
2464 size += aligned_size((u64)addr, sg->length);
2465 }
2466
2467 start_addr = iova->pfn_lo << PAGE_SHIFT;
2468
2469 /* clear the whole page */
2470 dma_pte_clear_range(domain, start_addr, start_addr + size);
2471 /* free page tables */
2472 dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2473
2474 if (iommu_flush_iotlb_psi(iommu, domain->id, start_addr,
2475 size >> VTD_PAGE_SHIFT, 0))
2476 iommu_flush_write_buffer(iommu);
2477
2478 /* free iova */
2479 __free_iova(&domain->iovad, iova);
2480 }
2481
2482 static int intel_nontranslate_map_sg(struct device *hddev,
2483 struct scatterlist *sglist, int nelems, int dir)
2484 {
2485 int i;
2486 struct scatterlist *sg;
2487
2488 for_each_sg(sglist, sg, nelems, i) {
2489 BUG_ON(!sg_page(sg));
2490 sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg));
2491 sg->dma_length = sg->length;
2492 }
2493 return nelems;
2494 }
2495
2496 int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2497 int dir)
2498 {
2499 void *addr;
2500 int i;
2501 struct pci_dev *pdev = to_pci_dev(hwdev);
2502 struct dmar_domain *domain;
2503 size_t size = 0;
2504 int prot = 0;
2505 size_t offset = 0;
2506 struct iova *iova = NULL;
2507 int ret;
2508 struct scatterlist *sg;
2509 unsigned long start_addr;
2510 struct intel_iommu *iommu;
2511
2512 BUG_ON(dir == DMA_NONE);
2513 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2514 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2515
2516 domain = get_valid_domain_for_dev(pdev);
2517 if (!domain)
2518 return 0;
2519
2520 iommu = domain_get_iommu(domain);
2521
2522 for_each_sg(sglist, sg, nelems, i) {
2523 addr = SG_ENT_VIRT_ADDRESS(sg);
2524 addr = (void *)virt_to_phys(addr);
2525 size += aligned_size((u64)addr, sg->length);
2526 }
2527
2528 iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2529 if (!iova) {
2530 sglist->dma_length = 0;
2531 return 0;
2532 }
2533
2534 /*
2535 * Check if DMAR supports zero-length reads on write only
2536 * mappings..
2537 */
2538 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2539 !cap_zlr(iommu->cap))
2540 prot |= DMA_PTE_READ;
2541 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2542 prot |= DMA_PTE_WRITE;
2543
2544 start_addr = iova->pfn_lo << PAGE_SHIFT;
2545 offset = 0;
2546 for_each_sg(sglist, sg, nelems, i) {
2547 addr = SG_ENT_VIRT_ADDRESS(sg);
2548 addr = (void *)virt_to_phys(addr);
2549 size = aligned_size((u64)addr, sg->length);
2550 ret = domain_page_mapping(domain, start_addr + offset,
2551 ((u64)addr) & PAGE_MASK,
2552 size, prot);
2553 if (ret) {
2554 /* clear the page */
2555 dma_pte_clear_range(domain, start_addr,
2556 start_addr + offset);
2557 /* free page tables */
2558 dma_pte_free_pagetable(domain, start_addr,
2559 start_addr + offset);
2560 /* free iova */
2561 __free_iova(&domain->iovad, iova);
2562 return 0;
2563 }
2564 sg->dma_address = start_addr + offset +
2565 ((u64)addr & (~PAGE_MASK));
2566 sg->dma_length = sg->length;
2567 offset += size;
2568 }
2569
2570 /* it's a non-present to present mapping */
2571 if (iommu_flush_iotlb_psi(iommu, domain->id,
2572 start_addr, offset >> VTD_PAGE_SHIFT, 1))
2573 iommu_flush_write_buffer(iommu);
2574 return nelems;
2575 }
2576
2577 static struct dma_mapping_ops intel_dma_ops = {
2578 .alloc_coherent = intel_alloc_coherent,
2579 .free_coherent = intel_free_coherent,
2580 .map_single = intel_map_single,
2581 .unmap_single = intel_unmap_single,
2582 .map_sg = intel_map_sg,
2583 .unmap_sg = intel_unmap_sg,
2584 };
2585
2586 static inline int iommu_domain_cache_init(void)
2587 {
2588 int ret = 0;
2589
2590 iommu_domain_cache = kmem_cache_create("iommu_domain",
2591 sizeof(struct dmar_domain),
2592 0,
2593 SLAB_HWCACHE_ALIGN,
2594
2595 NULL);
2596 if (!iommu_domain_cache) {
2597 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2598 ret = -ENOMEM;
2599 }
2600
2601 return ret;
2602 }
2603
2604 static inline int iommu_devinfo_cache_init(void)
2605 {
2606 int ret = 0;
2607
2608 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2609 sizeof(struct device_domain_info),
2610 0,
2611 SLAB_HWCACHE_ALIGN,
2612 NULL);
2613 if (!iommu_devinfo_cache) {
2614 printk(KERN_ERR "Couldn't create devinfo cache\n");
2615 ret = -ENOMEM;
2616 }
2617
2618 return ret;
2619 }
2620
2621 static inline int iommu_iova_cache_init(void)
2622 {
2623 int ret = 0;
2624
2625 iommu_iova_cache = kmem_cache_create("iommu_iova",
2626 sizeof(struct iova),
2627 0,
2628 SLAB_HWCACHE_ALIGN,
2629 NULL);
2630 if (!iommu_iova_cache) {
2631 printk(KERN_ERR "Couldn't create iova cache\n");
2632 ret = -ENOMEM;
2633 }
2634
2635 return ret;
2636 }
2637
2638 static int __init iommu_init_mempool(void)
2639 {
2640 int ret;
2641 ret = iommu_iova_cache_init();
2642 if (ret)
2643 return ret;
2644
2645 ret = iommu_domain_cache_init();
2646 if (ret)
2647 goto domain_error;
2648
2649 ret = iommu_devinfo_cache_init();
2650 if (!ret)
2651 return ret;
2652
2653 kmem_cache_destroy(iommu_domain_cache);
2654 domain_error:
2655 kmem_cache_destroy(iommu_iova_cache);
2656
2657 return -ENOMEM;
2658 }
2659
2660 static void __init iommu_exit_mempool(void)
2661 {
2662 kmem_cache_destroy(iommu_devinfo_cache);
2663 kmem_cache_destroy(iommu_domain_cache);
2664 kmem_cache_destroy(iommu_iova_cache);
2665
2666 }
2667
2668 static void __init init_no_remapping_devices(void)
2669 {
2670 struct dmar_drhd_unit *drhd;
2671
2672 for_each_drhd_unit(drhd) {
2673 if (!drhd->include_all) {
2674 int i;
2675 for (i = 0; i < drhd->devices_cnt; i++)
2676 if (drhd->devices[i] != NULL)
2677 break;
2678 /* ignore DMAR unit if no pci devices exist */
2679 if (i == drhd->devices_cnt)
2680 drhd->ignored = 1;
2681 }
2682 }
2683
2684 if (dmar_map_gfx)
2685 return;
2686
2687 for_each_drhd_unit(drhd) {
2688 int i;
2689 if (drhd->ignored || drhd->include_all)
2690 continue;
2691
2692 for (i = 0; i < drhd->devices_cnt; i++)
2693 if (drhd->devices[i] &&
2694 !IS_GFX_DEVICE(drhd->devices[i]))
2695 break;
2696
2697 if (i < drhd->devices_cnt)
2698 continue;
2699
2700 /* bypass IOMMU if it is just for gfx devices */
2701 drhd->ignored = 1;
2702 for (i = 0; i < drhd->devices_cnt; i++) {
2703 if (!drhd->devices[i])
2704 continue;
2705 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
2706 }
2707 }
2708 }
2709
2710 int __init intel_iommu_init(void)
2711 {
2712 int ret = 0;
2713
2714 if (dmar_table_init())
2715 return -ENODEV;
2716
2717 if (dmar_dev_scope_init())
2718 return -ENODEV;
2719
2720 /*
2721 * Check the need for DMA-remapping initialization now.
2722 * Above initialization will also be used by Interrupt-remapping.
2723 */
2724 if (no_iommu || swiotlb || dmar_disabled)
2725 return -ENODEV;
2726
2727 iommu_init_mempool();
2728 dmar_init_reserved_ranges();
2729
2730 init_no_remapping_devices();
2731
2732 ret = init_dmars();
2733 if (ret) {
2734 printk(KERN_ERR "IOMMU: dmar init failed\n");
2735 put_iova_domain(&reserved_iova_list);
2736 iommu_exit_mempool();
2737 return ret;
2738 }
2739 printk(KERN_INFO
2740 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2741
2742 init_timer(&unmap_timer);
2743 force_iommu = 1;
2744 dma_ops = &intel_dma_ops;
2745
2746 register_iommu(&intel_iommu_ops);
2747
2748 return 0;
2749 }
2750
2751 static int vm_domain_add_dev_info(struct dmar_domain *domain,
2752 struct pci_dev *pdev)
2753 {
2754 struct device_domain_info *info;
2755 unsigned long flags;
2756
2757 info = alloc_devinfo_mem();
2758 if (!info)
2759 return -ENOMEM;
2760
2761 info->bus = pdev->bus->number;
2762 info->devfn = pdev->devfn;
2763 info->dev = pdev;
2764 info->domain = domain;
2765
2766 spin_lock_irqsave(&device_domain_lock, flags);
2767 list_add(&info->link, &domain->devices);
2768 list_add(&info->global, &device_domain_list);
2769 pdev->dev.archdata.iommu = info;
2770 spin_unlock_irqrestore(&device_domain_lock, flags);
2771
2772 return 0;
2773 }
2774
2775 static void vm_domain_remove_one_dev_info(struct dmar_domain *domain,
2776 struct pci_dev *pdev)
2777 {
2778 struct device_domain_info *info;
2779 struct intel_iommu *iommu;
2780 unsigned long flags;
2781 int found = 0;
2782 struct list_head *entry, *tmp;
2783
2784 iommu = device_to_iommu(pdev->bus->number, pdev->devfn);
2785 if (!iommu)
2786 return;
2787
2788 spin_lock_irqsave(&device_domain_lock, flags);
2789 list_for_each_safe(entry, tmp, &domain->devices) {
2790 info = list_entry(entry, struct device_domain_info, link);
2791 if (info->bus == pdev->bus->number &&
2792 info->devfn == pdev->devfn) {
2793 list_del(&info->link);
2794 list_del(&info->global);
2795 if (info->dev)
2796 info->dev->dev.archdata.iommu = NULL;
2797 spin_unlock_irqrestore(&device_domain_lock, flags);
2798
2799 iommu_detach_dev(iommu, info->bus, info->devfn);
2800 free_devinfo_mem(info);
2801
2802 spin_lock_irqsave(&device_domain_lock, flags);
2803
2804 if (found)
2805 break;
2806 else
2807 continue;
2808 }
2809
2810 /* if there is no other devices under the same iommu
2811 * owned by this domain, clear this iommu in iommu_bmp
2812 * update iommu count and coherency
2813 */
2814 if (device_to_iommu(info->bus, info->devfn) == iommu)
2815 found = 1;
2816 }
2817
2818 if (found == 0) {
2819 unsigned long tmp_flags;
2820 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
2821 clear_bit(iommu->seq_id, &domain->iommu_bmp);
2822 domain->iommu_count--;
2823 domain_update_iommu_coherency(domain);
2824 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
2825 }
2826
2827 spin_unlock_irqrestore(&device_domain_lock, flags);
2828 }
2829
2830 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
2831 {
2832 struct device_domain_info *info;
2833 struct intel_iommu *iommu;
2834 unsigned long flags1, flags2;
2835
2836 spin_lock_irqsave(&device_domain_lock, flags1);
2837 while (!list_empty(&domain->devices)) {
2838 info = list_entry(domain->devices.next,
2839 struct device_domain_info, link);
2840 list_del(&info->link);
2841 list_del(&info->global);
2842 if (info->dev)
2843 info->dev->dev.archdata.iommu = NULL;
2844
2845 spin_unlock_irqrestore(&device_domain_lock, flags1);
2846
2847 iommu = device_to_iommu(info->bus, info->devfn);
2848 iommu_detach_dev(iommu, info->bus, info->devfn);
2849
2850 /* clear this iommu in iommu_bmp, update iommu count
2851 * and coherency
2852 */
2853 spin_lock_irqsave(&domain->iommu_lock, flags2);
2854 if (test_and_clear_bit(iommu->seq_id,
2855 &domain->iommu_bmp)) {
2856 domain->iommu_count--;
2857 domain_update_iommu_coherency(domain);
2858 }
2859 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
2860
2861 free_devinfo_mem(info);
2862 spin_lock_irqsave(&device_domain_lock, flags1);
2863 }
2864 spin_unlock_irqrestore(&device_domain_lock, flags1);
2865 }
2866
2867 /* domain id for virtual machine, it won't be set in context */
2868 static unsigned long vm_domid;
2869
2870 static int vm_domain_min_agaw(struct dmar_domain *domain)
2871 {
2872 int i;
2873 int min_agaw = domain->agaw;
2874
2875 i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
2876 for (; i < g_num_of_iommus; ) {
2877 if (min_agaw > g_iommus[i]->agaw)
2878 min_agaw = g_iommus[i]->agaw;
2879
2880 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
2881 }
2882
2883 return min_agaw;
2884 }
2885
2886 static struct dmar_domain *iommu_alloc_vm_domain(void)
2887 {
2888 struct dmar_domain *domain;
2889
2890 domain = alloc_domain_mem();
2891 if (!domain)
2892 return NULL;
2893
2894 domain->id = vm_domid++;
2895 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
2896 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
2897
2898 return domain;
2899 }
2900
2901 static int vm_domain_init(struct dmar_domain *domain, int guest_width)
2902 {
2903 int adjust_width;
2904
2905 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
2906 spin_lock_init(&domain->mapping_lock);
2907 spin_lock_init(&domain->iommu_lock);
2908
2909 domain_reserve_special_ranges(domain);
2910
2911 /* calculate AGAW */
2912 domain->gaw = guest_width;
2913 adjust_width = guestwidth_to_adjustwidth(guest_width);
2914 domain->agaw = width_to_agaw(adjust_width);
2915
2916 INIT_LIST_HEAD(&domain->devices);
2917
2918 domain->iommu_count = 0;
2919 domain->iommu_coherency = 0;
2920 domain->max_addr = 0;
2921
2922 /* always allocate the top pgd */
2923 domain->pgd = (struct dma_pte *)alloc_pgtable_page();
2924 if (!domain->pgd)
2925 return -ENOMEM;
2926 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
2927 return 0;
2928 }
2929
2930 static void iommu_free_vm_domain(struct dmar_domain *domain)
2931 {
2932 unsigned long flags;
2933 struct dmar_drhd_unit *drhd;
2934 struct intel_iommu *iommu;
2935 unsigned long i;
2936 unsigned long ndomains;
2937
2938 for_each_drhd_unit(drhd) {
2939 if (drhd->ignored)
2940 continue;
2941 iommu = drhd->iommu;
2942
2943 ndomains = cap_ndoms(iommu->cap);
2944 i = find_first_bit(iommu->domain_ids, ndomains);
2945 for (; i < ndomains; ) {
2946 if (iommu->domains[i] == domain) {
2947 spin_lock_irqsave(&iommu->lock, flags);
2948 clear_bit(i, iommu->domain_ids);
2949 iommu->domains[i] = NULL;
2950 spin_unlock_irqrestore(&iommu->lock, flags);
2951 break;
2952 }
2953 i = find_next_bit(iommu->domain_ids, ndomains, i+1);
2954 }
2955 }
2956 }
2957
2958 static void vm_domain_exit(struct dmar_domain *domain)
2959 {
2960 u64 end;
2961
2962 /* Domain 0 is reserved, so dont process it */
2963 if (!domain)
2964 return;
2965
2966 vm_domain_remove_all_dev_info(domain);
2967 /* destroy iovas */
2968 put_iova_domain(&domain->iovad);
2969 end = DOMAIN_MAX_ADDR(domain->gaw);
2970 end = end & (~VTD_PAGE_MASK);
2971
2972 /* clear ptes */
2973 dma_pte_clear_range(domain, 0, end);
2974
2975 /* free page tables */
2976 dma_pte_free_pagetable(domain, 0, end);
2977
2978 iommu_free_vm_domain(domain);
2979 free_domain_mem(domain);
2980 }
2981
2982 static int intel_iommu_domain_init(struct iommu_domain *domain)
2983 {
2984 struct dmar_domain *dmar_domain;
2985
2986 dmar_domain = iommu_alloc_vm_domain();
2987 if (!dmar_domain) {
2988 printk(KERN_ERR
2989 "intel_iommu_domain_init: dmar_domain == NULL\n");
2990 return -ENOMEM;
2991 }
2992 if (vm_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2993 printk(KERN_ERR
2994 "intel_iommu_domain_init() failed\n");
2995 vm_domain_exit(dmar_domain);
2996 return -ENOMEM;
2997 }
2998 domain->priv = dmar_domain;
2999
3000 return 0;
3001 }
3002
3003 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
3004 {
3005 struct dmar_domain *dmar_domain = domain->priv;
3006
3007 domain->priv = NULL;
3008 vm_domain_exit(dmar_domain);
3009 }
3010
3011 static int intel_iommu_attach_device(struct iommu_domain *domain,
3012 struct device *dev)
3013 {
3014 struct dmar_domain *dmar_domain = domain->priv;
3015 struct pci_dev *pdev = to_pci_dev(dev);
3016 struct intel_iommu *iommu;
3017 int addr_width;
3018 u64 end;
3019 int ret;
3020
3021 /* normally pdev is not mapped */
3022 if (unlikely(domain_context_mapped(pdev))) {
3023 struct dmar_domain *old_domain;
3024
3025 old_domain = find_domain(pdev);
3026 if (old_domain) {
3027 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
3028 vm_domain_remove_one_dev_info(old_domain, pdev);
3029 else
3030 domain_remove_dev_info(old_domain);
3031 }
3032 }
3033
3034 iommu = device_to_iommu(pdev->bus->number, pdev->devfn);
3035 if (!iommu)
3036 return -ENODEV;
3037
3038 /* check if this iommu agaw is sufficient for max mapped address */
3039 addr_width = agaw_to_width(iommu->agaw);
3040 end = DOMAIN_MAX_ADDR(addr_width);
3041 end = end & VTD_PAGE_MASK;
3042 if (end < dmar_domain->max_addr) {
3043 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3044 "sufficient for the mapped address (%llx)\n",
3045 __func__, iommu->agaw, dmar_domain->max_addr);
3046 return -EFAULT;
3047 }
3048
3049 ret = domain_context_mapping(dmar_domain, pdev);
3050 if (ret)
3051 return ret;
3052
3053 ret = vm_domain_add_dev_info(dmar_domain, pdev);
3054 return ret;
3055 }
3056
3057 static void intel_iommu_detach_device(struct iommu_domain *domain,
3058 struct device *dev)
3059 {
3060 struct dmar_domain *dmar_domain = domain->priv;
3061 struct pci_dev *pdev = to_pci_dev(dev);
3062
3063 vm_domain_remove_one_dev_info(dmar_domain, pdev);
3064 }
3065
3066 static int intel_iommu_map_range(struct iommu_domain *domain,
3067 unsigned long iova, phys_addr_t hpa,
3068 size_t size, int iommu_prot)
3069 {
3070 struct dmar_domain *dmar_domain = domain->priv;
3071 u64 max_addr;
3072 int addr_width;
3073 int prot = 0;
3074 int ret;
3075
3076 if (iommu_prot & IOMMU_READ)
3077 prot |= DMA_PTE_READ;
3078 if (iommu_prot & IOMMU_WRITE)
3079 prot |= DMA_PTE_WRITE;
3080
3081 max_addr = (iova & VTD_PAGE_MASK) + VTD_PAGE_ALIGN(size);
3082 if (dmar_domain->max_addr < max_addr) {
3083 int min_agaw;
3084 u64 end;
3085
3086 /* check if minimum agaw is sufficient for mapped address */
3087 min_agaw = vm_domain_min_agaw(dmar_domain);
3088 addr_width = agaw_to_width(min_agaw);
3089 end = DOMAIN_MAX_ADDR(addr_width);
3090 end = end & VTD_PAGE_MASK;
3091 if (end < max_addr) {
3092 printk(KERN_ERR "%s: iommu agaw (%d) is not "
3093 "sufficient for the mapped address (%llx)\n",
3094 __func__, min_agaw, max_addr);
3095 return -EFAULT;
3096 }
3097 dmar_domain->max_addr = max_addr;
3098 }
3099
3100 ret = domain_page_mapping(dmar_domain, iova, hpa, size, prot);
3101 return ret;
3102 }
3103
3104 static void intel_iommu_unmap_range(struct iommu_domain *domain,
3105 unsigned long iova, size_t size)
3106 {
3107 struct dmar_domain *dmar_domain = domain->priv;
3108 dma_addr_t base;
3109
3110 /* The address might not be aligned */
3111 base = iova & VTD_PAGE_MASK;
3112 size = VTD_PAGE_ALIGN(size);
3113 dma_pte_clear_range(dmar_domain, base, base + size);
3114
3115 if (dmar_domain->max_addr == base + size)
3116 dmar_domain->max_addr = base;
3117 }
3118
3119 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3120 unsigned long iova)
3121 {
3122 struct dmar_domain *dmar_domain = domain->priv;
3123 struct dma_pte *pte;
3124 u64 phys = 0;
3125
3126 pte = addr_to_dma_pte(dmar_domain, iova);
3127 if (pte)
3128 phys = dma_pte_addr(pte);
3129
3130 return phys;
3131 }
3132
3133 static struct iommu_ops intel_iommu_ops = {
3134 .domain_init = intel_iommu_domain_init,
3135 .domain_destroy = intel_iommu_domain_destroy,
3136 .attach_dev = intel_iommu_attach_device,
3137 .detach_dev = intel_iommu_detach_device,
3138 .map = intel_iommu_map_range,
3139 .unmap = intel_iommu_unmap_range,
3140 .iova_to_phys = intel_iommu_iova_to_phys,
3141 };
3142
3143 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3144 {
3145 /*
3146 * Mobile 4 Series Chipset neglects to set RWBF capability,
3147 * but needs it:
3148 */
3149 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3150 rwbf_quirk = 1;
3151 }
3152
3153 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
Support for the Chinese Samsung S3C2440 based development boards