?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
GUI: Fix Tomato RAF theme for all builds. Compilation typo.
[tomato.git] / release / src-rt-6.x.4708 / linux / linux-2.6.36 / drivers / pci / dmar.c
bloba286959db67e5ee2eef89a9db596c570d81d64dc
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 *
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */
28
29 #include <linux/pci.h>
30 #include <linux/dmar.h>
31 #include <linux/iova.h>
32 #include <linux/intel-iommu.h>
33 #include <linux/timer.h>
34 #include <linux/irq.h>
35 #include <linux/interrupt.h>
36 #include <linux/tboot.h>
37 #include <linux/dmi.h>
38 #include <linux/slab.h>
39
40 #define PREFIX "DMAR: "
41
42 /* No locks are needed as DMA remapping hardware unit
43 * list is constructed at boot time and hotplug of
44 * these units are not supported by the architecture.
45 */
46 LIST_HEAD(dmar_drhd_units);
47
48 static struct acpi_table_header * __initdata dmar_tbl;
49 static acpi_size dmar_tbl_size;
50
51 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
52 {
53 /*
54 * add INCLUDE_ALL at the tail, so scan the list will find it at
55 * the very end.
56 */
57 if (drhd->include_all)
58 list_add_tail(&drhd->list, &dmar_drhd_units);
59 else
60 list_add(&drhd->list, &dmar_drhd_units);
61 }
62
63 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
64 struct pci_dev **dev, u16 segment)
65 {
66 struct pci_bus *bus;
67 struct pci_dev *pdev = NULL;
68 struct acpi_dmar_pci_path *path;
69 int count;
70
71 bus = pci_find_bus(segment, scope->bus);
72 path = (struct acpi_dmar_pci_path *)(scope + 1);
73 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
74 / sizeof(struct acpi_dmar_pci_path);
75
76 while (count) {
77 if (pdev)
78 pci_dev_put(pdev);
79 /*
80 * Some BIOSes list non-exist devices in DMAR table, just
81 * ignore it
82 */
83 if (!bus) {
84 printk(KERN_WARNING
85 PREFIX "Device scope bus [%d] not found\n",
86 scope->bus);
87 break;
88 }
89 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
90 if (!pdev) {
91 printk(KERN_WARNING PREFIX
92 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
93 segment, bus->number, path->dev, path->fn);
94 break;
95 }
96 path ++;
97 count --;
98 bus = pdev->subordinate;
99 }
100 if (!pdev) {
101 printk(KERN_WARNING PREFIX
102 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
103 segment, scope->bus, path->dev, path->fn);
104 *dev = NULL;
105 return 0;
106 }
107 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
108 pdev->subordinate) || (scope->entry_type == \
109 ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
110 pci_dev_put(pdev);
111 printk(KERN_WARNING PREFIX
112 "Device scope type does not match for %s\n",
113 pci_name(pdev));
114 return -EINVAL;
115 }
116 *dev = pdev;
117 return 0;
118 }
119
120 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
121 struct pci_dev ***devices, u16 segment)
122 {
123 struct acpi_dmar_device_scope *scope;
124 void * tmp = start;
125 int index;
126 int ret;
127
128 *cnt = 0;
129 while (start < end) {
130 scope = start;
131 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
132 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
133 (*cnt)++;
134 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
135 printk(KERN_WARNING PREFIX
136 "Unsupported device scope\n");
137 }
138 start += scope->length;
139 }
140 if (*cnt == 0)
141 return 0;
142
143 *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
144 if (!*devices)
145 return -ENOMEM;
146
147 start = tmp;
148 index = 0;
149 while (start < end) {
150 scope = start;
151 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
152 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
153 ret = dmar_parse_one_dev_scope(scope,
154 &(*devices)[index], segment);
155 if (ret) {
156 kfree(*devices);
157 return ret;
158 }
159 index ++;
160 }
161 start += scope->length;
162 }
163
164 return 0;
165 }
166
167 /**
168 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
169 * structure which uniquely represent one DMA remapping hardware unit
170 * present in the platform
171 */
172 static int __init
173 dmar_parse_one_drhd(struct acpi_dmar_header *header)
174 {
175 struct acpi_dmar_hardware_unit *drhd;
176 struct dmar_drhd_unit *dmaru;
177 int ret = 0;
178
179 drhd = (struct acpi_dmar_hardware_unit *)header;
180 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
181 if (!dmaru)
182 return -ENOMEM;
183
184 dmaru->hdr = header;
185 dmaru->reg_base_addr = drhd->address;
186 dmaru->segment = drhd->segment;
187 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
188
189 ret = alloc_iommu(dmaru);
190 if (ret) {
191 kfree(dmaru);
192 return ret;
193 }
194 dmar_register_drhd_unit(dmaru);
195 return 0;
196 }
197
198 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
199 {
200 struct acpi_dmar_hardware_unit *drhd;
201 int ret = 0;
202
203 drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
204
205 if (dmaru->include_all)
206 return 0;
207
208 ret = dmar_parse_dev_scope((void *)(drhd + 1),
209 ((void *)drhd) + drhd->header.length,
210 &dmaru->devices_cnt, &dmaru->devices,
211 drhd->segment);
212 if (ret) {
213 list_del(&dmaru->list);
214 kfree(dmaru);
215 }
216 return ret;
217 }
218
219 #ifdef CONFIG_DMAR
220 LIST_HEAD(dmar_rmrr_units);
221
222 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
223 {
224 list_add(&rmrr->list, &dmar_rmrr_units);
225 }
226
227
228 static int __init
229 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
230 {
231 struct acpi_dmar_reserved_memory *rmrr;
232 struct dmar_rmrr_unit *rmrru;
233
234 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
235 if (!rmrru)
236 return -ENOMEM;
237
238 rmrru->hdr = header;
239 rmrr = (struct acpi_dmar_reserved_memory *)header;
240 rmrru->base_address = rmrr->base_address;
241 rmrru->end_address = rmrr->end_address;
242
243 dmar_register_rmrr_unit(rmrru);
244 return 0;
245 }
246
247 static int __init
248 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
249 {
250 struct acpi_dmar_reserved_memory *rmrr;
251 int ret;
252
253 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
254 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
255 ((void *)rmrr) + rmrr->header.length,
256 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
257
258 if (ret || (rmrru->devices_cnt == 0)) {
259 list_del(&rmrru->list);
260 kfree(rmrru);
261 }
262 return ret;
263 }
264
265 static LIST_HEAD(dmar_atsr_units);
266
267 static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
268 {
269 struct acpi_dmar_atsr *atsr;
270 struct dmar_atsr_unit *atsru;
271
272 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
273 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
274 if (!atsru)
275 return -ENOMEM;
276
277 atsru->hdr = hdr;
278 atsru->include_all = atsr->flags & 0x1;
279
280 list_add(&atsru->list, &dmar_atsr_units);
281
282 return 0;
283 }
284
285 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
286 {
287 int rc;
288 struct acpi_dmar_atsr *atsr;
289
290 if (atsru->include_all)
291 return 0;
292
293 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
294 rc = dmar_parse_dev_scope((void *)(atsr + 1),
295 (void *)atsr + atsr->header.length,
296 &atsru->devices_cnt, &atsru->devices,
297 atsr->segment);
298 if (rc || !atsru->devices_cnt) {
299 list_del(&atsru->list);
300 kfree(atsru);
301 }
302
303 return rc;
304 }
305
306 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
307 {
308 int i;
309 struct pci_bus *bus;
310 struct acpi_dmar_atsr *atsr;
311 struct dmar_atsr_unit *atsru;
312
313 dev = pci_physfn(dev);
314
315 list_for_each_entry(atsru, &dmar_atsr_units, list) {
316 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
317 if (atsr->segment == pci_domain_nr(dev->bus))
318 goto found;
319 }
320
321 return 0;
322
323 found:
324 for (bus = dev->bus; bus; bus = bus->parent) {
325 struct pci_dev *bridge = bus->self;
326
327 if (!bridge || !pci_is_pcie(bridge) ||
328 bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
329 return 0;
330
331 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
332 for (i = 0; i < atsru->devices_cnt; i++)
333 if (atsru->devices[i] == bridge)
334 return 1;
335 break;
336 }
337 }
338
339 if (atsru->include_all)
340 return 1;
341
342 return 0;
343 }
344 #endif
345
346 #ifdef CONFIG_ACPI_NUMA
347 static int __init
348 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
349 {
350 struct acpi_dmar_rhsa *rhsa;
351 struct dmar_drhd_unit *drhd;
352
353 rhsa = (struct acpi_dmar_rhsa *)header;
354 for_each_drhd_unit(drhd) {
355 if (drhd->reg_base_addr == rhsa->base_address) {
356 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
357
358 if (!node_online(node))
359 node = -1;
360 drhd->iommu->node = node;
361 return 0;
362 }
363 }
364 WARN_TAINT(
365 1, TAINT_FIRMWARE_WORKAROUND,
366 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
367 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
368 drhd->reg_base_addr,
369 dmi_get_system_info(DMI_BIOS_VENDOR),
370 dmi_get_system_info(DMI_BIOS_VERSION),
371 dmi_get_system_info(DMI_PRODUCT_VERSION));
372
373 return 0;
374 }
375 #endif
376
377 static void __init
378 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
379 {
380 struct acpi_dmar_hardware_unit *drhd;
381 struct acpi_dmar_reserved_memory *rmrr;
382 struct acpi_dmar_atsr *atsr;
383 struct acpi_dmar_rhsa *rhsa;
384
385 switch (header->type) {
386 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
387 drhd = container_of(header, struct acpi_dmar_hardware_unit,
388 header);
389 printk (KERN_INFO PREFIX
390 "DRHD base: %#016Lx flags: %#x\n",
391 (unsigned long long)drhd->address, drhd->flags);
392 break;
393 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
394 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
395 header);
396 printk (KERN_INFO PREFIX
397 "RMRR base: %#016Lx end: %#016Lx\n",
398 (unsigned long long)rmrr->base_address,
399 (unsigned long long)rmrr->end_address);
400 break;
401 case ACPI_DMAR_TYPE_ATSR:
402 atsr = container_of(header, struct acpi_dmar_atsr, header);
403 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
404 break;
405 case ACPI_DMAR_HARDWARE_AFFINITY:
406 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
407 printk(KERN_INFO PREFIX "RHSA base: %#016Lx proximity domain: %#x\n",
408 (unsigned long long)rhsa->base_address,
409 rhsa->proximity_domain);
410 break;
411 }
412 }
413
414 /**
415 * dmar_table_detect - checks to see if the platform supports DMAR devices
416 */
417 static int __init dmar_table_detect(void)
418 {
419 acpi_status status = AE_OK;
420
421 /* if we could find DMAR table, then there are DMAR devices */
422 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
423 (struct acpi_table_header **)&dmar_tbl,
424 &dmar_tbl_size);
425
426 if (ACPI_SUCCESS(status) && !dmar_tbl) {
427 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
428 status = AE_NOT_FOUND;
429 }
430
431 return (ACPI_SUCCESS(status) ? 1 : 0);
432 }
433
434 /**
435 * parse_dmar_table - parses the DMA reporting table
436 */
437 static int __init
438 parse_dmar_table(void)
439 {
440 struct acpi_table_dmar *dmar;
441 struct acpi_dmar_header *entry_header;
442 int ret = 0;
443
444 /*
445 * Do it again, earlier dmar_tbl mapping could be mapped with
446 * fixed map.
447 */
448 dmar_table_detect();
449
450 /*
451 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
452 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
453 */
454 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
455
456 dmar = (struct acpi_table_dmar *)dmar_tbl;
457 if (!dmar)
458 return -ENODEV;
459
460 if (dmar->width < PAGE_SHIFT - 1) {
461 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
462 return -EINVAL;
463 }
464
465 printk (KERN_INFO PREFIX "Host address width %d\n",
466 dmar->width + 1);
467
468 entry_header = (struct acpi_dmar_header *)(dmar + 1);
469 while (((unsigned long)entry_header) <
470 (((unsigned long)dmar) + dmar_tbl->length)) {
471 /* Avoid looping forever on bad ACPI tables */
472 if (entry_header->length == 0) {
473 printk(KERN_WARNING PREFIX
474 "Invalid 0-length structure\n");
475 ret = -EINVAL;
476 break;
477 }
478
479 dmar_table_print_dmar_entry(entry_header);
480
481 switch (entry_header->type) {
482 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
483 ret = dmar_parse_one_drhd(entry_header);
484 break;
485 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
486 #ifdef CONFIG_DMAR
487 ret = dmar_parse_one_rmrr(entry_header);
488 #endif
489 break;
490 case ACPI_DMAR_TYPE_ATSR:
491 #ifdef CONFIG_DMAR
492 ret = dmar_parse_one_atsr(entry_header);
493 #endif
494 break;
495 case ACPI_DMAR_HARDWARE_AFFINITY:
496 #ifdef CONFIG_ACPI_NUMA
497 ret = dmar_parse_one_rhsa(entry_header);
498 #endif
499 break;
500 default:
501 printk(KERN_WARNING PREFIX
502 "Unknown DMAR structure type %d\n",
503 entry_header->type);
504 ret = 0; /* for forward compatibility */
505 break;
506 }
507 if (ret)
508 break;
509
510 entry_header = ((void *)entry_header + entry_header->length);
511 }
512 return ret;
513 }
514
515 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
516 struct pci_dev *dev)
517 {
518 int index;
519
520 while (dev) {
521 for (index = 0; index < cnt; index++)
522 if (dev == devices[index])
523 return 1;
524
525 /* Check our parent */
526 dev = dev->bus->self;
527 }
528
529 return 0;
530 }
531
532 struct dmar_drhd_unit *
533 dmar_find_matched_drhd_unit(struct pci_dev *dev)
534 {
535 struct dmar_drhd_unit *dmaru = NULL;
536 struct acpi_dmar_hardware_unit *drhd;
537
538 dev = pci_physfn(dev);
539
540 list_for_each_entry(dmaru, &dmar_drhd_units, list) {
541 drhd = container_of(dmaru->hdr,
542 struct acpi_dmar_hardware_unit,
543 header);
544
545 if (dmaru->include_all &&
546 drhd->segment == pci_domain_nr(dev->bus))
547 return dmaru;
548
549 if (dmar_pci_device_match(dmaru->devices,
550 dmaru->devices_cnt, dev))
551 return dmaru;
552 }
553
554 return NULL;
555 }
556
557 int __init dmar_dev_scope_init(void)
558 {
559 struct dmar_drhd_unit *drhd, *drhd_n;
560 int ret = -ENODEV;
561
562 list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
563 ret = dmar_parse_dev(drhd);
564 if (ret)
565 return ret;
566 }
567
568 #ifdef CONFIG_DMAR
569 {
570 struct dmar_rmrr_unit *rmrr, *rmrr_n;
571 struct dmar_atsr_unit *atsr, *atsr_n;
572
573 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
574 ret = rmrr_parse_dev(rmrr);
575 if (ret)
576 return ret;
577 }
578
579 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
580 ret = atsr_parse_dev(atsr);
581 if (ret)
582 return ret;
583 }
584 }
585 #endif
586
587 return ret;
588 }
589
590
591 int __init dmar_table_init(void)
592 {
593 static int dmar_table_initialized;
594 int ret;
595
596 if (dmar_table_initialized)
597 return 0;
598
599 dmar_table_initialized = 1;
600
601 ret = parse_dmar_table();
602 if (ret) {
603 if (ret != -ENODEV)
604 printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
605 return ret;
606 }
607
608 if (list_empty(&dmar_drhd_units)) {
609 printk(KERN_INFO PREFIX "No DMAR devices found\n");
610 return -ENODEV;
611 }
612
613 #ifdef CONFIG_DMAR
614 if (list_empty(&dmar_rmrr_units))
615 printk(KERN_INFO PREFIX "No RMRR found\n");
616
617 if (list_empty(&dmar_atsr_units))
618 printk(KERN_INFO PREFIX "No ATSR found\n");
619 #endif
620
621 return 0;
622 }
623
624 static void warn_invalid_dmar(u64 addr, const char *message)
625 {
626 WARN_TAINT_ONCE(
627 1, TAINT_FIRMWARE_WORKAROUND,
628 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
629 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
630 addr, message,
631 dmi_get_system_info(DMI_BIOS_VENDOR),
632 dmi_get_system_info(DMI_BIOS_VERSION),
633 dmi_get_system_info(DMI_PRODUCT_VERSION));
634 }
635
636 int __init check_zero_address(void)
637 {
638 struct acpi_table_dmar *dmar;
639 struct acpi_dmar_header *entry_header;
640 struct acpi_dmar_hardware_unit *drhd;
641
642 dmar = (struct acpi_table_dmar *)dmar_tbl;
643 entry_header = (struct acpi_dmar_header *)(dmar + 1);
644
645 while (((unsigned long)entry_header) <
646 (((unsigned long)dmar) + dmar_tbl->length)) {
647 /* Avoid looping forever on bad ACPI tables */
648 if (entry_header->length == 0) {
649 printk(KERN_WARNING PREFIX
650 "Invalid 0-length structure\n");
651 return 0;
652 }
653
654 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
655 void __iomem *addr;
656 u64 cap, ecap;
657
658 drhd = (void *)entry_header;
659 if (!drhd->address) {
660 warn_invalid_dmar(0, "");
661 goto failed;
662 }
663
664 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
665 if (!addr ) {
666 printk("IOMMU: can't validate: %llx\n", drhd->address);
667 goto failed;
668 }
669 cap = dmar_readq(addr + DMAR_CAP_REG);
670 ecap = dmar_readq(addr + DMAR_ECAP_REG);
671 early_iounmap(addr, VTD_PAGE_SIZE);
672 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
673 warn_invalid_dmar(drhd->address,
674 " returns all ones");
675 goto failed;
676 }
677 }
678
679 entry_header = ((void *)entry_header + entry_header->length);
680 }
681 return 1;
682
683 failed:
684 #ifdef CONFIG_DMAR
685 dmar_disabled = 1;
686 #endif
687 return 0;
688 }
689
690 void __init detect_intel_iommu(void)
691 {
692 int ret;
693
694 ret = dmar_table_detect();
695 if (ret)
696 ret = check_zero_address();
697 {
698 #ifdef CONFIG_INTR_REMAP
699 struct acpi_table_dmar *dmar;
700 /*
701 * for now we will disable dma-remapping when interrupt
702 * remapping is enabled.
703 * When support for queued invalidation for IOTLB invalidation
704 * is added, we will not need this any more.
705 */
706 dmar = (struct acpi_table_dmar *) dmar_tbl;
707 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
708 printk(KERN_INFO
709 "Queued invalidation will be enabled to support "
710 "x2apic and Intr-remapping.\n");
711 #endif
712 #ifdef CONFIG_DMAR
713 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
714 iommu_detected = 1;
715 /* Make sure ACS will be enabled */
716 pci_request_acs();
717 }
718 #endif
719 #ifdef CONFIG_X86
720 if (ret)
721 x86_init.iommu.iommu_init = intel_iommu_init;
722 #endif
723 }
724 early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
725 dmar_tbl = NULL;
726 }
727
728
729 int alloc_iommu(struct dmar_drhd_unit *drhd)
730 {
731 struct intel_iommu *iommu;
732 int map_size;
733 u32 ver;
734 static int iommu_allocated = 0;
735 int agaw = 0;
736 int msagaw = 0;
737
738 if (!drhd->reg_base_addr) {
739 warn_invalid_dmar(0, "");
740 return -EINVAL;
741 }
742
743 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
744 if (!iommu)
745 return -ENOMEM;
746
747 iommu->seq_id = iommu_allocated++;
748 sprintf (iommu->name, "dmar%d", iommu->seq_id);
749
750 iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
751 if (!iommu->reg) {
752 printk(KERN_ERR "IOMMU: can't map the region\n");
753 goto error;
754 }
755 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
756 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
757
758 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
759 warn_invalid_dmar(drhd->reg_base_addr, " returns all ones");
760 goto err_unmap;
761 }
762
763 #ifdef CONFIG_DMAR
764 agaw = iommu_calculate_agaw(iommu);
765 if (agaw < 0) {
766 printk(KERN_ERR
767 "Cannot get a valid agaw for iommu (seq_id = %d)\n",
768 iommu->seq_id);
769 goto err_unmap;
770 }
771 msagaw = iommu_calculate_max_sagaw(iommu);
772 if (msagaw < 0) {
773 printk(KERN_ERR
774 "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
775 iommu->seq_id);
776 goto err_unmap;
777 }
778 #endif
779 iommu->agaw = agaw;
780 iommu->msagaw = msagaw;
781
782 iommu->node = -1;
783
784 /* the registers might be more than one page */
785 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
786 cap_max_fault_reg_offset(iommu->cap));
787 map_size = VTD_PAGE_ALIGN(map_size);
788 if (map_size > VTD_PAGE_SIZE) {
789 iounmap(iommu->reg);
790 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
791 if (!iommu->reg) {
792 printk(KERN_ERR "IOMMU: can't map the region\n");
793 goto error;
794 }
795 }
796
797 ver = readl(iommu->reg + DMAR_VER_REG);
798 pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
799 iommu->seq_id,
800 (unsigned long long)drhd->reg_base_addr,
801 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
802 (unsigned long long)iommu->cap,
803 (unsigned long long)iommu->ecap);
804
805 spin_lock_init(&iommu->register_lock);
806
807 drhd->iommu = iommu;
808 return 0;
809
810 err_unmap:
811 iounmap(iommu->reg);
812 error:
813 kfree(iommu);
814 return -1;
815 }
816
817 void free_iommu(struct intel_iommu *iommu)
818 {
819 if (!iommu)
820 return;
821
822 #ifdef CONFIG_DMAR
823 free_dmar_iommu(iommu);
824 #endif
825
826 if (iommu->reg)
827 iounmap(iommu->reg);
828 kfree(iommu);
829 }
830
831 /*
832 * Reclaim all the submitted descriptors which have completed its work.
833 */
834 static inline void reclaim_free_desc(struct q_inval *qi)
835 {
836 while (qi->desc_status[qi->free_tail] == QI_DONE ||
837 qi->desc_status[qi->free_tail] == QI_ABORT) {
838 qi->desc_status[qi->free_tail] = QI_FREE;
839 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
840 qi->free_cnt++;
841 }
842 }
843
844 static int qi_check_fault(struct intel_iommu *iommu, int index)
845 {
846 u32 fault;
847 int head, tail;
848 struct q_inval *qi = iommu->qi;
849 int wait_index = (index + 1) % QI_LENGTH;
850
851 if (qi->desc_status[wait_index] == QI_ABORT)
852 return -EAGAIN;
853
854 fault = readl(iommu->reg + DMAR_FSTS_REG);
855
856 /*
857 * If IQE happens, the head points to the descriptor associated
858 * with the error. No new descriptors are fetched until the IQE
859 * is cleared.
860 */
861 if (fault & DMA_FSTS_IQE) {
862 head = readl(iommu->reg + DMAR_IQH_REG);
863 if ((head >> DMAR_IQ_SHIFT) == index) {
864 printk(KERN_ERR "VT-d detected invalid descriptor: "
865 "low=%llx, high=%llx\n",
866 (unsigned long long)qi->desc[index].low,
867 (unsigned long long)qi->desc[index].high);
868 memcpy(&qi->desc[index], &qi->desc[wait_index],
869 sizeof(struct qi_desc));
870 __iommu_flush_cache(iommu, &qi->desc[index],
871 sizeof(struct qi_desc));
872 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
873 return -EINVAL;
874 }
875 }
876
877 /*
878 * If ITE happens, all pending wait_desc commands are aborted.
879 * No new descriptors are fetched until the ITE is cleared.
880 */
881 if (fault & DMA_FSTS_ITE) {
882 head = readl(iommu->reg + DMAR_IQH_REG);
883 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
884 head |= 1;
885 tail = readl(iommu->reg + DMAR_IQT_REG);
886 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
887
888 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
889
890 do {
891 if (qi->desc_status[head] == QI_IN_USE)
892 qi->desc_status[head] = QI_ABORT;
893 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
894 } while (head != tail);
895
896 if (qi->desc_status[wait_index] == QI_ABORT)
897 return -EAGAIN;
898 }
899
900 if (fault & DMA_FSTS_ICE)
901 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
902
903 return 0;
904 }
905
906 /*
907 * Submit the queued invalidation descriptor to the remapping
908 * hardware unit and wait for its completion.
909 */
910 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
911 {
912 int rc;
913 struct q_inval *qi = iommu->qi;
914 struct qi_desc *hw, wait_desc;
915 int wait_index, index;
916 unsigned long flags;
917
918 if (!qi)
919 return 0;
920
921 hw = qi->desc;
922
923 restart:
924 rc = 0;
925
926 spin_lock_irqsave(&qi->q_lock, flags);
927 while (qi->free_cnt < 3) {
928 spin_unlock_irqrestore(&qi->q_lock, flags);
929 cpu_relax();
930 spin_lock_irqsave(&qi->q_lock, flags);
931 }
932
933 index = qi->free_head;
934 wait_index = (index + 1) % QI_LENGTH;
935
936 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
937
938 hw[index] = *desc;
939
940 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
941 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
942 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
943
944 hw[wait_index] = wait_desc;
945
946 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
947 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
948
949 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
950 qi->free_cnt -= 2;
951
952 /*
953 * update the HW tail register indicating the presence of
954 * new descriptors.
955 */
956 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
957
958 while (qi->desc_status[wait_index] != QI_DONE) {
959 /*
960 * We will leave the interrupts disabled, to prevent interrupt
961 * context to queue another cmd while a cmd is already submitted
962 * and waiting for completion on this cpu. This is to avoid
963 * a deadlock where the interrupt context can wait indefinitely
964 * for free slots in the queue.
965 */
966 rc = qi_check_fault(iommu, index);
967 if (rc)
968 break;
969
970 spin_unlock(&qi->q_lock);
971 cpu_relax();
972 spin_lock(&qi->q_lock);
973 }
974
975 qi->desc_status[index] = QI_DONE;
976
977 reclaim_free_desc(qi);
978 spin_unlock_irqrestore(&qi->q_lock, flags);
979
980 if (rc == -EAGAIN)
981 goto restart;
982
983 return rc;
984 }
985
986 /*
987 * Flush the global interrupt entry cache.
988 */
989 void qi_global_iec(struct intel_iommu *iommu)
990 {
991 struct qi_desc desc;
992
993 desc.low = QI_IEC_TYPE;
994 desc.high = 0;
995
996 /* should never fail */
997 qi_submit_sync(&desc, iommu);
998 }
999
1000 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1001 u64 type)
1002 {
1003 struct qi_desc desc;
1004
1005 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1006 | QI_CC_GRAN(type) | QI_CC_TYPE;
1007 desc.high = 0;
1008
1009 qi_submit_sync(&desc, iommu);
1010 }
1011
1012 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1013 unsigned int size_order, u64 type)
1014 {
1015 u8 dw = 0, dr = 0;
1016
1017 struct qi_desc desc;
1018 int ih = 0;
1019
1020 if (cap_write_drain(iommu->cap))
1021 dw = 1;
1022
1023 if (cap_read_drain(iommu->cap))
1024 dr = 1;
1025
1026 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1027 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1028 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1029 | QI_IOTLB_AM(size_order);
1030
1031 qi_submit_sync(&desc, iommu);
1032 }
1033
1034 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1035 u64 addr, unsigned mask)
1036 {
1037 struct qi_desc desc;
1038
1039 if (mask) {
1040 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1041 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1042 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1043 } else
1044 desc.high = QI_DEV_IOTLB_ADDR(addr);
1045
1046 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1047 qdep = 0;
1048
1049 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1050 QI_DIOTLB_TYPE;
1051
1052 qi_submit_sync(&desc, iommu);
1053 }
1054
1055 /*
1056 * Disable Queued Invalidation interface.
1057 */
1058 void dmar_disable_qi(struct intel_iommu *iommu)
1059 {
1060 unsigned long flags;
1061 u32 sts;
1062 cycles_t start_time = get_cycles();
1063
1064 if (!ecap_qis(iommu->ecap))
1065 return;
1066
1067 spin_lock_irqsave(&iommu->register_lock, flags);
1068
1069 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
1070 if (!(sts & DMA_GSTS_QIES))
1071 goto end;
1072
1073 /*
1074 * Give a chance to HW to complete the pending invalidation requests.
1075 */
1076 while ((readl(iommu->reg + DMAR_IQT_REG) !=
1077 readl(iommu->reg + DMAR_IQH_REG)) &&
1078 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1079 cpu_relax();
1080
1081 iommu->gcmd &= ~DMA_GCMD_QIE;
1082 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1083
1084 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1085 !(sts & DMA_GSTS_QIES), sts);
1086 end:
1087 spin_unlock_irqrestore(&iommu->register_lock, flags);
1088 }
1089
1090 /*
1091 * Enable queued invalidation.
1092 */
1093 static void __dmar_enable_qi(struct intel_iommu *iommu)
1094 {
1095 u32 sts;
1096 unsigned long flags;
1097 struct q_inval *qi = iommu->qi;
1098
1099 qi->free_head = qi->free_tail = 0;
1100 qi->free_cnt = QI_LENGTH;
1101
1102 spin_lock_irqsave(&iommu->register_lock, flags);
1103
1104 /* write zero to the tail reg */
1105 writel(0, iommu->reg + DMAR_IQT_REG);
1106
1107 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1108
1109 iommu->gcmd |= DMA_GCMD_QIE;
1110 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1111
1112 /* Make sure hardware complete it */
1113 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1114
1115 spin_unlock_irqrestore(&iommu->register_lock, flags);
1116 }
1117
1118 /*
1119 * Enable Queued Invalidation interface. This is a must to support
1120 * interrupt-remapping. Also used by DMA-remapping, which replaces
1121 * register based IOTLB invalidation.
1122 */
1123 int dmar_enable_qi(struct intel_iommu *iommu)
1124 {
1125 struct q_inval *qi;
1126 struct page *desc_page;
1127
1128 if (!ecap_qis(iommu->ecap))
1129 return -ENOENT;
1130
1131 /*
1132 * queued invalidation is already setup and enabled.
1133 */
1134 if (iommu->qi)
1135 return 0;
1136
1137 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1138 if (!iommu->qi)
1139 return -ENOMEM;
1140
1141 qi = iommu->qi;
1142
1143
1144 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1145 if (!desc_page) {
1146 kfree(qi);
1147 iommu->qi = 0;
1148 return -ENOMEM;
1149 }
1150
1151 qi->desc = page_address(desc_page);
1152
1153 qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1154 if (!qi->desc_status) {
1155 free_page((unsigned long) qi->desc);
1156 kfree(qi);
1157 iommu->qi = 0;
1158 return -ENOMEM;
1159 }
1160
1161 qi->free_head = qi->free_tail = 0;
1162 qi->free_cnt = QI_LENGTH;
1163
1164 spin_lock_init(&qi->q_lock);
1165
1166 __dmar_enable_qi(iommu);
1167
1168 return 0;
1169 }
1170
1171 /* iommu interrupt handling. Most stuff are MSI-like. */
1172
1173 enum faulttype {
1174 DMA_REMAP,
1175 INTR_REMAP,
1176 UNKNOWN,
1177 };
1178
1179 static const char *dma_remap_fault_reasons[] =
1180 {
1181 "Software",
1182 "Present bit in root entry is clear",
1183 "Present bit in context entry is clear",
1184 "Invalid context entry",
1185 "Access beyond MGAW",
1186 "PTE Write access is not set",
1187 "PTE Read access is not set",
1188 "Next page table ptr is invalid",
1189 "Root table address invalid",
1190 "Context table ptr is invalid",
1191 "non-zero reserved fields in RTP",
1192 "non-zero reserved fields in CTP",
1193 "non-zero reserved fields in PTE",
1194 };
1195
1196 static const char *intr_remap_fault_reasons[] =
1197 {
1198 "Detected reserved fields in the decoded interrupt-remapped request",
1199 "Interrupt index exceeded the interrupt-remapping table size",
1200 "Present field in the IRTE entry is clear",
1201 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1202 "Detected reserved fields in the IRTE entry",
1203 "Blocked a compatibility format interrupt request",
1204 "Blocked an interrupt request due to source-id verification failure",
1205 };
1206
1207 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
1208
1209 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1210 {
1211 if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1212 ARRAY_SIZE(intr_remap_fault_reasons))) {
1213 *fault_type = INTR_REMAP;
1214 return intr_remap_fault_reasons[fault_reason - 0x20];
1215 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1216 *fault_type = DMA_REMAP;
1217 return dma_remap_fault_reasons[fault_reason];
1218 } else {
1219 *fault_type = UNKNOWN;
1220 return "Unknown";
1221 }
1222 }
1223
1224 void dmar_msi_unmask(unsigned int irq)
1225 {
1226 struct intel_iommu *iommu = get_irq_data(irq);
1227 unsigned long flag;
1228
1229 /* unmask it */
1230 spin_lock_irqsave(&iommu->register_lock, flag);
1231 writel(0, iommu->reg + DMAR_FECTL_REG);
1232 /* Read a reg to force flush the post write */
1233 readl(iommu->reg + DMAR_FECTL_REG);
1234 spin_unlock_irqrestore(&iommu->register_lock, flag);
1235 }
1236
1237 void dmar_msi_mask(unsigned int irq)
1238 {
1239 unsigned long flag;
1240 struct intel_iommu *iommu = get_irq_data(irq);
1241
1242 /* mask it */
1243 spin_lock_irqsave(&iommu->register_lock, flag);
1244 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1245 /* Read a reg to force flush the post write */
1246 readl(iommu->reg + DMAR_FECTL_REG);
1247 spin_unlock_irqrestore(&iommu->register_lock, flag);
1248 }
1249
1250 void dmar_msi_write(int irq, struct msi_msg *msg)
1251 {
1252 struct intel_iommu *iommu = get_irq_data(irq);
1253 unsigned long flag;
1254
1255 spin_lock_irqsave(&iommu->register_lock, flag);
1256 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1257 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1258 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1259 spin_unlock_irqrestore(&iommu->register_lock, flag);
1260 }
1261
1262 void dmar_msi_read(int irq, struct msi_msg *msg)
1263 {
1264 struct intel_iommu *iommu = get_irq_data(irq);
1265 unsigned long flag;
1266
1267 spin_lock_irqsave(&iommu->register_lock, flag);
1268 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1269 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1270 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1271 spin_unlock_irqrestore(&iommu->register_lock, flag);
1272 }
1273
1274 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1275 u8 fault_reason, u16 source_id, unsigned long long addr)
1276 {
1277 const char *reason;
1278 int fault_type;
1279
1280 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1281
1282 if (fault_type == INTR_REMAP)
1283 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1284 "fault index %llx\n"
1285 "INTR-REMAP:[fault reason %02d] %s\n",
1286 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1287 PCI_FUNC(source_id & 0xFF), addr >> 48,
1288 fault_reason, reason);
1289 else
1290 printk(KERN_ERR
1291 "DMAR:[%s] Request device [%02x:%02x.%d] "
1292 "fault addr %llx \n"
1293 "DMAR:[fault reason %02d] %s\n",
1294 (type ? "DMA Read" : "DMA Write"),
1295 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1296 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1297 return 0;
1298 }
1299
1300 #define PRIMARY_FAULT_REG_LEN (16)
1301 irqreturn_t dmar_fault(int irq, void *dev_id)
1302 {
1303 struct intel_iommu *iommu = dev_id;
1304 int reg, fault_index;
1305 u32 fault_status;
1306 unsigned long flag;
1307
1308 spin_lock_irqsave(&iommu->register_lock, flag);
1309 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1310 if (fault_status)
1311 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1312 fault_status);
1313
1314 /* TBD: ignore advanced fault log currently */
1315 if (!(fault_status & DMA_FSTS_PPF))
1316 goto clear_rest;
1317
1318 fault_index = dma_fsts_fault_record_index(fault_status);
1319 reg = cap_fault_reg_offset(iommu->cap);
1320 while (1) {
1321 u8 fault_reason;
1322 u16 source_id;
1323 u64 guest_addr;
1324 int type;
1325 u32 data;
1326
1327 /* highest 32 bits */
1328 data = readl(iommu->reg + reg +
1329 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1330 if (!(data & DMA_FRCD_F))
1331 break;
1332
1333 fault_reason = dma_frcd_fault_reason(data);
1334 type = dma_frcd_type(data);
1335
1336 data = readl(iommu->reg + reg +
1337 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1338 source_id = dma_frcd_source_id(data);
1339
1340 guest_addr = dmar_readq(iommu->reg + reg +
1341 fault_index * PRIMARY_FAULT_REG_LEN);
1342 guest_addr = dma_frcd_page_addr(guest_addr);
1343 /* clear the fault */
1344 writel(DMA_FRCD_F, iommu->reg + reg +
1345 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1346
1347 spin_unlock_irqrestore(&iommu->register_lock, flag);
1348
1349 dmar_fault_do_one(iommu, type, fault_reason,
1350 source_id, guest_addr);
1351
1352 fault_index++;
1353 if (fault_index >= cap_num_fault_regs(iommu->cap))
1354 fault_index = 0;
1355 spin_lock_irqsave(&iommu->register_lock, flag);
1356 }
1357 clear_rest:
1358 /* clear all the other faults */
1359 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1360 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1361
1362 spin_unlock_irqrestore(&iommu->register_lock, flag);
1363 return IRQ_HANDLED;
1364 }
1365
1366 int dmar_set_interrupt(struct intel_iommu *iommu)
1367 {
1368 int irq, ret;
1369
1370 /*
1371 * Check if the fault interrupt is already initialized.
1372 */
1373 if (iommu->irq)
1374 return 0;
1375
1376 irq = create_irq();
1377 if (!irq) {
1378 printk(KERN_ERR "IOMMU: no free vectors\n");
1379 return -EINVAL;
1380 }
1381
1382 set_irq_data(irq, iommu);
1383 iommu->irq = irq;
1384
1385 ret = arch_setup_dmar_msi(irq);
1386 if (ret) {
1387 set_irq_data(irq, NULL);
1388 iommu->irq = 0;
1389 destroy_irq(irq);
1390 return ret;
1391 }
1392
1393 ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1394 if (ret)
1395 printk(KERN_ERR "IOMMU: can't request irq\n");
1396 return ret;
1397 }
1398
1399 int __init enable_drhd_fault_handling(void)
1400 {
1401 struct dmar_drhd_unit *drhd;
1402
1403 /*
1404 * Enable fault control interrupt.
1405 */
1406 for_each_drhd_unit(drhd) {
1407 int ret;
1408 struct intel_iommu *iommu = drhd->iommu;
1409 ret = dmar_set_interrupt(iommu);
1410
1411 if (ret) {
1412 printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1413 " interrupt, ret %d\n",
1414 (unsigned long long)drhd->reg_base_addr, ret);
1415 return -1;
1416 }
1417
1418 /*
1419 * Clear any previous faults.
1420 */
1421 dmar_fault(iommu->irq, iommu);
1422 }
1423
1424 return 0;
1425 }
1426
1427 /*
1428 * Re-enable Queued Invalidation interface.
1429 */
1430 int dmar_reenable_qi(struct intel_iommu *iommu)
1431 {
1432 if (!ecap_qis(iommu->ecap))
1433 return -ENOENT;
1434
1435 if (!iommu->qi)
1436 return -ENOENT;
1437
1438 /*
1439 * First disable queued invalidation.
1440 */
1441 dmar_disable_qi(iommu);
1442 /*
1443 * Then enable queued invalidation again. Since there is no pending
1444 * invalidation requests now, it's safe to re-enable queued
1445 * invalidation.
1446 */
1447 __dmar_enable_qi(iommu);
1448
1449 return 0;
1450 }
1451
1452 /*
1453 * Check interrupt remapping support in DMAR table description.
1454 */
1455 int __init dmar_ir_support(void)
1456 {
1457 struct acpi_table_dmar *dmar;
1458 dmar = (struct acpi_table_dmar *)dmar_tbl;
1459 if (!dmar)
1460 return 0;
1461 return dmar->flags & 0x1;
1462 }
Tomato firmware and modifications.