?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
intel-iommu: Cope with broken HP DC7900 BIOS
[linux-2.6/linux-2.6-openrd.git] / drivers / pci / dmar.c
blob380b60e677e038b51c781609cf3eb03f5d11bc0c
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 *
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */
28
29 #include <linux/pci.h>
30 #include <linux/dmar.h>
31 #include <linux/iova.h>
32 #include <linux/intel-iommu.h>
33 #include <linux/timer.h>
34 #include <linux/irq.h>
35 #include <linux/interrupt.h>
36
37 #undef PREFIX
38 #define PREFIX "DMAR:"
39
40 /* No locks are needed as DMA remapping hardware unit
41 * list is constructed at boot time and hotplug of
42 * these units are not supported by the architecture.
43 */
44 LIST_HEAD(dmar_drhd_units);
45
46 static struct acpi_table_header * __initdata dmar_tbl;
47 static acpi_size dmar_tbl_size;
48
49 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
50 {
51 /*
52 * add INCLUDE_ALL at the tail, so scan the list will find it at
53 * the very end.
54 */
55 if (drhd->include_all)
56 list_add_tail(&drhd->list, &dmar_drhd_units);
57 else
58 list_add(&drhd->list, &dmar_drhd_units);
59 }
60
61 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
62 struct pci_dev **dev, u16 segment)
63 {
64 struct pci_bus *bus;
65 struct pci_dev *pdev = NULL;
66 struct acpi_dmar_pci_path *path;
67 int count;
68
69 bus = pci_find_bus(segment, scope->bus);
70 path = (struct acpi_dmar_pci_path *)(scope + 1);
71 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
72 / sizeof(struct acpi_dmar_pci_path);
73
74 while (count) {
75 if (pdev)
76 pci_dev_put(pdev);
77 /*
78 * Some BIOSes list non-exist devices in DMAR table, just
79 * ignore it
80 */
81 if (!bus) {
82 printk(KERN_WARNING
83 PREFIX "Device scope bus [%d] not found\n",
84 scope->bus);
85 break;
86 }
87 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
88 if (!pdev) {
89 printk(KERN_WARNING PREFIX
90 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
91 segment, bus->number, path->dev, path->fn);
92 break;
93 }
94 path ++;
95 count --;
96 bus = pdev->subordinate;
97 }
98 if (!pdev) {
99 printk(KERN_WARNING PREFIX
100 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
101 segment, scope->bus, path->dev, path->fn);
102 *dev = NULL;
103 return 0;
104 }
105 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
106 pdev->subordinate) || (scope->entry_type == \
107 ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
108 pci_dev_put(pdev);
109 printk(KERN_WARNING PREFIX
110 "Device scope type does not match for %s\n",
111 pci_name(pdev));
112 return -EINVAL;
113 }
114 *dev = pdev;
115 return 0;
116 }
117
118 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
119 struct pci_dev ***devices, u16 segment)
120 {
121 struct acpi_dmar_device_scope *scope;
122 void * tmp = start;
123 int index;
124 int ret;
125
126 *cnt = 0;
127 while (start < end) {
128 scope = start;
129 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
130 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
131 (*cnt)++;
132 else
133 printk(KERN_WARNING PREFIX
134 "Unsupported device scope\n");
135 start += scope->length;
136 }
137 if (*cnt == 0)
138 return 0;
139
140 *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
141 if (!*devices)
142 return -ENOMEM;
143
144 start = tmp;
145 index = 0;
146 while (start < end) {
147 scope = start;
148 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
149 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
150 ret = dmar_parse_one_dev_scope(scope,
151 &(*devices)[index], segment);
152 if (ret) {
153 kfree(*devices);
154 return ret;
155 }
156 index ++;
157 }
158 start += scope->length;
159 }
160
161 return 0;
162 }
163
164 /**
165 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
166 * structure which uniquely represent one DMA remapping hardware unit
167 * present in the platform
168 */
169 static int __init
170 dmar_parse_one_drhd(struct acpi_dmar_header *header)
171 {
172 struct acpi_dmar_hardware_unit *drhd;
173 struct dmar_drhd_unit *dmaru;
174 int ret = 0;
175
176 drhd = (struct acpi_dmar_hardware_unit *)header;
177 if (!drhd->address) {
178 /* Promote an attitude of violence to a BIOS engineer today */
179 WARN(1, "Your BIOS is broken; DMAR reported at address zero!\n"
180 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
181 dmi_get_system_info(DMI_BIOS_VENDOR),
182 dmi_get_system_info(DMI_BIOS_VERSION),
183 dmi_get_system_info(DMI_PRODUCT_VERSION));
184 return -ENODEV;
185 }
186 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
187 if (!dmaru)
188 return -ENOMEM;
189
190 dmaru->hdr = header;
191 dmaru->reg_base_addr = drhd->address;
192 dmaru->segment = drhd->segment;
193 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
194
195 ret = alloc_iommu(dmaru);
196 if (ret) {
197 kfree(dmaru);
198 return ret;
199 }
200 dmar_register_drhd_unit(dmaru);
201 return 0;
202 }
203
204 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
205 {
206 struct acpi_dmar_hardware_unit *drhd;
207 int ret = 0;
208
209 drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
210
211 if (dmaru->include_all)
212 return 0;
213
214 ret = dmar_parse_dev_scope((void *)(drhd + 1),
215 ((void *)drhd) + drhd->header.length,
216 &dmaru->devices_cnt, &dmaru->devices,
217 drhd->segment);
218 if (ret) {
219 list_del(&dmaru->list);
220 kfree(dmaru);
221 }
222 return ret;
223 }
224
225 #ifdef CONFIG_DMAR
226 LIST_HEAD(dmar_rmrr_units);
227
228 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
229 {
230 list_add(&rmrr->list, &dmar_rmrr_units);
231 }
232
233
234 static int __init
235 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
236 {
237 struct acpi_dmar_reserved_memory *rmrr;
238 struct dmar_rmrr_unit *rmrru;
239
240 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
241 if (!rmrru)
242 return -ENOMEM;
243
244 rmrru->hdr = header;
245 rmrr = (struct acpi_dmar_reserved_memory *)header;
246 rmrru->base_address = rmrr->base_address;
247 rmrru->end_address = rmrr->end_address;
248
249 dmar_register_rmrr_unit(rmrru);
250 return 0;
251 }
252
253 static int __init
254 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
255 {
256 struct acpi_dmar_reserved_memory *rmrr;
257 int ret;
258
259 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
260 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
261 ((void *)rmrr) + rmrr->header.length,
262 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
263
264 if (ret || (rmrru->devices_cnt == 0)) {
265 list_del(&rmrru->list);
266 kfree(rmrru);
267 }
268 return ret;
269 }
270
271 static LIST_HEAD(dmar_atsr_units);
272
273 static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
274 {
275 struct acpi_dmar_atsr *atsr;
276 struct dmar_atsr_unit *atsru;
277
278 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
279 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
280 if (!atsru)
281 return -ENOMEM;
282
283 atsru->hdr = hdr;
284 atsru->include_all = atsr->flags & 0x1;
285
286 list_add(&atsru->list, &dmar_atsr_units);
287
288 return 0;
289 }
290
291 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
292 {
293 int rc;
294 struct acpi_dmar_atsr *atsr;
295
296 if (atsru->include_all)
297 return 0;
298
299 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
300 rc = dmar_parse_dev_scope((void *)(atsr + 1),
301 (void *)atsr + atsr->header.length,
302 &atsru->devices_cnt, &atsru->devices,
303 atsr->segment);
304 if (rc || !atsru->devices_cnt) {
305 list_del(&atsru->list);
306 kfree(atsru);
307 }
308
309 return rc;
310 }
311
312 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
313 {
314 int i;
315 struct pci_bus *bus;
316 struct acpi_dmar_atsr *atsr;
317 struct dmar_atsr_unit *atsru;
318
319 list_for_each_entry(atsru, &dmar_atsr_units, list) {
320 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
321 if (atsr->segment == pci_domain_nr(dev->bus))
322 goto found;
323 }
324
325 return 0;
326
327 found:
328 for (bus = dev->bus; bus; bus = bus->parent) {
329 struct pci_dev *bridge = bus->self;
330
331 if (!bridge || !bridge->is_pcie ||
332 bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
333 return 0;
334
335 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
336 for (i = 0; i < atsru->devices_cnt; i++)
337 if (atsru->devices[i] == bridge)
338 return 1;
339 break;
340 }
341 }
342
343 if (atsru->include_all)
344 return 1;
345
346 return 0;
347 }
348 #endif
349
350 static void __init
351 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
352 {
353 struct acpi_dmar_hardware_unit *drhd;
354 struct acpi_dmar_reserved_memory *rmrr;
355 struct acpi_dmar_atsr *atsr;
356
357 switch (header->type) {
358 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
359 drhd = container_of(header, struct acpi_dmar_hardware_unit,
360 header);
361 printk (KERN_INFO PREFIX
362 "DRHD base: %#016Lx flags: %#x\n",
363 (unsigned long long)drhd->address, drhd->flags);
364 break;
365 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
366 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
367 header);
368 printk (KERN_INFO PREFIX
369 "RMRR base: %#016Lx end: %#016Lx\n",
370 (unsigned long long)rmrr->base_address,
371 (unsigned long long)rmrr->end_address);
372 break;
373 case ACPI_DMAR_TYPE_ATSR:
374 atsr = container_of(header, struct acpi_dmar_atsr, header);
375 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
376 break;
377 }
378 }
379
380 /**
381 * dmar_table_detect - checks to see if the platform supports DMAR devices
382 */
383 static int __init dmar_table_detect(void)
384 {
385 acpi_status status = AE_OK;
386
387 /* if we could find DMAR table, then there are DMAR devices */
388 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
389 (struct acpi_table_header **)&dmar_tbl,
390 &dmar_tbl_size);
391
392 if (ACPI_SUCCESS(status) && !dmar_tbl) {
393 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
394 status = AE_NOT_FOUND;
395 }
396
397 return (ACPI_SUCCESS(status) ? 1 : 0);
398 }
399
400 /**
401 * parse_dmar_table - parses the DMA reporting table
402 */
403 static int __init
404 parse_dmar_table(void)
405 {
406 struct acpi_table_dmar *dmar;
407 struct acpi_dmar_header *entry_header;
408 int ret = 0;
409
410 /*
411 * Do it again, earlier dmar_tbl mapping could be mapped with
412 * fixed map.
413 */
414 dmar_table_detect();
415
416 dmar = (struct acpi_table_dmar *)dmar_tbl;
417 if (!dmar)
418 return -ENODEV;
419
420 if (dmar->width < PAGE_SHIFT - 1) {
421 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
422 return -EINVAL;
423 }
424
425 printk (KERN_INFO PREFIX "Host address width %d\n",
426 dmar->width + 1);
427
428 entry_header = (struct acpi_dmar_header *)(dmar + 1);
429 while (((unsigned long)entry_header) <
430 (((unsigned long)dmar) + dmar_tbl->length)) {
431 /* Avoid looping forever on bad ACPI tables */
432 if (entry_header->length == 0) {
433 printk(KERN_WARNING PREFIX
434 "Invalid 0-length structure\n");
435 ret = -EINVAL;
436 break;
437 }
438
439 dmar_table_print_dmar_entry(entry_header);
440
441 switch (entry_header->type) {
442 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
443 ret = dmar_parse_one_drhd(entry_header);
444 break;
445 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
446 #ifdef CONFIG_DMAR
447 ret = dmar_parse_one_rmrr(entry_header);
448 #endif
449 break;
450 case ACPI_DMAR_TYPE_ATSR:
451 #ifdef CONFIG_DMAR
452 ret = dmar_parse_one_atsr(entry_header);
453 #endif
454 break;
455 default:
456 printk(KERN_WARNING PREFIX
457 "Unknown DMAR structure type\n");
458 ret = 0; /* for forward compatibility */
459 break;
460 }
461 if (ret)
462 break;
463
464 entry_header = ((void *)entry_header + entry_header->length);
465 }
466 return ret;
467 }
468
469 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
470 struct pci_dev *dev)
471 {
472 int index;
473
474 while (dev) {
475 for (index = 0; index < cnt; index++)
476 if (dev == devices[index])
477 return 1;
478
479 /* Check our parent */
480 dev = dev->bus->self;
481 }
482
483 return 0;
484 }
485
486 struct dmar_drhd_unit *
487 dmar_find_matched_drhd_unit(struct pci_dev *dev)
488 {
489 struct dmar_drhd_unit *dmaru = NULL;
490 struct acpi_dmar_hardware_unit *drhd;
491
492 list_for_each_entry(dmaru, &dmar_drhd_units, list) {
493 drhd = container_of(dmaru->hdr,
494 struct acpi_dmar_hardware_unit,
495 header);
496
497 if (dmaru->include_all &&
498 drhd->segment == pci_domain_nr(dev->bus))
499 return dmaru;
500
501 if (dmar_pci_device_match(dmaru->devices,
502 dmaru->devices_cnt, dev))
503 return dmaru;
504 }
505
506 return NULL;
507 }
508
509 int __init dmar_dev_scope_init(void)
510 {
511 struct dmar_drhd_unit *drhd, *drhd_n;
512 int ret = -ENODEV;
513
514 list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
515 ret = dmar_parse_dev(drhd);
516 if (ret)
517 return ret;
518 }
519
520 #ifdef CONFIG_DMAR
521 {
522 struct dmar_rmrr_unit *rmrr, *rmrr_n;
523 struct dmar_atsr_unit *atsr, *atsr_n;
524
525 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
526 ret = rmrr_parse_dev(rmrr);
527 if (ret)
528 return ret;
529 }
530
531 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
532 ret = atsr_parse_dev(atsr);
533 if (ret)
534 return ret;
535 }
536 }
537 #endif
538
539 return ret;
540 }
541
542
543 int __init dmar_table_init(void)
544 {
545 static int dmar_table_initialized;
546 int ret;
547
548 if (dmar_table_initialized)
549 return 0;
550
551 dmar_table_initialized = 1;
552
553 ret = parse_dmar_table();
554 if (ret) {
555 if (ret != -ENODEV)
556 printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
557 return ret;
558 }
559
560 if (list_empty(&dmar_drhd_units)) {
561 printk(KERN_INFO PREFIX "No DMAR devices found\n");
562 return -ENODEV;
563 }
564
565 #ifdef CONFIG_DMAR
566 if (list_empty(&dmar_rmrr_units))
567 printk(KERN_INFO PREFIX "No RMRR found\n");
568
569 if (list_empty(&dmar_atsr_units))
570 printk(KERN_INFO PREFIX "No ATSR found\n");
571 #endif
572
573 #ifdef CONFIG_INTR_REMAP
574 parse_ioapics_under_ir();
575 #endif
576 return 0;
577 }
578
579 void __init detect_intel_iommu(void)
580 {
581 int ret;
582
583 ret = dmar_table_detect();
584
585 {
586 #ifdef CONFIG_INTR_REMAP
587 struct acpi_table_dmar *dmar;
588 /*
589 * for now we will disable dma-remapping when interrupt
590 * remapping is enabled.
591 * When support for queued invalidation for IOTLB invalidation
592 * is added, we will not need this any more.
593 */
594 dmar = (struct acpi_table_dmar *) dmar_tbl;
595 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
596 printk(KERN_INFO
597 "Queued invalidation will be enabled to support "
598 "x2apic and Intr-remapping.\n");
599 #endif
600 #ifdef CONFIG_DMAR
601 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
602 !dmar_disabled)
603 iommu_detected = 1;
604 #endif
605 }
606 early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
607 dmar_tbl = NULL;
608 }
609
610
611 int alloc_iommu(struct dmar_drhd_unit *drhd)
612 {
613 struct intel_iommu *iommu;
614 int map_size;
615 u32 ver;
616 static int iommu_allocated = 0;
617 int agaw = 0;
618 int msagaw = 0;
619
620 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
621 if (!iommu)
622 return -ENOMEM;
623
624 iommu->seq_id = iommu_allocated++;
625 sprintf (iommu->name, "dmar%d", iommu->seq_id);
626
627 iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
628 if (!iommu->reg) {
629 printk(KERN_ERR "IOMMU: can't map the region\n");
630 goto error;
631 }
632 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
633 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
634
635 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
636 /* Promote an attitude of violence to a BIOS engineer today */
637 WARN(1, "Your BIOS is broken; DMAR reported at address %llx returns all ones!\n"
638 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
639 drhd->reg_base_addr,
640 dmi_get_system_info(DMI_BIOS_VENDOR),
641 dmi_get_system_info(DMI_BIOS_VERSION),
642 dmi_get_system_info(DMI_PRODUCT_VERSION));
643 goto err_unmap;
644 }
645
646 #ifdef CONFIG_DMAR
647 agaw = iommu_calculate_agaw(iommu);
648 if (agaw < 0) {
649 printk(KERN_ERR
650 "Cannot get a valid agaw for iommu (seq_id = %d)\n",
651 iommu->seq_id);
652 goto err_unmap;
653 }
654 msagaw = iommu_calculate_max_sagaw(iommu);
655 if (msagaw < 0) {
656 printk(KERN_ERR
657 "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
658 iommu->seq_id);
659 goto err_unmap;
660 }
661 #endif
662 iommu->agaw = agaw;
663 iommu->msagaw = msagaw;
664
665 /* the registers might be more than one page */
666 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
667 cap_max_fault_reg_offset(iommu->cap));
668 map_size = VTD_PAGE_ALIGN(map_size);
669 if (map_size > VTD_PAGE_SIZE) {
670 iounmap(iommu->reg);
671 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
672 if (!iommu->reg) {
673 printk(KERN_ERR "IOMMU: can't map the region\n");
674 goto error;
675 }
676 }
677
678 ver = readl(iommu->reg + DMAR_VER_REG);
679 pr_info("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
680 (unsigned long long)drhd->reg_base_addr,
681 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
682 (unsigned long long)iommu->cap,
683 (unsigned long long)iommu->ecap);
684
685 spin_lock_init(&iommu->register_lock);
686
687 drhd->iommu = iommu;
688 return 0;
689
690 err_unmap:
691 iounmap(iommu->reg);
692 error:
693 kfree(iommu);
694 return -1;
695 }
696
697 void free_iommu(struct intel_iommu *iommu)
698 {
699 if (!iommu)
700 return;
701
702 #ifdef CONFIG_DMAR
703 free_dmar_iommu(iommu);
704 #endif
705
706 if (iommu->reg)
707 iounmap(iommu->reg);
708 kfree(iommu);
709 }
710
711 /*
712 * Reclaim all the submitted descriptors which have completed its work.
713 */
714 static inline void reclaim_free_desc(struct q_inval *qi)
715 {
716 while (qi->desc_status[qi->free_tail] == QI_DONE ||
717 qi->desc_status[qi->free_tail] == QI_ABORT) {
718 qi->desc_status[qi->free_tail] = QI_FREE;
719 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
720 qi->free_cnt++;
721 }
722 }
723
724 static int qi_check_fault(struct intel_iommu *iommu, int index)
725 {
726 u32 fault;
727 int head, tail;
728 struct q_inval *qi = iommu->qi;
729 int wait_index = (index + 1) % QI_LENGTH;
730
731 if (qi->desc_status[wait_index] == QI_ABORT)
732 return -EAGAIN;
733
734 fault = readl(iommu->reg + DMAR_FSTS_REG);
735
736 /*
737 * If IQE happens, the head points to the descriptor associated
738 * with the error. No new descriptors are fetched until the IQE
739 * is cleared.
740 */
741 if (fault & DMA_FSTS_IQE) {
742 head = readl(iommu->reg + DMAR_IQH_REG);
743 if ((head >> DMAR_IQ_SHIFT) == index) {
744 printk(KERN_ERR "VT-d detected invalid descriptor: "
745 "low=%llx, high=%llx\n",
746 (unsigned long long)qi->desc[index].low,
747 (unsigned long long)qi->desc[index].high);
748 memcpy(&qi->desc[index], &qi->desc[wait_index],
749 sizeof(struct qi_desc));
750 __iommu_flush_cache(iommu, &qi->desc[index],
751 sizeof(struct qi_desc));
752 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
753 return -EINVAL;
754 }
755 }
756
757 /*
758 * If ITE happens, all pending wait_desc commands are aborted.
759 * No new descriptors are fetched until the ITE is cleared.
760 */
761 if (fault & DMA_FSTS_ITE) {
762 head = readl(iommu->reg + DMAR_IQH_REG);
763 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
764 head |= 1;
765 tail = readl(iommu->reg + DMAR_IQT_REG);
766 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
767
768 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
769
770 do {
771 if (qi->desc_status[head] == QI_IN_USE)
772 qi->desc_status[head] = QI_ABORT;
773 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
774 } while (head != tail);
775
776 if (qi->desc_status[wait_index] == QI_ABORT)
777 return -EAGAIN;
778 }
779
780 if (fault & DMA_FSTS_ICE)
781 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
782
783 return 0;
784 }
785
786 /*
787 * Submit the queued invalidation descriptor to the remapping
788 * hardware unit and wait for its completion.
789 */
790 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
791 {
792 int rc;
793 struct q_inval *qi = iommu->qi;
794 struct qi_desc *hw, wait_desc;
795 int wait_index, index;
796 unsigned long flags;
797
798 if (!qi)
799 return 0;
800
801 hw = qi->desc;
802
803 restart:
804 rc = 0;
805
806 spin_lock_irqsave(&qi->q_lock, flags);
807 while (qi->free_cnt < 3) {
808 spin_unlock_irqrestore(&qi->q_lock, flags);
809 cpu_relax();
810 spin_lock_irqsave(&qi->q_lock, flags);
811 }
812
813 index = qi->free_head;
814 wait_index = (index + 1) % QI_LENGTH;
815
816 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
817
818 hw[index] = *desc;
819
820 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
821 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
822 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
823
824 hw[wait_index] = wait_desc;
825
826 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
827 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
828
829 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
830 qi->free_cnt -= 2;
831
832 /*
833 * update the HW tail register indicating the presence of
834 * new descriptors.
835 */
836 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
837
838 while (qi->desc_status[wait_index] != QI_DONE) {
839 /*
840 * We will leave the interrupts disabled, to prevent interrupt
841 * context to queue another cmd while a cmd is already submitted
842 * and waiting for completion on this cpu. This is to avoid
843 * a deadlock where the interrupt context can wait indefinitely
844 * for free slots in the queue.
845 */
846 rc = qi_check_fault(iommu, index);
847 if (rc)
848 break;
849
850 spin_unlock(&qi->q_lock);
851 cpu_relax();
852 spin_lock(&qi->q_lock);
853 }
854
855 qi->desc_status[index] = QI_DONE;
856
857 reclaim_free_desc(qi);
858 spin_unlock_irqrestore(&qi->q_lock, flags);
859
860 if (rc == -EAGAIN)
861 goto restart;
862
863 return rc;
864 }
865
866 /*
867 * Flush the global interrupt entry cache.
868 */
869 void qi_global_iec(struct intel_iommu *iommu)
870 {
871 struct qi_desc desc;
872
873 desc.low = QI_IEC_TYPE;
874 desc.high = 0;
875
876 /* should never fail */
877 qi_submit_sync(&desc, iommu);
878 }
879
880 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
881 u64 type)
882 {
883 struct qi_desc desc;
884
885 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
886 | QI_CC_GRAN(type) | QI_CC_TYPE;
887 desc.high = 0;
888
889 qi_submit_sync(&desc, iommu);
890 }
891
892 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
893 unsigned int size_order, u64 type)
894 {
895 u8 dw = 0, dr = 0;
896
897 struct qi_desc desc;
898 int ih = 0;
899
900 if (cap_write_drain(iommu->cap))
901 dw = 1;
902
903 if (cap_read_drain(iommu->cap))
904 dr = 1;
905
906 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
907 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
908 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
909 | QI_IOTLB_AM(size_order);
910
911 qi_submit_sync(&desc, iommu);
912 }
913
914 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
915 u64 addr, unsigned mask)
916 {
917 struct qi_desc desc;
918
919 if (mask) {
920 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
921 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
922 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
923 } else
924 desc.high = QI_DEV_IOTLB_ADDR(addr);
925
926 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
927 qdep = 0;
928
929 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
930 QI_DIOTLB_TYPE;
931
932 qi_submit_sync(&desc, iommu);
933 }
934
935 /*
936 * Disable Queued Invalidation interface.
937 */
938 void dmar_disable_qi(struct intel_iommu *iommu)
939 {
940 unsigned long flags;
941 u32 sts;
942 cycles_t start_time = get_cycles();
943
944 if (!ecap_qis(iommu->ecap))
945 return;
946
947 spin_lock_irqsave(&iommu->register_lock, flags);
948
949 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
950 if (!(sts & DMA_GSTS_QIES))
951 goto end;
952
953 /*
954 * Give a chance to HW to complete the pending invalidation requests.
955 */
956 while ((readl(iommu->reg + DMAR_IQT_REG) !=
957 readl(iommu->reg + DMAR_IQH_REG)) &&
958 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
959 cpu_relax();
960
961 iommu->gcmd &= ~DMA_GCMD_QIE;
962 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
963
964 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
965 !(sts & DMA_GSTS_QIES), sts);
966 end:
967 spin_unlock_irqrestore(&iommu->register_lock, flags);
968 }
969
970 /*
971 * Enable queued invalidation.
972 */
973 static void __dmar_enable_qi(struct intel_iommu *iommu)
974 {
975 u32 sts;
976 unsigned long flags;
977 struct q_inval *qi = iommu->qi;
978
979 qi->free_head = qi->free_tail = 0;
980 qi->free_cnt = QI_LENGTH;
981
982 spin_lock_irqsave(&iommu->register_lock, flags);
983
984 /* write zero to the tail reg */
985 writel(0, iommu->reg + DMAR_IQT_REG);
986
987 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
988
989 iommu->gcmd |= DMA_GCMD_QIE;
990 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
991
992 /* Make sure hardware complete it */
993 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
994
995 spin_unlock_irqrestore(&iommu->register_lock, flags);
996 }
997
998 /*
999 * Enable Queued Invalidation interface. This is a must to support
1000 * interrupt-remapping. Also used by DMA-remapping, which replaces
1001 * register based IOTLB invalidation.
1002 */
1003 int dmar_enable_qi(struct intel_iommu *iommu)
1004 {
1005 struct q_inval *qi;
1006
1007 if (!ecap_qis(iommu->ecap))
1008 return -ENOENT;
1009
1010 /*
1011 * queued invalidation is already setup and enabled.
1012 */
1013 if (iommu->qi)
1014 return 0;
1015
1016 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1017 if (!iommu->qi)
1018 return -ENOMEM;
1019
1020 qi = iommu->qi;
1021
1022 qi->desc = (void *)(get_zeroed_page(GFP_ATOMIC));
1023 if (!qi->desc) {
1024 kfree(qi);
1025 iommu->qi = 0;
1026 return -ENOMEM;
1027 }
1028
1029 qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1030 if (!qi->desc_status) {
1031 free_page((unsigned long) qi->desc);
1032 kfree(qi);
1033 iommu->qi = 0;
1034 return -ENOMEM;
1035 }
1036
1037 qi->free_head = qi->free_tail = 0;
1038 qi->free_cnt = QI_LENGTH;
1039
1040 spin_lock_init(&qi->q_lock);
1041
1042 __dmar_enable_qi(iommu);
1043
1044 return 0;
1045 }
1046
1047 /* iommu interrupt handling. Most stuff are MSI-like. */
1048
1049 enum faulttype {
1050 DMA_REMAP,
1051 INTR_REMAP,
1052 UNKNOWN,
1053 };
1054
1055 static const char *dma_remap_fault_reasons[] =
1056 {
1057 "Software",
1058 "Present bit in root entry is clear",
1059 "Present bit in context entry is clear",
1060 "Invalid context entry",
1061 "Access beyond MGAW",
1062 "PTE Write access is not set",
1063 "PTE Read access is not set",
1064 "Next page table ptr is invalid",
1065 "Root table address invalid",
1066 "Context table ptr is invalid",
1067 "non-zero reserved fields in RTP",
1068 "non-zero reserved fields in CTP",
1069 "non-zero reserved fields in PTE",
1070 };
1071
1072 static const char *intr_remap_fault_reasons[] =
1073 {
1074 "Detected reserved fields in the decoded interrupt-remapped request",
1075 "Interrupt index exceeded the interrupt-remapping table size",
1076 "Present field in the IRTE entry is clear",
1077 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1078 "Detected reserved fields in the IRTE entry",
1079 "Blocked a compatibility format interrupt request",
1080 "Blocked an interrupt request due to source-id verification failure",
1081 };
1082
1083 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
1084
1085 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1086 {
1087 if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1088 ARRAY_SIZE(intr_remap_fault_reasons))) {
1089 *fault_type = INTR_REMAP;
1090 return intr_remap_fault_reasons[fault_reason - 0x20];
1091 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1092 *fault_type = DMA_REMAP;
1093 return dma_remap_fault_reasons[fault_reason];
1094 } else {
1095 *fault_type = UNKNOWN;
1096 return "Unknown";
1097 }
1098 }
1099
1100 void dmar_msi_unmask(unsigned int irq)
1101 {
1102 struct intel_iommu *iommu = get_irq_data(irq);
1103 unsigned long flag;
1104
1105 /* unmask it */
1106 spin_lock_irqsave(&iommu->register_lock, flag);
1107 writel(0, iommu->reg + DMAR_FECTL_REG);
1108 /* Read a reg to force flush the post write */
1109 readl(iommu->reg + DMAR_FECTL_REG);
1110 spin_unlock_irqrestore(&iommu->register_lock, flag);
1111 }
1112
1113 void dmar_msi_mask(unsigned int irq)
1114 {
1115 unsigned long flag;
1116 struct intel_iommu *iommu = get_irq_data(irq);
1117
1118 /* mask it */
1119 spin_lock_irqsave(&iommu->register_lock, flag);
1120 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1121 /* Read a reg to force flush the post write */
1122 readl(iommu->reg + DMAR_FECTL_REG);
1123 spin_unlock_irqrestore(&iommu->register_lock, flag);
1124 }
1125
1126 void dmar_msi_write(int irq, struct msi_msg *msg)
1127 {
1128 struct intel_iommu *iommu = get_irq_data(irq);
1129 unsigned long flag;
1130
1131 spin_lock_irqsave(&iommu->register_lock, flag);
1132 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1133 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1134 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1135 spin_unlock_irqrestore(&iommu->register_lock, flag);
1136 }
1137
1138 void dmar_msi_read(int irq, struct msi_msg *msg)
1139 {
1140 struct intel_iommu *iommu = get_irq_data(irq);
1141 unsigned long flag;
1142
1143 spin_lock_irqsave(&iommu->register_lock, flag);
1144 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1145 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1146 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1147 spin_unlock_irqrestore(&iommu->register_lock, flag);
1148 }
1149
1150 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1151 u8 fault_reason, u16 source_id, unsigned long long addr)
1152 {
1153 const char *reason;
1154 int fault_type;
1155
1156 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1157
1158 if (fault_type == INTR_REMAP)
1159 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1160 "fault index %llx\n"
1161 "INTR-REMAP:[fault reason %02d] %s\n",
1162 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1163 PCI_FUNC(source_id & 0xFF), addr >> 48,
1164 fault_reason, reason);
1165 else
1166 printk(KERN_ERR
1167 "DMAR:[%s] Request device [%02x:%02x.%d] "
1168 "fault addr %llx \n"
1169 "DMAR:[fault reason %02d] %s\n",
1170 (type ? "DMA Read" : "DMA Write"),
1171 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1172 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1173 return 0;
1174 }
1175
1176 #define PRIMARY_FAULT_REG_LEN (16)
1177 irqreturn_t dmar_fault(int irq, void *dev_id)
1178 {
1179 struct intel_iommu *iommu = dev_id;
1180 int reg, fault_index;
1181 u32 fault_status;
1182 unsigned long flag;
1183
1184 spin_lock_irqsave(&iommu->register_lock, flag);
1185 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1186 if (fault_status)
1187 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1188 fault_status);
1189
1190 /* TBD: ignore advanced fault log currently */
1191 if (!(fault_status & DMA_FSTS_PPF))
1192 goto clear_rest;
1193
1194 fault_index = dma_fsts_fault_record_index(fault_status);
1195 reg = cap_fault_reg_offset(iommu->cap);
1196 while (1) {
1197 u8 fault_reason;
1198 u16 source_id;
1199 u64 guest_addr;
1200 int type;
1201 u32 data;
1202
1203 /* highest 32 bits */
1204 data = readl(iommu->reg + reg +
1205 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1206 if (!(data & DMA_FRCD_F))
1207 break;
1208
1209 fault_reason = dma_frcd_fault_reason(data);
1210 type = dma_frcd_type(data);
1211
1212 data = readl(iommu->reg + reg +
1213 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1214 source_id = dma_frcd_source_id(data);
1215
1216 guest_addr = dmar_readq(iommu->reg + reg +
1217 fault_index * PRIMARY_FAULT_REG_LEN);
1218 guest_addr = dma_frcd_page_addr(guest_addr);
1219 /* clear the fault */
1220 writel(DMA_FRCD_F, iommu->reg + reg +
1221 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1222
1223 spin_unlock_irqrestore(&iommu->register_lock, flag);
1224
1225 dmar_fault_do_one(iommu, type, fault_reason,
1226 source_id, guest_addr);
1227
1228 fault_index++;
1229 if (fault_index > cap_num_fault_regs(iommu->cap))
1230 fault_index = 0;
1231 spin_lock_irqsave(&iommu->register_lock, flag);
1232 }
1233 clear_rest:
1234 /* clear all the other faults */
1235 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1236 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1237
1238 spin_unlock_irqrestore(&iommu->register_lock, flag);
1239 return IRQ_HANDLED;
1240 }
1241
1242 int dmar_set_interrupt(struct intel_iommu *iommu)
1243 {
1244 int irq, ret;
1245
1246 /*
1247 * Check if the fault interrupt is already initialized.
1248 */
1249 if (iommu->irq)
1250 return 0;
1251
1252 irq = create_irq();
1253 if (!irq) {
1254 printk(KERN_ERR "IOMMU: no free vectors\n");
1255 return -EINVAL;
1256 }
1257
1258 set_irq_data(irq, iommu);
1259 iommu->irq = irq;
1260
1261 ret = arch_setup_dmar_msi(irq);
1262 if (ret) {
1263 set_irq_data(irq, NULL);
1264 iommu->irq = 0;
1265 destroy_irq(irq);
1266 return ret;
1267 }
1268
1269 ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1270 if (ret)
1271 printk(KERN_ERR "IOMMU: can't request irq\n");
1272 return ret;
1273 }
1274
1275 int __init enable_drhd_fault_handling(void)
1276 {
1277 struct dmar_drhd_unit *drhd;
1278
1279 /*
1280 * Enable fault control interrupt.
1281 */
1282 for_each_drhd_unit(drhd) {
1283 int ret;
1284 struct intel_iommu *iommu = drhd->iommu;
1285 ret = dmar_set_interrupt(iommu);
1286
1287 if (ret) {
1288 printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1289 " interrupt, ret %d\n",
1290 (unsigned long long)drhd->reg_base_addr, ret);
1291 return -1;
1292 }
1293 }
1294
1295 return 0;
1296 }
1297
1298 /*
1299 * Re-enable Queued Invalidation interface.
1300 */
1301 int dmar_reenable_qi(struct intel_iommu *iommu)
1302 {
1303 if (!ecap_qis(iommu->ecap))
1304 return -ENOENT;
1305
1306 if (!iommu->qi)
1307 return -ENOENT;
1308
1309 /*
1310 * First disable queued invalidation.
1311 */
1312 dmar_disable_qi(iommu);
1313 /*
1314 * Then enable queued invalidation again. Since there is no pending
1315 * invalidation requests now, it's safe to re-enable queued
1316 * invalidation.
1317 */
1318 __dmar_enable_qi(iommu);
1319
1320 return 0;
1321 }
linux 2.6 git repository for openrd