?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Staging: comedi: Remove references to dead EXPORT_SYMTAB
[linux-2.6/x86.git] / drivers / pci / dmar.c
blobab99783dccec9570c7aecd9e8141490a97cadc14
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 *
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */
28
29 #include <linux/pci.h>
30 #include <linux/dmar.h>
31 #include <linux/iova.h>
32 #include <linux/intel-iommu.h>
33 #include <linux/timer.h>
34 #include <linux/irq.h>
35 #include <linux/interrupt.h>
36 #include <linux/tboot.h>
37
38 #undef PREFIX
39 #define PREFIX "DMAR:"
40
41 /* No locks are needed as DMA remapping hardware unit
42 * list is constructed at boot time and hotplug of
43 * these units are not supported by the architecture.
44 */
45 LIST_HEAD(dmar_drhd_units);
46
47 static struct acpi_table_header * __initdata dmar_tbl;
48 static acpi_size dmar_tbl_size;
49
50 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
51 {
52 /*
53 * add INCLUDE_ALL at the tail, so scan the list will find it at
54 * the very end.
55 */
56 if (drhd->include_all)
57 list_add_tail(&drhd->list, &dmar_drhd_units);
58 else
59 list_add(&drhd->list, &dmar_drhd_units);
60 }
61
62 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
63 struct pci_dev **dev, u16 segment)
64 {
65 struct pci_bus *bus;
66 struct pci_dev *pdev = NULL;
67 struct acpi_dmar_pci_path *path;
68 int count;
69
70 bus = pci_find_bus(segment, scope->bus);
71 path = (struct acpi_dmar_pci_path *)(scope + 1);
72 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
73 / sizeof(struct acpi_dmar_pci_path);
74
75 while (count) {
76 if (pdev)
77 pci_dev_put(pdev);
78 /*
79 * Some BIOSes list non-exist devices in DMAR table, just
80 * ignore it
81 */
82 if (!bus) {
83 printk(KERN_WARNING
84 PREFIX "Device scope bus [%d] not found\n",
85 scope->bus);
86 break;
87 }
88 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
89 if (!pdev) {
90 printk(KERN_WARNING PREFIX
91 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
92 segment, bus->number, path->dev, path->fn);
93 break;
94 }
95 path ++;
96 count --;
97 bus = pdev->subordinate;
98 }
99 if (!pdev) {
100 printk(KERN_WARNING PREFIX
101 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
102 segment, scope->bus, path->dev, path->fn);
103 *dev = NULL;
104 return 0;
105 }
106 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
107 pdev->subordinate) || (scope->entry_type == \
108 ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
109 pci_dev_put(pdev);
110 printk(KERN_WARNING PREFIX
111 "Device scope type does not match for %s\n",
112 pci_name(pdev));
113 return -EINVAL;
114 }
115 *dev = pdev;
116 return 0;
117 }
118
119 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
120 struct pci_dev ***devices, u16 segment)
121 {
122 struct acpi_dmar_device_scope *scope;
123 void * tmp = start;
124 int index;
125 int ret;
126
127 *cnt = 0;
128 while (start < end) {
129 scope = start;
130 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
131 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
132 (*cnt)++;
133 else
134 printk(KERN_WARNING PREFIX
135 "Unsupported device scope\n");
136 start += scope->length;
137 }
138 if (*cnt == 0)
139 return 0;
140
141 *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
142 if (!*devices)
143 return -ENOMEM;
144
145 start = tmp;
146 index = 0;
147 while (start < end) {
148 scope = start;
149 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
150 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
151 ret = dmar_parse_one_dev_scope(scope,
152 &(*devices)[index], segment);
153 if (ret) {
154 kfree(*devices);
155 return ret;
156 }
157 index ++;
158 }
159 start += scope->length;
160 }
161
162 return 0;
163 }
164
165 /**
166 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
167 * structure which uniquely represent one DMA remapping hardware unit
168 * present in the platform
169 */
170 static int __init
171 dmar_parse_one_drhd(struct acpi_dmar_header *header)
172 {
173 struct acpi_dmar_hardware_unit *drhd;
174 struct dmar_drhd_unit *dmaru;
175 int ret = 0;
176
177 drhd = (struct acpi_dmar_hardware_unit *)header;
178 if (!drhd->address) {
179 /* Promote an attitude of violence to a BIOS engineer today */
180 WARN(1, "Your BIOS is broken; DMAR reported at address zero!\n"
181 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
182 dmi_get_system_info(DMI_BIOS_VENDOR),
183 dmi_get_system_info(DMI_BIOS_VERSION),
184 dmi_get_system_info(DMI_PRODUCT_VERSION));
185 return -ENODEV;
186 }
187 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
188 if (!dmaru)
189 return -ENOMEM;
190
191 dmaru->hdr = header;
192 dmaru->reg_base_addr = drhd->address;
193 dmaru->segment = drhd->segment;
194 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
195
196 ret = alloc_iommu(dmaru);
197 if (ret) {
198 kfree(dmaru);
199 return ret;
200 }
201 dmar_register_drhd_unit(dmaru);
202 return 0;
203 }
204
205 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
206 {
207 struct acpi_dmar_hardware_unit *drhd;
208 int ret = 0;
209
210 drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
211
212 if (dmaru->include_all)
213 return 0;
214
215 ret = dmar_parse_dev_scope((void *)(drhd + 1),
216 ((void *)drhd) + drhd->header.length,
217 &dmaru->devices_cnt, &dmaru->devices,
218 drhd->segment);
219 if (ret) {
220 list_del(&dmaru->list);
221 kfree(dmaru);
222 }
223 return ret;
224 }
225
226 #ifdef CONFIG_DMAR
227 LIST_HEAD(dmar_rmrr_units);
228
229 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
230 {
231 list_add(&rmrr->list, &dmar_rmrr_units);
232 }
233
234
235 static int __init
236 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
237 {
238 struct acpi_dmar_reserved_memory *rmrr;
239 struct dmar_rmrr_unit *rmrru;
240
241 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
242 if (!rmrru)
243 return -ENOMEM;
244
245 rmrru->hdr = header;
246 rmrr = (struct acpi_dmar_reserved_memory *)header;
247 rmrru->base_address = rmrr->base_address;
248 rmrru->end_address = rmrr->end_address;
249
250 dmar_register_rmrr_unit(rmrru);
251 return 0;
252 }
253
254 static int __init
255 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
256 {
257 struct acpi_dmar_reserved_memory *rmrr;
258 int ret;
259
260 rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
261 ret = dmar_parse_dev_scope((void *)(rmrr + 1),
262 ((void *)rmrr) + rmrr->header.length,
263 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
264
265 if (ret || (rmrru->devices_cnt == 0)) {
266 list_del(&rmrru->list);
267 kfree(rmrru);
268 }
269 return ret;
270 }
271
272 static LIST_HEAD(dmar_atsr_units);
273
274 static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
275 {
276 struct acpi_dmar_atsr *atsr;
277 struct dmar_atsr_unit *atsru;
278
279 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
280 atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
281 if (!atsru)
282 return -ENOMEM;
283
284 atsru->hdr = hdr;
285 atsru->include_all = atsr->flags & 0x1;
286
287 list_add(&atsru->list, &dmar_atsr_units);
288
289 return 0;
290 }
291
292 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
293 {
294 int rc;
295 struct acpi_dmar_atsr *atsr;
296
297 if (atsru->include_all)
298 return 0;
299
300 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
301 rc = dmar_parse_dev_scope((void *)(atsr + 1),
302 (void *)atsr + atsr->header.length,
303 &atsru->devices_cnt, &atsru->devices,
304 atsr->segment);
305 if (rc || !atsru->devices_cnt) {
306 list_del(&atsru->list);
307 kfree(atsru);
308 }
309
310 return rc;
311 }
312
313 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
314 {
315 int i;
316 struct pci_bus *bus;
317 struct acpi_dmar_atsr *atsr;
318 struct dmar_atsr_unit *atsru;
319
320 list_for_each_entry(atsru, &dmar_atsr_units, list) {
321 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
322 if (atsr->segment == pci_domain_nr(dev->bus))
323 goto found;
324 }
325
326 return 0;
327
328 found:
329 for (bus = dev->bus; bus; bus = bus->parent) {
330 struct pci_dev *bridge = bus->self;
331
332 if (!bridge || !bridge->is_pcie ||
333 bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
334 return 0;
335
336 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
337 for (i = 0; i < atsru->devices_cnt; i++)
338 if (atsru->devices[i] == bridge)
339 return 1;
340 break;
341 }
342 }
343
344 if (atsru->include_all)
345 return 1;
346
347 return 0;
348 }
349 #endif
350
351 static void __init
352 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
353 {
354 struct acpi_dmar_hardware_unit *drhd;
355 struct acpi_dmar_reserved_memory *rmrr;
356 struct acpi_dmar_atsr *atsr;
357
358 switch (header->type) {
359 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
360 drhd = container_of(header, struct acpi_dmar_hardware_unit,
361 header);
362 printk (KERN_INFO PREFIX
363 "DRHD base: %#016Lx flags: %#x\n",
364 (unsigned long long)drhd->address, drhd->flags);
365 break;
366 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
367 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
368 header);
369 printk (KERN_INFO PREFIX
370 "RMRR base: %#016Lx end: %#016Lx\n",
371 (unsigned long long)rmrr->base_address,
372 (unsigned long long)rmrr->end_address);
373 break;
374 case ACPI_DMAR_TYPE_ATSR:
375 atsr = container_of(header, struct acpi_dmar_atsr, header);
376 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
377 break;
378 }
379 }
380
381 /**
382 * dmar_table_detect - checks to see if the platform supports DMAR devices
383 */
384 static int __init dmar_table_detect(void)
385 {
386 acpi_status status = AE_OK;
387
388 /* if we could find DMAR table, then there are DMAR devices */
389 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
390 (struct acpi_table_header **)&dmar_tbl,
391 &dmar_tbl_size);
392
393 if (ACPI_SUCCESS(status) && !dmar_tbl) {
394 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
395 status = AE_NOT_FOUND;
396 }
397
398 return (ACPI_SUCCESS(status) ? 1 : 0);
399 }
400
401 /**
402 * parse_dmar_table - parses the DMA reporting table
403 */
404 static int __init
405 parse_dmar_table(void)
406 {
407 struct acpi_table_dmar *dmar;
408 struct acpi_dmar_header *entry_header;
409 int ret = 0;
410
411 /*
412 * Do it again, earlier dmar_tbl mapping could be mapped with
413 * fixed map.
414 */
415 dmar_table_detect();
416
417 /*
418 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
419 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
420 */
421 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
422
423 dmar = (struct acpi_table_dmar *)dmar_tbl;
424 if (!dmar)
425 return -ENODEV;
426
427 if (dmar->width < PAGE_SHIFT - 1) {
428 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
429 return -EINVAL;
430 }
431
432 printk (KERN_INFO PREFIX "Host address width %d\n",
433 dmar->width + 1);
434
435 entry_header = (struct acpi_dmar_header *)(dmar + 1);
436 while (((unsigned long)entry_header) <
437 (((unsigned long)dmar) + dmar_tbl->length)) {
438 /* Avoid looping forever on bad ACPI tables */
439 if (entry_header->length == 0) {
440 printk(KERN_WARNING PREFIX
441 "Invalid 0-length structure\n");
442 ret = -EINVAL;
443 break;
444 }
445
446 dmar_table_print_dmar_entry(entry_header);
447
448 switch (entry_header->type) {
449 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
450 ret = dmar_parse_one_drhd(entry_header);
451 break;
452 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
453 #ifdef CONFIG_DMAR
454 ret = dmar_parse_one_rmrr(entry_header);
455 #endif
456 break;
457 case ACPI_DMAR_TYPE_ATSR:
458 #ifdef CONFIG_DMAR
459 ret = dmar_parse_one_atsr(entry_header);
460 #endif
461 break;
462 default:
463 printk(KERN_WARNING PREFIX
464 "Unknown DMAR structure type\n");
465 ret = 0; /* for forward compatibility */
466 break;
467 }
468 if (ret)
469 break;
470
471 entry_header = ((void *)entry_header + entry_header->length);
472 }
473 return ret;
474 }
475
476 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
477 struct pci_dev *dev)
478 {
479 int index;
480
481 while (dev) {
482 for (index = 0; index < cnt; index++)
483 if (dev == devices[index])
484 return 1;
485
486 /* Check our parent */
487 dev = dev->bus->self;
488 }
489
490 return 0;
491 }
492
493 struct dmar_drhd_unit *
494 dmar_find_matched_drhd_unit(struct pci_dev *dev)
495 {
496 struct dmar_drhd_unit *dmaru = NULL;
497 struct acpi_dmar_hardware_unit *drhd;
498
499 list_for_each_entry(dmaru, &dmar_drhd_units, list) {
500 drhd = container_of(dmaru->hdr,
501 struct acpi_dmar_hardware_unit,
502 header);
503
504 if (dmaru->include_all &&
505 drhd->segment == pci_domain_nr(dev->bus))
506 return dmaru;
507
508 if (dmar_pci_device_match(dmaru->devices,
509 dmaru->devices_cnt, dev))
510 return dmaru;
511 }
512
513 return NULL;
514 }
515
516 int __init dmar_dev_scope_init(void)
517 {
518 struct dmar_drhd_unit *drhd, *drhd_n;
519 int ret = -ENODEV;
520
521 list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
522 ret = dmar_parse_dev(drhd);
523 if (ret)
524 return ret;
525 }
526
527 #ifdef CONFIG_DMAR
528 {
529 struct dmar_rmrr_unit *rmrr, *rmrr_n;
530 struct dmar_atsr_unit *atsr, *atsr_n;
531
532 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
533 ret = rmrr_parse_dev(rmrr);
534 if (ret)
535 return ret;
536 }
537
538 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
539 ret = atsr_parse_dev(atsr);
540 if (ret)
541 return ret;
542 }
543 }
544 #endif
545
546 return ret;
547 }
548
549
550 int __init dmar_table_init(void)
551 {
552 static int dmar_table_initialized;
553 int ret;
554
555 if (dmar_table_initialized)
556 return 0;
557
558 dmar_table_initialized = 1;
559
560 ret = parse_dmar_table();
561 if (ret) {
562 if (ret != -ENODEV)
563 printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
564 return ret;
565 }
566
567 if (list_empty(&dmar_drhd_units)) {
568 printk(KERN_INFO PREFIX "No DMAR devices found\n");
569 return -ENODEV;
570 }
571
572 #ifdef CONFIG_DMAR
573 if (list_empty(&dmar_rmrr_units))
574 printk(KERN_INFO PREFIX "No RMRR found\n");
575
576 if (list_empty(&dmar_atsr_units))
577 printk(KERN_INFO PREFIX "No ATSR found\n");
578 #endif
579
580 #ifdef CONFIG_INTR_REMAP
581 parse_ioapics_under_ir();
582 #endif
583 return 0;
584 }
585
586 void __init detect_intel_iommu(void)
587 {
588 int ret;
589
590 ret = dmar_table_detect();
591
592 {
593 #ifdef CONFIG_INTR_REMAP
594 struct acpi_table_dmar *dmar;
595 /*
596 * for now we will disable dma-remapping when interrupt
597 * remapping is enabled.
598 * When support for queued invalidation for IOTLB invalidation
599 * is added, we will not need this any more.
600 */
601 dmar = (struct acpi_table_dmar *) dmar_tbl;
602 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
603 printk(KERN_INFO
604 "Queued invalidation will be enabled to support "
605 "x2apic and Intr-remapping.\n");
606 #endif
607 #ifdef CONFIG_DMAR
608 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
609 !dmar_disabled)
610 iommu_detected = 1;
611 #endif
612 }
613 early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
614 dmar_tbl = NULL;
615 }
616
617
618 int alloc_iommu(struct dmar_drhd_unit *drhd)
619 {
620 struct intel_iommu *iommu;
621 int map_size;
622 u32 ver;
623 static int iommu_allocated = 0;
624 int agaw = 0;
625 int msagaw = 0;
626
627 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
628 if (!iommu)
629 return -ENOMEM;
630
631 iommu->seq_id = iommu_allocated++;
632 sprintf (iommu->name, "dmar%d", iommu->seq_id);
633
634 iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
635 if (!iommu->reg) {
636 printk(KERN_ERR "IOMMU: can't map the region\n");
637 goto error;
638 }
639 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
640 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
641
642 #ifdef CONFIG_DMAR
643 agaw = iommu_calculate_agaw(iommu);
644 if (agaw < 0) {
645 printk(KERN_ERR
646 "Cannot get a valid agaw for iommu (seq_id = %d)\n",
647 iommu->seq_id);
648 goto error;
649 }
650 msagaw = iommu_calculate_max_sagaw(iommu);
651 if (msagaw < 0) {
652 printk(KERN_ERR
653 "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
654 iommu->seq_id);
655 goto error;
656 }
657 #endif
658 iommu->agaw = agaw;
659 iommu->msagaw = msagaw;
660
661 /* the registers might be more than one page */
662 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
663 cap_max_fault_reg_offset(iommu->cap));
664 map_size = VTD_PAGE_ALIGN(map_size);
665 if (map_size > VTD_PAGE_SIZE) {
666 iounmap(iommu->reg);
667 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
668 if (!iommu->reg) {
669 printk(KERN_ERR "IOMMU: can't map the region\n");
670 goto error;
671 }
672 }
673
674 ver = readl(iommu->reg + DMAR_VER_REG);
675 pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
676 (unsigned long long)drhd->reg_base_addr,
677 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
678 (unsigned long long)iommu->cap,
679 (unsigned long long)iommu->ecap);
680
681 spin_lock_init(&iommu->register_lock);
682
683 drhd->iommu = iommu;
684 return 0;
685 error:
686 kfree(iommu);
687 return -1;
688 }
689
690 void free_iommu(struct intel_iommu *iommu)
691 {
692 if (!iommu)
693 return;
694
695 #ifdef CONFIG_DMAR
696 free_dmar_iommu(iommu);
697 #endif
698
699 if (iommu->reg)
700 iounmap(iommu->reg);
701 kfree(iommu);
702 }
703
704 /*
705 * Reclaim all the submitted descriptors which have completed its work.
706 */
707 static inline void reclaim_free_desc(struct q_inval *qi)
708 {
709 while (qi->desc_status[qi->free_tail] == QI_DONE ||
710 qi->desc_status[qi->free_tail] == QI_ABORT) {
711 qi->desc_status[qi->free_tail] = QI_FREE;
712 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
713 qi->free_cnt++;
714 }
715 }
716
717 static int qi_check_fault(struct intel_iommu *iommu, int index)
718 {
719 u32 fault;
720 int head, tail;
721 struct q_inval *qi = iommu->qi;
722 int wait_index = (index + 1) % QI_LENGTH;
723
724 if (qi->desc_status[wait_index] == QI_ABORT)
725 return -EAGAIN;
726
727 fault = readl(iommu->reg + DMAR_FSTS_REG);
728
729 /*
730 * If IQE happens, the head points to the descriptor associated
731 * with the error. No new descriptors are fetched until the IQE
732 * is cleared.
733 */
734 if (fault & DMA_FSTS_IQE) {
735 head = readl(iommu->reg + DMAR_IQH_REG);
736 if ((head >> DMAR_IQ_SHIFT) == index) {
737 printk(KERN_ERR "VT-d detected invalid descriptor: "
738 "low=%llx, high=%llx\n",
739 (unsigned long long)qi->desc[index].low,
740 (unsigned long long)qi->desc[index].high);
741 memcpy(&qi->desc[index], &qi->desc[wait_index],
742 sizeof(struct qi_desc));
743 __iommu_flush_cache(iommu, &qi->desc[index],
744 sizeof(struct qi_desc));
745 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
746 return -EINVAL;
747 }
748 }
749
750 /*
751 * If ITE happens, all pending wait_desc commands are aborted.
752 * No new descriptors are fetched until the ITE is cleared.
753 */
754 if (fault & DMA_FSTS_ITE) {
755 head = readl(iommu->reg + DMAR_IQH_REG);
756 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
757 head |= 1;
758 tail = readl(iommu->reg + DMAR_IQT_REG);
759 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
760
761 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
762
763 do {
764 if (qi->desc_status[head] == QI_IN_USE)
765 qi->desc_status[head] = QI_ABORT;
766 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
767 } while (head != tail);
768
769 if (qi->desc_status[wait_index] == QI_ABORT)
770 return -EAGAIN;
771 }
772
773 if (fault & DMA_FSTS_ICE)
774 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
775
776 return 0;
777 }
778
779 /*
780 * Submit the queued invalidation descriptor to the remapping
781 * hardware unit and wait for its completion.
782 */
783 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
784 {
785 int rc;
786 struct q_inval *qi = iommu->qi;
787 struct qi_desc *hw, wait_desc;
788 int wait_index, index;
789 unsigned long flags;
790
791 if (!qi)
792 return 0;
793
794 hw = qi->desc;
795
796 restart:
797 rc = 0;
798
799 spin_lock_irqsave(&qi->q_lock, flags);
800 while (qi->free_cnt < 3) {
801 spin_unlock_irqrestore(&qi->q_lock, flags);
802 cpu_relax();
803 spin_lock_irqsave(&qi->q_lock, flags);
804 }
805
806 index = qi->free_head;
807 wait_index = (index + 1) % QI_LENGTH;
808
809 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
810
811 hw[index] = *desc;
812
813 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
814 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
815 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
816
817 hw[wait_index] = wait_desc;
818
819 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
820 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
821
822 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
823 qi->free_cnt -= 2;
824
825 /*
826 * update the HW tail register indicating the presence of
827 * new descriptors.
828 */
829 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
830
831 while (qi->desc_status[wait_index] != QI_DONE) {
832 /*
833 * We will leave the interrupts disabled, to prevent interrupt
834 * context to queue another cmd while a cmd is already submitted
835 * and waiting for completion on this cpu. This is to avoid
836 * a deadlock where the interrupt context can wait indefinitely
837 * for free slots in the queue.
838 */
839 rc = qi_check_fault(iommu, index);
840 if (rc)
841 break;
842
843 spin_unlock(&qi->q_lock);
844 cpu_relax();
845 spin_lock(&qi->q_lock);
846 }
847
848 qi->desc_status[index] = QI_DONE;
849
850 reclaim_free_desc(qi);
851 spin_unlock_irqrestore(&qi->q_lock, flags);
852
853 if (rc == -EAGAIN)
854 goto restart;
855
856 return rc;
857 }
858
859 /*
860 * Flush the global interrupt entry cache.
861 */
862 void qi_global_iec(struct intel_iommu *iommu)
863 {
864 struct qi_desc desc;
865
866 desc.low = QI_IEC_TYPE;
867 desc.high = 0;
868
869 /* should never fail */
870 qi_submit_sync(&desc, iommu);
871 }
872
873 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
874 u64 type)
875 {
876 struct qi_desc desc;
877
878 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
879 | QI_CC_GRAN(type) | QI_CC_TYPE;
880 desc.high = 0;
881
882 qi_submit_sync(&desc, iommu);
883 }
884
885 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
886 unsigned int size_order, u64 type)
887 {
888 u8 dw = 0, dr = 0;
889
890 struct qi_desc desc;
891 int ih = 0;
892
893 if (cap_write_drain(iommu->cap))
894 dw = 1;
895
896 if (cap_read_drain(iommu->cap))
897 dr = 1;
898
899 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
900 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
901 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
902 | QI_IOTLB_AM(size_order);
903
904 qi_submit_sync(&desc, iommu);
905 }
906
907 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
908 u64 addr, unsigned mask)
909 {
910 struct qi_desc desc;
911
912 if (mask) {
913 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
914 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
915 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
916 } else
917 desc.high = QI_DEV_IOTLB_ADDR(addr);
918
919 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
920 qdep = 0;
921
922 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
923 QI_DIOTLB_TYPE;
924
925 qi_submit_sync(&desc, iommu);
926 }
927
928 /*
929 * Disable Queued Invalidation interface.
930 */
931 void dmar_disable_qi(struct intel_iommu *iommu)
932 {
933 unsigned long flags;
934 u32 sts;
935 cycles_t start_time = get_cycles();
936
937 if (!ecap_qis(iommu->ecap))
938 return;
939
940 spin_lock_irqsave(&iommu->register_lock, flags);
941
942 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
943 if (!(sts & DMA_GSTS_QIES))
944 goto end;
945
946 /*
947 * Give a chance to HW to complete the pending invalidation requests.
948 */
949 while ((readl(iommu->reg + DMAR_IQT_REG) !=
950 readl(iommu->reg + DMAR_IQH_REG)) &&
951 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
952 cpu_relax();
953
954 iommu->gcmd &= ~DMA_GCMD_QIE;
955 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
956
957 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
958 !(sts & DMA_GSTS_QIES), sts);
959 end:
960 spin_unlock_irqrestore(&iommu->register_lock, flags);
961 }
962
963 /*
964 * Enable queued invalidation.
965 */
966 static void __dmar_enable_qi(struct intel_iommu *iommu)
967 {
968 u32 sts;
969 unsigned long flags;
970 struct q_inval *qi = iommu->qi;
971
972 qi->free_head = qi->free_tail = 0;
973 qi->free_cnt = QI_LENGTH;
974
975 spin_lock_irqsave(&iommu->register_lock, flags);
976
977 /* write zero to the tail reg */
978 writel(0, iommu->reg + DMAR_IQT_REG);
979
980 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
981
982 iommu->gcmd |= DMA_GCMD_QIE;
983 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
984
985 /* Make sure hardware complete it */
986 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
987
988 spin_unlock_irqrestore(&iommu->register_lock, flags);
989 }
990
991 /*
992 * Enable Queued Invalidation interface. This is a must to support
993 * interrupt-remapping. Also used by DMA-remapping, which replaces
994 * register based IOTLB invalidation.
995 */
996 int dmar_enable_qi(struct intel_iommu *iommu)
997 {
998 struct q_inval *qi;
999
1000 if (!ecap_qis(iommu->ecap))
1001 return -ENOENT;
1002
1003 /*
1004 * queued invalidation is already setup and enabled.
1005 */
1006 if (iommu->qi)
1007 return 0;
1008
1009 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1010 if (!iommu->qi)
1011 return -ENOMEM;
1012
1013 qi = iommu->qi;
1014
1015 qi->desc = (void *)(get_zeroed_page(GFP_ATOMIC));
1016 if (!qi->desc) {
1017 kfree(qi);
1018 iommu->qi = 0;
1019 return -ENOMEM;
1020 }
1021
1022 qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1023 if (!qi->desc_status) {
1024 free_page((unsigned long) qi->desc);
1025 kfree(qi);
1026 iommu->qi = 0;
1027 return -ENOMEM;
1028 }
1029
1030 qi->free_head = qi->free_tail = 0;
1031 qi->free_cnt = QI_LENGTH;
1032
1033 spin_lock_init(&qi->q_lock);
1034
1035 __dmar_enable_qi(iommu);
1036
1037 return 0;
1038 }
1039
1040 /* iommu interrupt handling. Most stuff are MSI-like. */
1041
1042 enum faulttype {
1043 DMA_REMAP,
1044 INTR_REMAP,
1045 UNKNOWN,
1046 };
1047
1048 static const char *dma_remap_fault_reasons[] =
1049 {
1050 "Software",
1051 "Present bit in root entry is clear",
1052 "Present bit in context entry is clear",
1053 "Invalid context entry",
1054 "Access beyond MGAW",
1055 "PTE Write access is not set",
1056 "PTE Read access is not set",
1057 "Next page table ptr is invalid",
1058 "Root table address invalid",
1059 "Context table ptr is invalid",
1060 "non-zero reserved fields in RTP",
1061 "non-zero reserved fields in CTP",
1062 "non-zero reserved fields in PTE",
1063 };
1064
1065 static const char *intr_remap_fault_reasons[] =
1066 {
1067 "Detected reserved fields in the decoded interrupt-remapped request",
1068 "Interrupt index exceeded the interrupt-remapping table size",
1069 "Present field in the IRTE entry is clear",
1070 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1071 "Detected reserved fields in the IRTE entry",
1072 "Blocked a compatibility format interrupt request",
1073 "Blocked an interrupt request due to source-id verification failure",
1074 };
1075
1076 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
1077
1078 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1079 {
1080 if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1081 ARRAY_SIZE(intr_remap_fault_reasons))) {
1082 *fault_type = INTR_REMAP;
1083 return intr_remap_fault_reasons[fault_reason - 0x20];
1084 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1085 *fault_type = DMA_REMAP;
1086 return dma_remap_fault_reasons[fault_reason];
1087 } else {
1088 *fault_type = UNKNOWN;
1089 return "Unknown";
1090 }
1091 }
1092
1093 void dmar_msi_unmask(unsigned int irq)
1094 {
1095 struct intel_iommu *iommu = get_irq_data(irq);
1096 unsigned long flag;
1097
1098 /* unmask it */
1099 spin_lock_irqsave(&iommu->register_lock, flag);
1100 writel(0, iommu->reg + DMAR_FECTL_REG);
1101 /* Read a reg to force flush the post write */
1102 readl(iommu->reg + DMAR_FECTL_REG);
1103 spin_unlock_irqrestore(&iommu->register_lock, flag);
1104 }
1105
1106 void dmar_msi_mask(unsigned int irq)
1107 {
1108 unsigned long flag;
1109 struct intel_iommu *iommu = get_irq_data(irq);
1110
1111 /* mask it */
1112 spin_lock_irqsave(&iommu->register_lock, flag);
1113 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1114 /* Read a reg to force flush the post write */
1115 readl(iommu->reg + DMAR_FECTL_REG);
1116 spin_unlock_irqrestore(&iommu->register_lock, flag);
1117 }
1118
1119 void dmar_msi_write(int irq, struct msi_msg *msg)
1120 {
1121 struct intel_iommu *iommu = get_irq_data(irq);
1122 unsigned long flag;
1123
1124 spin_lock_irqsave(&iommu->register_lock, flag);
1125 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1126 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1127 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1128 spin_unlock_irqrestore(&iommu->register_lock, flag);
1129 }
1130
1131 void dmar_msi_read(int irq, struct msi_msg *msg)
1132 {
1133 struct intel_iommu *iommu = get_irq_data(irq);
1134 unsigned long flag;
1135
1136 spin_lock_irqsave(&iommu->register_lock, flag);
1137 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1138 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1139 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1140 spin_unlock_irqrestore(&iommu->register_lock, flag);
1141 }
1142
1143 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1144 u8 fault_reason, u16 source_id, unsigned long long addr)
1145 {
1146 const char *reason;
1147 int fault_type;
1148
1149 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1150
1151 if (fault_type == INTR_REMAP)
1152 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1153 "fault index %llx\n"
1154 "INTR-REMAP:[fault reason %02d] %s\n",
1155 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1156 PCI_FUNC(source_id & 0xFF), addr >> 48,
1157 fault_reason, reason);
1158 else
1159 printk(KERN_ERR
1160 "DMAR:[%s] Request device [%02x:%02x.%d] "
1161 "fault addr %llx \n"
1162 "DMAR:[fault reason %02d] %s\n",
1163 (type ? "DMA Read" : "DMA Write"),
1164 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1165 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1166 return 0;
1167 }
1168
1169 #define PRIMARY_FAULT_REG_LEN (16)
1170 irqreturn_t dmar_fault(int irq, void *dev_id)
1171 {
1172 struct intel_iommu *iommu = dev_id;
1173 int reg, fault_index;
1174 u32 fault_status;
1175 unsigned long flag;
1176
1177 spin_lock_irqsave(&iommu->register_lock, flag);
1178 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1179 if (fault_status)
1180 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1181 fault_status);
1182
1183 /* TBD: ignore advanced fault log currently */
1184 if (!(fault_status & DMA_FSTS_PPF))
1185 goto clear_rest;
1186
1187 fault_index = dma_fsts_fault_record_index(fault_status);
1188 reg = cap_fault_reg_offset(iommu->cap);
1189 while (1) {
1190 u8 fault_reason;
1191 u16 source_id;
1192 u64 guest_addr;
1193 int type;
1194 u32 data;
1195
1196 /* highest 32 bits */
1197 data = readl(iommu->reg + reg +
1198 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1199 if (!(data & DMA_FRCD_F))
1200 break;
1201
1202 fault_reason = dma_frcd_fault_reason(data);
1203 type = dma_frcd_type(data);
1204
1205 data = readl(iommu->reg + reg +
1206 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1207 source_id = dma_frcd_source_id(data);
1208
1209 guest_addr = dmar_readq(iommu->reg + reg +
1210 fault_index * PRIMARY_FAULT_REG_LEN);
1211 guest_addr = dma_frcd_page_addr(guest_addr);
1212 /* clear the fault */
1213 writel(DMA_FRCD_F, iommu->reg + reg +
1214 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1215
1216 spin_unlock_irqrestore(&iommu->register_lock, flag);
1217
1218 dmar_fault_do_one(iommu, type, fault_reason,
1219 source_id, guest_addr);
1220
1221 fault_index++;
1222 if (fault_index > cap_num_fault_regs(iommu->cap))
1223 fault_index = 0;
1224 spin_lock_irqsave(&iommu->register_lock, flag);
1225 }
1226 clear_rest:
1227 /* clear all the other faults */
1228 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1229 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1230
1231 spin_unlock_irqrestore(&iommu->register_lock, flag);
1232 return IRQ_HANDLED;
1233 }
1234
1235 int dmar_set_interrupt(struct intel_iommu *iommu)
1236 {
1237 int irq, ret;
1238
1239 /*
1240 * Check if the fault interrupt is already initialized.
1241 */
1242 if (iommu->irq)
1243 return 0;
1244
1245 irq = create_irq();
1246 if (!irq) {
1247 printk(KERN_ERR "IOMMU: no free vectors\n");
1248 return -EINVAL;
1249 }
1250
1251 set_irq_data(irq, iommu);
1252 iommu->irq = irq;
1253
1254 ret = arch_setup_dmar_msi(irq);
1255 if (ret) {
1256 set_irq_data(irq, NULL);
1257 iommu->irq = 0;
1258 destroy_irq(irq);
1259 return ret;
1260 }
1261
1262 ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1263 if (ret)
1264 printk(KERN_ERR "IOMMU: can't request irq\n");
1265 return ret;
1266 }
1267
1268 int __init enable_drhd_fault_handling(void)
1269 {
1270 struct dmar_drhd_unit *drhd;
1271
1272 /*
1273 * Enable fault control interrupt.
1274 */
1275 for_each_drhd_unit(drhd) {
1276 int ret;
1277 struct intel_iommu *iommu = drhd->iommu;
1278 ret = dmar_set_interrupt(iommu);
1279
1280 if (ret) {
1281 printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1282 " interrupt, ret %d\n",
1283 (unsigned long long)drhd->reg_base_addr, ret);
1284 return -1;
1285 }
1286 }
1287
1288 return 0;
1289 }
1290
1291 /*
1292 * Re-enable Queued Invalidation interface.
1293 */
1294 int dmar_reenable_qi(struct intel_iommu *iommu)
1295 {
1296 if (!ecap_qis(iommu->ecap))
1297 return -ENOENT;
1298
1299 if (!iommu->qi)
1300 return -ENOENT;
1301
1302 /*
1303 * First disable queued invalidation.
1304 */
1305 dmar_disable_qi(iommu);
1306 /*
1307 * Then enable queued invalidation again. Since there is no pending
1308 * invalidation requests now, it's safe to re-enable queued
1309 * invalidation.
1310 */
1311 __dmar_enable_qi(iommu);
1312
1313 return 0;
1314 }
x86 "subsystem" repository