?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
i2c-designware-pci: use dev_err() instead of printk()
[linux-2.6/btrfs-unstable.git] / drivers / iommu / dmar.c
blobe5cdaf87822c3a5e53b07feedb47c33ffc5d2d09
1 /*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
21 *
22 * This file implements early detection/parsing of Remapping Devices
23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
24 * tables.
25 *
26 * These routines are used by both DMA-remapping and Interrupt-remapping
27 */
28
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt /* has to precede printk.h */
30
31 #include <linux/pci.h>
32 #include <linux/dmar.h>
33 #include <linux/iova.h>
34 #include <linux/intel-iommu.h>
35 #include <linux/timer.h>
36 #include <linux/irq.h>
37 #include <linux/interrupt.h>
38 #include <linux/tboot.h>
39 #include <linux/dmi.h>
40 #include <linux/slab.h>
41 #include <asm/irq_remapping.h>
42 #include <asm/iommu_table.h>
43
44 #include "irq_remapping.h"
45
46 /* No locks are needed as DMA remapping hardware unit
47 * list is constructed at boot time and hotplug of
48 * these units are not supported by the architecture.
49 */
50 LIST_HEAD(dmar_drhd_units);
51
52 struct acpi_table_header * __initdata dmar_tbl;
53 static acpi_size dmar_tbl_size;
54
55 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
56 {
57 /*
58 * add INCLUDE_ALL at the tail, so scan the list will find it at
59 * the very end.
60 */
61 if (drhd->include_all)
62 list_add_tail(&drhd->list, &dmar_drhd_units);
63 else
64 list_add(&drhd->list, &dmar_drhd_units);
65 }
66
67 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
68 struct pci_dev **dev, u16 segment)
69 {
70 struct pci_bus *bus;
71 struct pci_dev *pdev = NULL;
72 struct acpi_dmar_pci_path *path;
73 int count;
74
75 bus = pci_find_bus(segment, scope->bus);
76 path = (struct acpi_dmar_pci_path *)(scope + 1);
77 count = (scope->length - sizeof(struct acpi_dmar_device_scope))
78 / sizeof(struct acpi_dmar_pci_path);
79
80 while (count) {
81 if (pdev)
82 pci_dev_put(pdev);
83 /*
84 * Some BIOSes list non-exist devices in DMAR table, just
85 * ignore it
86 */
87 if (!bus) {
88 pr_warn("Device scope bus [%d] not found\n", scope->bus);
89 break;
90 }
91 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
92 if (!pdev) {
93 /* warning will be printed below */
94 break;
95 }
96 path ++;
97 count --;
98 bus = pdev->subordinate;
99 }
100 if (!pdev) {
101 pr_warn("Device scope device [%04x:%02x:%02x.%02x] not found\n",
102 segment, scope->bus, path->dev, path->fn);
103 *dev = NULL;
104 return 0;
105 }
106 if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
107 pdev->subordinate) || (scope->entry_type == \
108 ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
109 pci_dev_put(pdev);
110 pr_warn("Device scope type does not match for %s\n",
111 pci_name(pdev));
112 return -EINVAL;
113 }
114 *dev = pdev;
115 return 0;
116 }
117
118 int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
119 struct pci_dev ***devices, u16 segment)
120 {
121 struct acpi_dmar_device_scope *scope;
122 void * tmp = start;
123 int index;
124 int ret;
125
126 *cnt = 0;
127 while (start < end) {
128 scope = start;
129 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
130 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
131 (*cnt)++;
132 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
133 pr_warn("Unsupported device scope\n");
134 }
135 start += scope->length;
136 }
137 if (*cnt == 0)
138 return 0;
139
140 *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
141 if (!*devices)
142 return -ENOMEM;
143
144 start = tmp;
145 index = 0;
146 while (start < end) {
147 scope = start;
148 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
149 scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
150 ret = dmar_parse_one_dev_scope(scope,
151 &(*devices)[index], segment);
152 if (ret) {
153 kfree(*devices);
154 return ret;
155 }
156 index ++;
157 }
158 start += scope->length;
159 }
160
161 return 0;
162 }
163
164 /**
165 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
166 * structure which uniquely represent one DMA remapping hardware unit
167 * present in the platform
168 */
169 static int __init
170 dmar_parse_one_drhd(struct acpi_dmar_header *header)
171 {
172 struct acpi_dmar_hardware_unit *drhd;
173 struct dmar_drhd_unit *dmaru;
174 int ret = 0;
175
176 drhd = (struct acpi_dmar_hardware_unit *)header;
177 dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
178 if (!dmaru)
179 return -ENOMEM;
180
181 dmaru->hdr = header;
182 dmaru->reg_base_addr = drhd->address;
183 dmaru->segment = drhd->segment;
184 dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
185
186 ret = alloc_iommu(dmaru);
187 if (ret) {
188 kfree(dmaru);
189 return ret;
190 }
191 dmar_register_drhd_unit(dmaru);
192 return 0;
193 }
194
195 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
196 {
197 struct acpi_dmar_hardware_unit *drhd;
198 int ret = 0;
199
200 drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
201
202 if (dmaru->include_all)
203 return 0;
204
205 ret = dmar_parse_dev_scope((void *)(drhd + 1),
206 ((void *)drhd) + drhd->header.length,
207 &dmaru->devices_cnt, &dmaru->devices,
208 drhd->segment);
209 if (ret) {
210 list_del(&dmaru->list);
211 kfree(dmaru);
212 }
213 return ret;
214 }
215
216 #ifdef CONFIG_ACPI_NUMA
217 static int __init
218 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
219 {
220 struct acpi_dmar_rhsa *rhsa;
221 struct dmar_drhd_unit *drhd;
222
223 rhsa = (struct acpi_dmar_rhsa *)header;
224 for_each_drhd_unit(drhd) {
225 if (drhd->reg_base_addr == rhsa->base_address) {
226 int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
227
228 if (!node_online(node))
229 node = -1;
230 drhd->iommu->node = node;
231 return 0;
232 }
233 }
234 WARN_TAINT(
235 1, TAINT_FIRMWARE_WORKAROUND,
236 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
237 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
238 drhd->reg_base_addr,
239 dmi_get_system_info(DMI_BIOS_VENDOR),
240 dmi_get_system_info(DMI_BIOS_VERSION),
241 dmi_get_system_info(DMI_PRODUCT_VERSION));
242
243 return 0;
244 }
245 #endif
246
247 static void __init
248 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
249 {
250 struct acpi_dmar_hardware_unit *drhd;
251 struct acpi_dmar_reserved_memory *rmrr;
252 struct acpi_dmar_atsr *atsr;
253 struct acpi_dmar_rhsa *rhsa;
254
255 switch (header->type) {
256 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
257 drhd = container_of(header, struct acpi_dmar_hardware_unit,
258 header);
259 pr_info("DRHD base: %#016Lx flags: %#x\n",
260 (unsigned long long)drhd->address, drhd->flags);
261 break;
262 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
263 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
264 header);
265 pr_info("RMRR base: %#016Lx end: %#016Lx\n",
266 (unsigned long long)rmrr->base_address,
267 (unsigned long long)rmrr->end_address);
268 break;
269 case ACPI_DMAR_TYPE_ATSR:
270 atsr = container_of(header, struct acpi_dmar_atsr, header);
271 pr_info("ATSR flags: %#x\n", atsr->flags);
272 break;
273 case ACPI_DMAR_HARDWARE_AFFINITY:
274 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
275 pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
276 (unsigned long long)rhsa->base_address,
277 rhsa->proximity_domain);
278 break;
279 }
280 }
281
282 /**
283 * dmar_table_detect - checks to see if the platform supports DMAR devices
284 */
285 static int __init dmar_table_detect(void)
286 {
287 acpi_status status = AE_OK;
288
289 /* if we could find DMAR table, then there are DMAR devices */
290 status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
291 (struct acpi_table_header **)&dmar_tbl,
292 &dmar_tbl_size);
293
294 if (ACPI_SUCCESS(status) && !dmar_tbl) {
295 pr_warn("Unable to map DMAR\n");
296 status = AE_NOT_FOUND;
297 }
298
299 return (ACPI_SUCCESS(status) ? 1 : 0);
300 }
301
302 /**
303 * parse_dmar_table - parses the DMA reporting table
304 */
305 static int __init
306 parse_dmar_table(void)
307 {
308 struct acpi_table_dmar *dmar;
309 struct acpi_dmar_header *entry_header;
310 int ret = 0;
311
312 /*
313 * Do it again, earlier dmar_tbl mapping could be mapped with
314 * fixed map.
315 */
316 dmar_table_detect();
317
318 /*
319 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
320 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
321 */
322 dmar_tbl = tboot_get_dmar_table(dmar_tbl);
323
324 dmar = (struct acpi_table_dmar *)dmar_tbl;
325 if (!dmar)
326 return -ENODEV;
327
328 if (dmar->width < PAGE_SHIFT - 1) {
329 pr_warn("Invalid DMAR haw\n");
330 return -EINVAL;
331 }
332
333 pr_info("Host address width %d\n", dmar->width + 1);
334
335 entry_header = (struct acpi_dmar_header *)(dmar + 1);
336 while (((unsigned long)entry_header) <
337 (((unsigned long)dmar) + dmar_tbl->length)) {
338 /* Avoid looping forever on bad ACPI tables */
339 if (entry_header->length == 0) {
340 pr_warn("Invalid 0-length structure\n");
341 ret = -EINVAL;
342 break;
343 }
344
345 dmar_table_print_dmar_entry(entry_header);
346
347 switch (entry_header->type) {
348 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
349 ret = dmar_parse_one_drhd(entry_header);
350 break;
351 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
352 ret = dmar_parse_one_rmrr(entry_header);
353 break;
354 case ACPI_DMAR_TYPE_ATSR:
355 ret = dmar_parse_one_atsr(entry_header);
356 break;
357 case ACPI_DMAR_HARDWARE_AFFINITY:
358 #ifdef CONFIG_ACPI_NUMA
359 ret = dmar_parse_one_rhsa(entry_header);
360 #endif
361 break;
362 default:
363 pr_warn("Unknown DMAR structure type %d\n",
364 entry_header->type);
365 ret = 0; /* for forward compatibility */
366 break;
367 }
368 if (ret)
369 break;
370
371 entry_header = ((void *)entry_header + entry_header->length);
372 }
373 return ret;
374 }
375
376 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
377 struct pci_dev *dev)
378 {
379 int index;
380
381 while (dev) {
382 for (index = 0; index < cnt; index++)
383 if (dev == devices[index])
384 return 1;
385
386 /* Check our parent */
387 dev = dev->bus->self;
388 }
389
390 return 0;
391 }
392
393 struct dmar_drhd_unit *
394 dmar_find_matched_drhd_unit(struct pci_dev *dev)
395 {
396 struct dmar_drhd_unit *dmaru = NULL;
397 struct acpi_dmar_hardware_unit *drhd;
398
399 dev = pci_physfn(dev);
400
401 list_for_each_entry(dmaru, &dmar_drhd_units, list) {
402 drhd = container_of(dmaru->hdr,
403 struct acpi_dmar_hardware_unit,
404 header);
405
406 if (dmaru->include_all &&
407 drhd->segment == pci_domain_nr(dev->bus))
408 return dmaru;
409
410 if (dmar_pci_device_match(dmaru->devices,
411 dmaru->devices_cnt, dev))
412 return dmaru;
413 }
414
415 return NULL;
416 }
417
418 int __init dmar_dev_scope_init(void)
419 {
420 static int dmar_dev_scope_initialized;
421 struct dmar_drhd_unit *drhd, *drhd_n;
422 int ret = -ENODEV;
423
424 if (dmar_dev_scope_initialized)
425 return dmar_dev_scope_initialized;
426
427 if (list_empty(&dmar_drhd_units))
428 goto fail;
429
430 list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
431 ret = dmar_parse_dev(drhd);
432 if (ret)
433 goto fail;
434 }
435
436 ret = dmar_parse_rmrr_atsr_dev();
437 if (ret)
438 goto fail;
439
440 dmar_dev_scope_initialized = 1;
441 return 0;
442
443 fail:
444 dmar_dev_scope_initialized = ret;
445 return ret;
446 }
447
448
449 int __init dmar_table_init(void)
450 {
451 static int dmar_table_initialized;
452 int ret;
453
454 if (dmar_table_initialized)
455 return 0;
456
457 dmar_table_initialized = 1;
458
459 ret = parse_dmar_table();
460 if (ret) {
461 if (ret != -ENODEV)
462 pr_info("parse DMAR table failure.\n");
463 return ret;
464 }
465
466 if (list_empty(&dmar_drhd_units)) {
467 pr_info("No DMAR devices found\n");
468 return -ENODEV;
469 }
470
471 return 0;
472 }
473
474 static void warn_invalid_dmar(u64 addr, const char *message)
475 {
476 WARN_TAINT_ONCE(
477 1, TAINT_FIRMWARE_WORKAROUND,
478 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
479 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
480 addr, message,
481 dmi_get_system_info(DMI_BIOS_VENDOR),
482 dmi_get_system_info(DMI_BIOS_VERSION),
483 dmi_get_system_info(DMI_PRODUCT_VERSION));
484 }
485
486 int __init check_zero_address(void)
487 {
488 struct acpi_table_dmar *dmar;
489 struct acpi_dmar_header *entry_header;
490 struct acpi_dmar_hardware_unit *drhd;
491
492 dmar = (struct acpi_table_dmar *)dmar_tbl;
493 entry_header = (struct acpi_dmar_header *)(dmar + 1);
494
495 while (((unsigned long)entry_header) <
496 (((unsigned long)dmar) + dmar_tbl->length)) {
497 /* Avoid looping forever on bad ACPI tables */
498 if (entry_header->length == 0) {
499 pr_warn("Invalid 0-length structure\n");
500 return 0;
501 }
502
503 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
504 void __iomem *addr;
505 u64 cap, ecap;
506
507 drhd = (void *)entry_header;
508 if (!drhd->address) {
509 warn_invalid_dmar(0, "");
510 goto failed;
511 }
512
513 addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
514 if (!addr ) {
515 printk("IOMMU: can't validate: %llx\n", drhd->address);
516 goto failed;
517 }
518 cap = dmar_readq(addr + DMAR_CAP_REG);
519 ecap = dmar_readq(addr + DMAR_ECAP_REG);
520 early_iounmap(addr, VTD_PAGE_SIZE);
521 if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
522 warn_invalid_dmar(drhd->address,
523 " returns all ones");
524 goto failed;
525 }
526 }
527
528 entry_header = ((void *)entry_header + entry_header->length);
529 }
530 return 1;
531
532 failed:
533 return 0;
534 }
535
536 int __init detect_intel_iommu(void)
537 {
538 int ret;
539
540 ret = dmar_table_detect();
541 if (ret)
542 ret = check_zero_address();
543 {
544 struct acpi_table_dmar *dmar;
545
546 dmar = (struct acpi_table_dmar *) dmar_tbl;
547
548 if (ret && irq_remapping_enabled && cpu_has_x2apic &&
549 dmar->flags & 0x1)
550 pr_info("Queued invalidation will be enabled to support x2apic and Intr-remapping.\n");
551
552 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
553 iommu_detected = 1;
554 /* Make sure ACS will be enabled */
555 pci_request_acs();
556 }
557
558 #ifdef CONFIG_X86
559 if (ret)
560 x86_init.iommu.iommu_init = intel_iommu_init;
561 #endif
562 }
563 early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
564 dmar_tbl = NULL;
565
566 return ret ? 1 : -ENODEV;
567 }
568
569
570 static void unmap_iommu(struct intel_iommu *iommu)
571 {
572 iounmap(iommu->reg);
573 release_mem_region(iommu->reg_phys, iommu->reg_size);
574 }
575
576 /**
577 * map_iommu: map the iommu's registers
578 * @iommu: the iommu to map
579 * @phys_addr: the physical address of the base resgister
580 *
581 * Memory map the iommu's registers. Start w/ a single page, and
582 * possibly expand if that turns out to be insufficent.
583 */
584 static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
585 {
586 int map_size, err=0;
587
588 iommu->reg_phys = phys_addr;
589 iommu->reg_size = VTD_PAGE_SIZE;
590
591 if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
592 pr_err("IOMMU: can't reserve memory\n");
593 err = -EBUSY;
594 goto out;
595 }
596
597 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
598 if (!iommu->reg) {
599 pr_err("IOMMU: can't map the region\n");
600 err = -ENOMEM;
601 goto release;
602 }
603
604 iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
605 iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
606
607 if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
608 err = -EINVAL;
609 warn_invalid_dmar(phys_addr, " returns all ones");
610 goto unmap;
611 }
612
613 /* the registers might be more than one page */
614 map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
615 cap_max_fault_reg_offset(iommu->cap));
616 map_size = VTD_PAGE_ALIGN(map_size);
617 if (map_size > iommu->reg_size) {
618 iounmap(iommu->reg);
619 release_mem_region(iommu->reg_phys, iommu->reg_size);
620 iommu->reg_size = map_size;
621 if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
622 iommu->name)) {
623 pr_err("IOMMU: can't reserve memory\n");
624 err = -EBUSY;
625 goto out;
626 }
627 iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
628 if (!iommu->reg) {
629 pr_err("IOMMU: can't map the region\n");
630 err = -ENOMEM;
631 goto release;
632 }
633 }
634 err = 0;
635 goto out;
636
637 unmap:
638 iounmap(iommu->reg);
639 release:
640 release_mem_region(iommu->reg_phys, iommu->reg_size);
641 out:
642 return err;
643 }
644
645 int alloc_iommu(struct dmar_drhd_unit *drhd)
646 {
647 struct intel_iommu *iommu;
648 u32 ver;
649 static int iommu_allocated = 0;
650 int agaw = 0;
651 int msagaw = 0;
652 int err;
653
654 if (!drhd->reg_base_addr) {
655 warn_invalid_dmar(0, "");
656 return -EINVAL;
657 }
658
659 iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
660 if (!iommu)
661 return -ENOMEM;
662
663 iommu->seq_id = iommu_allocated++;
664 sprintf (iommu->name, "dmar%d", iommu->seq_id);
665
666 err = map_iommu(iommu, drhd->reg_base_addr);
667 if (err) {
668 pr_err("IOMMU: failed to map %s\n", iommu->name);
669 goto error;
670 }
671
672 err = -EINVAL;
673 agaw = iommu_calculate_agaw(iommu);
674 if (agaw < 0) {
675 pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
676 iommu->seq_id);
677 goto err_unmap;
678 }
679 msagaw = iommu_calculate_max_sagaw(iommu);
680 if (msagaw < 0) {
681 pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
682 iommu->seq_id);
683 goto err_unmap;
684 }
685 iommu->agaw = agaw;
686 iommu->msagaw = msagaw;
687
688 iommu->node = -1;
689
690 ver = readl(iommu->reg + DMAR_VER_REG);
691 pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
692 iommu->seq_id,
693 (unsigned long long)drhd->reg_base_addr,
694 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
695 (unsigned long long)iommu->cap,
696 (unsigned long long)iommu->ecap);
697
698 raw_spin_lock_init(&iommu->register_lock);
699
700 drhd->iommu = iommu;
701 return 0;
702
703 err_unmap:
704 unmap_iommu(iommu);
705 error:
706 kfree(iommu);
707 return err;
708 }
709
710 void free_iommu(struct intel_iommu *iommu)
711 {
712 if (!iommu)
713 return;
714
715 free_dmar_iommu(iommu);
716
717 if (iommu->reg)
718 unmap_iommu(iommu);
719
720 kfree(iommu);
721 }
722
723 /*
724 * Reclaim all the submitted descriptors which have completed its work.
725 */
726 static inline void reclaim_free_desc(struct q_inval *qi)
727 {
728 while (qi->desc_status[qi->free_tail] == QI_DONE ||
729 qi->desc_status[qi->free_tail] == QI_ABORT) {
730 qi->desc_status[qi->free_tail] = QI_FREE;
731 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
732 qi->free_cnt++;
733 }
734 }
735
736 static int qi_check_fault(struct intel_iommu *iommu, int index)
737 {
738 u32 fault;
739 int head, tail;
740 struct q_inval *qi = iommu->qi;
741 int wait_index = (index + 1) % QI_LENGTH;
742
743 if (qi->desc_status[wait_index] == QI_ABORT)
744 return -EAGAIN;
745
746 fault = readl(iommu->reg + DMAR_FSTS_REG);
747
748 /*
749 * If IQE happens, the head points to the descriptor associated
750 * with the error. No new descriptors are fetched until the IQE
751 * is cleared.
752 */
753 if (fault & DMA_FSTS_IQE) {
754 head = readl(iommu->reg + DMAR_IQH_REG);
755 if ((head >> DMAR_IQ_SHIFT) == index) {
756 pr_err("VT-d detected invalid descriptor: "
757 "low=%llx, high=%llx\n",
758 (unsigned long long)qi->desc[index].low,
759 (unsigned long long)qi->desc[index].high);
760 memcpy(&qi->desc[index], &qi->desc[wait_index],
761 sizeof(struct qi_desc));
762 __iommu_flush_cache(iommu, &qi->desc[index],
763 sizeof(struct qi_desc));
764 writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
765 return -EINVAL;
766 }
767 }
768
769 /*
770 * If ITE happens, all pending wait_desc commands are aborted.
771 * No new descriptors are fetched until the ITE is cleared.
772 */
773 if (fault & DMA_FSTS_ITE) {
774 head = readl(iommu->reg + DMAR_IQH_REG);
775 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
776 head |= 1;
777 tail = readl(iommu->reg + DMAR_IQT_REG);
778 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
779
780 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
781
782 do {
783 if (qi->desc_status[head] == QI_IN_USE)
784 qi->desc_status[head] = QI_ABORT;
785 head = (head - 2 + QI_LENGTH) % QI_LENGTH;
786 } while (head != tail);
787
788 if (qi->desc_status[wait_index] == QI_ABORT)
789 return -EAGAIN;
790 }
791
792 if (fault & DMA_FSTS_ICE)
793 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
794
795 return 0;
796 }
797
798 /*
799 * Submit the queued invalidation descriptor to the remapping
800 * hardware unit and wait for its completion.
801 */
802 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
803 {
804 int rc;
805 struct q_inval *qi = iommu->qi;
806 struct qi_desc *hw, wait_desc;
807 int wait_index, index;
808 unsigned long flags;
809
810 if (!qi)
811 return 0;
812
813 hw = qi->desc;
814
815 restart:
816 rc = 0;
817
818 raw_spin_lock_irqsave(&qi->q_lock, flags);
819 while (qi->free_cnt < 3) {
820 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
821 cpu_relax();
822 raw_spin_lock_irqsave(&qi->q_lock, flags);
823 }
824
825 index = qi->free_head;
826 wait_index = (index + 1) % QI_LENGTH;
827
828 qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
829
830 hw[index] = *desc;
831
832 wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
833 QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
834 wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
835
836 hw[wait_index] = wait_desc;
837
838 __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
839 __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
840
841 qi->free_head = (qi->free_head + 2) % QI_LENGTH;
842 qi->free_cnt -= 2;
843
844 /*
845 * update the HW tail register indicating the presence of
846 * new descriptors.
847 */
848 writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
849
850 while (qi->desc_status[wait_index] != QI_DONE) {
851 /*
852 * We will leave the interrupts disabled, to prevent interrupt
853 * context to queue another cmd while a cmd is already submitted
854 * and waiting for completion on this cpu. This is to avoid
855 * a deadlock where the interrupt context can wait indefinitely
856 * for free slots in the queue.
857 */
858 rc = qi_check_fault(iommu, index);
859 if (rc)
860 break;
861
862 raw_spin_unlock(&qi->q_lock);
863 cpu_relax();
864 raw_spin_lock(&qi->q_lock);
865 }
866
867 qi->desc_status[index] = QI_DONE;
868
869 reclaim_free_desc(qi);
870 raw_spin_unlock_irqrestore(&qi->q_lock, flags);
871
872 if (rc == -EAGAIN)
873 goto restart;
874
875 return rc;
876 }
877
878 /*
879 * Flush the global interrupt entry cache.
880 */
881 void qi_global_iec(struct intel_iommu *iommu)
882 {
883 struct qi_desc desc;
884
885 desc.low = QI_IEC_TYPE;
886 desc.high = 0;
887
888 /* should never fail */
889 qi_submit_sync(&desc, iommu);
890 }
891
892 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
893 u64 type)
894 {
895 struct qi_desc desc;
896
897 desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
898 | QI_CC_GRAN(type) | QI_CC_TYPE;
899 desc.high = 0;
900
901 qi_submit_sync(&desc, iommu);
902 }
903
904 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
905 unsigned int size_order, u64 type)
906 {
907 u8 dw = 0, dr = 0;
908
909 struct qi_desc desc;
910 int ih = 0;
911
912 if (cap_write_drain(iommu->cap))
913 dw = 1;
914
915 if (cap_read_drain(iommu->cap))
916 dr = 1;
917
918 desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
919 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
920 desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
921 | QI_IOTLB_AM(size_order);
922
923 qi_submit_sync(&desc, iommu);
924 }
925
926 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
927 u64 addr, unsigned mask)
928 {
929 struct qi_desc desc;
930
931 if (mask) {
932 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
933 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
934 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
935 } else
936 desc.high = QI_DEV_IOTLB_ADDR(addr);
937
938 if (qdep >= QI_DEV_IOTLB_MAX_INVS)
939 qdep = 0;
940
941 desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
942 QI_DIOTLB_TYPE;
943
944 qi_submit_sync(&desc, iommu);
945 }
946
947 /*
948 * Disable Queued Invalidation interface.
949 */
950 void dmar_disable_qi(struct intel_iommu *iommu)
951 {
952 unsigned long flags;
953 u32 sts;
954 cycles_t start_time = get_cycles();
955
956 if (!ecap_qis(iommu->ecap))
957 return;
958
959 raw_spin_lock_irqsave(&iommu->register_lock, flags);
960
961 sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
962 if (!(sts & DMA_GSTS_QIES))
963 goto end;
964
965 /*
966 * Give a chance to HW to complete the pending invalidation requests.
967 */
968 while ((readl(iommu->reg + DMAR_IQT_REG) !=
969 readl(iommu->reg + DMAR_IQH_REG)) &&
970 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
971 cpu_relax();
972
973 iommu->gcmd &= ~DMA_GCMD_QIE;
974 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
975
976 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
977 !(sts & DMA_GSTS_QIES), sts);
978 end:
979 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
980 }
981
982 /*
983 * Enable queued invalidation.
984 */
985 static void __dmar_enable_qi(struct intel_iommu *iommu)
986 {
987 u32 sts;
988 unsigned long flags;
989 struct q_inval *qi = iommu->qi;
990
991 qi->free_head = qi->free_tail = 0;
992 qi->free_cnt = QI_LENGTH;
993
994 raw_spin_lock_irqsave(&iommu->register_lock, flags);
995
996 /* write zero to the tail reg */
997 writel(0, iommu->reg + DMAR_IQT_REG);
998
999 dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1000
1001 iommu->gcmd |= DMA_GCMD_QIE;
1002 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1003
1004 /* Make sure hardware complete it */
1005 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1006
1007 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1008 }
1009
1010 /*
1011 * Enable Queued Invalidation interface. This is a must to support
1012 * interrupt-remapping. Also used by DMA-remapping, which replaces
1013 * register based IOTLB invalidation.
1014 */
1015 int dmar_enable_qi(struct intel_iommu *iommu)
1016 {
1017 struct q_inval *qi;
1018 struct page *desc_page;
1019
1020 if (!ecap_qis(iommu->ecap))
1021 return -ENOENT;
1022
1023 /*
1024 * queued invalidation is already setup and enabled.
1025 */
1026 if (iommu->qi)
1027 return 0;
1028
1029 iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1030 if (!iommu->qi)
1031 return -ENOMEM;
1032
1033 qi = iommu->qi;
1034
1035
1036 desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1037 if (!desc_page) {
1038 kfree(qi);
1039 iommu->qi = 0;
1040 return -ENOMEM;
1041 }
1042
1043 qi->desc = page_address(desc_page);
1044
1045 qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1046 if (!qi->desc_status) {
1047 free_page((unsigned long) qi->desc);
1048 kfree(qi);
1049 iommu->qi = 0;
1050 return -ENOMEM;
1051 }
1052
1053 qi->free_head = qi->free_tail = 0;
1054 qi->free_cnt = QI_LENGTH;
1055
1056 raw_spin_lock_init(&qi->q_lock);
1057
1058 __dmar_enable_qi(iommu);
1059
1060 return 0;
1061 }
1062
1063 /* iommu interrupt handling. Most stuff are MSI-like. */
1064
1065 enum faulttype {
1066 DMA_REMAP,
1067 INTR_REMAP,
1068 UNKNOWN,
1069 };
1070
1071 static const char *dma_remap_fault_reasons[] =
1072 {
1073 "Software",
1074 "Present bit in root entry is clear",
1075 "Present bit in context entry is clear",
1076 "Invalid context entry",
1077 "Access beyond MGAW",
1078 "PTE Write access is not set",
1079 "PTE Read access is not set",
1080 "Next page table ptr is invalid",
1081 "Root table address invalid",
1082 "Context table ptr is invalid",
1083 "non-zero reserved fields in RTP",
1084 "non-zero reserved fields in CTP",
1085 "non-zero reserved fields in PTE",
1086 "PCE for translation request specifies blocking",
1087 };
1088
1089 static const char *irq_remap_fault_reasons[] =
1090 {
1091 "Detected reserved fields in the decoded interrupt-remapped request",
1092 "Interrupt index exceeded the interrupt-remapping table size",
1093 "Present field in the IRTE entry is clear",
1094 "Error accessing interrupt-remapping table pointed by IRTA_REG",
1095 "Detected reserved fields in the IRTE entry",
1096 "Blocked a compatibility format interrupt request",
1097 "Blocked an interrupt request due to source-id verification failure",
1098 };
1099
1100 #define MAX_FAULT_REASON_IDX (ARRAY_SIZE(fault_reason_strings) - 1)
1101
1102 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1103 {
1104 if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1105 ARRAY_SIZE(irq_remap_fault_reasons))) {
1106 *fault_type = INTR_REMAP;
1107 return irq_remap_fault_reasons[fault_reason - 0x20];
1108 } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1109 *fault_type = DMA_REMAP;
1110 return dma_remap_fault_reasons[fault_reason];
1111 } else {
1112 *fault_type = UNKNOWN;
1113 return "Unknown";
1114 }
1115 }
1116
1117 void dmar_msi_unmask(struct irq_data *data)
1118 {
1119 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1120 unsigned long flag;
1121
1122 /* unmask it */
1123 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1124 writel(0, iommu->reg + DMAR_FECTL_REG);
1125 /* Read a reg to force flush the post write */
1126 readl(iommu->reg + DMAR_FECTL_REG);
1127 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1128 }
1129
1130 void dmar_msi_mask(struct irq_data *data)
1131 {
1132 unsigned long flag;
1133 struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1134
1135 /* mask it */
1136 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1137 writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1138 /* Read a reg to force flush the post write */
1139 readl(iommu->reg + DMAR_FECTL_REG);
1140 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1141 }
1142
1143 void dmar_msi_write(int irq, struct msi_msg *msg)
1144 {
1145 struct intel_iommu *iommu = irq_get_handler_data(irq);
1146 unsigned long flag;
1147
1148 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1149 writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1150 writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1151 writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1152 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1153 }
1154
1155 void dmar_msi_read(int irq, struct msi_msg *msg)
1156 {
1157 struct intel_iommu *iommu = irq_get_handler_data(irq);
1158 unsigned long flag;
1159
1160 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1161 msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1162 msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1163 msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1164 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1165 }
1166
1167 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1168 u8 fault_reason, u16 source_id, unsigned long long addr)
1169 {
1170 const char *reason;
1171 int fault_type;
1172
1173 reason = dmar_get_fault_reason(fault_reason, &fault_type);
1174
1175 if (fault_type == INTR_REMAP)
1176 pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1177 "fault index %llx\n"
1178 "INTR-REMAP:[fault reason %02d] %s\n",
1179 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1180 PCI_FUNC(source_id & 0xFF), addr >> 48,
1181 fault_reason, reason);
1182 else
1183 pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
1184 "fault addr %llx \n"
1185 "DMAR:[fault reason %02d] %s\n",
1186 (type ? "DMA Read" : "DMA Write"),
1187 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1188 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1189 return 0;
1190 }
1191
1192 #define PRIMARY_FAULT_REG_LEN (16)
1193 irqreturn_t dmar_fault(int irq, void *dev_id)
1194 {
1195 struct intel_iommu *iommu = dev_id;
1196 int reg, fault_index;
1197 u32 fault_status;
1198 unsigned long flag;
1199
1200 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1201 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1202 if (fault_status)
1203 pr_err("DRHD: handling fault status reg %x\n", fault_status);
1204
1205 /* TBD: ignore advanced fault log currently */
1206 if (!(fault_status & DMA_FSTS_PPF))
1207 goto clear_rest;
1208
1209 fault_index = dma_fsts_fault_record_index(fault_status);
1210 reg = cap_fault_reg_offset(iommu->cap);
1211 while (1) {
1212 u8 fault_reason;
1213 u16 source_id;
1214 u64 guest_addr;
1215 int type;
1216 u32 data;
1217
1218 /* highest 32 bits */
1219 data = readl(iommu->reg + reg +
1220 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1221 if (!(data & DMA_FRCD_F))
1222 break;
1223
1224 fault_reason = dma_frcd_fault_reason(data);
1225 type = dma_frcd_type(data);
1226
1227 data = readl(iommu->reg + reg +
1228 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1229 source_id = dma_frcd_source_id(data);
1230
1231 guest_addr = dmar_readq(iommu->reg + reg +
1232 fault_index * PRIMARY_FAULT_REG_LEN);
1233 guest_addr = dma_frcd_page_addr(guest_addr);
1234 /* clear the fault */
1235 writel(DMA_FRCD_F, iommu->reg + reg +
1236 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1237
1238 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1239
1240 dmar_fault_do_one(iommu, type, fault_reason,
1241 source_id, guest_addr);
1242
1243 fault_index++;
1244 if (fault_index >= cap_num_fault_regs(iommu->cap))
1245 fault_index = 0;
1246 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1247 }
1248 clear_rest:
1249 /* clear all the other faults */
1250 fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1251 writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1252
1253 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1254 return IRQ_HANDLED;
1255 }
1256
1257 int dmar_set_interrupt(struct intel_iommu *iommu)
1258 {
1259 int irq, ret;
1260
1261 /*
1262 * Check if the fault interrupt is already initialized.
1263 */
1264 if (iommu->irq)
1265 return 0;
1266
1267 irq = create_irq();
1268 if (!irq) {
1269 pr_err("IOMMU: no free vectors\n");
1270 return -EINVAL;
1271 }
1272
1273 irq_set_handler_data(irq, iommu);
1274 iommu->irq = irq;
1275
1276 ret = arch_setup_dmar_msi(irq);
1277 if (ret) {
1278 irq_set_handler_data(irq, NULL);
1279 iommu->irq = 0;
1280 destroy_irq(irq);
1281 return ret;
1282 }
1283
1284 ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1285 if (ret)
1286 pr_err("IOMMU: can't request irq\n");
1287 return ret;
1288 }
1289
1290 int __init enable_drhd_fault_handling(void)
1291 {
1292 struct dmar_drhd_unit *drhd;
1293
1294 /*
1295 * Enable fault control interrupt.
1296 */
1297 for_each_drhd_unit(drhd) {
1298 int ret;
1299 struct intel_iommu *iommu = drhd->iommu;
1300 ret = dmar_set_interrupt(iommu);
1301
1302 if (ret) {
1303 pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
1304 (unsigned long long)drhd->reg_base_addr, ret);
1305 return -1;
1306 }
1307
1308 /*
1309 * Clear any previous faults.
1310 */
1311 dmar_fault(iommu->irq, iommu);
1312 }
1313
1314 return 0;
1315 }
1316
1317 /*
1318 * Re-enable Queued Invalidation interface.
1319 */
1320 int dmar_reenable_qi(struct intel_iommu *iommu)
1321 {
1322 if (!ecap_qis(iommu->ecap))
1323 return -ENOENT;
1324
1325 if (!iommu->qi)
1326 return -ENOENT;
1327
1328 /*
1329 * First disable queued invalidation.
1330 */
1331 dmar_disable_qi(iommu);
1332 /*
1333 * Then enable queued invalidation again. Since there is no pending
1334 * invalidation requests now, it's safe to re-enable queued
1335 * invalidation.
1336 */
1337 __dmar_enable_qi(iommu);
1338
1339 return 0;
1340 }
1341
1342 /*
1343 * Check interrupt remapping support in DMAR table description.
1344 */
1345 int __init dmar_ir_support(void)
1346 {
1347 struct acpi_table_dmar *dmar;
1348 dmar = (struct acpi_table_dmar *)dmar_tbl;
1349 if (!dmar)
1350 return 0;
1351 return dmar->flags & 0x1;
1352 }
1353 IOMMU_INIT_POST(detect_intel_iommu);
(deprecated) Btrfs devel tree