drivers/pci/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #include <linux/pci.h>
  30 #include <linux/dmar.h>
  31 #include <linux/iova.h>
  32 #include <linux/intel-iommu.h>
  33 #include <linux/timer.h>
  34 #include <linux/irq.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/tboot.h>
  37 #include <linux/dmi.h>
  38 #include <linux/slab.h>
  39 #include <asm/iommu_table.h>
  40
  41 #define PREFIX "DMAR: "
  42
  43 /* No locks are needed as DMA remapping hardware unit
  44  * list is constructed at boot time and hotplug of
  45  * these units are not supported by the architecture.
  46  */
  47 LIST_HEAD(dmar_drhd_units);
  48
  49 static struct acpi_table_header * __initdata dmar_tbl;
  50 static acpi_size dmar_tbl_size;
  51
  52 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  53 {
  54         /*
  55          * add INCLUDE_ALL at the tail, so scan the list will find it at
  56          * the very end.
  57          */
  58         if (drhd->include_all)
  59                 list_add_tail(&drhd->list, &dmar_drhd_units);
  60         else
  61                 list_add(&drhd->list, &dmar_drhd_units);
  62 }
  63
  64 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  65                                            struct pci_dev **dev, u16 segment)
  66 {
  67         struct pci_bus *bus;
  68         struct pci_dev *pdev = NULL;
  69         struct acpi_dmar_pci_path *path;
  70         int count;
  71
  72         bus = pci_find_bus(segment, scope->bus);
  73         path = (struct acpi_dmar_pci_path *)(scope + 1);
  74         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  75                 / sizeof(struct acpi_dmar_pci_path);
  76
  77         while (count) {
  78                 if (pdev)
  79                         pci_dev_put(pdev);
  80                 /*
  81                  * Some BIOSes list non-exist devices in DMAR table, just
  82                  * ignore it
  83                  */
  84                 if (!bus) {
  85                         printk(KERN_WARNING
  86                         PREFIX "Device scope bus [%d] not found\n",
  87                         scope->bus);
  88                         break;
  89                 }
  90                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  91                 if (!pdev) {
  92                         printk(KERN_WARNING PREFIX
  93                         "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  94                                 segment, bus->number, path->dev, path->fn);
  95                         break;
  96                 }
  97                 path ++;
  98                 count --;
  99                 bus = pdev->subordinate;
 100         }
 101         if (!pdev) {
 102                 printk(KERN_WARNING PREFIX
 103                 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
 104                 segment, scope->bus, path->dev, path->fn);
 105                 *dev = NULL;
 106                 return 0;
 107         }
 108         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 109                         pdev->subordinate) || (scope->entry_type == \
 110                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 111                 pci_dev_put(pdev);
 112                 printk(KERN_WARNING PREFIX
 113                         "Device scope type does not match for %s\n",
 114                          pci_name(pdev));
 115                 return -EINVAL;
 116         }
 117         *dev = pdev;
 118         return 0;
 119 }
 120
 121 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 122                                        struct pci_dev ***devices, u16 segment)
 123 {
 124         struct acpi_dmar_device_scope *scope;
 125         void * tmp = start;
 126         int index;
 127         int ret;
 128
 129         *cnt = 0;
 130         while (start < end) {
 131                 scope = start;
 132                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 133                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 134                         (*cnt)++;
 135                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
 136                         printk(KERN_WARNING PREFIX
 137                                "Unsupported device scope\n");
 138                 }
 139                 start += scope->length;
 140         }
 141         if (*cnt == 0)
 142                 return 0;
 143
 144         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 145         if (!*devices)
 146                 return -ENOMEM;
 147
 148         start = tmp;
 149         index = 0;
 150         while (start < end) {
 151                 scope = start;
 152                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 153                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 154                         ret = dmar_parse_one_dev_scope(scope,
 155                                 &(*devices)[index], segment);
 156                         if (ret) {
 157                                 kfree(*devices);
 158                                 return ret;
 159                         }
 160                         index ++;
 161                 }
 162                 start += scope->length;
 163         }
 164
 165         return 0;
 166 }
 167
 168 /**
 169  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 170  * structure which uniquely represent one DMA remapping hardware unit
 171  * present in the platform
 172  */
 173 static int __init
 174 dmar_parse_one_drhd(struct acpi_dmar_header *header)
 175 {
 176         struct acpi_dmar_hardware_unit *drhd;
 177         struct dmar_drhd_unit *dmaru;
 178         int ret = 0;
 179
 180         drhd = (struct acpi_dmar_hardware_unit *)header;
 181         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 182         if (!dmaru)
 183                 return -ENOMEM;
 184
 185         dmaru->hdr = header;
 186         dmaru->reg_base_addr = drhd->address;
 187         dmaru->segment = drhd->segment;
 188         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 189
 190         ret = alloc_iommu(dmaru);
 191         if (ret) {
 192                 kfree(dmaru);
 193                 return ret;
 194         }
 195         dmar_register_drhd_unit(dmaru);
 196         return 0;
 197 }
 198
 199 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 200 {
 201         struct acpi_dmar_hardware_unit *drhd;
 202         int ret = 0;
 203
 204         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 205
 206         if (dmaru->include_all)
 207                 return 0;
 208
 209         ret = dmar_parse_dev_scope((void *)(drhd + 1),
 210                                 ((void *)drhd) + drhd->header.length,
 211                                 &dmaru->devices_cnt, &dmaru->devices,
 212                                 drhd->segment);
 213         if (ret) {
 214                 list_del(&dmaru->list);
 215                 kfree(dmaru);
 216         }
 217         return ret;
 218 }
 219
 220 #ifdef CONFIG_DMAR
 221 LIST_HEAD(dmar_rmrr_units);
 222
 223 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
 224 {
 225         list_add(&rmrr->list, &dmar_rmrr_units);
 226 }
 227
 228
 229 static int __init
 230 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
 231 {
 232         struct acpi_dmar_reserved_memory *rmrr;
 233         struct dmar_rmrr_unit *rmrru;
 234
 235         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
 236         if (!rmrru)
 237                 return -ENOMEM;
 238
 239         rmrru->hdr = header;
 240         rmrr = (struct acpi_dmar_reserved_memory *)header;
 241         rmrru->base_address = rmrr->base_address;
 242         rmrru->end_address = rmrr->end_address;
 243
 244         dmar_register_rmrr_unit(rmrru);
 245         return 0;
 246 }
 247
 248 static int __init
 249 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
 250 {
 251         struct acpi_dmar_reserved_memory *rmrr;
 252         int ret;
 253
 254         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
 255         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
 256                 ((void *)rmrr) + rmrr->header.length,
 257                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
 258
 259         if (ret || (rmrru->devices_cnt == 0)) {
 260                 list_del(&rmrru->list);
 261                 kfree(rmrru);
 262         }
 263         return ret;
 264 }
 265
 266 static LIST_HEAD(dmar_atsr_units);
 267
 268 static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
 269 {
 270         struct acpi_dmar_atsr *atsr;
 271         struct dmar_atsr_unit *atsru;
 272
 273         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
 274         atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
 275         if (!atsru)
 276                 return -ENOMEM;
 277
 278         atsru->hdr = hdr;
 279         atsru->include_all = atsr->flags & 0x1;
 280
 281         list_add(&atsru->list, &dmar_atsr_units);
 282
 283         return 0;
 284 }
 285
 286 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
 287 {
 288         int rc;
 289         struct acpi_dmar_atsr *atsr;
 290
 291         if (atsru->include_all)
 292                 return 0;
 293
 294         atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
 295         rc = dmar_parse_dev_scope((void *)(atsr + 1),
 296                                 (void *)atsr + atsr->header.length,
 297                                 &atsru->devices_cnt, &atsru->devices,
 298                                 atsr->segment);
 299         if (rc || !atsru->devices_cnt) {
 300                 list_del(&atsru->list);
 301                 kfree(atsru);
 302         }
 303
 304         return rc;
 305 }
 306
 307 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
 308 {
 309         int i;
 310         struct pci_bus *bus;
 311         struct acpi_dmar_atsr *atsr;
 312         struct dmar_atsr_unit *atsru;
 313
 314         dev = pci_physfn(dev);
 315
 316         list_for_each_entry(atsru, &dmar_atsr_units, list) {
 317                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
 318                 if (atsr->segment == pci_domain_nr(dev->bus))
 319                         goto found;
 320         }
 321
 322         return 0;
 323
 324 found:
 325         for (bus = dev->bus; bus; bus = bus->parent) {
 326                 struct pci_dev *bridge = bus->self;
 327
 328                 if (!bridge || !pci_is_pcie(bridge) ||
 329                     bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
 330                         return 0;
 331
 332                 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
 333                         for (i = 0; i < atsru->devices_cnt; i++)
 334                                 if (atsru->devices[i] == bridge)
 335                                         return 1;
 336                         break;
 337                 }
 338         }
 339
 340         if (atsru->include_all)
 341                 return 1;
 342
 343         return 0;
 344 }
 345 #endif
 346
 347 #ifdef CONFIG_ACPI_NUMA
 348 static int __init
 349 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
 350 {
 351         struct acpi_dmar_rhsa *rhsa;
 352         struct dmar_drhd_unit *drhd;
 353
 354         rhsa = (struct acpi_dmar_rhsa *)header;
 355         for_each_drhd_unit(drhd) {
 356                 if (drhd->reg_base_addr == rhsa->base_address) {
 357                         int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 358
 359                         if (!node_online(node))
 360                                 node = -1;
 361                         drhd->iommu->node = node;
 362                         return 0;
 363                 }
 364         }
 365         WARN_TAINT(
 366                 1, TAINT_FIRMWARE_WORKAROUND,
 367                 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 368                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 369                 drhd->reg_base_addr,
 370                 dmi_get_system_info(DMI_BIOS_VENDOR),
 371                 dmi_get_system_info(DMI_BIOS_VERSION),
 372                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 373
 374         return 0;
 375 }
 376 #endif
 377
 378 static void __init
 379 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 380 {
 381         struct acpi_dmar_hardware_unit *drhd;
 382         struct acpi_dmar_reserved_memory *rmrr;
 383         struct acpi_dmar_atsr *atsr;
 384         struct acpi_dmar_rhsa *rhsa;
 385
 386         switch (header->type) {
 387         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 388                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 389                                     header);
 390                 printk (KERN_INFO PREFIX
 391                         "DRHD base: %#016Lx flags: %#x\n",
 392                         (unsigned long long)drhd->address, drhd->flags);
 393                 break;
 394         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 395                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 396                                     header);
 397                 printk (KERN_INFO PREFIX
 398                         "RMRR base: %#016Lx end: %#016Lx\n",
 399                         (unsigned long long)rmrr->base_address,
 400                         (unsigned long long)rmrr->end_address);
 401                 break;
 402         case ACPI_DMAR_TYPE_ATSR:
 403                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 404                 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
 405                 break;
 406         case ACPI_DMAR_HARDWARE_AFFINITY:
 407                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 408                 printk(KERN_INFO PREFIX "RHSA base: %#016Lx proximity domain: %#x\n",
 409                        (unsigned long long)rhsa->base_address,
 410                        rhsa->proximity_domain);
 411                 break;
 412         }
 413 }
 414
 415 /**
 416  * dmar_table_detect - checks to see if the platform supports DMAR devices
 417  */
 418 static int __init dmar_table_detect(void)
 419 {
 420         acpi_status status = AE_OK;
 421
 422         /* if we could find DMAR table, then there are DMAR devices */
 423         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 424                                 (struct acpi_table_header **)&dmar_tbl,
 425                                 &dmar_tbl_size);
 426
 427         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 428                 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
 429                 status = AE_NOT_FOUND;
 430         }
 431
 432         return (ACPI_SUCCESS(status) ? 1 : 0);
 433 }
 434
 435 /**
 436  * parse_dmar_table - parses the DMA reporting table
 437  */
 438 static int __init
 439 parse_dmar_table(void)
 440 {
 441         struct acpi_table_dmar *dmar;
 442         struct acpi_dmar_header *entry_header;
 443         int ret = 0;
 444
 445         /*
 446          * Do it again, earlier dmar_tbl mapping could be mapped with
 447          * fixed map.
 448          */
 449         dmar_table_detect();
 450
 451         /*
 452          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 453          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 454          */
 455         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 456
 457         dmar = (struct acpi_table_dmar *)dmar_tbl;
 458         if (!dmar)
 459                 return -ENODEV;
 460
 461         if (dmar->width < PAGE_SHIFT - 1) {
 462                 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
 463                 return -EINVAL;
 464         }
 465
 466         printk (KERN_INFO PREFIX "Host address width %d\n",
 467                 dmar->width + 1);
 468
 469         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 470         while (((unsigned long)entry_header) <
 471                         (((unsigned long)dmar) + dmar_tbl->length)) {
 472                 /* Avoid looping forever on bad ACPI tables */
 473                 if (entry_header->length == 0) {
 474                         printk(KERN_WARNING PREFIX
 475                                 "Invalid 0-length structure\n");
 476                         ret = -EINVAL;
 477                         break;
 478                 }
 479
 480                 dmar_table_print_dmar_entry(entry_header);
 481
 482                 switch (entry_header->type) {
 483                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 484                         ret = dmar_parse_one_drhd(entry_header);
 485                         break;
 486                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 487 #ifdef CONFIG_DMAR
 488                         ret = dmar_parse_one_rmrr(entry_header);
 489 #endif
 490                         break;
 491                 case ACPI_DMAR_TYPE_ATSR:
 492 #ifdef CONFIG_DMAR
 493                         ret = dmar_parse_one_atsr(entry_header);
 494 #endif
 495                         break;
 496                 case ACPI_DMAR_HARDWARE_AFFINITY:
 497 #ifdef CONFIG_ACPI_NUMA
 498                         ret = dmar_parse_one_rhsa(entry_header);
 499 #endif
 500                         break;
 501                 default:
 502                         printk(KERN_WARNING PREFIX
 503                                 "Unknown DMAR structure type %d\n",
 504                                 entry_header->type);
 505                         ret = 0; /* for forward compatibility */
 506                         break;
 507                 }
 508                 if (ret)
 509                         break;
 510
 511                 entry_header = ((void *)entry_header + entry_header->length);
 512         }
 513         return ret;
 514 }
 515
 516 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 517                           struct pci_dev *dev)
 518 {
 519         int index;
 520
 521         while (dev) {
 522                 for (index = 0; index < cnt; index++)
 523                         if (dev == devices[index])
 524                                 return 1;
 525
 526                 /* Check our parent */
 527                 dev = dev->bus->self;
 528         }
 529
 530         return 0;
 531 }
 532
 533 struct dmar_drhd_unit *
 534 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 535 {
 536         struct dmar_drhd_unit *dmaru = NULL;
 537         struct acpi_dmar_hardware_unit *drhd;
 538
 539         dev = pci_physfn(dev);
 540
 541         list_for_each_entry(dmaru, &dmar_drhd_units, list) {
 542                 drhd = container_of(dmaru->hdr,
 543                                     struct acpi_dmar_hardware_unit,
 544                                     header);
 545
 546                 if (dmaru->include_all &&
 547                     drhd->segment == pci_domain_nr(dev->bus))
 548                         return dmaru;
 549
 550                 if (dmar_pci_device_match(dmaru->devices,
 551                                           dmaru->devices_cnt, dev))
 552                         return dmaru;
 553         }
 554
 555         return NULL;
 556 }
 557
 558 int __init dmar_dev_scope_init(void)
 559 {
 560         struct dmar_drhd_unit *drhd, *drhd_n;
 561         int ret = -ENODEV;
 562
 563         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 564                 ret = dmar_parse_dev(drhd);
 565                 if (ret)
 566                         return ret;
 567         }
 568
 569 #ifdef CONFIG_DMAR
 570         {
 571                 struct dmar_rmrr_unit *rmrr, *rmrr_n;
 572                 struct dmar_atsr_unit *atsr, *atsr_n;
 573
 574                 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
 575                         ret = rmrr_parse_dev(rmrr);
 576                         if (ret)
 577                                 return ret;
 578                 }
 579
 580                 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
 581                         ret = atsr_parse_dev(atsr);
 582                         if (ret)
 583                                 return ret;
 584                 }
 585         }
 586 #endif
 587
 588         return ret;
 589 }
 590
 591
 592 int __init dmar_table_init(void)
 593 {
 594         static int dmar_table_initialized;
 595         int ret;
 596
 597         if (dmar_table_initialized)
 598                 return 0;
 599
 600         dmar_table_initialized = 1;
 601
 602         ret = parse_dmar_table();
 603         if (ret) {
 604                 if (ret != -ENODEV)
 605                         printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
 606                 return ret;
 607         }
 608
 609         if (list_empty(&dmar_drhd_units)) {
 610                 printk(KERN_INFO PREFIX "No DMAR devices found\n");
 611                 return -ENODEV;
 612         }
 613
 614 #ifdef CONFIG_DMAR
 615         if (list_empty(&dmar_rmrr_units))
 616                 printk(KERN_INFO PREFIX "No RMRR found\n");
 617
 618         if (list_empty(&dmar_atsr_units))
 619                 printk(KERN_INFO PREFIX "No ATSR found\n");
 620 #endif
 621
 622         return 0;
 623 }
 624
 625 static void warn_invalid_dmar(u64 addr, const char *message)
 626 {
 627         WARN_TAINT_ONCE(
 628                 1, TAINT_FIRMWARE_WORKAROUND,
 629                 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 630                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 631                 addr, message,
 632                 dmi_get_system_info(DMI_BIOS_VENDOR),
 633                 dmi_get_system_info(DMI_BIOS_VERSION),
 634                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 635 }
 636
 637 int __init check_zero_address(void)
 638 {
 639         struct acpi_table_dmar *dmar;
 640         struct acpi_dmar_header *entry_header;
 641         struct acpi_dmar_hardware_unit *drhd;
 642
 643         dmar = (struct acpi_table_dmar *)dmar_tbl;
 644         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 645
 646         while (((unsigned long)entry_header) <
 647                         (((unsigned long)dmar) + dmar_tbl->length)) {
 648                 /* Avoid looping forever on bad ACPI tables */
 649                 if (entry_header->length == 0) {
 650                         printk(KERN_WARNING PREFIX
 651                                 "Invalid 0-length structure\n");
 652                         return 0;
 653                 }
 654
 655                 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
 656                         void __iomem *addr;
 657                         u64 cap, ecap;
 658
 659                         drhd = (void *)entry_header;
 660                         if (!drhd->address) {
 661                                 warn_invalid_dmar(0, "");
 662                                 goto failed;
 663                         }
 664
 665                         addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 666                         if (!addr ) {
 667                                 printk("IOMMU: can't validate: %llx\n", drhd->address);
 668                                 goto failed;
 669                         }
 670                         cap = dmar_readq(addr + DMAR_CAP_REG);
 671                         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 672                         early_iounmap(addr, VTD_PAGE_SIZE);
 673                         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 674                                 warn_invalid_dmar(drhd->address,
 675                                                   " returns all ones");
 676                                 goto failed;
 677                         }
 678                 }
 679
 680                 entry_header = ((void *)entry_header + entry_header->length);
 681         }
 682         return 1;
 683
 684 failed:
 685 #ifdef CONFIG_DMAR
 686         dmar_disabled = 1;
 687 #endif
 688         return 0;
 689 }
 690
 691 int __init detect_intel_iommu(void)
 692 {
 693         int ret;
 694
 695         ret = dmar_table_detect();
 696         if (ret)
 697                 ret = check_zero_address();
 698         {
 699 #ifdef CONFIG_INTR_REMAP
 700                 struct acpi_table_dmar *dmar;
 701                 /*
 702                  * for now we will disable dma-remapping when interrupt
 703                  * remapping is enabled.
 704                  * When support for queued invalidation for IOTLB invalidation
 705                  * is added, we will not need this any more.
 706                  */
 707                 dmar = (struct acpi_table_dmar *) dmar_tbl;
 708                 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
 709                         printk(KERN_INFO
 710                                "Queued invalidation will be enabled to support "
 711                                "x2apic and Intr-remapping.\n");
 712 #endif
 713 #ifdef CONFIG_DMAR
 714                 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 715                         iommu_detected = 1;
 716                         /* Make sure ACS will be enabled */
 717                         pci_request_acs();
 718                 }
 719 #endif
 720 #ifdef CONFIG_X86
 721                 if (ret)
 722                         x86_init.iommu.iommu_init = intel_iommu_init;
 723 #endif
 724         }
 725         early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
 726         dmar_tbl = NULL;
 727
 728         return ret ? 1 : -ENODEV;
 729 }
 730
 731
 732 int alloc_iommu(struct dmar_drhd_unit *drhd)
 733 {
 734         struct intel_iommu *iommu;
 735         int map_size;
 736         u32 ver;
 737         static int iommu_allocated = 0;
 738         int agaw = 0;
 739         int msagaw = 0;
 740
 741         if (!drhd->reg_base_addr) {
 742                 warn_invalid_dmar(0, "");
 743                 return -EINVAL;
 744         }
 745
 746         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 747         if (!iommu)
 748                 return -ENOMEM;
 749
 750         iommu->seq_id = iommu_allocated++;
 751         sprintf (iommu->name, "dmar%d", iommu->seq_id);
 752
 753         iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
 754         if (!iommu->reg) {
 755                 printk(KERN_ERR "IOMMU: can't map the region\n");
 756                 goto error;
 757         }
 758         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 759         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 760
 761         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 762                 warn_invalid_dmar(drhd->reg_base_addr, " returns all ones");
 763                 goto err_unmap;
 764         }
 765
 766 #ifdef CONFIG_DMAR
 767         agaw = iommu_calculate_agaw(iommu);
 768         if (agaw < 0) {
 769                 printk(KERN_ERR
 770                        "Cannot get a valid agaw for iommu (seq_id = %d)\n",
 771                        iommu->seq_id);
 772                 goto err_unmap;
 773         }
 774         msagaw = iommu_calculate_max_sagaw(iommu);
 775         if (msagaw < 0) {
 776                 printk(KERN_ERR
 777                         "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
 778                         iommu->seq_id);
 779                 goto err_unmap;
 780         }
 781 #endif
 782         iommu->agaw = agaw;
 783         iommu->msagaw = msagaw;
 784
 785         iommu->node = -1;
 786
 787         /* the registers might be more than one page */
 788         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 789                 cap_max_fault_reg_offset(iommu->cap));
 790         map_size = VTD_PAGE_ALIGN(map_size);
 791         if (map_size > VTD_PAGE_SIZE) {
 792                 iounmap(iommu->reg);
 793                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
 794                 if (!iommu->reg) {
 795                         printk(KERN_ERR "IOMMU: can't map the region\n");
 796                         goto error;
 797                 }
 798         }
 799
 800         ver = readl(iommu->reg + DMAR_VER_REG);
 801         pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
 802                 iommu->seq_id,
 803                 (unsigned long long)drhd->reg_base_addr,
 804                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 805                 (unsigned long long)iommu->cap,
 806                 (unsigned long long)iommu->ecap);
 807
 808         spin_lock_init(&iommu->register_lock);
 809
 810         drhd->iommu = iommu;
 811         return 0;
 812
 813  err_unmap:
 814         iounmap(iommu->reg);
 815  error:
 816         kfree(iommu);
 817         return -1;
 818 }
 819
 820 void free_iommu(struct intel_iommu *iommu)
 821 {
 822         if (!iommu)
 823                 return;
 824
 825 #ifdef CONFIG_DMAR
 826         free_dmar_iommu(iommu);
 827 #endif
 828
 829         if (iommu->reg)
 830                 iounmap(iommu->reg);
 831         kfree(iommu);
 832 }
 833
 834 /*
 835  * Reclaim all the submitted descriptors which have completed its work.
 836  */
 837 static inline void reclaim_free_desc(struct q_inval *qi)
 838 {
 839         while (qi->desc_status[qi->free_tail] == QI_DONE ||
 840                qi->desc_status[qi->free_tail] == QI_ABORT) {
 841                 qi->desc_status[qi->free_tail] = QI_FREE;
 842                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 843                 qi->free_cnt++;
 844         }
 845 }
 846
 847 static int qi_check_fault(struct intel_iommu *iommu, int index)
 848 {
 849         u32 fault;
 850         int head, tail;
 851         struct q_inval *qi = iommu->qi;
 852         int wait_index = (index + 1) % QI_LENGTH;
 853
 854         if (qi->desc_status[wait_index] == QI_ABORT)
 855                 return -EAGAIN;
 856
 857         fault = readl(iommu->reg + DMAR_FSTS_REG);
 858
 859         /*
 860          * If IQE happens, the head points to the descriptor associated
 861          * with the error. No new descriptors are fetched until the IQE
 862          * is cleared.
 863          */
 864         if (fault & DMA_FSTS_IQE) {
 865                 head = readl(iommu->reg + DMAR_IQH_REG);
 866                 if ((head >> DMAR_IQ_SHIFT) == index) {
 867                         printk(KERN_ERR "VT-d detected invalid descriptor: "
 868                                 "low=%llx, high=%llx\n",
 869                                 (unsigned long long)qi->desc[index].low,
 870                                 (unsigned long long)qi->desc[index].high);
 871                         memcpy(&qi->desc[index], &qi->desc[wait_index],
 872                                         sizeof(struct qi_desc));
 873                         __iommu_flush_cache(iommu, &qi->desc[index],
 874                                         sizeof(struct qi_desc));
 875                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 876                         return -EINVAL;
 877                 }
 878         }
 879
 880         /*
 881          * If ITE happens, all pending wait_desc commands are aborted.
 882          * No new descriptors are fetched until the ITE is cleared.
 883          */
 884         if (fault & DMA_FSTS_ITE) {
 885                 head = readl(iommu->reg + DMAR_IQH_REG);
 886                 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 887                 head |= 1;
 888                 tail = readl(iommu->reg + DMAR_IQT_REG);
 889                 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 890
 891                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
 892
 893                 do {
 894                         if (qi->desc_status[head] == QI_IN_USE)
 895                                 qi->desc_status[head] = QI_ABORT;
 896                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
 897                 } while (head != tail);
 898
 899                 if (qi->desc_status[wait_index] == QI_ABORT)
 900                         return -EAGAIN;
 901         }
 902
 903         if (fault & DMA_FSTS_ICE)
 904                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
 905
 906         return 0;
 907 }
 908
 909 /*
 910  * Submit the queued invalidation descriptor to the remapping
 911  * hardware unit and wait for its completion.
 912  */
 913 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 914 {
 915         int rc;
 916         struct q_inval *qi = iommu->qi;
 917         struct qi_desc *hw, wait_desc;
 918         int wait_index, index;
 919         unsigned long flags;
 920
 921         if (!qi)
 922                 return 0;
 923
 924         hw = qi->desc;
 925
 926 restart:
 927         rc = 0;
 928
 929         spin_lock_irqsave(&qi->q_lock, flags);
 930         while (qi->free_cnt < 3) {
 931                 spin_unlock_irqrestore(&qi->q_lock, flags);
 932                 cpu_relax();
 933                 spin_lock_irqsave(&qi->q_lock, flags);
 934         }
 935
 936         index = qi->free_head;
 937         wait_index = (index + 1) % QI_LENGTH;
 938
 939         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 940
 941         hw[index] = *desc;
 942
 943         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 944                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 945         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 946
 947         hw[wait_index] = wait_desc;
 948
 949         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 950         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 951
 952         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 953         qi->free_cnt -= 2;
 954
 955         /*
 956          * update the HW tail register indicating the presence of
 957          * new descriptors.
 958          */
 959         writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
 960
 961         while (qi->desc_status[wait_index] != QI_DONE) {
 962                 /*
 963                  * We will leave the interrupts disabled, to prevent interrupt
 964                  * context to queue another cmd while a cmd is already submitted
 965                  * and waiting for completion on this cpu. This is to avoid
 966                  * a deadlock where the interrupt context can wait indefinitely
 967                  * for free slots in the queue.
 968                  */
 969                 rc = qi_check_fault(iommu, index);
 970                 if (rc)
 971                         break;
 972
 973                 spin_unlock(&qi->q_lock);
 974                 cpu_relax();
 975                 spin_lock(&qi->q_lock);
 976         }
 977
 978         qi->desc_status[index] = QI_DONE;
 979
 980         reclaim_free_desc(qi);
 981         spin_unlock_irqrestore(&qi->q_lock, flags);
 982
 983         if (rc == -EAGAIN)
 984                 goto restart;
 985
 986         return rc;
 987 }
 988
 989 /*
 990  * Flush the global interrupt entry cache.
 991  */
 992 void qi_global_iec(struct intel_iommu *iommu)
 993 {
 994         struct qi_desc desc;
 995
 996         desc.low = QI_IEC_TYPE;
 997         desc.high = 0;
 998
 999         /* should never fail */
1000         qi_submit_sync(&desc, iommu);
1001 }
1002
1003 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1004                       u64 type)
1005 {
1006         struct qi_desc desc;
1007
1008         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1009                         | QI_CC_GRAN(type) | QI_CC_TYPE;
1010         desc.high = 0;
1011
1012         qi_submit_sync(&desc, iommu);
1013 }
1014
1015 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1016                     unsigned int size_order, u64 type)
1017 {
1018         u8 dw = 0, dr = 0;
1019
1020         struct qi_desc desc;
1021         int ih = 0;
1022
1023         if (cap_write_drain(iommu->cap))
1024                 dw = 1;
1025
1026         if (cap_read_drain(iommu->cap))
1027                 dr = 1;
1028
1029         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1030                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1031         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1032                 | QI_IOTLB_AM(size_order);
1033
1034         qi_submit_sync(&desc, iommu);
1035 }
1036
1037 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1038                         u64 addr, unsigned mask)
1039 {
1040         struct qi_desc desc;
1041
1042         if (mask) {
1043                 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1044                 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1045                 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1046         } else
1047                 desc.high = QI_DEV_IOTLB_ADDR(addr);
1048
1049         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1050                 qdep = 0;
1051
1052         desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1053                    QI_DIOTLB_TYPE;
1054
1055         qi_submit_sync(&desc, iommu);
1056 }
1057
1058 /*
1059  * Disable Queued Invalidation interface.
1060  */
1061 void dmar_disable_qi(struct intel_iommu *iommu)
1062 {
1063         unsigned long flags;
1064         u32 sts;
1065         cycles_t start_time = get_cycles();
1066
1067         if (!ecap_qis(iommu->ecap))
1068                 return;
1069
1070         spin_lock_irqsave(&iommu->register_lock, flags);
1071
1072         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
1073         if (!(sts & DMA_GSTS_QIES))
1074                 goto end;
1075
1076         /*
1077          * Give a chance to HW to complete the pending invalidation requests.
1078          */
1079         while ((readl(iommu->reg + DMAR_IQT_REG) !=
1080                 readl(iommu->reg + DMAR_IQH_REG)) &&
1081                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1082                 cpu_relax();
1083
1084         iommu->gcmd &= ~DMA_GCMD_QIE;
1085         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1086
1087         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1088                       !(sts & DMA_GSTS_QIES), sts);
1089 end:
1090         spin_unlock_irqrestore(&iommu->register_lock, flags);
1091 }
1092
1093 /*
1094  * Enable queued invalidation.
1095  */
1096 static void __dmar_enable_qi(struct intel_iommu *iommu)
1097 {
1098         u32 sts;
1099         unsigned long flags;
1100         struct q_inval *qi = iommu->qi;
1101
1102         qi->free_head = qi->free_tail = 0;
1103         qi->free_cnt = QI_LENGTH;
1104
1105         spin_lock_irqsave(&iommu->register_lock, flags);
1106
1107         /* write zero to the tail reg */
1108         writel(0, iommu->reg + DMAR_IQT_REG);
1109
1110         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1111
1112         iommu->gcmd |= DMA_GCMD_QIE;
1113         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1114
1115         /* Make sure hardware complete it */
1116         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1117
1118         spin_unlock_irqrestore(&iommu->register_lock, flags);
1119 }
1120
1121 /*
1122  * Enable Queued Invalidation interface. This is a must to support
1123  * interrupt-remapping. Also used by DMA-remapping, which replaces
1124  * register based IOTLB invalidation.
1125  */
1126 int dmar_enable_qi(struct intel_iommu *iommu)
1127 {
1128         struct q_inval *qi;
1129         struct page *desc_page;
1130
1131         if (!ecap_qis(iommu->ecap))
1132                 return -ENOENT;
1133
1134         /*
1135          * queued invalidation is already setup and enabled.
1136          */
1137         if (iommu->qi)
1138                 return 0;
1139
1140         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1141         if (!iommu->qi)
1142                 return -ENOMEM;
1143
1144         qi = iommu->qi;
1145
1146
1147         desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1148         if (!desc_page) {
1149                 kfree(qi);
1150                 iommu->qi = 0;
1151                 return -ENOMEM;
1152         }
1153
1154         qi->desc = page_address(desc_page);
1155
1156         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1157         if (!qi->desc_status) {
1158                 free_page((unsigned long) qi->desc);
1159                 kfree(qi);
1160                 iommu->qi = 0;
1161                 return -ENOMEM;
1162         }
1163
1164         qi->free_head = qi->free_tail = 0;
1165         qi->free_cnt = QI_LENGTH;
1166
1167         spin_lock_init(&qi->q_lock);
1168
1169         __dmar_enable_qi(iommu);
1170
1171         return 0;
1172 }
1173
1174 /* iommu interrupt handling. Most stuff are MSI-like. */
1175
1176 enum faulttype {
1177         DMA_REMAP,
1178         INTR_REMAP,
1179         UNKNOWN,
1180 };
1181
1182 static const char *dma_remap_fault_reasons[] =
1183 {
1184         "Software",
1185         "Present bit in root entry is clear",
1186         "Present bit in context entry is clear",
1187         "Invalid context entry",
1188         "Access beyond MGAW",
1189         "PTE Write access is not set",
1190         "PTE Read access is not set",
1191         "Next page table ptr is invalid",
1192         "Root table address invalid",
1193         "Context table ptr is invalid",
1194         "non-zero reserved fields in RTP",
1195         "non-zero reserved fields in CTP",
1196         "non-zero reserved fields in PTE",
1197 };
1198
1199 static const char *intr_remap_fault_reasons[] =
1200 {
1201         "Detected reserved fields in the decoded interrupt-remapped request",
1202         "Interrupt index exceeded the interrupt-remapping table size",
1203         "Present field in the IRTE entry is clear",
1204         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1205         "Detected reserved fields in the IRTE entry",
1206         "Blocked a compatibility format interrupt request",
1207         "Blocked an interrupt request due to source-id verification failure",
1208 };
1209
1210 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
1211
1212 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1213 {
1214         if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1215                                      ARRAY_SIZE(intr_remap_fault_reasons))) {
1216                 *fault_type = INTR_REMAP;
1217                 return intr_remap_fault_reasons[fault_reason - 0x20];
1218         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1219                 *fault_type = DMA_REMAP;
1220                 return dma_remap_fault_reasons[fault_reason];
1221         } else {
1222                 *fault_type = UNKNOWN;
1223                 return "Unknown";
1224         }
1225 }
1226
1227 void dmar_msi_unmask(struct irq_data *data)
1228 {
1229         struct intel_iommu *iommu = irq_data_get_irq_data(data);
1230         unsigned long flag;
1231
1232         /* unmask it */
1233         spin_lock_irqsave(&iommu->register_lock, flag);
1234         writel(0, iommu->reg + DMAR_FECTL_REG);
1235         /* Read a reg to force flush the post write */
1236         readl(iommu->reg + DMAR_FECTL_REG);
1237         spin_unlock_irqrestore(&iommu->register_lock, flag);
1238 }
1239
1240 void dmar_msi_mask(struct irq_data *data)
1241 {
1242         unsigned long flag;
1243         struct intel_iommu *iommu = irq_data_get_irq_data(data);
1244
1245         /* mask it */
1246         spin_lock_irqsave(&iommu->register_lock, flag);
1247         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1248         /* Read a reg to force flush the post write */
1249         readl(iommu->reg + DMAR_FECTL_REG);
1250         spin_unlock_irqrestore(&iommu->register_lock, flag);
1251 }
1252
1253 void dmar_msi_write(int irq, struct msi_msg *msg)
1254 {
1255         struct intel_iommu *iommu = get_irq_data(irq);
1256         unsigned long flag;
1257
1258         spin_lock_irqsave(&iommu->register_lock, flag);
1259         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1260         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1261         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1262         spin_unlock_irqrestore(&iommu->register_lock, flag);
1263 }
1264
1265 void dmar_msi_read(int irq, struct msi_msg *msg)
1266 {
1267         struct intel_iommu *iommu = get_irq_data(irq);
1268         unsigned long flag;
1269
1270         spin_lock_irqsave(&iommu->register_lock, flag);
1271         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1272         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1273         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1274         spin_unlock_irqrestore(&iommu->register_lock, flag);
1275 }
1276
1277 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1278                 u8 fault_reason, u16 source_id, unsigned long long addr)
1279 {
1280         const char *reason;
1281         int fault_type;
1282
1283         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1284
1285         if (fault_type == INTR_REMAP)
1286                 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1287                        "fault index %llx\n"
1288                         "INTR-REMAP:[fault reason %02d] %s\n",
1289                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1290                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1291                         fault_reason, reason);
1292         else
1293                 printk(KERN_ERR
1294                        "DMAR:[%s] Request device [%02x:%02x.%d] "
1295                        "fault addr %llx \n"
1296                        "DMAR:[fault reason %02d] %s\n",
1297                        (type ? "DMA Read" : "DMA Write"),
1298                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1299                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1300         return 0;
1301 }
1302
1303 #define PRIMARY_FAULT_REG_LEN (16)
1304 irqreturn_t dmar_fault(int irq, void *dev_id)
1305 {
1306         struct intel_iommu *iommu = dev_id;
1307         int reg, fault_index;
1308         u32 fault_status;
1309         unsigned long flag;
1310
1311         spin_lock_irqsave(&iommu->register_lock, flag);
1312         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1313         if (fault_status)
1314                 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1315                        fault_status);
1316
1317         /* TBD: ignore advanced fault log currently */
1318         if (!(fault_status & DMA_FSTS_PPF))
1319                 goto clear_rest;
1320
1321         fault_index = dma_fsts_fault_record_index(fault_status);
1322         reg = cap_fault_reg_offset(iommu->cap);
1323         while (1) {
1324                 u8 fault_reason;
1325                 u16 source_id;
1326                 u64 guest_addr;
1327                 int type;
1328                 u32 data;
1329
1330                 /* highest 32 bits */
1331                 data = readl(iommu->reg + reg +
1332                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1333                 if (!(data & DMA_FRCD_F))
1334                         break;
1335
1336                 fault_reason = dma_frcd_fault_reason(data);
1337                 type = dma_frcd_type(data);
1338
1339                 data = readl(iommu->reg + reg +
1340                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1341                 source_id = dma_frcd_source_id(data);
1342
1343                 guest_addr = dmar_readq(iommu->reg + reg +
1344                                 fault_index * PRIMARY_FAULT_REG_LEN);
1345                 guest_addr = dma_frcd_page_addr(guest_addr);
1346                 /* clear the fault */
1347                 writel(DMA_FRCD_F, iommu->reg + reg +
1348                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1349
1350                 spin_unlock_irqrestore(&iommu->register_lock, flag);
1351
1352                 dmar_fault_do_one(iommu, type, fault_reason,
1353                                 source_id, guest_addr);
1354
1355                 fault_index++;
1356                 if (fault_index >= cap_num_fault_regs(iommu->cap))
1357                         fault_index = 0;
1358                 spin_lock_irqsave(&iommu->register_lock, flag);
1359         }
1360 clear_rest:
1361         /* clear all the other faults */
1362         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1363         writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1364
1365         spin_unlock_irqrestore(&iommu->register_lock, flag);
1366         return IRQ_HANDLED;
1367 }
1368
1369 int dmar_set_interrupt(struct intel_iommu *iommu)
1370 {
1371         int irq, ret;
1372
1373         /*
1374          * Check if the fault interrupt is already initialized.
1375          */
1376         if (iommu->irq)
1377                 return 0;
1378
1379         irq = create_irq();
1380         if (!irq) {
1381                 printk(KERN_ERR "IOMMU: no free vectors\n");
1382                 return -EINVAL;
1383         }
1384
1385         set_irq_data(irq, iommu);
1386         iommu->irq = irq;
1387
1388         ret = arch_setup_dmar_msi(irq);
1389         if (ret) {
1390                 set_irq_data(irq, NULL);
1391                 iommu->irq = 0;
1392                 destroy_irq(irq);
1393                 return ret;
1394         }
1395
1396         ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1397         if (ret)
1398                 printk(KERN_ERR "IOMMU: can't request irq\n");
1399         return ret;
1400 }
1401
1402 int __init enable_drhd_fault_handling(void)
1403 {
1404         struct dmar_drhd_unit *drhd;
1405
1406         /*
1407          * Enable fault control interrupt.
1408          */
1409         for_each_drhd_unit(drhd) {
1410                 int ret;
1411                 struct intel_iommu *iommu = drhd->iommu;
1412                 ret = dmar_set_interrupt(iommu);
1413
1414                 if (ret) {
1415                         printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1416                                " interrupt, ret %d\n",
1417                                (unsigned long long)drhd->reg_base_addr, ret);
1418                         return -1;
1419                 }
1420         }
1421
1422         return 0;
1423 }
1424
1425 /*
1426  * Re-enable Queued Invalidation interface.
1427  */
1428 int dmar_reenable_qi(struct intel_iommu *iommu)
1429 {
1430         if (!ecap_qis(iommu->ecap))
1431                 return -ENOENT;
1432
1433         if (!iommu->qi)
1434                 return -ENOENT;
1435
1436         /*
1437          * First disable queued invalidation.
1438          */
1439         dmar_disable_qi(iommu);
1440         /*
1441          * Then enable queued invalidation again. Since there is no pending
1442          * invalidation requests now, it's safe to re-enable queued
1443          * invalidation.
1444          */
1445         __dmar_enable_qi(iommu);
1446
1447         return 0;
1448 }
1449
1450 /*
1451  * Check interrupt remapping support in DMAR table description.
1452  */
1453 int __init dmar_ir_support(void)
1454 {
1455         struct acpi_table_dmar *dmar;
1456         dmar = (struct acpi_table_dmar *)dmar_tbl;
1457         if (!dmar)
1458                 return 0;
1459         return dmar->flags & 0x1;
1460 }
1461 IOMMU_INIT_POST(detect_intel_iommu);