drivers/pci/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #include <linux/pci.h>
  30 #include <linux/dmar.h>
  31 #include <linux/iova.h>
  32 #include <linux/intel-iommu.h>
  33 #include <linux/timer.h>
  34 #include <linux/irq.h>
  35 #include <linux/interrupt.h>
  36 #include <linux/tboot.h>
  37 #include <linux/dmi.h>
  38 #include <linux/slab.h>
  39
  40 #define PREFIX "DMAR: "
  41
  42 /* No locks are needed as DMA remapping hardware unit
  43  * list is constructed at boot time and hotplug of
  44  * these units are not supported by the architecture.
  45  */
  46 LIST_HEAD(dmar_drhd_units);
  47
  48 static struct acpi_table_header * __initdata dmar_tbl;
  49 static acpi_size dmar_tbl_size;
  50
  51 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  52 {
  53         /*
  54          * add INCLUDE_ALL at the tail, so scan the list will find it at
  55          * the very end.
  56          */
  57         if (drhd->include_all)
  58                 list_add_tail(&drhd->list, &dmar_drhd_units);
  59         else
  60                 list_add(&drhd->list, &dmar_drhd_units);
  61 }
  62
  63 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  64                                            struct pci_dev **dev, u16 segment)
  65 {
  66         struct pci_bus *bus;
  67         struct pci_dev *pdev = NULL;
  68         struct acpi_dmar_pci_path *path;
  69         int count;
  70
  71         bus = pci_find_bus(segment, scope->bus);
  72         path = (struct acpi_dmar_pci_path *)(scope + 1);
  73         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  74                 / sizeof(struct acpi_dmar_pci_path);
  75
  76         while (count) {
  77                 if (pdev)
  78                         pci_dev_put(pdev);
  79                 /*
  80                  * Some BIOSes list non-exist devices in DMAR table, just
  81                  * ignore it
  82                  */
  83                 if (!bus) {
  84                         printk(KERN_WARNING
  85                         PREFIX "Device scope bus [%d] not found\n",
  86                         scope->bus);
  87                         break;
  88                 }
  89                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  90                 if (!pdev) {
  91                         printk(KERN_WARNING PREFIX
  92                         "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  93                                 segment, bus->number, path->dev, path->fn);
  94                         break;
  95                 }
  96                 path ++;
  97                 count --;
  98                 bus = pdev->subordinate;
  99         }
 100         if (!pdev) {
 101                 printk(KERN_WARNING PREFIX
 102                 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
 103                 segment, scope->bus, path->dev, path->fn);
 104                 *dev = NULL;
 105                 return 0;
 106         }
 107         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 108                         pdev->subordinate) || (scope->entry_type == \
 109                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 110                 pci_dev_put(pdev);
 111                 printk(KERN_WARNING PREFIX
 112                         "Device scope type does not match for %s\n",
 113                          pci_name(pdev));
 114                 return -EINVAL;
 115         }
 116         *dev = pdev;
 117         return 0;
 118 }
 119
 120 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 121                                        struct pci_dev ***devices, u16 segment)
 122 {
 123         struct acpi_dmar_device_scope *scope;
 124         void * tmp = start;
 125         int index;
 126         int ret;
 127
 128         *cnt = 0;
 129         while (start < end) {
 130                 scope = start;
 131                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 132                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 133                         (*cnt)++;
 134                 else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
 135                         printk(KERN_WARNING PREFIX
 136                                "Unsupported device scope\n");
 137                 }
 138                 start += scope->length;
 139         }
 140         if (*cnt == 0)
 141                 return 0;
 142
 143         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 144         if (!*devices)
 145                 return -ENOMEM;
 146
 147         start = tmp;
 148         index = 0;
 149         while (start < end) {
 150                 scope = start;
 151                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 152                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 153                         ret = dmar_parse_one_dev_scope(scope,
 154                                 &(*devices)[index], segment);
 155                         if (ret) {
 156                                 kfree(*devices);
 157                                 return ret;
 158                         }
 159                         index ++;
 160                 }
 161                 start += scope->length;
 162         }
 163
 164         return 0;
 165 }
 166
 167 /**
 168  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 169  * structure which uniquely represent one DMA remapping hardware unit
 170  * present in the platform
 171  */
 172 static int __init
 173 dmar_parse_one_drhd(struct acpi_dmar_header *header)
 174 {
 175         struct acpi_dmar_hardware_unit *drhd;
 176         struct dmar_drhd_unit *dmaru;
 177         int ret = 0;
 178
 179         drhd = (struct acpi_dmar_hardware_unit *)header;
 180         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 181         if (!dmaru)
 182                 return -ENOMEM;
 183
 184         dmaru->hdr = header;
 185         dmaru->reg_base_addr = drhd->address;
 186         dmaru->segment = drhd->segment;
 187         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 188
 189         ret = alloc_iommu(dmaru);
 190         if (ret) {
 191                 kfree(dmaru);
 192                 return ret;
 193         }
 194         dmar_register_drhd_unit(dmaru);
 195         return 0;
 196 }
 197
 198 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 199 {
 200         struct acpi_dmar_hardware_unit *drhd;
 201         int ret = 0;
 202
 203         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 204
 205         if (dmaru->include_all)
 206                 return 0;
 207
 208         ret = dmar_parse_dev_scope((void *)(drhd + 1),
 209                                 ((void *)drhd) + drhd->header.length,
 210                                 &dmaru->devices_cnt, &dmaru->devices,
 211                                 drhd->segment);
 212         if (ret) {
 213                 list_del(&dmaru->list);
 214                 kfree(dmaru);
 215         }
 216         return ret;
 217 }
 218
 219 #ifdef CONFIG_DMAR
 220 LIST_HEAD(dmar_rmrr_units);
 221
 222 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
 223 {
 224         list_add(&rmrr->list, &dmar_rmrr_units);
 225 }
 226
 227
 228 static int __init
 229 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
 230 {
 231         struct acpi_dmar_reserved_memory *rmrr;
 232         struct dmar_rmrr_unit *rmrru;
 233
 234         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
 235         if (!rmrru)
 236                 return -ENOMEM;
 237
 238         rmrru->hdr = header;
 239         rmrr = (struct acpi_dmar_reserved_memory *)header;
 240         rmrru->base_address = rmrr->base_address;
 241         rmrru->end_address = rmrr->end_address;
 242
 243         dmar_register_rmrr_unit(rmrru);
 244         return 0;
 245 }
 246
 247 static int __init
 248 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
 249 {
 250         struct acpi_dmar_reserved_memory *rmrr;
 251         int ret;
 252
 253         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
 254         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
 255                 ((void *)rmrr) + rmrr->header.length,
 256                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
 257
 258         if (ret || (rmrru->devices_cnt == 0)) {
 259                 list_del(&rmrru->list);
 260                 kfree(rmrru);
 261         }
 262         return ret;
 263 }
 264
 265 static LIST_HEAD(dmar_atsr_units);
 266
 267 static int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr)
 268 {
 269         struct acpi_dmar_atsr *atsr;
 270         struct dmar_atsr_unit *atsru;
 271
 272         atsr = container_of(hdr, struct acpi_dmar_atsr, header);
 273         atsru = kzalloc(sizeof(*atsru), GFP_KERNEL);
 274         if (!atsru)
 275                 return -ENOMEM;
 276
 277         atsru->hdr = hdr;
 278         atsru->include_all = atsr->flags & 0x1;
 279
 280         list_add(&atsru->list, &dmar_atsr_units);
 281
 282         return 0;
 283 }
 284
 285 static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru)
 286 {
 287         int rc;
 288         struct acpi_dmar_atsr *atsr;
 289
 290         if (atsru->include_all)
 291                 return 0;
 292
 293         atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
 294         rc = dmar_parse_dev_scope((void *)(atsr + 1),
 295                                 (void *)atsr + atsr->header.length,
 296                                 &atsru->devices_cnt, &atsru->devices,
 297                                 atsr->segment);
 298         if (rc || !atsru->devices_cnt) {
 299                 list_del(&atsru->list);
 300                 kfree(atsru);
 301         }
 302
 303         return rc;
 304 }
 305
 306 int dmar_find_matched_atsr_unit(struct pci_dev *dev)
 307 {
 308         int i;
 309         struct pci_bus *bus;
 310         struct acpi_dmar_atsr *atsr;
 311         struct dmar_atsr_unit *atsru;
 312
 313         dev = pci_physfn(dev);
 314
 315         list_for_each_entry(atsru, &dmar_atsr_units, list) {
 316                 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
 317                 if (atsr->segment == pci_domain_nr(dev->bus))
 318                         goto found;
 319         }
 320
 321         return 0;
 322
 323 found:
 324         for (bus = dev->bus; bus; bus = bus->parent) {
 325                 struct pci_dev *bridge = bus->self;
 326
 327                 if (!bridge || !pci_is_pcie(bridge) ||
 328                     bridge->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
 329                         return 0;
 330
 331                 if (bridge->pcie_type == PCI_EXP_TYPE_ROOT_PORT) {
 332                         for (i = 0; i < atsru->devices_cnt; i++)
 333                                 if (atsru->devices[i] == bridge)
 334                                         return 1;
 335                         break;
 336                 }
 337         }
 338
 339         if (atsru->include_all)
 340                 return 1;
 341
 342         return 0;
 343 }
 344 #endif
 345
 346 #ifdef CONFIG_ACPI_NUMA
 347 static int __init
 348 dmar_parse_one_rhsa(struct acpi_dmar_header *header)
 349 {
 350         struct acpi_dmar_rhsa *rhsa;
 351         struct dmar_drhd_unit *drhd;
 352
 353         rhsa = (struct acpi_dmar_rhsa *)header;
 354         for_each_drhd_unit(drhd) {
 355                 if (drhd->reg_base_addr == rhsa->base_address) {
 356                         int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 357
 358                         if (!node_online(node))
 359                                 node = -1;
 360                         drhd->iommu->node = node;
 361                         return 0;
 362                 }
 363         }
 364         WARN_TAINT(
 365                 1, TAINT_FIRMWARE_WORKAROUND,
 366                 "Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 367                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 368                 drhd->reg_base_addr,
 369                 dmi_get_system_info(DMI_BIOS_VENDOR),
 370                 dmi_get_system_info(DMI_BIOS_VERSION),
 371                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 372
 373         return 0;
 374 }
 375 #endif
 376
 377 static void __init
 378 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 379 {
 380         struct acpi_dmar_hardware_unit *drhd;
 381         struct acpi_dmar_reserved_memory *rmrr;
 382         struct acpi_dmar_atsr *atsr;
 383         struct acpi_dmar_rhsa *rhsa;
 384
 385         switch (header->type) {
 386         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 387                 drhd = container_of(header, struct acpi_dmar_hardware_unit,
 388                                     header);
 389                 printk (KERN_INFO PREFIX
 390                         "DRHD base: %#016Lx flags: %#x\n",
 391                         (unsigned long long)drhd->address, drhd->flags);
 392                 break;
 393         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 394                 rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 395                                     header);
 396                 printk (KERN_INFO PREFIX
 397                         "RMRR base: %#016Lx end: %#016Lx\n",
 398                         (unsigned long long)rmrr->base_address,
 399                         (unsigned long long)rmrr->end_address);
 400                 break;
 401         case ACPI_DMAR_TYPE_ATSR:
 402                 atsr = container_of(header, struct acpi_dmar_atsr, header);
 403                 printk(KERN_INFO PREFIX "ATSR flags: %#x\n", atsr->flags);
 404                 break;
 405         case ACPI_DMAR_HARDWARE_AFFINITY:
 406                 rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 407                 printk(KERN_INFO PREFIX "RHSA base: %#016Lx proximity domain: %#x\n",
 408                        (unsigned long long)rhsa->base_address,
 409                        rhsa->proximity_domain);
 410                 break;
 411         }
 412 }
 413
 414 /**
 415  * dmar_table_detect - checks to see if the platform supports DMAR devices
 416  */
 417 static int __init dmar_table_detect(void)
 418 {
 419         acpi_status status = AE_OK;
 420
 421         /* if we could find DMAR table, then there are DMAR devices */
 422         status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 423                                 (struct acpi_table_header **)&dmar_tbl,
 424                                 &dmar_tbl_size);
 425
 426         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 427                 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
 428                 status = AE_NOT_FOUND;
 429         }
 430
 431         return (ACPI_SUCCESS(status) ? 1 : 0);
 432 }
 433
 434 /**
 435  * parse_dmar_table - parses the DMA reporting table
 436  */
 437 static int __init
 438 parse_dmar_table(void)
 439 {
 440         struct acpi_table_dmar *dmar;
 441         struct acpi_dmar_header *entry_header;
 442         int ret = 0;
 443
 444         /*
 445          * Do it again, earlier dmar_tbl mapping could be mapped with
 446          * fixed map.
 447          */
 448         dmar_table_detect();
 449
 450         /*
 451          * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 452          * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 453          */
 454         dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 455
 456         dmar = (struct acpi_table_dmar *)dmar_tbl;
 457         if (!dmar)
 458                 return -ENODEV;
 459
 460         if (dmar->width < PAGE_SHIFT - 1) {
 461                 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
 462                 return -EINVAL;
 463         }
 464
 465         printk (KERN_INFO PREFIX "Host address width %d\n",
 466                 dmar->width + 1);
 467
 468         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 469         while (((unsigned long)entry_header) <
 470                         (((unsigned long)dmar) + dmar_tbl->length)) {
 471                 /* Avoid looping forever on bad ACPI tables */
 472                 if (entry_header->length == 0) {
 473                         printk(KERN_WARNING PREFIX
 474                                 "Invalid 0-length structure\n");
 475                         ret = -EINVAL;
 476                         break;
 477                 }
 478
 479                 dmar_table_print_dmar_entry(entry_header);
 480
 481                 switch (entry_header->type) {
 482                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 483                         ret = dmar_parse_one_drhd(entry_header);
 484                         break;
 485                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 486 #ifdef CONFIG_DMAR
 487                         ret = dmar_parse_one_rmrr(entry_header);
 488 #endif
 489                         break;
 490                 case ACPI_DMAR_TYPE_ATSR:
 491 #ifdef CONFIG_DMAR
 492                         ret = dmar_parse_one_atsr(entry_header);
 493 #endif
 494                         break;
 495                 case ACPI_DMAR_HARDWARE_AFFINITY:
 496 #ifdef CONFIG_ACPI_NUMA
 497                         ret = dmar_parse_one_rhsa(entry_header);
 498 #endif
 499                         break;
 500                 default:
 501                         printk(KERN_WARNING PREFIX
 502                                 "Unknown DMAR structure type %d\n",
 503                                 entry_header->type);
 504                         ret = 0; /* for forward compatibility */
 505                         break;
 506                 }
 507                 if (ret)
 508                         break;
 509
 510                 entry_header = ((void *)entry_header + entry_header->length);
 511         }
 512         return ret;
 513 }
 514
 515 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 516                           struct pci_dev *dev)
 517 {
 518         int index;
 519
 520         while (dev) {
 521                 for (index = 0; index < cnt; index++)
 522                         if (dev == devices[index])
 523                                 return 1;
 524
 525                 /* Check our parent */
 526                 dev = dev->bus->self;
 527         }
 528
 529         return 0;
 530 }
 531
 532 struct dmar_drhd_unit *
 533 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 534 {
 535         struct dmar_drhd_unit *dmaru = NULL;
 536         struct acpi_dmar_hardware_unit *drhd;
 537
 538         dev = pci_physfn(dev);
 539
 540         list_for_each_entry(dmaru, &dmar_drhd_units, list) {
 541                 drhd = container_of(dmaru->hdr,
 542                                     struct acpi_dmar_hardware_unit,
 543                                     header);
 544
 545                 if (dmaru->include_all &&
 546                     drhd->segment == pci_domain_nr(dev->bus))
 547                         return dmaru;
 548
 549                 if (dmar_pci_device_match(dmaru->devices,
 550                                           dmaru->devices_cnt, dev))
 551                         return dmaru;
 552         }
 553
 554         return NULL;
 555 }
 556
 557 int __init dmar_dev_scope_init(void)
 558 {
 559         struct dmar_drhd_unit *drhd, *drhd_n;
 560         int ret = -ENODEV;
 561
 562         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 563                 ret = dmar_parse_dev(drhd);
 564                 if (ret)
 565                         return ret;
 566         }
 567
 568 #ifdef CONFIG_DMAR
 569         {
 570                 struct dmar_rmrr_unit *rmrr, *rmrr_n;
 571                 struct dmar_atsr_unit *atsr, *atsr_n;
 572
 573                 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
 574                         ret = rmrr_parse_dev(rmrr);
 575                         if (ret)
 576                                 return ret;
 577                 }
 578
 579                 list_for_each_entry_safe(atsr, atsr_n, &dmar_atsr_units, list) {
 580                         ret = atsr_parse_dev(atsr);
 581                         if (ret)
 582                                 return ret;
 583                 }
 584         }
 585 #endif
 586
 587         return ret;
 588 }
 589
 590
 591 int __init dmar_table_init(void)
 592 {
 593         static int dmar_table_initialized;
 594         int ret;
 595
 596         if (dmar_table_initialized)
 597                 return 0;
 598
 599         dmar_table_initialized = 1;
 600
 601         ret = parse_dmar_table();
 602         if (ret) {
 603                 if (ret != -ENODEV)
 604                         printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
 605                 return ret;
 606         }
 607
 608         if (list_empty(&dmar_drhd_units)) {
 609                 printk(KERN_INFO PREFIX "No DMAR devices found\n");
 610                 return -ENODEV;
 611         }
 612
 613 #ifdef CONFIG_DMAR
 614         if (list_empty(&dmar_rmrr_units))
 615                 printk(KERN_INFO PREFIX "No RMRR found\n");
 616
 617         if (list_empty(&dmar_atsr_units))
 618                 printk(KERN_INFO PREFIX "No ATSR found\n");
 619 #endif
 620
 621         return 0;
 622 }
 623
 624 static void warn_invalid_dmar(u64 addr, const char *message)
 625 {
 626         WARN_TAINT_ONCE(
 627                 1, TAINT_FIRMWARE_WORKAROUND,
 628                 "Your BIOS is broken; DMAR reported at address %llx%s!\n"
 629                 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 630                 addr, message,
 631                 dmi_get_system_info(DMI_BIOS_VENDOR),
 632                 dmi_get_system_info(DMI_BIOS_VERSION),
 633                 dmi_get_system_info(DMI_PRODUCT_VERSION));
 634 }
 635
 636 int __init check_zero_address(void)
 637 {
 638         struct acpi_table_dmar *dmar;
 639         struct acpi_dmar_header *entry_header;
 640         struct acpi_dmar_hardware_unit *drhd;
 641
 642         dmar = (struct acpi_table_dmar *)dmar_tbl;
 643         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 644
 645         while (((unsigned long)entry_header) <
 646                         (((unsigned long)dmar) + dmar_tbl->length)) {
 647                 /* Avoid looping forever on bad ACPI tables */
 648                 if (entry_header->length == 0) {
 649                         printk(KERN_WARNING PREFIX
 650                                 "Invalid 0-length structure\n");
 651                         return 0;
 652                 }
 653
 654                 if (entry_header->type == ACPI_DMAR_TYPE_HARDWARE_UNIT) {
 655                         void __iomem *addr;
 656                         u64 cap, ecap;
 657
 658                         drhd = (void *)entry_header;
 659                         if (!drhd->address) {
 660                                 warn_invalid_dmar(0, "");
 661                                 goto failed;
 662                         }
 663
 664                         addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 665                         if (!addr ) {
 666                                 printk("IOMMU: can't validate: %llx\n", drhd->address);
 667                                 goto failed;
 668                         }
 669                         cap = dmar_readq(addr + DMAR_CAP_REG);
 670                         ecap = dmar_readq(addr + DMAR_ECAP_REG);
 671                         early_iounmap(addr, VTD_PAGE_SIZE);
 672                         if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 673                                 warn_invalid_dmar(drhd->address,
 674                                                   " returns all ones");
 675                                 goto failed;
 676                         }
 677                 }
 678
 679                 entry_header = ((void *)entry_header + entry_header->length);
 680         }
 681         return 1;
 682
 683 failed:
 684 #ifdef CONFIG_DMAR
 685         dmar_disabled = 1;
 686 #endif
 687         return 0;
 688 }
 689
 690 int __init detect_intel_iommu(void)
 691 {
 692         int ret;
 693
 694         ret = dmar_table_detect();
 695         if (ret)
 696                 ret = check_zero_address();
 697         {
 698 #ifdef CONFIG_INTR_REMAP
 699                 struct acpi_table_dmar *dmar;
 700                 /*
 701                  * for now we will disable dma-remapping when interrupt
 702                  * remapping is enabled.
 703                  * When support for queued invalidation for IOTLB invalidation
 704                  * is added, we will not need this any more.
 705                  */
 706                 dmar = (struct acpi_table_dmar *) dmar_tbl;
 707                 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
 708                         printk(KERN_INFO
 709                                "Queued invalidation will be enabled to support "
 710                                "x2apic and Intr-remapping.\n");
 711 #endif
 712 #ifdef CONFIG_DMAR
 713                 if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 714                         iommu_detected = 1;
 715                         /* Make sure ACS will be enabled */
 716                         pci_request_acs();
 717                 }
 718 #endif
 719 #ifdef CONFIG_X86
 720                 if (ret)
 721                         x86_init.iommu.iommu_init = intel_iommu_init;
 722 #endif
 723         }
 724         early_acpi_os_unmap_memory(dmar_tbl, dmar_tbl_size);
 725         dmar_tbl = NULL;
 726
 727         return (ret ? 1 : -ENODEV);
 728 }
 729
 730
 731 int alloc_iommu(struct dmar_drhd_unit *drhd)
 732 {
 733         struct intel_iommu *iommu;
 734         int map_size;
 735         u32 ver;
 736         static int iommu_allocated = 0;
 737         int agaw = 0;
 738         int msagaw = 0;
 739
 740         if (!drhd->reg_base_addr) {
 741                 warn_invalid_dmar(0, "");
 742                 return -EINVAL;
 743         }
 744
 745         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 746         if (!iommu)
 747                 return -ENOMEM;
 748
 749         iommu->seq_id = iommu_allocated++;
 750         sprintf (iommu->name, "dmar%d", iommu->seq_id);
 751
 752         iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
 753         if (!iommu->reg) {
 754                 printk(KERN_ERR "IOMMU: can't map the region\n");
 755                 goto error;
 756         }
 757         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 758         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 759
 760         if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 761                 warn_invalid_dmar(drhd->reg_base_addr, " returns all ones");
 762                 goto err_unmap;
 763         }
 764
 765 #ifdef CONFIG_DMAR
 766         agaw = iommu_calculate_agaw(iommu);
 767         if (agaw < 0) {
 768                 printk(KERN_ERR
 769                        "Cannot get a valid agaw for iommu (seq_id = %d)\n",
 770                        iommu->seq_id);
 771                 goto err_unmap;
 772         }
 773         msagaw = iommu_calculate_max_sagaw(iommu);
 774         if (msagaw < 0) {
 775                 printk(KERN_ERR
 776                         "Cannot get a valid max agaw for iommu (seq_id = %d)\n",
 777                         iommu->seq_id);
 778                 goto err_unmap;
 779         }
 780 #endif
 781         iommu->agaw = agaw;
 782         iommu->msagaw = msagaw;
 783
 784         iommu->node = -1;
 785
 786         /* the registers might be more than one page */
 787         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 788                 cap_max_fault_reg_offset(iommu->cap));
 789         map_size = VTD_PAGE_ALIGN(map_size);
 790         if (map_size > VTD_PAGE_SIZE) {
 791                 iounmap(iommu->reg);
 792                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
 793                 if (!iommu->reg) {
 794                         printk(KERN_ERR "IOMMU: can't map the region\n");
 795                         goto error;
 796                 }
 797         }
 798
 799         ver = readl(iommu->reg + DMAR_VER_REG);
 800         pr_info("IOMMU %d: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
 801                 iommu->seq_id,
 802                 (unsigned long long)drhd->reg_base_addr,
 803                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 804                 (unsigned long long)iommu->cap,
 805                 (unsigned long long)iommu->ecap);
 806
 807         spin_lock_init(&iommu->register_lock);
 808
 809         drhd->iommu = iommu;
 810         return 0;
 811
 812  err_unmap:
 813         iounmap(iommu->reg);
 814  error:
 815         kfree(iommu);
 816         return -1;
 817 }
 818
 819 void free_iommu(struct intel_iommu *iommu)
 820 {
 821         if (!iommu)
 822                 return;
 823
 824 #ifdef CONFIG_DMAR
 825         free_dmar_iommu(iommu);
 826 #endif
 827
 828         if (iommu->reg)
 829                 iounmap(iommu->reg);
 830         kfree(iommu);
 831 }
 832
 833 /*
 834  * Reclaim all the submitted descriptors which have completed its work.
 835  */
 836 static inline void reclaim_free_desc(struct q_inval *qi)
 837 {
 838         while (qi->desc_status[qi->free_tail] == QI_DONE ||
 839                qi->desc_status[qi->free_tail] == QI_ABORT) {
 840                 qi->desc_status[qi->free_tail] = QI_FREE;
 841                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 842                 qi->free_cnt++;
 843         }
 844 }
 845
 846 static int qi_check_fault(struct intel_iommu *iommu, int index)
 847 {
 848         u32 fault;
 849         int head, tail;
 850         struct q_inval *qi = iommu->qi;
 851         int wait_index = (index + 1) % QI_LENGTH;
 852
 853         if (qi->desc_status[wait_index] == QI_ABORT)
 854                 return -EAGAIN;
 855
 856         fault = readl(iommu->reg + DMAR_FSTS_REG);
 857
 858         /*
 859          * If IQE happens, the head points to the descriptor associated
 860          * with the error. No new descriptors are fetched until the IQE
 861          * is cleared.
 862          */
 863         if (fault & DMA_FSTS_IQE) {
 864                 head = readl(iommu->reg + DMAR_IQH_REG);
 865                 if ((head >> DMAR_IQ_SHIFT) == index) {
 866                         printk(KERN_ERR "VT-d detected invalid descriptor: "
 867                                 "low=%llx, high=%llx\n",
 868                                 (unsigned long long)qi->desc[index].low,
 869                                 (unsigned long long)qi->desc[index].high);
 870                         memcpy(&qi->desc[index], &qi->desc[wait_index],
 871                                         sizeof(struct qi_desc));
 872                         __iommu_flush_cache(iommu, &qi->desc[index],
 873                                         sizeof(struct qi_desc));
 874                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 875                         return -EINVAL;
 876                 }
 877         }
 878
 879         /*
 880          * If ITE happens, all pending wait_desc commands are aborted.
 881          * No new descriptors are fetched until the ITE is cleared.
 882          */
 883         if (fault & DMA_FSTS_ITE) {
 884                 head = readl(iommu->reg + DMAR_IQH_REG);
 885                 head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 886                 head |= 1;
 887                 tail = readl(iommu->reg + DMAR_IQT_REG);
 888                 tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
 889
 890                 writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
 891
 892                 do {
 893                         if (qi->desc_status[head] == QI_IN_USE)
 894                                 qi->desc_status[head] = QI_ABORT;
 895                         head = (head - 2 + QI_LENGTH) % QI_LENGTH;
 896                 } while (head != tail);
 897
 898                 if (qi->desc_status[wait_index] == QI_ABORT)
 899                         return -EAGAIN;
 900         }
 901
 902         if (fault & DMA_FSTS_ICE)
 903                 writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
 904
 905         return 0;
 906 }
 907
 908 /*
 909  * Submit the queued invalidation descriptor to the remapping
 910  * hardware unit and wait for its completion.
 911  */
 912 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 913 {
 914         int rc;
 915         struct q_inval *qi = iommu->qi;
 916         struct qi_desc *hw, wait_desc;
 917         int wait_index, index;
 918         unsigned long flags;
 919
 920         if (!qi)
 921                 return 0;
 922
 923         hw = qi->desc;
 924
 925 restart:
 926         rc = 0;
 927
 928         spin_lock_irqsave(&qi->q_lock, flags);
 929         while (qi->free_cnt < 3) {
 930                 spin_unlock_irqrestore(&qi->q_lock, flags);
 931                 cpu_relax();
 932                 spin_lock_irqsave(&qi->q_lock, flags);
 933         }
 934
 935         index = qi->free_head;
 936         wait_index = (index + 1) % QI_LENGTH;
 937
 938         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 939
 940         hw[index] = *desc;
 941
 942         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 943                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 944         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 945
 946         hw[wait_index] = wait_desc;
 947
 948         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 949         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 950
 951         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 952         qi->free_cnt -= 2;
 953
 954         /*
 955          * update the HW tail register indicating the presence of
 956          * new descriptors.
 957          */
 958         writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
 959
 960         while (qi->desc_status[wait_index] != QI_DONE) {
 961                 /*
 962                  * We will leave the interrupts disabled, to prevent interrupt
 963                  * context to queue another cmd while a cmd is already submitted
 964                  * and waiting for completion on this cpu. This is to avoid
 965                  * a deadlock where the interrupt context can wait indefinitely
 966                  * for free slots in the queue.
 967                  */
 968                 rc = qi_check_fault(iommu, index);
 969                 if (rc)
 970                         break;
 971
 972                 spin_unlock(&qi->q_lock);
 973                 cpu_relax();
 974                 spin_lock(&qi->q_lock);
 975         }
 976
 977         qi->desc_status[index] = QI_DONE;
 978
 979         reclaim_free_desc(qi);
 980         spin_unlock_irqrestore(&qi->q_lock, flags);
 981
 982         if (rc == -EAGAIN)
 983                 goto restart;
 984
 985         return rc;
 986 }
 987
 988 /*
 989  * Flush the global interrupt entry cache.
 990  */
 991 void qi_global_iec(struct intel_iommu *iommu)
 992 {
 993         struct qi_desc desc;
 994
 995         desc.low = QI_IEC_TYPE;
 996         desc.high = 0;
 997
 998         /* should never fail */
 999         qi_submit_sync(&desc, iommu);
1000 }
1001
1002 void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1003                       u64 type)
1004 {
1005         struct qi_desc desc;
1006
1007         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1008                         | QI_CC_GRAN(type) | QI_CC_TYPE;
1009         desc.high = 0;
1010
1011         qi_submit_sync(&desc, iommu);
1012 }
1013
1014 void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1015                     unsigned int size_order, u64 type)
1016 {
1017         u8 dw = 0, dr = 0;
1018
1019         struct qi_desc desc;
1020         int ih = 0;
1021
1022         if (cap_write_drain(iommu->cap))
1023                 dw = 1;
1024
1025         if (cap_read_drain(iommu->cap))
1026                 dr = 1;
1027
1028         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1029                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1030         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1031                 | QI_IOTLB_AM(size_order);
1032
1033         qi_submit_sync(&desc, iommu);
1034 }
1035
1036 void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1037                         u64 addr, unsigned mask)
1038 {
1039         struct qi_desc desc;
1040
1041         if (mask) {
1042                 BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1043                 addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1044                 desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1045         } else
1046                 desc.high = QI_DEV_IOTLB_ADDR(addr);
1047
1048         if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1049                 qdep = 0;
1050
1051         desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1052                    QI_DIOTLB_TYPE;
1053
1054         qi_submit_sync(&desc, iommu);
1055 }
1056
1057 /*
1058  * Disable Queued Invalidation interface.
1059  */
1060 void dmar_disable_qi(struct intel_iommu *iommu)
1061 {
1062         unsigned long flags;
1063         u32 sts;
1064         cycles_t start_time = get_cycles();
1065
1066         if (!ecap_qis(iommu->ecap))
1067                 return;
1068
1069         spin_lock_irqsave(&iommu->register_lock, flags);
1070
1071         sts =  dmar_readq(iommu->reg + DMAR_GSTS_REG);
1072         if (!(sts & DMA_GSTS_QIES))
1073                 goto end;
1074
1075         /*
1076          * Give a chance to HW to complete the pending invalidation requests.
1077          */
1078         while ((readl(iommu->reg + DMAR_IQT_REG) !=
1079                 readl(iommu->reg + DMAR_IQH_REG)) &&
1080                 (DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1081                 cpu_relax();
1082
1083         iommu->gcmd &= ~DMA_GCMD_QIE;
1084         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1085
1086         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1087                       !(sts & DMA_GSTS_QIES), sts);
1088 end:
1089         spin_unlock_irqrestore(&iommu->register_lock, flags);
1090 }
1091
1092 /*
1093  * Enable queued invalidation.
1094  */
1095 static void __dmar_enable_qi(struct intel_iommu *iommu)
1096 {
1097         u32 sts;
1098         unsigned long flags;
1099         struct q_inval *qi = iommu->qi;
1100
1101         qi->free_head = qi->free_tail = 0;
1102         qi->free_cnt = QI_LENGTH;
1103
1104         spin_lock_irqsave(&iommu->register_lock, flags);
1105
1106         /* write zero to the tail reg */
1107         writel(0, iommu->reg + DMAR_IQT_REG);
1108
1109         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1110
1111         iommu->gcmd |= DMA_GCMD_QIE;
1112         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1113
1114         /* Make sure hardware complete it */
1115         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1116
1117         spin_unlock_irqrestore(&iommu->register_lock, flags);
1118 }
1119
1120 /*
1121  * Enable Queued Invalidation interface. This is a must to support
1122  * interrupt-remapping. Also used by DMA-remapping, which replaces
1123  * register based IOTLB invalidation.
1124  */
1125 int dmar_enable_qi(struct intel_iommu *iommu)
1126 {
1127         struct q_inval *qi;
1128         struct page *desc_page;
1129
1130         if (!ecap_qis(iommu->ecap))
1131                 return -ENOENT;
1132
1133         /*
1134          * queued invalidation is already setup and enabled.
1135          */
1136         if (iommu->qi)
1137                 return 0;
1138
1139         iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1140         if (!iommu->qi)
1141                 return -ENOMEM;
1142
1143         qi = iommu->qi;
1144
1145
1146         desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1147         if (!desc_page) {
1148                 kfree(qi);
1149                 iommu->qi = 0;
1150                 return -ENOMEM;
1151         }
1152
1153         qi->desc = page_address(desc_page);
1154
1155         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1156         if (!qi->desc_status) {
1157                 free_page((unsigned long) qi->desc);
1158                 kfree(qi);
1159                 iommu->qi = 0;
1160                 return -ENOMEM;
1161         }
1162
1163         qi->free_head = qi->free_tail = 0;
1164         qi->free_cnt = QI_LENGTH;
1165
1166         spin_lock_init(&qi->q_lock);
1167
1168         __dmar_enable_qi(iommu);
1169
1170         return 0;
1171 }
1172
1173 /* iommu interrupt handling. Most stuff are MSI-like. */
1174
1175 enum faulttype {
1176         DMA_REMAP,
1177         INTR_REMAP,
1178         UNKNOWN,
1179 };
1180
1181 static const char *dma_remap_fault_reasons[] =
1182 {
1183         "Software",
1184         "Present bit in root entry is clear",
1185         "Present bit in context entry is clear",
1186         "Invalid context entry",
1187         "Access beyond MGAW",
1188         "PTE Write access is not set",
1189         "PTE Read access is not set",
1190         "Next page table ptr is invalid",
1191         "Root table address invalid",
1192         "Context table ptr is invalid",
1193         "non-zero reserved fields in RTP",
1194         "non-zero reserved fields in CTP",
1195         "non-zero reserved fields in PTE",
1196 };
1197
1198 static const char *intr_remap_fault_reasons[] =
1199 {
1200         "Detected reserved fields in the decoded interrupt-remapped request",
1201         "Interrupt index exceeded the interrupt-remapping table size",
1202         "Present field in the IRTE entry is clear",
1203         "Error accessing interrupt-remapping table pointed by IRTA_REG",
1204         "Detected reserved fields in the IRTE entry",
1205         "Blocked a compatibility format interrupt request",
1206         "Blocked an interrupt request due to source-id verification failure",
1207 };
1208
1209 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
1210
1211 const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1212 {
1213         if (fault_reason >= 0x20 && (fault_reason <= 0x20 +
1214                                      ARRAY_SIZE(intr_remap_fault_reasons))) {
1215                 *fault_type = INTR_REMAP;
1216                 return intr_remap_fault_reasons[fault_reason - 0x20];
1217         } else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1218                 *fault_type = DMA_REMAP;
1219                 return dma_remap_fault_reasons[fault_reason];
1220         } else {
1221                 *fault_type = UNKNOWN;
1222                 return "Unknown";
1223         }
1224 }
1225
1226 void dmar_msi_unmask(unsigned int irq)
1227 {
1228         struct intel_iommu *iommu = get_irq_data(irq);
1229         unsigned long flag;
1230
1231         /* unmask it */
1232         spin_lock_irqsave(&iommu->register_lock, flag);
1233         writel(0, iommu->reg + DMAR_FECTL_REG);
1234         /* Read a reg to force flush the post write */
1235         readl(iommu->reg + DMAR_FECTL_REG);
1236         spin_unlock_irqrestore(&iommu->register_lock, flag);
1237 }
1238
1239 void dmar_msi_mask(unsigned int irq)
1240 {
1241         unsigned long flag;
1242         struct intel_iommu *iommu = get_irq_data(irq);
1243
1244         /* mask it */
1245         spin_lock_irqsave(&iommu->register_lock, flag);
1246         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
1247         /* Read a reg to force flush the post write */
1248         readl(iommu->reg + DMAR_FECTL_REG);
1249         spin_unlock_irqrestore(&iommu->register_lock, flag);
1250 }
1251
1252 void dmar_msi_write(int irq, struct msi_msg *msg)
1253 {
1254         struct intel_iommu *iommu = get_irq_data(irq);
1255         unsigned long flag;
1256
1257         spin_lock_irqsave(&iommu->register_lock, flag);
1258         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
1259         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
1260         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
1261         spin_unlock_irqrestore(&iommu->register_lock, flag);
1262 }
1263
1264 void dmar_msi_read(int irq, struct msi_msg *msg)
1265 {
1266         struct intel_iommu *iommu = get_irq_data(irq);
1267         unsigned long flag;
1268
1269         spin_lock_irqsave(&iommu->register_lock, flag);
1270         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
1271         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
1272         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
1273         spin_unlock_irqrestore(&iommu->register_lock, flag);
1274 }
1275
1276 static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1277                 u8 fault_reason, u16 source_id, unsigned long long addr)
1278 {
1279         const char *reason;
1280         int fault_type;
1281
1282         reason = dmar_get_fault_reason(fault_reason, &fault_type);
1283
1284         if (fault_type == INTR_REMAP)
1285                 printk(KERN_ERR "INTR-REMAP: Request device [[%02x:%02x.%d] "
1286                        "fault index %llx\n"
1287                         "INTR-REMAP:[fault reason %02d] %s\n",
1288                         (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1289                         PCI_FUNC(source_id & 0xFF), addr >> 48,
1290                         fault_reason, reason);
1291         else
1292                 printk(KERN_ERR
1293                        "DMAR:[%s] Request device [%02x:%02x.%d] "
1294                        "fault addr %llx \n"
1295                        "DMAR:[fault reason %02d] %s\n",
1296                        (type ? "DMA Read" : "DMA Write"),
1297                        (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1298                        PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1299         return 0;
1300 }
1301
1302 #define PRIMARY_FAULT_REG_LEN (16)
1303 irqreturn_t dmar_fault(int irq, void *dev_id)
1304 {
1305         struct intel_iommu *iommu = dev_id;
1306         int reg, fault_index;
1307         u32 fault_status;
1308         unsigned long flag;
1309
1310         spin_lock_irqsave(&iommu->register_lock, flag);
1311         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1312         if (fault_status)
1313                 printk(KERN_ERR "DRHD: handling fault status reg %x\n",
1314                        fault_status);
1315
1316         /* TBD: ignore advanced fault log currently */
1317         if (!(fault_status & DMA_FSTS_PPF))
1318                 goto clear_rest;
1319
1320         fault_index = dma_fsts_fault_record_index(fault_status);
1321         reg = cap_fault_reg_offset(iommu->cap);
1322         while (1) {
1323                 u8 fault_reason;
1324                 u16 source_id;
1325                 u64 guest_addr;
1326                 int type;
1327                 u32 data;
1328
1329                 /* highest 32 bits */
1330                 data = readl(iommu->reg + reg +
1331                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
1332                 if (!(data & DMA_FRCD_F))
1333                         break;
1334
1335                 fault_reason = dma_frcd_fault_reason(data);
1336                 type = dma_frcd_type(data);
1337
1338                 data = readl(iommu->reg + reg +
1339                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
1340                 source_id = dma_frcd_source_id(data);
1341
1342                 guest_addr = dmar_readq(iommu->reg + reg +
1343                                 fault_index * PRIMARY_FAULT_REG_LEN);
1344                 guest_addr = dma_frcd_page_addr(guest_addr);
1345                 /* clear the fault */
1346                 writel(DMA_FRCD_F, iommu->reg + reg +
1347                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
1348
1349                 spin_unlock_irqrestore(&iommu->register_lock, flag);
1350
1351                 dmar_fault_do_one(iommu, type, fault_reason,
1352                                 source_id, guest_addr);
1353
1354                 fault_index++;
1355                 if (fault_index >= cap_num_fault_regs(iommu->cap))
1356                         fault_index = 0;
1357                 spin_lock_irqsave(&iommu->register_lock, flag);
1358         }
1359 clear_rest:
1360         /* clear all the other faults */
1361         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1362         writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1363
1364         spin_unlock_irqrestore(&iommu->register_lock, flag);
1365         return IRQ_HANDLED;
1366 }
1367
1368 int dmar_set_interrupt(struct intel_iommu *iommu)
1369 {
1370         int irq, ret;
1371
1372         /*
1373          * Check if the fault interrupt is already initialized.
1374          */
1375         if (iommu->irq)
1376                 return 0;
1377
1378         irq = create_irq();
1379         if (!irq) {
1380                 printk(KERN_ERR "IOMMU: no free vectors\n");
1381                 return -EINVAL;
1382         }
1383
1384         set_irq_data(irq, iommu);
1385         iommu->irq = irq;
1386
1387         ret = arch_setup_dmar_msi(irq);
1388         if (ret) {
1389                 set_irq_data(irq, NULL);
1390                 iommu->irq = 0;
1391                 destroy_irq(irq);
1392                 return ret;
1393         }
1394
1395         ret = request_irq(irq, dmar_fault, 0, iommu->name, iommu);
1396         if (ret)
1397                 printk(KERN_ERR "IOMMU: can't request irq\n");
1398         return ret;
1399 }
1400
1401 int __init enable_drhd_fault_handling(void)
1402 {
1403         struct dmar_drhd_unit *drhd;
1404
1405         /*
1406          * Enable fault control interrupt.
1407          */
1408         for_each_drhd_unit(drhd) {
1409                 int ret;
1410                 struct intel_iommu *iommu = drhd->iommu;
1411                 ret = dmar_set_interrupt(iommu);
1412
1413                 if (ret) {
1414                         printk(KERN_ERR "DRHD %Lx: failed to enable fault, "
1415                                " interrupt, ret %d\n",
1416                                (unsigned long long)drhd->reg_base_addr, ret);
1417                         return -1;
1418                 }
1419         }
1420
1421         return 0;
1422 }
1423
1424 /*
1425  * Re-enable Queued Invalidation interface.
1426  */
1427 int dmar_reenable_qi(struct intel_iommu *iommu)
1428 {
1429         if (!ecap_qis(iommu->ecap))
1430                 return -ENOENT;
1431
1432         if (!iommu->qi)
1433                 return -ENOENT;
1434
1435         /*
1436          * First disable queued invalidation.
1437          */
1438         dmar_disable_qi(iommu);
1439         /*
1440          * Then enable queued invalidation again. Since there is no pending
1441          * invalidation requests now, it's safe to re-enable queued
1442          * invalidation.
1443          */
1444         __dmar_enable_qi(iommu);
1445
1446         return 0;
1447 }
1448
1449 /*
1450  * Check interrupt remapping support in DMAR table description.
1451  */
1452 int __init dmar_ir_support(void)
1453 {
1454         struct acpi_table_dmar *dmar;
1455         dmar = (struct acpi_table_dmar *)dmar_tbl;
1456         if (!dmar)
1457                 return 0;
1458         return dmar->flags & 0x1;
1459 }