drivers/pci/dmar.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  *
  22  * This file implements early detection/parsing of Remapping Devices
  23  * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24  * tables.
  25  *
  26  * These routines are used by both DMA-remapping and Interrupt-remapping
  27  */
  28
  29 #include <linux/pci.h>
  30 #include <linux/dmar.h>
  31 #include <linux/iova.h>
  32 #include <linux/intel-iommu.h>
  33 #include <linux/timer.h>
  34
  35 #undef PREFIX
  36 #define PREFIX "DMAR:"
  37
  38 /* No locks are needed as DMA remapping hardware unit
  39  * list is constructed at boot time and hotplug of
  40  * these units are not supported by the architecture.
  41  */
  42 LIST_HEAD(dmar_drhd_units);
  43
  44 static struct acpi_table_header * __initdata dmar_tbl;
  45
  46 static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  47 {
  48         /*
  49          * add INCLUDE_ALL at the tail, so scan the list will find it at
  50          * the very end.
  51          */
  52         if (drhd->include_all)
  53                 list_add_tail(&drhd->list, &dmar_drhd_units);
  54         else
  55                 list_add(&drhd->list, &dmar_drhd_units);
  56 }
  57
  58 static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope,
  59                                            struct pci_dev **dev, u16 segment)
  60 {
  61         struct pci_bus *bus;
  62         struct pci_dev *pdev = NULL;
  63         struct acpi_dmar_pci_path *path;
  64         int count;
  65
  66         bus = pci_find_bus(segment, scope->bus);
  67         path = (struct acpi_dmar_pci_path *)(scope + 1);
  68         count = (scope->length - sizeof(struct acpi_dmar_device_scope))
  69                 / sizeof(struct acpi_dmar_pci_path);
  70
  71         while (count) {
  72                 if (pdev)
  73                         pci_dev_put(pdev);
  74                 /*
  75                  * Some BIOSes list non-exist devices in DMAR table, just
  76                  * ignore it
  77                  */
  78                 if (!bus) {
  79                         printk(KERN_WARNING
  80                         PREFIX "Device scope bus [%d] not found\n",
  81                         scope->bus);
  82                         break;
  83                 }
  84                 pdev = pci_get_slot(bus, PCI_DEVFN(path->dev, path->fn));
  85                 if (!pdev) {
  86                         printk(KERN_WARNING PREFIX
  87                         "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  88                                 segment, bus->number, path->dev, path->fn);
  89                         break;
  90                 }
  91                 path ++;
  92                 count --;
  93                 bus = pdev->subordinate;
  94         }
  95         if (!pdev) {
  96                 printk(KERN_WARNING PREFIX
  97                 "Device scope device [%04x:%02x:%02x.%02x] not found\n",
  98                 segment, scope->bus, path->dev, path->fn);
  99                 *dev = NULL;
 100                 return 0;
 101         }
 102         if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \
 103                         pdev->subordinate) || (scope->entry_type == \
 104                         ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) {
 105                 pci_dev_put(pdev);
 106                 printk(KERN_WARNING PREFIX
 107                         "Device scope type does not match for %s\n",
 108                          pci_name(pdev));
 109                 return -EINVAL;
 110         }
 111         *dev = pdev;
 112         return 0;
 113 }
 114
 115 static int __init dmar_parse_dev_scope(void *start, void *end, int *cnt,
 116                                        struct pci_dev ***devices, u16 segment)
 117 {
 118         struct acpi_dmar_device_scope *scope;
 119         void * tmp = start;
 120         int index;
 121         int ret;
 122
 123         *cnt = 0;
 124         while (start < end) {
 125                 scope = start;
 126                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 127                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 128                         (*cnt)++;
 129                 else
 130                         printk(KERN_WARNING PREFIX
 131                                 "Unsupported device scope\n");
 132                 start += scope->length;
 133         }
 134         if (*cnt == 0)
 135                 return 0;
 136
 137         *devices = kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL);
 138         if (!*devices)
 139                 return -ENOMEM;
 140
 141         start = tmp;
 142         index = 0;
 143         while (start < end) {
 144                 scope = start;
 145                 if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
 146                     scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) {
 147                         ret = dmar_parse_one_dev_scope(scope,
 148                                 &(*devices)[index], segment);
 149                         if (ret) {
 150                                 kfree(*devices);
 151                                 return ret;
 152                         }
 153                         index ++;
 154                 }
 155                 start += scope->length;
 156         }
 157
 158         return 0;
 159 }
 160
 161 /**
 162  * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 163  * structure which uniquely represent one DMA remapping hardware unit
 164  * present in the platform
 165  */
 166 static int __init
 167 dmar_parse_one_drhd(struct acpi_dmar_header *header)
 168 {
 169         struct acpi_dmar_hardware_unit *drhd;
 170         struct dmar_drhd_unit *dmaru;
 171         int ret = 0;
 172
 173         dmaru = kzalloc(sizeof(*dmaru), GFP_KERNEL);
 174         if (!dmaru)
 175                 return -ENOMEM;
 176
 177         dmaru->hdr = header;
 178         drhd = (struct acpi_dmar_hardware_unit *)header;
 179         dmaru->reg_base_addr = drhd->address;
 180         dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 181
 182         ret = alloc_iommu(dmaru);
 183         if (ret) {
 184                 kfree(dmaru);
 185                 return ret;
 186         }
 187         dmar_register_drhd_unit(dmaru);
 188         return 0;
 189 }
 190
 191 static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru)
 192 {
 193         struct acpi_dmar_hardware_unit *drhd;
 194         int ret = 0;
 195
 196         drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr;
 197
 198         if (dmaru->include_all)
 199                 return 0;
 200
 201         ret = dmar_parse_dev_scope((void *)(drhd + 1),
 202                                 ((void *)drhd) + drhd->header.length,
 203                                 &dmaru->devices_cnt, &dmaru->devices,
 204                                 drhd->segment);
 205         if (ret) {
 206                 list_del(&dmaru->list);
 207                 kfree(dmaru);
 208         }
 209         return ret;
 210 }
 211
 212 #ifdef CONFIG_DMAR
 213 LIST_HEAD(dmar_rmrr_units);
 214
 215 static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr)
 216 {
 217         list_add(&rmrr->list, &dmar_rmrr_units);
 218 }
 219
 220
 221 static int __init
 222 dmar_parse_one_rmrr(struct acpi_dmar_header *header)
 223 {
 224         struct acpi_dmar_reserved_memory *rmrr;
 225         struct dmar_rmrr_unit *rmrru;
 226
 227         rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
 228         if (!rmrru)
 229                 return -ENOMEM;
 230
 231         rmrru->hdr = header;
 232         rmrr = (struct acpi_dmar_reserved_memory *)header;
 233         rmrru->base_address = rmrr->base_address;
 234         rmrru->end_address = rmrr->end_address;
 235
 236         dmar_register_rmrr_unit(rmrru);
 237         return 0;
 238 }
 239
 240 static int __init
 241 rmrr_parse_dev(struct dmar_rmrr_unit *rmrru)
 242 {
 243         struct acpi_dmar_reserved_memory *rmrr;
 244         int ret;
 245
 246         rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr;
 247         ret = dmar_parse_dev_scope((void *)(rmrr + 1),
 248                 ((void *)rmrr) + rmrr->header.length,
 249                 &rmrru->devices_cnt, &rmrru->devices, rmrr->segment);
 250
 251         if (ret || (rmrru->devices_cnt == 0)) {
 252                 list_del(&rmrru->list);
 253                 kfree(rmrru);
 254         }
 255         return ret;
 256 }
 257 #endif
 258
 259 static void __init
 260 dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 261 {
 262         struct acpi_dmar_hardware_unit *drhd;
 263         struct acpi_dmar_reserved_memory *rmrr;
 264
 265         switch (header->type) {
 266         case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 267                 drhd = (struct acpi_dmar_hardware_unit *)header;
 268                 printk (KERN_INFO PREFIX
 269                         "DRHD (flags: 0x%08x)base: 0x%016Lx\n",
 270                         drhd->flags, (unsigned long long)drhd->address);
 271                 break;
 272         case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 273                 rmrr = (struct acpi_dmar_reserved_memory *)header;
 274
 275                 printk (KERN_INFO PREFIX
 276                         "RMRR base: 0x%016Lx end: 0x%016Lx\n",
 277                         (unsigned long long)rmrr->base_address,
 278                         (unsigned long long)rmrr->end_address);
 279                 break;
 280         }
 281 }
 282
 283 /**
 284  * dmar_table_detect - checks to see if the platform supports DMAR devices
 285  */
 286 static int __init dmar_table_detect(void)
 287 {
 288         acpi_status status = AE_OK;
 289
 290         /* if we could find DMAR table, then there are DMAR devices */
 291         status = acpi_get_table(ACPI_SIG_DMAR, 0,
 292                                 (struct acpi_table_header **)&dmar_tbl);
 293
 294         if (ACPI_SUCCESS(status) && !dmar_tbl) {
 295                 printk (KERN_WARNING PREFIX "Unable to map DMAR\n");
 296                 status = AE_NOT_FOUND;
 297         }
 298
 299         return (ACPI_SUCCESS(status) ? 1 : 0);
 300 }
 301
 302 /**
 303  * parse_dmar_table - parses the DMA reporting table
 304  */
 305 static int __init
 306 parse_dmar_table(void)
 307 {
 308         struct acpi_table_dmar *dmar;
 309         struct acpi_dmar_header *entry_header;
 310         int ret = 0;
 311
 312         /*
 313          * Do it again, earlier dmar_tbl mapping could be mapped with
 314          * fixed map.
 315          */
 316         dmar_table_detect();
 317
 318         dmar = (struct acpi_table_dmar *)dmar_tbl;
 319         if (!dmar)
 320                 return -ENODEV;
 321
 322         if (dmar->width < PAGE_SHIFT - 1) {
 323                 printk(KERN_WARNING PREFIX "Invalid DMAR haw\n");
 324                 return -EINVAL;
 325         }
 326
 327         printk (KERN_INFO PREFIX "Host address width %d\n",
 328                 dmar->width + 1);
 329
 330         entry_header = (struct acpi_dmar_header *)(dmar + 1);
 331         while (((unsigned long)entry_header) <
 332                         (((unsigned long)dmar) + dmar_tbl->length)) {
 333                 /* Avoid looping forever on bad ACPI tables */
 334                 if (entry_header->length == 0) {
 335                         printk(KERN_WARNING PREFIX
 336                                 "Invalid 0-length structure\n");
 337                         ret = -EINVAL;
 338                         break;
 339                 }
 340
 341                 dmar_table_print_dmar_entry(entry_header);
 342
 343                 switch (entry_header->type) {
 344                 case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 345                         ret = dmar_parse_one_drhd(entry_header);
 346                         break;
 347                 case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 348 #ifdef CONFIG_DMAR
 349                         ret = dmar_parse_one_rmrr(entry_header);
 350 #endif
 351                         break;
 352                 default:
 353                         printk(KERN_WARNING PREFIX
 354                                 "Unknown DMAR structure type\n");
 355                         ret = 0; /* for forward compatibility */
 356                         break;
 357                 }
 358                 if (ret)
 359                         break;
 360
 361                 entry_header = ((void *)entry_header + entry_header->length);
 362         }
 363         return ret;
 364 }
 365
 366 int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
 367                           struct pci_dev *dev)
 368 {
 369         int index;
 370
 371         while (dev) {
 372                 for (index = 0; index < cnt; index++)
 373                         if (dev == devices[index])
 374                                 return 1;
 375
 376                 /* Check our parent */
 377                 dev = dev->bus->self;
 378         }
 379
 380         return 0;
 381 }
 382
 383 struct dmar_drhd_unit *
 384 dmar_find_matched_drhd_unit(struct pci_dev *dev)
 385 {
 386         struct dmar_drhd_unit *dmaru = NULL;
 387         struct acpi_dmar_hardware_unit *drhd;
 388
 389         list_for_each_entry(dmaru, &dmar_drhd_units, list) {
 390                 drhd = container_of(dmaru->hdr,
 391                                     struct acpi_dmar_hardware_unit,
 392                                     header);
 393
 394                 if (dmaru->include_all &&
 395                     drhd->segment == pci_domain_nr(dev->bus))
 396                         return dmaru;
 397
 398                 if (dmar_pci_device_match(dmaru->devices,
 399                                           dmaru->devices_cnt, dev))
 400                         return dmaru;
 401         }
 402
 403         return NULL;
 404 }
 405
 406 int __init dmar_dev_scope_init(void)
 407 {
 408         struct dmar_drhd_unit *drhd, *drhd_n;
 409         int ret = -ENODEV;
 410
 411         list_for_each_entry_safe(drhd, drhd_n, &dmar_drhd_units, list) {
 412                 ret = dmar_parse_dev(drhd);
 413                 if (ret)
 414                         return ret;
 415         }
 416
 417 #ifdef CONFIG_DMAR
 418         {
 419                 struct dmar_rmrr_unit *rmrr, *rmrr_n;
 420                 list_for_each_entry_safe(rmrr, rmrr_n, &dmar_rmrr_units, list) {
 421                         ret = rmrr_parse_dev(rmrr);
 422                         if (ret)
 423                                 return ret;
 424                 }
 425         }
 426 #endif
 427
 428         return ret;
 429 }
 430
 431
 432 int __init dmar_table_init(void)
 433 {
 434         static int dmar_table_initialized;
 435         int ret;
 436
 437         if (dmar_table_initialized)
 438                 return 0;
 439
 440         dmar_table_initialized = 1;
 441
 442         ret = parse_dmar_table();
 443         if (ret) {
 444                 if (ret != -ENODEV)
 445                         printk(KERN_INFO PREFIX "parse DMAR table failure.\n");
 446                 return ret;
 447         }
 448
 449         if (list_empty(&dmar_drhd_units)) {
 450                 printk(KERN_INFO PREFIX "No DMAR devices found\n");
 451                 return -ENODEV;
 452         }
 453
 454 #ifdef CONFIG_DMAR
 455         if (list_empty(&dmar_rmrr_units))
 456                 printk(KERN_INFO PREFIX "No RMRR found\n");
 457 #endif
 458
 459 #ifdef CONFIG_INTR_REMAP
 460         parse_ioapics_under_ir();
 461 #endif
 462         return 0;
 463 }
 464
 465 void __init detect_intel_iommu(void)
 466 {
 467         int ret;
 468
 469         ret = dmar_table_detect();
 470
 471         {
 472 #ifdef CONFIG_INTR_REMAP
 473                 struct acpi_table_dmar *dmar;
 474                 /*
 475                  * for now we will disable dma-remapping when interrupt
 476                  * remapping is enabled.
 477                  * When support for queued invalidation for IOTLB invalidation
 478                  * is added, we will not need this any more.
 479                  */
 480                 dmar = (struct acpi_table_dmar *) dmar_tbl;
 481                 if (ret && cpu_has_x2apic && dmar->flags & 0x1)
 482                         printk(KERN_INFO
 483                                "Queued invalidation will be enabled to support "
 484                                "x2apic and Intr-remapping.\n");
 485 #endif
 486 #ifdef CONFIG_DMAR
 487                 if (ret && !no_iommu && !iommu_detected && !swiotlb &&
 488                     !dmar_disabled)
 489                         iommu_detected = 1;
 490 #endif
 491         }
 492         dmar_tbl = NULL;
 493 }
 494
 495
 496 int alloc_iommu(struct dmar_drhd_unit *drhd)
 497 {
 498         struct intel_iommu *iommu;
 499         int map_size;
 500         u32 ver;
 501         static int iommu_allocated = 0;
 502         int agaw = 0;
 503
 504         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 505         if (!iommu)
 506                 return -ENOMEM;
 507
 508         iommu->seq_id = iommu_allocated++;
 509
 510         iommu->reg = ioremap(drhd->reg_base_addr, VTD_PAGE_SIZE);
 511         if (!iommu->reg) {
 512                 printk(KERN_ERR "IOMMU: can't map the region\n");
 513                 goto error;
 514         }
 515         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 516         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 517
 518 #ifdef CONFIG_DMAR
 519         agaw = iommu_calculate_agaw(iommu);
 520         if (agaw < 0) {
 521                 printk(KERN_ERR
 522                         "Cannot get a valid agaw for iommu (seq_id = %d)\n",
 523                         iommu->seq_id);
 524                 goto error;
 525         }
 526 #endif
 527         iommu->agaw = agaw;
 528
 529         /* the registers might be more than one page */
 530         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 531                 cap_max_fault_reg_offset(iommu->cap));
 532         map_size = VTD_PAGE_ALIGN(map_size);
 533         if (map_size > VTD_PAGE_SIZE) {
 534                 iounmap(iommu->reg);
 535                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
 536                 if (!iommu->reg) {
 537                         printk(KERN_ERR "IOMMU: can't map the region\n");
 538                         goto error;
 539                 }
 540         }
 541
 542         ver = readl(iommu->reg + DMAR_VER_REG);
 543         pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
 544                 (unsigned long long)drhd->reg_base_addr,
 545                 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
 546                 (unsigned long long)iommu->cap,
 547                 (unsigned long long)iommu->ecap);
 548
 549         spin_lock_init(&iommu->register_lock);
 550
 551         drhd->iommu = iommu;
 552         return 0;
 553 error:
 554         kfree(iommu);
 555         return -1;
 556 }
 557
 558 void free_iommu(struct intel_iommu *iommu)
 559 {
 560         if (!iommu)
 561                 return;
 562
 563 #ifdef CONFIG_DMAR
 564         free_dmar_iommu(iommu);
 565 #endif
 566
 567         if (iommu->reg)
 568                 iounmap(iommu->reg);
 569         kfree(iommu);
 570 }
 571
 572 /*
 573  * Reclaim all the submitted descriptors which have completed its work.
 574  */
 575 static inline void reclaim_free_desc(struct q_inval *qi)
 576 {
 577         while (qi->desc_status[qi->free_tail] == QI_DONE) {
 578                 qi->desc_status[qi->free_tail] = QI_FREE;
 579                 qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
 580                 qi->free_cnt++;
 581         }
 582 }
 583
 584 static int qi_check_fault(struct intel_iommu *iommu, int index)
 585 {
 586         u32 fault;
 587         int head;
 588         struct q_inval *qi = iommu->qi;
 589         int wait_index = (index + 1) % QI_LENGTH;
 590
 591         fault = readl(iommu->reg + DMAR_FSTS_REG);
 592
 593         /*
 594          * If IQE happens, the head points to the descriptor associated
 595          * with the error. No new descriptors are fetched until the IQE
 596          * is cleared.
 597          */
 598         if (fault & DMA_FSTS_IQE) {
 599                 head = readl(iommu->reg + DMAR_IQH_REG);
 600                 if ((head >> 4) == index) {
 601                         memcpy(&qi->desc[index], &qi->desc[wait_index],
 602                                         sizeof(struct qi_desc));
 603                         __iommu_flush_cache(iommu, &qi->desc[index],
 604                                         sizeof(struct qi_desc));
 605                         writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
 606                         return -EINVAL;
 607                 }
 608         }
 609
 610         return 0;
 611 }
 612
 613 /*
 614  * Submit the queued invalidation descriptor to the remapping
 615  * hardware unit and wait for its completion.
 616  */
 617 int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
 618 {
 619         int rc = 0;
 620         struct q_inval *qi = iommu->qi;
 621         struct qi_desc *hw, wait_desc;
 622         int wait_index, index;
 623         unsigned long flags;
 624
 625         if (!qi)
 626                 return 0;
 627
 628         hw = qi->desc;
 629
 630         spin_lock_irqsave(&qi->q_lock, flags);
 631         while (qi->free_cnt < 3) {
 632                 spin_unlock_irqrestore(&qi->q_lock, flags);
 633                 cpu_relax();
 634                 spin_lock_irqsave(&qi->q_lock, flags);
 635         }
 636
 637         index = qi->free_head;
 638         wait_index = (index + 1) % QI_LENGTH;
 639
 640         qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 641
 642         hw[index] = *desc;
 643
 644         wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
 645                         QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
 646         wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
 647
 648         hw[wait_index] = wait_desc;
 649
 650         __iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
 651         __iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
 652
 653         qi->free_head = (qi->free_head + 2) % QI_LENGTH;
 654         qi->free_cnt -= 2;
 655
 656         /*
 657          * update the HW tail register indicating the presence of
 658          * new descriptors.
 659          */
 660         writel(qi->free_head << 4, iommu->reg + DMAR_IQT_REG);
 661
 662         while (qi->desc_status[wait_index] != QI_DONE) {
 663                 /*
 664                  * We will leave the interrupts disabled, to prevent interrupt
 665                  * context to queue another cmd while a cmd is already submitted
 666                  * and waiting for completion on this cpu. This is to avoid
 667                  * a deadlock where the interrupt context can wait indefinitely
 668                  * for free slots in the queue.
 669                  */
 670                 rc = qi_check_fault(iommu, index);
 671                 if (rc)
 672                         goto out;
 673
 674                 spin_unlock(&qi->q_lock);
 675                 cpu_relax();
 676                 spin_lock(&qi->q_lock);
 677         }
 678 out:
 679         qi->desc_status[index] = qi->desc_status[wait_index] = QI_DONE;
 680
 681         reclaim_free_desc(qi);
 682         spin_unlock_irqrestore(&qi->q_lock, flags);
 683
 684         return rc;
 685 }
 686
 687 /*
 688  * Flush the global interrupt entry cache.
 689  */
 690 void qi_global_iec(struct intel_iommu *iommu)
 691 {
 692         struct qi_desc desc;
 693
 694         desc.low = QI_IEC_TYPE;
 695         desc.high = 0;
 696
 697         /* should never fail */
 698         qi_submit_sync(&desc, iommu);
 699 }
 700
 701 int qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
 702                      u64 type, int non_present_entry_flush)
 703 {
 704         struct qi_desc desc;
 705
 706         if (non_present_entry_flush) {
 707                 if (!cap_caching_mode(iommu->cap))
 708                         return 1;
 709                 else
 710                         did = 0;
 711         }
 712
 713         desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
 714                         | QI_CC_GRAN(type) | QI_CC_TYPE;
 715         desc.high = 0;
 716
 717         return qi_submit_sync(&desc, iommu);
 718 }
 719
 720 int qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
 721                    unsigned int size_order, u64 type,
 722                    int non_present_entry_flush)
 723 {
 724         u8 dw = 0, dr = 0;
 725
 726         struct qi_desc desc;
 727         int ih = 0;
 728
 729         if (non_present_entry_flush) {
 730                 if (!cap_caching_mode(iommu->cap))
 731                         return 1;
 732                 else
 733                         did = 0;
 734         }
 735
 736         if (cap_write_drain(iommu->cap))
 737                 dw = 1;
 738
 739         if (cap_read_drain(iommu->cap))
 740                 dr = 1;
 741
 742         desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
 743                 | QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
 744         desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
 745                 | QI_IOTLB_AM(size_order);
 746
 747         return qi_submit_sync(&desc, iommu);
 748 }
 749
 750 /*
 751  * Enable Queued Invalidation interface. This is a must to support
 752  * interrupt-remapping. Also used by DMA-remapping, which replaces
 753  * register based IOTLB invalidation.
 754  */
 755 int dmar_enable_qi(struct intel_iommu *iommu)
 756 {
 757         u32 cmd, sts;
 758         unsigned long flags;
 759         struct q_inval *qi;
 760
 761         if (!ecap_qis(iommu->ecap))
 762                 return -ENOENT;
 763
 764         /*
 765          * queued invalidation is already setup and enabled.
 766          */
 767         if (iommu->qi)
 768                 return 0;
 769
 770         iommu->qi = kmalloc(sizeof(*qi), GFP_KERNEL);
 771         if (!iommu->qi)
 772                 return -ENOMEM;
 773
 774         qi = iommu->qi;
 775
 776         qi->desc = (void *)(get_zeroed_page(GFP_KERNEL));
 777         if (!qi->desc) {
 778                 kfree(qi);
 779                 iommu->qi = 0;
 780                 return -ENOMEM;
 781         }
 782
 783         qi->desc_status = kmalloc(QI_LENGTH * sizeof(int), GFP_KERNEL);
 784         if (!qi->desc_status) {
 785                 free_page((unsigned long) qi->desc);
 786                 kfree(qi);
 787                 iommu->qi = 0;
 788                 return -ENOMEM;
 789         }
 790
 791         qi->free_head = qi->free_tail = 0;
 792         qi->free_cnt = QI_LENGTH;
 793
 794         spin_lock_init(&qi->q_lock);
 795
 796         spin_lock_irqsave(&iommu->register_lock, flags);
 797         /* write zero to the tail reg */
 798         writel(0, iommu->reg + DMAR_IQT_REG);
 799
 800         dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
 801
 802         cmd = iommu->gcmd | DMA_GCMD_QIE;
 803         iommu->gcmd |= DMA_GCMD_QIE;
 804         writel(cmd, iommu->reg + DMAR_GCMD_REG);
 805
 806         /* Make sure hardware complete it */
 807         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
 808         spin_unlock_irqrestore(&iommu->register_lock, flags);
 809
 810         return 0;
 811 }