arch/x86_64/kernel/pci-gart.c

   1 /*
   2  * Dynamic DMA mapping support for AMD Hammer.
   3  *
   4  * Use the integrated AGP GART in the Hammer northbridge as an IOMMU for PCI.
   5  * This allows to use PCI devices that only support 32bit addresses on systems
   6  * with more than 4GB.
   7  *
   8  * See Documentation/DMA-mapping.txt for the interface specification.
   9  *
  10  * Copyright 2002 Andi Kleen, SuSE Labs.
  11  */
  12
  13 #include <linux/config.h>
  14 #include <linux/types.h>
  15 #include <linux/ctype.h>
  16 #include <linux/agp_backend.h>
  17 #include <linux/init.h>
  18 #include <linux/mm.h>
  19 #include <linux/string.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/pci.h>
  22 #include <linux/module.h>
  23 #include <linux/topology.h>
  24 #include <linux/interrupt.h>
  25 #include <linux/bitops.h>
  26 #include <asm/atomic.h>
  27 #include <asm/io.h>
  28 #include <asm/mtrr.h>
  29 #include <asm/pgtable.h>
  30 #include <asm/proto.h>
  31 #include <asm/cacheflush.h>
  32 #include <asm/kdebug.h>
  33 #include <asm/swiotlb.h>
  34 #include <asm/dma.h>
  35
  36 unsigned long iommu_bus_base;   /* GART remapping area (physical) */
  37 static unsigned long iommu_size;        /* size of remapping area bytes */
  38 static unsigned long iommu_pages;       /* .. and in pages */
  39
  40 u32 *iommu_gatt_base;           /* Remapping table */
  41
  42 /* If this is disabled the IOMMU will use an optimized flushing strategy
  43    of only flushing when an mapping is reused. With it true the GART is flushed
  44    for every mapping. Problem is that doing the lazy flush seems to trigger
  45    bugs with some popular PCI cards, in particular 3ware (but has been also
  46    also seen with Qlogic at least). */
  47 int iommu_fullflush = 1;
  48
  49 #define MAX_NB 8
  50
  51 /* Allocation bitmap for the remapping area */
  52 static DEFINE_SPINLOCK(iommu_bitmap_lock);
  53 static unsigned long *iommu_gart_bitmap; /* guarded by iommu_bitmap_lock */
  54
  55 static u32 gart_unmapped_entry;
  56
  57 #define GPTE_VALID    1
  58 #define GPTE_COHERENT 2
  59 #define GPTE_ENCODE(x) \
  60         (((x) & 0xfffff000) | (((x) >> 32) << 4) | GPTE_VALID | GPTE_COHERENT)
  61 #define GPTE_DECODE(x) (((x) & 0xfffff000) | (((u64)(x) & 0xff0) << 28))
  62
  63 #define to_pages(addr,size) \
  64         (round_up(((addr) & ~PAGE_MASK) + (size), PAGE_SIZE) >> PAGE_SHIFT)
  65
  66 #define for_all_nb(dev) \
  67         dev = NULL;     \
  68         while ((dev = pci_get_device(PCI_VENDOR_ID_AMD, 0x1103, dev))!=NULL)\
  69              if (dev->bus->number == 0 &&                                    \
  70                     (PCI_SLOT(dev->devfn) >= 24) && (PCI_SLOT(dev->devfn) <= 31))
  71
  72 static struct pci_dev *northbridges[MAX_NB];
  73 static u32 northbridge_flush_word[MAX_NB];
  74
  75 #define EMERGENCY_PAGES 32 /* = 128KB */
  76
  77 #ifdef CONFIG_AGP
  78 #define AGPEXTERN extern
  79 #else
  80 #define AGPEXTERN
  81 #endif
  82
  83 /* backdoor interface to AGP driver */
  84 AGPEXTERN int agp_memory_reserved;
  85 AGPEXTERN __u32 *agp_gatt_table;
  86
  87 static unsigned long next_bit;  /* protected by iommu_bitmap_lock */
  88 static int need_flush;          /* global flush state. set for each gart wrap */
  89
  90 static unsigned long alloc_iommu(int size)
  91 {
  92         unsigned long offset, flags;
  93
  94         spin_lock_irqsave(&iommu_bitmap_lock, flags);
  95         offset = find_next_zero_string(iommu_gart_bitmap,next_bit,iommu_pages,size);
  96         if (offset == -1) {
  97                 need_flush = 1;
  98                 offset = find_next_zero_string(iommu_gart_bitmap,0,next_bit,size);
  99         }
 100         if (offset != -1) {
 101                 set_bit_string(iommu_gart_bitmap, offset, size);
 102                 next_bit = offset+size;
 103                 if (next_bit >= iommu_pages) {
 104                         next_bit = 0;
 105                         need_flush = 1;
 106                 }
 107         }
 108         if (iommu_fullflush)
 109                 need_flush = 1;
 110         spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
 111         return offset;
 112 }
 113
 114 static void free_iommu(unsigned long offset, int size)
 115 {
 116         unsigned long flags;
 117         if (size == 1) {
 118                 clear_bit(offset, iommu_gart_bitmap);
 119                 return;
 120         }
 121         spin_lock_irqsave(&iommu_bitmap_lock, flags);
 122         __clear_bit_string(iommu_gart_bitmap, offset, size);
 123         spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
 124 }
 125
 126 /*
 127  * Use global flush state to avoid races with multiple flushers.
 128  */
 129 static void flush_gart(struct device *dev)
 130 {
 131         unsigned long flags;
 132         int flushed = 0;
 133         int i, max;
 134
 135         spin_lock_irqsave(&iommu_bitmap_lock, flags);
 136         if (need_flush) {
 137                 max = 0;
 138                 for (i = 0; i < MAX_NB; i++) {
 139                         if (!northbridges[i])
 140                                 continue;
 141                         pci_write_config_dword(northbridges[i], 0x9c,
 142                                                northbridge_flush_word[i] | 1);
 143                         flushed++;
 144                         max = i;
 145                 }
 146                 for (i = 0; i <= max; i++) {
 147                         u32 w;
 148                         if (!northbridges[i])
 149                                 continue;
 150                         /* Make sure the hardware actually executed the flush. */
 151                         for (;;) {
 152                                 pci_read_config_dword(northbridges[i], 0x9c, &w);
 153                                 if (!(w & 1))
 154                                         break;
 155                                 cpu_relax();
 156                         }
 157                 }
 158                 if (!flushed)
 159                         printk("nothing to flush?\n");
 160                 need_flush = 0;
 161         }
 162         spin_unlock_irqrestore(&iommu_bitmap_lock, flags);
 163 }
 164
 165
 166
 167 #ifdef CONFIG_IOMMU_LEAK
 168
 169 #define SET_LEAK(x) if (iommu_leak_tab) \
 170                         iommu_leak_tab[x] = __builtin_return_address(0);
 171 #define CLEAR_LEAK(x) if (iommu_leak_tab) \
 172                         iommu_leak_tab[x] = NULL;
 173
 174 /* Debugging aid for drivers that don't free their IOMMU tables */
 175 static void **iommu_leak_tab;
 176 static int leak_trace;
 177 int iommu_leak_pages = 20;
 178 void dump_leak(void)
 179 {
 180         int i;
 181         static int dump;
 182         if (dump || !iommu_leak_tab) return;
 183         dump = 1;
 184         show_stack(NULL,NULL);
 185         /* Very crude. dump some from the end of the table too */
 186         printk("Dumping %d pages from end of IOMMU:\n", iommu_leak_pages);
 187         for (i = 0; i < iommu_leak_pages; i+=2) {
 188                 printk("%lu: ", iommu_pages-i);
 189                 printk_address((unsigned long) iommu_leak_tab[iommu_pages-i]);
 190                 printk("%c", (i+1)%2 == 0 ? '\n' : ' ');
 191         }
 192         printk("\n");
 193 }
 194 #else
 195 #define SET_LEAK(x)
 196 #define CLEAR_LEAK(x)
 197 #endif
 198
 199 static void iommu_full(struct device *dev, size_t size, int dir)
 200 {
 201         /*
 202          * Ran out of IOMMU space for this operation. This is very bad.
 203          * Unfortunately the drivers cannot handle this operation properly.
 204          * Return some non mapped prereserved space in the aperture and
 205          * let the Northbridge deal with it. This will result in garbage
 206          * in the IO operation. When the size exceeds the prereserved space
 207          * memory corruption will occur or random memory will be DMAed
 208          * out. Hopefully no network devices use single mappings that big.
 209          */
 210
 211         printk(KERN_ERR
 212   "PCI-DMA: Out of IOMMU space for %lu bytes at device %s\n",
 213                size, dev->bus_id);
 214
 215         if (size > PAGE_SIZE*EMERGENCY_PAGES) {
 216                 if (dir == PCI_DMA_FROMDEVICE || dir == PCI_DMA_BIDIRECTIONAL)
 217                         panic("PCI-DMA: Memory would be corrupted\n");
 218                 if (dir == PCI_DMA_TODEVICE || dir == PCI_DMA_BIDIRECTIONAL)
 219                         panic(KERN_ERR "PCI-DMA: Random memory would be DMAed\n");
 220         }
 221
 222 #ifdef CONFIG_IOMMU_LEAK
 223         dump_leak();
 224 #endif
 225 }
 226
 227 static inline int need_iommu(struct device *dev, unsigned long addr, size_t size)
 228 {
 229         u64 mask = *dev->dma_mask;
 230         int high = addr + size >= mask;
 231         int mmu = high;
 232         if (force_iommu)
 233                 mmu = 1;
 234         return mmu;
 235 }
 236
 237 static inline int nonforced_iommu(struct device *dev, unsigned long addr, size_t size)
 238 {
 239         u64 mask = *dev->dma_mask;
 240         int high = addr + size >= mask;
 241         int mmu = high;
 242         return mmu;
 243 }
 244
 245 /* Map a single continuous physical area into the IOMMU.
 246  * Caller needs to check if the iommu is needed and flush.
 247  */
 248 static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem,
 249                                 size_t size, int dir)
 250 {
 251         unsigned long npages = to_pages(phys_mem, size);
 252         unsigned long iommu_page = alloc_iommu(npages);
 253         int i;
 254         if (iommu_page == -1) {
 255                 if (!nonforced_iommu(dev, phys_mem, size))
 256                         return phys_mem;
 257                 if (panic_on_overflow)
 258                         panic("dma_map_area overflow %lu bytes\n", size);
 259                 iommu_full(dev, size, dir);
 260                 return bad_dma_address;
 261         }
 262
 263         for (i = 0; i < npages; i++) {
 264                 iommu_gatt_base[iommu_page + i] = GPTE_ENCODE(phys_mem);
 265                 SET_LEAK(iommu_page + i);
 266                 phys_mem += PAGE_SIZE;
 267         }
 268         return iommu_bus_base + iommu_page*PAGE_SIZE + (phys_mem & ~PAGE_MASK);
 269 }
 270
 271 static dma_addr_t gart_map_simple(struct device *dev, char *buf,
 272                                  size_t size, int dir)
 273 {
 274         dma_addr_t map = dma_map_area(dev, virt_to_bus(buf), size, dir);
 275         flush_gart(dev);
 276         return map;
 277 }
 278
 279 /* Map a single area into the IOMMU */
 280 dma_addr_t gart_map_single(struct device *dev, void *addr, size_t size, int dir)
 281 {
 282         unsigned long phys_mem, bus;
 283
 284         BUG_ON(dir == DMA_NONE);
 285
 286         if (!dev)
 287                 dev = &fallback_dev;
 288
 289         phys_mem = virt_to_phys(addr);
 290         if (!need_iommu(dev, phys_mem, size))
 291                 return phys_mem;
 292
 293         bus = gart_map_simple(dev, addr, size, dir);
 294         return bus;
 295 }
 296
 297 /*
 298  * Wrapper for pci_unmap_single working with scatterlists.
 299  */
 300 void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
 301 {
 302         int i;
 303
 304         for (i = 0; i < nents; i++) {
 305                 struct scatterlist *s = &sg[i];
 306                 if (!s->dma_length || !s->length)
 307                         break;
 308                 dma_unmap_single(dev, s->dma_address, s->dma_length, dir);
 309         }
 310 }
 311
 312 /* Fallback for dma_map_sg in case of overflow */
 313 static int dma_map_sg_nonforce(struct device *dev, struct scatterlist *sg,
 314                                int nents, int dir)
 315 {
 316         int i;
 317
 318 #ifdef CONFIG_IOMMU_DEBUG
 319         printk(KERN_DEBUG "dma_map_sg overflow\n");
 320 #endif
 321
 322         for (i = 0; i < nents; i++ ) {
 323                 struct scatterlist *s = &sg[i];
 324                 unsigned long addr = page_to_phys(s->page) + s->offset;
 325                 if (nonforced_iommu(dev, addr, s->length)) {
 326                         addr = dma_map_area(dev, addr, s->length, dir);
 327                         if (addr == bad_dma_address) {
 328                                 if (i > 0)
 329                                         gart_unmap_sg(dev, sg, i, dir);
 330                                 nents = 0;
 331                                 sg[0].dma_length = 0;
 332                                 break;
 333                         }
 334                 }
 335                 s->dma_address = addr;
 336                 s->dma_length = s->length;
 337         }
 338         flush_gart(dev);
 339         return nents;
 340 }
 341
 342 /* Map multiple scatterlist entries continuous into the first. */
 343 static int __dma_map_cont(struct scatterlist *sg, int start, int stopat,
 344                       struct scatterlist *sout, unsigned long pages)
 345 {
 346         unsigned long iommu_start = alloc_iommu(pages);
 347         unsigned long iommu_page = iommu_start;
 348         int i;
 349
 350         if (iommu_start == -1)
 351                 return -1;
 352
 353         for (i = start; i < stopat; i++) {
 354                 struct scatterlist *s = &sg[i];
 355                 unsigned long pages, addr;
 356                 unsigned long phys_addr = s->dma_address;
 357
 358                 BUG_ON(i > start && s->offset);
 359                 if (i == start) {
 360                         *sout = *s;
 361                         sout->dma_address = iommu_bus_base;
 362                         sout->dma_address += iommu_page*PAGE_SIZE + s->offset;
 363                         sout->dma_length = s->length;
 364                 } else {
 365                         sout->dma_length += s->length;
 366                 }
 367
 368                 addr = phys_addr;
 369                 pages = to_pages(s->offset, s->length);
 370                 while (pages--) {
 371                         iommu_gatt_base[iommu_page] = GPTE_ENCODE(addr);
 372                         SET_LEAK(iommu_page);
 373                         addr += PAGE_SIZE;
 374                         iommu_page++;
 375                 }
 376         }
 377         BUG_ON(iommu_page - iommu_start != pages);
 378         return 0;
 379 }
 380
 381 static inline int dma_map_cont(struct scatterlist *sg, int start, int stopat,
 382                       struct scatterlist *sout,
 383                       unsigned long pages, int need)
 384 {
 385         if (!need) {
 386                 BUG_ON(stopat - start != 1);
 387                 *sout = sg[start];
 388                 sout->dma_length = sg[start].length;
 389                 return 0;
 390         }
 391         return __dma_map_cont(sg, start, stopat, sout, pages);
 392 }
 393
 394 /*
 395  * DMA map all entries in a scatterlist.
 396  * Merge chunks that have page aligned sizes into a continuous mapping.
 397  */
 398 int gart_map_sg(struct device *dev, struct scatterlist *sg, int nents, int dir)
 399 {
 400         int i;
 401         int out;
 402         int start;
 403         unsigned long pages = 0;
 404         int need = 0, nextneed;
 405
 406         BUG_ON(dir == DMA_NONE);
 407         if (nents == 0)
 408                 return 0;
 409
 410         if (!dev)
 411                 dev = &fallback_dev;
 412
 413         out = 0;
 414         start = 0;
 415         for (i = 0; i < nents; i++) {
 416                 struct scatterlist *s = &sg[i];
 417                 dma_addr_t addr = page_to_phys(s->page) + s->offset;
 418                 s->dma_address = addr;
 419                 BUG_ON(s->length == 0);
 420
 421                 nextneed = need_iommu(dev, addr, s->length);
 422
 423                 /* Handle the previous not yet processed entries */
 424                 if (i > start) {
 425                         struct scatterlist *ps = &sg[i-1];
 426                         /* Can only merge when the last chunk ends on a page
 427                            boundary and the new one doesn't have an offset. */
 428                         if (!iommu_merge || !nextneed || !need || s->offset ||
 429                             (ps->offset + ps->length) % PAGE_SIZE) {
 430                                 if (dma_map_cont(sg, start, i, sg+out, pages,
 431                                                  need) < 0)
 432                                         goto error;
 433                                 out++;
 434                                 pages = 0;
 435                                 start = i;
 436                         }
 437                 }
 438
 439                 need = nextneed;
 440                 pages += to_pages(s->offset, s->length);
 441         }
 442         if (dma_map_cont(sg, start, i, sg+out, pages, need) < 0)
 443                 goto error;
 444         out++;
 445         flush_gart(dev);
 446         if (out < nents)
 447                 sg[out].dma_length = 0;
 448         return out;
 449
 450 error:
 451         flush_gart(NULL);
 452         gart_unmap_sg(dev, sg, nents, dir);
 453         /* When it was forced or merged try again in a dumb way */
 454         if (force_iommu || iommu_merge) {
 455                 out = dma_map_sg_nonforce(dev, sg, nents, dir);
 456                 if (out > 0)
 457                         return out;
 458         }
 459         if (panic_on_overflow)
 460                 panic("dma_map_sg: overflow on %lu pages\n", pages);
 461         iommu_full(dev, pages << PAGE_SHIFT, dir);
 462         for (i = 0; i < nents; i++)
 463                 sg[i].dma_address = bad_dma_address;
 464         return 0;
 465 }
 466
 467 /*
 468  * Free a DMA mapping.
 469  */
 470 void gart_unmap_single(struct device *dev, dma_addr_t dma_addr,
 471                       size_t size, int direction)
 472 {
 473         unsigned long iommu_page;
 474         int npages;
 475         int i;
 476
 477         if (dma_addr < iommu_bus_base + EMERGENCY_PAGES*PAGE_SIZE ||
 478             dma_addr >= iommu_bus_base + iommu_size)
 479                 return;
 480         iommu_page = (dma_addr - iommu_bus_base)>>PAGE_SHIFT;
 481         npages = to_pages(dma_addr, size);
 482         for (i = 0; i < npages; i++) {
 483                 iommu_gatt_base[iommu_page + i] = gart_unmapped_entry;
 484                 CLEAR_LEAK(iommu_page + i);
 485         }
 486         free_iommu(iommu_page, npages);
 487 }
 488
 489 static int no_agp;
 490
 491 static __init unsigned long check_iommu_size(unsigned long aper, u64 aper_size)
 492 {
 493         unsigned long a;
 494         if (!iommu_size) {
 495                 iommu_size = aper_size;
 496                 if (!no_agp)
 497                         iommu_size /= 2;
 498         }
 499
 500         a = aper + iommu_size;
 501         iommu_size -= round_up(a, LARGE_PAGE_SIZE) - a;
 502
 503         if (iommu_size < 64*1024*1024)
 504                 printk(KERN_WARNING
 505   "PCI-DMA: Warning: Small IOMMU %luMB. Consider increasing the AGP aperture in BIOS\n",iommu_size>>20);
 506
 507         return iommu_size;
 508 }
 509
 510 static __init unsigned read_aperture(struct pci_dev *dev, u32 *size)
 511 {
 512         unsigned aper_size = 0, aper_base_32;
 513         u64 aper_base;
 514         unsigned aper_order;
 515
 516         pci_read_config_dword(dev, 0x94, &aper_base_32);
 517         pci_read_config_dword(dev, 0x90, &aper_order);
 518         aper_order = (aper_order >> 1) & 7;
 519
 520         aper_base = aper_base_32 & 0x7fff;
 521         aper_base <<= 25;
 522
 523         aper_size = (32 * 1024 * 1024) << aper_order;
 524         if (aper_base + aper_size >= 0xffffffff || !aper_size)
 525                 aper_base = 0;
 526
 527         *size = aper_size;
 528         return aper_base;
 529 }
 530
 531 /*
 532  * Private Northbridge GATT initialization in case we cannot use the
 533  * AGP driver for some reason.
 534  */
 535 static __init int init_k8_gatt(struct agp_kern_info *info)
 536 {
 537         struct pci_dev *dev;
 538         void *gatt;
 539         unsigned aper_base, new_aper_base;
 540         unsigned aper_size, gatt_size, new_aper_size;
 541
 542         printk(KERN_INFO "PCI-DMA: Disabling AGP.\n");
 543         aper_size = aper_base = info->aper_size = 0;
 544         for_all_nb(dev) {
 545                 new_aper_base = read_aperture(dev, &new_aper_size);
 546                 if (!new_aper_base)
 547                         goto nommu;
 548
 549                 if (!aper_base) {
 550                         aper_size = new_aper_size;
 551                         aper_base = new_aper_base;
 552                 }
 553                 if (aper_size != new_aper_size || aper_base != new_aper_base)
 554                         goto nommu;
 555         }
 556         if (!aper_base)
 557                 goto nommu;
 558         info->aper_base = aper_base;
 559         info->aper_size = aper_size>>20;
 560
 561         gatt_size = (aper_size >> PAGE_SHIFT) * sizeof(u32);
 562         gatt = (void *)__get_free_pages(GFP_KERNEL, get_order(gatt_size));
 563         if (!gatt)
 564                 panic("Cannot allocate GATT table");
 565         memset(gatt, 0, gatt_size);
 566         agp_gatt_table = gatt;
 567
 568         for_all_nb(dev) {
 569                 u32 ctl;
 570                 u32 gatt_reg;
 571
 572                 gatt_reg = __pa(gatt) >> 12;
 573                 gatt_reg <<= 4;
 574                 pci_write_config_dword(dev, 0x98, gatt_reg);
 575                 pci_read_config_dword(dev, 0x90, &ctl);
 576
 577                 ctl |= 1;
 578                 ctl &= ~((1<<4) | (1<<5));
 579
 580                 pci_write_config_dword(dev, 0x90, ctl);
 581         }
 582         flush_gart(NULL);
 583
 584         printk("PCI-DMA: aperture base @ %x size %u KB\n",aper_base, aper_size>>10);
 585         return 0;
 586
 587  nommu:
 588         /* Should not happen anymore */
 589         printk(KERN_ERR "PCI-DMA: More than 4GB of RAM and no IOMMU\n"
 590                KERN_ERR "PCI-DMA: 32bit PCI IO may malfunction.\n");
 591         return -1;
 592 }
 593
 594 extern int agp_amd64_init(void);
 595
 596 static struct dma_mapping_ops gart_dma_ops = {
 597         .mapping_error = NULL,
 598         .map_single = gart_map_single,
 599         .map_simple = gart_map_simple,
 600         .unmap_single = gart_unmap_single,
 601         .sync_single_for_cpu = NULL,
 602         .sync_single_for_device = NULL,
 603         .sync_single_range_for_cpu = NULL,
 604         .sync_single_range_for_device = NULL,
 605         .sync_sg_for_cpu = NULL,
 606         .sync_sg_for_device = NULL,
 607         .map_sg = gart_map_sg,
 608         .unmap_sg = gart_unmap_sg,
 609 };
 610
 611 static int __init pci_iommu_init(void)
 612 {
 613         struct agp_kern_info info;
 614         unsigned long aper_size;
 615         unsigned long iommu_start;
 616         struct pci_dev *dev;
 617         unsigned long scratch;
 618         long i;
 619
 620 #ifndef CONFIG_AGP_AMD64
 621         no_agp = 1;
 622 #else
 623         /* Makefile puts PCI initialization via subsys_initcall first. */
 624         /* Add other K8 AGP bridge drivers here */
 625         no_agp = no_agp ||
 626                 (agp_amd64_init() < 0) ||
 627                 (agp_copy_info(agp_bridge, &info) < 0);
 628 #endif
 629
 630         if (swiotlb)
 631                 return -1;
 632
 633         if (no_iommu ||
 634             (!force_iommu && end_pfn <= MAX_DMA32_PFN) ||
 635             !iommu_aperture ||
 636             (no_agp && init_k8_gatt(&info) < 0)) {
 637                 printk(KERN_INFO "PCI-DMA: Disabling IOMMU.\n");
 638                 if (end_pfn > MAX_DMA32_PFN) {
 639                         printk(KERN_ERR "WARNING more than 4GB of memory "
 640                                         "but IOMMU not compiled in.\n"
 641                                KERN_ERR "WARNING 32bit PCI may malfunction.\n"
 642                                KERN_ERR "You might want to enable "
 643                                         "CONFIG_GART_IOMMU\n");
 644                 }
 645                 return -1;
 646         }
 647
 648         printk(KERN_INFO "PCI-DMA: using GART IOMMU.\n");
 649         aper_size = info.aper_size * 1024 * 1024;
 650         iommu_size = check_iommu_size(info.aper_base, aper_size);
 651         iommu_pages = iommu_size >> PAGE_SHIFT;
 652
 653         iommu_gart_bitmap = (void*)__get_free_pages(GFP_KERNEL,
 654                                                     get_order(iommu_pages/8));
 655         if (!iommu_gart_bitmap)
 656                 panic("Cannot allocate iommu bitmap\n");
 657         memset(iommu_gart_bitmap, 0, iommu_pages/8);
 658
 659 #ifdef CONFIG_IOMMU_LEAK
 660         if (leak_trace) {
 661                 iommu_leak_tab = (void *)__get_free_pages(GFP_KERNEL,
 662                                   get_order(iommu_pages*sizeof(void *)));
 663                 if (iommu_leak_tab)
 664                         memset(iommu_leak_tab, 0, iommu_pages * 8);
 665                 else
 666                         printk("PCI-DMA: Cannot allocate leak trace area\n");
 667         }
 668 #endif
 669
 670         /*
 671          * Out of IOMMU space handling.
 672          * Reserve some invalid pages at the beginning of the GART.
 673          */
 674         set_bit_string(iommu_gart_bitmap, 0, EMERGENCY_PAGES);
 675
 676         agp_memory_reserved = iommu_size;
 677         printk(KERN_INFO
 678                "PCI-DMA: Reserving %luMB of IOMMU area in the AGP aperture\n",
 679                iommu_size>>20);
 680
 681         iommu_start = aper_size - iommu_size;
 682         iommu_bus_base = info.aper_base + iommu_start;
 683         bad_dma_address = iommu_bus_base;
 684         iommu_gatt_base = agp_gatt_table + (iommu_start>>PAGE_SHIFT);
 685
 686         /*
 687          * Unmap the IOMMU part of the GART. The alias of the page is
 688          * always mapped with cache enabled and there is no full cache
 689          * coherency across the GART remapping. The unmapping avoids
 690          * automatic prefetches from the CPU allocating cache lines in
 691          * there. All CPU accesses are done via the direct mapping to
 692          * the backing memory. The GART address is only used by PCI
 693          * devices.
 694          */
 695         clear_kernel_mapping((unsigned long)__va(iommu_bus_base), iommu_size);
 696
 697         /*
 698          * Try to workaround a bug (thanks to BenH)
 699          * Set unmapped entries to a scratch page instead of 0.
 700          * Any prefetches that hit unmapped entries won't get an bus abort
 701          * then.
 702          */
 703         scratch = get_zeroed_page(GFP_KERNEL);
 704         if (!scratch)
 705                 panic("Cannot allocate iommu scratch page");
 706         gart_unmapped_entry = GPTE_ENCODE(__pa(scratch));
 707         for (i = EMERGENCY_PAGES; i < iommu_pages; i++)
 708                 iommu_gatt_base[i] = gart_unmapped_entry;
 709
 710         for_all_nb(dev) {
 711                 u32 flag;
 712                 int cpu = PCI_SLOT(dev->devfn) - 24;
 713                 if (cpu >= MAX_NB)
 714                         continue;
 715                 northbridges[cpu] = dev;
 716                 pci_read_config_dword(dev, 0x9c, &flag); /* cache flush word */
 717                 northbridge_flush_word[cpu] = flag;
 718         }
 719
 720         flush_gart(NULL);
 721
 722         dma_ops = &gart_dma_ops;
 723
 724         return 0;
 725 }
 726
 727 /* Must execute after PCI subsystem */
 728 fs_initcall(pci_iommu_init);
 729
 730 void gart_parse_options(char *p)
 731 {
 732         int arg;
 733
 734 #ifdef CONFIG_IOMMU_LEAK
 735         if (!strncmp(p,"leak",4)) {
 736                 leak_trace = 1;
 737                 p += 4;
 738                 if (*p == '=') ++p;
 739                 if (isdigit(*p) && get_option(&p, &arg))
 740                         iommu_leak_pages = arg;
 741         }
 742 #endif
 743         if (isdigit(*p) && get_option(&p, &arg))
 744                 iommu_size = arg;
 745         if (!strncmp(p, "fullflush",8))
 746                 iommu_fullflush = 1;
 747         if (!strncmp(p, "nofullflush",11))
 748                 iommu_fullflush = 0;
 749         if (!strncmp(p,"noagp",5))
 750                 no_agp = 1;
 751         if (!strncmp(p, "noaperture",10))
 752                 fix_aperture = 0;
 753         /* duplicated from pci-dma.c */
 754         if (!strncmp(p,"force",5))
 755                 iommu_aperture_allowed = 1;
 756         if (!strncmp(p,"allowed",7))
 757                 iommu_aperture_allowed = 1;
 758         if (!strncmp(p, "memaper", 7)) {
 759                 fallback_aper_force = 1;
 760                 p += 7;
 761                 if (*p == '=') {
 762                         ++p;
 763                         if (get_option(&p, &arg))
 764                                 fallback_aper_order = arg;
 765                 }
 766         }
 767 }