arch/x86_64/kernel/pci-dma.c

   1 /*
   2  * Dynamic DMA mapping support.
   3  */
   4
   5 #include <linux/types.h>
   6 #include <linux/mm.h>
   7 #include <linux/string.h>
   8 #include <linux/pci.h>
   9 #include <linux/module.h>
  10 #include <asm/io.h>
  11 #include <asm/proto.h>
  12 #include <asm/calgary.h>
  13
  14 int iommu_merge __read_mostly = 0;
  15 EXPORT_SYMBOL(iommu_merge);
  16
  17 dma_addr_t bad_dma_address __read_mostly;
  18 EXPORT_SYMBOL(bad_dma_address);
  19
  20 /* This tells the BIO block layer to assume merging. Default to off
  21    because we cannot guarantee merging later. */
  22 int iommu_bio_merge __read_mostly = 0;
  23 EXPORT_SYMBOL(iommu_bio_merge);
  24
  25 int iommu_sac_force __read_mostly = 0;
  26 EXPORT_SYMBOL(iommu_sac_force);
  27
  28 int no_iommu __read_mostly;
  29 #ifdef CONFIG_IOMMU_DEBUG
  30 int panic_on_overflow __read_mostly = 1;
  31 int force_iommu __read_mostly = 1;
  32 #else
  33 int panic_on_overflow __read_mostly = 0;
  34 int force_iommu __read_mostly= 0;
  35 #endif
  36
  37 /* Set this to 1 if there is a HW IOMMU in the system */
  38 int iommu_detected __read_mostly = 0;
  39
  40 /* Dummy device used for NULL arguments (normally ISA). Better would
  41    be probably a smaller DMA mask, but this is bug-to-bug compatible
  42    to i386. */
  43 struct device fallback_dev = {
  44         .bus_id = "fallback device",
  45         .coherent_dma_mask = DMA_32BIT_MASK,
  46         .dma_mask = &fallback_dev.coherent_dma_mask,
  47 };
  48
  49 /* Allocate DMA memory on node near device */
  50 noinline static void *
  51 dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
  52 {
  53         struct page *page;
  54         int node;
  55 #ifdef CONFIG_PCI
  56         if (dev->bus == &pci_bus_type)
  57                 node = pcibus_to_node(to_pci_dev(dev)->bus);
  58         else
  59 #endif
  60                 node = numa_node_id();
  61
  62         if (node < first_node(node_online_map))
  63                 node = first_node(node_online_map);
  64
  65         page = alloc_pages_node(node, gfp, order);
  66         return page ? page_address(page) : NULL;
  67 }
  68
  69 /*
  70  * Allocate memory for a coherent mapping.
  71  */
  72 void *
  73 dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
  74                    gfp_t gfp)
  75 {
  76         void *memory;
  77         unsigned long dma_mask = 0;
  78         u64 bus;
  79
  80         if (!dev)
  81                 dev = &fallback_dev;
  82         dma_mask = dev->coherent_dma_mask;
  83         if (dma_mask == 0)
  84                 dma_mask = DMA_32BIT_MASK;
  85
  86         /* Don't invoke OOM killer */
  87         gfp |= __GFP_NORETRY;
  88
  89         /* Kludge to make it bug-to-bug compatible with i386. i386
  90            uses the normal dma_mask for alloc_coherent. */
  91         dma_mask &= *dev->dma_mask;
  92
  93         /* Why <=? Even when the mask is smaller than 4GB it is often
  94            larger than 16MB and in this case we have a chance of
  95            finding fitting memory in the next higher zone first. If
  96            not retry with true GFP_DMA. -AK */
  97         if (dma_mask <= DMA_32BIT_MASK)
  98                 gfp |= GFP_DMA32;
  99
 100  again:
 101         memory = dma_alloc_pages(dev, gfp, get_order(size));
 102         if (memory == NULL)
 103                 return NULL;
 104
 105         {
 106                 int high, mmu;
 107                 bus = virt_to_bus(memory);
 108                 high = (bus + size) >= dma_mask;
 109                 mmu = high;
 110                 if (force_iommu && !(gfp & GFP_DMA))
 111                         mmu = 1;
 112                 else if (high) {
 113                         free_pages((unsigned long)memory,
 114                                    get_order(size));
 115
 116                         /* Don't use the 16MB ZONE_DMA unless absolutely
 117                            needed. It's better to use remapping first. */
 118                         if (dma_mask < DMA_32BIT_MASK && !(gfp & GFP_DMA)) {
 119                                 gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
 120                                 goto again;
 121                         }
 122
 123                         /* Let low level make its own zone decisions */
 124                         gfp &= ~(GFP_DMA32|GFP_DMA);
 125
 126                         if (dma_ops->alloc_coherent)
 127                                 return dma_ops->alloc_coherent(dev, size,
 128                                                            dma_handle, gfp);
 129                         return NULL;
 130                 }
 131
 132                 memset(memory, 0, size);
 133                 if (!mmu) {
 134                         *dma_handle = virt_to_bus(memory);
 135                         return memory;
 136                 }
 137         }
 138
 139         if (dma_ops->alloc_coherent) {
 140                 free_pages((unsigned long)memory, get_order(size));
 141                 gfp &= ~(GFP_DMA|GFP_DMA32);
 142                 return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
 143         }
 144
 145         if (dma_ops->map_simple) {
 146                 *dma_handle = dma_ops->map_simple(dev, memory,
 147                                               size,
 148                                               PCI_DMA_BIDIRECTIONAL);
 149                 if (*dma_handle != bad_dma_address)
 150                         return memory;
 151         }
 152
 153         if (panic_on_overflow)
 154                 panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",size);
 155         free_pages((unsigned long)memory, get_order(size));
 156         return NULL;
 157 }
 158 EXPORT_SYMBOL(dma_alloc_coherent);
 159
 160 /*
 161  * Unmap coherent memory.
 162  * The caller must ensure that the device has finished accessing the mapping.
 163  */
 164 void dma_free_coherent(struct device *dev, size_t size,
 165                          void *vaddr, dma_addr_t bus)
 166 {
 167         if (dma_ops->unmap_single)
 168                 dma_ops->unmap_single(dev, bus, size, 0);
 169         free_pages((unsigned long)vaddr, get_order(size));
 170 }
 171 EXPORT_SYMBOL(dma_free_coherent);
 172
 173 int dma_supported(struct device *dev, u64 mask)
 174 {
 175         if (dma_ops->dma_supported)
 176                 return dma_ops->dma_supported(dev, mask);
 177
 178         /* Copied from i386. Doesn't make much sense, because it will
 179            only work for pci_alloc_coherent.
 180            The caller just has to use GFP_DMA in this case. */
 181         if (mask < DMA_24BIT_MASK)
 182                 return 0;
 183
 184         /* Tell the device to use SAC when IOMMU force is on.  This
 185            allows the driver to use cheaper accesses in some cases.
 186
 187            Problem with this is that if we overflow the IOMMU area and
 188            return DAC as fallback address the device may not handle it
 189            correctly.
 190
 191            As a special case some controllers have a 39bit address
 192            mode that is as efficient as 32bit (aic79xx). Don't force
 193            SAC for these.  Assume all masks <= 40 bits are of this
 194            type. Normally this doesn't make any difference, but gives
 195            more gentle handling of IOMMU overflow. */
 196         if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
 197                 printk(KERN_INFO "%s: Force SAC with mask %Lx\n", dev->bus_id,mask);
 198                 return 0;
 199         }
 200
 201         return 1;
 202 }
 203 EXPORT_SYMBOL(dma_supported);
 204
 205 int dma_set_mask(struct device *dev, u64 mask)
 206 {
 207         if (!dev->dma_mask || !dma_supported(dev, mask))
 208                 return -EIO;
 209         *dev->dma_mask = mask;
 210         return 0;
 211 }
 212 EXPORT_SYMBOL(dma_set_mask);
 213
 214 /* iommu=[size][,noagp][,off][,force][,noforce][,leak][,memaper[=order]][,merge]
 215          [,forcesac][,fullflush][,nomerge][,biomerge]
 216    size  set size of iommu (in bytes)
 217    noagp don't initialize the AGP driver and use full aperture.
 218    off   don't use the IOMMU
 219    leak  turn on simple iommu leak tracing (only when CONFIG_IOMMU_LEAK is on)
 220    memaper[=order] allocate an own aperture over RAM with size 32MB^order.
 221    noforce don't force IOMMU usage. Default.
 222    force  Force IOMMU.
 223    merge  Do lazy merging. This may improve performance on some block devices.
 224           Implies force (experimental)
 225    biomerge Do merging at the BIO layer. This is more efficient than merge,
 226             but should be only done with very big IOMMUs. Implies merge,force.
 227    nomerge Don't do SG merging.
 228    forcesac For SAC mode for masks <40bits  (experimental)
 229    fullflush Flush IOMMU on each allocation (default)
 230    nofullflush Don't use IOMMU fullflush
 231    allowed  overwrite iommu off workarounds for specific chipsets.
 232    soft  Use software bounce buffering (default for Intel machines)
 233    noaperture Don't touch the aperture for AGP.
 234 */
 235 __init int iommu_setup(char *p)
 236 {
 237     iommu_merge = 1;
 238
 239         if (!p)
 240                 return -EINVAL;
 241
 242     while (*p) {
 243             if (!strncmp(p,"off",3))
 244                     no_iommu = 1;
 245             /* gart_parse_options has more force support */
 246             if (!strncmp(p,"force",5))
 247                     force_iommu = 1;
 248             if (!strncmp(p,"noforce",7)) {
 249                     iommu_merge = 0;
 250                     force_iommu = 0;
 251             }
 252
 253             if (!strncmp(p, "biomerge",8)) {
 254                     iommu_bio_merge = 4096;
 255                     iommu_merge = 1;
 256                     force_iommu = 1;
 257             }
 258             if (!strncmp(p, "panic",5))
 259                     panic_on_overflow = 1;
 260             if (!strncmp(p, "nopanic",7))
 261                     panic_on_overflow = 0;
 262             if (!strncmp(p, "merge",5)) {
 263                     iommu_merge = 1;
 264                     force_iommu = 1;
 265             }
 266             if (!strncmp(p, "nomerge",7))
 267                     iommu_merge = 0;
 268             if (!strncmp(p, "forcesac",8))
 269                     iommu_sac_force = 1;
 270
 271 #ifdef CONFIG_SWIOTLB
 272             if (!strncmp(p, "soft",4))
 273                     swiotlb = 1;
 274 #endif
 275
 276 #ifdef CONFIG_IOMMU
 277             gart_parse_options(p);
 278 #endif
 279
 280             p += strcspn(p, ",");
 281             if (*p == ',')
 282                     ++p;
 283     }
 284     return 0;
 285 }
 286 early_param("iommu", iommu_setup);
 287
 288 void __init pci_iommu_alloc(void)
 289 {
 290         /*
 291          * The order of these functions is important for
 292          * fall-back/fail-over reasons
 293          */
 294 #ifdef CONFIG_IOMMU
 295         iommu_hole_init();
 296 #endif
 297
 298 #ifdef CONFIG_CALGARY_IOMMU
 299         detect_calgary();
 300 #endif
 301
 302 #ifdef CONFIG_SWIOTLB
 303         pci_swiotlb_init();
 304 #endif
 305 }
 306
 307 static int __init pci_iommu_init(void)
 308 {
 309 #ifdef CONFIG_CALGARY_IOMMU
 310         calgary_iommu_init();
 311 #endif
 312
 313 #ifdef CONFIG_IOMMU
 314         gart_iommu_init();
 315 #endif
 316
 317         no_iommu_init();
 318         return 0;
 319 }
 320
 321 /* Must execute after PCI subsystem */
 322 fs_initcall(pci_iommu_init);