arch/x86_64/kernel/pci-dma.c

   1 /*
   2  * Dynamic DMA mapping support.
   3  */
   4
   5 #include <linux/types.h>
   6 #include <linux/mm.h>
   7 #include <linux/string.h>
   8 #include <linux/pci.h>
   9 #include <linux/module.h>
  10 #include <asm/io.h>
  11 #include <asm/proto.h>
  12
  13 int iommu_merge __read_mostly = 0;
  14 EXPORT_SYMBOL(iommu_merge);
  15
  16 dma_addr_t bad_dma_address __read_mostly;
  17 EXPORT_SYMBOL(bad_dma_address);
  18
  19 /* This tells the BIO block layer to assume merging. Default to off
  20    because we cannot guarantee merging later. */
  21 int iommu_bio_merge __read_mostly = 0;
  22 EXPORT_SYMBOL(iommu_bio_merge);
  23
  24 int iommu_sac_force __read_mostly = 0;
  25 EXPORT_SYMBOL(iommu_sac_force);
  26
  27 int no_iommu __read_mostly;
  28 #ifdef CONFIG_IOMMU_DEBUG
  29 int panic_on_overflow __read_mostly = 1;
  30 int force_iommu __read_mostly = 1;
  31 #else
  32 int panic_on_overflow __read_mostly = 0;
  33 int force_iommu __read_mostly= 0;
  34 #endif
  35
  36 /* Dummy device used for NULL arguments (normally ISA). Better would
  37    be probably a smaller DMA mask, but this is bug-to-bug compatible
  38    to i386. */
  39 struct device fallback_dev = {
  40         .bus_id = "fallback device",
  41         .coherent_dma_mask = 0xffffffff,
  42         .dma_mask = &fallback_dev.coherent_dma_mask,
  43 };
  44
  45 /* Allocate DMA memory on node near device */
  46 noinline static void *
  47 dma_alloc_pages(struct device *dev, gfp_t gfp, unsigned order)
  48 {
  49         struct page *page;
  50         int node;
  51 #ifdef CONFIG_PCI
  52         if (dev->bus == &pci_bus_type)
  53                 node = pcibus_to_node(to_pci_dev(dev)->bus);
  54         else
  55 #endif
  56                 node = numa_node_id();
  57         page = alloc_pages_node(node, gfp, order);
  58         return page ? page_address(page) : NULL;
  59 }
  60
  61 /*
  62  * Allocate memory for a coherent mapping.
  63  */
  64 void *
  65 dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
  66                    gfp_t gfp)
  67 {
  68         void *memory;
  69         unsigned long dma_mask = 0;
  70         u64 bus;
  71
  72         if (!dev)
  73                 dev = &fallback_dev;
  74         dma_mask = dev->coherent_dma_mask;
  75         if (dma_mask == 0)
  76                 dma_mask = 0xffffffff;
  77
  78         /* Don't invoke OOM killer */
  79         gfp |= __GFP_NORETRY;
  80
  81         /* Kludge to make it bug-to-bug compatible with i386. i386
  82            uses the normal dma_mask for alloc_coherent. */
  83         dma_mask &= *dev->dma_mask;
  84
  85         /* Why <=? Even when the mask is smaller than 4GB it is often
  86            larger than 16MB and in this case we have a chance of
  87            finding fitting memory in the next higher zone first. If
  88            not retry with true GFP_DMA. -AK */
  89         if (dma_mask <= 0xffffffff)
  90                 gfp |= GFP_DMA32;
  91
  92  again:
  93         memory = dma_alloc_pages(dev, gfp, get_order(size));
  94         if (memory == NULL)
  95                 return NULL;
  96
  97         {
  98                 int high, mmu;
  99                 bus = virt_to_bus(memory);
 100                 high = (bus + size) >= dma_mask;
 101                 mmu = high;
 102                 if (force_iommu && !(gfp & GFP_DMA))
 103                         mmu = 1;
 104                 else if (high) {
 105                         free_pages((unsigned long)memory,
 106                                    get_order(size));
 107
 108                         /* Don't use the 16MB ZONE_DMA unless absolutely
 109                            needed. It's better to use remapping first. */
 110                         if (dma_mask < 0xffffffff && !(gfp & GFP_DMA)) {
 111                                 gfp = (gfp & ~GFP_DMA32) | GFP_DMA;
 112                                 goto again;
 113                         }
 114
 115                         /* Let low level make its own zone decisions */
 116                         gfp &= ~(GFP_DMA32|GFP_DMA);
 117
 118                         if (dma_ops->alloc_coherent)
 119                                 return dma_ops->alloc_coherent(dev, size,
 120                                                            dma_handle, gfp);
 121                         return NULL;
 122                 }
 123
 124                 memset(memory, 0, size);
 125                 if (!mmu) {
 126                         *dma_handle = virt_to_bus(memory);
 127                         return memory;
 128                 }
 129         }
 130
 131         if (dma_ops->alloc_coherent) {
 132                 free_pages((unsigned long)memory, get_order(size));
 133                 gfp &= ~(GFP_DMA|GFP_DMA32);
 134                 return dma_ops->alloc_coherent(dev, size, dma_handle, gfp);
 135         }
 136
 137         if (dma_ops->map_simple) {
 138                 *dma_handle = dma_ops->map_simple(dev, memory,
 139                                               size,
 140                                               PCI_DMA_BIDIRECTIONAL);
 141                 if (*dma_handle != bad_dma_address)
 142                         return memory;
 143         }
 144
 145         if (panic_on_overflow)
 146                 panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",size);
 147         free_pages((unsigned long)memory, get_order(size));
 148         return NULL;
 149 }
 150 EXPORT_SYMBOL(dma_alloc_coherent);
 151
 152 /*
 153  * Unmap coherent memory.
 154  * The caller must ensure that the device has finished accessing the mapping.
 155  */
 156 void dma_free_coherent(struct device *dev, size_t size,
 157                          void *vaddr, dma_addr_t bus)
 158 {
 159         if (dma_ops->unmap_single)
 160                 dma_ops->unmap_single(dev, bus, size, 0);
 161         free_pages((unsigned long)vaddr, get_order(size));
 162 }
 163 EXPORT_SYMBOL(dma_free_coherent);
 164
 165 int dma_supported(struct device *dev, u64 mask)
 166 {
 167         if (dma_ops->dma_supported)
 168                 return dma_ops->dma_supported(dev, mask);
 169
 170         /* Copied from i386. Doesn't make much sense, because it will
 171            only work for pci_alloc_coherent.
 172            The caller just has to use GFP_DMA in this case. */
 173         if (mask < 0x00ffffff)
 174                 return 0;
 175
 176         /* Tell the device to use SAC when IOMMU force is on.  This
 177            allows the driver to use cheaper accesses in some cases.
 178
 179            Problem with this is that if we overflow the IOMMU area and
 180            return DAC as fallback address the device may not handle it
 181            correctly.
 182
 183            As a special case some controllers have a 39bit address
 184            mode that is as efficient as 32bit (aic79xx). Don't force
 185            SAC for these.  Assume all masks <= 40 bits are of this
 186            type. Normally this doesn't make any difference, but gives
 187            more gentle handling of IOMMU overflow. */
 188         if (iommu_sac_force && (mask >= 0xffffffffffULL)) {
 189                 printk(KERN_INFO "%s: Force SAC with mask %Lx\n", dev->bus_id,mask);
 190                 return 0;
 191         }
 192
 193         return 1;
 194 }
 195 EXPORT_SYMBOL(dma_supported);
 196
 197 int dma_set_mask(struct device *dev, u64 mask)
 198 {
 199         if (!dev->dma_mask || !dma_supported(dev, mask))
 200                 return -EIO;
 201         *dev->dma_mask = mask;
 202         return 0;
 203 }
 204 EXPORT_SYMBOL(dma_set_mask);
 205
 206 /* iommu=[size][,noagp][,off][,force][,noforce][,leak][,memaper[=order]][,merge]
 207          [,forcesac][,fullflush][,nomerge][,biomerge]
 208    size  set size of iommu (in bytes)
 209    noagp don't initialize the AGP driver and use full aperture.
 210    off   don't use the IOMMU
 211    leak  turn on simple iommu leak tracing (only when CONFIG_IOMMU_LEAK is on)
 212    memaper[=order] allocate an own aperture over RAM with size 32MB^order.
 213    noforce don't force IOMMU usage. Default.
 214    force  Force IOMMU.
 215    merge  Do lazy merging. This may improve performance on some block devices.
 216           Implies force (experimental)
 217    biomerge Do merging at the BIO layer. This is more efficient than merge,
 218             but should be only done with very big IOMMUs. Implies merge,force.
 219    nomerge Don't do SG merging.
 220    forcesac For SAC mode for masks <40bits  (experimental)
 221    fullflush Flush IOMMU on each allocation (default)
 222    nofullflush Don't use IOMMU fullflush
 223    allowed  overwrite iommu off workarounds for specific chipsets.
 224    soft  Use software bounce buffering (default for Intel machines)
 225    noaperture Don't touch the aperture for AGP.
 226 */
 227 __init int iommu_setup(char *p)
 228 {
 229     iommu_merge = 1;
 230
 231     while (*p) {
 232             if (!strncmp(p,"off",3))
 233                     no_iommu = 1;
 234             /* gart_parse_options has more force support */
 235             if (!strncmp(p,"force",5))
 236                     force_iommu = 1;
 237             if (!strncmp(p,"noforce",7)) {
 238                     iommu_merge = 0;
 239                     force_iommu = 0;
 240             }
 241
 242             if (!strncmp(p, "biomerge",8)) {
 243                     iommu_bio_merge = 4096;
 244                     iommu_merge = 1;
 245                     force_iommu = 1;
 246             }
 247             if (!strncmp(p, "panic",5))
 248                     panic_on_overflow = 1;
 249             if (!strncmp(p, "nopanic",7))
 250                     panic_on_overflow = 0;
 251             if (!strncmp(p, "merge",5)) {
 252                     iommu_merge = 1;
 253                     force_iommu = 1;
 254             }
 255             if (!strncmp(p, "nomerge",7))
 256                     iommu_merge = 0;
 257             if (!strncmp(p, "forcesac",8))
 258                     iommu_sac_force = 1;
 259
 260 #ifdef CONFIG_SWIOTLB
 261             if (!strncmp(p, "soft",4))
 262                     swiotlb = 1;
 263 #endif
 264
 265 #ifdef CONFIG_GART_IOMMU
 266             gart_parse_options(p);
 267 #endif
 268
 269             p += strcspn(p, ",");
 270             if (*p == ',')
 271                     ++p;
 272     }
 273     return 1;
 274 }