arch/x86/kernel/pci-dma.c

   1 #include <linux/dma-mapping.h>
   2 #include <linux/dma-debug.h>
   3 #include <linux/dmar.h>
   4 #include <linux/bootmem.h>
   5 #include <linux/pci.h>
   6
   7 #include <asm/proto.h>
   8 #include <asm/dma.h>
   9 #include <asm/iommu.h>
  10 #include <asm/gart.h>
  11 #include <asm/calgary.h>
  12 #include <asm/amd_iommu.h>
  13
  14 static int forbid_dac __read_mostly;
  15
  16 struct dma_map_ops *dma_ops;
  17 EXPORT_SYMBOL(dma_ops);
  18
  19 static int iommu_sac_force __read_mostly;
  20
  21 #ifdef CONFIG_IOMMU_DEBUG
  22 int panic_on_overflow __read_mostly = 1;
  23 int force_iommu __read_mostly = 1;
  24 #else
  25 int panic_on_overflow __read_mostly = 0;
  26 int force_iommu __read_mostly = 0;
  27 #endif
  28
  29 int iommu_merge __read_mostly = 0;
  30
  31 int no_iommu __read_mostly;
  32 /* Set this to 1 if there is a HW IOMMU in the system */
  33 int iommu_detected __read_mostly = 0;
  34
  35 int iommu_pass_through;
  36
  37 dma_addr_t bad_dma_address __read_mostly = 0;
  38 EXPORT_SYMBOL(bad_dma_address);
  39
  40 /* Dummy device used for NULL arguments (normally ISA). Better would
  41    be probably a smaller DMA mask, but this is bug-to-bug compatible
  42    to older i386. */
  43 struct device x86_dma_fallback_dev = {
  44         .init_name = "fallback device",
  45         .coherent_dma_mask = DMA_BIT_MASK(32),
  46         .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
  47 };
  48 EXPORT_SYMBOL(x86_dma_fallback_dev);
  49
  50 /* Number of entries preallocated for DMA-API debugging */
  51 #define PREALLOC_DMA_DEBUG_ENTRIES       32768
  52
  53 int dma_set_mask(struct device *dev, u64 mask)
  54 {
  55         if (!dev->dma_mask || !dma_supported(dev, mask))
  56                 return -EIO;
  57
  58         *dev->dma_mask = mask;
  59
  60         return 0;
  61 }
  62 EXPORT_SYMBOL(dma_set_mask);
  63
  64 #ifdef CONFIG_X86_64
  65 static __initdata void *dma32_bootmem_ptr;
  66 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
  67
  68 static int __init parse_dma32_size_opt(char *p)
  69 {
  70         if (!p)
  71                 return -EINVAL;
  72         dma32_bootmem_size = memparse(p, &p);
  73         return 0;
  74 }
  75 early_param("dma32_size", parse_dma32_size_opt);
  76
  77 void __init dma32_reserve_bootmem(void)
  78 {
  79         unsigned long size, align;
  80         if (max_pfn <= MAX_DMA32_PFN)
  81                 return;
  82
  83         /*
  84          * check aperture_64.c allocate_aperture() for reason about
  85          * using 512M as goal
  86          */
  87         align = 64ULL<<20;
  88         size = roundup(dma32_bootmem_size, align);
  89         dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
  90                                  512ULL<<20);
  91         if (dma32_bootmem_ptr)
  92                 dma32_bootmem_size = size;
  93         else
  94                 dma32_bootmem_size = 0;
  95 }
  96 static void __init dma32_free_bootmem(void)
  97 {
  98
  99         if (max_pfn <= MAX_DMA32_PFN)
 100                 return;
 101
 102         if (!dma32_bootmem_ptr)
 103                 return;
 104
 105         free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
 106
 107         dma32_bootmem_ptr = NULL;
 108         dma32_bootmem_size = 0;
 109 }
 110 #endif
 111
 112 void __init pci_iommu_alloc(void)
 113 {
 114 #ifdef CONFIG_X86_64
 115         /* free the range so iommu could get some range less than 4G */
 116         dma32_free_bootmem();
 117 #endif
 118
 119         /*
 120          * The order of these functions is important for
 121          * fall-back/fail-over reasons
 122          */
 123         gart_iommu_hole_init();
 124
 125         detect_calgary();
 126
 127         detect_intel_iommu();
 128
 129         amd_iommu_detect();
 130
 131         pci_swiotlb_init();
 132 }
 133
 134 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
 135                                  dma_addr_t *dma_addr, gfp_t flag)
 136 {
 137         unsigned long dma_mask;
 138         struct page *page;
 139         dma_addr_t addr;
 140
 141         dma_mask = dma_alloc_coherent_mask(dev, flag);
 142
 143         flag |= __GFP_ZERO;
 144 again:
 145         page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
 146         if (!page)
 147                 return NULL;
 148
 149         addr = page_to_phys(page);
 150         if (!is_buffer_dma_capable(dma_mask, addr, size)) {
 151                 __free_pages(page, get_order(size));
 152
 153                 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
 154                         flag = (flag & ~GFP_DMA32) | GFP_DMA;
 155                         goto again;
 156                 }
 157
 158                 return NULL;
 159         }
 160
 161         *dma_addr = addr;
 162         return page_address(page);
 163 }
 164
 165 /*
 166  * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 167  * documentation.
 168  */
 169 static __init int iommu_setup(char *p)
 170 {
 171         iommu_merge = 1;
 172
 173         if (!p)
 174                 return -EINVAL;
 175
 176         while (*p) {
 177                 if (!strncmp(p, "off", 3))
 178                         no_iommu = 1;
 179                 /* gart_parse_options has more force support */
 180                 if (!strncmp(p, "force", 5))
 181                         force_iommu = 1;
 182                 if (!strncmp(p, "noforce", 7)) {
 183                         iommu_merge = 0;
 184                         force_iommu = 0;
 185                 }
 186
 187                 if (!strncmp(p, "biomerge", 8)) {
 188                         iommu_merge = 1;
 189                         force_iommu = 1;
 190                 }
 191                 if (!strncmp(p, "panic", 5))
 192                         panic_on_overflow = 1;
 193                 if (!strncmp(p, "nopanic", 7))
 194                         panic_on_overflow = 0;
 195                 if (!strncmp(p, "merge", 5)) {
 196                         iommu_merge = 1;
 197                         force_iommu = 1;
 198                 }
 199                 if (!strncmp(p, "nomerge", 7))
 200                         iommu_merge = 0;
 201                 if (!strncmp(p, "forcesac", 8))
 202                         iommu_sac_force = 1;
 203                 if (!strncmp(p, "allowdac", 8))
 204                         forbid_dac = 0;
 205                 if (!strncmp(p, "nodac", 5))
 206                         forbid_dac = -1;
 207                 if (!strncmp(p, "usedac", 6)) {
 208                         forbid_dac = -1;
 209                         return 1;
 210                 }
 211 #ifdef CONFIG_SWIOTLB
 212                 if (!strncmp(p, "soft", 4))
 213                         swiotlb = 1;
 214 #endif
 215                 if (!strncmp(p, "pt", 2))
 216                         iommu_pass_through = 1;
 217
 218                 gart_parse_options(p);
 219
 220 #ifdef CONFIG_CALGARY_IOMMU
 221                 if (!strncmp(p, "calgary", 7))
 222                         use_calgary = 1;
 223 #endif /* CONFIG_CALGARY_IOMMU */
 224
 225                 p += strcspn(p, ",");
 226                 if (*p == ',')
 227                         ++p;
 228         }
 229         return 0;
 230 }
 231 early_param("iommu", iommu_setup);
 232
 233 int dma_supported(struct device *dev, u64 mask)
 234 {
 235         struct dma_map_ops *ops = get_dma_ops(dev);
 236
 237 #ifdef CONFIG_PCI
 238         if (mask > 0xffffffff && forbid_dac > 0) {
 239                 dev_info(dev, "PCI: Disallowing DAC for device\n");
 240                 return 0;
 241         }
 242 #endif
 243
 244         if (ops->dma_supported)
 245                 return ops->dma_supported(dev, mask);
 246
 247         /* Copied from i386. Doesn't make much sense, because it will
 248            only work for pci_alloc_coherent.
 249            The caller just has to use GFP_DMA in this case. */
 250         if (mask < DMA_BIT_MASK(24))
 251                 return 0;
 252
 253         /* Tell the device to use SAC when IOMMU force is on.  This
 254            allows the driver to use cheaper accesses in some cases.
 255
 256            Problem with this is that if we overflow the IOMMU area and
 257            return DAC as fallback address the device may not handle it
 258            correctly.
 259
 260            As a special case some controllers have a 39bit address
 261            mode that is as efficient as 32bit (aic79xx). Don't force
 262            SAC for these.  Assume all masks <= 40 bits are of this
 263            type. Normally this doesn't make any difference, but gives
 264            more gentle handling of IOMMU overflow. */
 265         if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
 266                 dev_info(dev, "Force SAC with mask %Lx\n", mask);
 267                 return 0;
 268         }
 269
 270         return 1;
 271 }
 272 EXPORT_SYMBOL(dma_supported);
 273
 274 static int __init pci_iommu_init(void)
 275 {
 276         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 277
 278 #ifdef CONFIG_PCI
 279         dma_debug_add_bus(&pci_bus_type);
 280 #endif
 281
 282         calgary_iommu_init();
 283
 284         intel_iommu_init();
 285
 286         amd_iommu_init();
 287
 288         gart_iommu_init();
 289
 290         no_iommu_init();
 291         return 0;
 292 }
 293
 294 void pci_iommu_shutdown(void)
 295 {
 296         gart_iommu_shutdown();
 297
 298         amd_iommu_shutdown();
 299 }
 300 /* Must execute after PCI subsystem */
 301 fs_initcall(pci_iommu_init);
 302
 303 #ifdef CONFIG_PCI
 304 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
 305
 306 static __devinit void via_no_dac(struct pci_dev *dev)
 307 {
 308         if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
 309                 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
 310                 forbid_dac = 1;
 311         }
 312 }
 313 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
 314 #endif