arch/x86/kernel/pci-dma.c

   1 #include <linux/dma-mapping.h>
   2 #include <linux/dma-debug.h>
   3 #include <linux/dmar.h>
   4 #include <linux/bootmem.h>
   5 #include <linux/pci.h>
   6 #include <linux/kmemleak.h>
   7
   8 #include <asm/proto.h>
   9 #include <asm/dma.h>
  10 #include <asm/iommu.h>
  11 #include <asm/gart.h>
  12 #include <asm/calgary.h>
  13 #include <asm/amd_iommu.h>
  14 #include <asm/x86_init.h>
  15
  16 static int forbid_dac __read_mostly;
  17
  18 struct dma_map_ops *dma_ops = &nommu_dma_ops;
  19 EXPORT_SYMBOL(dma_ops);
  20
  21 static int iommu_sac_force __read_mostly;
  22
  23 #ifdef CONFIG_IOMMU_DEBUG
  24 int panic_on_overflow __read_mostly = 1;
  25 int force_iommu __read_mostly = 1;
  26 #else
  27 int panic_on_overflow __read_mostly = 0;
  28 int force_iommu __read_mostly = 0;
  29 #endif
  30
  31 int iommu_merge __read_mostly = 0;
  32
  33 int no_iommu __read_mostly;
  34 /* Set this to 1 if there is a HW IOMMU in the system */
  35 int iommu_detected __read_mostly = 0;
  36
  37 /*
  38  * This variable becomes 1 if iommu=pt is passed on the kernel command line.
  39  * If this variable is 1, IOMMU implementations do no DMA translation for
  40  * devices and allow every device to access to whole physical memory. This is
  41  * useful if a user want to use an IOMMU only for KVM device assignment to
  42  * guests and not for driver dma translation.
  43  */
  44 int iommu_pass_through __read_mostly;
  45
  46 /* Dummy device used for NULL arguments (normally ISA). */
  47 struct device x86_dma_fallback_dev = {
  48         .init_name = "fallback device",
  49         .coherent_dma_mask = ISA_DMA_BIT_MASK,
  50         .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
  51 };
  52 EXPORT_SYMBOL(x86_dma_fallback_dev);
  53
  54 /* Number of entries preallocated for DMA-API debugging */
  55 #define PREALLOC_DMA_DEBUG_ENTRIES       32768
  56
  57 int dma_set_mask(struct device *dev, u64 mask)
  58 {
  59         if (!dev->dma_mask || !dma_supported(dev, mask))
  60                 return -EIO;
  61
  62         *dev->dma_mask = mask;
  63
  64         return 0;
  65 }
  66 EXPORT_SYMBOL(dma_set_mask);
  67
  68 #ifdef CONFIG_X86_64
  69 static __initdata void *dma32_bootmem_ptr;
  70 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
  71
  72 static int __init parse_dma32_size_opt(char *p)
  73 {
  74         if (!p)
  75                 return -EINVAL;
  76         dma32_bootmem_size = memparse(p, &p);
  77         return 0;
  78 }
  79 early_param("dma32_size", parse_dma32_size_opt);
  80
  81 void __init dma32_reserve_bootmem(void)
  82 {
  83         unsigned long size, align;
  84         if (max_pfn <= MAX_DMA32_PFN)
  85                 return;
  86
  87         /*
  88          * check aperture_64.c allocate_aperture() for reason about
  89          * using 512M as goal
  90          */
  91         align = 64ULL<<20;
  92         size = roundup(dma32_bootmem_size, align);
  93         dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
  94                                  512ULL<<20);
  95         /*
  96          * Kmemleak should not scan this block as it may not be mapped via the
  97          * kernel direct mapping.
  98          */
  99         kmemleak_ignore(dma32_bootmem_ptr);
 100         if (dma32_bootmem_ptr)
 101                 dma32_bootmem_size = size;
 102         else
 103                 dma32_bootmem_size = 0;
 104 }
 105 static void __init dma32_free_bootmem(void)
 106 {
 107
 108         if (max_pfn <= MAX_DMA32_PFN)
 109                 return;
 110
 111         if (!dma32_bootmem_ptr)
 112                 return;
 113
 114         free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
 115
 116         dma32_bootmem_ptr = NULL;
 117         dma32_bootmem_size = 0;
 118 }
 119 #endif
 120
 121 void __init pci_iommu_alloc(void)
 122 {
 123 #ifdef CONFIG_X86_64
 124         /* free the range so iommu could get some range less than 4G */
 125         dma32_free_bootmem();
 126 #endif
 127         if (pci_swiotlb_detect())
 128                 goto out;
 129
 130         gart_iommu_hole_init();
 131
 132         detect_calgary();
 133
 134         detect_intel_iommu();
 135
 136         /* needs to be called after gart_iommu_hole_init */
 137         amd_iommu_detect();
 138 out:
 139         pci_swiotlb_init();
 140 }
 141
 142 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
 143                                  dma_addr_t *dma_addr, gfp_t flag)
 144 {
 145         unsigned long dma_mask;
 146         struct page *page;
 147         dma_addr_t addr;
 148
 149         dma_mask = dma_alloc_coherent_mask(dev, flag);
 150
 151         flag |= __GFP_ZERO;
 152 again:
 153         page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
 154         if (!page)
 155                 return NULL;
 156
 157         addr = page_to_phys(page);
 158         if (addr + size > dma_mask) {
 159                 __free_pages(page, get_order(size));
 160
 161                 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
 162                         flag = (flag & ~GFP_DMA32) | GFP_DMA;
 163                         goto again;
 164                 }
 165
 166                 return NULL;
 167         }
 168
 169         *dma_addr = addr;
 170         return page_address(page);
 171 }
 172
 173 /*
 174  * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 175  * documentation.
 176  */
 177 static __init int iommu_setup(char *p)
 178 {
 179         iommu_merge = 1;
 180
 181         if (!p)
 182                 return -EINVAL;
 183
 184         while (*p) {
 185                 if (!strncmp(p, "off", 3))
 186                         no_iommu = 1;
 187                 /* gart_parse_options has more force support */
 188                 if (!strncmp(p, "force", 5))
 189                         force_iommu = 1;
 190                 if (!strncmp(p, "noforce", 7)) {
 191                         iommu_merge = 0;
 192                         force_iommu = 0;
 193                 }
 194
 195                 if (!strncmp(p, "biomerge", 8)) {
 196                         iommu_merge = 1;
 197                         force_iommu = 1;
 198                 }
 199                 if (!strncmp(p, "panic", 5))
 200                         panic_on_overflow = 1;
 201                 if (!strncmp(p, "nopanic", 7))
 202                         panic_on_overflow = 0;
 203                 if (!strncmp(p, "merge", 5)) {
 204                         iommu_merge = 1;
 205                         force_iommu = 1;
 206                 }
 207                 if (!strncmp(p, "nomerge", 7))
 208                         iommu_merge = 0;
 209                 if (!strncmp(p, "forcesac", 8))
 210                         iommu_sac_force = 1;
 211                 if (!strncmp(p, "allowdac", 8))
 212                         forbid_dac = 0;
 213                 if (!strncmp(p, "nodac", 5))
 214                         forbid_dac = 1;
 215                 if (!strncmp(p, "usedac", 6)) {
 216                         forbid_dac = -1;
 217                         return 1;
 218                 }
 219 #ifdef CONFIG_SWIOTLB
 220                 if (!strncmp(p, "soft", 4))
 221                         swiotlb = 1;
 222 #endif
 223                 if (!strncmp(p, "pt", 2))
 224                         iommu_pass_through = 1;
 225
 226                 gart_parse_options(p);
 227
 228 #ifdef CONFIG_CALGARY_IOMMU
 229                 if (!strncmp(p, "calgary", 7))
 230                         use_calgary = 1;
 231 #endif /* CONFIG_CALGARY_IOMMU */
 232
 233                 p += strcspn(p, ",");
 234                 if (*p == ',')
 235                         ++p;
 236         }
 237         return 0;
 238 }
 239 early_param("iommu", iommu_setup);
 240
 241 int dma_supported(struct device *dev, u64 mask)
 242 {
 243         struct dma_map_ops *ops = get_dma_ops(dev);
 244
 245 #ifdef CONFIG_PCI
 246         if (mask > 0xffffffff && forbid_dac > 0) {
 247                 dev_info(dev, "PCI: Disallowing DAC for device\n");
 248                 return 0;
 249         }
 250 #endif
 251
 252         if (ops->dma_supported)
 253                 return ops->dma_supported(dev, mask);
 254
 255         /* Copied from i386. Doesn't make much sense, because it will
 256            only work for pci_alloc_coherent.
 257            The caller just has to use GFP_DMA in this case. */
 258         if (mask < DMA_BIT_MASK(24))
 259                 return 0;
 260
 261         /* Tell the device to use SAC when IOMMU force is on.  This
 262            allows the driver to use cheaper accesses in some cases.
 263
 264            Problem with this is that if we overflow the IOMMU area and
 265            return DAC as fallback address the device may not handle it
 266            correctly.
 267
 268            As a special case some controllers have a 39bit address
 269            mode that is as efficient as 32bit (aic79xx). Don't force
 270            SAC for these.  Assume all masks <= 40 bits are of this
 271            type. Normally this doesn't make any difference, but gives
 272            more gentle handling of IOMMU overflow. */
 273         if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
 274                 dev_info(dev, "Force SAC with mask %Lx\n", mask);
 275                 return 0;
 276         }
 277
 278         return 1;
 279 }
 280 EXPORT_SYMBOL(dma_supported);
 281
 282 static int __init pci_iommu_init(void)
 283 {
 284         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 285
 286 #ifdef CONFIG_PCI
 287         dma_debug_add_bus(&pci_bus_type);
 288 #endif
 289         x86_init.iommu.iommu_init();
 290
 291         if (swiotlb) {
 292                 printk(KERN_INFO "PCI-DMA: "
 293                        "Using software bounce buffering for IO (SWIOTLB)\n");
 294                 swiotlb_print_info();
 295         } else
 296                 swiotlb_free();
 297
 298         return 0;
 299 }
 300 /* Must execute after PCI subsystem */
 301 rootfs_initcall(pci_iommu_init);
 302
 303 #ifdef CONFIG_PCI
 304 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
 305
 306 static __devinit void via_no_dac(struct pci_dev *dev)
 307 {
 308         if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
 309                 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
 310                 forbid_dac = 1;
 311         }
 312 }
 313 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
 314 #endif