arch/x86/kernel/pci-dma.c

   1 #include <linux/dma-mapping.h>
   2 #include <linux/dma-debug.h>
   3 #include <linux/dmar.h>
   4 #include <linux/bootmem.h>
   5 #include <linux/pci.h>
   6 #include <linux/kmemleak.h>
   7
   8 #include <asm/proto.h>
   9 #include <asm/dma.h>
  10 #include <asm/iommu.h>
  11 #include <asm/gart.h>
  12 #include <asm/calgary.h>
  13 #include <asm/amd_iommu.h>
  14 #include <asm/x86_init.h>
  15
  16 static int forbid_dac __read_mostly;
  17
  18 struct dma_map_ops *dma_ops = &nommu_dma_ops;
  19 EXPORT_SYMBOL(dma_ops);
  20
  21 static int iommu_sac_force __read_mostly;
  22
  23 #ifdef CONFIG_IOMMU_DEBUG
  24 int panic_on_overflow __read_mostly = 1;
  25 int force_iommu __read_mostly = 1;
  26 #else
  27 int panic_on_overflow __read_mostly = 0;
  28 int force_iommu __read_mostly = 0;
  29 #endif
  30
  31 int iommu_merge __read_mostly = 0;
  32
  33 int no_iommu __read_mostly;
  34 /* Set this to 1 if there is a HW IOMMU in the system */
  35 int iommu_detected __read_mostly = 0;
  36
  37 /*
  38  * This variable becomes 1 if iommu=pt is passed on the kernel command line.
  39  * If this variable is 1, IOMMU implementations do no DMA translation for
  40  * devices and allow every device to access to whole physical memory. This is
  41  * useful if a user want to use an IOMMU only for KVM device assignment to
  42  * guests and not for driver dma translation.
  43  */
  44 int iommu_pass_through __read_mostly;
  45
  46 /* Dummy device used for NULL arguments (normally ISA). */
  47 struct device x86_dma_fallback_dev = {
  48         .init_name = "fallback device",
  49         .coherent_dma_mask = ISA_DMA_BIT_MASK,
  50         .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
  51 };
  52 EXPORT_SYMBOL(x86_dma_fallback_dev);
  53
  54 /* Number of entries preallocated for DMA-API debugging */
  55 #define PREALLOC_DMA_DEBUG_ENTRIES       32768
  56
  57 int dma_set_mask(struct device *dev, u64 mask)
  58 {
  59         if (!dev->dma_mask || !dma_supported(dev, mask))
  60                 return -EIO;
  61
  62         *dev->dma_mask = mask;
  63
  64         return 0;
  65 }
  66 EXPORT_SYMBOL(dma_set_mask);
  67
  68 #ifdef CONFIG_X86_64
  69 static __initdata void *dma32_bootmem_ptr;
  70 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
  71
  72 static int __init parse_dma32_size_opt(char *p)
  73 {
  74         if (!p)
  75                 return -EINVAL;
  76         dma32_bootmem_size = memparse(p, &p);
  77         return 0;
  78 }
  79 early_param("dma32_size", parse_dma32_size_opt);
  80
  81 void __init dma32_reserve_bootmem(void)
  82 {
  83         unsigned long size, align;
  84         if (max_pfn <= MAX_DMA32_PFN)
  85                 return;
  86
  87         /*
  88          * check aperture_64.c allocate_aperture() for reason about
  89          * using 512M as goal
  90          */
  91         align = 64ULL<<20;
  92         size = roundup(dma32_bootmem_size, align);
  93         dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
  94                                  512ULL<<20);
  95         /*
  96          * Kmemleak should not scan this block as it may not be mapped via the
  97          * kernel direct mapping.
  98          */
  99         kmemleak_ignore(dma32_bootmem_ptr);
 100         if (dma32_bootmem_ptr)
 101                 dma32_bootmem_size = size;
 102         else
 103                 dma32_bootmem_size = 0;
 104 }
 105 static void __init dma32_free_bootmem(void)
 106 {
 107
 108         if (max_pfn <= MAX_DMA32_PFN)
 109                 return;
 110
 111         if (!dma32_bootmem_ptr)
 112                 return;
 113
 114         free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
 115
 116         dma32_bootmem_ptr = NULL;
 117         dma32_bootmem_size = 0;
 118 }
 119 #endif
 120
 121 void __init pci_iommu_alloc(void)
 122 {
 123 #ifdef CONFIG_X86_64
 124         /* free the range so iommu could get some range less than 4G */
 125         dma32_free_bootmem();
 126 #endif
 127         if (pci_swiotlb_init())
 128                 return;
 129
 130         gart_iommu_hole_init();
 131
 132         detect_calgary();
 133
 134         detect_intel_iommu();
 135
 136         /* needs to be called after gart_iommu_hole_init */
 137         amd_iommu_detect();
 138 }
 139
 140 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
 141                                  dma_addr_t *dma_addr, gfp_t flag)
 142 {
 143         unsigned long dma_mask;
 144         struct page *page;
 145         dma_addr_t addr;
 146
 147         dma_mask = dma_alloc_coherent_mask(dev, flag);
 148
 149         flag |= __GFP_ZERO;
 150 again:
 151         page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
 152         if (!page)
 153                 return NULL;
 154
 155         addr = page_to_phys(page);
 156         if (addr + size > dma_mask) {
 157                 __free_pages(page, get_order(size));
 158
 159                 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
 160                         flag = (flag & ~GFP_DMA32) | GFP_DMA;
 161                         goto again;
 162                 }
 163
 164                 return NULL;
 165         }
 166
 167         *dma_addr = addr;
 168         return page_address(page);
 169 }
 170
 171 /*
 172  * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 173  * documentation.
 174  */
 175 static __init int iommu_setup(char *p)
 176 {
 177         iommu_merge = 1;
 178
 179         if (!p)
 180                 return -EINVAL;
 181
 182         while (*p) {
 183                 if (!strncmp(p, "off", 3))
 184                         no_iommu = 1;
 185                 /* gart_parse_options has more force support */
 186                 if (!strncmp(p, "force", 5))
 187                         force_iommu = 1;
 188                 if (!strncmp(p, "noforce", 7)) {
 189                         iommu_merge = 0;
 190                         force_iommu = 0;
 191                 }
 192
 193                 if (!strncmp(p, "biomerge", 8)) {
 194                         iommu_merge = 1;
 195                         force_iommu = 1;
 196                 }
 197                 if (!strncmp(p, "panic", 5))
 198                         panic_on_overflow = 1;
 199                 if (!strncmp(p, "nopanic", 7))
 200                         panic_on_overflow = 0;
 201                 if (!strncmp(p, "merge", 5)) {
 202                         iommu_merge = 1;
 203                         force_iommu = 1;
 204                 }
 205                 if (!strncmp(p, "nomerge", 7))
 206                         iommu_merge = 0;
 207                 if (!strncmp(p, "forcesac", 8))
 208                         iommu_sac_force = 1;
 209                 if (!strncmp(p, "allowdac", 8))
 210                         forbid_dac = 0;
 211                 if (!strncmp(p, "nodac", 5))
 212                         forbid_dac = 1;
 213                 if (!strncmp(p, "usedac", 6)) {
 214                         forbid_dac = -1;
 215                         return 1;
 216                 }
 217 #ifdef CONFIG_SWIOTLB
 218                 if (!strncmp(p, "soft", 4))
 219                         swiotlb = 1;
 220 #endif
 221                 if (!strncmp(p, "pt", 2))
 222                         iommu_pass_through = 1;
 223
 224                 gart_parse_options(p);
 225
 226 #ifdef CONFIG_CALGARY_IOMMU
 227                 if (!strncmp(p, "calgary", 7))
 228                         use_calgary = 1;
 229 #endif /* CONFIG_CALGARY_IOMMU */
 230
 231                 p += strcspn(p, ",");
 232                 if (*p == ',')
 233                         ++p;
 234         }
 235         return 0;
 236 }
 237 early_param("iommu", iommu_setup);
 238
 239 int dma_supported(struct device *dev, u64 mask)
 240 {
 241         struct dma_map_ops *ops = get_dma_ops(dev);
 242
 243 #ifdef CONFIG_PCI
 244         if (mask > 0xffffffff && forbid_dac > 0) {
 245                 dev_info(dev, "PCI: Disallowing DAC for device\n");
 246                 return 0;
 247         }
 248 #endif
 249
 250         if (ops->dma_supported)
 251                 return ops->dma_supported(dev, mask);
 252
 253         /* Copied from i386. Doesn't make much sense, because it will
 254            only work for pci_alloc_coherent.
 255            The caller just has to use GFP_DMA in this case. */
 256         if (mask < DMA_BIT_MASK(24))
 257                 return 0;
 258
 259         /* Tell the device to use SAC when IOMMU force is on.  This
 260            allows the driver to use cheaper accesses in some cases.
 261
 262            Problem with this is that if we overflow the IOMMU area and
 263            return DAC as fallback address the device may not handle it
 264            correctly.
 265
 266            As a special case some controllers have a 39bit address
 267            mode that is as efficient as 32bit (aic79xx). Don't force
 268            SAC for these.  Assume all masks <= 40 bits are of this
 269            type. Normally this doesn't make any difference, but gives
 270            more gentle handling of IOMMU overflow. */
 271         if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
 272                 dev_info(dev, "Force SAC with mask %Lx\n", mask);
 273                 return 0;
 274         }
 275
 276         return 1;
 277 }
 278 EXPORT_SYMBOL(dma_supported);
 279
 280 static int __init pci_iommu_init(void)
 281 {
 282         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 283
 284 #ifdef CONFIG_PCI
 285         dma_debug_add_bus(&pci_bus_type);
 286 #endif
 287         x86_init.iommu.iommu_init();
 288
 289         if (swiotlb) {
 290                 printk(KERN_INFO "PCI-DMA: "
 291                        "Using software bounce buffering for IO (SWIOTLB)\n");
 292                 swiotlb_print_info();
 293         } else
 294                 swiotlb_free();
 295
 296         return 0;
 297 }
 298 /* Must execute after PCI subsystem */
 299 rootfs_initcall(pci_iommu_init);
 300
 301 #ifdef CONFIG_PCI
 302 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
 303
 304 static __devinit void via_no_dac(struct pci_dev *dev)
 305 {
 306         if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
 307                 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
 308                 forbid_dac = 1;
 309         }
 310 }
 311 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
 312 #endif