arch/x86/kernel/pci-dma.c

   1 #include <linux/dma-mapping.h>
   2 #include <linux/dma-debug.h>
   3 #include <linux/dmar.h>
   4 #include <linux/bootmem.h>
   5 #include <linux/pci.h>
   6 #include <linux/kmemleak.h>
   7
   8 #include <asm/proto.h>
   9 #include <asm/dma.h>
  10 #include <asm/iommu.h>
  11 #include <asm/gart.h>
  12 #include <asm/calgary.h>
  13 #include <asm/amd_iommu.h>
  14 #include <asm/x86_init.h>
  15
  16 static int forbid_dac __read_mostly;
  17
  18 struct dma_map_ops *dma_ops = &nommu_dma_ops;
  19 EXPORT_SYMBOL(dma_ops);
  20
  21 static int iommu_sac_force __read_mostly;
  22
  23 #ifdef CONFIG_IOMMU_DEBUG
  24 int panic_on_overflow __read_mostly = 1;
  25 int force_iommu __read_mostly = 1;
  26 #else
  27 int panic_on_overflow __read_mostly = 0;
  28 int force_iommu __read_mostly = 0;
  29 #endif
  30
  31 int iommu_merge __read_mostly = 0;
  32
  33 int no_iommu __read_mostly;
  34 /* Set this to 1 if there is a HW IOMMU in the system */
  35 int iommu_detected __read_mostly = 0;
  36
  37 /*
  38  * This variable becomes 1 if iommu=pt is passed on the kernel command line.
  39  * If this variable is 1, IOMMU implementations do no DMA translation for
  40  * devices and allow every device to access to whole physical memory. This is
  41  * useful if a user want to use an IOMMU only for KVM device assignment to
  42  * guests and not for driver dma translation.
  43  */
  44 int iommu_pass_through __read_mostly;
  45
  46 /* Dummy device used for NULL arguments (normally ISA). */
  47 struct device x86_dma_fallback_dev = {
  48         .init_name = "fallback device",
  49         .coherent_dma_mask = ISA_DMA_BIT_MASK,
  50         .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
  51 };
  52 EXPORT_SYMBOL(x86_dma_fallback_dev);
  53
  54 /* Number of entries preallocated for DMA-API debugging */
  55 #define PREALLOC_DMA_DEBUG_ENTRIES       32768
  56
  57 int dma_set_mask(struct device *dev, u64 mask)
  58 {
  59         if (!dev->dma_mask || !dma_supported(dev, mask))
  60                 return -EIO;
  61
  62         *dev->dma_mask = mask;
  63
  64         return 0;
  65 }
  66 EXPORT_SYMBOL(dma_set_mask);
  67
  68 #if defined(CONFIG_X86_64) && !defined(CONFIG_NUMA)
  69 static __initdata void *dma32_bootmem_ptr;
  70 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
  71
  72 static int __init parse_dma32_size_opt(char *p)
  73 {
  74         if (!p)
  75                 return -EINVAL;
  76         dma32_bootmem_size = memparse(p, &p);
  77         return 0;
  78 }
  79 early_param("dma32_size", parse_dma32_size_opt);
  80
  81 void __init dma32_reserve_bootmem(void)
  82 {
  83         unsigned long size, align;
  84         if (max_pfn <= MAX_DMA32_PFN)
  85                 return;
  86
  87         /*
  88          * check aperture_64.c allocate_aperture() for reason about
  89          * using 512M as goal
  90          */
  91         align = 64ULL<<20;
  92         size = roundup(dma32_bootmem_size, align);
  93         dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
  94                                  512ULL<<20);
  95         /*
  96          * Kmemleak should not scan this block as it may not be mapped via the
  97          * kernel direct mapping.
  98          */
  99         kmemleak_ignore(dma32_bootmem_ptr);
 100         if (dma32_bootmem_ptr)
 101                 dma32_bootmem_size = size;
 102         else
 103                 dma32_bootmem_size = 0;
 104 }
 105 static void __init dma32_free_bootmem(void)
 106 {
 107
 108         if (max_pfn <= MAX_DMA32_PFN)
 109                 return;
 110
 111         if (!dma32_bootmem_ptr)
 112                 return;
 113
 114         free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
 115
 116         dma32_bootmem_ptr = NULL;
 117         dma32_bootmem_size = 0;
 118 }
 119 #else
 120 void __init dma32_reserve_bootmem(void)
 121 {
 122 }
 123 static void __init dma32_free_bootmem(void)
 124 {
 125 }
 126
 127 #endif
 128
 129 void __init pci_iommu_alloc(void)
 130 {
 131         /* free the range so iommu could get some range less than 4G */
 132         dma32_free_bootmem();
 133
 134         if (pci_swiotlb_detect())
 135                 goto out;
 136
 137         gart_iommu_hole_init();
 138
 139         detect_calgary();
 140
 141         detect_intel_iommu();
 142
 143         /* needs to be called after gart_iommu_hole_init */
 144         amd_iommu_detect();
 145 out:
 146         pci_swiotlb_init();
 147 }
 148
 149 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
 150                                  dma_addr_t *dma_addr, gfp_t flag)
 151 {
 152         unsigned long dma_mask;
 153         struct page *page;
 154         dma_addr_t addr;
 155
 156         dma_mask = dma_alloc_coherent_mask(dev, flag);
 157
 158         flag |= __GFP_ZERO;
 159 again:
 160         page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
 161         if (!page)
 162                 return NULL;
 163
 164         addr = page_to_phys(page);
 165         if (addr + size > dma_mask) {
 166                 __free_pages(page, get_order(size));
 167
 168                 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
 169                         flag = (flag & ~GFP_DMA32) | GFP_DMA;
 170                         goto again;
 171                 }
 172
 173                 return NULL;
 174         }
 175
 176         *dma_addr = addr;
 177         return page_address(page);
 178 }
 179
 180 /*
 181  * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 182  * documentation.
 183  */
 184 static __init int iommu_setup(char *p)
 185 {
 186         iommu_merge = 1;
 187
 188         if (!p)
 189                 return -EINVAL;
 190
 191         while (*p) {
 192                 if (!strncmp(p, "off", 3))
 193                         no_iommu = 1;
 194                 /* gart_parse_options has more force support */
 195                 if (!strncmp(p, "force", 5))
 196                         force_iommu = 1;
 197                 if (!strncmp(p, "noforce", 7)) {
 198                         iommu_merge = 0;
 199                         force_iommu = 0;
 200                 }
 201
 202                 if (!strncmp(p, "biomerge", 8)) {
 203                         iommu_merge = 1;
 204                         force_iommu = 1;
 205                 }
 206                 if (!strncmp(p, "panic", 5))
 207                         panic_on_overflow = 1;
 208                 if (!strncmp(p, "nopanic", 7))
 209                         panic_on_overflow = 0;
 210                 if (!strncmp(p, "merge", 5)) {
 211                         iommu_merge = 1;
 212                         force_iommu = 1;
 213                 }
 214                 if (!strncmp(p, "nomerge", 7))
 215                         iommu_merge = 0;
 216                 if (!strncmp(p, "forcesac", 8))
 217                         iommu_sac_force = 1;
 218                 if (!strncmp(p, "allowdac", 8))
 219                         forbid_dac = 0;
 220                 if (!strncmp(p, "nodac", 5))
 221                         forbid_dac = 1;
 222                 if (!strncmp(p, "usedac", 6)) {
 223                         forbid_dac = -1;
 224                         return 1;
 225                 }
 226 #ifdef CONFIG_SWIOTLB
 227                 if (!strncmp(p, "soft", 4))
 228                         swiotlb = 1;
 229 #endif
 230                 if (!strncmp(p, "pt", 2))
 231                         iommu_pass_through = 1;
 232
 233                 gart_parse_options(p);
 234
 235 #ifdef CONFIG_CALGARY_IOMMU
 236                 if (!strncmp(p, "calgary", 7))
 237                         use_calgary = 1;
 238 #endif /* CONFIG_CALGARY_IOMMU */
 239
 240                 p += strcspn(p, ",");
 241                 if (*p == ',')
 242                         ++p;
 243         }
 244         return 0;
 245 }
 246 early_param("iommu", iommu_setup);
 247
 248 int dma_supported(struct device *dev, u64 mask)
 249 {
 250         struct dma_map_ops *ops = get_dma_ops(dev);
 251
 252 #ifdef CONFIG_PCI
 253         if (mask > 0xffffffff && forbid_dac > 0) {
 254                 dev_info(dev, "PCI: Disallowing DAC for device\n");
 255                 return 0;
 256         }
 257 #endif
 258
 259         if (ops->dma_supported)
 260                 return ops->dma_supported(dev, mask);
 261
 262         /* Copied from i386. Doesn't make much sense, because it will
 263            only work for pci_alloc_coherent.
 264            The caller just has to use GFP_DMA in this case. */
 265         if (mask < DMA_BIT_MASK(24))
 266                 return 0;
 267
 268         /* Tell the device to use SAC when IOMMU force is on.  This
 269            allows the driver to use cheaper accesses in some cases.
 270
 271            Problem with this is that if we overflow the IOMMU area and
 272            return DAC as fallback address the device may not handle it
 273            correctly.
 274
 275            As a special case some controllers have a 39bit address
 276            mode that is as efficient as 32bit (aic79xx). Don't force
 277            SAC for these.  Assume all masks <= 40 bits are of this
 278            type. Normally this doesn't make any difference, but gives
 279            more gentle handling of IOMMU overflow. */
 280         if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
 281                 dev_info(dev, "Force SAC with mask %Lx\n", mask);
 282                 return 0;
 283         }
 284
 285         return 1;
 286 }
 287 EXPORT_SYMBOL(dma_supported);
 288
 289 static int __init pci_iommu_init(void)
 290 {
 291         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 292
 293 #ifdef CONFIG_PCI
 294         dma_debug_add_bus(&pci_bus_type);
 295 #endif
 296         x86_init.iommu.iommu_init();
 297
 298         if (swiotlb) {
 299                 printk(KERN_INFO "PCI-DMA: "
 300                        "Using software bounce buffering for IO (SWIOTLB)\n");
 301                 swiotlb_print_info();
 302         } else
 303                 swiotlb_free();
 304
 305         return 0;
 306 }
 307 /* Must execute after PCI subsystem */
 308 rootfs_initcall(pci_iommu_init);
 309
 310 #ifdef CONFIG_PCI
 311 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
 312
 313 static __devinit void via_no_dac(struct pci_dev *dev)
 314 {
 315         if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
 316                 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
 317                 forbid_dac = 1;
 318         }
 319 }
 320 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
 321 #endif