arch/x86/kernel/pci-dma.c

   1 #include <linux/dma-mapping.h>
   2 #include <linux/dma-debug.h>
   3 #include <linux/dmar.h>
   4 #include <linux/bootmem.h>
   5 #include <linux/pci.h>
   6 #include <linux/kmemleak.h>
   7
   8 #include <asm/proto.h>
   9 #include <asm/dma.h>
  10 #include <asm/iommu.h>
  11 #include <asm/gart.h>
  12 #include <asm/calgary.h>
  13 #include <asm/amd_iommu.h>
  14
  15 static int forbid_dac __read_mostly;
  16
  17 struct dma_map_ops *dma_ops;
  18 EXPORT_SYMBOL(dma_ops);
  19
  20 static int iommu_sac_force __read_mostly;
  21
  22 #ifdef CONFIG_IOMMU_DEBUG
  23 int panic_on_overflow __read_mostly = 1;
  24 int force_iommu __read_mostly = 1;
  25 #else
  26 int panic_on_overflow __read_mostly = 0;
  27 int force_iommu __read_mostly = 0;
  28 #endif
  29
  30 int iommu_merge __read_mostly = 0;
  31
  32 int no_iommu __read_mostly;
  33 /* Set this to 1 if there is a HW IOMMU in the system */
  34 int iommu_detected __read_mostly = 0;
  35
  36 /*
  37  * This variable becomes 1 if iommu=pt is passed on the kernel command line.
  38  * If this variable is 1, IOMMU implementations do no DMA translation for
  39  * devices and allow every device to access to whole physical memory. This is
  40  * useful if a user want to use an IOMMU only for KVM device assignment to
  41  * guests and not for driver dma translation.
  42  */
  43 int iommu_pass_through __read_mostly;
  44
  45 dma_addr_t bad_dma_address __read_mostly = 0;
  46 EXPORT_SYMBOL(bad_dma_address);
  47
  48 /* Dummy device used for NULL arguments (normally ISA). */
  49 struct device x86_dma_fallback_dev = {
  50         .init_name = "fallback device",
  51         .coherent_dma_mask = ISA_DMA_BIT_MASK,
  52         .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
  53 };
  54 EXPORT_SYMBOL(x86_dma_fallback_dev);
  55
  56 /* Number of entries preallocated for DMA-API debugging */
  57 #define PREALLOC_DMA_DEBUG_ENTRIES       32768
  58
  59 int dma_set_mask(struct device *dev, u64 mask)
  60 {
  61         if (!dev->dma_mask || !dma_supported(dev, mask))
  62                 return -EIO;
  63
  64         *dev->dma_mask = mask;
  65
  66         return 0;
  67 }
  68 EXPORT_SYMBOL(dma_set_mask);
  69
  70 #ifdef CONFIG_X86_64
  71 static __initdata void *dma32_bootmem_ptr;
  72 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
  73
  74 static int __init parse_dma32_size_opt(char *p)
  75 {
  76         if (!p)
  77                 return -EINVAL;
  78         dma32_bootmem_size = memparse(p, &p);
  79         return 0;
  80 }
  81 early_param("dma32_size", parse_dma32_size_opt);
  82
  83 void __init dma32_reserve_bootmem(void)
  84 {
  85         unsigned long size, align;
  86         if (max_pfn <= MAX_DMA32_PFN)
  87                 return;
  88
  89         /*
  90          * check aperture_64.c allocate_aperture() for reason about
  91          * using 512M as goal
  92          */
  93         align = 64ULL<<20;
  94         size = roundup(dma32_bootmem_size, align);
  95         dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
  96                                  512ULL<<20);
  97         /*
  98          * Kmemleak should not scan this block as it may not be mapped via the
  99          * kernel direct mapping.
 100          */
 101         kmemleak_ignore(dma32_bootmem_ptr);
 102         if (dma32_bootmem_ptr)
 103                 dma32_bootmem_size = size;
 104         else
 105                 dma32_bootmem_size = 0;
 106 }
 107 static void __init dma32_free_bootmem(void)
 108 {
 109
 110         if (max_pfn <= MAX_DMA32_PFN)
 111                 return;
 112
 113         if (!dma32_bootmem_ptr)
 114                 return;
 115
 116         free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
 117
 118         dma32_bootmem_ptr = NULL;
 119         dma32_bootmem_size = 0;
 120 }
 121 #endif
 122
 123 void __init pci_iommu_alloc(void)
 124 {
 125 #ifdef CONFIG_X86_64
 126         /* free the range so iommu could get some range less than 4G */
 127         dma32_free_bootmem();
 128 #endif
 129
 130         /*
 131          * The order of these functions is important for
 132          * fall-back/fail-over reasons
 133          */
 134         gart_iommu_hole_init();
 135
 136         detect_calgary();
 137
 138         detect_intel_iommu();
 139
 140         amd_iommu_detect();
 141
 142         pci_swiotlb_init();
 143 }
 144
 145 void *dma_generic_alloc_coherent(struct device *dev, size_t size,
 146                                  dma_addr_t *dma_addr, gfp_t flag)
 147 {
 148         unsigned long dma_mask;
 149         struct page *page;
 150         dma_addr_t addr;
 151
 152         dma_mask = dma_alloc_coherent_mask(dev, flag);
 153
 154         flag |= __GFP_ZERO;
 155 again:
 156         page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
 157         if (!page)
 158                 return NULL;
 159
 160         addr = page_to_phys(page);
 161         if (addr + size > dma_mask) {
 162                 __free_pages(page, get_order(size));
 163
 164                 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) {
 165                         flag = (flag & ~GFP_DMA32) | GFP_DMA;
 166                         goto again;
 167                 }
 168
 169                 return NULL;
 170         }
 171
 172         *dma_addr = addr;
 173         return page_address(page);
 174 }
 175
 176 /*
 177  * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 178  * documentation.
 179  */
 180 static __init int iommu_setup(char *p)
 181 {
 182         iommu_merge = 1;
 183
 184         if (!p)
 185                 return -EINVAL;
 186
 187         while (*p) {
 188                 if (!strncmp(p, "off", 3))
 189                         no_iommu = 1;
 190                 /* gart_parse_options has more force support */
 191                 if (!strncmp(p, "force", 5))
 192                         force_iommu = 1;
 193                 if (!strncmp(p, "noforce", 7)) {
 194                         iommu_merge = 0;
 195                         force_iommu = 0;
 196                 }
 197
 198                 if (!strncmp(p, "biomerge", 8)) {
 199                         iommu_merge = 1;
 200                         force_iommu = 1;
 201                 }
 202                 if (!strncmp(p, "panic", 5))
 203                         panic_on_overflow = 1;
 204                 if (!strncmp(p, "nopanic", 7))
 205                         panic_on_overflow = 0;
 206                 if (!strncmp(p, "merge", 5)) {
 207                         iommu_merge = 1;
 208                         force_iommu = 1;
 209                 }
 210                 if (!strncmp(p, "nomerge", 7))
 211                         iommu_merge = 0;
 212                 if (!strncmp(p, "forcesac", 8))
 213                         iommu_sac_force = 1;
 214                 if (!strncmp(p, "allowdac", 8))
 215                         forbid_dac = 0;
 216                 if (!strncmp(p, "nodac", 5))
 217                         forbid_dac = 1;
 218                 if (!strncmp(p, "usedac", 6)) {
 219                         forbid_dac = -1;
 220                         return 1;
 221                 }
 222 #ifdef CONFIG_SWIOTLB
 223                 if (!strncmp(p, "soft", 4))
 224                         swiotlb = 1;
 225 #endif
 226                 if (!strncmp(p, "pt", 2))
 227                         iommu_pass_through = 1;
 228
 229                 gart_parse_options(p);
 230
 231 #ifdef CONFIG_CALGARY_IOMMU
 232                 if (!strncmp(p, "calgary", 7))
 233                         use_calgary = 1;
 234 #endif /* CONFIG_CALGARY_IOMMU */
 235
 236                 p += strcspn(p, ",");
 237                 if (*p == ',')
 238                         ++p;
 239         }
 240         return 0;
 241 }
 242 early_param("iommu", iommu_setup);
 243
 244 int dma_supported(struct device *dev, u64 mask)
 245 {
 246         struct dma_map_ops *ops = get_dma_ops(dev);
 247
 248 #ifdef CONFIG_PCI
 249         if (mask > 0xffffffff && forbid_dac > 0) {
 250                 dev_info(dev, "PCI: Disallowing DAC for device\n");
 251                 return 0;
 252         }
 253 #endif
 254
 255         if (ops->dma_supported)
 256                 return ops->dma_supported(dev, mask);
 257
 258         /* Copied from i386. Doesn't make much sense, because it will
 259            only work for pci_alloc_coherent.
 260            The caller just has to use GFP_DMA in this case. */
 261         if (mask < DMA_BIT_MASK(24))
 262                 return 0;
 263
 264         /* Tell the device to use SAC when IOMMU force is on.  This
 265            allows the driver to use cheaper accesses in some cases.
 266
 267            Problem with this is that if we overflow the IOMMU area and
 268            return DAC as fallback address the device may not handle it
 269            correctly.
 270
 271            As a special case some controllers have a 39bit address
 272            mode that is as efficient as 32bit (aic79xx). Don't force
 273            SAC for these.  Assume all masks <= 40 bits are of this
 274            type. Normally this doesn't make any difference, but gives
 275            more gentle handling of IOMMU overflow. */
 276         if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) {
 277                 dev_info(dev, "Force SAC with mask %Lx\n", mask);
 278                 return 0;
 279         }
 280
 281         return 1;
 282 }
 283 EXPORT_SYMBOL(dma_supported);
 284
 285 static int __init pci_iommu_init(void)
 286 {
 287         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
 288
 289 #ifdef CONFIG_PCI
 290         dma_debug_add_bus(&pci_bus_type);
 291 #endif
 292
 293         calgary_iommu_init();
 294
 295         intel_iommu_init();
 296
 297         amd_iommu_init();
 298
 299         gart_iommu_init();
 300
 301         no_iommu_init();
 302         return 0;
 303 }
 304
 305 void pci_iommu_shutdown(void)
 306 {
 307         gart_iommu_shutdown();
 308
 309         amd_iommu_shutdown();
 310 }
 311 /* Must execute after PCI subsystem */
 312 rootfs_initcall(pci_iommu_init);
 313
 314 #ifdef CONFIG_PCI
 315 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
 316
 317 static __devinit void via_no_dac(struct pci_dev *dev)
 318 {
 319         if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
 320                 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n");
 321                 forbid_dac = 1;
 322         }
 323 }
 324 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
 325 #endif