1 #include <linux/dma-mapping.h>
2 #include <linux/dma-debug.h>
3 #include <linux/dmar.h>
4 #include <linux/bootmem.h>
11 #include <asm/calgary.h>
12 #include <asm/amd_iommu.h>
14 static int forbid_dac __read_mostly
;
16 struct dma_map_ops
*dma_ops
;
17 EXPORT_SYMBOL(dma_ops
);
19 static int iommu_sac_force __read_mostly
;
21 #ifdef CONFIG_IOMMU_DEBUG
22 int panic_on_overflow __read_mostly
= 1;
23 int force_iommu __read_mostly
= 1;
25 int panic_on_overflow __read_mostly
= 0;
26 int force_iommu __read_mostly
= 0;
29 int iommu_merge __read_mostly
= 0;
31 int no_iommu __read_mostly
;
32 /* Set this to 1 if there is a HW IOMMU in the system */
33 int iommu_detected __read_mostly
= 0;
35 dma_addr_t bad_dma_address __read_mostly
= 0;
36 EXPORT_SYMBOL(bad_dma_address
);
38 /* Dummy device used for NULL arguments (normally ISA). Better would
39 be probably a smaller DMA mask, but this is bug-to-bug compatible
41 struct device x86_dma_fallback_dev
= {
42 .init_name
= "fallback device",
43 .coherent_dma_mask
= DMA_BIT_MASK(32),
44 .dma_mask
= &x86_dma_fallback_dev
.coherent_dma_mask
,
46 EXPORT_SYMBOL(x86_dma_fallback_dev
);
48 /* Number of entries preallocated for DMA-API debugging */
49 #define PREALLOC_DMA_DEBUG_ENTRIES 32768
51 int dma_set_mask(struct device
*dev
, u64 mask
)
53 if (!dev
->dma_mask
|| !dma_supported(dev
, mask
))
56 *dev
->dma_mask
= mask
;
60 EXPORT_SYMBOL(dma_set_mask
);
63 static __initdata
void *dma32_bootmem_ptr
;
64 static unsigned long dma32_bootmem_size __initdata
= (128ULL<<20);
66 static int __init
parse_dma32_size_opt(char *p
)
70 dma32_bootmem_size
= memparse(p
, &p
);
73 early_param("dma32_size", parse_dma32_size_opt
);
75 void __init
dma32_reserve_bootmem(void)
77 unsigned long size
, align
;
78 if (max_pfn
<= MAX_DMA32_PFN
)
82 * check aperture_64.c allocate_aperture() for reason about
86 size
= roundup(dma32_bootmem_size
, align
);
87 dma32_bootmem_ptr
= __alloc_bootmem_nopanic(size
, align
,
89 if (dma32_bootmem_ptr
)
90 dma32_bootmem_size
= size
;
92 dma32_bootmem_size
= 0;
94 static void __init
dma32_free_bootmem(void)
97 if (max_pfn
<= MAX_DMA32_PFN
)
100 if (!dma32_bootmem_ptr
)
103 free_bootmem(__pa(dma32_bootmem_ptr
), dma32_bootmem_size
);
105 dma32_bootmem_ptr
= NULL
;
106 dma32_bootmem_size
= 0;
110 void __init
pci_iommu_alloc(void)
113 /* free the range so iommu could get some range less than 4G */
114 dma32_free_bootmem();
118 * The order of these functions is important for
119 * fall-back/fail-over reasons
121 gart_iommu_hole_init();
125 detect_intel_iommu();
132 void *dma_generic_alloc_coherent(struct device
*dev
, size_t size
,
133 dma_addr_t
*dma_addr
, gfp_t flag
)
135 unsigned long dma_mask
;
139 dma_mask
= dma_alloc_coherent_mask(dev
, flag
);
143 page
= alloc_pages_node(dev_to_node(dev
), flag
, get_order(size
));
147 addr
= page_to_phys(page
);
148 if (!is_buffer_dma_capable(dma_mask
, addr
, size
)) {
149 __free_pages(page
, get_order(size
));
151 if (dma_mask
< DMA_BIT_MASK(32) && !(flag
& GFP_DMA
)) {
152 flag
= (flag
& ~GFP_DMA32
) | GFP_DMA
;
160 return page_address(page
);
163 extern int iommu_pass_through
;
166 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
169 static __init
int iommu_setup(char *p
)
177 if (!strncmp(p
, "off", 3))
179 /* gart_parse_options has more force support */
180 if (!strncmp(p
, "force", 5))
182 if (!strncmp(p
, "noforce", 7)) {
187 if (!strncmp(p
, "biomerge", 8)) {
191 if (!strncmp(p
, "panic", 5))
192 panic_on_overflow
= 1;
193 if (!strncmp(p
, "nopanic", 7))
194 panic_on_overflow
= 0;
195 if (!strncmp(p
, "merge", 5)) {
199 if (!strncmp(p
, "nomerge", 7))
201 if (!strncmp(p
, "forcesac", 8))
203 if (!strncmp(p
, "allowdac", 8))
205 if (!strncmp(p
, "nodac", 5))
207 if (!strncmp(p
, "usedac", 6)) {
211 #ifdef CONFIG_SWIOTLB
212 if (!strncmp(p
, "soft", 4))
214 if (!strncmp(p
, "pt", 2)) {
215 iommu_pass_through
= 1;
220 gart_parse_options(p
);
222 #ifdef CONFIG_CALGARY_IOMMU
223 if (!strncmp(p
, "calgary", 7))
225 #endif /* CONFIG_CALGARY_IOMMU */
227 p
+= strcspn(p
, ",");
233 early_param("iommu", iommu_setup
);
235 int dma_supported(struct device
*dev
, u64 mask
)
237 struct dma_map_ops
*ops
= get_dma_ops(dev
);
240 if (mask
> 0xffffffff && forbid_dac
> 0) {
241 dev_info(dev
, "PCI: Disallowing DAC for device\n");
246 if (ops
->dma_supported
)
247 return ops
->dma_supported(dev
, mask
);
249 /* Copied from i386. Doesn't make much sense, because it will
250 only work for pci_alloc_coherent.
251 The caller just has to use GFP_DMA in this case. */
252 if (mask
< DMA_BIT_MASK(24))
255 /* Tell the device to use SAC when IOMMU force is on. This
256 allows the driver to use cheaper accesses in some cases.
258 Problem with this is that if we overflow the IOMMU area and
259 return DAC as fallback address the device may not handle it
262 As a special case some controllers have a 39bit address
263 mode that is as efficient as 32bit (aic79xx). Don't force
264 SAC for these. Assume all masks <= 40 bits are of this
265 type. Normally this doesn't make any difference, but gives
266 more gentle handling of IOMMU overflow. */
267 if (iommu_sac_force
&& (mask
>= DMA_BIT_MASK(40))) {
268 dev_info(dev
, "Force SAC with mask %Lx\n", mask
);
274 EXPORT_SYMBOL(dma_supported
);
276 static int __init
pci_iommu_init(void)
278 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES
);
281 dma_debug_add_bus(&pci_bus_type
);
284 calgary_iommu_init();
296 void pci_iommu_shutdown(void)
298 gart_iommu_shutdown();
300 /* Must execute after PCI subsystem */
301 fs_initcall(pci_iommu_init
);
304 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */
306 static __devinit
void via_no_dac(struct pci_dev
*dev
)
308 if ((dev
->class >> 8) == PCI_CLASS_BRIDGE_PCI
&& forbid_dac
== 0) {
309 dev_info(&dev
->dev
, "disabling DAC on VIA PCI bridge\n");
313 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA
, PCI_ANY_ID
, via_no_dac
);