2 * Dynamic DMA mapping support.
5 #include <linux/types.h>
7 #include <linux/string.h>
9 #include <linux/module.h>
11 #include <asm/proto.h>
13 int iommu_merge __read_mostly
= 0;
14 EXPORT_SYMBOL(iommu_merge
);
16 dma_addr_t bad_dma_address __read_mostly
;
17 EXPORT_SYMBOL(bad_dma_address
);
19 /* This tells the BIO block layer to assume merging. Default to off
20 because we cannot guarantee merging later. */
21 int iommu_bio_merge __read_mostly
= 0;
22 EXPORT_SYMBOL(iommu_bio_merge
);
24 int iommu_sac_force __read_mostly
= 0;
25 EXPORT_SYMBOL(iommu_sac_force
);
27 int no_iommu __read_mostly
;
28 #ifdef CONFIG_IOMMU_DEBUG
29 int panic_on_overflow __read_mostly
= 1;
30 int force_iommu __read_mostly
= 1;
32 int panic_on_overflow __read_mostly
= 0;
33 int force_iommu __read_mostly
= 0;
36 /* Dummy device used for NULL arguments (normally ISA). Better would
37 be probably a smaller DMA mask, but this is bug-to-bug compatible
39 struct device fallback_dev
= {
40 .bus_id
= "fallback device",
41 .coherent_dma_mask
= 0xffffffff,
42 .dma_mask
= &fallback_dev
.coherent_dma_mask
,
45 /* Allocate DMA memory on node near device */
46 noinline
static void *
47 dma_alloc_pages(struct device
*dev
, gfp_t gfp
, unsigned order
)
51 if (dev
->bus
== &pci_bus_type
)
52 node
= pcibus_to_node(to_pci_dev(dev
)->bus
);
54 node
= numa_node_id();
55 page
= alloc_pages_node(node
, gfp
, order
);
56 return page
? page_address(page
) : NULL
;
60 * Allocate memory for a coherent mapping.
63 dma_alloc_coherent(struct device
*dev
, size_t size
, dma_addr_t
*dma_handle
,
67 unsigned long dma_mask
= 0;
72 dma_mask
= dev
->coherent_dma_mask
;
74 dma_mask
= 0xffffffff;
76 /* Kludge to make it bug-to-bug compatible with i386. i386
77 uses the normal dma_mask for alloc_coherent. */
78 dma_mask
&= *dev
->dma_mask
;
80 /* Why <=? Even when the mask is smaller than 4GB it is often
81 larger than 16MB and in this case we have a chance of
82 finding fitting memory in the next higher zone first. If
83 not retry with true GFP_DMA. -AK */
84 if (dma_mask
<= 0xffffffff)
88 memory
= dma_alloc_pages(dev
, gfp
, get_order(size
));
94 bus
= virt_to_bus(memory
);
95 high
= (bus
+ size
) >= dma_mask
;
97 if (force_iommu
&& !(gfp
& GFP_DMA
))
100 free_pages((unsigned long)memory
,
103 /* Don't use the 16MB ZONE_DMA unless absolutely
104 needed. It's better to use remapping first. */
105 if (dma_mask
< 0xffffffff && !(gfp
& GFP_DMA
)) {
106 gfp
= (gfp
& ~GFP_DMA32
) | GFP_DMA
;
110 /* Let low level make its own zone decisions */
111 gfp
&= ~(GFP_DMA32
|GFP_DMA
);
113 if (dma_ops
->alloc_coherent
)
114 return dma_ops
->alloc_coherent(dev
, size
,
119 memset(memory
, 0, size
);
121 *dma_handle
= virt_to_bus(memory
);
126 if (dma_ops
->alloc_coherent
) {
127 free_pages((unsigned long)memory
, get_order(size
));
128 gfp
&= ~(GFP_DMA
|GFP_DMA32
);
129 return dma_ops
->alloc_coherent(dev
, size
, dma_handle
, gfp
);
132 if (dma_ops
->map_simple
) {
133 *dma_handle
= dma_ops
->map_simple(dev
, memory
,
135 PCI_DMA_BIDIRECTIONAL
);
136 if (*dma_handle
!= bad_dma_address
)
140 if (panic_on_overflow
)
141 panic("dma_alloc_coherent: IOMMU overflow by %lu bytes\n",size
);
142 free_pages((unsigned long)memory
, get_order(size
));
145 EXPORT_SYMBOL(dma_alloc_coherent
);
148 * Unmap coherent memory.
149 * The caller must ensure that the device has finished accessing the mapping.
151 void dma_free_coherent(struct device
*dev
, size_t size
,
152 void *vaddr
, dma_addr_t bus
)
154 if (dma_ops
->unmap_single
)
155 dma_ops
->unmap_single(dev
, bus
, size
, 0);
156 free_pages((unsigned long)vaddr
, get_order(size
));
158 EXPORT_SYMBOL(dma_free_coherent
);
160 int dma_supported(struct device
*dev
, u64 mask
)
162 if (dma_ops
->dma_supported
)
163 return dma_ops
->dma_supported(dev
, mask
);
165 /* Copied from i386. Doesn't make much sense, because it will
166 only work for pci_alloc_coherent.
167 The caller just has to use GFP_DMA in this case. */
168 if (mask
< 0x00ffffff)
171 /* Tell the device to use SAC when IOMMU force is on. This
172 allows the driver to use cheaper accesses in some cases.
174 Problem with this is that if we overflow the IOMMU area and
175 return DAC as fallback address the device may not handle it
178 As a special case some controllers have a 39bit address
179 mode that is as efficient as 32bit (aic79xx). Don't force
180 SAC for these. Assume all masks <= 40 bits are of this
181 type. Normally this doesn't make any difference, but gives
182 more gentle handling of IOMMU overflow. */
183 if (iommu_sac_force
&& (mask
>= 0xffffffffffULL
)) {
184 printk(KERN_INFO
"%s: Force SAC with mask %Lx\n", dev
->bus_id
,mask
);
190 EXPORT_SYMBOL(dma_supported
);
192 int dma_set_mask(struct device
*dev
, u64 mask
)
194 if (!dev
->dma_mask
|| !dma_supported(dev
, mask
))
196 *dev
->dma_mask
= mask
;
199 EXPORT_SYMBOL(dma_set_mask
);
201 /* iommu=[size][,noagp][,off][,force][,noforce][,leak][,memaper[=order]][,merge]
202 [,forcesac][,fullflush][,nomerge][,biomerge]
203 size set size of iommu (in bytes)
204 noagp don't initialize the AGP driver and use full aperture.
205 off don't use the IOMMU
206 leak turn on simple iommu leak tracing (only when CONFIG_IOMMU_LEAK is on)
207 memaper[=order] allocate an own aperture over RAM with size 32MB^order.
208 noforce don't force IOMMU usage. Default.
210 merge Do lazy merging. This may improve performance on some block devices.
211 Implies force (experimental)
212 biomerge Do merging at the BIO layer. This is more efficient than merge,
213 but should be only done with very big IOMMUs. Implies merge,force.
214 nomerge Don't do SG merging.
215 forcesac For SAC mode for masks <40bits (experimental)
216 fullflush Flush IOMMU on each allocation (default)
217 nofullflush Don't use IOMMU fullflush
218 allowed overwrite iommu off workarounds for specific chipsets.
219 soft Use software bounce buffering (default for Intel machines)
220 noaperture Don't touch the aperture for AGP.
222 __init
int iommu_setup(char *p
)
227 if (!strncmp(p
,"off",3))
229 /* gart_parse_options has more force support */
230 if (!strncmp(p
,"force",5))
232 if (!strncmp(p
,"noforce",7)) {
237 if (!strncmp(p
, "biomerge",8)) {
238 iommu_bio_merge
= 4096;
242 if (!strncmp(p
, "panic",5))
243 panic_on_overflow
= 1;
244 if (!strncmp(p
, "nopanic",7))
245 panic_on_overflow
= 0;
246 if (!strncmp(p
, "merge",5)) {
250 if (!strncmp(p
, "nomerge",7))
252 if (!strncmp(p
, "forcesac",8))
255 #ifdef CONFIG_SWIOTLB
256 if (!strncmp(p
, "soft",4))
260 #ifdef CONFIG_GART_IOMMU
261 gart_parse_options(p
);
264 p
+= strcspn(p
, ",");