2 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
4 * Rewrite, cleanup, new allocation schemes, virtual merging:
5 * Copyright (C) 2004 Olof Johansson, IBM Corporation
6 * and Ben. Herrenschmidt, IBM Corporation
8 * Dynamic DMA mapping support, bus-independent parts.
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 #include <linux/init.h>
27 #include <linux/types.h>
28 #include <linux/slab.h>
30 #include <linux/spinlock.h>
31 #include <linux/string.h>
32 #include <linux/dma-mapping.h>
33 #include <linux/init.h>
34 #include <linux/bitops.h>
37 #include <asm/iommu.h>
38 #include <asm/pci-bridge.h>
39 #include <asm/machdep.h>
40 #include <asm/kdump.h>
44 #ifdef CONFIG_IOMMU_VMERGE
45 static int novmerge
= 0;
47 static int novmerge
= 1;
50 static int protect4gb
= 1;
52 static inline unsigned long iommu_num_pages(unsigned long vaddr
,
57 npages
= IOMMU_PAGE_ALIGN(vaddr
+ slen
) - (vaddr
& IOMMU_PAGE_MASK
);
58 npages
>>= IOMMU_PAGE_SHIFT
;
63 static int __init
setup_protect4gb(char *str
)
65 if (strcmp(str
, "on") == 0)
67 else if (strcmp(str
, "off") == 0)
73 static int __init
setup_iommu(char *str
)
75 if (!strcmp(str
, "novmerge"))
77 else if (!strcmp(str
, "vmerge"))
82 __setup("protect4gb=", setup_protect4gb
);
83 __setup("iommu=", setup_iommu
);
85 static unsigned long iommu_range_alloc(struct iommu_table
*tbl
,
87 unsigned long *handle
,
89 unsigned int align_order
)
91 unsigned long n
, end
, i
, start
;
93 int largealloc
= npages
> 15;
95 unsigned long align_mask
;
97 align_mask
= 0xffffffffffffffffl
>> (64 - align_order
);
99 /* This allocator was derived from x86_64's bit string search */
102 if (unlikely(npages
== 0)) {
103 if (printk_ratelimit())
105 return DMA_ERROR_CODE
;
108 if (handle
&& *handle
)
111 start
= largealloc
? tbl
->it_largehint
: tbl
->it_hint
;
113 /* Use only half of the table for small allocs (15 pages or less) */
114 limit
= largealloc
? tbl
->it_size
: tbl
->it_halfpoint
;
116 if (largealloc
&& start
< tbl
->it_halfpoint
)
117 start
= tbl
->it_halfpoint
;
119 /* The case below can happen if we have a small segment appended
120 * to a large, or when the previous alloc was at the very end of
121 * the available space. If so, go back to the initial start.
124 start
= largealloc
? tbl
->it_largehint
: tbl
->it_hint
;
128 if (limit
+ tbl
->it_offset
> mask
) {
129 limit
= mask
- tbl
->it_offset
+ 1;
130 /* If we're constrained on address range, first try
131 * at the masked hint to avoid O(n) search complexity,
132 * but on second pass, start at 0.
134 if ((start
& mask
) >= limit
|| pass
> 0)
140 n
= find_next_zero_bit(tbl
->it_map
, limit
, start
);
142 /* Align allocation */
143 n
= (n
+ align_mask
) & ~align_mask
;
147 if (unlikely(end
>= limit
)) {
148 if (likely(pass
< 2)) {
149 /* First failure, just rescan the half of the table.
150 * Second failure, rescan the other half of the table.
152 start
= (largealloc
^ pass
) ? tbl
->it_halfpoint
: 0;
153 limit
= pass
? tbl
->it_size
: limit
;
157 /* Third failure, give up */
158 return DMA_ERROR_CODE
;
162 for (i
= n
; i
< end
; i
++)
163 if (test_bit(i
, tbl
->it_map
)) {
168 for (i
= n
; i
< end
; i
++)
169 __set_bit(i
, tbl
->it_map
);
171 /* Bump the hint to a new block for small allocs. */
173 /* Don't bump to new block to avoid fragmentation */
174 tbl
->it_largehint
= end
;
176 /* Overflow will be taken care of at the next allocation */
177 tbl
->it_hint
= (end
+ tbl
->it_blocksize
- 1) &
178 ~(tbl
->it_blocksize
- 1);
181 /* Update handle for SG allocations */
188 static dma_addr_t
iommu_alloc(struct iommu_table
*tbl
, void *page
,
189 unsigned int npages
, enum dma_data_direction direction
,
190 unsigned long mask
, unsigned int align_order
)
192 unsigned long entry
, flags
;
193 dma_addr_t ret
= DMA_ERROR_CODE
;
195 spin_lock_irqsave(&(tbl
->it_lock
), flags
);
197 entry
= iommu_range_alloc(tbl
, npages
, NULL
, mask
, align_order
);
199 if (unlikely(entry
== DMA_ERROR_CODE
)) {
200 spin_unlock_irqrestore(&(tbl
->it_lock
), flags
);
201 return DMA_ERROR_CODE
;
204 entry
+= tbl
->it_offset
; /* Offset into real TCE table */
205 ret
= entry
<< IOMMU_PAGE_SHIFT
; /* Set the return dma address */
207 /* Put the TCEs in the HW table */
208 ppc_md
.tce_build(tbl
, entry
, npages
, (unsigned long)page
& IOMMU_PAGE_MASK
,
212 /* Flush/invalidate TLB caches if necessary */
213 if (ppc_md
.tce_flush
)
214 ppc_md
.tce_flush(tbl
);
216 spin_unlock_irqrestore(&(tbl
->it_lock
), flags
);
218 /* Make sure updates are seen by hardware */
224 static void __iommu_free(struct iommu_table
*tbl
, dma_addr_t dma_addr
,
227 unsigned long entry
, free_entry
;
230 entry
= dma_addr
>> IOMMU_PAGE_SHIFT
;
231 free_entry
= entry
- tbl
->it_offset
;
233 if (((free_entry
+ npages
) > tbl
->it_size
) ||
234 (entry
< tbl
->it_offset
)) {
235 if (printk_ratelimit()) {
236 printk(KERN_INFO
"iommu_free: invalid entry\n");
237 printk(KERN_INFO
"\tentry = 0x%lx\n", entry
);
238 printk(KERN_INFO
"\tdma_addr = 0x%lx\n", (u64
)dma_addr
);
239 printk(KERN_INFO
"\tTable = 0x%lx\n", (u64
)tbl
);
240 printk(KERN_INFO
"\tbus# = 0x%lx\n", (u64
)tbl
->it_busno
);
241 printk(KERN_INFO
"\tsize = 0x%lx\n", (u64
)tbl
->it_size
);
242 printk(KERN_INFO
"\tstartOff = 0x%lx\n", (u64
)tbl
->it_offset
);
243 printk(KERN_INFO
"\tindex = 0x%lx\n", (u64
)tbl
->it_index
);
249 ppc_md
.tce_free(tbl
, entry
, npages
);
251 for (i
= 0; i
< npages
; i
++)
252 __clear_bit(free_entry
+i
, tbl
->it_map
);
255 static void iommu_free(struct iommu_table
*tbl
, dma_addr_t dma_addr
,
260 spin_lock_irqsave(&(tbl
->it_lock
), flags
);
262 __iommu_free(tbl
, dma_addr
, npages
);
264 /* Make sure TLB cache is flushed if the HW needs it. We do
265 * not do an mb() here on purpose, it is not needed on any of
266 * the current platforms.
268 if (ppc_md
.tce_flush
)
269 ppc_md
.tce_flush(tbl
);
271 spin_unlock_irqrestore(&(tbl
->it_lock
), flags
);
274 int iommu_map_sg(struct iommu_table
*tbl
, struct scatterlist
*sglist
,
275 int nelems
, unsigned long mask
,
276 enum dma_data_direction direction
)
278 dma_addr_t dma_next
= 0, dma_addr
;
280 struct scatterlist
*s
, *outs
, *segstart
;
281 int outcount
, incount
;
282 unsigned long handle
;
284 BUG_ON(direction
== DMA_NONE
);
286 if ((nelems
== 0) || !tbl
)
289 outs
= s
= segstart
= &sglist
[0];
294 /* Init first segment length for backout at failure */
295 outs
->dma_length
= 0;
297 DBG("sg mapping %d elements:\n", nelems
);
299 spin_lock_irqsave(&(tbl
->it_lock
), flags
);
301 for (s
= outs
; nelems
; nelems
--, s
++) {
302 unsigned long vaddr
, npages
, entry
, slen
;
310 /* Allocate iommu entries for that segment */
311 vaddr
= (unsigned long)page_address(s
->page
) + s
->offset
;
312 npages
= iommu_num_pages(vaddr
, slen
);
313 entry
= iommu_range_alloc(tbl
, npages
, &handle
, mask
>> IOMMU_PAGE_SHIFT
, 0);
315 DBG(" - vaddr: %lx, size: %lx\n", vaddr
, slen
);
318 if (unlikely(entry
== DMA_ERROR_CODE
)) {
319 if (printk_ratelimit())
320 printk(KERN_INFO
"iommu_alloc failed, tbl %p vaddr %lx"
321 " npages %lx\n", tbl
, vaddr
, npages
);
325 /* Convert entry to a dma_addr_t */
326 entry
+= tbl
->it_offset
;
327 dma_addr
= entry
<< IOMMU_PAGE_SHIFT
;
328 dma_addr
|= (s
->offset
& ~IOMMU_PAGE_MASK
);
330 DBG(" - %lu pages, entry: %lx, dma_addr: %lx\n",
331 npages
, entry
, dma_addr
);
333 /* Insert into HW table */
334 ppc_md
.tce_build(tbl
, entry
, npages
, vaddr
& IOMMU_PAGE_MASK
, direction
);
336 /* If we are in an open segment, try merging */
338 DBG(" - trying merge...\n");
339 /* We cannot merge if:
340 * - allocated dma_addr isn't contiguous to previous allocation
342 if (novmerge
|| (dma_addr
!= dma_next
)) {
343 /* Can't merge: create a new segment */
346 DBG(" can't merge, new segment.\n");
348 outs
->dma_length
+= s
->length
;
349 DBG(" merged, new len: %ux\n", outs
->dma_length
);
354 /* This is a new segment, fill entries */
355 DBG(" - filling new segment.\n");
356 outs
->dma_address
= dma_addr
;
357 outs
->dma_length
= slen
;
360 /* Calculate next page pointer for contiguous check */
361 dma_next
= dma_addr
+ slen
;
363 DBG(" - dma next is: %lx\n", dma_next
);
366 /* Flush/invalidate TLB caches if necessary */
367 if (ppc_md
.tce_flush
)
368 ppc_md
.tce_flush(tbl
);
370 spin_unlock_irqrestore(&(tbl
->it_lock
), flags
);
372 DBG("mapped %d elements:\n", outcount
);
374 /* For the sake of iommu_unmap_sg, we clear out the length in the
375 * next entry of the sglist if we didn't fill the list completely
377 if (outcount
< incount
) {
379 outs
->dma_address
= DMA_ERROR_CODE
;
380 outs
->dma_length
= 0;
383 /* Make sure updates are seen by hardware */
389 for (s
= &sglist
[0]; s
<= outs
; s
++) {
390 if (s
->dma_length
!= 0) {
391 unsigned long vaddr
, npages
;
393 vaddr
= s
->dma_address
& IOMMU_PAGE_MASK
;
394 npages
= iommu_num_pages(s
->dma_address
, s
->dma_length
);
395 __iommu_free(tbl
, vaddr
, npages
);
396 s
->dma_address
= DMA_ERROR_CODE
;
400 spin_unlock_irqrestore(&(tbl
->it_lock
), flags
);
405 void iommu_unmap_sg(struct iommu_table
*tbl
, struct scatterlist
*sglist
,
406 int nelems
, enum dma_data_direction direction
)
410 BUG_ON(direction
== DMA_NONE
);
415 spin_lock_irqsave(&(tbl
->it_lock
), flags
);
419 dma_addr_t dma_handle
= sglist
->dma_address
;
421 if (sglist
->dma_length
== 0)
423 npages
= iommu_num_pages(dma_handle
,sglist
->dma_length
);
424 __iommu_free(tbl
, dma_handle
, npages
);
428 /* Flush/invalidate TLBs if necessary. As for iommu_free(), we
429 * do not do an mb() here, the affected platforms do not need it
432 if (ppc_md
.tce_flush
)
433 ppc_md
.tce_flush(tbl
);
435 spin_unlock_irqrestore(&(tbl
->it_lock
), flags
);
439 * Build a iommu_table structure. This contains a bit map which
440 * is used to manage allocation of the tce space.
442 struct iommu_table
*iommu_init_table(struct iommu_table
*tbl
, int nid
)
445 unsigned long start_index
, end_index
;
446 unsigned long entries_per_4g
;
448 static int welcomed
= 0;
451 /* Set aside 1/4 of the table for large allocations. */
452 tbl
->it_halfpoint
= tbl
->it_size
* 3 / 4;
454 /* number of bytes needed for the bitmap */
455 sz
= (tbl
->it_size
+ 7) >> 3;
457 page
= alloc_pages_node(nid
, GFP_ATOMIC
, get_order(sz
));
459 panic("iommu_init_table: Can't allocate %ld bytes\n", sz
);
460 tbl
->it_map
= page_address(page
);
461 memset(tbl
->it_map
, 0, sz
);
464 tbl
->it_largehint
= tbl
->it_halfpoint
;
465 spin_lock_init(&tbl
->it_lock
);
467 #ifdef CONFIG_CRASH_DUMP
468 if (ppc_md
.tce_get
) {
469 unsigned long tceval
;
470 unsigned long tcecount
= 0;
473 * Reserve the existing mappings left by the first kernel.
475 for (index
= 0; index
< tbl
->it_size
; index
++) {
476 tceval
= ppc_md
.tce_get(tbl
, index
+ tbl
->it_offset
);
478 * Freed TCE entry contains 0x7fffffffffffffff on JS20
480 if (tceval
&& (tceval
!= 0x7fffffffffffffffUL
)) {
481 __set_bit(index
, tbl
->it_map
);
485 if ((tbl
->it_size
- tcecount
) < KDUMP_MIN_TCE_ENTRIES
) {
486 printk(KERN_WARNING
"TCE table is full; ");
487 printk(KERN_WARNING
"freeing %d entries for the kdump boot\n",
488 KDUMP_MIN_TCE_ENTRIES
);
489 for (index
= tbl
->it_size
- KDUMP_MIN_TCE_ENTRIES
;
490 index
< tbl
->it_size
; index
++)
491 __clear_bit(index
, tbl
->it_map
);
495 /* Clear the hardware table in case firmware left allocations in it */
496 ppc_md
.tce_free(tbl
, tbl
->it_offset
, tbl
->it_size
);
500 * DMA cannot cross 4 GB boundary. Mark last entry of each 4
501 * GB chunk as reserved.
504 entries_per_4g
= 0x100000000l
>> IOMMU_PAGE_SHIFT
;
506 /* Mark the last bit before a 4GB boundary as used */
507 start_index
= tbl
->it_offset
| (entries_per_4g
- 1);
508 start_index
-= tbl
->it_offset
;
510 end_index
= tbl
->it_size
;
512 for (index
= start_index
; index
< end_index
- 1; index
+= entries_per_4g
)
513 __set_bit(index
, tbl
->it_map
);
517 printk(KERN_INFO
"IOMMU table initialized, virtual merging %s\n",
518 novmerge
? "disabled" : "enabled");
525 void iommu_free_table(struct device_node
*dn
)
527 struct pci_dn
*pdn
= dn
->data
;
528 struct iommu_table
*tbl
= pdn
->iommu_table
;
529 unsigned long bitmap_sz
, i
;
532 if (!tbl
|| !tbl
->it_map
) {
533 printk(KERN_ERR
"%s: expected TCE map for %s\n", __FUNCTION__
,
538 /* verify that table contains no entries */
539 /* it_size is in entries, and we're examining 64 at a time */
540 for (i
= 0; i
< (tbl
->it_size
/64); i
++) {
541 if (tbl
->it_map
[i
] != 0) {
542 printk(KERN_WARNING
"%s: Unexpected TCEs for %s\n",
543 __FUNCTION__
, dn
->full_name
);
548 /* calculate bitmap size in bytes */
549 bitmap_sz
= (tbl
->it_size
+ 7) / 8;
552 order
= get_order(bitmap_sz
);
553 free_pages((unsigned long) tbl
->it_map
, order
);
559 /* Creates TCEs for a user provided buffer. The user buffer must be
560 * contiguous real kernel storage (not vmalloc). The address of the buffer
561 * passed here is the kernel (virtual) address of the buffer. The buffer
562 * need not be page aligned, the dma_addr_t returned will point to the same
563 * byte within the page as vaddr.
565 dma_addr_t
iommu_map_single(struct iommu_table
*tbl
, void *vaddr
,
566 size_t size
, unsigned long mask
,
567 enum dma_data_direction direction
)
569 dma_addr_t dma_handle
= DMA_ERROR_CODE
;
573 BUG_ON(direction
== DMA_NONE
);
575 uaddr
= (unsigned long)vaddr
;
576 npages
= iommu_num_pages(uaddr
, size
);
579 dma_handle
= iommu_alloc(tbl
, vaddr
, npages
, direction
,
580 mask
>> IOMMU_PAGE_SHIFT
, 0);
581 if (dma_handle
== DMA_ERROR_CODE
) {
582 if (printk_ratelimit()) {
583 printk(KERN_INFO
"iommu_alloc failed, "
584 "tbl %p vaddr %p npages %d\n",
588 dma_handle
|= (uaddr
& ~IOMMU_PAGE_MASK
);
594 void iommu_unmap_single(struct iommu_table
*tbl
, dma_addr_t dma_handle
,
595 size_t size
, enum dma_data_direction direction
)
599 BUG_ON(direction
== DMA_NONE
);
602 npages
= iommu_num_pages(dma_handle
, size
);
603 iommu_free(tbl
, dma_handle
, npages
);
607 /* Allocates a contiguous real buffer and creates mappings over it.
608 * Returns the virtual address of the buffer and sets dma_handle
609 * to the dma address (mapping) of the first page.
611 void *iommu_alloc_coherent(struct iommu_table
*tbl
, size_t size
,
612 dma_addr_t
*dma_handle
, unsigned long mask
, gfp_t flag
, int node
)
617 unsigned int nio_pages
, io_order
;
620 size
= PAGE_ALIGN(size
);
621 order
= get_order(size
);
624 * Client asked for way too much space. This is checked later
625 * anyway. It is easier to debug here for the drivers than in
628 if (order
>= IOMAP_MAX_ORDER
) {
629 printk("iommu_alloc_consistent size too large: 0x%lx\n", size
);
636 /* Alloc enough pages (and possibly more) */
637 page
= alloc_pages_node(node
, flag
, order
);
640 ret
= page_address(page
);
641 memset(ret
, 0, size
);
643 /* Set up tces to cover the allocated range */
644 nio_pages
= size
>> IOMMU_PAGE_SHIFT
;
645 io_order
= get_iommu_order(size
);
646 mapping
= iommu_alloc(tbl
, ret
, nio_pages
, DMA_BIDIRECTIONAL
,
647 mask
>> IOMMU_PAGE_SHIFT
, io_order
);
648 if (mapping
== DMA_ERROR_CODE
) {
649 free_pages((unsigned long)ret
, order
);
652 *dma_handle
= mapping
;
656 void iommu_free_coherent(struct iommu_table
*tbl
, size_t size
,
657 void *vaddr
, dma_addr_t dma_handle
)
660 unsigned int nio_pages
;
662 size
= PAGE_ALIGN(size
);
663 nio_pages
= size
>> IOMMU_PAGE_SHIFT
;
664 iommu_free(tbl
, dma_handle
, nio_pages
);
665 size
= PAGE_ALIGN(size
);
666 free_pages((unsigned long)vaddr
, get_order(size
));