2 * Copyright (c) 1997, 1998 Justin T. Gibbs.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions, and the following disclaimer,
10 * without modification, immediately at the beginning of the file.
11 * 2. The name of the author may not be used to endorse or promote products
12 * derived from this software without specific prior written permission.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * $FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.94 2008/08/15 20:51:31 kmacy Exp $
27 * $DragonFly: src/sys/platform/pc32/i386/busdma_machdep.c,v 1.23 2008/06/05 18:06:32 swildner Exp $
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/malloc.h>
35 #include <sys/bus_dma.h>
36 #include <sys/kernel.h>
37 #include <sys/sysctl.h>
40 #include <sys/thread2.h>
41 #include <sys/spinlock2.h>
42 #include <sys/mplock2.h>
45 #include <vm/vm_page.h>
47 /* XXX needed for to access pmap to convert per-proc virtual to physical */
50 #include <vm/vm_map.h>
52 #include <machine/md_var.h>
54 #define MAX_BPAGES 1024
57 * 16 x N declared on stack.
59 #define BUS_DMA_CACHE_SEGMENTS 8
70 bus_dma_filter_t
*filter
;
78 bus_dma_segment_t
*segments
;
79 struct bounce_zone
*bounce_zone
;
88 * bus_dma_tag private flags
90 #define BUS_DMA_BOUNCE_ALIGN BUS_DMA_BUS2
91 #define BUS_DMA_BOUNCE_LOWADDR BUS_DMA_BUS3
92 #define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4
94 #define BUS_DMA_COULD_BOUNCE (BUS_DMA_BOUNCE_LOWADDR | BUS_DMA_BOUNCE_ALIGN)
96 #define BUS_DMAMEM_KMALLOC(dmat) \
97 ((dmat)->maxsize <= PAGE_SIZE && \
98 (dmat)->alignment <= PAGE_SIZE && \
99 (dmat)->lowaddr >= ptoa(Maxmem))
102 vm_offset_t vaddr
; /* kva of bounce buffer */
103 bus_addr_t busaddr
; /* Physical address */
104 vm_offset_t datavaddr
; /* kva of client data */
105 bus_size_t datacount
; /* client data count */
106 STAILQ_ENTRY(bounce_page
) links
;
110 STAILQ_ENTRY(bounce_zone
) links
;
111 STAILQ_HEAD(bp_list
, bounce_page
) bounce_page_list
;
112 STAILQ_HEAD(, bus_dmamap
) bounce_map_waitinglist
;
114 struct spinlock spin
;
125 bus_size_t alignment
;
129 struct sysctl_ctx_list sysctl_ctx
;
130 struct sysctl_oid
*sysctl_tree
;
134 #define BZ_LOCK(bz) spin_lock_wr(&(bz)->spin)
135 #define BZ_UNLOCK(bz) spin_unlock_wr(&(bz)->spin)
137 #define BZ_LOCK(bz) crit_enter()
138 #define BZ_UNLOCK(bz) crit_exit()
141 static struct lwkt_token bounce_zone_tok
=
142 LWKT_TOKEN_INITIALIZER(bounce_zone_tok
);
143 static int busdma_zonecount
;
144 static STAILQ_HEAD(, bounce_zone
) bounce_zone_list
=
145 STAILQ_HEAD_INITIALIZER(bounce_zone_list
);
147 int busdma_swi_pending
;
148 static int total_bounce_pages
;
149 static int max_bounce_pages
= MAX_BPAGES
;
150 static int bounce_alignment
= 1; /* XXX temporary */
152 TUNABLE_INT("hw.busdma.max_bpages", &max_bounce_pages
);
153 TUNABLE_INT("hw.busdma.bounce_alignment", &bounce_alignment
);
156 struct bp_list bpages
;
160 void *buf
; /* unmapped buffer pointer */
161 bus_size_t buflen
; /* unmapped buffer length */
162 bus_dmamap_callback_t
*callback
;
164 STAILQ_ENTRY(bus_dmamap
) links
;
167 static STAILQ_HEAD(, bus_dmamap
) bounce_map_callbacklist
=
168 STAILQ_HEAD_INITIALIZER(bounce_map_callbacklist
);
170 static struct bus_dmamap nobounce_dmamap
;
172 static int alloc_bounce_zone(bus_dma_tag_t
);
173 static int alloc_bounce_pages(bus_dma_tag_t
, u_int
, int);
174 static int reserve_bounce_pages(bus_dma_tag_t
, bus_dmamap_t
, int);
175 static void return_bounce_pages(bus_dma_tag_t
, bus_dmamap_t
);
176 static bus_addr_t
add_bounce_page(bus_dma_tag_t
, bus_dmamap_t
,
177 vm_offset_t
, bus_size_t
);
178 static void free_bounce_page(bus_dma_tag_t
, struct bounce_page
*);
180 static bus_dmamap_t
get_map_waiting(bus_dma_tag_t
);
181 static void add_map_callback(bus_dmamap_t
);
183 SYSCTL_NODE(_hw
, OID_AUTO
, busdma
, CTLFLAG_RD
, 0, "Busdma parameters");
184 SYSCTL_INT(_hw_busdma
, OID_AUTO
, total_bpages
, CTLFLAG_RD
, &total_bounce_pages
,
185 0, "Total bounce pages");
186 SYSCTL_INT(_hw_busdma
, OID_AUTO
, max_bpages
, CTLFLAG_RD
, &max_bounce_pages
,
187 0, "Max bounce pages per bounce zone");
188 SYSCTL_INT(_hw_busdma
, OID_AUTO
, bounce_alignment
, CTLFLAG_RD
,
189 &bounce_alignment
, 0, "Obey alignment constraint");
192 run_filter(bus_dma_tag_t dmat
, bus_addr_t paddr
)
198 if (((paddr
> dmat
->lowaddr
&& paddr
<= dmat
->highaddr
) ||
199 (bounce_alignment
&& (paddr
& (dmat
->alignment
- 1)) != 0))
200 && (dmat
->filter
== NULL
||
201 dmat
->filter(dmat
->filterarg
, paddr
) != 0))
205 } while (retval
== 0 && dmat
!= NULL
);
211 bus_dma_tag_lock(bus_dma_tag_t tag
, bus_dma_segment_t
*cache
)
213 if (tag
->nsegments
<= BUS_DMA_CACHE_SEGMENTS
)
216 spin_lock_wr(&tag
->spin
);
218 return(tag
->segments
);
223 bus_dma_tag_unlock(bus_dma_tag_t tag
)
226 if (tag
->nsegments
> BUS_DMA_CACHE_SEGMENTS
)
227 spin_unlock_wr(&tag
->spin
);
232 * Allocate a device specific dma_tag.
235 bus_dma_tag_create(bus_dma_tag_t parent
, bus_size_t alignment
,
236 bus_size_t boundary
, bus_addr_t lowaddr
,
237 bus_addr_t highaddr
, bus_dma_filter_t
*filter
,
238 void *filterarg
, bus_size_t maxsize
, int nsegments
,
239 bus_size_t maxsegsz
, int flags
, bus_dma_tag_t
*dmat
)
241 bus_dma_tag_t newtag
;
250 if (alignment
& (alignment
- 1))
251 panic("alignment must be power of 2\n");
254 if (boundary
& (boundary
- 1))
255 panic("boundary must be power of 2\n");
256 if (boundary
< maxsegsz
) {
257 kprintf("boundary < maxsegsz:\n");
263 /* Return a NULL tag on failure */
266 newtag
= kmalloc(sizeof(*newtag
), M_DEVBUF
, M_INTWAIT
| M_ZERO
);
269 spin_init(&newtag
->spin
);
271 newtag
->parent
= parent
;
272 newtag
->alignment
= alignment
;
273 newtag
->boundary
= boundary
;
274 newtag
->lowaddr
= trunc_page((vm_paddr_t
)lowaddr
) + (PAGE_SIZE
- 1);
275 newtag
->highaddr
= trunc_page((vm_paddr_t
)highaddr
) + (PAGE_SIZE
- 1);
276 newtag
->filter
= filter
;
277 newtag
->filterarg
= filterarg
;
278 newtag
->maxsize
= maxsize
;
279 newtag
->nsegments
= nsegments
;
280 newtag
->maxsegsz
= maxsegsz
;
281 newtag
->flags
= flags
;
282 newtag
->ref_count
= 1; /* Count ourself */
283 newtag
->map_count
= 0;
284 newtag
->segments
= NULL
;
285 newtag
->bounce_zone
= NULL
;
287 /* Take into account any restrictions imposed by our parent tag */
288 if (parent
!= NULL
) {
289 newtag
->lowaddr
= MIN(parent
->lowaddr
, newtag
->lowaddr
);
290 newtag
->highaddr
= MAX(parent
->highaddr
, newtag
->highaddr
);
292 if (newtag
->boundary
== 0) {
293 newtag
->boundary
= parent
->boundary
;
294 } else if (parent
->boundary
!= 0) {
295 newtag
->boundary
= MIN(parent
->boundary
,
300 newtag
->alignment
= MAX(parent
->alignment
, newtag
->alignment
);
303 if (newtag
->filter
== NULL
) {
305 * Short circuit looking at our parent directly
306 * since we have encapsulated all of its information
308 newtag
->filter
= parent
->filter
;
309 newtag
->filterarg
= parent
->filterarg
;
310 newtag
->parent
= parent
->parent
;
312 if (newtag
->parent
!= NULL
)
316 if (newtag
->lowaddr
< ptoa(Maxmem
))
317 newtag
->flags
|= BUS_DMA_BOUNCE_LOWADDR
;
318 if (bounce_alignment
&& newtag
->alignment
> 1 &&
319 !(newtag
->flags
& BUS_DMA_ALIGNED
))
320 newtag
->flags
|= BUS_DMA_BOUNCE_ALIGN
;
322 if ((newtag
->flags
& BUS_DMA_COULD_BOUNCE
) &&
323 (flags
& BUS_DMA_ALLOCNOW
) != 0) {
324 struct bounce_zone
*bz
;
328 error
= alloc_bounce_zone(newtag
);
331 bz
= newtag
->bounce_zone
;
333 if (ptoa(bz
->total_bpages
) < maxsize
) {
336 if (flags
& BUS_DMA_ONEBPAGE
) {
339 pages
= atop(round_page(maxsize
)) -
341 pages
= MAX(pages
, 1);
344 /* Add pages to our bounce pool */
345 if (alloc_bounce_pages(newtag
, pages
, flags
) < pages
)
348 /* Performed initial allocation */
349 newtag
->flags
|= BUS_DMA_MIN_ALLOC_COMP
;
354 kfree(newtag
, M_DEVBUF
);
361 bus_dma_tag_destroy(bus_dma_tag_t dmat
)
364 if (dmat
->map_count
!= 0)
367 while (dmat
!= NULL
) {
368 bus_dma_tag_t parent
;
370 parent
= dmat
->parent
;
372 if (dmat
->ref_count
== 0) {
373 if (dmat
->segments
!= NULL
)
374 kfree(dmat
->segments
, M_DEVBUF
);
375 kfree(dmat
, M_DEVBUF
);
377 * Last reference count, so
378 * release our reference
379 * count on our parent.
390 bus_dma_tag_getmaxsize(bus_dma_tag_t tag
)
392 return(tag
->maxsize
);
396 * Allocate a handle for mapping from kva/uva/physical
397 * address space into bus device space.
400 bus_dmamap_create(bus_dma_tag_t dmat
, int flags
, bus_dmamap_t
*mapp
)
406 if (dmat
->segments
== NULL
) {
407 KKASSERT(dmat
->nsegments
&& dmat
->nsegments
< 16384);
408 dmat
->segments
= kmalloc(sizeof(bus_dma_segment_t
) *
409 dmat
->nsegments
, M_DEVBUF
, M_INTWAIT
);
412 if (dmat
->flags
& BUS_DMA_COULD_BOUNCE
) {
413 struct bounce_zone
*bz
;
418 if (dmat
->bounce_zone
== NULL
) {
419 error
= alloc_bounce_zone(dmat
);
423 bz
= dmat
->bounce_zone
;
425 *mapp
= kmalloc(sizeof(**mapp
), M_DEVBUF
, M_INTWAIT
| M_ZERO
);
427 /* Initialize the new map */
428 STAILQ_INIT(&((*mapp
)->bpages
));
431 * Attempt to add pages to our pool on a per-instance
432 * basis up to a sane limit.
434 if (dmat
->flags
& BUS_DMA_BOUNCE_ALIGN
) {
435 maxpages
= max_bounce_pages
;
437 maxpages
= MIN(max_bounce_pages
,
438 Maxmem
- atop(dmat
->lowaddr
));
440 if ((dmat
->flags
& BUS_DMA_MIN_ALLOC_COMP
) == 0
441 || (dmat
->map_count
> 0
442 && bz
->total_bpages
< maxpages
)) {
445 if (flags
& BUS_DMA_ONEBPAGE
) {
448 pages
= atop(round_page(dmat
->maxsize
));
449 pages
= MIN(maxpages
- bz
->total_bpages
, pages
);
450 pages
= MAX(pages
, 1);
452 if (alloc_bounce_pages(dmat
, pages
, flags
) < pages
)
455 if ((dmat
->flags
& BUS_DMA_MIN_ALLOC_COMP
) == 0) {
457 dmat
->flags
|= BUS_DMA_MIN_ALLOC_COMP
;
471 * Destroy a handle for mapping from kva/uva/physical
472 * address space into bus device space.
475 bus_dmamap_destroy(bus_dma_tag_t dmat
, bus_dmamap_t map
)
478 if (STAILQ_FIRST(&map
->bpages
) != NULL
)
480 kfree(map
, M_DEVBUF
);
486 static __inline bus_size_t
487 check_kmalloc(bus_dma_tag_t dmat
, const void *vaddr0
, int verify
)
489 bus_size_t maxsize
= 0;
490 uintptr_t vaddr
= (uintptr_t)vaddr0
;
492 if ((vaddr
^ (vaddr
+ dmat
->maxsize
- 1)) & ~PAGE_MASK
) {
493 if (verify
|| bootverbose
)
494 kprintf("boundary check failed\n");
496 print_backtrace(); /* XXX panic */
497 maxsize
= dmat
->maxsize
;
499 if (vaddr
& (dmat
->alignment
- 1)) {
500 if (verify
|| bootverbose
)
501 kprintf("alignment check failed\n");
503 print_backtrace(); /* XXX panic */
504 if (dmat
->maxsize
< dmat
->alignment
)
505 maxsize
= dmat
->alignment
;
507 maxsize
= dmat
->maxsize
;
513 * Allocate a piece of memory that can be efficiently mapped into
514 * bus device space based on the constraints lited in the dma tag.
516 * mapp is degenerate. By definition this allocation should not require
517 * bounce buffers so do not allocate a dma map.
520 bus_dmamem_alloc(bus_dma_tag_t dmat
, void **vaddr
, int flags
,
525 /* If we succeed, no mapping/bouncing will be required */
528 if (dmat
->segments
== NULL
) {
529 KKASSERT(dmat
->nsegments
< 16384);
530 dmat
->segments
= kmalloc(sizeof(bus_dma_segment_t
) *
531 dmat
->nsegments
, M_DEVBUF
, M_INTWAIT
);
534 if (flags
& BUS_DMA_NOWAIT
)
538 if (flags
& BUS_DMA_ZERO
)
541 if (BUS_DMAMEM_KMALLOC(dmat
)) {
544 *vaddr
= kmalloc(dmat
->maxsize
, M_DEVBUF
, mflags
);
548 * Check whether the allocation
549 * - crossed a page boundary
551 * Retry with power-of-2 alignment in the above cases.
553 maxsize
= check_kmalloc(dmat
, *vaddr
, 0);
557 kfree(*vaddr
, M_DEVBUF
);
558 /* XXX check for overflow? */
559 for (size
= 1; size
<= maxsize
; size
<<= 1)
561 *vaddr
= kmalloc(size
, M_DEVBUF
, mflags
);
562 check_kmalloc(dmat
, *vaddr
, 1);
566 * XXX Use Contigmalloc until it is merged into this facility
567 * and handles multi-seg allocations. Nobody is doing
568 * multi-seg allocations yet though.
570 *vaddr
= contigmalloc(dmat
->maxsize
, M_DEVBUF
, mflags
,
571 0ul, dmat
->lowaddr
, dmat
->alignment
, dmat
->boundary
);
579 * Free a piece of memory and it's allociated dmamap, that was allocated
580 * via bus_dmamem_alloc. Make the same choice for free/contigfree.
583 bus_dmamem_free(bus_dma_tag_t dmat
, void *vaddr
, bus_dmamap_t map
)
586 * dmamem does not need to be bounced, so the map should be
590 panic("bus_dmamem_free: Invalid map freed\n");
591 if (BUS_DMAMEM_KMALLOC(dmat
))
592 kfree(vaddr
, M_DEVBUF
);
594 contigfree(vaddr
, dmat
->maxsize
, M_DEVBUF
);
597 static __inline vm_paddr_t
598 _bus_dma_extract(pmap_t pmap
, vm_offset_t vaddr
)
601 return pmap_extract(pmap
, vaddr
);
603 return pmap_kextract(vaddr
);
607 * Utility function to load a linear buffer. lastaddrp holds state
608 * between invocations (for multiple-buffer loads). segp contains
609 * the segment following the starting one on entrace, and the ending
610 * segment on exit. first indicates if this is the first invocation
614 _bus_dmamap_load_buffer(bus_dma_tag_t dmat
,
616 void *buf
, bus_size_t buflen
,
617 bus_dma_segment_t
*segments
,
621 vm_paddr_t
*lastpaddrp
,
626 vm_paddr_t paddr
, nextpaddr
;
627 bus_dma_segment_t
*sg
;
632 map
= &nobounce_dmamap
;
635 if (dmat
->flags
& BUS_DMA_ALIGNED
)
636 KKASSERT(((uintptr_t)buf
& (dmat
->alignment
- 1)) == 0);
640 * If we are being called during a callback, pagesneeded will
641 * be non-zero, so we can avoid doing the work twice.
643 if ((dmat
->flags
& BUS_DMA_COULD_BOUNCE
) &&
644 map
!= &nobounce_dmamap
&& map
->pagesneeded
== 0) {
645 vm_offset_t vendaddr
;
648 * Count the number of bounce pages
649 * needed in order to complete this transfer
651 vaddr
= (vm_offset_t
)buf
;
652 vendaddr
= (vm_offset_t
)buf
+ buflen
;
654 while (vaddr
< vendaddr
) {
655 paddr
= _bus_dma_extract(pmap
, vaddr
);
656 if (run_filter(dmat
, paddr
) != 0)
658 vaddr
+= (PAGE_SIZE
- ((vm_offset_t
)vaddr
& PAGE_MASK
));
662 /* Reserve Necessary Bounce Pages */
663 if (map
->pagesneeded
!= 0) {
664 struct bounce_zone
*bz
;
666 bz
= dmat
->bounce_zone
;
668 if (flags
& BUS_DMA_NOWAIT
) {
669 if (reserve_bounce_pages(dmat
, map
, 0) != 0) {
675 if (reserve_bounce_pages(dmat
, map
, 1) != 0) {
676 /* Queue us for resources */
679 map
->buflen
= buflen
;
682 &dmat
->bounce_zone
->bounce_map_waitinglist
,
686 return (EINPROGRESS
);
692 KKASSERT(*segp
>= 1 && *segp
<= nsegments
);
694 sg
= &segments
[seg
- 1];
696 vaddr
= (vm_offset_t
)buf
;
697 nextpaddr
= *lastpaddrp
;
698 bmask
= ~(dmat
->boundary
- 1); /* note: will be 0 if boundary is 0 */
700 /* force at least one segment */
707 paddr
= _bus_dma_extract(pmap
, vaddr
);
708 size
= PAGE_SIZE
- (paddr
& PAGE_MASK
);
711 if (map
->pagesneeded
!= 0 && run_filter(dmat
, paddr
)) {
713 * note: this paddr has the same in-page offset
714 * as vaddr and thus the paddr above, so the
715 * size does not have to be recalculated
717 paddr
= add_bounce_page(dmat
, map
, vaddr
, size
);
721 * Fill in the bus_dma_segment
727 } else if (paddr
== nextpaddr
) {
737 nextpaddr
= paddr
+ size
;
740 * Handle maxsegsz and boundary issues with a nested loop
746 * Limit to the boundary and maximum segment size
748 if (((nextpaddr
- 1) ^ sg
->ds_addr
) & bmask
) {
749 tmpsize
= dmat
->boundary
-
750 (sg
->ds_addr
& ~bmask
);
751 if (tmpsize
> dmat
->maxsegsz
)
752 tmpsize
= dmat
->maxsegsz
;
753 KKASSERT(tmpsize
< sg
->ds_len
);
754 } else if (sg
->ds_len
> dmat
->maxsegsz
) {
755 tmpsize
= dmat
->maxsegsz
;
761 * Futz, split the data into a new segment.
763 if (seg
>= nsegments
)
765 sg
[1].ds_len
= sg
[0].ds_len
- tmpsize
;
766 sg
[1].ds_addr
= sg
[0].ds_addr
+ tmpsize
;
767 sg
[0].ds_len
= tmpsize
;
777 } while (buflen
> 0);
783 *lastpaddrp
= nextpaddr
;
786 if (error
&& (dmat
->flags
& BUS_DMA_COULD_BOUNCE
) &&
787 map
!= &nobounce_dmamap
) {
788 _bus_dmamap_unload(dmat
, map
);
789 return_bounce_pages(dmat
, map
);
795 * Map the buffer buf into bus space using the dmamap map.
798 bus_dmamap_load(bus_dma_tag_t dmat
, bus_dmamap_t map
, void *buf
,
799 bus_size_t buflen
, bus_dmamap_callback_t
*callback
,
800 void *callback_arg
, int flags
)
802 bus_dma_segment_t cache_segments
[BUS_DMA_CACHE_SEGMENTS
];
803 bus_dma_segment_t
*segments
;
804 vm_paddr_t lastaddr
= 0;
805 int error
, nsegs
= 1;
810 * Follow old semantics. Once all of the callers are fixed,
811 * we should get rid of these internal flag "adjustment".
813 flags
&= ~BUS_DMA_NOWAIT
;
814 flags
|= BUS_DMA_WAITOK
;
816 map
->callback
= callback
;
817 map
->callback_arg
= callback_arg
;
820 segments
= bus_dma_tag_lock(dmat
, cache_segments
);
821 error
= _bus_dmamap_load_buffer(dmat
, map
, buf
, buflen
,
822 segments
, dmat
->nsegments
,
823 NULL
, flags
, &lastaddr
, &nsegs
, 1);
824 if (error
== EINPROGRESS
) {
825 bus_dma_tag_unlock(dmat
);
828 callback(callback_arg
, segments
, nsegs
, error
);
829 bus_dma_tag_unlock(dmat
);
834 * Like _bus_dmamap_load(), but for mbufs.
837 bus_dmamap_load_mbuf(bus_dma_tag_t dmat
, bus_dmamap_t map
,
839 bus_dmamap_callback2_t
*callback
, void *callback_arg
,
842 bus_dma_segment_t cache_segments
[BUS_DMA_CACHE_SEGMENTS
];
843 bus_dma_segment_t
*segments
;
848 * Follow old semantics. Once all of the callers are fixed,
849 * we should get rid of these internal flag "adjustment".
851 flags
&= ~BUS_DMA_WAITOK
;
852 flags
|= BUS_DMA_NOWAIT
;
854 segments
= bus_dma_tag_lock(dmat
, cache_segments
);
855 error
= bus_dmamap_load_mbuf_segment(dmat
, map
, m0
,
856 segments
, dmat
->nsegments
, &nsegs
, flags
);
858 /* force "no valid mappings" in callback */
859 callback(callback_arg
, segments
, 0,
862 callback(callback_arg
, segments
, nsegs
,
863 m0
->m_pkthdr
.len
, error
);
865 bus_dma_tag_unlock(dmat
);
870 bus_dmamap_load_mbuf_segment(bus_dma_tag_t dmat
, bus_dmamap_t map
,
872 bus_dma_segment_t
*segs
, int maxsegs
,
873 int *nsegs
, int flags
)
879 KASSERT(maxsegs
>= 1, ("invalid maxsegs %d\n", maxsegs
));
880 KASSERT(maxsegs
<= dmat
->nsegments
,
881 ("%d too many segments, dmat only support %d segments\n",
882 maxsegs
, dmat
->nsegments
));
883 KASSERT(flags
& BUS_DMA_NOWAIT
,
884 ("only BUS_DMA_NOWAIT is supported\n"));
886 if (m0
->m_pkthdr
.len
<= dmat
->maxsize
) {
888 vm_paddr_t lastaddr
= 0;
893 for (m
= m0
; m
!= NULL
&& error
== 0; m
= m
->m_next
) {
897 error
= _bus_dmamap_load_buffer(dmat
, map
,
900 NULL
, flags
, &lastaddr
,
902 if (error
== ENOMEM
&& !first
) {
904 * Out of bounce pages due to too many
905 * fragments in the mbuf chain; return
914 KKASSERT(*nsegs
<= maxsegs
&& *nsegs
>= 1);
920 KKASSERT(error
!= EINPROGRESS
);
925 * Like _bus_dmamap_load(), but for uios.
928 bus_dmamap_load_uio(bus_dma_tag_t dmat
, bus_dmamap_t map
,
930 bus_dmamap_callback2_t
*callback
, void *callback_arg
,
934 int nsegs
, error
, first
, i
;
938 bus_dma_segment_t cache_segments
[BUS_DMA_CACHE_SEGMENTS
];
939 bus_dma_segment_t
*segments
;
940 bus_dma_segment_t
*segs
;
943 if (dmat
->nsegments
<= BUS_DMA_CACHE_SEGMENTS
)
944 segments
= cache_segments
;
946 segments
= kmalloc(sizeof(bus_dma_segment_t
) * dmat
->nsegments
,
947 M_DEVBUF
, M_WAITOK
| M_ZERO
);
951 * Follow old semantics. Once all of the callers are fixed,
952 * we should get rid of these internal flag "adjustment".
954 flags
&= ~BUS_DMA_WAITOK
;
955 flags
|= BUS_DMA_NOWAIT
;
957 resid
= (bus_size_t
)uio
->uio_resid
;
961 nsegs_left
= dmat
->nsegments
;
963 if (uio
->uio_segflg
== UIO_USERSPACE
) {
967 KASSERT(td
!= NULL
&& td
->td_proc
!= NULL
,
968 ("bus_dmamap_load_uio: USERSPACE but no proc"));
969 pmap
= vmspace_pmap(td
->td_proc
->p_vmspace
);
978 for (i
= 0; i
< uio
->uio_iovcnt
&& resid
!= 0 && !error
; i
++) {
980 * Now at the first iovec to load. Load each iovec
981 * until we have exhausted the residual count.
984 resid
< iov
[i
].iov_len
? resid
: iov
[i
].iov_len
;
985 caddr_t addr
= (caddr_t
) iov
[i
].iov_base
;
987 error
= _bus_dmamap_load_buffer(dmat
, map
, addr
, minlen
,
989 pmap
, flags
, &lastaddr
, &nsegs
, first
);
1000 * Minimum one DMA segment, even if 0-length buffer.
1002 if (nsegs_left
== dmat
->nsegments
)
1006 /* force "no valid mappings" in callback */
1007 callback(callback_arg
, segments
, 0,
1010 callback(callback_arg
, segments
, dmat
->nsegments
- nsegs_left
,
1011 (bus_size_t
)uio
->uio_resid
, error
);
1013 if (dmat
->nsegments
> BUS_DMA_CACHE_SEGMENTS
)
1014 kfree(segments
, M_DEVBUF
);
1019 * Release the mapping held by map.
1022 _bus_dmamap_unload(bus_dma_tag_t dmat
, bus_dmamap_t map
)
1024 struct bounce_page
*bpage
;
1026 while ((bpage
= STAILQ_FIRST(&map
->bpages
)) != NULL
) {
1027 STAILQ_REMOVE_HEAD(&map
->bpages
, links
);
1028 free_bounce_page(dmat
, bpage
);
1033 _bus_dmamap_sync(bus_dma_tag_t dmat
, bus_dmamap_t map
, bus_dmasync_op_t op
)
1035 struct bounce_page
*bpage
;
1037 if ((bpage
= STAILQ_FIRST(&map
->bpages
)) != NULL
) {
1039 * Handle data bouncing. We might also
1040 * want to add support for invalidating
1041 * the caches on broken hardware
1044 case BUS_DMASYNC_PREWRITE
:
1045 while (bpage
!= NULL
) {
1046 bcopy((void *)bpage
->datavaddr
,
1047 (void *)bpage
->vaddr
,
1049 bpage
= STAILQ_NEXT(bpage
, links
);
1051 dmat
->bounce_zone
->total_bounced
++;
1054 case BUS_DMASYNC_POSTREAD
:
1055 while (bpage
!= NULL
) {
1056 bcopy((void *)bpage
->vaddr
,
1057 (void *)bpage
->datavaddr
,
1059 bpage
= STAILQ_NEXT(bpage
, links
);
1061 dmat
->bounce_zone
->total_bounced
++;
1064 case BUS_DMASYNC_PREREAD
:
1065 case BUS_DMASYNC_POSTWRITE
:
1073 alloc_bounce_zone(bus_dma_tag_t dmat
)
1075 struct bounce_zone
*bz
, *new_bz
;
1078 KASSERT(dmat
->bounce_zone
== NULL
,
1079 ("bounce zone was already assigned\n"));
1081 new_bz
= kmalloc(sizeof(*new_bz
), M_DEVBUF
, M_INTWAIT
| M_ZERO
);
1083 lwkt_gettoken(&ref
, &bounce_zone_tok
);
1085 /* Check to see if we already have a suitable zone */
1086 STAILQ_FOREACH(bz
, &bounce_zone_list
, links
) {
1087 if (dmat
->alignment
<= bz
->alignment
&&
1088 dmat
->lowaddr
>= bz
->lowaddr
) {
1089 lwkt_reltoken(&ref
);
1091 dmat
->bounce_zone
= bz
;
1092 kfree(new_bz
, M_DEVBUF
);
1099 spin_init(&bz
->spin
);
1101 STAILQ_INIT(&bz
->bounce_page_list
);
1102 STAILQ_INIT(&bz
->bounce_map_waitinglist
);
1103 bz
->free_bpages
= 0;
1104 bz
->reserved_bpages
= 0;
1105 bz
->active_bpages
= 0;
1106 bz
->lowaddr
= dmat
->lowaddr
;
1107 bz
->alignment
= round_page(dmat
->alignment
);
1108 ksnprintf(bz
->zoneid
, 8, "zone%d", busdma_zonecount
);
1110 ksnprintf(bz
->lowaddrid
, 18, "%#jx", (uintmax_t)bz
->lowaddr
);
1111 STAILQ_INSERT_TAIL(&bounce_zone_list
, bz
, links
);
1113 lwkt_reltoken(&ref
);
1115 dmat
->bounce_zone
= bz
;
1117 sysctl_ctx_init(&bz
->sysctl_ctx
);
1118 bz
->sysctl_tree
= SYSCTL_ADD_NODE(&bz
->sysctl_ctx
,
1119 SYSCTL_STATIC_CHILDREN(_hw_busdma
), OID_AUTO
, bz
->zoneid
,
1121 if (bz
->sysctl_tree
== NULL
) {
1122 sysctl_ctx_free(&bz
->sysctl_ctx
);
1123 return 0; /* XXX error code? */
1126 SYSCTL_ADD_INT(&bz
->sysctl_ctx
,
1127 SYSCTL_CHILDREN(bz
->sysctl_tree
), OID_AUTO
,
1128 "total_bpages", CTLFLAG_RD
, &bz
->total_bpages
, 0,
1129 "Total bounce pages");
1130 SYSCTL_ADD_INT(&bz
->sysctl_ctx
,
1131 SYSCTL_CHILDREN(bz
->sysctl_tree
), OID_AUTO
,
1132 "free_bpages", CTLFLAG_RD
, &bz
->free_bpages
, 0,
1133 "Free bounce pages");
1134 SYSCTL_ADD_INT(&bz
->sysctl_ctx
,
1135 SYSCTL_CHILDREN(bz
->sysctl_tree
), OID_AUTO
,
1136 "reserved_bpages", CTLFLAG_RD
, &bz
->reserved_bpages
, 0,
1137 "Reserved bounce pages");
1138 SYSCTL_ADD_INT(&bz
->sysctl_ctx
,
1139 SYSCTL_CHILDREN(bz
->sysctl_tree
), OID_AUTO
,
1140 "active_bpages", CTLFLAG_RD
, &bz
->active_bpages
, 0,
1141 "Active bounce pages");
1142 SYSCTL_ADD_INT(&bz
->sysctl_ctx
,
1143 SYSCTL_CHILDREN(bz
->sysctl_tree
), OID_AUTO
,
1144 "total_bounced", CTLFLAG_RD
, &bz
->total_bounced
, 0,
1145 "Total bounce requests");
1146 SYSCTL_ADD_INT(&bz
->sysctl_ctx
,
1147 SYSCTL_CHILDREN(bz
->sysctl_tree
), OID_AUTO
,
1148 "total_deferred", CTLFLAG_RD
, &bz
->total_deferred
, 0,
1149 "Total bounce requests that were deferred");
1150 SYSCTL_ADD_INT(&bz
->sysctl_ctx
,
1151 SYSCTL_CHILDREN(bz
->sysctl_tree
), OID_AUTO
,
1152 "reserve_failed", CTLFLAG_RD
, &bz
->reserve_failed
, 0,
1153 "Total bounce page reservations that were failed");
1154 SYSCTL_ADD_STRING(&bz
->sysctl_ctx
,
1155 SYSCTL_CHILDREN(bz
->sysctl_tree
), OID_AUTO
,
1156 "lowaddr", CTLFLAG_RD
, bz
->lowaddrid
, 0, "");
1157 SYSCTL_ADD_INT(&bz
->sysctl_ctx
,
1158 SYSCTL_CHILDREN(bz
->sysctl_tree
), OID_AUTO
,
1159 "alignment", CTLFLAG_RD
, &bz
->alignment
, 0, "");
1165 alloc_bounce_pages(bus_dma_tag_t dmat
, u_int numpages
, int flags
)
1167 struct bounce_zone
*bz
= dmat
->bounce_zone
;
1168 int count
= 0, mflags
;
1170 if (flags
& BUS_DMA_NOWAIT
)
1175 while (numpages
> 0) {
1176 struct bounce_page
*bpage
;
1178 bpage
= kmalloc(sizeof(*bpage
), M_DEVBUF
, M_INTWAIT
| M_ZERO
);
1180 bpage
->vaddr
= (vm_offset_t
)contigmalloc(PAGE_SIZE
, M_DEVBUF
,
1184 if (bpage
->vaddr
== 0) {
1185 kfree(bpage
, M_DEVBUF
);
1188 bpage
->busaddr
= pmap_kextract(bpage
->vaddr
);
1191 STAILQ_INSERT_TAIL(&bz
->bounce_page_list
, bpage
, links
);
1192 total_bounce_pages
++;
1203 /* Assume caller holds bounce zone spinlock */
1205 reserve_bounce_pages(bus_dma_tag_t dmat
, bus_dmamap_t map
, int commit
)
1207 struct bounce_zone
*bz
= dmat
->bounce_zone
;
1210 pages
= MIN(bz
->free_bpages
, map
->pagesneeded
- map
->pagesreserved
);
1211 if (!commit
&& map
->pagesneeded
> (map
->pagesreserved
+ pages
)) {
1212 bz
->reserve_failed
++;
1213 return (map
->pagesneeded
- (map
->pagesreserved
+ pages
));
1216 bz
->free_bpages
-= pages
;
1218 bz
->reserved_bpages
+= pages
;
1219 KKASSERT(bz
->reserved_bpages
<= bz
->total_bpages
);
1221 map
->pagesreserved
+= pages
;
1222 pages
= map
->pagesneeded
- map
->pagesreserved
;
1228 return_bounce_pages(bus_dma_tag_t dmat
, bus_dmamap_t map
)
1230 struct bounce_zone
*bz
= dmat
->bounce_zone
;
1231 int reserved
= map
->pagesreserved
;
1232 bus_dmamap_t wait_map
;
1234 map
->pagesreserved
= 0;
1235 map
->pagesneeded
= 0;
1242 bz
->free_bpages
+= reserved
;
1243 KKASSERT(bz
->free_bpages
<= bz
->total_bpages
);
1245 KKASSERT(bz
->reserved_bpages
>= reserved
);
1246 bz
->reserved_bpages
-= reserved
;
1248 wait_map
= get_map_waiting(dmat
);
1252 if (wait_map
!= NULL
)
1253 add_map_callback(map
);
1257 add_bounce_page(bus_dma_tag_t dmat
, bus_dmamap_t map
, vm_offset_t vaddr
,
1260 struct bounce_zone
*bz
= dmat
->bounce_zone
;
1261 struct bounce_page
*bpage
;
1263 KASSERT(map
->pagesneeded
> 0, ("map doesn't need any pages"));
1266 KASSERT(map
->pagesreserved
> 0, ("map doesn't reserve any pages"));
1267 map
->pagesreserved
--;
1271 bpage
= STAILQ_FIRST(&bz
->bounce_page_list
);
1272 KASSERT(bpage
!= NULL
, ("free page list is empty"));
1273 STAILQ_REMOVE_HEAD(&bz
->bounce_page_list
, links
);
1275 KKASSERT(bz
->reserved_bpages
> 0);
1276 bz
->reserved_bpages
--;
1278 bz
->active_bpages
++;
1279 KKASSERT(bz
->active_bpages
<= bz
->total_bpages
);
1283 bpage
->datavaddr
= vaddr
;
1284 bpage
->datacount
= size
;
1285 STAILQ_INSERT_TAIL(&map
->bpages
, bpage
, links
);
1286 return bpage
->busaddr
;
1290 free_bounce_page(bus_dma_tag_t dmat
, struct bounce_page
*bpage
)
1292 struct bounce_zone
*bz
= dmat
->bounce_zone
;
1295 bpage
->datavaddr
= 0;
1296 bpage
->datacount
= 0;
1300 STAILQ_INSERT_HEAD(&bz
->bounce_page_list
, bpage
, links
);
1303 KKASSERT(bz
->free_bpages
<= bz
->total_bpages
);
1305 KKASSERT(bz
->active_bpages
> 0);
1306 bz
->active_bpages
--;
1308 map
= get_map_waiting(dmat
);
1313 add_map_callback(map
);
1316 /* Assume caller holds bounce zone spinlock */
1318 get_map_waiting(bus_dma_tag_t dmat
)
1320 struct bounce_zone
*bz
= dmat
->bounce_zone
;
1323 map
= STAILQ_FIRST(&bz
->bounce_map_waitinglist
);
1325 if (reserve_bounce_pages(map
->dmat
, map
, 1) == 0) {
1326 STAILQ_REMOVE_HEAD(&bz
->bounce_map_waitinglist
, links
);
1327 bz
->total_deferred
++;
1336 add_map_callback(bus_dmamap_t map
)
1338 /* XXX callbacklist is not MPSAFE */
1341 STAILQ_INSERT_TAIL(&bounce_map_callbacklist
, map
, links
);
1342 busdma_swi_pending
= 1;
1354 while ((map
= STAILQ_FIRST(&bounce_map_callbacklist
)) != NULL
) {
1355 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist
, links
);
1357 bus_dmamap_load(map
->dmat
, map
, map
->buf
, map
->buflen
,
1358 map
->callback
, map
->callback_arg
, /*flags*/0);