2 * Copyright (c) 2012 EMC Corp.
5 * Copyright (c) 1997, 1998 Justin T. Gibbs.
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
35 #include <sys/param.h>
37 #include <sys/systm.h>
40 #include <sys/callout.h>
42 #include <sys/memdesc.h>
47 #include <vm/vm_page.h>
48 #include <vm/vm_map.h>
52 #include <cam/cam_ccb.h>
54 #include <machine/bus.h>
57 * Load up data starting at offset within a region specified by a
58 * list of virtual address ranges until either length or the region
62 _bus_dmamap_load_vlist(bus_dma_tag_t dmat
, bus_dmamap_t map
,
63 bus_dma_segment_t
*list
, int sglist_cnt
, struct pmap
*pmap
, int *nsegs
,
64 int flags
, size_t offset
, size_t length
)
69 for (; sglist_cnt
> 0 && length
!= 0; sglist_cnt
--, list
++) {
73 KASSERT((offset
< list
->ds_len
),
74 ("Invalid mid-segment offset"));
75 addr
= (char *)(uintptr_t)list
->ds_addr
+ offset
;
76 ds_len
= list
->ds_len
- offset
;
81 KASSERT((ds_len
!= 0), ("Segment length is zero"));
82 error
= _bus_dmamap_load_buffer(dmat
, map
, addr
, ds_len
, pmap
,
91 * Load a list of physical addresses.
94 _bus_dmamap_load_plist(bus_dma_tag_t dmat
, bus_dmamap_t map
,
95 bus_dma_segment_t
*list
, int sglist_cnt
, int *nsegs
, int flags
)
100 for (; sglist_cnt
> 0; sglist_cnt
--, list
++) {
101 error
= _bus_dmamap_load_phys(dmat
, map
,
102 (vm_paddr_t
)list
->ds_addr
, list
->ds_len
, flags
, NULL
,
111 * Load an mbuf chain.
114 _bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat
, bus_dmamap_t map
,
115 struct mbuf
*m0
, bus_dma_segment_t
*segs
, int *nsegs
, int flags
)
121 for (m
= m0
; m
!= NULL
&& error
== 0; m
= m
->m_next
) {
123 error
= _bus_dmamap_load_buffer(dmat
, map
, m
->m_data
,
124 m
->m_len
, kernel_pmap
, flags
| BUS_DMA_LOAD_MBUF
,
128 CTR5(KTR_BUSDMA
, "%s: tag %p tag flags 0x%x error %d nsegs %d",
129 __func__
, dmat
, flags
, error
, *nsegs
);
134 * Load tlen data starting at offset within a region specified by a list of
138 _bus_dmamap_load_pages(bus_dma_tag_t dmat
, bus_dmamap_t map
,
139 vm_page_t
*pages
, bus_size_t tlen
, int offset
, int *nsegs
, int flags
)
145 for (i
= 0, error
= 0; error
== 0 && tlen
> 0; i
++, tlen
-= len
) {
146 len
= min(PAGE_SIZE
- offset
, tlen
);
147 paddr
= VM_PAGE_TO_PHYS(pages
[i
]) + offset
;
148 error
= _bus_dmamap_load_phys(dmat
, map
, paddr
, len
,
156 * Load from block io.
159 _bus_dmamap_load_bio(bus_dma_tag_t dmat
, bus_dmamap_t map
, struct bio
*bio
,
160 int *nsegs
, int flags
)
163 if ((bio
->bio_flags
& BIO_VLIST
) != 0) {
164 bus_dma_segment_t
*segs
= (bus_dma_segment_t
*)bio
->bio_data
;
165 return (_bus_dmamap_load_vlist(dmat
, map
, segs
, bio
->bio_ma_n
,
166 kernel_pmap
, nsegs
, flags
, bio
->bio_ma_offset
,
170 if ((bio
->bio_flags
& BIO_UNMAPPED
) != 0)
171 return (_bus_dmamap_load_pages(dmat
, map
, bio
->bio_ma
,
172 bio
->bio_bcount
, bio
->bio_ma_offset
, nsegs
, flags
));
174 return (_bus_dmamap_load_buffer(dmat
, map
, bio
->bio_data
,
175 bio
->bio_bcount
, kernel_pmap
, flags
, NULL
, nsegs
));
179 bus_dmamap_load_ma_triv(bus_dma_tag_t dmat
, bus_dmamap_t map
,
180 struct vm_page
**ma
, bus_size_t tlen
, int ma_offs
, int flags
,
181 bus_dma_segment_t
*segs
, int *segp
)
188 for (i
= 0; tlen
> 0; i
++, tlen
-= len
) {
189 len
= min(PAGE_SIZE
- ma_offs
, tlen
);
190 paddr
= VM_PAGE_TO_PHYS(ma
[i
]) + ma_offs
;
191 error
= _bus_dmamap_load_phys(dmat
, map
, paddr
, len
,
201 * Load a cam control block.
204 _bus_dmamap_load_ccb(bus_dma_tag_t dmat
, bus_dmamap_t map
, union ccb
*ccb
,
205 int *nsegs
, int flags
)
207 struct ccb_hdr
*ccb_h
;
215 switch (ccb_h
->func_code
) {
217 struct ccb_scsiio
*csio
;
220 data_ptr
= csio
->data_ptr
;
221 dxfer_len
= csio
->dxfer_len
;
222 sglist_cnt
= csio
->sglist_cnt
;
225 case XPT_CONT_TARGET_IO
: {
226 struct ccb_scsiio
*ctio
;
229 data_ptr
= ctio
->data_ptr
;
230 dxfer_len
= ctio
->dxfer_len
;
231 sglist_cnt
= ctio
->sglist_cnt
;
235 struct ccb_ataio
*ataio
;
238 data_ptr
= ataio
->data_ptr
;
239 dxfer_len
= ataio
->dxfer_len
;
244 panic("_bus_dmamap_load_ccb: Unsupported func code %d",
248 switch ((ccb_h
->flags
& CAM_DATA_MASK
)) {
250 error
= _bus_dmamap_load_buffer(dmat
, map
, data_ptr
, dxfer_len
,
251 kernel_pmap
, flags
, NULL
, nsegs
);
254 error
= _bus_dmamap_load_phys(dmat
, map
,
255 (vm_paddr_t
)(uintptr_t)data_ptr
, dxfer_len
, flags
, NULL
,
259 error
= _bus_dmamap_load_vlist(dmat
, map
,
260 (bus_dma_segment_t
*)data_ptr
, sglist_cnt
, kernel_pmap
,
261 nsegs
, flags
, 0, dxfer_len
);
263 case CAM_DATA_SG_PADDR
:
264 error
= _bus_dmamap_load_plist(dmat
, map
,
265 (bus_dma_segment_t
*)data_ptr
, sglist_cnt
, nsegs
, flags
);
268 error
= _bus_dmamap_load_bio(dmat
, map
, (struct bio
*)data_ptr
,
272 panic("_bus_dmamap_load_ccb: flags 0x%X unimplemented",
282 _bus_dmamap_load_uio(bus_dma_tag_t dmat
, bus_dmamap_t map
, struct uio
*uio
,
283 int *nsegs
, int flags
)
292 if (uio
->uio_segflg
== UIO_USERSPACE
) {
293 KASSERT(uio
->uio_td
!= NULL
,
294 ("bus_dmamap_load_uio: USERSPACE but no proc"));
295 pmap
= vmspace_pmap(uio
->uio_td
->td_proc
->p_vmspace
);
298 resid
= uio
->uio_resid
;
302 for (i
= 0; i
< uio
->uio_iovcnt
&& resid
!= 0 && !error
; i
++) {
304 * Now at the first iovec to load. Load each iovec
305 * until we have exhausted the residual count.
308 addr
= (caddr_t
) iov
[i
].iov_base
;
309 minlen
= resid
< iov
[i
].iov_len
? resid
: iov
[i
].iov_len
;
311 error
= _bus_dmamap_load_buffer(dmat
, map
, addr
,
312 minlen
, pmap
, flags
, NULL
, nsegs
);
321 * Map the buffer buf into bus space using the dmamap map.
324 bus_dmamap_load(bus_dma_tag_t dmat
, bus_dmamap_t map
, void *buf
,
325 bus_size_t buflen
, bus_dmamap_callback_t
*callback
,
326 void *callback_arg
, int flags
)
328 bus_dma_segment_t
*segs
;
333 if ((flags
& BUS_DMA_NOWAIT
) == 0) {
334 mem
= memdesc_vaddr(buf
, buflen
);
335 _bus_dmamap_waitok(dmat
, map
, &mem
, callback
, callback_arg
);
339 error
= _bus_dmamap_load_buffer(dmat
, map
, buf
, buflen
, kernel_pmap
,
340 flags
, NULL
, &nsegs
);
343 CTR5(KTR_BUSDMA
, "%s: tag %p tag flags 0x%x error %d nsegs %d",
344 __func__
, dmat
, flags
, error
, nsegs
);
346 if (error
== EINPROGRESS
)
349 segs
= _bus_dmamap_complete(dmat
, map
, NULL
, nsegs
, error
);
351 (*callback
)(callback_arg
, segs
, 0, error
);
353 (*callback
)(callback_arg
, segs
, nsegs
, 0);
356 * Return ENOMEM to the caller so that it can pass it up the stack.
357 * This error only happens when NOWAIT is set, so deferral is disabled.
366 bus_dmamap_load_mbuf(bus_dma_tag_t dmat
, bus_dmamap_t map
, struct mbuf
*m0
,
367 bus_dmamap_callback2_t
*callback
, void *callback_arg
, int flags
)
369 bus_dma_segment_t
*segs
;
374 flags
|= BUS_DMA_NOWAIT
;
376 error
= _bus_dmamap_load_mbuf_sg(dmat
, map
, m0
, NULL
, &nsegs
, flags
);
379 segs
= _bus_dmamap_complete(dmat
, map
, NULL
, nsegs
, error
);
381 (*callback
)(callback_arg
, segs
, 0, 0, error
);
383 (*callback
)(callback_arg
, segs
, nsegs
, m0
->m_pkthdr
.len
, error
);
385 CTR5(KTR_BUSDMA
, "%s: tag %p tag flags 0x%x error %d nsegs %d",
386 __func__
, dmat
, flags
, error
, nsegs
);
391 bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat
, bus_dmamap_t map
, struct mbuf
*m0
,
392 bus_dma_segment_t
*segs
, int *nsegs
, int flags
)
396 flags
|= BUS_DMA_NOWAIT
;
398 error
= _bus_dmamap_load_mbuf_sg(dmat
, map
, m0
, segs
, nsegs
, flags
);
400 _bus_dmamap_complete(dmat
, map
, segs
, *nsegs
, error
);
405 bus_dmamap_load_uio(bus_dma_tag_t dmat
, bus_dmamap_t map
, struct uio
*uio
,
406 bus_dmamap_callback2_t
*callback
, void *callback_arg
, int flags
)
408 bus_dma_segment_t
*segs
;
411 flags
|= BUS_DMA_NOWAIT
;
413 error
= _bus_dmamap_load_uio(dmat
, map
, uio
, &nsegs
, flags
);
416 segs
= _bus_dmamap_complete(dmat
, map
, NULL
, nsegs
, error
);
418 (*callback
)(callback_arg
, segs
, 0, 0, error
);
420 (*callback
)(callback_arg
, segs
, nsegs
, uio
->uio_resid
, error
);
422 CTR5(KTR_BUSDMA
, "%s: tag %p tag flags 0x%x error %d nsegs %d",
423 __func__
, dmat
, flags
, error
, nsegs
);
428 bus_dmamap_load_ccb(bus_dma_tag_t dmat
, bus_dmamap_t map
, union ccb
*ccb
,
429 bus_dmamap_callback_t
*callback
, void *callback_arg
,
432 bus_dma_segment_t
*segs
;
433 struct ccb_hdr
*ccb_h
;
439 if ((ccb_h
->flags
& CAM_DIR_MASK
) == CAM_DIR_NONE
) {
440 callback(callback_arg
, NULL
, 0, 0);
443 if ((flags
& BUS_DMA_NOWAIT
) == 0) {
444 mem
= memdesc_ccb(ccb
);
445 _bus_dmamap_waitok(dmat
, map
, &mem
, callback
, callback_arg
);
448 error
= _bus_dmamap_load_ccb(dmat
, map
, ccb
, &nsegs
, flags
);
451 CTR5(KTR_BUSDMA
, "%s: tag %p tag flags 0x%x error %d nsegs %d",
452 __func__
, dmat
, flags
, error
, nsegs
);
454 if (error
== EINPROGRESS
)
457 segs
= _bus_dmamap_complete(dmat
, map
, NULL
, nsegs
, error
);
459 (*callback
)(callback_arg
, segs
, 0, error
);
461 (*callback
)(callback_arg
, segs
, nsegs
, error
);
463 * Return ENOMEM to the caller so that it can pass it up the stack.
464 * This error only happens when NOWAIT is set, so deferral is disabled.
473 bus_dmamap_load_bio(bus_dma_tag_t dmat
, bus_dmamap_t map
, struct bio
*bio
,
474 bus_dmamap_callback_t
*callback
, void *callback_arg
,
477 bus_dma_segment_t
*segs
;
482 if ((flags
& BUS_DMA_NOWAIT
) == 0) {
483 mem
= memdesc_bio(bio
);
484 _bus_dmamap_waitok(dmat
, map
, &mem
, callback
, callback_arg
);
487 error
= _bus_dmamap_load_bio(dmat
, map
, bio
, &nsegs
, flags
);
490 CTR5(KTR_BUSDMA
, "%s: tag %p tag flags 0x%x error %d nsegs %d",
491 __func__
, dmat
, flags
, error
, nsegs
);
493 if (error
== EINPROGRESS
)
496 segs
= _bus_dmamap_complete(dmat
, map
, NULL
, nsegs
, error
);
498 (*callback
)(callback_arg
, segs
, 0, error
);
500 (*callback
)(callback_arg
, segs
, nsegs
, error
);
502 * Return ENOMEM to the caller so that it can pass it up the stack.
503 * This error only happens when NOWAIT is set, so deferral is disabled.
512 bus_dmamap_load_mem(bus_dma_tag_t dmat
, bus_dmamap_t map
,
513 struct memdesc
*mem
, bus_dmamap_callback_t
*callback
,
514 void *callback_arg
, int flags
)
516 bus_dma_segment_t
*segs
;
520 if ((flags
& BUS_DMA_NOWAIT
) == 0)
521 _bus_dmamap_waitok(dmat
, map
, mem
, callback
, callback_arg
);
525 switch (mem
->md_type
) {
527 error
= _bus_dmamap_load_buffer(dmat
, map
, mem
->u
.md_vaddr
,
528 mem
->md_opaque
, kernel_pmap
, flags
, NULL
, &nsegs
);
531 error
= _bus_dmamap_load_phys(dmat
, map
, mem
->u
.md_paddr
,
532 mem
->md_opaque
, flags
, NULL
, &nsegs
);
535 error
= _bus_dmamap_load_vlist(dmat
, map
, mem
->u
.md_list
,
536 mem
->md_opaque
, kernel_pmap
, &nsegs
, flags
, 0, SIZE_T_MAX
);
539 error
= _bus_dmamap_load_plist(dmat
, map
, mem
->u
.md_list
,
540 mem
->md_opaque
, &nsegs
, flags
);
543 error
= _bus_dmamap_load_bio(dmat
, map
, mem
->u
.md_bio
,
547 error
= _bus_dmamap_load_uio(dmat
, map
, mem
->u
.md_uio
,
551 error
= _bus_dmamap_load_mbuf_sg(dmat
, map
, mem
->u
.md_mbuf
,
552 NULL
, &nsegs
, flags
);
555 error
= _bus_dmamap_load_ccb(dmat
, map
, mem
->u
.md_ccb
, &nsegs
,
561 CTR5(KTR_BUSDMA
, "%s: tag %p tag flags 0x%x error %d nsegs %d",
562 __func__
, dmat
, flags
, error
, nsegs
);
564 if (error
== EINPROGRESS
)
567 segs
= _bus_dmamap_complete(dmat
, map
, NULL
, nsegs
, error
);
569 (*callback
)(callback_arg
, segs
, 0, error
);
571 (*callback
)(callback_arg
, segs
, nsegs
, 0);
574 * Return ENOMEM to the caller so that it can pass it up the stack.
575 * This error only happens when NOWAIT is set, so deferral is disabled.