1 /* $FreeBSD: head/sys/dev/usb/usb_busdma.c 261505 2014-02-05 08:02:52Z hselasky $ */
3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 #include <sys/stdint.h>
28 #include <sys/param.h>
29 #include <sys/queue.h>
30 #include <sys/types.h>
31 #include <sys/systm.h>
32 #include <sys/kernel.h>
34 #include <sys/module.h>
36 #include <sys/mutex.h>
37 #include <sys/mutex2.h>
38 #include <sys/condvar.h>
39 #include <sys/sysctl.h>
40 #include <sys/unistd.h>
41 #include <sys/callout.h>
42 #include <sys/malloc.h>
45 #include <bus/u4b/usb.h>
46 #include <bus/u4b/usbdi.h>
47 #include <bus/u4b/usbdi_util.h>
49 #define USB_DEBUG_VAR usb_debug
51 #include <bus/u4b/usb_core.h>
52 #include <bus/u4b/usb_busdma.h>
53 #include <bus/u4b/usb_process.h>
54 #include <bus/u4b/usb_transfer.h>
55 #include <bus/u4b/usb_device.h>
56 #include <bus/u4b/usb_util.h>
57 #include <bus/u4b/usb_debug.h>
59 #include <bus/u4b/usb_controller.h>
60 #include <bus/u4b/usb_bus.h>
63 static void usb_dma_tag_create(struct usb_dma_tag
*, usb_size_t
, usb_size_t
);
64 static void usb_dma_tag_destroy(struct usb_dma_tag
*);
66 static void usb_dma_lock_cb(void *, bus_dma_lock_op_t
);
68 static void usb_pc_alloc_mem_cb(void *, bus_dma_segment_t
*, int, int);
69 static void usb_pc_load_mem_cb(void *, bus_dma_segment_t
*, int, int);
70 static void usb_pc_common_mem_cb(void *, bus_dma_segment_t
*, int, int,
74 /*------------------------------------------------------------------------*
75 * usbd_get_page - lookup DMA-able memory for the given offset
77 * NOTE: Only call this function when the "page_cache" structure has
78 * been properly initialized !
79 *------------------------------------------------------------------------*/
81 usbd_get_page(struct usb_page_cache
*pc
, usb_frlength_t offset
,
82 struct usb_page_search
*res
)
85 struct usb_page
*page
;
89 /* Case 1 - something has been loaded into DMA */
93 /* Case 1a - Kernel Virtual Address */
95 res
->buffer
= USB_ADD_BYTES(pc
->buffer
, offset
);
97 offset
+= pc
->page_offset_buf
;
99 /* compute destination page */
101 page
= pc
->page_start
;
103 if (pc
->ismultiseg
) {
105 page
+= (offset
/ USB_PAGE_SIZE
);
107 offset
%= USB_PAGE_SIZE
;
109 res
->length
= USB_PAGE_SIZE
- offset
;
110 res
->physaddr
= page
->physaddr
+ offset
;
112 res
->length
= (usb_size_t
)-1;
113 res
->physaddr
= page
->physaddr
+ offset
;
117 /* Case 1b - Non Kernel Virtual Address */
119 res
->buffer
= USB_ADD_BYTES(page
->buffer
, offset
);
124 /* Case 2 - Plain PIO */
126 res
->buffer
= USB_ADD_BYTES(pc
->buffer
, offset
);
127 res
->length
= (usb_size_t
)-1;
133 /*------------------------------------------------------------------------*
134 * usbd_copy_in - copy directly to DMA-able memory
135 *------------------------------------------------------------------------*/
137 usbd_copy_in(struct usb_page_cache
*cache
, usb_frlength_t offset
,
138 const void *ptr
, usb_frlength_t len
)
140 struct usb_page_search buf_res
;
144 usbd_get_page(cache
, offset
, &buf_res
);
146 if (buf_res
.length
> len
) {
147 buf_res
.length
= len
;
149 memcpy(buf_res
.buffer
, ptr
, buf_res
.length
);
151 offset
+= buf_res
.length
;
152 len
-= buf_res
.length
;
153 ptr
= USB_ADD_BYTES(ptr
, buf_res
.length
);
157 /*------------------------------------------------------------------------*
158 * usbd_copy_in_user - copy directly to DMA-able memory from userland
163 *------------------------------------------------------------------------*/
166 usbd_copy_in_user(struct usb_page_cache
*cache
, usb_frlength_t offset
,
167 const void *ptr
, usb_frlength_t len
)
169 struct usb_page_search buf_res
;
174 usbd_get_page(cache
, offset
, &buf_res
);
176 if (buf_res
.length
> len
) {
177 buf_res
.length
= len
;
179 error
= copyin(ptr
, buf_res
.buffer
, buf_res
.length
);
183 offset
+= buf_res
.length
;
184 len
-= buf_res
.length
;
185 ptr
= USB_ADD_BYTES(ptr
, buf_res
.length
);
187 return (0); /* success */
191 /*------------------------------------------------------------------------*
192 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory
193 *------------------------------------------------------------------------*/
195 struct usb_m_copy_in_arg
{
196 struct usb_page_cache
*cache
;
197 usb_frlength_t dst_offset
;
201 usbd_m_copy_in_cb(void *arg
, void *src
, uint32_t count
)
203 register struct usb_m_copy_in_arg
*ua
= arg
;
205 usbd_copy_in(ua
->cache
, ua
->dst_offset
, src
, count
);
206 ua
->dst_offset
+= count
;
211 usbd_m_copy_in(struct usb_page_cache
*cache
, usb_frlength_t dst_offset
,
212 struct mbuf
*m
, usb_size_t src_offset
, usb_frlength_t src_len
)
214 struct usb_m_copy_in_arg arg
= {cache
, dst_offset
};
215 (void) m_apply(m
, src_offset
, src_len
, &usbd_m_copy_in_cb
, &arg
);
219 /*------------------------------------------------------------------------*
220 * usb_uiomove - factored out code
221 *------------------------------------------------------------------------*/
224 usb_uiomove(struct usb_page_cache
*pc
, struct uio
*uio
,
225 usb_frlength_t pc_offset
, usb_frlength_t len
)
227 struct usb_page_search res
;
232 usbd_get_page(pc
, pc_offset
, &res
);
234 if (res
.length
> len
) {
238 * "uiomove()" can sleep so one needs to make a wrapper,
239 * exiting the mutex and checking things
241 error
= uiomove(res
.buffer
, res
.length
, uio
);
246 pc_offset
+= res
.length
;
253 /*------------------------------------------------------------------------*
254 * usbd_copy_out - copy directly from DMA-able memory
255 *------------------------------------------------------------------------*/
257 usbd_copy_out(struct usb_page_cache
*cache
, usb_frlength_t offset
,
258 void *ptr
, usb_frlength_t len
)
260 struct usb_page_search res
;
264 usbd_get_page(cache
, offset
, &res
);
266 if (res
.length
> len
) {
269 memcpy(ptr
, res
.buffer
, res
.length
);
271 offset
+= res
.length
;
273 ptr
= USB_ADD_BYTES(ptr
, res
.length
);
277 /*------------------------------------------------------------------------*
278 * usbd_copy_out_user - copy directly from DMA-able memory to userland
283 *------------------------------------------------------------------------*/
286 usbd_copy_out_user(struct usb_page_cache
*cache
, usb_frlength_t offset
,
287 void *ptr
, usb_frlength_t len
)
289 struct usb_page_search res
;
294 usbd_get_page(cache
, offset
, &res
);
296 if (res
.length
> len
) {
299 error
= copyout(res
.buffer
, ptr
, res
.length
);
303 offset
+= res
.length
;
305 ptr
= USB_ADD_BYTES(ptr
, res
.length
);
307 return (0); /* success */
311 /*------------------------------------------------------------------------*
312 * usbd_frame_zero - zero DMA-able memory
313 *------------------------------------------------------------------------*/
315 usbd_frame_zero(struct usb_page_cache
*cache
, usb_frlength_t offset
,
318 struct usb_page_search res
;
322 usbd_get_page(cache
, offset
, &res
);
324 if (res
.length
> len
) {
327 memset(res
.buffer
, 0, res
.length
);
329 offset
+= res
.length
;
336 /*------------------------------------------------------------------------*
337 * usb_dma_lock_cb - dummy callback
338 *------------------------------------------------------------------------*/
341 usb_dma_lock_cb(void *arg
, bus_dma_lock_op_t op
)
343 /* we use "mtx_owned()" instead of this function */
347 /*------------------------------------------------------------------------*
348 * usb_dma_tag_create - allocate a DMA tag
350 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will
351 * allow multi-segment mappings. Else all mappings are single-segment.
352 *------------------------------------------------------------------------*/
354 usb_dma_tag_create(struct usb_dma_tag
*udt
,
355 usb_size_t size
, usb_size_t align
)
359 if (bus_dma_tag_create
360 ( /* parent */ udt
->tag_parent
->tag
,
361 /* alignment */ align
,
363 /* lowaddr */ (2ULL << (udt
->tag_parent
->dma_bits
- 1)) - 1,
364 /* highaddr */ BUS_SPACE_MAXADDR
,
366 /* filterarg */ NULL
,
368 /* nsegments */ (align
== 1 && size
> 1) ? (2 + (size
/ USB_PAGE_SIZE
)) : 1,
369 /* maxsegsz */ (align
== 1 && size
> USB_PAGE_SIZE
) ? USB_PAGE_SIZE
: size
,
370 /* flags */ BUS_DMA_KEEP_PG_OFFSET
,
377 /*------------------------------------------------------------------------*
378 * usb_dma_tag_free - free a DMA tag
379 *------------------------------------------------------------------------*/
381 usb_dma_tag_destroy(struct usb_dma_tag
*udt
)
383 bus_dma_tag_destroy(udt
->tag
);
386 /*------------------------------------------------------------------------*
387 * usb_pc_alloc_mem_cb - BUS-DMA callback function
388 *------------------------------------------------------------------------*/
390 usb_pc_alloc_mem_cb(void *arg
, bus_dma_segment_t
*segs
,
393 usb_pc_common_mem_cb(arg
, segs
, nseg
, error
, 0);
396 /*------------------------------------------------------------------------*
397 * usb_pc_load_mem_cb - BUS-DMA callback function
398 *------------------------------------------------------------------------*/
400 usb_pc_load_mem_cb(void *arg
, bus_dma_segment_t
*segs
,
403 usb_pc_common_mem_cb(arg
, segs
, nseg
, error
, 1);
406 /*------------------------------------------------------------------------*
407 * usb_pc_common_mem_cb - BUS-DMA callback function
408 *------------------------------------------------------------------------*/
410 usb_pc_common_mem_cb(void *arg
, bus_dma_segment_t
*segs
,
411 int nseg
, int error
, uint8_t isload
)
413 struct usb_dma_parent_tag
*uptag
;
414 struct usb_page_cache
*pc
;
421 uptag
= pc
->tag_parent
;
424 * XXX There is sometimes recursive locking here.
425 * XXX We should try to find a better solution.
426 * XXX Until further the "owned" variable does
436 pg
->physaddr
= segs
->ds_addr
& ~(USB_PAGE_SIZE
- 1);
437 rem
= segs
->ds_addr
& (USB_PAGE_SIZE
- 1);
438 pc
->page_offset_buf
= rem
;
439 pc
->page_offset_end
+= rem
;
441 if (rem
!= (USB_P2U(pc
->buffer
) & (USB_PAGE_SIZE
- 1))) {
443 * This check verifies that the physical address is correct:
445 DPRINTFN(0, "Page offset was not preserved\n");
450 while (pc
->ismultiseg
) {
451 off
+= USB_PAGE_SIZE
;
452 if (off
>= (segs
->ds_len
+ rem
)) {
462 pg
->physaddr
= (segs
->ds_addr
+ off
) & ~(USB_PAGE_SIZE
- 1);
466 owned
= lockowned(uptag
->lock
);
468 lockmgr(uptag
->lock
, LK_EXCLUSIVE
);
470 uptag
->dma_error
= (error
? 1 : 0);
472 (uptag
->func
) (uptag
);
474 cv_broadcast(uptag
->cv
);
477 lockmgr(uptag
->lock
, LK_RELEASE
);
480 /*------------------------------------------------------------------------*
481 * usb_pc_alloc_mem - allocate DMA'able memory
486 *------------------------------------------------------------------------*/
488 usb_pc_alloc_mem(struct usb_page_cache
*pc
, struct usb_page
*pg
,
489 usb_size_t size
, usb_size_t align
)
491 struct usb_dma_parent_tag
*uptag
;
492 struct usb_dma_tag
*utag
;
497 uptag
= pc
->tag_parent
;
501 * The alignment must be greater or equal to the
502 * "size" else the object can be split between two
503 * memory pages and we get a problem!
505 while (align
< size
) {
513 * XXX BUS-DMA workaround - FIXME later:
515 * We assume that that the aligment at this point of
516 * the code is greater than or equal to the size and
517 * less than two times the size, so that if we double
518 * the size, the size will be greater than the
521 * The bus-dma system has a check for "alignment"
522 * being less than "size". If that check fails we end
523 * up using contigmalloc which is page based even for
524 * small allocations. Try to avoid that to save
525 * memory, hence we sometimes to a large number of
528 if (size
<= (USB_PAGE_SIZE
/ 2)) {
533 /* get the correct DMA tag */
534 utag
= usb_dma_tag_find(uptag
, size
, align
);
538 /* allocate memory */
539 if (bus_dmamem_alloc(
540 utag
->tag
, &ptr
, (BUS_DMA_WAITOK
| BUS_DMA_COHERENT
), &map
)) {
543 /* setup page cache */
546 pc
->page_offset_buf
= 0;
547 pc
->page_offset_end
= size
;
550 pc
->ismultiseg
= (align
== 1);
552 lockmgr(uptag
->lock
, LK_EXCLUSIVE
);
554 /* load memory into DMA */
555 err
= bus_dmamap_load(
556 utag
->tag
, map
, ptr
, size
, &usb_pc_alloc_mem_cb
,
557 pc
, (BUS_DMA_WAITOK
| BUS_DMA_COHERENT
));
559 if (err
== EINPROGRESS
) {
560 cv_wait(uptag
->cv
, uptag
->lock
);
563 lockmgr(uptag
->lock
, LK_RELEASE
);
565 if (err
|| uptag
->dma_error
) {
566 bus_dmamem_free(utag
->tag
, ptr
, map
);
569 memset(ptr
, 0, size
);
571 usb_pc_cpu_flush(pc
);
576 /* reset most of the page cache */
578 pc
->page_start
= NULL
;
579 pc
->page_offset_buf
= 0;
580 pc
->page_offset_end
= 0;
586 /*------------------------------------------------------------------------*
587 * usb_pc_free_mem - free DMA memory
589 * This function is NULL safe.
590 *------------------------------------------------------------------------*/
592 usb_pc_free_mem(struct usb_page_cache
*pc
)
594 if (pc
&& pc
->buffer
) {
596 bus_dmamap_unload(pc
->tag
, pc
->map
);
598 bus_dmamem_free(pc
->tag
, pc
->buffer
, pc
->map
);
604 /*------------------------------------------------------------------------*
605 * usb_pc_load_mem - load virtual memory into DMA
610 *------------------------------------------------------------------------*/
612 usb_pc_load_mem(struct usb_page_cache
*pc
, usb_size_t size
, uint8_t sync
)
614 /* setup page cache */
615 pc
->page_offset_buf
= 0;
616 pc
->page_offset_end
= size
;
619 KKASSERT(lockowned(pc
->tag_parent
->lock
));
623 struct usb_dma_parent_tag
*uptag
;
626 uptag
= pc
->tag_parent
;
629 * We have to unload the previous loaded DMA
630 * pages before trying to load a new one!
632 bus_dmamap_unload(pc
->tag
, pc
->map
);
635 * Try to load memory into DMA.
637 err
= bus_dmamap_load(
638 pc
->tag
, pc
->map
, pc
->buffer
, size
,
639 &usb_pc_alloc_mem_cb
, pc
, BUS_DMA_WAITOK
);
640 if (err
== EINPROGRESS
) {
641 cv_wait(uptag
->cv
, uptag
->lock
);
644 if (err
|| uptag
->dma_error
) {
650 * We have to unload the previous loaded DMA
651 * pages before trying to load a new one!
653 bus_dmamap_unload(pc
->tag
, pc
->map
);
656 * Try to load memory into DMA. The callback
657 * will be called in all cases:
660 pc
->tag
, pc
->map
, pc
->buffer
, size
,
661 &usb_pc_load_mem_cb
, pc
, BUS_DMA_WAITOK
)) {
667 * Call callback so that refcount is decremented
670 pc
->tag_parent
->dma_error
= 0;
671 (pc
->tag_parent
->func
) (pc
->tag_parent
);
677 /*------------------------------------------------------------------------*
678 * usb_pc_cpu_invalidate - invalidate CPU cache
679 *------------------------------------------------------------------------*/
681 usb_pc_cpu_invalidate(struct usb_page_cache
*pc
)
683 if (pc
->page_offset_end
== pc
->page_offset_buf
) {
684 /* nothing has been loaded into this page cache! */
689 * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the
690 * same time, but in the future we should try to isolate the
691 * different cases to optimise the code. --HPS
693 bus_dmamap_sync(pc
->tag
, pc
->map
, BUS_DMASYNC_POSTREAD
);
694 bus_dmamap_sync(pc
->tag
, pc
->map
, BUS_DMASYNC_PREREAD
);
697 /*------------------------------------------------------------------------*
698 * usb_pc_cpu_flush - flush CPU cache
699 *------------------------------------------------------------------------*/
701 usb_pc_cpu_flush(struct usb_page_cache
*pc
)
703 if (pc
->page_offset_end
== pc
->page_offset_buf
) {
704 /* nothing has been loaded into this page cache! */
707 bus_dmamap_sync(pc
->tag
, pc
->map
, BUS_DMASYNC_PREWRITE
);
710 /*------------------------------------------------------------------------*
711 * usb_pc_dmamap_create - create a DMA map
716 *------------------------------------------------------------------------*/
718 usb_pc_dmamap_create(struct usb_page_cache
*pc
, usb_size_t size
)
720 struct usb_xfer_root
*info
;
721 struct usb_dma_tag
*utag
;
724 info
= USB_DMATAG_TO_XROOT(pc
->tag_parent
);
730 utag
= usb_dma_tag_find(pc
->tag_parent
, size
, 1);
735 if (bus_dmamap_create(utag
->tag
, 0, &pc
->map
)) {
739 return 0; /* success */
744 return 1; /* failure */
747 /*------------------------------------------------------------------------*
748 * usb_pc_dmamap_destroy
750 * This function is NULL safe.
751 *------------------------------------------------------------------------*/
753 usb_pc_dmamap_destroy(struct usb_page_cache
*pc
)
756 bus_dmamap_destroy(pc
->tag
, pc
->map
);
762 /*------------------------------------------------------------------------*
763 * usb_dma_tag_find - factored out code
764 *------------------------------------------------------------------------*/
766 usb_dma_tag_find(struct usb_dma_parent_tag
*udpt
,
767 usb_size_t size
, usb_size_t align
)
769 struct usb_dma_tag
*udt
;
772 USB_ASSERT(align
> 0, ("Invalid parameter align = 0\n"));
773 USB_ASSERT(size
> 0, ("Invalid parameter size = 0\n"));
775 udt
= udpt
->utag_first
;
776 nudt
= udpt
->utag_max
;
780 if (udt
->align
== 0) {
781 usb_dma_tag_create(udt
, size
, align
);
782 if (udt
->tag
== NULL
) {
789 if ((udt
->align
== align
) && (udt
->size
== size
)) {
797 /*------------------------------------------------------------------------*
798 * usb_dma_tag_setup - initialise USB DMA tags
799 *------------------------------------------------------------------------*/
801 usb_dma_tag_setup(struct usb_dma_parent_tag
*udpt
,
802 struct usb_dma_tag
*udt
, bus_dma_tag_t dmat
,
803 struct lock
*lock
, usb_dma_callback_t
*func
,
804 uint8_t ndmabits
, uint8_t nudt
)
806 memset(udpt
, 0, sizeof(*udpt
));
808 /* sanity checking */
812 /* something is corrupt */
815 /* initialise condition variable */
816 cv_init(udpt
->cv
, "USB DMA CV");
818 /* store some information */
822 udpt
->utag_first
= udt
;
823 udpt
->utag_max
= nudt
;
824 udpt
->dma_bits
= ndmabits
;
827 memset(udt
, 0, sizeof(*udt
));
828 udt
->tag_parent
= udpt
;
833 /*------------------------------------------------------------------------*
834 * usb_bus_tag_unsetup - factored out code
835 *------------------------------------------------------------------------*/
837 usb_dma_tag_unsetup(struct usb_dma_parent_tag
*udpt
)
839 struct usb_dma_tag
*udt
;
842 udt
= udpt
->utag_first
;
843 nudt
= udpt
->utag_max
;
848 /* destroy the USB DMA tag */
849 usb_dma_tag_destroy(udt
);
855 if (udpt
->utag_max
) {
856 /* destroy the condition variable */
857 cv_destroy(udpt
->cv
);
861 /*------------------------------------------------------------------------*
864 * This function handles loading of virtual buffers into DMA and is
865 * only called when "dma_refcount" is zero.
866 *------------------------------------------------------------------------*/
868 usb_bdma_work_loop(struct usb_xfer_queue
*pq
)
870 struct usb_xfer_root
*info
;
871 struct usb_xfer
*xfer
;
872 usb_frcount_t nframes
;
877 KKASSERT(lockowned(info
->xfer_lock
));
880 /* some error happened */
881 USB_BUS_LOCK(info
->bus
);
882 usbd_transfer_done(xfer
, 0);
883 USB_BUS_UNLOCK(info
->bus
);
886 if (!xfer
->flags_int
.bdma_setup
) {
888 usb_frlength_t frlength_0
;
891 xfer
->flags_int
.bdma_setup
= 1;
893 /* reset BUS-DMA load state */
897 if (xfer
->flags_int
.isochronous_xfr
) {
898 /* only one frame buffer */
900 frlength_0
= xfer
->sumlen
;
902 /* can be multiple frame buffers */
903 nframes
= xfer
->nframes
;
904 frlength_0
= xfer
->frlengths
[0];
908 * Set DMA direction first. This is needed to
909 * select the correct cache invalidate and cache
912 isread
= USB_GET_DATA_ISREAD(xfer
);
913 pg
= xfer
->dma_page_ptr
;
915 if (xfer
->flags_int
.control_xfr
&&
916 xfer
->flags_int
.control_hdr
) {
918 if (xfer
->flags_int
.usb_mode
== USB_MODE_DEVICE
) {
919 /* The device controller writes to memory */
920 xfer
->frbuffers
[0].isread
= 1;
922 /* The host controller reads from memory */
923 xfer
->frbuffers
[0].isread
= 0;
927 xfer
->frbuffers
[0].isread
= isread
;
931 * Setup the "page_start" pointer which points to an array of
932 * USB pages where information about the physical address of a
933 * page will be stored. Also initialise the "isread" field of
934 * the USB page caches.
936 xfer
->frbuffers
[0].page_start
= pg
;
938 info
->dma_nframes
= nframes
;
939 info
->dma_currframe
= 0;
940 info
->dma_frlength_0
= frlength_0
;
942 pg
+= (frlength_0
/ USB_PAGE_SIZE
);
945 while (--nframes
> 0) {
946 xfer
->frbuffers
[nframes
].isread
= isread
;
947 xfer
->frbuffers
[nframes
].page_start
= pg
;
949 pg
+= (xfer
->frlengths
[nframes
] / USB_PAGE_SIZE
);
954 if (info
->dma_error
) {
955 USB_BUS_LOCK(info
->bus
);
956 usbd_transfer_done(xfer
, USB_ERR_DMA_LOAD_FAILED
);
957 USB_BUS_UNLOCK(info
->bus
);
960 if (info
->dma_currframe
!= info
->dma_nframes
) {
962 if (info
->dma_currframe
== 0) {
964 usb_pc_load_mem(xfer
->frbuffers
,
965 info
->dma_frlength_0
, 0);
968 nframes
= info
->dma_currframe
;
969 usb_pc_load_mem(xfer
->frbuffers
+ nframes
,
970 xfer
->frlengths
[nframes
], 0);
973 /* advance frame index */
974 info
->dma_currframe
++;
979 usb_bdma_pre_sync(xfer
);
981 /* start loading next USB transfer, if any */
982 usb_command_wrapper(pq
, NULL
);
984 /* finally start the hardware */
985 usbd_pipe_enter(xfer
);
988 /*------------------------------------------------------------------------*
989 * usb_bdma_done_event
991 * This function is called when the BUS-DMA has loaded virtual memory
993 *------------------------------------------------------------------------*/
995 usb_bdma_done_event(struct usb_dma_parent_tag
*udpt
)
997 struct usb_xfer_root
*info
;
999 info
= USB_DMATAG_TO_XROOT(udpt
);
1001 KKASSERT(lockowned(info
->xfer_lock
));
1004 info
->dma_error
= udpt
->dma_error
;
1006 /* enter workloop again */
1007 usb_command_wrapper(&info
->dma_q
,
1011 /*------------------------------------------------------------------------*
1014 * This function handles DMA synchronisation that must be done before
1015 * an USB transfer is started.
1016 *------------------------------------------------------------------------*/
1018 usb_bdma_pre_sync(struct usb_xfer
*xfer
)
1020 struct usb_page_cache
*pc
;
1021 usb_frcount_t nframes
;
1023 if (xfer
->flags_int
.isochronous_xfr
) {
1024 /* only one frame buffer */
1027 /* can be multiple frame buffers */
1028 nframes
= xfer
->nframes
;
1031 pc
= xfer
->frbuffers
;
1036 usb_pc_cpu_invalidate(pc
);
1038 usb_pc_cpu_flush(pc
);
1044 /*------------------------------------------------------------------------*
1045 * usb_bdma_post_sync
1047 * This function handles DMA synchronisation that must be done after
1048 * an USB transfer is complete.
1049 *------------------------------------------------------------------------*/
1051 usb_bdma_post_sync(struct usb_xfer
*xfer
)
1053 struct usb_page_cache
*pc
;
1054 usb_frcount_t nframes
;
1056 if (xfer
->flags_int
.isochronous_xfr
) {
1057 /* only one frame buffer */
1060 /* can be multiple frame buffers */
1061 nframes
= xfer
->nframes
;
1064 pc
= xfer
->frbuffers
;
1068 usb_pc_cpu_invalidate(pc
);