2 * Copyright (c) 2010 Hans Petter Selasky. All rights reserved.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * USB eXtensible Host Controller Interface, a.k.a. USB 3.0 controller.
29 * The XHCI 1.0 spec can be found at
30 * http://www.intel.com/technology/usb/download/xHCI_Specification_for_USB.pdf
31 * and the USB 3.0 spec at
32 * http://www.usb.org/developers/docs/usb_30_spec_060910.zip
36 * A few words about the design implementation: This driver emulates
37 * the concept about TDs which is found in EHCI specification. This
38 * way we achieve that the USB controller drivers look similar to
39 * eachother which makes it easier to understand the code.
42 #include <sys/stdint.h>
43 #include <sys/param.h>
44 #include <sys/queue.h>
45 #include <sys/types.h>
46 #include <sys/systm.h>
47 #include <sys/kernel.h>
49 #include <sys/module.h>
51 #include <sys/condvar.h>
52 #include <sys/sysctl.h>
53 #include <sys/unistd.h>
54 #include <sys/callout.h>
55 #include <sys/malloc.h>
58 #include <bus/u4b/usb.h>
59 #include <bus/u4b/usbdi.h>
61 #define USB_DEBUG_VAR xhcidebug
63 #include <bus/u4b/usb_core.h>
64 #include <bus/u4b/usb_debug.h>
65 #include <bus/u4b/usb_busdma.h>
66 #include <bus/u4b/usb_process.h>
67 #include <bus/u4b/usb_transfer.h>
68 #include <bus/u4b/usb_device.h>
69 #include <bus/u4b/usb_hub.h>
70 #include <bus/u4b/usb_util.h>
72 #include <bus/u4b/usb_controller.h>
73 #include <bus/u4b/usb_bus.h>
74 #include <bus/u4b/controller/xhci.h>
75 #include <bus/u4b/controller/xhcireg.h>
77 #define XHCI_BUS2SC(bus) \
78 ((struct xhci_softc *)(((uint8_t *)(bus)) - \
79 ((uint8_t *)&(((struct xhci_softc *)0)->sc_bus))))
82 static int xhcidebug
= 0;
83 static int xhciroute
= 0;
84 static int xhcipolling
= 0;
86 static SYSCTL_NODE(_hw_usb
, OID_AUTO
, xhci
, CTLFLAG_RW
, 0, "USB XHCI");
87 SYSCTL_INT(_hw_usb_xhci
, OID_AUTO
, debug
, CTLFLAG_RW
,
88 &xhcidebug
, 0, "Debug level");
89 TUNABLE_INT("hw.usb.xhci.debug", &xhcidebug
);
90 SYSCTL_INT(_hw_usb_xhci
, OID_AUTO
, xhci_port_route
, CTLFLAG_RW
,
91 &xhciroute
, 0, "Routing bitmap for switching EHCI ports to XHCI controller");
92 TUNABLE_INT("hw.usb.xhci.xhci_port_route", &xhciroute
);
93 SYSCTL_INT(_hw_usb_xhci
, OID_AUTO
, use_polling
, CTLFLAG_RW
,
94 &xhcipolling
, 0, "Set to enable software interrupt polling for XHCI controller");
95 TUNABLE_INT("hw.usb.xhci.use_polling", &xhcipolling
);
100 #define XHCI_INTR_ENDPT 1
102 struct xhci_std_temp
{
103 struct xhci_softc
*sc
;
104 struct usb_page_cache
*pc
;
106 struct xhci_td
*td_next
;
109 uint32_t max_packet_size
;
121 uint8_t do_isoc_sync
;
124 static void xhci_do_poll(struct usb_bus
*);
125 static void xhci_device_done(struct usb_xfer
*, usb_error_t
);
126 static void xhci_root_intr(struct xhci_softc
*);
127 static void xhci_free_device_ext(struct usb_device
*);
128 static struct xhci_endpoint_ext
*xhci_get_endpoint_ext(struct usb_device
*,
129 struct usb_endpoint_descriptor
*);
130 static usb_proc_callback_t xhci_configure_msg
;
131 static usb_error_t
xhci_configure_device(struct usb_device
*);
132 static usb_error_t
xhci_configure_endpoint(struct usb_device
*,
133 struct usb_endpoint_descriptor
*, struct xhci_endpoint_ext
*,
134 uint16_t, uint8_t, uint8_t, uint8_t, uint16_t, uint16_t,
136 static usb_error_t
xhci_configure_mask(struct usb_device
*,
138 static usb_error_t
xhci_cmd_evaluate_ctx(struct xhci_softc
*,
140 static void xhci_endpoint_doorbell(struct usb_xfer
*);
141 static void xhci_ctx_set_le32(struct xhci_softc
*sc
, volatile uint32_t *ptr
, uint32_t val
);
142 static uint32_t xhci_ctx_get_le32(struct xhci_softc
*sc
, volatile uint32_t *ptr
);
143 static void xhci_ctx_set_le64(struct xhci_softc
*sc
, volatile uint64_t *ptr
, uint64_t val
);
145 static uint64_t xhci_ctx_get_le64(struct xhci_softc
*sc
, volatile uint64_t *ptr
);
148 static const struct usb_bus_methods xhci_bus_methods
;
152 xhci_dump_trb(struct xhci_trb
*trb
)
154 DPRINTFN(5, "trb = %p\n", trb
);
155 DPRINTFN(5, "qwTrb0 = 0x%016llx\n", (long long)le64toh(trb
->qwTrb0
));
156 DPRINTFN(5, "dwTrb2 = 0x%08x\n", le32toh(trb
->dwTrb2
));
157 DPRINTFN(5, "dwTrb3 = 0x%08x\n", le32toh(trb
->dwTrb3
));
161 xhci_dump_endpoint(struct xhci_softc
*sc
, struct xhci_endp_ctx
*pep
)
163 DPRINTFN(5, "pep = %p\n", pep
);
164 DPRINTFN(5, "dwEpCtx0=0x%08x\n", xhci_ctx_get_le32(sc
, &pep
->dwEpCtx0
));
165 DPRINTFN(5, "dwEpCtx1=0x%08x\n", xhci_ctx_get_le32(sc
, &pep
->dwEpCtx1
));
166 DPRINTFN(5, "qwEpCtx2=0x%016llx\n", (long long)xhci_ctx_get_le64(sc
, &pep
->qwEpCtx2
));
167 DPRINTFN(5, "dwEpCtx4=0x%08x\n", xhci_ctx_get_le32(sc
, &pep
->dwEpCtx4
));
168 DPRINTFN(5, "dwEpCtx5=0x%08x\n", xhci_ctx_get_le32(sc
, &pep
->dwEpCtx5
));
169 DPRINTFN(5, "dwEpCtx6=0x%08x\n", xhci_ctx_get_le32(sc
, &pep
->dwEpCtx6
));
170 DPRINTFN(5, "dwEpCtx7=0x%08x\n", xhci_ctx_get_le32(sc
, &pep
->dwEpCtx7
));
174 xhci_dump_device(struct xhci_softc
*sc
, struct xhci_slot_ctx
*psl
)
176 DPRINTFN(5, "psl = %p\n", psl
);
177 DPRINTFN(5, "dwSctx0=0x%08x\n", xhci_ctx_get_le32(sc
, &psl
->dwSctx0
));
178 DPRINTFN(5, "dwSctx1=0x%08x\n", xhci_ctx_get_le32(sc
, &psl
->dwSctx1
));
179 DPRINTFN(5, "dwSctx2=0x%08x\n", xhci_ctx_get_le32(sc
, &psl
->dwSctx2
));
180 DPRINTFN(5, "dwSctx3=0x%08x\n", xhci_ctx_get_le32(sc
, &psl
->dwSctx3
));
185 xhci_use_polling(void)
188 return (xhcipolling
!= 0);
195 xhci_iterate_hw_softc(struct usb_bus
*bus
, usb_bus_mem_sub_cb_t
*cb
)
197 struct xhci_softc
*sc
= XHCI_BUS2SC(bus
);
200 cb(bus
, &sc
->sc_hw
.root_pc
, &sc
->sc_hw
.root_pg
,
201 sizeof(struct xhci_hw_root
), XHCI_PAGE_SIZE
);
203 cb(bus
, &sc
->sc_hw
.ctx_pc
, &sc
->sc_hw
.ctx_pg
,
204 sizeof(struct xhci_dev_ctx_addr
), XHCI_PAGE_SIZE
);
206 for (i
= 0; i
!= XHCI_MAX_SCRATCHPADS
; i
++) {
207 cb(bus
, &sc
->sc_hw
.scratch_pc
[i
], &sc
->sc_hw
.scratch_pg
[i
],
208 XHCI_PAGE_SIZE
, XHCI_PAGE_SIZE
);
213 xhci_ctx_set_le32(struct xhci_softc
*sc
, volatile uint32_t *ptr
, uint32_t val
)
215 if (sc
->sc_ctx_is_64_byte
) {
217 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
218 /* all contexts are initially 32-bytes */
219 offset
= ((uintptr_t)ptr
) & ((XHCI_PAGE_SIZE
- 1) & ~(31U));
220 ptr
= (volatile uint32_t *)(((volatile uint8_t *)ptr
) + offset
);
226 xhci_ctx_get_le32(struct xhci_softc
*sc
, volatile uint32_t *ptr
)
228 if (sc
->sc_ctx_is_64_byte
) {
230 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
231 /* all contexts are initially 32-bytes */
232 offset
= ((uintptr_t)ptr
) & ((XHCI_PAGE_SIZE
- 1) & ~(31U));
233 ptr
= (volatile uint32_t *)(((volatile uint8_t *)ptr
) + offset
);
235 return (le32toh(*ptr
));
239 xhci_ctx_set_le64(struct xhci_softc
*sc
, volatile uint64_t *ptr
, uint64_t val
)
241 if (sc
->sc_ctx_is_64_byte
) {
243 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
244 /* all contexts are initially 32-bytes */
245 offset
= ((uintptr_t)ptr
) & ((XHCI_PAGE_SIZE
- 1) & ~(31U));
246 ptr
= (volatile uint64_t *)(((volatile uint8_t *)ptr
) + offset
);
253 xhci_ctx_get_le64(struct xhci_softc
*sc
, volatile uint64_t *ptr
)
255 if (sc
->sc_ctx_is_64_byte
) {
257 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */
258 /* all contexts are initially 32-bytes */
259 offset
= ((uintptr_t)ptr
) & ((XHCI_PAGE_SIZE
- 1) & ~(31U));
260 ptr
= (volatile uint64_t *)(((volatile uint8_t *)ptr
) + offset
);
262 return (le64toh(*ptr
));
267 xhci_reset_command_queue_locked(struct xhci_softc
*sc
)
269 struct usb_page_search buf_res
;
270 struct xhci_hw_root
*phwr
;
276 temp
= XREAD4(sc
, oper
, XHCI_CRCR_LO
);
277 if (temp
& XHCI_CRCR_LO_CRR
) {
278 DPRINTF("Command ring running\n");
279 temp
&= ~(XHCI_CRCR_LO_CS
| XHCI_CRCR_LO_CA
);
282 * Try to abort the last command as per section
283 * 4.6.1.2 "Aborting a Command" of the XHCI
287 /* stop and cancel */
288 XWRITE4(sc
, oper
, XHCI_CRCR_LO
, temp
| XHCI_CRCR_LO_CS
);
289 XWRITE4(sc
, oper
, XHCI_CRCR_HI
, 0);
291 XWRITE4(sc
, oper
, XHCI_CRCR_LO
, temp
| XHCI_CRCR_LO_CA
);
292 XWRITE4(sc
, oper
, XHCI_CRCR_HI
, 0);
295 usb_pause_mtx(&sc
->sc_bus
.bus_lock
, hz
/ 4);
297 /* check if command ring is still running */
298 temp
= XREAD4(sc
, oper
, XHCI_CRCR_LO
);
299 if (temp
& XHCI_CRCR_LO_CRR
) {
300 DPRINTF("Comand ring still running\n");
301 return (USB_ERR_IOERROR
);
305 /* reset command ring */
306 sc
->sc_command_ccs
= 1;
307 sc
->sc_command_idx
= 0;
309 usbd_get_page(&sc
->sc_hw
.root_pc
, 0, &buf_res
);
311 /* setup command ring control base address */
312 addr
= buf_res
.physaddr
;
313 phwr
= buf_res
.buffer
;
314 addr
+= (uintptr_t)&((struct xhci_hw_root
*)0)->hwr_commands
[0];
316 DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr
);
318 memset(phwr
->hwr_commands
, 0, sizeof(phwr
->hwr_commands
));
319 phwr
->hwr_commands
[XHCI_MAX_COMMANDS
- 1].qwTrb0
= htole64(addr
);
321 usb_pc_cpu_flush(&sc
->sc_hw
.root_pc
);
323 XWRITE4(sc
, oper
, XHCI_CRCR_LO
, ((uint32_t)addr
) | XHCI_CRCR_LO_RCS
);
324 XWRITE4(sc
, oper
, XHCI_CRCR_HI
, (uint32_t)(addr
>> 32));
330 xhci_start_controller(struct xhci_softc
*sc
)
332 struct usb_page_search buf_res
;
333 struct xhci_hw_root
*phwr
;
334 struct xhci_dev_ctx_addr
*pdctxa
;
342 sc
->sc_oper_off
= XREAD1(sc
, capa
, XHCI_CAPLENGTH
);
343 sc
->sc_runt_off
= XREAD4(sc
, capa
, XHCI_RTSOFF
) & ~0x1F;
344 sc
->sc_door_off
= XREAD4(sc
, capa
, XHCI_DBOFF
) & ~0x3;
346 DPRINTF("CAPLENGTH=0x%x\n", sc
->sc_oper_off
);
347 DPRINTF("RUNTIMEOFFSET=0x%x\n", sc
->sc_runt_off
);
348 DPRINTF("DOOROFFSET=0x%x\n", sc
->sc_door_off
);
350 sc
->sc_event_ccs
= 1;
351 sc
->sc_event_idx
= 0;
352 sc
->sc_command_ccs
= 1;
353 sc
->sc_command_idx
= 0;
355 DPRINTF("xHCI version = 0x%04x\n", XREAD2(sc
, capa
, XHCI_HCIVERSION
));
357 temp
= XREAD4(sc
, capa
, XHCI_HCSPARAMS0
);
359 DPRINTF("HCS0 = 0x%08x\n", temp
);
361 if (XHCI_HCS0_CSZ(temp
)) {
362 sc
->sc_ctx_is_64_byte
= 1;
363 device_printf(sc
->sc_bus
.parent
, "64 byte context size.\n");
365 sc
->sc_ctx_is_64_byte
= 0;
366 device_printf(sc
->sc_bus
.parent
, "32 byte context size.\n");
369 /* Reset controller */
370 XWRITE4(sc
, oper
, XHCI_USBCMD
, XHCI_CMD_HCRST
);
372 for (i
= 0; i
!= 100; i
++) {
373 usb_pause_mtx(NULL
, hz
/ 100);
374 temp
= (XREAD4(sc
, oper
, XHCI_USBCMD
) & XHCI_CMD_HCRST
) |
375 (XREAD4(sc
, oper
, XHCI_USBSTS
) & XHCI_STS_CNR
);
381 device_printf(sc
->sc_bus
.parent
, "Controller "
383 return (USB_ERR_IOERROR
);
386 if (!(XREAD4(sc
, oper
, XHCI_PAGESIZE
) & XHCI_PAGESIZE_4K
)) {
387 device_printf(sc
->sc_bus
.parent
, "Controller does "
388 "not support 4K page size.\n");
389 return (USB_ERR_IOERROR
);
392 temp
= XREAD4(sc
, capa
, XHCI_HCSPARAMS1
);
394 i
= XHCI_HCS1_N_PORTS(temp
);
397 device_printf(sc
->sc_bus
.parent
, "Invalid number "
398 "of ports: %u\n", i
);
399 return (USB_ERR_IOERROR
);
403 sc
->sc_noslot
= XHCI_HCS1_DEVSLOT_MAX(temp
);
405 if (sc
->sc_noslot
> XHCI_MAX_DEVICES
)
406 sc
->sc_noslot
= XHCI_MAX_DEVICES
;
408 /* setup number of device slots */
410 DPRINTF("CONFIG=0x%08x -> 0x%08x\n",
411 XREAD4(sc
, oper
, XHCI_CONFIG
), sc
->sc_noslot
);
413 XWRITE4(sc
, oper
, XHCI_CONFIG
, sc
->sc_noslot
);
415 DPRINTF("Max slots: %u\n", sc
->sc_noslot
);
417 temp
= XREAD4(sc
, capa
, XHCI_HCSPARAMS2
);
419 sc
->sc_noscratch
= XHCI_HCS2_SPB_MAX(temp
);
421 if (sc
->sc_noscratch
> XHCI_MAX_SCRATCHPADS
) {
422 device_printf(sc
->sc_bus
.parent
, "XHCI request "
423 "too many scratchpads\n");
424 return (USB_ERR_NOMEM
);
427 DPRINTF("Max scratch: %u\n", sc
->sc_noscratch
);
429 temp
= XREAD4(sc
, capa
, XHCI_HCSPARAMS3
);
431 sc
->sc_exit_lat_max
= XHCI_HCS3_U1_DEL(temp
) +
432 XHCI_HCS3_U2_DEL(temp
) + 250 /* us */;
434 temp
= XREAD4(sc
, oper
, XHCI_USBSTS
);
436 /* clear interrupts */
437 XWRITE4(sc
, oper
, XHCI_USBSTS
, temp
);
438 /* disable all device notifications */
439 XWRITE4(sc
, oper
, XHCI_DNCTRL
, 0);
441 /* setup device context base address */
442 usbd_get_page(&sc
->sc_hw
.ctx_pc
, 0, &buf_res
);
443 pdctxa
= buf_res
.buffer
;
444 memset(pdctxa
, 0, sizeof(*pdctxa
));
446 addr
= buf_res
.physaddr
;
447 addr
+= (uintptr_t)&((struct xhci_dev_ctx_addr
*)0)->qwSpBufPtr
[0];
449 /* slot 0 points to the table of scratchpad pointers */
450 pdctxa
->qwBaaDevCtxAddr
[0] = htole64(addr
);
452 for (i
= 0; i
!= sc
->sc_noscratch
; i
++) {
453 struct usb_page_search buf_scp
;
454 usbd_get_page(&sc
->sc_hw
.scratch_pc
[i
], 0, &buf_scp
);
455 pdctxa
->qwSpBufPtr
[i
] = htole64((uint64_t)buf_scp
.physaddr
);
458 addr
= buf_res
.physaddr
;
460 XWRITE4(sc
, oper
, XHCI_DCBAAP_LO
, (uint32_t)addr
);
461 XWRITE4(sc
, oper
, XHCI_DCBAAP_HI
, (uint32_t)(addr
>> 32));
462 XWRITE4(sc
, oper
, XHCI_DCBAAP_LO
, (uint32_t)addr
);
463 XWRITE4(sc
, oper
, XHCI_DCBAAP_HI
, (uint32_t)(addr
>> 32));
465 /* Setup event table size */
467 temp
= XREAD4(sc
, capa
, XHCI_HCSPARAMS2
);
469 DPRINTF("HCS2=0x%08x\n", temp
);
471 temp
= XHCI_HCS2_ERST_MAX(temp
);
473 if (temp
> XHCI_MAX_RSEG
)
474 temp
= XHCI_MAX_RSEG
;
476 sc
->sc_erst_max
= temp
;
478 DPRINTF("ERSTSZ=0x%08x -> 0x%08x\n",
479 XREAD4(sc
, runt
, XHCI_ERSTSZ(0)), temp
);
481 XWRITE4(sc
, runt
, XHCI_ERSTSZ(0), XHCI_ERSTS_SET(temp
));
483 /* Setup interrupt rate */
484 XWRITE4(sc
, runt
, XHCI_IMOD(0), XHCI_IMOD_DEFAULT
);
486 usbd_get_page(&sc
->sc_hw
.root_pc
, 0, &buf_res
);
488 phwr
= buf_res
.buffer
;
489 addr
= buf_res
.physaddr
;
490 addr
+= (uintptr_t)&((struct xhci_hw_root
*)0)->hwr_events
[0];
492 /* reset hardware root structure */
493 memset(phwr
, 0, sizeof(*phwr
));
495 phwr
->hwr_ring_seg
[0].qwEvrsTablePtr
= htole64(addr
);
496 phwr
->hwr_ring_seg
[0].dwEvrsTableSize
= htole32(XHCI_MAX_EVENTS
);
498 DPRINTF("ERDP(0)=0x%016llx\n", (unsigned long long)addr
);
500 XWRITE4(sc
, runt
, XHCI_ERDP_LO(0), (uint32_t)addr
);
501 XWRITE4(sc
, runt
, XHCI_ERDP_HI(0), (uint32_t)(addr
>> 32));
503 addr
= (uint64_t)buf_res
.physaddr
;
505 DPRINTF("ERSTBA(0)=0x%016llx\n", (unsigned long long)addr
);
507 XWRITE4(sc
, runt
, XHCI_ERSTBA_LO(0), (uint32_t)addr
);
508 XWRITE4(sc
, runt
, XHCI_ERSTBA_HI(0), (uint32_t)(addr
>> 32));
510 /* Setup interrupter registers */
512 temp
= XREAD4(sc
, runt
, XHCI_IMAN(0));
513 temp
|= XHCI_IMAN_INTR_ENA
;
514 XWRITE4(sc
, runt
, XHCI_IMAN(0), temp
);
516 /* setup command ring control base address */
517 addr
= buf_res
.physaddr
;
518 addr
+= (uintptr_t)&((struct xhci_hw_root
*)0)->hwr_commands
[0];
520 DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr
);
522 XWRITE4(sc
, oper
, XHCI_CRCR_LO
, ((uint32_t)addr
) | XHCI_CRCR_LO_RCS
);
523 XWRITE4(sc
, oper
, XHCI_CRCR_HI
, (uint32_t)(addr
>> 32));
525 phwr
->hwr_commands
[XHCI_MAX_COMMANDS
- 1].qwTrb0
= htole64(addr
);
527 usb_bus_mem_flush_all(&sc
->sc_bus
, &xhci_iterate_hw_softc
);
530 XWRITE4(sc
, oper
, XHCI_USBCMD
, XHCI_CMD_RS
|
531 XHCI_CMD_INTE
| XHCI_CMD_HSEE
);
533 for (i
= 0; i
!= 100; i
++) {
534 usb_pause_mtx(NULL
, hz
/ 100);
535 temp
= XREAD4(sc
, oper
, XHCI_USBSTS
) & XHCI_STS_HCH
;
540 XWRITE4(sc
, oper
, XHCI_USBCMD
, 0);
541 device_printf(sc
->sc_bus
.parent
, "Run timeout.\n");
542 return (USB_ERR_IOERROR
);
545 /* catch any lost interrupts */
546 xhci_do_poll(&sc
->sc_bus
);
548 if (sc
->sc_port_route
!= NULL
) {
549 /* Route all ports to the XHCI by default */
550 sc
->sc_port_route(sc
->sc_bus
.parent
,
551 ~xhciroute
, xhciroute
);
557 xhci_halt_controller(struct xhci_softc
*sc
)
565 sc
->sc_oper_off
= XREAD1(sc
, capa
, XHCI_CAPLENGTH
);
566 sc
->sc_runt_off
= XREAD4(sc
, capa
, XHCI_RTSOFF
) & ~0xF;
567 sc
->sc_door_off
= XREAD4(sc
, capa
, XHCI_DBOFF
) & ~0x3;
569 /* Halt controller */
570 XWRITE4(sc
, oper
, XHCI_USBCMD
, 0);
572 for (i
= 0; i
!= 100; i
++) {
573 usb_pause_mtx(NULL
, hz
/ 100);
574 temp
= XREAD4(sc
, oper
, XHCI_USBSTS
) & XHCI_STS_HCH
;
580 device_printf(sc
->sc_bus
.parent
, "Controller halt timeout.\n");
581 return (USB_ERR_IOERROR
);
587 xhci_init(struct xhci_softc
*sc
, device_t self
)
589 /* initialise some bus fields */
590 sc
->sc_bus
.parent
= self
;
592 /* set the bus revision */
593 sc
->sc_bus
.usbrev
= USB_REV_3_0
;
595 /* set up the bus struct */
596 sc
->sc_bus
.methods
= &xhci_bus_methods
;
598 /* setup devices array */
599 sc
->sc_bus
.devices
= sc
->sc_devices
;
600 sc
->sc_bus
.devices_max
= XHCI_MAX_DEVICES
;
602 /* setup command queue mutex and condition varible */
603 cv_init(&sc
->sc_cmd_cv
, "CMDQ");
604 lockinit(&sc
->sc_cmd_lock
, "CMDQ lock", 0, 0);
606 /* get all DMA memory */
607 if (usb_bus_mem_alloc_all(&sc
->sc_bus
,
608 USB_GET_DMA_TAG(self
), &xhci_iterate_hw_softc
)) {
612 sc
->sc_config_msg
[0].hdr
.pm_callback
= &xhci_configure_msg
;
613 sc
->sc_config_msg
[0].bus
= &sc
->sc_bus
;
614 sc
->sc_config_msg
[1].hdr
.pm_callback
= &xhci_configure_msg
;
615 sc
->sc_config_msg
[1].bus
= &sc
->sc_bus
;
621 xhci_uninit(struct xhci_softc
*sc
)
624 * NOTE: At this point the control transfer process is gone
625 * and "xhci_configure_msg" is no longer called. Consequently
626 * waiting for the configuration messages to complete is not
629 usb_bus_mem_free_all(&sc
->sc_bus
, &xhci_iterate_hw_softc
);
631 cv_destroy(&sc
->sc_cmd_cv
);
632 lockuninit(&sc
->sc_cmd_lock
);
636 xhci_set_hw_power_sleep(struct usb_bus
*bus
, uint32_t state
)
638 struct xhci_softc
*sc
= XHCI_BUS2SC(bus
);
641 case USB_HW_POWER_SUSPEND
:
642 DPRINTF("Stopping the XHCI\n");
643 xhci_halt_controller(sc
);
645 case USB_HW_POWER_SHUTDOWN
:
646 DPRINTF("Stopping the XHCI\n");
647 xhci_halt_controller(sc
);
649 case USB_HW_POWER_RESUME
:
650 DPRINTF("Starting the XHCI\n");
651 xhci_start_controller(sc
);
659 xhci_generic_done_sub(struct usb_xfer
*xfer
)
662 struct xhci_td
*td_alt_next
;
666 td
= xfer
->td_transfer_cache
;
667 td_alt_next
= td
->alt_next
;
669 if (xfer
->aframes
!= xfer
->nframes
)
670 usbd_xfer_set_frame_len(xfer
, xfer
->aframes
, 0);
674 usb_pc_cpu_invalidate(td
->page_cache
);
679 DPRINTFN(4, "xfer=%p[%u/%u] rem=%u/%u status=%u\n",
680 xfer
, (unsigned int)xfer
->aframes
,
681 (unsigned int)xfer
->nframes
,
682 (unsigned int)len
, (unsigned int)td
->len
,
683 (unsigned int)status
);
686 * Verify the status length and
687 * add the length to "frlengths[]":
690 /* should not happen */
691 DPRINTF("Invalid status length, "
692 "0x%04x/0x%04x bytes\n", len
, td
->len
);
693 status
= XHCI_TRB_ERROR_LENGTH
;
694 } else if (xfer
->aframes
!= xfer
->nframes
) {
695 xfer
->frlengths
[xfer
->aframes
] += td
->len
- len
;
697 /* Check for last transfer */
698 if (((void *)td
) == xfer
->td_transfer_last
) {
702 /* Check for transfer error */
703 if (status
!= XHCI_TRB_ERROR_SHORT_PKT
&&
704 status
!= XHCI_TRB_ERROR_SUCCESS
) {
705 /* the transfer is finished */
709 /* Check for short transfer */
711 if (xfer
->flags_int
.short_frames_ok
||
712 xfer
->flags_int
.isochronous_xfr
||
713 xfer
->flags_int
.control_xfr
) {
714 /* follow alt next */
717 /* the transfer is finished */
724 if (td
->alt_next
!= td_alt_next
) {
725 /* this USB frame is complete */
730 /* update transfer cache */
732 xfer
->td_transfer_cache
= td
;
734 return ((status
== XHCI_TRB_ERROR_STALL
) ? USB_ERR_STALLED
:
735 (status
!= XHCI_TRB_ERROR_SHORT_PKT
&&
736 status
!= XHCI_TRB_ERROR_SUCCESS
) ? USB_ERR_IOERROR
:
737 USB_ERR_NORMAL_COMPLETION
);
741 xhci_generic_done(struct usb_xfer
*xfer
)
745 DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n",
746 xfer
, xfer
->endpoint
);
750 xfer
->td_transfer_cache
= xfer
->td_transfer_first
;
752 if (xfer
->flags_int
.control_xfr
) {
754 if (xfer
->flags_int
.control_hdr
)
755 err
= xhci_generic_done_sub(xfer
);
759 if (xfer
->td_transfer_cache
== NULL
)
763 while (xfer
->aframes
!= xfer
->nframes
) {
765 err
= xhci_generic_done_sub(xfer
);
768 if (xfer
->td_transfer_cache
== NULL
)
772 if (xfer
->flags_int
.control_xfr
&&
773 !xfer
->flags_int
.control_act
)
774 err
= xhci_generic_done_sub(xfer
);
776 /* transfer is complete */
777 xhci_device_done(xfer
, err
);
781 xhci_activate_transfer(struct usb_xfer
*xfer
)
785 td
= xfer
->td_transfer_cache
;
787 usb_pc_cpu_invalidate(td
->page_cache
);
789 if (!(td
->td_trb
[0].dwTrb3
& htole32(XHCI_TRB_3_CYCLE_BIT
))) {
791 /* activate the transfer */
793 td
->td_trb
[0].dwTrb3
|= htole32(XHCI_TRB_3_CYCLE_BIT
);
794 usb_pc_cpu_flush(td
->page_cache
);
796 xhci_endpoint_doorbell(xfer
);
801 xhci_skip_transfer(struct usb_xfer
*xfer
)
804 struct xhci_td
*td_last
;
806 td
= xfer
->td_transfer_cache
;
807 td_last
= xfer
->td_transfer_last
;
811 usb_pc_cpu_invalidate(td
->page_cache
);
813 if (!(td
->td_trb
[0].dwTrb3
& htole32(XHCI_TRB_3_CYCLE_BIT
))) {
815 usb_pc_cpu_invalidate(td_last
->page_cache
);
817 /* copy LINK TRB to current waiting location */
819 td
->td_trb
[0].qwTrb0
= td_last
->td_trb
[td_last
->ntrb
].qwTrb0
;
820 td
->td_trb
[0].dwTrb2
= td_last
->td_trb
[td_last
->ntrb
].dwTrb2
;
821 usb_pc_cpu_flush(td
->page_cache
);
823 td
->td_trb
[0].dwTrb3
= td_last
->td_trb
[td_last
->ntrb
].dwTrb3
;
824 usb_pc_cpu_flush(td
->page_cache
);
826 xhci_endpoint_doorbell(xfer
);
830 /*------------------------------------------------------------------------*
831 * xhci_check_transfer
832 *------------------------------------------------------------------------*/
834 xhci_check_transfer(struct xhci_softc
*sc
, struct xhci_trb
*trb
)
836 struct xhci_endpoint_ext
*pepext
;
849 td_event
= le64toh(trb
->qwTrb0
);
850 temp
= le32toh(trb
->dwTrb2
);
852 remainder
= XHCI_TRB_2_REM_GET(temp
);
853 status
= XHCI_TRB_2_ERROR_GET(temp
);
854 stream_id
= XHCI_TRB_2_STREAM_GET(temp
);
856 temp
= le32toh(trb
->dwTrb3
);
857 epno
= XHCI_TRB_3_EP_GET(temp
);
858 index
= XHCI_TRB_3_SLOT_GET(temp
);
860 /* check if error means halted */
861 halted
= (status
!= XHCI_TRB_ERROR_SHORT_PKT
&&
862 status
!= XHCI_TRB_ERROR_SUCCESS
);
864 DPRINTF("slot=%u epno=%u stream=%u remainder=%u status=%u\n",
865 index
, epno
, stream_id
, remainder
, status
);
867 if (index
> sc
->sc_noslot
) {
868 DPRINTF("Invalid slot.\n");
872 if ((epno
== 0) || (epno
>= XHCI_MAX_ENDPOINTS
)) {
873 DPRINTF("Invalid endpoint.\n");
877 pepext
= &sc
->sc_hw
.devs
[index
].endp
[epno
];
879 if (pepext
->trb_ep_mode
!= USB_EP_MODE_STREAMS
) {
881 DPRINTF("stream_id=0\n");
882 } else if (stream_id
>= XHCI_MAX_STREAMS
) {
883 DPRINTF("Invalid stream ID.\n");
887 /* try to find the USB transfer that generated the event */
888 for (i
= 0; i
!= (XHCI_MAX_TRANSFERS
- 1); i
++) {
889 struct usb_xfer
*xfer
;
892 xfer
= pepext
->xfer
[i
+ (XHCI_MAX_TRANSFERS
* stream_id
)];
896 td
= xfer
->td_transfer_cache
;
898 DPRINTFN(5, "Checking if 0x%016llx == (0x%016llx .. 0x%016llx)\n",
900 (long long)td
->td_self
,
901 (long long)td
->td_self
+ sizeof(td
->td_trb
));
904 * NOTE: Some XHCI implementations might not trigger
905 * an event on the last LINK TRB so we need to
906 * consider both the last and second last event
907 * address as conditions for a successful transfer.
909 * NOTE: We assume that the XHCI will only trigger one
910 * event per chain of TRBs.
913 offset
= td_event
- td
->td_self
;
916 offset
< (int64_t)sizeof(td
->td_trb
)) {
918 usb_pc_cpu_invalidate(td
->page_cache
);
920 /* compute rest of remainder, if any */
921 for (i
= (offset
/ 16) + 1; i
< td
->ntrb
; i
++) {
922 temp
= le32toh(td
->td_trb
[i
].dwTrb2
);
923 remainder
+= XHCI_TRB_2_BYTES_GET(temp
);
926 DPRINTFN(5, "New remainder: %u\n", remainder
);
928 /* clear isochronous transfer errors */
929 if (xfer
->flags_int
.isochronous_xfr
) {
932 status
= XHCI_TRB_ERROR_SUCCESS
;
937 /* "td->remainder" is verified later */
938 td
->remainder
= remainder
;
941 usb_pc_cpu_flush(td
->page_cache
);
944 * 1) Last transfer descriptor makes the
947 if (((void *)td
) == xfer
->td_transfer_last
) {
948 DPRINTF("TD is last\n");
949 xhci_generic_done(xfer
);
954 * 2) Any kind of error makes the transfer
958 DPRINTF("TD has I/O error\n");
959 xhci_generic_done(xfer
);
964 * 3) If there is no alternate next transfer,
965 * a short packet also makes the transfer done
967 if (td
->remainder
> 0) {
968 if (td
->alt_next
== NULL
) {
970 "short TD has no alternate next\n");
971 xhci_generic_done(xfer
);
974 DPRINTF("TD has short pkt\n");
975 if (xfer
->flags_int
.short_frames_ok
||
976 xfer
->flags_int
.isochronous_xfr
||
977 xfer
->flags_int
.control_xfr
) {
978 /* follow the alt next */
979 xfer
->td_transfer_cache
= td
->alt_next
;
980 xhci_activate_transfer(xfer
);
983 xhci_skip_transfer(xfer
);
984 xhci_generic_done(xfer
);
989 * 4) Transfer complete - go to next TD
991 DPRINTF("Following next TD\n");
992 xfer
->td_transfer_cache
= td
->obj_next
;
993 xhci_activate_transfer(xfer
);
994 break; /* there should only be one match */
1000 xhci_check_command(struct xhci_softc
*sc
, struct xhci_trb
*trb
)
1002 if (sc
->sc_cmd_addr
== trb
->qwTrb0
) {
1003 DPRINTF("Received command event\n");
1004 sc
->sc_cmd_result
[0] = trb
->dwTrb2
;
1005 sc
->sc_cmd_result
[1] = trb
->dwTrb3
;
1006 cv_signal(&sc
->sc_cmd_cv
);
1007 return (1); /* command match */
1013 xhci_interrupt_poll(struct xhci_softc
*sc
)
1015 struct usb_page_search buf_res
;
1016 struct xhci_hw_root
*phwr
;
1026 usbd_get_page(&sc
->sc_hw
.root_pc
, 0, &buf_res
);
1028 phwr
= buf_res
.buffer
;
1030 /* Receive any events */
1032 usb_pc_cpu_invalidate(&sc
->sc_hw
.root_pc
);
1034 i
= sc
->sc_event_idx
;
1035 j
= sc
->sc_event_ccs
;
1040 temp
= le32toh(phwr
->hwr_events
[i
].dwTrb3
);
1042 k
= (temp
& XHCI_TRB_3_CYCLE_BIT
) ? 1 : 0;
1047 event
= XHCI_TRB_3_TYPE_GET(temp
);
1049 DPRINTFN(10, "event[%u] = %u (0x%016llx 0x%08lx 0x%08lx)\n",
1050 i
, event
, (long long)le64toh(phwr
->hwr_events
[i
].qwTrb0
),
1051 (long)le32toh(phwr
->hwr_events
[i
].dwTrb2
),
1052 (long)le32toh(phwr
->hwr_events
[i
].dwTrb3
));
1055 case XHCI_TRB_EVENT_TRANSFER
:
1056 xhci_check_transfer(sc
, &phwr
->hwr_events
[i
]);
1058 case XHCI_TRB_EVENT_CMD_COMPLETE
:
1059 retval
|= xhci_check_command(sc
, &phwr
->hwr_events
[i
]);
1062 DPRINTF("Unhandled event = %u\n", event
);
1068 if (i
== XHCI_MAX_EVENTS
) {
1072 /* check for timeout */
1078 sc
->sc_event_idx
= i
;
1079 sc
->sc_event_ccs
= j
;
1082 * NOTE: The Event Ring Dequeue Pointer Register is 64-bit
1083 * latched. That means to activate the register we need to
1084 * write both the low and high double word of the 64-bit
1088 addr
= (uint32_t)buf_res
.physaddr
;
1089 addr
+= (uintptr_t)&((struct xhci_hw_root
*)0)->hwr_events
[i
];
1091 /* try to clear busy bit */
1092 addr
|= XHCI_ERDP_LO_BUSY
;
1094 XWRITE4(sc
, runt
, XHCI_ERDP_LO(0), (uint32_t)addr
);
1095 XWRITE4(sc
, runt
, XHCI_ERDP_HI(0), (uint32_t)(addr
>> 32));
1101 xhci_do_command(struct xhci_softc
*sc
, struct xhci_trb
*trb
,
1102 uint16_t timeout_ms
)
1104 struct usb_page_search buf_res
;
1105 struct xhci_hw_root
*phwr
;
1110 uint8_t timeout
= 0;
1113 XHCI_CMD_ASSERT_LOCKED(sc
);
1115 /* get hardware root structure */
1117 usbd_get_page(&sc
->sc_hw
.root_pc
, 0, &buf_res
);
1119 phwr
= buf_res
.buffer
;
1123 USB_BUS_LOCK(&sc
->sc_bus
);
1125 i
= sc
->sc_command_idx
;
1126 j
= sc
->sc_command_ccs
;
1128 DPRINTFN(10, "command[%u] = %u (0x%016llx, 0x%08lx, 0x%08lx)\n",
1129 i
, XHCI_TRB_3_TYPE_GET(le32toh(trb
->dwTrb3
)),
1130 (long long)le64toh(trb
->qwTrb0
),
1131 (long)le32toh(trb
->dwTrb2
),
1132 (long)le32toh(trb
->dwTrb3
));
1134 phwr
->hwr_commands
[i
].qwTrb0
= trb
->qwTrb0
;
1135 phwr
->hwr_commands
[i
].dwTrb2
= trb
->dwTrb2
;
1137 usb_pc_cpu_flush(&sc
->sc_hw
.root_pc
);
1142 temp
|= htole32(XHCI_TRB_3_CYCLE_BIT
);
1144 temp
&= ~htole32(XHCI_TRB_3_CYCLE_BIT
);
1146 temp
&= ~htole32(XHCI_TRB_3_TC_BIT
);
1148 phwr
->hwr_commands
[i
].dwTrb3
= temp
;
1150 usb_pc_cpu_flush(&sc
->sc_hw
.root_pc
);
1152 addr
= buf_res
.physaddr
;
1153 addr
+= (uintptr_t)&((struct xhci_hw_root
*)0)->hwr_commands
[i
];
1155 sc
->sc_cmd_addr
= htole64(addr
);
1159 if (i
== (XHCI_MAX_COMMANDS
- 1)) {
1162 temp
= htole32(XHCI_TRB_3_TC_BIT
|
1163 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK
) |
1164 XHCI_TRB_3_CYCLE_BIT
);
1166 temp
= htole32(XHCI_TRB_3_TC_BIT
|
1167 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK
));
1170 phwr
->hwr_commands
[i
].dwTrb3
= temp
;
1172 usb_pc_cpu_flush(&sc
->sc_hw
.root_pc
);
1178 sc
->sc_command_idx
= i
;
1179 sc
->sc_command_ccs
= j
;
1181 XWRITE4(sc
, door
, XHCI_DOORBELL(0), 0);
1183 err
= cv_timedwait(&sc
->sc_cmd_cv
, &sc
->sc_bus
.bus_lock
,
1184 USB_MS_TO_TICKS(timeout_ms
));
1187 * In some error cases event interrupts are not generated.
1188 * Poll one time to see if the command has completed.
1190 if (err
!= 0 && xhci_interrupt_poll(sc
) != 0) {
1191 DPRINTF("Command was completed when polling\n");
1195 DPRINTF("Command timeout!\n");
1197 * After some weeks of continuous operation, it has
1198 * been observed that the ASMedia Technology, ASM1042
1199 * SuperSpeed USB Host Controller can suddenly stop
1200 * accepting commands via the command queue. Try to
1201 * first reset the command queue. If that fails do a
1202 * host controller reset.
1205 xhci_reset_command_queue_locked(sc
) == 0) {
1209 DPRINTF("Controller reset!\n");
1210 usb_bus_reset_async_locked(&sc
->sc_bus
);
1212 err
= USB_ERR_TIMEOUT
;
1216 temp
= le32toh(sc
->sc_cmd_result
[0]);
1217 if (XHCI_TRB_2_ERROR_GET(temp
) != XHCI_TRB_ERROR_SUCCESS
)
1218 err
= USB_ERR_IOERROR
;
1220 trb
->dwTrb2
= sc
->sc_cmd_result
[0];
1221 trb
->dwTrb3
= sc
->sc_cmd_result
[1];
1224 USB_BUS_UNLOCK(&sc
->sc_bus
);
1231 xhci_cmd_nop(struct xhci_softc
*sc
)
1233 struct xhci_trb trb
;
1240 temp
= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NOOP
);
1242 trb
.dwTrb3
= htole32(temp
);
1244 return (xhci_do_command(sc
, &trb
, 100 /* ms */));
1249 xhci_cmd_enable_slot(struct xhci_softc
*sc
, uint8_t *pslot
)
1251 struct xhci_trb trb
;
1259 trb
.dwTrb3
= htole32(XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT
));
1261 err
= xhci_do_command(sc
, &trb
, 100 /* ms */);
1265 temp
= le32toh(trb
.dwTrb3
);
1267 *pslot
= XHCI_TRB_3_SLOT_GET(temp
);
1274 xhci_cmd_disable_slot(struct xhci_softc
*sc
, uint8_t slot_id
)
1276 struct xhci_trb trb
;
1283 temp
= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT
) |
1284 XHCI_TRB_3_SLOT_SET(slot_id
);
1286 trb
.dwTrb3
= htole32(temp
);
1288 return (xhci_do_command(sc
, &trb
, 100 /* ms */));
1292 xhci_cmd_set_address(struct xhci_softc
*sc
, uint64_t input_ctx
,
1293 uint8_t bsr
, uint8_t slot_id
)
1295 struct xhci_trb trb
;
1300 trb
.qwTrb0
= htole64(input_ctx
);
1302 temp
= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE
) |
1303 XHCI_TRB_3_SLOT_SET(slot_id
);
1306 temp
|= XHCI_TRB_3_BSR_BIT
;
1308 trb
.dwTrb3
= htole32(temp
);
1310 return (xhci_do_command(sc
, &trb
, 500 /* ms */));
1314 xhci_set_address(struct usb_device
*udev
, struct lock
*lock
, uint16_t address
)
1316 struct usb_page_search buf_inp
;
1317 struct usb_page_search buf_dev
;
1318 struct xhci_softc
*sc
= XHCI_BUS2SC(udev
->bus
);
1319 struct xhci_hw_dev
*hdev
;
1320 struct xhci_dev_ctx
*pdev
;
1321 struct xhci_endpoint_ext
*pepext
;
1327 /* the root HUB case is not handled here */
1328 if (udev
->parent_hub
== NULL
)
1329 return (USB_ERR_INVAL
);
1331 index
= udev
->controller_slot_id
;
1333 hdev
= &sc
->sc_hw
.devs
[index
];
1336 lockmgr(lock
, LK_RELEASE
);
1340 switch (hdev
->state
) {
1341 case XHCI_ST_DEFAULT
:
1342 case XHCI_ST_ENABLED
:
1344 hdev
->state
= XHCI_ST_ENABLED
;
1346 /* set configure mask to slot and EP0 */
1347 xhci_configure_mask(udev
, 3, 0);
1349 /* configure input slot context structure */
1350 err
= xhci_configure_device(udev
);
1353 DPRINTF("Could not configure device\n");
1357 /* configure input endpoint context structure */
1358 switch (udev
->speed
) {
1360 case USB_SPEED_FULL
:
1363 case USB_SPEED_HIGH
:
1371 pepext
= xhci_get_endpoint_ext(udev
,
1372 &udev
->ctrl_ep_desc
);
1373 err
= xhci_configure_endpoint(udev
,
1374 &udev
->ctrl_ep_desc
, pepext
,
1375 0, 1, 1, 0, mps
, mps
, USB_EP_MODE_DEFAULT
);
1378 DPRINTF("Could not configure default endpoint\n");
1382 /* execute set address command */
1383 usbd_get_page(&hdev
->input_pc
, 0, &buf_inp
);
1385 err
= xhci_cmd_set_address(sc
, buf_inp
.physaddr
,
1386 (address
== 0), index
);
1389 temp
= le32toh(sc
->sc_cmd_result
[0]);
1390 if (address
== 0 && sc
->sc_port_route
!= NULL
&&
1391 XHCI_TRB_2_ERROR_GET(temp
) ==
1392 XHCI_TRB_ERROR_PARAMETER
) {
1393 /* LynxPoint XHCI - ports are not switchable */
1394 /* Un-route all ports from the XHCI */
1395 sc
->sc_port_route(sc
->sc_bus
.parent
, 0, ~0);
1397 DPRINTF("Could not set address "
1398 "for slot %u.\n", index
);
1403 /* update device address to new value */
1405 usbd_get_page(&hdev
->device_pc
, 0, &buf_dev
);
1406 pdev
= buf_dev
.buffer
;
1407 usb_pc_cpu_invalidate(&hdev
->device_pc
);
1409 temp
= xhci_ctx_get_le32(sc
, &pdev
->ctx_slot
.dwSctx3
);
1410 udev
->address
= XHCI_SCTX_3_DEV_ADDR_GET(temp
);
1412 /* update device state to new value */
1415 hdev
->state
= XHCI_ST_ADDRESSED
;
1417 hdev
->state
= XHCI_ST_DEFAULT
;
1421 DPRINTF("Wrong state for set address.\n");
1422 err
= USB_ERR_IOERROR
;
1425 XHCI_CMD_UNLOCK(sc
);
1428 lockmgr(lock
, LK_EXCLUSIVE
);
1434 xhci_cmd_configure_ep(struct xhci_softc
*sc
, uint64_t input_ctx
,
1435 uint8_t deconfigure
, uint8_t slot_id
)
1437 struct xhci_trb trb
;
1442 trb
.qwTrb0
= htole64(input_ctx
);
1444 temp
= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP
) |
1445 XHCI_TRB_3_SLOT_SET(slot_id
);
1448 temp
|= XHCI_TRB_3_DCEP_BIT
;
1450 trb
.dwTrb3
= htole32(temp
);
1452 return (xhci_do_command(sc
, &trb
, 100 /* ms */));
1456 xhci_cmd_evaluate_ctx(struct xhci_softc
*sc
, uint64_t input_ctx
,
1459 struct xhci_trb trb
;
1464 trb
.qwTrb0
= htole64(input_ctx
);
1466 temp
= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX
) |
1467 XHCI_TRB_3_SLOT_SET(slot_id
);
1468 trb
.dwTrb3
= htole32(temp
);
1470 return (xhci_do_command(sc
, &trb
, 100 /* ms */));
1474 xhci_cmd_reset_ep(struct xhci_softc
*sc
, uint8_t preserve
,
1475 uint8_t ep_id
, uint8_t slot_id
)
1477 struct xhci_trb trb
;
1484 temp
= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP
) |
1485 XHCI_TRB_3_SLOT_SET(slot_id
) |
1486 XHCI_TRB_3_EP_SET(ep_id
);
1489 temp
|= XHCI_TRB_3_PRSV_BIT
;
1491 trb
.dwTrb3
= htole32(temp
);
1493 return (xhci_do_command(sc
, &trb
, 100 /* ms */));
1497 xhci_cmd_set_tr_dequeue_ptr(struct xhci_softc
*sc
, uint64_t dequeue_ptr
,
1498 uint16_t stream_id
, uint8_t ep_id
, uint8_t slot_id
)
1500 struct xhci_trb trb
;
1505 trb
.qwTrb0
= htole64(dequeue_ptr
);
1507 temp
= XHCI_TRB_2_STREAM_SET(stream_id
);
1508 trb
.dwTrb2
= htole32(temp
);
1510 temp
= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE
) |
1511 XHCI_TRB_3_SLOT_SET(slot_id
) |
1512 XHCI_TRB_3_EP_SET(ep_id
);
1513 trb
.dwTrb3
= htole32(temp
);
1515 return (xhci_do_command(sc
, &trb
, 100 /* ms */));
1519 xhci_cmd_stop_ep(struct xhci_softc
*sc
, uint8_t suspend
,
1520 uint8_t ep_id
, uint8_t slot_id
)
1522 struct xhci_trb trb
;
1529 temp
= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP
) |
1530 XHCI_TRB_3_SLOT_SET(slot_id
) |
1531 XHCI_TRB_3_EP_SET(ep_id
);
1534 temp
|= XHCI_TRB_3_SUSP_EP_BIT
;
1536 trb
.dwTrb3
= htole32(temp
);
1538 return (xhci_do_command(sc
, &trb
, 100 /* ms */));
1542 xhci_cmd_reset_dev(struct xhci_softc
*sc
, uint8_t slot_id
)
1544 struct xhci_trb trb
;
1551 temp
= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_DEVICE
) |
1552 XHCI_TRB_3_SLOT_SET(slot_id
);
1554 trb
.dwTrb3
= htole32(temp
);
1556 return (xhci_do_command(sc
, &trb
, 100 /* ms */));
1559 /*------------------------------------------------------------------------*
1560 * xhci_interrupt - XHCI interrupt handler
1561 *------------------------------------------------------------------------*/
1563 xhci_interrupt(struct xhci_softc
*sc
)
1567 USB_BUS_LOCK(&sc
->sc_bus
);
1569 status
= XREAD4(sc
, oper
, XHCI_USBSTS
);
1574 /* acknowledge interrupts */
1576 XWRITE4(sc
, oper
, XHCI_USBSTS
, status
);
1578 DPRINTFN(16, "real interrupt (status=0x%08x)\n", status
);
1580 if (status
& XHCI_STS_EINT
) {
1581 /* check for event(s) */
1582 xhci_interrupt_poll(sc
);
1585 if (status
& (XHCI_STS_PCD
| XHCI_STS_HCH
|
1586 XHCI_STS_HSE
| XHCI_STS_HCE
)) {
1588 if (status
& XHCI_STS_PCD
) {
1592 if (status
& XHCI_STS_HCH
) {
1593 kprintf("%s: host controller halted\n",
1597 if (status
& XHCI_STS_HSE
) {
1598 kprintf("%s: host system error\n",
1602 if (status
& XHCI_STS_HCE
) {
1603 kprintf("%s: host controller error\n",
1608 USB_BUS_UNLOCK(&sc
->sc_bus
);
1611 /*------------------------------------------------------------------------*
1612 * xhci_timeout - XHCI timeout handler
1613 *------------------------------------------------------------------------*/
1615 xhci_timeout(void *arg
)
1617 struct usb_xfer
*xfer
= arg
;
1619 DPRINTF("xfer=%p\n", xfer
);
1621 USB_BUS_LOCK_ASSERT(xfer
->xroot
->bus
);
1623 /* transfer is transferred */
1624 xhci_device_done(xfer
, USB_ERR_TIMEOUT
);
1628 xhci_do_poll(struct usb_bus
*bus
)
1630 struct xhci_softc
*sc
= XHCI_BUS2SC(bus
);
1632 USB_BUS_LOCK(&sc
->sc_bus
);
1633 xhci_interrupt_poll(sc
);
1634 USB_BUS_UNLOCK(&sc
->sc_bus
);
1638 xhci_setup_generic_chain_sub(struct xhci_std_temp
*temp
)
1640 struct usb_page_search buf_res
;
1642 struct xhci_td
*td_next
;
1643 struct xhci_td
*td_alt_next
;
1644 struct xhci_td
*td_first
;
1645 uint32_t buf_offset
;
1650 uint8_t shortpkt_old
;
1656 shortpkt_old
= temp
->shortpkt
;
1657 len_old
= temp
->len
;
1664 td_next
= td_first
= temp
->td_next
;
1668 if (temp
->len
== 0) {
1673 /* send a Zero Length Packet, ZLP, last */
1680 average
= temp
->average
;
1682 if (temp
->len
< average
) {
1683 if (temp
->len
% temp
->max_packet_size
) {
1686 average
= temp
->len
;
1690 if (td_next
== NULL
)
1691 panic("%s: out of XHCI transfer descriptors!", __func__
);
1696 td_next
= td
->obj_next
;
1698 /* check if we are pre-computing */
1702 /* update remaining length */
1704 temp
->len
-= average
;
1708 /* fill out current TD */
1714 /* update remaining length */
1716 temp
->len
-= average
;
1718 /* reset TRB index */
1722 if (temp
->trb_type
== XHCI_TRB_TYPE_SETUP_STAGE
) {
1723 /* immediate data */
1728 td
->td_trb
[0].qwTrb0
= 0;
1730 usbd_copy_out(temp
->pc
, temp
->offset
+ buf_offset
,
1731 (uint8_t *)(uintptr_t)&td
->td_trb
[0].qwTrb0
,
1734 dword
= XHCI_TRB_2_BYTES_SET(8) |
1735 XHCI_TRB_2_TDSZ_SET(0) |
1736 XHCI_TRB_2_IRQ_SET(0);
1738 td
->td_trb
[0].dwTrb2
= htole32(dword
);
1740 dword
= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE
) |
1741 XHCI_TRB_3_IDT_BIT
| XHCI_TRB_3_CYCLE_BIT
;
1744 if (td
->td_trb
[0].qwTrb0
&
1745 htole64(XHCI_TRB_0_WLENGTH_MASK
)) {
1746 if (td
->td_trb
[0].qwTrb0
& htole64(1))
1747 dword
|= XHCI_TRB_3_TRT_IN
;
1749 dword
|= XHCI_TRB_3_TRT_OUT
;
1752 td
->td_trb
[0].dwTrb3
= htole32(dword
);
1754 xhci_dump_trb(&td
->td_trb
[x
]);
1762 /* fill out buffer pointers */
1765 memset(&buf_res
, 0, sizeof(buf_res
));
1767 usbd_get_page(temp
->pc
, temp
->offset
+
1768 buf_offset
, &buf_res
);
1770 /* get length to end of page */
1771 if (buf_res
.length
> average
)
1772 buf_res
.length
= average
;
1774 /* check for maximum length */
1775 if (buf_res
.length
> XHCI_TD_PAGE_SIZE
)
1776 buf_res
.length
= XHCI_TD_PAGE_SIZE
;
1778 npkt_off
+= buf_res
.length
;
1782 npkt
= (len_old
- npkt_off
+ temp
->max_packet_size
- 1) /
1783 temp
->max_packet_size
;
1790 /* fill out TRB's */
1791 td
->td_trb
[x
].qwTrb0
=
1792 htole64((uint64_t)buf_res
.physaddr
);
1795 XHCI_TRB_2_BYTES_SET(buf_res
.length
) |
1796 XHCI_TRB_2_TDSZ_SET(npkt
) |
1797 XHCI_TRB_2_IRQ_SET(0);
1799 td
->td_trb
[x
].dwTrb2
= htole32(dword
);
1801 switch (temp
->trb_type
) {
1802 case XHCI_TRB_TYPE_ISOCH
:
1803 dword
= XHCI_TRB_3_CHAIN_BIT
| XHCI_TRB_3_CYCLE_BIT
|
1804 XHCI_TRB_3_TBC_SET(temp
->tbc
) |
1805 XHCI_TRB_3_TLBPC_SET(temp
->tlbpc
);
1806 if (td
!= td_first
) {
1807 dword
|= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL
);
1808 } else if (temp
->do_isoc_sync
!= 0) {
1809 temp
->do_isoc_sync
= 0;
1810 /* wait until "isoc_frame" */
1811 dword
|= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH
) |
1812 XHCI_TRB_3_FRID_SET(temp
->isoc_frame
/ 8);
1814 /* start data transfer at next interval */
1815 dword
|= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH
) |
1816 XHCI_TRB_3_ISO_SIA_BIT
;
1818 if (temp
->direction
== UE_DIR_IN
)
1819 dword
|= XHCI_TRB_3_DIR_IN
| XHCI_TRB_3_ISP_BIT
;
1821 case XHCI_TRB_TYPE_DATA_STAGE
:
1822 dword
= XHCI_TRB_3_CHAIN_BIT
| XHCI_TRB_3_CYCLE_BIT
|
1823 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE
) |
1824 XHCI_TRB_3_TBC_SET(temp
->tbc
) |
1825 XHCI_TRB_3_TLBPC_SET(temp
->tlbpc
);
1826 if (temp
->direction
== UE_DIR_IN
)
1827 dword
|= XHCI_TRB_3_DIR_IN
| XHCI_TRB_3_ISP_BIT
;
1829 case XHCI_TRB_TYPE_STATUS_STAGE
:
1830 dword
= XHCI_TRB_3_CHAIN_BIT
| XHCI_TRB_3_CYCLE_BIT
|
1831 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE
) |
1832 XHCI_TRB_3_TBC_SET(temp
->tbc
) |
1833 XHCI_TRB_3_TLBPC_SET(temp
->tlbpc
);
1834 if (temp
->direction
== UE_DIR_IN
)
1835 dword
|= XHCI_TRB_3_DIR_IN
;
1837 default: /* XHCI_TRB_TYPE_NORMAL */
1838 dword
= XHCI_TRB_3_CHAIN_BIT
| XHCI_TRB_3_CYCLE_BIT
|
1839 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL
) |
1840 XHCI_TRB_3_TBC_SET(temp
->tbc
) |
1841 XHCI_TRB_3_TLBPC_SET(temp
->tlbpc
);
1842 if (temp
->direction
== UE_DIR_IN
)
1843 dword
|= XHCI_TRB_3_DIR_IN
| XHCI_TRB_3_ISP_BIT
;
1846 td
->td_trb
[x
].dwTrb3
= htole32(dword
);
1848 average
-= buf_res
.length
;
1849 buf_offset
+= buf_res
.length
;
1851 xhci_dump_trb(&td
->td_trb
[x
]);
1855 } while (average
!= 0);
1857 td
->td_trb
[x
-1].dwTrb3
|= htole32(XHCI_TRB_3_IOC_BIT
);
1859 /* store number of data TRB's */
1863 DPRINTF("NTRB=%u\n", x
);
1865 /* fill out link TRB */
1867 if (td_next
!= NULL
) {
1868 /* link the current TD with the next one */
1869 td
->td_trb
[x
].qwTrb0
= htole64((uint64_t)td_next
->td_self
);
1870 DPRINTF("LINK=0x%08llx\n", (long long)td_next
->td_self
);
1872 /* this field will get updated later */
1873 DPRINTF("NOLINK\n");
1876 dword
= XHCI_TRB_2_IRQ_SET(0);
1878 td
->td_trb
[x
].dwTrb2
= htole32(dword
);
1880 dword
= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK
) |
1881 XHCI_TRB_3_CYCLE_BIT
| XHCI_TRB_3_IOC_BIT
|
1883 * CHAIN-BIT: Ensure that a multi-TRB IN-endpoint
1884 * frame only receives a single short packet event
1885 * by setting the CHAIN bit in the LINK field. In
1886 * addition some XHCI controllers have problems
1887 * sending a ZLP unless the CHAIN-BIT is set in
1890 XHCI_TRB_3_CHAIN_BIT
;
1892 td
->td_trb
[x
].dwTrb3
= htole32(dword
);
1894 td
->alt_next
= td_alt_next
;
1896 xhci_dump_trb(&td
->td_trb
[x
]);
1898 usb_pc_cpu_flush(td
->page_cache
);
1904 /* setup alt next pointer, if any */
1905 if (temp
->last_frame
) {
1908 /* we use this field internally */
1909 td_alt_next
= td_next
;
1913 temp
->shortpkt
= shortpkt_old
;
1914 temp
->len
= len_old
;
1919 * Remove cycle bit from the first TRB if we are
1922 if (temp
->step_td
!= 0) {
1923 td_first
->td_trb
[0].dwTrb3
&= ~htole32(XHCI_TRB_3_CYCLE_BIT
);
1924 usb_pc_cpu_flush(td_first
->page_cache
);
1927 /* clear TD SIZE to zero, hence this is the last TRB */
1928 /* remove chain bit because this is the last data TRB in the chain */
1929 td
->td_trb
[td
->ntrb
- 1].dwTrb2
&= ~htole32(XHCI_TRB_2_TDSZ_SET(15));
1930 td
->td_trb
[td
->ntrb
- 1].dwTrb3
&= ~htole32(XHCI_TRB_3_CHAIN_BIT
);
1931 /* remove CHAIN-BIT from last LINK TRB */
1932 td
->td_trb
[td
->ntrb
].dwTrb3
&= ~htole32(XHCI_TRB_3_CHAIN_BIT
);
1934 usb_pc_cpu_flush(td
->page_cache
);
1937 temp
->td_next
= td_next
;
1941 xhci_setup_generic_chain(struct usb_xfer
*xfer
)
1943 struct xhci_std_temp temp
;
1949 temp
.do_isoc_sync
= 0;
1953 temp
.average
= xfer
->max_hc_frame_size
;
1954 temp
.max_packet_size
= xfer
->max_packet_size
;
1955 temp
.sc
= XHCI_BUS2SC(xfer
->xroot
->bus
);
1957 temp
.last_frame
= 0;
1959 temp
.multishort
= xfer
->flags_int
.isochronous_xfr
||
1960 xfer
->flags_int
.control_xfr
||
1961 xfer
->flags_int
.short_frames_ok
;
1963 /* toggle the DMA set we are using */
1964 xfer
->flags_int
.curr_dma_set
^= 1;
1966 /* get next DMA set */
1967 td
= xfer
->td_start
[xfer
->flags_int
.curr_dma_set
];
1972 xfer
->td_transfer_first
= td
;
1973 xfer
->td_transfer_cache
= td
;
1975 if (xfer
->flags_int
.isochronous_xfr
) {
1978 /* compute multiplier for ISOCHRONOUS transfers */
1979 mult
= xfer
->endpoint
->ecomp
?
1980 UE_GET_SS_ISO_MULT(xfer
->endpoint
->ecomp
->bmAttributes
)
1982 /* check for USB 2.0 multiplier */
1984 mult
= (xfer
->endpoint
->edesc
->
1985 wMaxPacketSize
[1] >> 3) & 3;
1993 x
= XREAD4(temp
.sc
, runt
, XHCI_MFINDEX
);
1995 DPRINTF("MFINDEX=0x%08x\n", x
);
1997 switch (usbd_get_speed(xfer
->xroot
->udev
)) {
1998 case USB_SPEED_FULL
:
2000 temp
.isoc_delta
= 8; /* 1ms */
2001 x
+= temp
.isoc_delta
- 1;
2002 x
&= ~(temp
.isoc_delta
- 1);
2005 shift
= usbd_xfer_get_fps_shift(xfer
);
2006 temp
.isoc_delta
= 1U << shift
;
2007 x
+= temp
.isoc_delta
- 1;
2008 x
&= ~(temp
.isoc_delta
- 1);
2009 /* simple frame load balancing */
2010 x
+= xfer
->endpoint
->usb_uframe
;
2014 y
= XHCI_MFINDEX_GET(x
- xfer
->endpoint
->isoc_next
);
2016 if ((xfer
->endpoint
->is_synced
== 0) ||
2017 (y
< (xfer
->nframes
<< shift
)) ||
2018 (XHCI_MFINDEX_GET(-y
) >= (128 * 8))) {
2020 * If there is data underflow or the pipe
2021 * queue is empty we schedule the transfer a
2022 * few frames ahead of the current frame
2023 * position. Else two isochronous transfers
2026 xfer
->endpoint
->isoc_next
= XHCI_MFINDEX_GET(x
+ (3 * 8));
2027 xfer
->endpoint
->is_synced
= 1;
2028 temp
.do_isoc_sync
= 1;
2030 DPRINTFN(3, "start next=%d\n", xfer
->endpoint
->isoc_next
);
2033 /* compute isochronous completion time */
2035 y
= XHCI_MFINDEX_GET(xfer
->endpoint
->isoc_next
- (x
& ~7));
2037 xfer
->isoc_time_complete
=
2038 usb_isoc_time_expand(&temp
.sc
->sc_bus
, x
/ 8) +
2039 (y
/ 8) + (((xfer
->nframes
<< shift
) + 7) / 8);
2042 temp
.isoc_frame
= xfer
->endpoint
->isoc_next
;
2043 temp
.trb_type
= XHCI_TRB_TYPE_ISOCH
;
2045 xfer
->endpoint
->isoc_next
+= xfer
->nframes
<< shift
;
2047 } else if (xfer
->flags_int
.control_xfr
) {
2049 /* check if we should prepend a setup message */
2051 if (xfer
->flags_int
.control_hdr
) {
2053 temp
.len
= xfer
->frlengths
[0];
2054 temp
.pc
= xfer
->frbuffers
+ 0;
2055 temp
.shortpkt
= temp
.len
? 1 : 0;
2056 temp
.trb_type
= XHCI_TRB_TYPE_SETUP_STAGE
;
2059 /* check for last frame */
2060 if (xfer
->nframes
== 1) {
2061 /* no STATUS stage yet, SETUP is last */
2062 if (xfer
->flags_int
.control_act
)
2063 temp
.last_frame
= 1;
2066 xhci_setup_generic_chain_sub(&temp
);
2070 temp
.isoc_delta
= 0;
2071 temp
.isoc_frame
= 0;
2072 temp
.trb_type
= XHCI_TRB_TYPE_DATA_STAGE
;
2076 temp
.isoc_delta
= 0;
2077 temp
.isoc_frame
= 0;
2078 temp
.trb_type
= XHCI_TRB_TYPE_NORMAL
;
2081 if (x
!= xfer
->nframes
) {
2082 /* setup page_cache pointer */
2083 temp
.pc
= xfer
->frbuffers
+ x
;
2084 /* set endpoint direction */
2085 temp
.direction
= UE_GET_DIR(xfer
->endpointno
);
2088 while (x
!= xfer
->nframes
) {
2090 /* DATA0 / DATA1 message */
2092 temp
.len
= xfer
->frlengths
[x
];
2093 temp
.step_td
= ((xfer
->endpointno
& UE_DIR_IN
) &&
2094 x
!= 0 && temp
.multishort
== 0);
2098 if (x
== xfer
->nframes
) {
2099 if (xfer
->flags_int
.control_xfr
) {
2100 /* no STATUS stage yet, DATA is last */
2101 if (xfer
->flags_int
.control_act
)
2102 temp
.last_frame
= 1;
2104 temp
.last_frame
= 1;
2107 if (temp
.len
== 0) {
2109 /* make sure that we send an USB packet */
2114 temp
.tlbpc
= mult
- 1;
2116 } else if (xfer
->flags_int
.isochronous_xfr
) {
2121 * Isochronous transfers don't have short
2122 * packet termination:
2127 /* isochronous transfers have a transfer limit */
2129 if (temp
.len
> xfer
->max_frame_size
)
2130 temp
.len
= xfer
->max_frame_size
;
2132 /* compute TD packet count */
2133 tdpc
= (temp
.len
+ xfer
->max_packet_size
- 1) /
2134 xfer
->max_packet_size
;
2136 temp
.tbc
= ((tdpc
+ mult
- 1) / mult
) - 1;
2137 temp
.tlbpc
= (tdpc
% mult
);
2139 if (temp
.tlbpc
== 0)
2140 temp
.tlbpc
= mult
- 1;
2145 /* regular data transfer */
2147 temp
.shortpkt
= xfer
->flags
.force_short_xfer
? 0 : 1;
2150 xhci_setup_generic_chain_sub(&temp
);
2152 if (xfer
->flags_int
.isochronous_xfr
) {
2153 temp
.offset
+= xfer
->frlengths
[x
- 1];
2154 temp
.isoc_frame
+= temp
.isoc_delta
;
2156 /* get next Page Cache pointer */
2157 temp
.pc
= xfer
->frbuffers
+ x
;
2161 /* check if we should append a status stage */
2163 if (xfer
->flags_int
.control_xfr
&&
2164 !xfer
->flags_int
.control_act
) {
2167 * Send a DATA1 message and invert the current
2168 * endpoint direction.
2170 temp
.step_td
= (xfer
->nframes
!= 0);
2171 temp
.direction
= UE_GET_DIR(xfer
->endpointno
) ^ UE_DIR_IN
;
2175 temp
.last_frame
= 1;
2176 temp
.trb_type
= XHCI_TRB_TYPE_STATUS_STAGE
;
2178 xhci_setup_generic_chain_sub(&temp
);
2183 /* must have at least one frame! */
2185 xfer
->td_transfer_last
= td
;
2187 DPRINTF("first=%p last=%p\n", xfer
->td_transfer_first
, td
);
2191 xhci_set_slot_pointer(struct xhci_softc
*sc
, uint8_t index
, uint64_t dev_addr
)
2193 struct usb_page_search buf_res
;
2194 struct xhci_dev_ctx_addr
*pdctxa
;
2196 usbd_get_page(&sc
->sc_hw
.ctx_pc
, 0, &buf_res
);
2198 pdctxa
= buf_res
.buffer
;
2200 DPRINTF("addr[%u]=0x%016llx\n", index
, (long long)dev_addr
);
2202 pdctxa
->qwBaaDevCtxAddr
[index
] = htole64(dev_addr
);
2204 usb_pc_cpu_flush(&sc
->sc_hw
.ctx_pc
);
2208 xhci_configure_mask(struct usb_device
*udev
, uint32_t mask
, uint8_t drop
)
2210 struct xhci_softc
*sc
= XHCI_BUS2SC(udev
->bus
);
2211 struct usb_page_search buf_inp
;
2212 struct xhci_input_dev_ctx
*pinp
;
2217 index
= udev
->controller_slot_id
;
2219 usbd_get_page(&sc
->sc_hw
.devs
[index
].input_pc
, 0, &buf_inp
);
2221 pinp
= buf_inp
.buffer
;
2224 mask
&= XHCI_INCTX_NON_CTRL_MASK
;
2225 xhci_ctx_set_le32(sc
, &pinp
->ctx_input
.dwInCtx0
, mask
);
2226 xhci_ctx_set_le32(sc
, &pinp
->ctx_input
.dwInCtx1
, 0);
2228 xhci_ctx_set_le32(sc
, &pinp
->ctx_input
.dwInCtx0
, 0);
2229 xhci_ctx_set_le32(sc
, &pinp
->ctx_input
.dwInCtx1
, mask
);
2231 /* find most significant set bit */
2232 for (x
= 31; x
!= 1; x
--) {
2233 if (mask
& (1 << x
))
2240 /* figure out maximum */
2241 if (x
> sc
->sc_hw
.devs
[index
].context_num
) {
2242 sc
->sc_hw
.devs
[index
].context_num
= x
;
2243 temp
= xhci_ctx_get_le32(sc
, &pinp
->ctx_slot
.dwSctx0
);
2244 temp
&= ~XHCI_SCTX_0_CTX_NUM_SET(31);
2245 temp
|= XHCI_SCTX_0_CTX_NUM_SET(x
+ 1);
2246 xhci_ctx_set_le32(sc
, &pinp
->ctx_slot
.dwSctx0
, temp
);
2253 xhci_configure_endpoint(struct usb_device
*udev
,
2254 struct usb_endpoint_descriptor
*edesc
, struct xhci_endpoint_ext
*pepext
,
2255 uint16_t interval
, uint8_t max_packet_count
,
2256 uint8_t mult
, uint8_t fps_shift
, uint16_t max_packet_size
,
2257 uint16_t max_frame_size
, uint8_t ep_mode
)
2259 struct usb_page_search buf_inp
;
2260 struct xhci_softc
*sc
= XHCI_BUS2SC(udev
->bus
);
2261 struct xhci_input_dev_ctx
*pinp
;
2262 uint64_t ring_addr
= pepext
->physaddr
;
2268 index
= udev
->controller_slot_id
;
2270 usbd_get_page(&sc
->sc_hw
.devs
[index
].input_pc
, 0, &buf_inp
);
2272 pinp
= buf_inp
.buffer
;
2274 epno
= edesc
->bEndpointAddress
;
2275 type
= edesc
->bmAttributes
& UE_XFERTYPE
;
2277 if (type
== UE_CONTROL
)
2280 epno
= XHCI_EPNO2EPID(epno
);
2283 return (USB_ERR_NO_PIPE
); /* invalid */
2285 if (max_packet_count
== 0)
2286 return (USB_ERR_BAD_BUFSIZE
);
2291 return (USB_ERR_BAD_BUFSIZE
);
2293 /* store endpoint mode */
2294 pepext
->trb_ep_mode
= ep_mode
;
2295 usb_pc_cpu_flush(pepext
->page_cache
);
2297 if (ep_mode
== USB_EP_MODE_STREAMS
) {
2298 temp
= XHCI_EPCTX_0_EPSTATE_SET(0) |
2299 XHCI_EPCTX_0_MAXP_STREAMS_SET(XHCI_MAX_STREAMS_LOG
- 1) |
2300 XHCI_EPCTX_0_LSA_SET(1);
2302 ring_addr
+= sizeof(struct xhci_trb
) *
2303 XHCI_MAX_TRANSFERS
* XHCI_MAX_STREAMS
;
2305 temp
= XHCI_EPCTX_0_EPSTATE_SET(0) |
2306 XHCI_EPCTX_0_MAXP_STREAMS_SET(0) |
2307 XHCI_EPCTX_0_LSA_SET(0);
2309 ring_addr
|= XHCI_EPCTX_2_DCS_SET(1);
2312 switch (udev
->speed
) {
2313 case USB_SPEED_FULL
:
2326 temp
|= XHCI_EPCTX_0_IVAL_SET(fps_shift
);
2328 case UE_ISOCHRONOUS
:
2329 temp
|= XHCI_EPCTX_0_IVAL_SET(fps_shift
);
2331 switch (udev
->speed
) {
2332 case USB_SPEED_SUPER
:
2335 temp
|= XHCI_EPCTX_0_MULT_SET(mult
- 1);
2336 max_packet_count
/= mult
;
2346 xhci_ctx_set_le32(sc
, &pinp
->ctx_ep
[epno
- 1].dwEpCtx0
, temp
);
2349 XHCI_EPCTX_1_HID_SET(0) |
2350 XHCI_EPCTX_1_MAXB_SET(max_packet_count
) |
2351 XHCI_EPCTX_1_MAXP_SIZE_SET(max_packet_size
);
2353 if ((udev
->parent_hs_hub
!= NULL
) || (udev
->address
!= 0)) {
2354 if (type
!= UE_ISOCHRONOUS
)
2355 temp
|= XHCI_EPCTX_1_CERR_SET(3);
2360 temp
|= XHCI_EPCTX_1_EPTYPE_SET(4);
2362 case UE_ISOCHRONOUS
:
2363 temp
|= XHCI_EPCTX_1_EPTYPE_SET(1);
2366 temp
|= XHCI_EPCTX_1_EPTYPE_SET(2);
2369 temp
|= XHCI_EPCTX_1_EPTYPE_SET(3);
2373 /* check for IN direction */
2375 temp
|= XHCI_EPCTX_1_EPTYPE_SET(4);
2377 xhci_ctx_set_le32(sc
, &pinp
->ctx_ep
[epno
- 1].dwEpCtx1
, temp
);
2378 xhci_ctx_set_le64(sc
, &pinp
->ctx_ep
[epno
- 1].qwEpCtx2
, ring_addr
);
2380 switch (edesc
->bmAttributes
& UE_XFERTYPE
) {
2382 case UE_ISOCHRONOUS
:
2383 temp
= XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(max_frame_size
) |
2384 XHCI_EPCTX_4_AVG_TRB_LEN_SET(MIN(XHCI_PAGE_SIZE
,
2388 temp
= XHCI_EPCTX_4_AVG_TRB_LEN_SET(8);
2391 temp
= XHCI_EPCTX_4_AVG_TRB_LEN_SET(XHCI_PAGE_SIZE
);
2395 xhci_ctx_set_le32(sc
, &pinp
->ctx_ep
[epno
- 1].dwEpCtx4
, temp
);
2398 xhci_dump_endpoint(sc
, &pinp
->ctx_ep
[epno
- 1]);
2400 usb_pc_cpu_flush(&sc
->sc_hw
.devs
[index
].input_pc
);
2402 return (0); /* success */
2406 xhci_configure_endpoint_by_xfer(struct usb_xfer
*xfer
)
2408 struct xhci_endpoint_ext
*pepext
;
2409 struct usb_endpoint_ss_comp_descriptor
*ecomp
;
2412 pepext
= xhci_get_endpoint_ext(xfer
->xroot
->udev
,
2413 xfer
->endpoint
->edesc
);
2415 ecomp
= xfer
->endpoint
->ecomp
;
2417 for (x
= 0; x
!= XHCI_MAX_STREAMS
; x
++) {
2420 /* halt any transfers */
2421 pepext
->trb
[x
* XHCI_MAX_TRANSFERS
].dwTrb3
= 0;
2423 /* compute start of TRB ring for stream "x" */
2424 temp
= pepext
->physaddr
+
2425 (x
* XHCI_MAX_TRANSFERS
* sizeof(struct xhci_trb
)) +
2426 XHCI_SCTX_0_SCT_SEC_TR_RING
;
2428 /* make tree structure */
2429 pepext
->trb
[(XHCI_MAX_TRANSFERS
*
2430 XHCI_MAX_STREAMS
) + x
].qwTrb0
= htole64(temp
);
2432 /* reserved fields */
2433 pepext
->trb
[(XHCI_MAX_TRANSFERS
*
2434 XHCI_MAX_STREAMS
) + x
].dwTrb2
= 0;
2435 pepext
->trb
[(XHCI_MAX_TRANSFERS
*
2436 XHCI_MAX_STREAMS
) + x
].dwTrb3
= 0;
2438 usb_pc_cpu_flush(pepext
->page_cache
);
2440 return (xhci_configure_endpoint(xfer
->xroot
->udev
,
2441 xfer
->endpoint
->edesc
, pepext
,
2442 xfer
->interval
, xfer
->max_packet_count
,
2443 (ecomp
!= NULL
) ? UE_GET_SS_ISO_MULT(ecomp
->bmAttributes
) + 1 : 1,
2444 usbd_xfer_get_fps_shift(xfer
), xfer
->max_packet_size
,
2445 xfer
->max_frame_size
, xfer
->endpoint
->ep_mode
));
2449 xhci_configure_device(struct usb_device
*udev
)
2451 struct xhci_softc
*sc
= XHCI_BUS2SC(udev
->bus
);
2452 struct usb_page_search buf_inp
;
2453 struct usb_page_cache
*pcinp
;
2454 struct xhci_input_dev_ctx
*pinp
;
2455 struct usb_device
*hubdev
;
2463 index
= udev
->controller_slot_id
;
2465 DPRINTF("index=%u\n", index
);
2467 pcinp
= &sc
->sc_hw
.devs
[index
].input_pc
;
2469 usbd_get_page(pcinp
, 0, &buf_inp
);
2471 pinp
= buf_inp
.buffer
;
2476 /* figure out route string and root HUB port number */
2478 for (hubdev
= udev
; hubdev
!= NULL
; hubdev
= hubdev
->parent_hub
) {
2480 if (hubdev
->parent_hub
== NULL
)
2483 depth
= hubdev
->parent_hub
->depth
;
2486 * NOTE: HS/FS/LS devices and the SS root HUB can have
2487 * more than 15 ports
2490 rh_port
= hubdev
->port_no
;
2499 route
|= rh_port
<< (4 * (depth
- 1));
2502 DPRINTF("Route=0x%08x\n", route
);
2504 temp
= XHCI_SCTX_0_ROUTE_SET(route
) |
2505 XHCI_SCTX_0_CTX_NUM_SET(
2506 sc
->sc_hw
.devs
[index
].context_num
+ 1);
2508 switch (udev
->speed
) {
2510 temp
|= XHCI_SCTX_0_SPEED_SET(2);
2511 if (udev
->parent_hs_hub
!= NULL
&&
2512 udev
->parent_hs_hub
->ddesc
.bDeviceProtocol
==
2514 DPRINTF("Device inherits MTT\n");
2515 temp
|= XHCI_SCTX_0_MTT_SET(1);
2518 case USB_SPEED_HIGH
:
2519 temp
|= XHCI_SCTX_0_SPEED_SET(3);
2520 if (sc
->sc_hw
.devs
[index
].nports
!= 0 &&
2521 udev
->ddesc
.bDeviceProtocol
== UDPROTO_HSHUBMTT
) {
2522 DPRINTF("HUB supports MTT\n");
2523 temp
|= XHCI_SCTX_0_MTT_SET(1);
2526 case USB_SPEED_FULL
:
2527 temp
|= XHCI_SCTX_0_SPEED_SET(1);
2528 if (udev
->parent_hs_hub
!= NULL
&&
2529 udev
->parent_hs_hub
->ddesc
.bDeviceProtocol
==
2531 DPRINTF("Device inherits MTT\n");
2532 temp
|= XHCI_SCTX_0_MTT_SET(1);
2536 temp
|= XHCI_SCTX_0_SPEED_SET(4);
2540 is_hub
= sc
->sc_hw
.devs
[index
].nports
!= 0 &&
2541 (udev
->speed
== USB_SPEED_SUPER
||
2542 udev
->speed
== USB_SPEED_HIGH
);
2545 temp
|= XHCI_SCTX_0_HUB_SET(1);
2548 xhci_ctx_set_le32(sc
, &pinp
->ctx_slot
.dwSctx0
, temp
);
2550 temp
= XHCI_SCTX_1_RH_PORT_SET(rh_port
);
2553 temp
|= XHCI_SCTX_1_NUM_PORTS_SET(
2554 sc
->sc_hw
.devs
[index
].nports
);
2557 switch (udev
->speed
) {
2558 case USB_SPEED_SUPER
:
2559 switch (sc
->sc_hw
.devs
[index
].state
) {
2560 case XHCI_ST_ADDRESSED
:
2561 case XHCI_ST_CONFIGURED
:
2562 /* enable power save */
2563 temp
|= XHCI_SCTX_1_MAX_EL_SET(sc
->sc_exit_lat_max
);
2566 /* disable power save */
2574 xhci_ctx_set_le32(sc
, &pinp
->ctx_slot
.dwSctx1
, temp
);
2576 temp
= XHCI_SCTX_2_IRQ_TARGET_SET(0);
2579 temp
|= XHCI_SCTX_2_TT_THINK_TIME_SET(
2580 sc
->sc_hw
.devs
[index
].tt
);
2583 hubdev
= udev
->parent_hs_hub
;
2585 /* check if we should activate the transaction translator */
2586 switch (udev
->speed
) {
2587 case USB_SPEED_FULL
:
2589 if (hubdev
!= NULL
) {
2590 temp
|= XHCI_SCTX_2_TT_HUB_SID_SET(
2591 hubdev
->controller_slot_id
);
2592 temp
|= XHCI_SCTX_2_TT_PORT_NUM_SET(
2600 xhci_ctx_set_le32(sc
, &pinp
->ctx_slot
.dwSctx2
, temp
);
2603 * These fields should be initialized to zero, according to
2604 * XHCI section 6.2.2 - slot context:
2606 temp
= XHCI_SCTX_3_DEV_ADDR_SET(0) |
2607 XHCI_SCTX_3_SLOT_STATE_SET(0);
2609 xhci_ctx_set_le32(sc
, &pinp
->ctx_slot
.dwSctx3
, temp
);
2612 xhci_dump_device(sc
, &pinp
->ctx_slot
);
2614 usb_pc_cpu_flush(pcinp
);
2616 return (0); /* success */
2620 xhci_alloc_device_ext(struct usb_device
*udev
)
2622 struct xhci_softc
*sc
= XHCI_BUS2SC(udev
->bus
);
2623 struct usb_page_search buf_dev
;
2624 struct usb_page_search buf_ep
;
2625 struct xhci_trb
*trb
;
2626 struct usb_page_cache
*pc
;
2627 struct usb_page
*pg
;
2632 index
= udev
->controller_slot_id
;
2634 pc
= &sc
->sc_hw
.devs
[index
].device_pc
;
2635 pg
= &sc
->sc_hw
.devs
[index
].device_pg
;
2637 /* need to initialize the page cache */
2638 pc
->tag_parent
= sc
->sc_bus
.dma_parent_tag
;
2640 if (usb_pc_alloc_mem(pc
, pg
, sc
->sc_ctx_is_64_byte
?
2641 (2 * sizeof(struct xhci_dev_ctx
)) :
2642 sizeof(struct xhci_dev_ctx
), XHCI_PAGE_SIZE
))
2645 usbd_get_page(pc
, 0, &buf_dev
);
2647 pc
= &sc
->sc_hw
.devs
[index
].input_pc
;
2648 pg
= &sc
->sc_hw
.devs
[index
].input_pg
;
2650 /* need to initialize the page cache */
2651 pc
->tag_parent
= sc
->sc_bus
.dma_parent_tag
;
2653 if (usb_pc_alloc_mem(pc
, pg
, sc
->sc_ctx_is_64_byte
?
2654 (2 * sizeof(struct xhci_input_dev_ctx
)) :
2655 sizeof(struct xhci_input_dev_ctx
), XHCI_PAGE_SIZE
)) {
2659 pc
= &sc
->sc_hw
.devs
[index
].endpoint_pc
;
2660 pg
= &sc
->sc_hw
.devs
[index
].endpoint_pg
;
2662 /* need to initialize the page cache */
2663 pc
->tag_parent
= sc
->sc_bus
.dma_parent_tag
;
2665 if (usb_pc_alloc_mem(pc
, pg
,
2666 sizeof(struct xhci_dev_endpoint_trbs
), XHCI_PAGE_SIZE
)) {
2670 /* initialise all endpoint LINK TRBs */
2672 for (i
= 0; i
!= XHCI_MAX_ENDPOINTS
; i
++) {
2674 /* lookup endpoint TRB ring */
2675 usbd_get_page(pc
, (uintptr_t)&
2676 ((struct xhci_dev_endpoint_trbs
*)0)->trb
[i
][0], &buf_ep
);
2678 /* get TRB pointer */
2679 trb
= buf_ep
.buffer
;
2680 trb
+= XHCI_MAX_TRANSFERS
- 1;
2682 /* get TRB start address */
2683 addr
= buf_ep
.physaddr
;
2685 /* create LINK TRB */
2686 trb
->qwTrb0
= htole64(addr
);
2687 trb
->dwTrb2
= htole32(XHCI_TRB_2_IRQ_SET(0));
2688 trb
->dwTrb3
= htole32(XHCI_TRB_3_CYCLE_BIT
|
2689 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK
));
2692 usb_pc_cpu_flush(pc
);
2694 xhci_set_slot_pointer(sc
, index
, buf_dev
.physaddr
);
2699 xhci_free_device_ext(udev
);
2701 return (USB_ERR_NOMEM
);
2705 xhci_free_device_ext(struct usb_device
*udev
)
2707 struct xhci_softc
*sc
= XHCI_BUS2SC(udev
->bus
);
2710 index
= udev
->controller_slot_id
;
2711 xhci_set_slot_pointer(sc
, index
, 0);
2713 usb_pc_free_mem(&sc
->sc_hw
.devs
[index
].device_pc
);
2714 usb_pc_free_mem(&sc
->sc_hw
.devs
[index
].input_pc
);
2715 usb_pc_free_mem(&sc
->sc_hw
.devs
[index
].endpoint_pc
);
2718 static struct xhci_endpoint_ext
*
2719 xhci_get_endpoint_ext(struct usb_device
*udev
, struct usb_endpoint_descriptor
*edesc
)
2721 struct xhci_softc
*sc
= XHCI_BUS2SC(udev
->bus
);
2722 struct xhci_endpoint_ext
*pepext
;
2723 struct usb_page_cache
*pc
;
2724 struct usb_page_search buf_ep
;
2728 epno
= edesc
->bEndpointAddress
;
2729 if ((edesc
->bmAttributes
& UE_XFERTYPE
) == UE_CONTROL
)
2732 epno
= XHCI_EPNO2EPID(epno
);
2734 index
= udev
->controller_slot_id
;
2736 pc
= &sc
->sc_hw
.devs
[index
].endpoint_pc
;
2738 usbd_get_page(pc
, (uintptr_t)&((struct xhci_dev_endpoint_trbs
*)0)->
2739 trb
[epno
][0], &buf_ep
);
2741 pepext
= &sc
->sc_hw
.devs
[index
].endp
[epno
];
2742 pepext
->page_cache
= pc
;
2743 pepext
->trb
= buf_ep
.buffer
;
2744 pepext
->physaddr
= buf_ep
.physaddr
;
2750 xhci_endpoint_doorbell(struct usb_xfer
*xfer
)
2752 struct xhci_softc
*sc
= XHCI_BUS2SC(xfer
->xroot
->bus
);
2756 epno
= xfer
->endpointno
;
2757 if (xfer
->flags_int
.control_xfr
)
2760 epno
= XHCI_EPNO2EPID(epno
);
2761 index
= xfer
->xroot
->udev
->controller_slot_id
;
2763 if (xfer
->xroot
->udev
->flags
.self_suspended
== 0) {
2764 XWRITE4(sc
, door
, XHCI_DOORBELL(index
),
2765 epno
| XHCI_DB_SID_SET(xfer
->stream_id
));
2770 xhci_transfer_remove(struct usb_xfer
*xfer
, usb_error_t error
)
2772 struct xhci_endpoint_ext
*pepext
;
2774 if (xfer
->flags_int
.bandwidth_reclaimed
) {
2775 xfer
->flags_int
.bandwidth_reclaimed
= 0;
2777 pepext
= xhci_get_endpoint_ext(xfer
->xroot
->udev
,
2778 xfer
->endpoint
->edesc
);
2780 pepext
->trb_used
[xfer
->stream_id
]--;
2782 pepext
->xfer
[xfer
->qh_pos
] = NULL
;
2784 if (error
&& pepext
->trb_running
!= 0) {
2785 pepext
->trb_halted
= 1;
2786 pepext
->trb_running
= 0;
2792 xhci_transfer_insert(struct usb_xfer
*xfer
)
2794 struct xhci_td
*td_first
;
2795 struct xhci_td
*td_last
;
2796 struct xhci_trb
*trb_link
;
2797 struct xhci_endpoint_ext
*pepext
;
2806 id
= xfer
->stream_id
;
2808 /* check if already inserted */
2809 if (xfer
->flags_int
.bandwidth_reclaimed
) {
2810 DPRINTFN(8, "Already in schedule\n");
2814 pepext
= xhci_get_endpoint_ext(xfer
->xroot
->udev
,
2815 xfer
->endpoint
->edesc
);
2817 td_first
= xfer
->td_transfer_first
;
2818 td_last
= xfer
->td_transfer_last
;
2819 addr
= pepext
->physaddr
;
2821 switch (xfer
->endpoint
->edesc
->bmAttributes
& UE_XFERTYPE
) {
2824 /* single buffered */
2828 /* multi buffered */
2829 trb_limit
= (XHCI_MAX_TRANSFERS
- 2);
2833 if (pepext
->trb_used
[id
] >= trb_limit
) {
2834 DPRINTFN(8, "Too many TDs queued.\n");
2835 return (USB_ERR_NOMEM
);
2838 /* check for stopped condition, after putting transfer on interrupt queue */
2839 if (pepext
->trb_running
== 0) {
2840 struct xhci_softc
*sc
= XHCI_BUS2SC(xfer
->xroot
->bus
);
2842 DPRINTFN(8, "Not running\n");
2844 /* start configuration */
2845 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc
->sc_bus
),
2846 &sc
->sc_config_msg
[0], &sc
->sc_config_msg
[1]);
2850 pepext
->trb_used
[id
]++;
2852 /* get current TRB index */
2853 i
= pepext
->trb_index
[id
];
2855 /* get next TRB index */
2858 /* the last entry of the ring is a hardcoded link TRB */
2859 if (inext
>= (XHCI_MAX_TRANSFERS
- 1))
2862 /* store next TRB index, before stream ID offset is added */
2863 pepext
->trb_index
[id
] = inext
;
2865 /* offset for stream */
2866 i
+= id
* XHCI_MAX_TRANSFERS
;
2867 inext
+= id
* XHCI_MAX_TRANSFERS
;
2869 /* compute terminating return address */
2870 addr
+= (inext
* sizeof(struct xhci_trb
));
2872 /* compute link TRB pointer */
2873 trb_link
= td_last
->td_trb
+ td_last
->ntrb
;
2875 /* update next pointer of last link TRB */
2876 trb_link
->qwTrb0
= htole64(addr
);
2877 trb_link
->dwTrb2
= htole32(XHCI_TRB_2_IRQ_SET(0));
2878 trb_link
->dwTrb3
= htole32(XHCI_TRB_3_IOC_BIT
|
2879 XHCI_TRB_3_CYCLE_BIT
|
2880 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK
));
2883 xhci_dump_trb(&td_last
->td_trb
[td_last
->ntrb
]);
2885 usb_pc_cpu_flush(td_last
->page_cache
);
2887 /* write ahead chain end marker */
2889 pepext
->trb
[inext
].qwTrb0
= 0;
2890 pepext
->trb
[inext
].dwTrb2
= 0;
2891 pepext
->trb
[inext
].dwTrb3
= 0;
2893 /* update next pointer of link TRB */
2895 pepext
->trb
[i
].qwTrb0
= htole64((uint64_t)td_first
->td_self
);
2896 pepext
->trb
[i
].dwTrb2
= htole32(XHCI_TRB_2_IRQ_SET(0));
2899 xhci_dump_trb(&pepext
->trb
[i
]);
2901 usb_pc_cpu_flush(pepext
->page_cache
);
2903 /* toggle cycle bit which activates the transfer chain */
2905 pepext
->trb
[i
].dwTrb3
= htole32(XHCI_TRB_3_CYCLE_BIT
|
2906 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK
));
2908 usb_pc_cpu_flush(pepext
->page_cache
);
2910 DPRINTF("qh_pos = %u\n", i
);
2912 pepext
->xfer
[i
] = xfer
;
2916 xfer
->flags_int
.bandwidth_reclaimed
= 1;
2918 xhci_endpoint_doorbell(xfer
);
2924 xhci_root_intr(struct xhci_softc
*sc
)
2928 USB_BUS_LOCK_ASSERT(&sc
->sc_bus
);
2930 /* clear any old interrupt data */
2931 memset(sc
->sc_hub_idata
, 0, sizeof(sc
->sc_hub_idata
));
2933 for (i
= 1; i
<= sc
->sc_noport
; i
++) {
2934 /* pick out CHANGE bits from the status register */
2935 if (XREAD4(sc
, oper
, XHCI_PORTSC(i
)) & (
2936 XHCI_PS_CSC
| XHCI_PS_PEC
|
2937 XHCI_PS_OCC
| XHCI_PS_WRC
|
2938 XHCI_PS_PRC
| XHCI_PS_PLC
|
2940 sc
->sc_hub_idata
[i
/ 8] |= 1 << (i
% 8);
2941 DPRINTF("port %d changed\n", i
);
2944 uhub_root_intr(&sc
->sc_bus
, sc
->sc_hub_idata
,
2945 sizeof(sc
->sc_hub_idata
));
2948 /*------------------------------------------------------------------------*
2949 * xhci_device_done - XHCI done handler
2951 * NOTE: This function can be called two times in a row on
2952 * the same USB transfer. From close and from interrupt.
2953 *------------------------------------------------------------------------*/
2955 xhci_device_done(struct usb_xfer
*xfer
, usb_error_t error
)
2957 DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n",
2958 xfer
, xfer
->endpoint
, error
);
2960 /* remove transfer from HW queue */
2961 xhci_transfer_remove(xfer
, error
);
2963 /* dequeue transfer and start next transfer */
2964 usbd_transfer_done(xfer
, error
);
2967 /*------------------------------------------------------------------------*
2968 * XHCI data transfer support (generic type)
2969 *------------------------------------------------------------------------*/
2971 xhci_device_generic_open(struct usb_xfer
*xfer
)
2973 if (xfer
->flags_int
.isochronous_xfr
) {
2974 switch (xfer
->xroot
->udev
->speed
) {
2975 case USB_SPEED_FULL
:
2978 usb_hs_bandwidth_alloc(xfer
);
2985 xhci_device_generic_close(struct usb_xfer
*xfer
)
2989 xhci_device_done(xfer
, USB_ERR_CANCELLED
);
2991 if (xfer
->flags_int
.isochronous_xfr
) {
2992 switch (xfer
->xroot
->udev
->speed
) {
2993 case USB_SPEED_FULL
:
2996 usb_hs_bandwidth_free(xfer
);
3003 xhci_device_generic_multi_enter(struct usb_endpoint
*ep
,
3004 usb_stream_t stream_id
, struct usb_xfer
*enter_xfer
)
3006 struct usb_xfer
*xfer
;
3008 /* check if there is a current transfer */
3009 xfer
= ep
->endpoint_q
[stream_id
].curr
;
3014 * Check if the current transfer is started and then pickup
3015 * the next one, if any. Else wait for next start event due to
3016 * block on failure feature.
3018 if (!xfer
->flags_int
.bandwidth_reclaimed
)
3021 xfer
= TAILQ_FIRST(&ep
->endpoint_q
[stream_id
].head
);
3024 * In case of enter we have to consider that the
3025 * transfer is queued by the USB core after the enter
3034 /* try to multi buffer */
3035 xhci_transfer_insert(xfer
);
3039 xhci_device_generic_enter(struct usb_xfer
*xfer
)
3043 /* setup TD's and QH */
3044 xhci_setup_generic_chain(xfer
);
3046 xhci_device_generic_multi_enter(xfer
->endpoint
,
3047 xfer
->stream_id
, xfer
);
3051 xhci_device_generic_start(struct usb_xfer
*xfer
)
3055 /* try to insert xfer on HW queue */
3056 xhci_transfer_insert(xfer
);
3058 /* try to multi buffer */
3059 xhci_device_generic_multi_enter(xfer
->endpoint
,
3060 xfer
->stream_id
, NULL
);
3062 /* add transfer last on interrupt queue */
3063 usbd_transfer_enqueue(&xfer
->xroot
->bus
->intr_q
, xfer
);
3065 /* start timeout, if any */
3066 if (xfer
->timeout
!= 0)
3067 usbd_transfer_timeout_ms(xfer
, &xhci_timeout
, xfer
->timeout
);
3070 static const struct usb_pipe_methods xhci_device_generic_methods
=
3072 .open
= xhci_device_generic_open
,
3073 .close
= xhci_device_generic_close
,
3074 .enter
= xhci_device_generic_enter
,
3075 .start
= xhci_device_generic_start
,
3078 /*------------------------------------------------------------------------*
3079 * xhci root HUB support
3080 *------------------------------------------------------------------------*
3081 * Simulate a hardware HUB by handling all the necessary requests.
3082 *------------------------------------------------------------------------*/
3084 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) }
3087 struct usb_device_descriptor xhci_devd
=
3089 .bLength
= sizeof(xhci_devd
),
3090 .bDescriptorType
= UDESC_DEVICE
, /* type */
3091 HSETW(.bcdUSB
, 0x0300), /* USB version */
3092 .bDeviceClass
= UDCLASS_HUB
, /* class */
3093 .bDeviceSubClass
= UDSUBCLASS_HUB
, /* subclass */
3094 .bDeviceProtocol
= UDPROTO_SSHUB
, /* protocol */
3095 .bMaxPacketSize
= 9, /* max packet size */
3096 HSETW(.idVendor
, 0x0000), /* vendor */
3097 HSETW(.idProduct
, 0x0000), /* product */
3098 HSETW(.bcdDevice
, 0x0100), /* device version */
3102 .bNumConfigurations
= 1, /* # of configurations */
3106 struct xhci_bos_desc xhci_bosd
= {
3108 .bLength
= sizeof(xhci_bosd
.bosd
),
3109 .bDescriptorType
= UDESC_BOS
,
3110 HSETW(.wTotalLength
, sizeof(xhci_bosd
)),
3111 .bNumDeviceCaps
= 3,
3114 .bLength
= sizeof(xhci_bosd
.usb2extd
),
3115 .bDescriptorType
= 1,
3116 .bDevCapabilityType
= 2,
3117 .bmAttributes
[0] = 2,
3120 .bLength
= sizeof(xhci_bosd
.usbdcd
),
3121 .bDescriptorType
= UDESC_DEVICE_CAPABILITY
,
3122 .bDevCapabilityType
= 3,
3123 .bmAttributes
= 0, /* XXX */
3124 HSETW(.wSpeedsSupported
, 0x000C),
3125 .bFunctionalitySupport
= 8,
3126 .bU1DevExitLat
= 255, /* dummy - not used */
3127 .wU2DevExitLat
= { 0x00, 0x08 },
3130 .bLength
= sizeof(xhci_bosd
.cidd
),
3131 .bDescriptorType
= 1,
3132 .bDevCapabilityType
= 4,
3134 .bContainerID
= 0, /* XXX */
3139 struct xhci_config_desc xhci_confd
= {
3141 .bLength
= sizeof(xhci_confd
.confd
),
3142 .bDescriptorType
= UDESC_CONFIG
,
3143 .wTotalLength
[0] = sizeof(xhci_confd
),
3145 .bConfigurationValue
= 1,
3146 .iConfiguration
= 0,
3147 .bmAttributes
= UC_SELF_POWERED
,
3148 .bMaxPower
= 0 /* max power */
3151 .bLength
= sizeof(xhci_confd
.ifcd
),
3152 .bDescriptorType
= UDESC_INTERFACE
,
3154 .bInterfaceClass
= UICLASS_HUB
,
3155 .bInterfaceSubClass
= UISUBCLASS_HUB
,
3156 .bInterfaceProtocol
= 0,
3159 .bLength
= sizeof(xhci_confd
.endpd
),
3160 .bDescriptorType
= UDESC_ENDPOINT
,
3161 .bEndpointAddress
= UE_DIR_IN
| XHCI_INTR_ENDPT
,
3162 .bmAttributes
= UE_INTERRUPT
,
3163 .wMaxPacketSize
[0] = 2, /* max 15 ports */
3167 .bLength
= sizeof(xhci_confd
.endpcd
),
3168 .bDescriptorType
= UDESC_ENDPOINT_SS_COMP
,
3175 struct usb_hub_ss_descriptor xhci_hubd
= {
3176 .bLength
= sizeof(xhci_hubd
),
3177 .bDescriptorType
= UDESC_SS_HUB
,
3181 xhci_roothub_exec(struct usb_device
*udev
,
3182 struct usb_device_request
*req
, const void **pptr
, uint16_t *plength
)
3184 struct xhci_softc
*sc
= XHCI_BUS2SC(udev
->bus
);
3185 const char *str_ptr
;
3196 USB_BUS_LOCK_ASSERT(&sc
->sc_bus
);
3199 ptr
= (const void *)&sc
->sc_hub_desc
;
3203 value
= UGETW(req
->wValue
);
3204 index
= UGETW(req
->wIndex
);
3206 DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x "
3207 "wValue=0x%04x wIndex=0x%04x\n",
3208 req
->bmRequestType
, req
->bRequest
,
3209 UGETW(req
->wLength
), value
, index
);
3211 #define C(x,y) ((x) | ((y) << 8))
3212 switch (C(req
->bRequest
, req
->bmRequestType
)) {
3213 case C(UR_CLEAR_FEATURE
, UT_WRITE_DEVICE
):
3214 case C(UR_CLEAR_FEATURE
, UT_WRITE_INTERFACE
):
3215 case C(UR_CLEAR_FEATURE
, UT_WRITE_ENDPOINT
):
3217 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops
3218 * for the integrated root hub.
3221 case C(UR_GET_CONFIG
, UT_READ_DEVICE
):
3223 sc
->sc_hub_desc
.temp
[0] = sc
->sc_conf
;
3225 case C(UR_GET_DESCRIPTOR
, UT_READ_DEVICE
):
3226 switch (value
>> 8) {
3228 if ((value
& 0xff) != 0) {
3229 err
= USB_ERR_IOERROR
;
3232 len
= sizeof(xhci_devd
);
3233 ptr
= (const void *)&xhci_devd
;
3237 if ((value
& 0xff) != 0) {
3238 err
= USB_ERR_IOERROR
;
3241 len
= sizeof(xhci_bosd
);
3242 ptr
= (const void *)&xhci_bosd
;
3246 if ((value
& 0xff) != 0) {
3247 err
= USB_ERR_IOERROR
;
3250 len
= sizeof(xhci_confd
);
3251 ptr
= (const void *)&xhci_confd
;
3255 switch (value
& 0xff) {
3256 case 0: /* Language table */
3260 case 1: /* Vendor */
3261 str_ptr
= sc
->sc_vendor
;
3264 case 2: /* Product */
3265 str_ptr
= "XHCI root HUB";
3273 len
= usb_make_str_desc(
3274 sc
->sc_hub_desc
.temp
,
3275 sizeof(sc
->sc_hub_desc
.temp
),
3280 err
= USB_ERR_IOERROR
;
3284 case C(UR_GET_INTERFACE
, UT_READ_INTERFACE
):
3286 sc
->sc_hub_desc
.temp
[0] = 0;
3288 case C(UR_GET_STATUS
, UT_READ_DEVICE
):
3290 USETW(sc
->sc_hub_desc
.stat
.wStatus
, UDS_SELF_POWERED
);
3292 case C(UR_GET_STATUS
, UT_READ_INTERFACE
):
3293 case C(UR_GET_STATUS
, UT_READ_ENDPOINT
):
3295 USETW(sc
->sc_hub_desc
.stat
.wStatus
, 0);
3297 case C(UR_SET_ADDRESS
, UT_WRITE_DEVICE
):
3298 if (value
>= XHCI_MAX_DEVICES
) {
3299 err
= USB_ERR_IOERROR
;
3303 case C(UR_SET_CONFIG
, UT_WRITE_DEVICE
):
3304 if (value
!= 0 && value
!= 1) {
3305 err
= USB_ERR_IOERROR
;
3308 sc
->sc_conf
= value
;
3310 case C(UR_SET_DESCRIPTOR
, UT_WRITE_DEVICE
):
3312 case C(UR_SET_FEATURE
, UT_WRITE_DEVICE
):
3313 case C(UR_SET_FEATURE
, UT_WRITE_INTERFACE
):
3314 case C(UR_SET_FEATURE
, UT_WRITE_ENDPOINT
):
3315 err
= USB_ERR_IOERROR
;
3317 case C(UR_SET_INTERFACE
, UT_WRITE_INTERFACE
):
3319 case C(UR_SYNCH_FRAME
, UT_WRITE_ENDPOINT
):
3322 case C(UR_CLEAR_FEATURE
, UT_WRITE_CLASS_DEVICE
):
3324 case C(UR_CLEAR_FEATURE
, UT_WRITE_CLASS_OTHER
):
3325 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n");
3328 (index
> sc
->sc_noport
)) {
3329 err
= USB_ERR_IOERROR
;
3332 port
= XHCI_PORTSC(index
);
3334 v
= XREAD4(sc
, oper
, port
);
3335 i
= XHCI_PS_PLS_GET(v
);
3336 v
&= ~XHCI_PS_CLEAR
;
3339 case UHF_C_BH_PORT_RESET
:
3340 XWRITE4(sc
, oper
, port
, v
| XHCI_PS_WRC
);
3342 case UHF_C_PORT_CONFIG_ERROR
:
3343 XWRITE4(sc
, oper
, port
, v
| XHCI_PS_CEC
);
3345 case UHF_C_PORT_SUSPEND
:
3346 case UHF_C_PORT_LINK_STATE
:
3347 XWRITE4(sc
, oper
, port
, v
| XHCI_PS_PLC
);
3349 case UHF_C_PORT_CONNECTION
:
3350 XWRITE4(sc
, oper
, port
, v
| XHCI_PS_CSC
);
3352 case UHF_C_PORT_ENABLE
:
3353 XWRITE4(sc
, oper
, port
, v
| XHCI_PS_PEC
);
3355 case UHF_C_PORT_OVER_CURRENT
:
3356 XWRITE4(sc
, oper
, port
, v
| XHCI_PS_OCC
);
3358 case UHF_C_PORT_RESET
:
3359 XWRITE4(sc
, oper
, port
, v
| XHCI_PS_PRC
);
3361 case UHF_PORT_ENABLE
:
3362 XWRITE4(sc
, oper
, port
, v
| XHCI_PS_PED
);
3364 case UHF_PORT_POWER
:
3365 XWRITE4(sc
, oper
, port
, v
& ~XHCI_PS_PP
);
3367 case UHF_PORT_INDICATOR
:
3368 XWRITE4(sc
, oper
, port
, v
& ~XHCI_PS_PIC_SET(3));
3370 case UHF_PORT_SUSPEND
:
3374 XWRITE4(sc
, oper
, port
, v
|
3375 XHCI_PS_PLS_SET(0xF) | XHCI_PS_LWS
);
3378 /* wait 20ms for resume sequence to complete */
3379 usb_pause_mtx(&sc
->sc_bus
.bus_lock
, hz
/ 50);
3382 XWRITE4(sc
, oper
, port
, v
|
3383 XHCI_PS_PLS_SET(0) | XHCI_PS_LWS
);
3386 err
= USB_ERR_IOERROR
;
3391 case C(UR_GET_DESCRIPTOR
, UT_READ_CLASS_DEVICE
):
3392 if ((value
& 0xff) != 0) {
3393 err
= USB_ERR_IOERROR
;
3397 v
= XREAD4(sc
, capa
, XHCI_HCSPARAMS0
);
3399 sc
->sc_hub_desc
.hubd
= xhci_hubd
;
3401 sc
->sc_hub_desc
.hubd
.bNbrPorts
= sc
->sc_noport
;
3403 if (XHCI_HCS0_PPC(v
))
3404 i
= UHD_PWR_INDIVIDUAL
;
3408 if (XHCI_HCS0_PIND(v
))
3411 i
|= UHD_OC_INDIVIDUAL
;
3413 USETW(sc
->sc_hub_desc
.hubd
.wHubCharacteristics
, i
);
3415 /* see XHCI section 5.4.9: */
3416 sc
->sc_hub_desc
.hubd
.bPwrOn2PwrGood
= 10;
3418 for (j
= 1; j
<= sc
->sc_noport
; j
++) {
3420 v
= XREAD4(sc
, oper
, XHCI_PORTSC(j
));
3421 if (v
& XHCI_PS_DR
) {
3422 sc
->sc_hub_desc
.hubd
.
3423 DeviceRemovable
[j
/ 8] |= 1U << (j
% 8);
3426 len
= sc
->sc_hub_desc
.hubd
.bLength
;
3429 case C(UR_GET_STATUS
, UT_READ_CLASS_DEVICE
):
3431 memset(sc
->sc_hub_desc
.temp
, 0, 16);
3434 case C(UR_GET_STATUS
, UT_READ_CLASS_OTHER
):
3435 DPRINTFN(9, "UR_GET_STATUS i=%d\n", index
);
3438 (index
> sc
->sc_noport
)) {
3439 err
= USB_ERR_IOERROR
;
3443 v
= XREAD4(sc
, oper
, XHCI_PORTSC(index
));
3445 DPRINTFN(9, "port status=0x%08x\n", v
);
3447 i
= UPS_PORT_LINK_STATE_SET(XHCI_PS_PLS_GET(v
));
3449 switch (XHCI_PS_SPEED_GET(v
)) {
3451 i
|= UPS_HIGH_SPEED
;
3460 i
|= UPS_OTHER_SPEED
;
3464 if (v
& XHCI_PS_CCS
)
3465 i
|= UPS_CURRENT_CONNECT_STATUS
;
3466 if (v
& XHCI_PS_PED
)
3467 i
|= UPS_PORT_ENABLED
;
3468 if (v
& XHCI_PS_OCA
)
3469 i
|= UPS_OVERCURRENT_INDICATOR
;
3472 if (v
& XHCI_PS_PP
) {
3474 * The USB 3.0 RH is using the
3475 * USB 2.0's power bit
3477 i
|= UPS_PORT_POWER
;
3479 USETW(sc
->sc_hub_desc
.ps
.wPortStatus
, i
);
3482 if (v
& XHCI_PS_CSC
)
3483 i
|= UPS_C_CONNECT_STATUS
;
3484 if (v
& XHCI_PS_PEC
)
3485 i
|= UPS_C_PORT_ENABLED
;
3486 if (v
& XHCI_PS_OCC
)
3487 i
|= UPS_C_OVERCURRENT_INDICATOR
;
3488 if (v
& XHCI_PS_WRC
)
3489 i
|= UPS_C_BH_PORT_RESET
;
3490 if (v
& XHCI_PS_PRC
)
3491 i
|= UPS_C_PORT_RESET
;
3492 if (v
& XHCI_PS_PLC
)
3493 i
|= UPS_C_PORT_LINK_STATE
;
3494 if (v
& XHCI_PS_CEC
)
3495 i
|= UPS_C_PORT_CONFIG_ERROR
;
3497 USETW(sc
->sc_hub_desc
.ps
.wPortChange
, i
);
3498 len
= sizeof(sc
->sc_hub_desc
.ps
);
3501 case C(UR_SET_DESCRIPTOR
, UT_WRITE_CLASS_DEVICE
):
3502 err
= USB_ERR_IOERROR
;
3505 case C(UR_SET_FEATURE
, UT_WRITE_CLASS_DEVICE
):
3508 case C(UR_SET_FEATURE
, UT_WRITE_CLASS_OTHER
):
3514 (index
> sc
->sc_noport
)) {
3515 err
= USB_ERR_IOERROR
;
3519 port
= XHCI_PORTSC(index
);
3520 v
= XREAD4(sc
, oper
, port
) & ~XHCI_PS_CLEAR
;
3523 case UHF_PORT_U1_TIMEOUT
:
3524 if (XHCI_PS_SPEED_GET(v
) != 4) {
3525 err
= USB_ERR_IOERROR
;
3528 port
= XHCI_PORTPMSC(index
);
3529 v
= XREAD4(sc
, oper
, port
);
3530 v
&= ~XHCI_PM3_U1TO_SET(0xFF);
3531 v
|= XHCI_PM3_U1TO_SET(i
);
3532 XWRITE4(sc
, oper
, port
, v
);
3534 case UHF_PORT_U2_TIMEOUT
:
3535 if (XHCI_PS_SPEED_GET(v
) != 4) {
3536 err
= USB_ERR_IOERROR
;
3539 port
= XHCI_PORTPMSC(index
);
3540 v
= XREAD4(sc
, oper
, port
);
3541 v
&= ~XHCI_PM3_U2TO_SET(0xFF);
3542 v
|= XHCI_PM3_U2TO_SET(i
);
3543 XWRITE4(sc
, oper
, port
, v
);
3545 case UHF_BH_PORT_RESET
:
3546 XWRITE4(sc
, oper
, port
, v
| XHCI_PS_WPR
);
3548 case UHF_PORT_LINK_STATE
:
3549 XWRITE4(sc
, oper
, port
, v
|
3550 XHCI_PS_PLS_SET(i
) | XHCI_PS_LWS
);
3551 /* 4ms settle time */
3552 usb_pause_mtx(&sc
->sc_bus
.bus_lock
, hz
/ 250);
3554 case UHF_PORT_ENABLE
:
3555 DPRINTFN(3, "set port enable %d\n", index
);
3557 case UHF_PORT_SUSPEND
:
3558 DPRINTFN(6, "suspend port %u (LPM=%u)\n", index
, i
);
3559 j
= XHCI_PS_SPEED_GET(v
);
3560 if ((j
< 1) || (j
> 3)) {
3561 /* non-supported speed */
3562 err
= USB_ERR_IOERROR
;
3565 XWRITE4(sc
, oper
, port
, v
|
3566 XHCI_PS_PLS_SET(i
? 2 /* LPM */ : 3) | XHCI_PS_LWS
);
3568 case UHF_PORT_RESET
:
3569 DPRINTFN(6, "reset port %d\n", index
);
3570 XWRITE4(sc
, oper
, port
, v
| XHCI_PS_PR
);
3572 case UHF_PORT_POWER
:
3573 DPRINTFN(3, "set port power %d\n", index
);
3574 XWRITE4(sc
, oper
, port
, v
| XHCI_PS_PP
);
3577 DPRINTFN(3, "set port test %d\n", index
);
3579 case UHF_PORT_INDICATOR
:
3580 DPRINTFN(3, "set port indicator %d\n", index
);
3582 v
&= ~XHCI_PS_PIC_SET(3);
3583 v
|= XHCI_PS_PIC_SET(1);
3585 XWRITE4(sc
, oper
, port
, v
);
3588 err
= USB_ERR_IOERROR
;
3593 case C(UR_CLEAR_TT_BUFFER
, UT_WRITE_CLASS_OTHER
):
3594 case C(UR_RESET_TT
, UT_WRITE_CLASS_OTHER
):
3595 case C(UR_GET_TT_STATE
, UT_READ_CLASS_OTHER
):
3596 case C(UR_STOP_TT
, UT_WRITE_CLASS_OTHER
):
3599 err
= USB_ERR_IOERROR
;
3609 xhci_xfer_setup(struct usb_setup_params
*parm
)
3611 struct usb_page_search page_info
;
3612 struct usb_page_cache
*pc
;
3613 struct xhci_softc
*sc
;
3614 struct usb_xfer
*xfer
;
3619 sc
= XHCI_BUS2SC(parm
->udev
->bus
);
3620 xfer
= parm
->curr_xfer
;
3623 * The proof for the "ntd" formula is illustrated like this:
3625 * +------------------------------------+
3629 * | | xxx | x | frm 0 |
3631 * | | xxx | xx | frm 1 |
3634 * +------------------------------------+
3636 * "xxx" means a completely full USB transfer descriptor
3638 * "x" and "xx" means a short USB packet
3640 * For the remainder of an USB transfer modulo
3641 * "max_data_length" we need two USB transfer descriptors.
3642 * One to transfer the remaining data and one to finalise with
3643 * a zero length packet in case the "force_short_xfer" flag is
3644 * set. We only need two USB transfer descriptors in the case
3645 * where the transfer length of the first one is a factor of
3646 * "max_frame_size". The rest of the needed USB transfer
3647 * descriptors is given by the buffer size divided by the
3648 * maximum data payload.
3650 parm
->hc_max_packet_size
= 0x400;
3651 parm
->hc_max_packet_count
= 16 * 3;
3652 parm
->hc_max_frame_size
= XHCI_TD_PAYLOAD_MAX
;
3654 xfer
->flags_int
.bdma_enable
= 1;
3656 usbd_transfer_setup_sub(parm
);
3658 if (xfer
->flags_int
.isochronous_xfr
) {
3659 ntd
= ((1 * xfer
->nframes
)
3660 + (xfer
->max_data_length
/ xfer
->max_hc_frame_size
));
3661 } else if (xfer
->flags_int
.control_xfr
) {
3662 ntd
= ((2 * xfer
->nframes
) + 1 /* STATUS */
3663 + (xfer
->max_data_length
/ xfer
->max_hc_frame_size
));
3665 ntd
= ((2 * xfer
->nframes
)
3666 + (xfer
->max_data_length
/ xfer
->max_hc_frame_size
));
3675 * Allocate queue heads and transfer descriptors
3679 if (usbd_transfer_setup_sub_malloc(
3680 parm
, &pc
, sizeof(struct xhci_td
),
3681 XHCI_TD_ALIGN
, ntd
)) {
3682 parm
->err
= USB_ERR_NOMEM
;
3686 for (n
= 0; n
!= ntd
; n
++) {
3689 usbd_get_page(pc
+ n
, 0, &page_info
);
3691 td
= page_info
.buffer
;
3694 td
->td_self
= page_info
.physaddr
;
3695 td
->obj_next
= last_obj
;
3696 td
->page_cache
= pc
+ n
;
3700 usb_pc_cpu_flush(pc
+ n
);
3703 xfer
->td_start
[xfer
->flags_int
.curr_dma_set
] = last_obj
;
3705 if (!xfer
->flags_int
.curr_dma_set
) {
3706 xfer
->flags_int
.curr_dma_set
= 1;
3712 xhci_configure_reset_endpoint(struct usb_xfer
*xfer
)
3714 struct xhci_softc
*sc
= XHCI_BUS2SC(xfer
->xroot
->bus
);
3715 struct usb_page_search buf_inp
;
3716 struct usb_device
*udev
;
3717 struct xhci_endpoint_ext
*pepext
;
3718 struct usb_endpoint_descriptor
*edesc
;
3719 struct usb_page_cache
*pcinp
;
3721 usb_stream_t stream_id
;
3725 pepext
= xhci_get_endpoint_ext(xfer
->xroot
->udev
,
3726 xfer
->endpoint
->edesc
);
3728 udev
= xfer
->xroot
->udev
;
3729 index
= udev
->controller_slot_id
;
3731 pcinp
= &sc
->sc_hw
.devs
[index
].input_pc
;
3733 usbd_get_page(pcinp
, 0, &buf_inp
);
3735 edesc
= xfer
->endpoint
->edesc
;
3737 epno
= edesc
->bEndpointAddress
;
3738 stream_id
= xfer
->stream_id
;
3740 if ((edesc
->bmAttributes
& UE_XFERTYPE
) == UE_CONTROL
)
3743 epno
= XHCI_EPNO2EPID(epno
);
3746 return (USB_ERR_NO_PIPE
); /* invalid */
3750 /* configure endpoint */
3752 err
= xhci_configure_endpoint_by_xfer(xfer
);
3755 XHCI_CMD_UNLOCK(sc
);
3760 * Get the endpoint into the stopped state according to the
3761 * endpoint context state diagram in the XHCI specification:
3764 err
= xhci_cmd_stop_ep(sc
, 0, epno
, index
);
3767 DPRINTF("Could not stop endpoint %u\n", epno
);
3769 err
= xhci_cmd_reset_ep(sc
, 0, epno
, index
);
3772 DPRINTF("Could not reset endpoint %u\n", epno
);
3774 err
= xhci_cmd_set_tr_dequeue_ptr(sc
,
3775 (pepext
->physaddr
+ (stream_id
* sizeof(struct xhci_trb
) *
3776 XHCI_MAX_TRANSFERS
)) | XHCI_EPCTX_2_DCS_SET(1),
3777 stream_id
, epno
, index
);
3780 DPRINTF("Could not set dequeue ptr for endpoint %u\n", epno
);
3783 * Get the endpoint into the running state according to the
3784 * endpoint context state diagram in the XHCI specification:
3787 xhci_configure_mask(udev
, (1U << epno
) | 1U, 0);
3789 err
= xhci_cmd_evaluate_ctx(sc
, buf_inp
.physaddr
, index
);
3792 DPRINTF("Could not configure endpoint %u\n", epno
);
3794 err
= xhci_cmd_configure_ep(sc
, buf_inp
.physaddr
, 0, index
);
3797 DPRINTF("Could not configure endpoint %u\n", epno
);
3799 XHCI_CMD_UNLOCK(sc
);
3805 xhci_xfer_unsetup(struct usb_xfer
*xfer
)
3811 xhci_start_dma_delay(struct usb_xfer
*xfer
)
3813 struct xhci_softc
*sc
= XHCI_BUS2SC(xfer
->xroot
->bus
);
3815 /* put transfer on interrupt queue (again) */
3816 usbd_transfer_enqueue(&sc
->sc_bus
.intr_q
, xfer
);
3818 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc
->sc_bus
),
3819 &sc
->sc_config_msg
[0], &sc
->sc_config_msg
[1]);
3823 xhci_configure_msg(struct usb_proc_msg
*pm
)
3825 struct xhci_softc
*sc
;
3826 struct xhci_endpoint_ext
*pepext
;
3827 struct usb_xfer
*xfer
;
3829 sc
= XHCI_BUS2SC(((struct usb_bus_msg
*)pm
)->bus
);
3832 TAILQ_FOREACH(xfer
, &sc
->sc_bus
.intr_q
.head
, wait_entry
) {
3834 pepext
= xhci_get_endpoint_ext(xfer
->xroot
->udev
,
3835 xfer
->endpoint
->edesc
);
3837 if ((pepext
->trb_halted
!= 0) ||
3838 (pepext
->trb_running
== 0)) {
3842 /* clear halted and running */
3843 pepext
->trb_halted
= 0;
3844 pepext
->trb_running
= 0;
3846 /* nuke remaining buffered transfers */
3848 for (i
= 0; i
!= (XHCI_MAX_TRANSFERS
*
3849 XHCI_MAX_STREAMS
); i
++) {
3851 * NOTE: We need to use the timeout
3852 * error code here else existing
3853 * isochronous clients can get
3856 if (pepext
->xfer
[i
] != NULL
) {
3857 xhci_device_done(pepext
->xfer
[i
],
3863 * NOTE: The USB transfer cannot vanish in
3867 USB_BUS_UNLOCK(&sc
->sc_bus
);
3869 xhci_configure_reset_endpoint(xfer
);
3871 USB_BUS_LOCK(&sc
->sc_bus
);
3873 /* check if halted is still cleared */
3874 if (pepext
->trb_halted
== 0) {
3875 pepext
->trb_running
= 1;
3876 memset(pepext
->trb_index
, 0,
3877 sizeof(pepext
->trb_index
));
3882 if (xfer
->flags_int
.did_dma_delay
) {
3884 /* remove transfer from interrupt queue (again) */
3885 usbd_transfer_dequeue(xfer
);
3887 /* we are finally done */
3888 usb_dma_delay_done_cb(xfer
);
3890 /* queue changed - restart */
3895 TAILQ_FOREACH(xfer
, &sc
->sc_bus
.intr_q
.head
, wait_entry
) {
3897 /* try to insert xfer on HW queue */
3898 xhci_transfer_insert(xfer
);
3900 /* try to multi buffer */
3901 xhci_device_generic_multi_enter(xfer
->endpoint
,
3902 xfer
->stream_id
, NULL
);
3907 xhci_ep_init(struct usb_device
*udev
, struct usb_endpoint_descriptor
*edesc
,
3908 struct usb_endpoint
*ep
)
3910 struct xhci_endpoint_ext
*pepext
;
3912 DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d\n",
3913 ep
, udev
->address
, edesc
->bEndpointAddress
, udev
->flags
.usb_mode
);
3915 if (udev
->parent_hub
== NULL
) {
3916 /* root HUB has special endpoint handling */
3920 ep
->methods
= &xhci_device_generic_methods
;
3922 pepext
= xhci_get_endpoint_ext(udev
, edesc
);
3924 USB_BUS_LOCK(udev
->bus
);
3925 pepext
->trb_halted
= 1;
3926 pepext
->trb_running
= 0;
3927 USB_BUS_UNLOCK(udev
->bus
);
3931 xhci_ep_uninit(struct usb_device
*udev
, struct usb_endpoint
*ep
)
3937 xhci_ep_clear_stall(struct usb_device
*udev
, struct usb_endpoint
*ep
)
3939 struct xhci_endpoint_ext
*pepext
;
3943 if (udev
->flags
.usb_mode
!= USB_MODE_HOST
) {
3947 if (udev
->parent_hub
== NULL
) {
3948 /* root HUB has special endpoint handling */
3952 pepext
= xhci_get_endpoint_ext(udev
, ep
->edesc
);
3954 USB_BUS_LOCK(udev
->bus
);
3955 pepext
->trb_halted
= 1;
3956 pepext
->trb_running
= 0;
3957 USB_BUS_UNLOCK(udev
->bus
);
3961 xhci_device_init(struct usb_device
*udev
)
3963 struct xhci_softc
*sc
= XHCI_BUS2SC(udev
->bus
);
3967 /* no init for root HUB */
3968 if (udev
->parent_hub
== NULL
)
3973 /* set invalid default */
3975 udev
->controller_slot_id
= sc
->sc_noslot
+ 1;
3977 /* try to get a new slot ID from the XHCI */
3979 err
= xhci_cmd_enable_slot(sc
, &temp
);
3982 XHCI_CMD_UNLOCK(sc
);
3986 if (temp
> sc
->sc_noslot
) {
3987 XHCI_CMD_UNLOCK(sc
);
3988 return (USB_ERR_BAD_ADDRESS
);
3991 if (sc
->sc_hw
.devs
[temp
].state
!= XHCI_ST_DISABLED
) {
3992 DPRINTF("slot %u already allocated.\n", temp
);
3993 XHCI_CMD_UNLOCK(sc
);
3994 return (USB_ERR_BAD_ADDRESS
);
3997 /* store slot ID for later reference */
3999 udev
->controller_slot_id
= temp
;
4001 /* reset data structure */
4003 memset(&sc
->sc_hw
.devs
[temp
], 0, sizeof(sc
->sc_hw
.devs
[0]));
4005 /* set mark slot allocated */
4007 sc
->sc_hw
.devs
[temp
].state
= XHCI_ST_ENABLED
;
4009 err
= xhci_alloc_device_ext(udev
);
4011 XHCI_CMD_UNLOCK(sc
);
4013 /* get device into default state */
4016 err
= xhci_set_address(udev
, NULL
, 0);
4022 xhci_device_uninit(struct usb_device
*udev
)
4024 struct xhci_softc
*sc
= XHCI_BUS2SC(udev
->bus
);
4027 /* no init for root HUB */
4028 if (udev
->parent_hub
== NULL
)
4033 index
= udev
->controller_slot_id
;
4035 if (index
<= sc
->sc_noslot
) {
4036 xhci_cmd_disable_slot(sc
, index
);
4037 sc
->sc_hw
.devs
[index
].state
= XHCI_ST_DISABLED
;
4039 /* free device extension */
4040 xhci_free_device_ext(udev
);
4043 XHCI_CMD_UNLOCK(sc
);
4047 xhci_get_dma_delay(struct usb_device
*udev
, uint32_t *pus
)
4050 * Wait until the hardware has finished any possible use of
4051 * the transfer descriptor(s)
4053 *pus
= 2048; /* microseconds */
4057 xhci_device_resume(struct usb_device
*udev
)
4059 struct xhci_softc
*sc
= XHCI_BUS2SC(udev
->bus
);
4066 /* check for root HUB */
4067 if (udev
->parent_hub
== NULL
)
4070 index
= udev
->controller_slot_id
;
4074 /* blindly resume all endpoints */
4076 USB_BUS_LOCK(udev
->bus
);
4078 for (n
= 1; n
!= XHCI_MAX_ENDPOINTS
; n
++) {
4079 for (p
= 0; p
!= XHCI_MAX_STREAMS
; p
++) {
4080 XWRITE4(sc
, door
, XHCI_DOORBELL(index
),
4081 n
| XHCI_DB_SID_SET(p
));
4085 USB_BUS_UNLOCK(udev
->bus
);
4087 XHCI_CMD_UNLOCK(sc
);
4091 xhci_device_suspend(struct usb_device
*udev
)
4093 struct xhci_softc
*sc
= XHCI_BUS2SC(udev
->bus
);
4100 /* check for root HUB */
4101 if (udev
->parent_hub
== NULL
)
4104 index
= udev
->controller_slot_id
;
4108 /* blindly suspend all endpoints */
4110 for (n
= 1; n
!= XHCI_MAX_ENDPOINTS
; n
++) {
4111 err
= xhci_cmd_stop_ep(sc
, 1, n
, index
);
4113 DPRINTF("Failed to suspend endpoint "
4114 "%u on slot %u (ignored).\n", n
, index
);
4118 XHCI_CMD_UNLOCK(sc
);
4122 xhci_set_hw_power(struct usb_bus
*bus
)
4128 xhci_device_state_change(struct usb_device
*udev
)
4130 struct xhci_softc
*sc
= XHCI_BUS2SC(udev
->bus
);
4131 struct usb_page_search buf_inp
;
4135 /* check for root HUB */
4136 if (udev
->parent_hub
== NULL
)
4139 index
= udev
->controller_slot_id
;
4143 if (usb_get_device_state(udev
) == USB_STATE_CONFIGURED
) {
4144 err
= uhub_query_info(udev
, &sc
->sc_hw
.devs
[index
].nports
,
4145 &sc
->sc_hw
.devs
[index
].tt
);
4147 sc
->sc_hw
.devs
[index
].nports
= 0;
4152 switch (usb_get_device_state(udev
)) {
4153 case USB_STATE_POWERED
:
4154 if (sc
->sc_hw
.devs
[index
].state
== XHCI_ST_DEFAULT
)
4157 /* set default state */
4158 sc
->sc_hw
.devs
[index
].state
= XHCI_ST_DEFAULT
;
4160 /* reset number of contexts */
4161 sc
->sc_hw
.devs
[index
].context_num
= 0;
4163 err
= xhci_cmd_reset_dev(sc
, index
);
4166 DPRINTF("Device reset failed "
4167 "for slot %u.\n", index
);
4171 case USB_STATE_ADDRESSED
:
4172 if (sc
->sc_hw
.devs
[index
].state
== XHCI_ST_ADDRESSED
)
4175 sc
->sc_hw
.devs
[index
].state
= XHCI_ST_ADDRESSED
;
4177 err
= xhci_cmd_configure_ep(sc
, 0, 1, index
);
4180 DPRINTF("Failed to deconfigure "
4181 "slot %u.\n", index
);
4185 case USB_STATE_CONFIGURED
:
4186 if (sc
->sc_hw
.devs
[index
].state
== XHCI_ST_CONFIGURED
)
4189 /* set configured state */
4190 sc
->sc_hw
.devs
[index
].state
= XHCI_ST_CONFIGURED
;
4192 /* reset number of contexts */
4193 sc
->sc_hw
.devs
[index
].context_num
= 0;
4195 usbd_get_page(&sc
->sc_hw
.devs
[index
].input_pc
, 0, &buf_inp
);
4197 xhci_configure_mask(udev
, 3, 0);
4199 err
= xhci_configure_device(udev
);
4201 DPRINTF("Could not configure device "
4202 "at slot %u.\n", index
);
4205 err
= xhci_cmd_evaluate_ctx(sc
, buf_inp
.physaddr
, index
);
4207 DPRINTF("Could not evaluate device "
4208 "context at slot %u.\n", index
);
4215 XHCI_CMD_UNLOCK(sc
);
4219 xhci_set_endpoint_mode(struct usb_device
*udev
, struct usb_endpoint
*ep
,
4223 case USB_EP_MODE_DEFAULT
:
4225 case USB_EP_MODE_STREAMS
:
4226 if (xhcistreams
== 0 ||
4227 (ep
->edesc
->bmAttributes
& UE_XFERTYPE
) != UE_BULK
||
4228 udev
->speed
!= USB_SPEED_SUPER
)
4229 return (USB_ERR_INVAL
);
4232 return (USB_ERR_INVAL
);
4236 static const struct usb_bus_methods xhci_bus_methods
= {
4237 .endpoint_init
= xhci_ep_init
,
4238 .endpoint_uninit
= xhci_ep_uninit
,
4239 .xfer_setup
= xhci_xfer_setup
,
4240 .xfer_unsetup
= xhci_xfer_unsetup
,
4241 .get_dma_delay
= xhci_get_dma_delay
,
4242 .device_init
= xhci_device_init
,
4243 .device_uninit
= xhci_device_uninit
,
4244 .device_resume
= xhci_device_resume
,
4245 .device_suspend
= xhci_device_suspend
,
4246 .set_hw_power
= xhci_set_hw_power
,
4247 .roothub_exec
= xhci_roothub_exec
,
4248 .xfer_poll
= xhci_do_poll
,
4249 .start_dma_delay
= xhci_start_dma_delay
,
4250 .set_address
= xhci_set_address
,
4251 .clear_stall
= xhci_ep_clear_stall
,
4252 .device_state_change
= xhci_device_state_change
,
4253 .set_hw_power_sleep
= xhci_set_hw_power_sleep
,
4254 .set_endpoint_mode
= xhci_set_endpoint_mode
,