2 * Copyright (c) 2001-2004 by David Brownell
3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License as published by the
7 * Free Software Foundation; either version 2 of the License, or (at your
8 * option) any later version.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software Foundation,
17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 /* this file is part of ehci-hcd.c */
22 /*-------------------------------------------------------------------------*/
25 * EHCI scheduled transaction support: interrupt, iso, split iso
26 * These are called "periodic" transactions in the EHCI spec.
28 * Note that for interrupt transfers, the QH/QTD manipulation is shared
29 * with the "asynchronous" transaction support (control/bulk transfers).
30 * The only real difference is in how interrupt transfers are scheduled.
32 * For ISO, we make an "iso_stream" head to serve the same role as a QH.
33 * It keeps track of every ITD (or SITD) that's linked, and holds enough
34 * pre-calculated schedule data to make appending to the queue be quick.
37 static int ehci_get_frame (struct usb_hcd
*hcd
);
39 /*-------------------------------------------------------------------------*/
42 * periodic_next_shadow - return "next" pointer on shadow list
43 * @periodic: host pointer to qh/itd/sitd
44 * @tag: hardware tag for type of this record
46 static union ehci_shadow
*
47 periodic_next_shadow(struct ehci_hcd
*ehci
, union ehci_shadow
*periodic
,
50 switch (hc32_to_cpu(ehci
, tag
)) {
52 return &periodic
->qh
->qh_next
;
54 return &periodic
->fstn
->fstn_next
;
56 return &periodic
->itd
->itd_next
;
59 return &periodic
->sitd
->sitd_next
;
63 /* caller must hold ehci->lock */
64 static void periodic_unlink (struct ehci_hcd
*ehci
, unsigned frame
, void *ptr
)
66 union ehci_shadow
*prev_p
= &ehci
->pshadow
[frame
];
67 __hc32
*hw_p
= &ehci
->periodic
[frame
];
68 union ehci_shadow here
= *prev_p
;
70 /* find predecessor of "ptr"; hw and shadow lists are in sync */
71 while (here
.ptr
&& here
.ptr
!= ptr
) {
72 prev_p
= periodic_next_shadow(ehci
, prev_p
,
73 Q_NEXT_TYPE(ehci
, *hw_p
));
77 /* an interrupt entry (at list end) could have been shared */
81 /* update shadow and hardware lists ... the old "next" pointers
82 * from ptr may still be in use, the caller updates them.
84 *prev_p
= *periodic_next_shadow(ehci
, &here
,
85 Q_NEXT_TYPE(ehci
, *hw_p
));
86 *hw_p
= *here
.hw_next
;
89 /* how many of the uframe's 125 usecs are allocated? */
91 periodic_usecs (struct ehci_hcd
*ehci
, unsigned frame
, unsigned uframe
)
93 __hc32
*hw_p
= &ehci
->periodic
[frame
];
94 union ehci_shadow
*q
= &ehci
->pshadow
[frame
];
98 switch (hc32_to_cpu(ehci
, Q_NEXT_TYPE(ehci
, *hw_p
))) {
100 /* is it in the S-mask? */
101 if (q
->qh
->hw_info2
& cpu_to_hc32(ehci
, 1 << uframe
))
102 usecs
+= q
->qh
->usecs
;
104 if (q
->qh
->hw_info2
& cpu_to_hc32(ehci
,
106 usecs
+= q
->qh
->c_usecs
;
107 hw_p
= &q
->qh
->hw_next
;
112 /* for "save place" FSTNs, count the relevant INTR
113 * bandwidth from the previous frame
115 if (q
->fstn
->hw_prev
!= EHCI_LIST_END(ehci
)) {
116 ehci_dbg (ehci
, "ignoring FSTN cost ...\n");
118 hw_p
= &q
->fstn
->hw_next
;
119 q
= &q
->fstn
->fstn_next
;
122 if (q
->itd
->hw_transaction
[uframe
])
123 usecs
+= q
->itd
->stream
->usecs
;
124 hw_p
= &q
->itd
->hw_next
;
125 q
= &q
->itd
->itd_next
;
128 /* is it in the S-mask? (count SPLIT, DATA) */
129 if (q
->sitd
->hw_uframe
& cpu_to_hc32(ehci
,
131 if (q
->sitd
->hw_fullspeed_ep
&
132 cpu_to_hc32(ehci
, 1<<31))
133 usecs
+= q
->sitd
->stream
->usecs
;
134 else /* worst case for OUT start-split */
135 usecs
+= HS_USECS_ISO (188);
138 /* ... C-mask? (count CSPLIT, DATA) */
139 if (q
->sitd
->hw_uframe
&
140 cpu_to_hc32(ehci
, 1 << (8 + uframe
))) {
141 /* worst case for IN complete-split */
142 usecs
+= q
->sitd
->stream
->c_usecs
;
145 hw_p
= &q
->sitd
->hw_next
;
146 q
= &q
->sitd
->sitd_next
;
152 ehci_err (ehci
, "uframe %d sched overrun: %d usecs\n",
153 frame
* 8 + uframe
, usecs
);
158 /*-------------------------------------------------------------------------*/
160 static int same_tt (struct usb_device
*dev1
, struct usb_device
*dev2
)
162 if (!dev1
->tt
|| !dev2
->tt
)
164 if (dev1
->tt
!= dev2
->tt
)
167 return dev1
->ttport
== dev2
->ttport
;
172 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
174 /* Which uframe does the low/fullspeed transfer start in?
176 * The parameter is the mask of ssplits in "H-frame" terms
177 * and this returns the transfer start uframe in "B-frame" terms,
178 * which allows both to match, e.g. a ssplit in "H-frame" uframe 0
179 * will cause a transfer in "B-frame" uframe 0. "B-frames" lag
180 * "H-frames" by 1 uframe. See the EHCI spec sec 4.5 and figure 4.7.
182 static inline unsigned char tt_start_uframe(struct ehci_hcd
*ehci
, __hc32 mask
)
184 unsigned char smask
= QH_SMASK
& hc32_to_cpu(ehci
, mask
);
186 ehci_err(ehci
, "invalid empty smask!\n");
187 /* uframe 7 can't have bw so this will indicate failure */
190 return ffs(smask
) - 1;
193 static const unsigned char
194 max_tt_usecs
[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
196 /* carryover low/fullspeed bandwidth that crosses uframe boundries */
197 static inline void carryover_tt_bandwidth(unsigned short tt_usecs
[8])
200 for (i
=0; i
<7; i
++) {
201 if (max_tt_usecs
[i
] < tt_usecs
[i
]) {
202 tt_usecs
[i
+1] += tt_usecs
[i
] - max_tt_usecs
[i
];
203 tt_usecs
[i
] = max_tt_usecs
[i
];
208 /* How many of the tt's periodic downstream 1000 usecs are allocated?
210 * While this measures the bandwidth in terms of usecs/uframe,
211 * the low/fullspeed bus has no notion of uframes, so any particular
212 * low/fullspeed transfer can "carry over" from one uframe to the next,
213 * since the TT just performs downstream transfers in sequence.
215 * For example two separate 100 usec transfers can start in the same uframe,
216 * and the second one would "carry over" 75 usecs into the next uframe.
220 struct ehci_hcd
*ehci
,
221 struct usb_device
*dev
,
223 unsigned short tt_usecs
[8]
226 __hc32
*hw_p
= &ehci
->periodic
[frame
];
227 union ehci_shadow
*q
= &ehci
->pshadow
[frame
];
230 memset(tt_usecs
, 0, 16);
233 switch (hc32_to_cpu(ehci
, Q_NEXT_TYPE(ehci
, *hw_p
))) {
235 hw_p
= &q
->itd
->hw_next
;
236 q
= &q
->itd
->itd_next
;
239 if (same_tt(dev
, q
->qh
->dev
)) {
240 uf
= tt_start_uframe(ehci
, q
->qh
->hw_info2
);
241 tt_usecs
[uf
] += q
->qh
->tt_usecs
;
243 hw_p
= &q
->qh
->hw_next
;
247 if (same_tt(dev
, q
->sitd
->urb
->dev
)) {
248 uf
= tt_start_uframe(ehci
, q
->sitd
->hw_uframe
);
249 tt_usecs
[uf
] += q
->sitd
->stream
->tt_usecs
;
251 hw_p
= &q
->sitd
->hw_next
;
252 q
= &q
->sitd
->sitd_next
;
256 ehci_dbg(ehci
, "ignoring periodic frame %d FSTN\n",
258 hw_p
= &q
->fstn
->hw_next
;
259 q
= &q
->fstn
->fstn_next
;
263 carryover_tt_bandwidth(tt_usecs
);
265 if (max_tt_usecs
[7] < tt_usecs
[7])
266 ehci_err(ehci
, "frame %d tt sched overrun: %d usecs\n",
267 frame
, tt_usecs
[7] - max_tt_usecs
[7]);
271 * Return true if the device's tt's downstream bus is available for a
272 * periodic transfer of the specified length (usecs), starting at the
273 * specified frame/uframe. Note that (as summarized in section 11.19
274 * of the usb 2.0 spec) TTs can buffer multiple transactions for each
277 * The uframe parameter is when the fullspeed/lowspeed transfer
278 * should be executed in "B-frame" terms, which is the same as the
279 * highspeed ssplit's uframe (which is in "H-frame" terms). For example
280 * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0.
281 * See the EHCI spec sec 4.5 and fig 4.7.
283 * This checks if the full/lowspeed bus, at the specified starting uframe,
284 * has the specified bandwidth available, according to rules listed
285 * in USB 2.0 spec section 11.18.1 fig 11-60.
287 * This does not check if the transfer would exceed the max ssplit
288 * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4,
289 * since proper scheduling limits ssplits to less than 16 per uframe.
291 static int tt_available (
292 struct ehci_hcd
*ehci
,
294 struct usb_device
*dev
,
300 if ((period
== 0) || (uframe
>= 7)) /* error */
303 for (; frame
< ehci
->periodic_size
; frame
+= period
) {
304 unsigned short tt_usecs
[8];
306 periodic_tt_usecs (ehci
, dev
, frame
, tt_usecs
);
308 ehci_vdbg(ehci
, "tt frame %d check %d usecs start uframe %d in"
309 " schedule %d/%d/%d/%d/%d/%d/%d/%d\n",
310 frame
, usecs
, uframe
,
311 tt_usecs
[0], tt_usecs
[1], tt_usecs
[2], tt_usecs
[3],
312 tt_usecs
[4], tt_usecs
[5], tt_usecs
[6], tt_usecs
[7]);
314 if (max_tt_usecs
[uframe
] <= tt_usecs
[uframe
]) {
315 ehci_vdbg(ehci
, "frame %d uframe %d fully scheduled\n",
320 /* special case for isoc transfers larger than 125us:
321 * the first and each subsequent fully used uframe
322 * must be empty, so as to not illegally delay
323 * already scheduled transactions
326 int ufs
= (usecs
/ 125) - 1;
328 for (i
= uframe
; i
< (uframe
+ ufs
) && i
< 8; i
++)
329 if (0 < tt_usecs
[i
]) {
331 "multi-uframe xfer can't fit "
332 "in frame %d uframe %d\n",
338 tt_usecs
[uframe
] += usecs
;
340 carryover_tt_bandwidth(tt_usecs
);
342 /* fail if the carryover pushed bw past the last uframe's limit */
343 if (max_tt_usecs
[7] < tt_usecs
[7]) {
345 "tt unavailable usecs %d frame %d uframe %d\n",
346 usecs
, frame
, uframe
);
356 /* return true iff the device's transaction translator is available
357 * for a periodic transfer starting at the specified frame, using
358 * all the uframes in the mask.
360 static int tt_no_collision (
361 struct ehci_hcd
*ehci
,
363 struct usb_device
*dev
,
368 if (period
== 0) /* error */
371 /* note bandwidth wastage: split never follows csplit
372 * (different dev or endpoint) until the next uframe.
373 * calling convention doesn't make that distinction.
375 for (; frame
< ehci
->periodic_size
; frame
+= period
) {
376 union ehci_shadow here
;
379 here
= ehci
->pshadow
[frame
];
380 type
= Q_NEXT_TYPE(ehci
, ehci
->periodic
[frame
]);
382 switch (hc32_to_cpu(ehci
, type
)) {
384 type
= Q_NEXT_TYPE(ehci
, here
.itd
->hw_next
);
385 here
= here
.itd
->itd_next
;
388 if (same_tt (dev
, here
.qh
->dev
)) {
391 mask
= hc32_to_cpu(ehci
,
393 /* "knows" no gap is needed */
398 type
= Q_NEXT_TYPE(ehci
, here
.qh
->hw_next
);
399 here
= here
.qh
->qh_next
;
402 if (same_tt (dev
, here
.sitd
->urb
->dev
)) {
405 mask
= hc32_to_cpu(ehci
, here
.sitd
407 /* FIXME assumes no gap for IN! */
412 type
= Q_NEXT_TYPE(ehci
, here
.sitd
->hw_next
);
413 here
= here
.sitd
->sitd_next
;
418 "periodic frame %d bogus type %d\n",
422 /* collision or error */
431 #endif /* CONFIG_USB_EHCI_TT_NEWSCHED */
433 /*-------------------------------------------------------------------------*/
435 static int enable_periodic (struct ehci_hcd
*ehci
)
440 if (ehci
->periodic_sched
++)
443 /* did clearing PSE did take effect yet?
444 * takes effect only at frame boundaries...
446 status
= handshake_on_error_set_halt(ehci
, &ehci
->regs
->status
,
447 STS_PSS
, 0, 9 * 125);
451 cmd
= ehci_readl(ehci
, &ehci
->regs
->command
) | CMD_PSE
;
452 ehci_writel(ehci
, cmd
, &ehci
->regs
->command
);
453 /* posted write ... PSS happens later */
454 ehci_to_hcd(ehci
)->state
= HC_STATE_RUNNING
;
456 /* make sure ehci_work scans these */
457 ehci
->next_uframe
= ehci_readl(ehci
, &ehci
->regs
->frame_index
)
458 % (ehci
->periodic_size
<< 3);
462 static int disable_periodic (struct ehci_hcd
*ehci
)
467 if (--ehci
->periodic_sched
)
470 /* did setting PSE not take effect yet?
471 * takes effect only at frame boundaries...
473 status
= handshake_on_error_set_halt(ehci
, &ehci
->regs
->status
,
474 STS_PSS
, STS_PSS
, 9 * 125);
478 cmd
= ehci_readl(ehci
, &ehci
->regs
->command
) & ~CMD_PSE
;
479 ehci_writel(ehci
, cmd
, &ehci
->regs
->command
);
480 /* posted write ... */
482 ehci
->next_uframe
= -1;
486 /*-------------------------------------------------------------------------*/
488 /* periodic schedule slots have iso tds (normal or split) first, then a
489 * sparse tree for active interrupt transfers.
491 * this just links in a qh; caller guarantees uframe masks are set right.
492 * no FSTN support (yet; ehci 0.96+)
494 static int qh_link_periodic (struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
497 unsigned period
= qh
->period
;
499 dev_dbg (&qh
->dev
->dev
,
500 "link qh%d-%04x/%p start %d [%d/%d us]\n",
501 period
, hc32_to_cpup(ehci
, &qh
->hw_info2
) & (QH_CMASK
| QH_SMASK
),
502 qh
, qh
->start
, qh
->usecs
, qh
->c_usecs
);
504 /* high bandwidth, or otherwise every microframe */
508 for (i
= qh
->start
; i
< ehci
->periodic_size
; i
+= period
) {
509 union ehci_shadow
*prev
= &ehci
->pshadow
[i
];
510 __hc32
*hw_p
= &ehci
->periodic
[i
];
511 union ehci_shadow here
= *prev
;
514 /* skip the iso nodes at list head */
516 type
= Q_NEXT_TYPE(ehci
, *hw_p
);
517 if (type
== cpu_to_hc32(ehci
, Q_TYPE_QH
))
519 prev
= periodic_next_shadow(ehci
, prev
, type
);
520 hw_p
= &here
.qh
->hw_next
;
524 /* sorting each branch by period (slow-->fast)
525 * enables sharing interior tree nodes
527 while (here
.ptr
&& qh
!= here
.qh
) {
528 if (qh
->period
> here
.qh
->period
)
530 prev
= &here
.qh
->qh_next
;
531 hw_p
= &here
.qh
->hw_next
;
534 /* link in this qh, unless some earlier pass did that */
541 *hw_p
= QH_NEXT (ehci
, qh
->qh_dma
);
544 qh
->qh_state
= QH_STATE_LINKED
;
547 /* update per-qh bandwidth for usbfs */
548 ehci_to_hcd(ehci
)->self
.bandwidth_allocated
+= qh
->period
549 ? ((qh
->usecs
+ qh
->c_usecs
) / qh
->period
)
552 /* maybe enable periodic schedule processing */
553 return enable_periodic(ehci
);
556 static int qh_unlink_periodic(struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
562 // IF this isn't high speed
563 // and this qh is active in the current uframe
564 // (and overlay token SplitXstate is false?)
566 // qh->hw_info1 |= __constant_cpu_to_hc32(1 << 7 /* "ignore" */);
568 /* high bandwidth, or otherwise part of every microframe */
569 if ((period
= qh
->period
) == 0)
572 for (i
= qh
->start
; i
< ehci
->periodic_size
; i
+= period
)
573 periodic_unlink (ehci
, i
, qh
);
575 /* update per-qh bandwidth for usbfs */
576 ehci_to_hcd(ehci
)->self
.bandwidth_allocated
-= qh
->period
577 ? ((qh
->usecs
+ qh
->c_usecs
) / qh
->period
)
580 dev_dbg (&qh
->dev
->dev
,
581 "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
583 hc32_to_cpup(ehci
, &qh
->hw_info2
) & (QH_CMASK
| QH_SMASK
),
584 qh
, qh
->start
, qh
->usecs
, qh
->c_usecs
);
586 /* qh->qh_next still "live" to HC */
587 qh
->qh_state
= QH_STATE_UNLINK
;
588 qh
->qh_next
.ptr
= NULL
;
591 /* maybe turn off periodic schedule */
592 return disable_periodic(ehci
);
595 static void intr_deschedule (struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
599 qh_unlink_periodic (ehci
, qh
);
601 /* simple/paranoid: always delay, expecting the HC needs to read
602 * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and
603 * expect khubd to clean up after any CSPLITs we won't issue.
604 * active high speed queues may need bigger delays...
606 if (list_empty (&qh
->qtd_list
)
607 || (cpu_to_hc32(ehci
, QH_CMASK
)
608 & qh
->hw_info2
) != 0)
611 wait
= 55; /* worst case: 3 * 1024 */
614 qh
->qh_state
= QH_STATE_IDLE
;
615 qh
->hw_next
= EHCI_LIST_END(ehci
);
619 /*-------------------------------------------------------------------------*/
621 static int check_period (
622 struct ehci_hcd
*ehci
,
630 /* complete split running into next frame?
631 * given FSTN support, we could sometimes check...
637 * 80% periodic == 100 usec/uframe available
638 * convert "usecs we need" to "max already claimed"
642 /* we "know" 2 and 4 uframe intervals were rejected; so
643 * for period 0, check _every_ microframe in the schedule.
645 if (unlikely (period
== 0)) {
647 for (uframe
= 0; uframe
< 7; uframe
++) {
648 claimed
= periodic_usecs (ehci
, frame
, uframe
);
652 } while ((frame
+= 1) < ehci
->periodic_size
);
654 /* just check the specified uframe, at that period */
657 claimed
= periodic_usecs (ehci
, frame
, uframe
);
660 } while ((frame
+= period
) < ehci
->periodic_size
);
667 static int check_intr_schedule (
668 struct ehci_hcd
*ehci
,
671 const struct ehci_qh
*qh
,
675 int retval
= -ENOSPC
;
678 if (qh
->c_usecs
&& uframe
>= 6) /* FSTN territory? */
681 if (!check_period (ehci
, frame
, uframe
, qh
->period
, qh
->usecs
))
689 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
690 if (tt_available (ehci
, qh
->period
, qh
->dev
, frame
, uframe
,
694 /* TODO : this may need FSTN for SSPLIT in uframe 5. */
695 for (i
=uframe
+1; i
<8 && i
<uframe
+4; i
++)
696 if (!check_period (ehci
, frame
, i
,
697 qh
->period
, qh
->c_usecs
))
704 *c_maskp
= cpu_to_hc32(ehci
, mask
<< 8);
707 /* Make sure this tt's buffer is also available for CSPLITs.
708 * We pessimize a bit; probably the typical full speed case
709 * doesn't need the second CSPLIT.
711 * NOTE: both SPLIT and CSPLIT could be checked in just
714 mask
= 0x03 << (uframe
+ qh
->gap_uf
);
715 *c_maskp
= cpu_to_hc32(ehci
, mask
<< 8);
718 if (tt_no_collision (ehci
, qh
->period
, qh
->dev
, frame
, mask
)) {
719 if (!check_period (ehci
, frame
, uframe
+ qh
->gap_uf
+ 1,
720 qh
->period
, qh
->c_usecs
))
722 if (!check_period (ehci
, frame
, uframe
+ qh
->gap_uf
,
723 qh
->period
, qh
->c_usecs
))
732 /* "first fit" scheduling policy used the first time through,
733 * or when the previous schedule slot can't be re-used.
735 static int qh_schedule(struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
740 unsigned frame
; /* 0..(qh->period - 1), or NO_FRAME */
742 qh_refresh(ehci
, qh
);
743 qh
->hw_next
= EHCI_LIST_END(ehci
);
746 /* reuse the previous schedule slots, if we can */
747 if (frame
< qh
->period
) {
748 uframe
= ffs(hc32_to_cpup(ehci
, &qh
->hw_info2
) & QH_SMASK
);
749 status
= check_intr_schedule (ehci
, frame
, --uframe
,
757 /* else scan the schedule to find a group of slots such that all
758 * uframes have enough periodic bandwidth available.
761 /* "normal" case, uframing flexible except with splits */
763 frame
= qh
->period
- 1;
765 for (uframe
= 0; uframe
< 8; uframe
++) {
766 status
= check_intr_schedule (ehci
,
772 } while (status
&& frame
--);
774 /* qh->period == 0 means every uframe */
777 status
= check_intr_schedule (ehci
, 0, 0, qh
, &c_mask
);
783 /* reset S-frame and (maybe) C-frame masks */
784 qh
->hw_info2
&= cpu_to_hc32(ehci
, ~(QH_CMASK
| QH_SMASK
));
785 qh
->hw_info2
|= qh
->period
786 ? cpu_to_hc32(ehci
, 1 << uframe
)
787 : cpu_to_hc32(ehci
, QH_SMASK
);
788 qh
->hw_info2
|= c_mask
;
790 ehci_dbg (ehci
, "reused qh %p schedule\n", qh
);
792 /* stuff into the periodic schedule */
793 status
= qh_link_periodic (ehci
, qh
);
798 static int intr_submit (
799 struct ehci_hcd
*ehci
,
801 struct list_head
*qtd_list
,
808 struct list_head empty
;
810 /* get endpoint and transfer/schedule data */
811 epnum
= urb
->ep
->desc
.bEndpointAddress
;
813 spin_lock_irqsave (&ehci
->lock
, flags
);
815 if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE
,
816 &ehci_to_hcd(ehci
)->flags
))) {
818 goto done_not_linked
;
820 status
= usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci
), urb
);
821 if (unlikely(status
))
822 goto done_not_linked
;
824 /* get qh and force any scheduling errors */
825 INIT_LIST_HEAD (&empty
);
826 qh
= qh_append_tds(ehci
, urb
, &empty
, epnum
, &urb
->ep
->hcpriv
);
831 if (qh
->qh_state
== QH_STATE_IDLE
) {
832 if ((status
= qh_schedule (ehci
, qh
)) != 0)
836 /* then queue the urb's tds to the qh */
837 qh
= qh_append_tds(ehci
, urb
, qtd_list
, epnum
, &urb
->ep
->hcpriv
);
840 /* ... update usbfs periodic stats */
841 ehci_to_hcd(ehci
)->self
.bandwidth_int_reqs
++;
844 if (unlikely(status
))
845 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci
), urb
);
847 spin_unlock_irqrestore (&ehci
->lock
, flags
);
849 qtd_list_free (ehci
, urb
, qtd_list
);
854 /*-------------------------------------------------------------------------*/
856 /* ehci_iso_stream ops work with both ITD and SITD */
858 static struct ehci_iso_stream
*
859 iso_stream_alloc (gfp_t mem_flags
)
861 struct ehci_iso_stream
*stream
;
863 stream
= kzalloc(sizeof *stream
, mem_flags
);
864 if (likely (stream
!= NULL
)) {
865 INIT_LIST_HEAD(&stream
->td_list
);
866 INIT_LIST_HEAD(&stream
->free_list
);
867 stream
->next_uframe
= -1;
868 stream
->refcount
= 1;
875 struct ehci_hcd
*ehci
,
876 struct ehci_iso_stream
*stream
,
877 struct usb_device
*dev
,
882 static const u8 smask_out
[] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
885 unsigned epnum
, maxp
;
890 * this might be a "high bandwidth" highspeed endpoint,
891 * as encoded in the ep descriptor's wMaxPacket field
893 epnum
= usb_pipeendpoint (pipe
);
894 is_input
= usb_pipein (pipe
) ? USB_DIR_IN
: 0;
895 maxp
= usb_maxpacket(dev
, pipe
, !is_input
);
902 /* knows about ITD vs SITD */
903 if (dev
->speed
== USB_SPEED_HIGH
) {
904 unsigned multi
= hb_mult(maxp
);
906 stream
->highspeed
= 1;
908 maxp
= max_packet(maxp
);
912 stream
->buf0
= cpu_to_hc32(ehci
, (epnum
<< 8) | dev
->devnum
);
913 stream
->buf1
= cpu_to_hc32(ehci
, buf1
);
914 stream
->buf2
= cpu_to_hc32(ehci
, multi
);
916 /* usbfs wants to report the average usecs per frame tied up
917 * when transfers on this endpoint are scheduled ...
919 stream
->usecs
= HS_USECS_ISO (maxp
);
920 bandwidth
= stream
->usecs
* 8;
921 bandwidth
/= interval
;
928 addr
= dev
->ttport
<< 24;
929 if (!ehci_is_TDI(ehci
)
931 ehci_to_hcd(ehci
)->self
.root_hub
))
932 addr
|= dev
->tt
->hub
->devnum
<< 16;
935 stream
->usecs
= HS_USECS_ISO (maxp
);
936 think_time
= dev
->tt
? dev
->tt
->think_time
: 0;
937 stream
->tt_usecs
= NS_TO_US (think_time
+ usb_calc_bus_time (
938 dev
->speed
, is_input
, 1, maxp
));
939 hs_transfers
= max (1u, (maxp
+ 187) / 188);
944 stream
->c_usecs
= stream
->usecs
;
945 stream
->usecs
= HS_USECS_ISO (1);
946 stream
->raw_mask
= 1;
948 /* c-mask as specified in USB 2.0 11.18.4 3.c */
949 tmp
= (1 << (hs_transfers
+ 2)) - 1;
950 stream
->raw_mask
|= tmp
<< (8 + 2);
952 stream
->raw_mask
= smask_out
[hs_transfers
- 1];
953 bandwidth
= stream
->usecs
+ stream
->c_usecs
;
954 bandwidth
/= interval
<< 3;
956 /* stream->splits gets created from raw_mask later */
957 stream
->address
= cpu_to_hc32(ehci
, addr
);
959 stream
->bandwidth
= bandwidth
;
963 stream
->bEndpointAddress
= is_input
| epnum
;
964 stream
->interval
= interval
;
969 iso_stream_put(struct ehci_hcd
*ehci
, struct ehci_iso_stream
*stream
)
973 /* free whenever just a dev->ep reference remains.
974 * not like a QH -- no persistent state (toggle, halt)
976 if (stream
->refcount
== 1) {
979 // BUG_ON (!list_empty(&stream->td_list));
981 while (!list_empty (&stream
->free_list
)) {
982 struct list_head
*entry
;
984 entry
= stream
->free_list
.next
;
987 /* knows about ITD vs SITD */
988 if (stream
->highspeed
) {
989 struct ehci_itd
*itd
;
991 itd
= list_entry (entry
, struct ehci_itd
,
993 dma_pool_free (ehci
->itd_pool
, itd
,
996 struct ehci_sitd
*sitd
;
998 sitd
= list_entry (entry
, struct ehci_sitd
,
1000 dma_pool_free (ehci
->sitd_pool
, sitd
,
1005 is_in
= (stream
->bEndpointAddress
& USB_DIR_IN
) ? 0x10 : 0;
1006 stream
->bEndpointAddress
&= 0x0f;
1008 stream
->ep
->hcpriv
= NULL
;
1010 if (stream
->rescheduled
) {
1011 ehci_info (ehci
, "ep%d%s-iso rescheduled "
1012 "%lu times in %lu seconds\n",
1013 stream
->bEndpointAddress
, is_in
? "in" : "out",
1014 stream
->rescheduled
,
1015 ((jiffies
- stream
->start
)/HZ
)
1023 static inline struct ehci_iso_stream
*
1024 iso_stream_get (struct ehci_iso_stream
*stream
)
1026 if (likely (stream
!= NULL
))
1031 static struct ehci_iso_stream
*
1032 iso_stream_find (struct ehci_hcd
*ehci
, struct urb
*urb
)
1035 struct ehci_iso_stream
*stream
;
1036 struct usb_host_endpoint
*ep
;
1037 unsigned long flags
;
1039 epnum
= usb_pipeendpoint (urb
->pipe
);
1040 if (usb_pipein(urb
->pipe
))
1041 ep
= urb
->dev
->ep_in
[epnum
];
1043 ep
= urb
->dev
->ep_out
[epnum
];
1045 spin_lock_irqsave (&ehci
->lock
, flags
);
1046 stream
= ep
->hcpriv
;
1048 if (unlikely (stream
== NULL
)) {
1049 stream
= iso_stream_alloc(GFP_ATOMIC
);
1050 if (likely (stream
!= NULL
)) {
1051 /* dev->ep owns the initial refcount */
1052 ep
->hcpriv
= stream
;
1054 iso_stream_init(ehci
, stream
, urb
->dev
, urb
->pipe
,
1058 /* if dev->ep [epnum] is a QH, info1.maxpacket is nonzero */
1059 } else if (unlikely (stream
->hw_info1
!= 0)) {
1060 ehci_dbg (ehci
, "dev %s ep%d%s, not iso??\n",
1061 urb
->dev
->devpath
, epnum
,
1062 usb_pipein(urb
->pipe
) ? "in" : "out");
1066 /* caller guarantees an eventual matching iso_stream_put */
1067 stream
= iso_stream_get (stream
);
1069 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1073 /*-------------------------------------------------------------------------*/
1075 /* ehci_iso_sched ops can be ITD-only or SITD-only */
1077 static struct ehci_iso_sched
*
1078 iso_sched_alloc (unsigned packets
, gfp_t mem_flags
)
1080 struct ehci_iso_sched
*iso_sched
;
1081 int size
= sizeof *iso_sched
;
1083 size
+= packets
* sizeof (struct ehci_iso_packet
);
1084 iso_sched
= kzalloc(size
, mem_flags
);
1085 if (likely (iso_sched
!= NULL
)) {
1086 INIT_LIST_HEAD (&iso_sched
->td_list
);
1093 struct ehci_hcd
*ehci
,
1094 struct ehci_iso_sched
*iso_sched
,
1095 struct ehci_iso_stream
*stream
,
1100 dma_addr_t dma
= urb
->transfer_dma
;
1102 /* how many uframes are needed for these transfers */
1103 iso_sched
->span
= urb
->number_of_packets
* stream
->interval
;
1105 /* figure out per-uframe itd fields that we'll need later
1106 * when we fit new itds into the schedule.
1108 for (i
= 0; i
< urb
->number_of_packets
; i
++) {
1109 struct ehci_iso_packet
*uframe
= &iso_sched
->packet
[i
];
1114 length
= urb
->iso_frame_desc
[i
].length
;
1115 buf
= dma
+ urb
->iso_frame_desc
[i
].offset
;
1117 trans
= EHCI_ISOC_ACTIVE
;
1118 trans
|= buf
& 0x0fff;
1119 if (unlikely (((i
+ 1) == urb
->number_of_packets
))
1120 && !(urb
->transfer_flags
& URB_NO_INTERRUPT
))
1121 trans
|= EHCI_ITD_IOC
;
1122 trans
|= length
<< 16;
1123 uframe
->transaction
= cpu_to_hc32(ehci
, trans
);
1125 /* might need to cross a buffer page within a uframe */
1126 uframe
->bufp
= (buf
& ~(u64
)0x0fff);
1128 if (unlikely ((uframe
->bufp
!= (buf
& ~(u64
)0x0fff))))
1135 struct ehci_iso_stream
*stream
,
1136 struct ehci_iso_sched
*iso_sched
1141 // caller must hold ehci->lock!
1142 list_splice (&iso_sched
->td_list
, &stream
->free_list
);
1147 itd_urb_transaction (
1148 struct ehci_iso_stream
*stream
,
1149 struct ehci_hcd
*ehci
,
1154 struct ehci_itd
*itd
;
1158 struct ehci_iso_sched
*sched
;
1159 unsigned long flags
;
1161 sched
= iso_sched_alloc (urb
->number_of_packets
, mem_flags
);
1162 if (unlikely (sched
== NULL
))
1165 itd_sched_init(ehci
, sched
, stream
, urb
);
1167 if (urb
->interval
< 8)
1168 num_itds
= 1 + (sched
->span
+ 7) / 8;
1170 num_itds
= urb
->number_of_packets
;
1172 /* allocate/init ITDs */
1173 spin_lock_irqsave (&ehci
->lock
, flags
);
1174 for (i
= 0; i
< num_itds
; i
++) {
1176 /* free_list.next might be cache-hot ... but maybe
1177 * the HC caches it too. avoid that issue for now.
1180 /* prefer previously-allocated itds */
1181 if (likely (!list_empty(&stream
->free_list
))) {
1182 itd
= list_entry (stream
->free_list
.prev
,
1183 struct ehci_itd
, itd_list
);
1184 list_del (&itd
->itd_list
);
1185 itd_dma
= itd
->itd_dma
;
1187 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1188 itd
= dma_pool_alloc (ehci
->itd_pool
, mem_flags
,
1190 spin_lock_irqsave (&ehci
->lock
, flags
);
1192 iso_sched_free(stream
, sched
);
1193 spin_unlock_irqrestore(&ehci
->lock
, flags
);
1198 memset (itd
, 0, sizeof *itd
);
1199 itd
->itd_dma
= itd_dma
;
1200 list_add (&itd
->itd_list
, &sched
->td_list
);
1202 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1204 /* temporarily store schedule info in hcpriv */
1205 urb
->hcpriv
= sched
;
1206 urb
->error_count
= 0;
1210 /*-------------------------------------------------------------------------*/
1214 struct ehci_hcd
*ehci
,
1223 /* can't commit more than 80% periodic == 100 usec */
1224 if (periodic_usecs (ehci
, uframe
>> 3, uframe
& 0x7)
1228 /* we know urb->interval is 2^N uframes */
1230 } while (uframe
< mod
);
1236 struct ehci_hcd
*ehci
,
1238 struct ehci_iso_stream
*stream
,
1240 struct ehci_iso_sched
*sched
,
1247 mask
= stream
->raw_mask
<< (uframe
& 7);
1249 /* for IN, don't wrap CSPLIT into the next frame */
1253 /* this multi-pass logic is simple, but performance may
1254 * suffer when the schedule data isn't cached.
1257 /* check bandwidth */
1258 uframe
%= period_uframes
;
1262 frame
= uframe
>> 3;
1265 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
1266 /* The tt's fullspeed bus bandwidth must be available.
1267 * tt_available scheduling guarantees 10+% for control/bulk.
1269 if (!tt_available (ehci
, period_uframes
<< 3,
1270 stream
->udev
, frame
, uf
, stream
->tt_usecs
))
1273 /* tt must be idle for start(s), any gap, and csplit.
1274 * assume scheduling slop leaves 10+% for control/bulk.
1276 if (!tt_no_collision (ehci
, period_uframes
<< 3,
1277 stream
->udev
, frame
, mask
))
1281 /* check starts (OUT uses more than one) */
1282 max_used
= 100 - stream
->usecs
;
1283 for (tmp
= stream
->raw_mask
& 0xff; tmp
; tmp
>>= 1, uf
++) {
1284 if (periodic_usecs (ehci
, frame
, uf
) > max_used
)
1288 /* for IN, check CSPLIT */
1289 if (stream
->c_usecs
) {
1291 max_used
= 100 - stream
->c_usecs
;
1295 if ((stream
->raw_mask
& tmp
) == 0)
1297 if (periodic_usecs (ehci
, frame
, uf
)
1303 /* we know urb->interval is 2^N uframes */
1304 uframe
+= period_uframes
;
1305 } while (uframe
< mod
);
1307 stream
->splits
= cpu_to_hc32(ehci
, stream
->raw_mask
<< (uframe
& 7));
1312 * This scheduler plans almost as far into the future as it has actual
1313 * periodic schedule slots. (Affected by TUNE_FLS, which defaults to
1314 * "as small as possible" to be cache-friendlier.) That limits the size
1315 * transfers you can stream reliably; avoid more than 64 msec per urb.
1316 * Also avoid queue depths of less than ehci's worst irq latency (affected
1317 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
1318 * and other factors); or more than about 230 msec total (for portability,
1319 * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler!
1322 #define SCHEDULE_SLOP 10 /* frames */
1325 iso_stream_schedule (
1326 struct ehci_hcd
*ehci
,
1328 struct ehci_iso_stream
*stream
1331 u32 now
, start
, max
, period
;
1333 unsigned mod
= ehci
->periodic_size
<< 3;
1334 struct ehci_iso_sched
*sched
= urb
->hcpriv
;
1336 if (sched
->span
> (mod
- 8 * SCHEDULE_SLOP
)) {
1337 ehci_dbg (ehci
, "iso request %p too long\n", urb
);
1342 if ((stream
->depth
+ sched
->span
) > mod
) {
1343 ehci_dbg (ehci
, "request %p would overflow (%d+%d>%d)\n",
1344 urb
, stream
->depth
, sched
->span
, mod
);
1349 now
= ehci_readl(ehci
, &ehci
->regs
->frame_index
) % mod
;
1351 /* when's the last uframe this urb could start? */
1354 /* Typical case: reuse current schedule, stream is still active.
1355 * Hopefully there are no gaps from the host falling behind
1356 * (irq delays etc), but if there are we'll take the next
1357 * slot in the schedule, implicitly assuming URB_ISO_ASAP.
1359 if (likely (!list_empty (&stream
->td_list
))) {
1360 start
= stream
->next_uframe
;
1364 /* Fell behind (by up to twice the slop amount)? */
1365 if (start
>= max
- 2 * 8 * SCHEDULE_SLOP
)
1366 start
+= stream
->interval
* DIV_ROUND_UP(
1367 max
- start
, stream
->interval
) - mod
;
1369 /* Tried to schedule too far into the future? */
1370 if (unlikely((start
+ sched
->span
) >= max
)) {
1377 /* need to schedule; when's the next (u)frame we could start?
1378 * this is bigger than ehci->i_thresh allows; scheduling itself
1379 * isn't free, the slop should handle reasonably slow cpus. it
1380 * can also help high bandwidth if the dma and irq loads don't
1381 * jump until after the queue is primed.
1383 start
= SCHEDULE_SLOP
* 8 + (now
& ~0x07);
1385 stream
->next_uframe
= start
;
1387 /* NOTE: assumes URB_ISO_ASAP, to limit complexity/bugs */
1389 period
= urb
->interval
;
1390 if (!stream
->highspeed
)
1393 /* find a uframe slot with enough bandwidth */
1394 for (; start
< (stream
->next_uframe
+ period
); start
++) {
1397 /* check schedule: enough space? */
1398 if (stream
->highspeed
)
1399 enough_space
= itd_slot_ok (ehci
, mod
, start
,
1400 stream
->usecs
, period
);
1402 if ((start
% 8) >= 6)
1404 enough_space
= sitd_slot_ok (ehci
, mod
, stream
,
1405 start
, sched
, period
);
1408 /* schedule it here if there's enough bandwidth */
1410 stream
->next_uframe
= start
% mod
;
1415 /* no room in the schedule */
1416 ehci_dbg (ehci
, "iso %ssched full %p (now %d max %d)\n",
1417 list_empty (&stream
->td_list
) ? "" : "re",
1422 iso_sched_free (stream
, sched
);
1427 /* report high speed start in uframes; full speed, in frames */
1428 urb
->start_frame
= stream
->next_uframe
;
1429 if (!stream
->highspeed
)
1430 urb
->start_frame
>>= 3;
1434 /*-------------------------------------------------------------------------*/
1437 itd_init(struct ehci_hcd
*ehci
, struct ehci_iso_stream
*stream
,
1438 struct ehci_itd
*itd
)
1442 /* it's been recently zeroed */
1443 itd
->hw_next
= EHCI_LIST_END(ehci
);
1444 itd
->hw_bufp
[0] = stream
->buf0
;
1445 itd
->hw_bufp
[1] = stream
->buf1
;
1446 itd
->hw_bufp
[2] = stream
->buf2
;
1448 for (i
= 0; i
< 8; i
++)
1451 /* All other fields are filled when scheduling */
1456 struct ehci_hcd
*ehci
,
1457 struct ehci_itd
*itd
,
1458 struct ehci_iso_sched
*iso_sched
,
1463 struct ehci_iso_packet
*uf
= &iso_sched
->packet
[index
];
1464 unsigned pg
= itd
->pg
;
1466 // BUG_ON (pg == 6 && uf->cross);
1469 itd
->index
[uframe
] = index
;
1471 itd
->hw_transaction
[uframe
] = uf
->transaction
;
1472 itd
->hw_transaction
[uframe
] |= cpu_to_hc32(ehci
, pg
<< 12);
1473 itd
->hw_bufp
[pg
] |= cpu_to_hc32(ehci
, uf
->bufp
& ~(u32
)0);
1474 itd
->hw_bufp_hi
[pg
] |= cpu_to_hc32(ehci
, (u32
)(uf
->bufp
>> 32));
1476 /* iso_frame_desc[].offset must be strictly increasing */
1477 if (unlikely (uf
->cross
)) {
1478 u64 bufp
= uf
->bufp
+ 4096;
1481 itd
->hw_bufp
[pg
] |= cpu_to_hc32(ehci
, bufp
& ~(u32
)0);
1482 itd
->hw_bufp_hi
[pg
] |= cpu_to_hc32(ehci
, (u32
)(bufp
>> 32));
1487 itd_link (struct ehci_hcd
*ehci
, unsigned frame
, struct ehci_itd
*itd
)
1489 /* always prepend ITD/SITD ... only QH tree is order-sensitive */
1490 itd
->itd_next
= ehci
->pshadow
[frame
];
1491 itd
->hw_next
= ehci
->periodic
[frame
];
1492 ehci
->pshadow
[frame
].itd
= itd
;
1495 ehci
->periodic
[frame
] = cpu_to_hc32(ehci
, itd
->itd_dma
| Q_TYPE_ITD
);
1498 /* fit urb's itds into the selected schedule slot; activate as needed */
1501 struct ehci_hcd
*ehci
,
1504 struct ehci_iso_stream
*stream
1508 unsigned next_uframe
, uframe
, frame
;
1509 struct ehci_iso_sched
*iso_sched
= urb
->hcpriv
;
1510 struct ehci_itd
*itd
;
1512 next_uframe
= stream
->next_uframe
% mod
;
1514 if (unlikely (list_empty(&stream
->td_list
))) {
1515 ehci_to_hcd(ehci
)->self
.bandwidth_allocated
1516 += stream
->bandwidth
;
1518 "schedule devp %s ep%d%s-iso period %d start %d.%d\n",
1519 urb
->dev
->devpath
, stream
->bEndpointAddress
& 0x0f,
1520 (stream
->bEndpointAddress
& USB_DIR_IN
) ? "in" : "out",
1522 next_uframe
>> 3, next_uframe
& 0x7);
1523 stream
->start
= jiffies
;
1525 ehci_to_hcd(ehci
)->self
.bandwidth_isoc_reqs
++;
1527 /* fill iTDs uframe by uframe */
1528 for (packet
= 0, itd
= NULL
; packet
< urb
->number_of_packets
; ) {
1530 /* ASSERT: we have all necessary itds */
1531 // BUG_ON (list_empty (&iso_sched->td_list));
1533 /* ASSERT: no itds for this endpoint in this uframe */
1535 itd
= list_entry (iso_sched
->td_list
.next
,
1536 struct ehci_itd
, itd_list
);
1537 list_move_tail (&itd
->itd_list
, &stream
->td_list
);
1538 itd
->stream
= iso_stream_get (stream
);
1540 itd_init (ehci
, stream
, itd
);
1543 uframe
= next_uframe
& 0x07;
1544 frame
= next_uframe
>> 3;
1546 itd_patch(ehci
, itd
, iso_sched
, packet
, uframe
);
1548 next_uframe
+= stream
->interval
;
1549 stream
->depth
+= stream
->interval
;
1553 /* link completed itds into the schedule */
1554 if (((next_uframe
>> 3) != frame
)
1555 || packet
== urb
->number_of_packets
) {
1556 itd_link (ehci
, frame
% ehci
->periodic_size
, itd
);
1560 stream
->next_uframe
= next_uframe
;
1562 /* don't need that schedule data any more */
1563 iso_sched_free (stream
, iso_sched
);
1566 timer_action (ehci
, TIMER_IO_WATCHDOG
);
1567 return enable_periodic(ehci
);
1570 #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1572 /* Process and recycle a completed ITD. Return true iff its urb completed,
1573 * and hence its completion callback probably added things to the hardware
1576 * Note that we carefully avoid recycling this descriptor until after any
1577 * completion callback runs, so that it won't be reused quickly. That is,
1578 * assuming (a) no more than two urbs per frame on this endpoint, and also
1579 * (b) only this endpoint's completions submit URBs. It seems some silicon
1580 * corrupts things if you reuse completed descriptors very quickly...
1584 struct ehci_hcd
*ehci
,
1585 struct ehci_itd
*itd
1587 struct urb
*urb
= itd
->urb
;
1588 struct usb_iso_packet_descriptor
*desc
;
1592 struct ehci_iso_stream
*stream
= itd
->stream
;
1593 struct usb_device
*dev
;
1594 unsigned retval
= false;
1596 /* for each uframe with a packet */
1597 for (uframe
= 0; uframe
< 8; uframe
++) {
1598 if (likely (itd
->index
[uframe
] == -1))
1600 urb_index
= itd
->index
[uframe
];
1601 desc
= &urb
->iso_frame_desc
[urb_index
];
1603 t
= hc32_to_cpup(ehci
, &itd
->hw_transaction
[uframe
]);
1604 itd
->hw_transaction
[uframe
] = 0;
1605 stream
->depth
-= stream
->interval
;
1607 /* report transfer status */
1608 if (unlikely (t
& ISO_ERRS
)) {
1610 if (t
& EHCI_ISOC_BUF_ERR
)
1611 desc
->status
= usb_pipein (urb
->pipe
)
1612 ? -ENOSR
/* hc couldn't read */
1613 : -ECOMM
; /* hc couldn't write */
1614 else if (t
& EHCI_ISOC_BABBLE
)
1615 desc
->status
= -EOVERFLOW
;
1616 else /* (t & EHCI_ISOC_XACTERR) */
1617 desc
->status
= -EPROTO
;
1619 /* HC need not update length with this error */
1620 if (!(t
& EHCI_ISOC_BABBLE
))
1621 desc
->actual_length
= EHCI_ITD_LENGTH (t
);
1622 } else if (likely ((t
& EHCI_ISOC_ACTIVE
) == 0)) {
1624 desc
->actual_length
= EHCI_ITD_LENGTH (t
);
1626 /* URB was too late */
1627 desc
->status
= -EXDEV
;
1631 /* handle completion now? */
1632 if (likely ((urb_index
+ 1) != urb
->number_of_packets
))
1635 /* ASSERT: it's really the last itd for this urb
1636 list_for_each_entry (itd, &stream->td_list, itd_list)
1637 BUG_ON (itd->urb == urb);
1640 /* give urb back to the driver; completion often (re)submits */
1642 ehci_urb_done(ehci
, urb
, 0);
1645 (void) disable_periodic(ehci
);
1646 ehci_to_hcd(ehci
)->self
.bandwidth_isoc_reqs
--;
1648 if (unlikely(list_is_singular(&stream
->td_list
))) {
1649 ehci_to_hcd(ehci
)->self
.bandwidth_allocated
1650 -= stream
->bandwidth
;
1652 "deschedule devp %s ep%d%s-iso\n",
1653 dev
->devpath
, stream
->bEndpointAddress
& 0x0f,
1654 (stream
->bEndpointAddress
& USB_DIR_IN
) ? "in" : "out");
1656 iso_stream_put (ehci
, stream
);
1660 if (ehci
->clock_frame
!= itd
->frame
|| itd
->index
[7] != -1) {
1661 /* OK to recycle this ITD now. */
1663 list_move(&itd
->itd_list
, &stream
->free_list
);
1664 iso_stream_put(ehci
, stream
);
1666 /* HW might remember this ITD, so we can't recycle it yet.
1667 * Move it to a safe place until a new frame starts.
1669 list_move(&itd
->itd_list
, &ehci
->cached_itd_list
);
1670 if (stream
->refcount
== 2) {
1671 /* If iso_stream_put() were called here, stream
1672 * would be freed. Instead, just prevent reuse.
1674 stream
->ep
->hcpriv
= NULL
;
1681 /*-------------------------------------------------------------------------*/
1683 static int itd_submit (struct ehci_hcd
*ehci
, struct urb
*urb
,
1686 int status
= -EINVAL
;
1687 unsigned long flags
;
1688 struct ehci_iso_stream
*stream
;
1690 /* Get iso_stream head */
1691 stream
= iso_stream_find (ehci
, urb
);
1692 if (unlikely (stream
== NULL
)) {
1693 ehci_dbg (ehci
, "can't get iso stream\n");
1696 if (unlikely (urb
->interval
!= stream
->interval
)) {
1697 ehci_dbg (ehci
, "can't change iso interval %d --> %d\n",
1698 stream
->interval
, urb
->interval
);
1702 #ifdef EHCI_URB_TRACE
1704 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1705 __func__
, urb
->dev
->devpath
, urb
,
1706 usb_pipeendpoint (urb
->pipe
),
1707 usb_pipein (urb
->pipe
) ? "in" : "out",
1708 urb
->transfer_buffer_length
,
1709 urb
->number_of_packets
, urb
->interval
,
1713 /* allocate ITDs w/o locking anything */
1714 status
= itd_urb_transaction (stream
, ehci
, urb
, mem_flags
);
1715 if (unlikely (status
< 0)) {
1716 ehci_dbg (ehci
, "can't init itds\n");
1720 /* schedule ... need to lock */
1721 spin_lock_irqsave (&ehci
->lock
, flags
);
1722 if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE
,
1723 &ehci_to_hcd(ehci
)->flags
))) {
1724 status
= -ESHUTDOWN
;
1725 goto done_not_linked
;
1727 status
= usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci
), urb
);
1728 if (unlikely(status
))
1729 goto done_not_linked
;
1730 status
= iso_stream_schedule(ehci
, urb
, stream
);
1731 if (likely (status
== 0))
1732 itd_link_urb (ehci
, urb
, ehci
->periodic_size
<< 3, stream
);
1734 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci
), urb
);
1736 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1739 if (unlikely (status
< 0))
1740 iso_stream_put (ehci
, stream
);
1744 /*-------------------------------------------------------------------------*/
1747 * "Split ISO TDs" ... used for USB 1.1 devices going through the
1748 * TTs in USB 2.0 hubs. These need microframe scheduling.
1753 struct ehci_hcd
*ehci
,
1754 struct ehci_iso_sched
*iso_sched
,
1755 struct ehci_iso_stream
*stream
,
1760 dma_addr_t dma
= urb
->transfer_dma
;
1762 /* how many frames are needed for these transfers */
1763 iso_sched
->span
= urb
->number_of_packets
* stream
->interval
;
1765 /* figure out per-frame sitd fields that we'll need later
1766 * when we fit new sitds into the schedule.
1768 for (i
= 0; i
< urb
->number_of_packets
; i
++) {
1769 struct ehci_iso_packet
*packet
= &iso_sched
->packet
[i
];
1774 length
= urb
->iso_frame_desc
[i
].length
& 0x03ff;
1775 buf
= dma
+ urb
->iso_frame_desc
[i
].offset
;
1777 trans
= SITD_STS_ACTIVE
;
1778 if (((i
+ 1) == urb
->number_of_packets
)
1779 && !(urb
->transfer_flags
& URB_NO_INTERRUPT
))
1781 trans
|= length
<< 16;
1782 packet
->transaction
= cpu_to_hc32(ehci
, trans
);
1784 /* might need to cross a buffer page within a td */
1786 packet
->buf1
= (buf
+ length
) & ~0x0fff;
1787 if (packet
->buf1
!= (buf
& ~(u64
)0x0fff))
1790 /* OUT uses multiple start-splits */
1791 if (stream
->bEndpointAddress
& USB_DIR_IN
)
1793 length
= (length
+ 187) / 188;
1794 if (length
> 1) /* BEGIN vs ALL */
1796 packet
->buf1
|= length
;
1801 sitd_urb_transaction (
1802 struct ehci_iso_stream
*stream
,
1803 struct ehci_hcd
*ehci
,
1808 struct ehci_sitd
*sitd
;
1809 dma_addr_t sitd_dma
;
1811 struct ehci_iso_sched
*iso_sched
;
1812 unsigned long flags
;
1814 iso_sched
= iso_sched_alloc (urb
->number_of_packets
, mem_flags
);
1815 if (iso_sched
== NULL
)
1818 sitd_sched_init(ehci
, iso_sched
, stream
, urb
);
1820 /* allocate/init sITDs */
1821 spin_lock_irqsave (&ehci
->lock
, flags
);
1822 for (i
= 0; i
< urb
->number_of_packets
; i
++) {
1824 /* NOTE: for now, we don't try to handle wraparound cases
1825 * for IN (using sitd->hw_backpointer, like a FSTN), which
1826 * means we never need two sitds for full speed packets.
1829 /* free_list.next might be cache-hot ... but maybe
1830 * the HC caches it too. avoid that issue for now.
1833 /* prefer previously-allocated sitds */
1834 if (!list_empty(&stream
->free_list
)) {
1835 sitd
= list_entry (stream
->free_list
.prev
,
1836 struct ehci_sitd
, sitd_list
);
1837 list_del (&sitd
->sitd_list
);
1838 sitd_dma
= sitd
->sitd_dma
;
1840 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1841 sitd
= dma_pool_alloc (ehci
->sitd_pool
, mem_flags
,
1843 spin_lock_irqsave (&ehci
->lock
, flags
);
1845 iso_sched_free(stream
, iso_sched
);
1846 spin_unlock_irqrestore(&ehci
->lock
, flags
);
1851 memset (sitd
, 0, sizeof *sitd
);
1852 sitd
->sitd_dma
= sitd_dma
;
1853 list_add (&sitd
->sitd_list
, &iso_sched
->td_list
);
1856 /* temporarily store schedule info in hcpriv */
1857 urb
->hcpriv
= iso_sched
;
1858 urb
->error_count
= 0;
1860 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1864 /*-------------------------------------------------------------------------*/
1868 struct ehci_hcd
*ehci
,
1869 struct ehci_iso_stream
*stream
,
1870 struct ehci_sitd
*sitd
,
1871 struct ehci_iso_sched
*iso_sched
,
1875 struct ehci_iso_packet
*uf
= &iso_sched
->packet
[index
];
1876 u64 bufp
= uf
->bufp
;
1878 sitd
->hw_next
= EHCI_LIST_END(ehci
);
1879 sitd
->hw_fullspeed_ep
= stream
->address
;
1880 sitd
->hw_uframe
= stream
->splits
;
1881 sitd
->hw_results
= uf
->transaction
;
1882 sitd
->hw_backpointer
= EHCI_LIST_END(ehci
);
1885 sitd
->hw_buf
[0] = cpu_to_hc32(ehci
, bufp
);
1886 sitd
->hw_buf_hi
[0] = cpu_to_hc32(ehci
, bufp
>> 32);
1888 sitd
->hw_buf
[1] = cpu_to_hc32(ehci
, uf
->buf1
);
1891 sitd
->hw_buf_hi
[1] = cpu_to_hc32(ehci
, bufp
>> 32);
1892 sitd
->index
= index
;
1896 sitd_link (struct ehci_hcd
*ehci
, unsigned frame
, struct ehci_sitd
*sitd
)
1898 /* note: sitd ordering could matter (CSPLIT then SSPLIT) */
1899 sitd
->sitd_next
= ehci
->pshadow
[frame
];
1900 sitd
->hw_next
= ehci
->periodic
[frame
];
1901 ehci
->pshadow
[frame
].sitd
= sitd
;
1902 sitd
->frame
= frame
;
1904 ehci
->periodic
[frame
] = cpu_to_hc32(ehci
, sitd
->sitd_dma
| Q_TYPE_SITD
);
1907 /* fit urb's sitds into the selected schedule slot; activate as needed */
1910 struct ehci_hcd
*ehci
,
1913 struct ehci_iso_stream
*stream
1917 unsigned next_uframe
;
1918 struct ehci_iso_sched
*sched
= urb
->hcpriv
;
1919 struct ehci_sitd
*sitd
;
1921 next_uframe
= stream
->next_uframe
;
1923 if (list_empty(&stream
->td_list
)) {
1924 /* usbfs ignores TT bandwidth */
1925 ehci_to_hcd(ehci
)->self
.bandwidth_allocated
1926 += stream
->bandwidth
;
1928 "sched devp %s ep%d%s-iso [%d] %dms/%04x\n",
1929 urb
->dev
->devpath
, stream
->bEndpointAddress
& 0x0f,
1930 (stream
->bEndpointAddress
& USB_DIR_IN
) ? "in" : "out",
1931 (next_uframe
>> 3) % ehci
->periodic_size
,
1932 stream
->interval
, hc32_to_cpu(ehci
, stream
->splits
));
1933 stream
->start
= jiffies
;
1935 ehci_to_hcd(ehci
)->self
.bandwidth_isoc_reqs
++;
1937 /* fill sITDs frame by frame */
1938 for (packet
= 0, sitd
= NULL
;
1939 packet
< urb
->number_of_packets
;
1942 /* ASSERT: we have all necessary sitds */
1943 BUG_ON (list_empty (&sched
->td_list
));
1945 /* ASSERT: no itds for this endpoint in this frame */
1947 sitd
= list_entry (sched
->td_list
.next
,
1948 struct ehci_sitd
, sitd_list
);
1949 list_move_tail (&sitd
->sitd_list
, &stream
->td_list
);
1950 sitd
->stream
= iso_stream_get (stream
);
1953 sitd_patch(ehci
, stream
, sitd
, sched
, packet
);
1954 sitd_link (ehci
, (next_uframe
>> 3) % ehci
->periodic_size
,
1957 next_uframe
+= stream
->interval
<< 3;
1958 stream
->depth
+= stream
->interval
<< 3;
1960 stream
->next_uframe
= next_uframe
% mod
;
1962 /* don't need that schedule data any more */
1963 iso_sched_free (stream
, sched
);
1966 timer_action (ehci
, TIMER_IO_WATCHDOG
);
1967 return enable_periodic(ehci
);
1970 /*-------------------------------------------------------------------------*/
1972 #define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
1973 | SITD_STS_XACT | SITD_STS_MMF)
1975 /* Process and recycle a completed SITD. Return true iff its urb completed,
1976 * and hence its completion callback probably added things to the hardware
1979 * Note that we carefully avoid recycling this descriptor until after any
1980 * completion callback runs, so that it won't be reused quickly. That is,
1981 * assuming (a) no more than two urbs per frame on this endpoint, and also
1982 * (b) only this endpoint's completions submit URBs. It seems some silicon
1983 * corrupts things if you reuse completed descriptors very quickly...
1987 struct ehci_hcd
*ehci
,
1988 struct ehci_sitd
*sitd
1990 struct urb
*urb
= sitd
->urb
;
1991 struct usb_iso_packet_descriptor
*desc
;
1994 struct ehci_iso_stream
*stream
= sitd
->stream
;
1995 struct usb_device
*dev
;
1996 unsigned retval
= false;
1998 urb_index
= sitd
->index
;
1999 desc
= &urb
->iso_frame_desc
[urb_index
];
2000 t
= hc32_to_cpup(ehci
, &sitd
->hw_results
);
2002 /* report transfer status */
2003 if (t
& SITD_ERRS
) {
2005 if (t
& SITD_STS_DBE
)
2006 desc
->status
= usb_pipein (urb
->pipe
)
2007 ? -ENOSR
/* hc couldn't read */
2008 : -ECOMM
; /* hc couldn't write */
2009 else if (t
& SITD_STS_BABBLE
)
2010 desc
->status
= -EOVERFLOW
;
2011 else /* XACT, MMF, etc */
2012 desc
->status
= -EPROTO
;
2015 desc
->actual_length
= desc
->length
- SITD_LENGTH (t
);
2017 stream
->depth
-= stream
->interval
<< 3;
2019 /* handle completion now? */
2020 if ((urb_index
+ 1) != urb
->number_of_packets
)
2023 /* ASSERT: it's really the last sitd for this urb
2024 list_for_each_entry (sitd, &stream->td_list, sitd_list)
2025 BUG_ON (sitd->urb == urb);
2028 /* give urb back to the driver; completion often (re)submits */
2030 ehci_urb_done(ehci
, urb
, 0);
2033 (void) disable_periodic(ehci
);
2034 ehci_to_hcd(ehci
)->self
.bandwidth_isoc_reqs
--;
2036 if (list_is_singular(&stream
->td_list
)) {
2037 ehci_to_hcd(ehci
)->self
.bandwidth_allocated
2038 -= stream
->bandwidth
;
2040 "deschedule devp %s ep%d%s-iso\n",
2041 dev
->devpath
, stream
->bEndpointAddress
& 0x0f,
2042 (stream
->bEndpointAddress
& USB_DIR_IN
) ? "in" : "out");
2044 iso_stream_put (ehci
, stream
);
2045 /* OK to recycle this SITD now that its completion callback ran. */
2048 sitd
->stream
= NULL
;
2049 list_move(&sitd
->sitd_list
, &stream
->free_list
);
2050 iso_stream_put(ehci
, stream
);
2056 static int sitd_submit (struct ehci_hcd
*ehci
, struct urb
*urb
,
2059 int status
= -EINVAL
;
2060 unsigned long flags
;
2061 struct ehci_iso_stream
*stream
;
2063 /* Get iso_stream head */
2064 stream
= iso_stream_find (ehci
, urb
);
2065 if (stream
== NULL
) {
2066 ehci_dbg (ehci
, "can't get iso stream\n");
2069 if (urb
->interval
!= stream
->interval
) {
2070 ehci_dbg (ehci
, "can't change iso interval %d --> %d\n",
2071 stream
->interval
, urb
->interval
);
2075 #ifdef EHCI_URB_TRACE
2077 "submit %p dev%s ep%d%s-iso len %d\n",
2078 urb
, urb
->dev
->devpath
,
2079 usb_pipeendpoint (urb
->pipe
),
2080 usb_pipein (urb
->pipe
) ? "in" : "out",
2081 urb
->transfer_buffer_length
);
2084 /* allocate SITDs */
2085 status
= sitd_urb_transaction (stream
, ehci
, urb
, mem_flags
);
2087 ehci_dbg (ehci
, "can't init sitds\n");
2091 /* schedule ... need to lock */
2092 spin_lock_irqsave (&ehci
->lock
, flags
);
2093 if (unlikely(!test_bit(HCD_FLAG_HW_ACCESSIBLE
,
2094 &ehci_to_hcd(ehci
)->flags
))) {
2095 status
= -ESHUTDOWN
;
2096 goto done_not_linked
;
2098 status
= usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci
), urb
);
2099 if (unlikely(status
))
2100 goto done_not_linked
;
2101 status
= iso_stream_schedule(ehci
, urb
, stream
);
2103 sitd_link_urb (ehci
, urb
, ehci
->periodic_size
<< 3, stream
);
2105 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci
), urb
);
2107 spin_unlock_irqrestore (&ehci
->lock
, flags
);
2111 iso_stream_put (ehci
, stream
);
2115 /*-------------------------------------------------------------------------*/
2117 static void free_cached_itd_list(struct ehci_hcd
*ehci
)
2119 struct ehci_itd
*itd
, *n
;
2121 list_for_each_entry_safe(itd
, n
, &ehci
->cached_itd_list
, itd_list
) {
2122 struct ehci_iso_stream
*stream
= itd
->stream
;
2124 list_move(&itd
->itd_list
, &stream
->free_list
);
2125 iso_stream_put(ehci
, stream
);
2129 /*-------------------------------------------------------------------------*/
2132 scan_periodic (struct ehci_hcd
*ehci
)
2134 unsigned now_uframe
, frame
, clock
, clock_frame
, mod
;
2137 mod
= ehci
->periodic_size
<< 3;
2140 * When running, scan from last scan point up to "now"
2141 * else clean up by scanning everything that's left.
2142 * Touches as few pages as possible: cache-friendly.
2144 now_uframe
= ehci
->next_uframe
;
2145 if (HC_IS_RUNNING(ehci_to_hcd(ehci
)->state
)) {
2146 clock
= ehci_readl(ehci
, &ehci
->regs
->frame_index
);
2147 clock_frame
= (clock
>> 3) % ehci
->periodic_size
;
2149 clock
= now_uframe
+ mod
- 1;
2152 if (ehci
->clock_frame
!= clock_frame
) {
2153 free_cached_itd_list(ehci
);
2154 ehci
->clock_frame
= clock_frame
;
2157 clock_frame
= clock
>> 3;
2160 union ehci_shadow q
, *q_p
;
2162 unsigned incomplete
= false;
2164 frame
= now_uframe
>> 3;
2167 /* scan each element in frame's queue for completions */
2168 q_p
= &ehci
->pshadow
[frame
];
2169 hw_p
= &ehci
->periodic
[frame
];
2171 type
= Q_NEXT_TYPE(ehci
, *hw_p
);
2174 while (q
.ptr
!= NULL
) {
2176 union ehci_shadow temp
;
2179 live
= HC_IS_RUNNING (ehci_to_hcd(ehci
)->state
);
2180 switch (hc32_to_cpu(ehci
, type
)) {
2182 /* handle any completions */
2183 temp
.qh
= qh_get (q
.qh
);
2184 type
= Q_NEXT_TYPE(ehci
, q
.qh
->hw_next
);
2186 modified
= qh_completions (ehci
, temp
.qh
);
2187 if (unlikely (list_empty (&temp
.qh
->qtd_list
)))
2188 intr_deschedule (ehci
, temp
.qh
);
2192 /* for "save place" FSTNs, look at QH entries
2193 * in the previous frame for completions.
2195 if (q
.fstn
->hw_prev
!= EHCI_LIST_END(ehci
)) {
2196 dbg ("ignoring completions from FSTNs");
2198 type
= Q_NEXT_TYPE(ehci
, q
.fstn
->hw_next
);
2199 q
= q
.fstn
->fstn_next
;
2202 /* If this ITD is still active, leave it for
2203 * later processing ... check the next entry.
2204 * No need to check for activity unless the
2207 if (frame
== clock_frame
&& live
) {
2209 for (uf
= 0; uf
< 8; uf
++) {
2210 if (q
.itd
->hw_transaction
[uf
] &
2216 q_p
= &q
.itd
->itd_next
;
2217 hw_p
= &q
.itd
->hw_next
;
2218 type
= Q_NEXT_TYPE(ehci
,
2225 /* Take finished ITDs out of the schedule
2226 * and process them: recycle, maybe report
2227 * URB completion. HC won't cache the
2228 * pointer for much longer, if at all.
2230 *q_p
= q
.itd
->itd_next
;
2231 *hw_p
= q
.itd
->hw_next
;
2232 type
= Q_NEXT_TYPE(ehci
, q
.itd
->hw_next
);
2234 modified
= itd_complete (ehci
, q
.itd
);
2238 /* If this SITD is still active, leave it for
2239 * later processing ... check the next entry.
2240 * No need to check for activity unless the
2243 if (frame
== clock_frame
&& live
&&
2244 (q
.sitd
->hw_results
&
2245 SITD_ACTIVE(ehci
))) {
2247 q_p
= &q
.sitd
->sitd_next
;
2248 hw_p
= &q
.sitd
->hw_next
;
2249 type
= Q_NEXT_TYPE(ehci
,
2255 /* Take finished SITDs out of the schedule
2256 * and process them: recycle, maybe report
2259 *q_p
= q
.sitd
->sitd_next
;
2260 *hw_p
= q
.sitd
->hw_next
;
2261 type
= Q_NEXT_TYPE(ehci
, q
.sitd
->hw_next
);
2263 modified
= sitd_complete (ehci
, q
.sitd
);
2267 dbg ("corrupt type %d frame %d shadow %p",
2268 type
, frame
, q
.ptr
);
2273 /* assume completion callbacks modify the queue */
2274 if (unlikely (modified
)) {
2275 if (likely(ehci
->periodic_sched
> 0))
2277 /* short-circuit this scan */
2283 /* If we can tell we caught up to the hardware, stop now.
2284 * We can't advance our scan without collecting the ISO
2285 * transfers that are still pending in this frame.
2287 if (incomplete
&& HC_IS_RUNNING(ehci_to_hcd(ehci
)->state
)) {
2288 ehci
->next_uframe
= now_uframe
;
2292 // FIXME: this assumes we won't get lapped when
2293 // latencies climb; that should be rare, but...
2294 // detect it, and just go all the way around.
2295 // FLR might help detect this case, so long as latencies
2296 // don't exceed periodic_size msec (default 1.024 sec).
2298 // FIXME: likewise assumes HC doesn't halt mid-scan
2300 if (now_uframe
== clock
) {
2303 if (!HC_IS_RUNNING (ehci_to_hcd(ehci
)->state
)
2304 || ehci
->periodic_sched
== 0)
2306 ehci
->next_uframe
= now_uframe
;
2307 now
= ehci_readl(ehci
, &ehci
->regs
->frame_index
) % mod
;
2308 if (now_uframe
== now
)
2311 /* rescan the rest of this frame, then ... */
2313 clock_frame
= clock
>> 3;
2314 if (ehci
->clock_frame
!= clock_frame
) {
2315 free_cached_itd_list(ehci
);
2316 ehci
->clock_frame
= clock_frame
;