2 * Copyright (c) 2001-2003 by David Brownell
3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License as published by the
7 * Free Software Foundation; either version 2 of the License, or (at your
8 * option) any later version.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software Foundation,
17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
20 /* this file is part of ehci-hcd.c */
22 /*-------------------------------------------------------------------------*/
25 * EHCI scheduled transaction support: interrupt, iso, split iso
26 * These are called "periodic" transactions in the EHCI spec.
28 * Note that for interrupt transfers, the QH/QTD manipulation is shared
29 * with the "asynchronous" transaction support (control/bulk transfers).
30 * The only real difference is in how interrupt transfers are scheduled.
32 * For ISO, we make an "iso_stream" head to serve the same role as a QH.
33 * It keeps track of every ITD (or SITD) that's linked, and holds enough
34 * pre-calculated schedule data to make appending to the queue be quick.
37 static int ehci_get_frame (struct usb_hcd
*hcd
);
39 /*-------------------------------------------------------------------------*/
42 * periodic_next_shadow - return "next" pointer on shadow list
43 * @periodic: host pointer to qh/itd/sitd
44 * @tag: hardware tag for type of this record
46 static union ehci_shadow
*
47 periodic_next_shadow (union ehci_shadow
*periodic
, __le32 tag
)
51 return &periodic
->qh
->qh_next
;
53 return &periodic
->fstn
->fstn_next
;
55 return &periodic
->itd
->itd_next
;
58 return &periodic
->sitd
->sitd_next
;
62 /* returns true after successful unlink */
63 /* caller must hold ehci->lock */
64 static int periodic_unlink (struct ehci_hcd
*ehci
, unsigned frame
, void *ptr
)
66 union ehci_shadow
*prev_p
= &ehci
->pshadow
[frame
];
67 __le32
*hw_p
= &ehci
->periodic
[frame
];
68 union ehci_shadow here
= *prev_p
;
69 union ehci_shadow
*next_p
;
71 /* find predecessor of "ptr"; hw and shadow lists are in sync */
72 while (here
.ptr
&& here
.ptr
!= ptr
) {
73 prev_p
= periodic_next_shadow (prev_p
, Q_NEXT_TYPE (*hw_p
));
74 hw_p
= &here
.qh
->hw_next
;
77 /* an interrupt entry (at list end) could have been shared */
79 dbg ("entry %p no longer on frame [%d]", ptr
, frame
);
82 // vdbg ("periodic unlink %p from frame %d", ptr, frame);
84 /* update hardware list ... HC may still know the old structure, so
85 * don't change hw_next until it'll have purged its cache
87 next_p
= periodic_next_shadow (&here
, Q_NEXT_TYPE (*hw_p
));
88 *hw_p
= here
.qh
->hw_next
;
90 /* unlink from shadow list; HCD won't see old structure again */
97 /* how many of the uframe's 125 usecs are allocated? */
99 periodic_usecs (struct ehci_hcd
*ehci
, unsigned frame
, unsigned uframe
)
101 __le32
*hw_p
= &ehci
->periodic
[frame
];
102 union ehci_shadow
*q
= &ehci
->pshadow
[frame
];
106 switch (Q_NEXT_TYPE (*hw_p
)) {
108 /* is it in the S-mask? */
109 if (q
->qh
->hw_info2
& cpu_to_le32 (1 << uframe
))
110 usecs
+= q
->qh
->usecs
;
112 if (q
->qh
->hw_info2
& cpu_to_le32 (1 << (8 + uframe
)))
113 usecs
+= q
->qh
->c_usecs
;
114 hw_p
= &q
->qh
->hw_next
;
118 /* for "save place" FSTNs, count the relevant INTR
119 * bandwidth from the previous frame
121 if (q
->fstn
->hw_prev
!= EHCI_LIST_END
) {
122 ehci_dbg (ehci
, "ignoring FSTN cost ...\n");
124 hw_p
= &q
->fstn
->hw_next
;
125 q
= &q
->fstn
->fstn_next
;
128 usecs
+= q
->itd
->usecs
[uframe
];
129 hw_p
= &q
->itd
->hw_next
;
130 q
= &q
->itd
->itd_next
;
133 /* is it in the S-mask? (count SPLIT, DATA) */
134 if (q
->sitd
->hw_uframe
& cpu_to_le32 (1 << uframe
)) {
135 if (q
->sitd
->hw_fullspeed_ep
&
136 __constant_cpu_to_le32 (1<<31))
137 usecs
+= q
->sitd
->stream
->usecs
;
138 else /* worst case for OUT start-split */
139 usecs
+= HS_USECS_ISO (188);
142 /* ... C-mask? (count CSPLIT, DATA) */
143 if (q
->sitd
->hw_uframe
&
144 cpu_to_le32 (1 << (8 + uframe
))) {
145 /* worst case for IN complete-split */
146 usecs
+= q
->sitd
->stream
->c_usecs
;
149 hw_p
= &q
->sitd
->hw_next
;
150 q
= &q
->sitd
->sitd_next
;
158 err ("overallocated uframe %d, periodic is %d usecs",
159 frame
* 8 + uframe
, usecs
);
164 /*-------------------------------------------------------------------------*/
166 static int same_tt (struct usb_device
*dev1
, struct usb_device
*dev2
)
168 if (!dev1
->tt
|| !dev2
->tt
)
170 if (dev1
->tt
!= dev2
->tt
)
173 return dev1
->ttport
== dev2
->ttport
;
178 /* return true iff the device's transaction translator is available
179 * for a periodic transfer starting at the specified frame, using
180 * all the uframes in the mask.
182 static int tt_no_collision (
183 struct ehci_hcd
*ehci
,
185 struct usb_device
*dev
,
190 if (period
== 0) /* error */
193 /* note bandwidth wastage: split never follows csplit
194 * (different dev or endpoint) until the next uframe.
195 * calling convention doesn't make that distinction.
197 for (; frame
< ehci
->periodic_size
; frame
+= period
) {
198 union ehci_shadow here
;
201 here
= ehci
->pshadow
[frame
];
202 type
= Q_NEXT_TYPE (ehci
->periodic
[frame
]);
206 type
= Q_NEXT_TYPE (here
.itd
->hw_next
);
207 here
= here
.itd
->itd_next
;
210 if (same_tt (dev
, here
.qh
->dev
)) {
213 mask
= le32_to_cpu (here
.qh
->hw_info2
);
214 /* "knows" no gap is needed */
219 type
= Q_NEXT_TYPE (here
.qh
->hw_next
);
220 here
= here
.qh
->qh_next
;
223 if (same_tt (dev
, here
.itd
->urb
->dev
)) {
226 mask
= le32_to_cpu (here
.sitd
228 /* FIXME assumes no gap for IN! */
233 type
= Q_NEXT_TYPE (here
.qh
->hw_next
);
234 here
= here
.sitd
->sitd_next
;
239 "periodic frame %d bogus type %d\n",
243 /* collision or error */
252 /*-------------------------------------------------------------------------*/
254 static int enable_periodic (struct ehci_hcd
*ehci
)
259 /* did clearing PSE did take effect yet?
260 * takes effect only at frame boundaries...
262 status
= handshake (&ehci
->regs
->status
, STS_PSS
, 0, 9 * 125);
264 ehci
->hcd
.state
= USB_STATE_HALT
;
268 cmd
= readl (&ehci
->regs
->command
) | CMD_PSE
;
269 writel (cmd
, &ehci
->regs
->command
);
270 /* posted write ... PSS happens later */
271 ehci
->hcd
.state
= USB_STATE_RUNNING
;
273 /* make sure ehci_work scans these */
274 ehci
->next_uframe
= readl (&ehci
->regs
->frame_index
)
275 % (ehci
->periodic_size
<< 3);
279 static int disable_periodic (struct ehci_hcd
*ehci
)
284 /* did setting PSE not take effect yet?
285 * takes effect only at frame boundaries...
287 status
= handshake (&ehci
->regs
->status
, STS_PSS
, STS_PSS
, 9 * 125);
289 ehci
->hcd
.state
= USB_STATE_HALT
;
293 cmd
= readl (&ehci
->regs
->command
) & ~CMD_PSE
;
294 writel (cmd
, &ehci
->regs
->command
);
295 /* posted write ... */
297 ehci
->next_uframe
= -1;
301 /*-------------------------------------------------------------------------*/
303 // FIXME microframe periods not yet handled
305 static void intr_deschedule (
306 struct ehci_hcd
*ehci
,
311 unsigned frame
= qh
->start
;
314 periodic_unlink (ehci
, frame
, qh
);
317 } while (frame
< ehci
->periodic_size
);
319 qh
->qh_state
= QH_STATE_UNLINK
;
320 qh
->qh_next
.ptr
= NULL
;
321 ehci
->periodic_sched
--;
323 /* maybe turn off periodic schedule */
324 if (!ehci
->periodic_sched
)
325 status
= disable_periodic (ehci
);
328 vdbg ("periodic schedule still enabled");
332 * If the hc may be looking at this qh, then delay a uframe
333 * (yeech!) to be sure it's done.
334 * No other threads may be mucking with this qh.
336 if (((ehci_get_frame (&ehci
->hcd
) - frame
) % qh
->period
) == 0) {
339 qh
->hw_next
= EHCI_LIST_END
;
341 /* we may not be IDLE yet, but if the qh is empty
342 * the race is very short. then if qh also isn't
343 * rescheduled soon, it won't matter. otherwise...
345 vdbg ("intr_deschedule...");
348 qh
->hw_next
= EHCI_LIST_END
;
350 qh
->qh_state
= QH_STATE_IDLE
;
352 /* update per-qh bandwidth utilization (for usbfs) */
353 hcd_to_bus (&ehci
->hcd
)->bandwidth_allocated
-=
354 (qh
->usecs
+ qh
->c_usecs
) / qh
->period
;
356 dbg ("descheduled qh %p, period = %d frame = %d count = %d, urbs = %d",
357 qh
, qh
->period
, frame
,
358 atomic_read (&qh
->kref
.refcount
), ehci
->periodic_sched
);
361 static int check_period (
362 struct ehci_hcd
*ehci
,
368 /* complete split running into next frame?
369 * given FSTN support, we could sometimes check...
375 * 80% periodic == 100 usec/uframe available
376 * convert "usecs we need" to "max already claimed"
383 // FIXME delete when intr_submit handles non-empty queues
384 // this gives us a one intr/frame limit (vs N/uframe)
385 // ... and also lets us avoid tracking split transactions
386 // that might collide at a given TT/hub.
387 if (ehci
->pshadow
[frame
].ptr
)
390 claimed
= periodic_usecs (ehci
, frame
, uframe
);
394 // FIXME update to handle sub-frame periods
395 } while ((frame
+= period
) < ehci
->periodic_size
);
401 static int check_intr_schedule (
402 struct ehci_hcd
*ehci
,
405 const struct ehci_qh
*qh
,
409 int retval
= -ENOSPC
;
411 if (!check_period (ehci
, frame
, uframe
, qh
->period
, qh
->usecs
))
419 /* This is a split transaction; check the bandwidth available for
420 * the completion too. Check both worst and best case gaps: worst
421 * case is SPLIT near uframe end, and CSPLIT near start ... best is
422 * vice versa. Difference can be almost two uframe times, but we
423 * reserve unnecessary bandwidth (waste it) this way. (Actually
424 * even better cases exist, like immediate device NAK.)
426 * FIXME don't even bother unless we know this TT is idle in that
427 * range of uframes ... for now, check_period() allows only one
428 * interrupt transfer per frame, so needn't check "TT busy" status
429 * when scheduling a split (QH, SITD, or FSTN).
431 * FIXME ehci 0.96 and above can use FSTNs
433 if (!check_period (ehci
, frame
, uframe
+ qh
->gap_uf
+ 1,
434 qh
->period
, qh
->c_usecs
))
436 if (!check_period (ehci
, frame
, uframe
+ qh
->gap_uf
,
437 qh
->period
, qh
->c_usecs
))
440 *c_maskp
= cpu_to_le32 (0x03 << (8 + uframe
+ qh
->gap_uf
));
446 static int qh_schedule (struct ehci_hcd
*ehci
, struct ehci_qh
*qh
)
451 unsigned frame
; /* 0..(qh->period - 1), or NO_FRAME */
453 qh
->hw_next
= EHCI_LIST_END
;
456 /* reuse the previous schedule slots, if we can */
457 if (frame
< qh
->period
) {
458 uframe
= ffs (le32_to_cpup (&qh
->hw_info2
) & 0x00ff);
459 status
= check_intr_schedule (ehci
, frame
, --uframe
,
467 /* else scan the schedule to find a group of slots such that all
468 * uframes have enough periodic bandwidth available.
471 frame
= qh
->period
- 1;
473 for (uframe
= 0; uframe
< 8; uframe
++) {
474 status
= check_intr_schedule (ehci
,
480 } while (status
&& frame
--);
485 /* reset S-frame and (maybe) C-frame masks */
486 qh
->hw_info2
&= ~__constant_cpu_to_le32(0xffff);
487 qh
->hw_info2
|= cpu_to_le32 (1 << uframe
) | c_mask
;
489 dbg ("reused previous qh %p schedule", qh
);
491 /* stuff into the periodic schedule */
492 qh
->qh_state
= QH_STATE_LINKED
;
493 dbg ("scheduled qh %p usecs %d/%d period %d.0 starting %d.%d (gap %d)",
494 qh
, qh
->usecs
, qh
->c_usecs
,
495 qh
->period
, frame
, uframe
, qh
->gap_uf
);
497 if (unlikely (ehci
->pshadow
[frame
].ptr
!= 0)) {
499 // FIXME -- just link toward the end, before any qh with a shorter period,
500 // AND accommodate it already having been linked here (after some other qh)
501 // AS WELL AS updating the schedule checking logic
505 ehci
->pshadow
[frame
].qh
= qh_get (qh
);
506 ehci
->periodic
[frame
] =
507 QH_NEXT (qh
->qh_dma
);
511 } while (frame
< ehci
->periodic_size
);
513 /* update per-qh bandwidth for usbfs */
514 hcd_to_bus (&ehci
->hcd
)->bandwidth_allocated
+=
515 (qh
->usecs
+ qh
->c_usecs
) / qh
->period
;
517 /* maybe enable periodic schedule processing */
518 if (!ehci
->periodic_sched
++)
519 status
= enable_periodic (ehci
);
524 static int intr_submit (
525 struct ehci_hcd
*ehci
,
527 struct list_head
*qtd_list
,
536 struct list_head empty
;
538 /* get endpoint and transfer/schedule data */
539 epnum
= usb_pipeendpoint (urb
->pipe
);
540 is_input
= usb_pipein (urb
->pipe
);
544 spin_lock_irqsave (&ehci
->lock
, flags
);
545 dev
= (struct hcd_dev
*)urb
->dev
->hcpriv
;
547 /* get qh and force any scheduling errors */
548 INIT_LIST_HEAD (&empty
);
549 qh
= qh_append_tds (ehci
, urb
, &empty
, epnum
, &dev
->ep
[epnum
]);
554 if (qh
->qh_state
== QH_STATE_IDLE
) {
555 if ((status
= qh_schedule (ehci
, qh
)) != 0)
559 /* then queue the urb's tds to the qh */
560 qh
= qh_append_tds (ehci
, urb
, qtd_list
, epnum
, &dev
->ep
[epnum
]);
563 /* ... update usbfs periodic stats */
564 hcd_to_bus (&ehci
->hcd
)->bandwidth_int_reqs
++;
567 spin_unlock_irqrestore (&ehci
->lock
, flags
);
569 qtd_list_free (ehci
, urb
, qtd_list
);
574 /*-------------------------------------------------------------------------*/
576 /* ehci_iso_stream ops work with both ITD and SITD */
578 static struct ehci_iso_stream
*
579 iso_stream_alloc (int mem_flags
)
581 struct ehci_iso_stream
*stream
;
583 stream
= kmalloc(sizeof *stream
, mem_flags
);
584 if (likely (stream
!= 0)) {
585 memset (stream
, 0, sizeof(*stream
));
586 INIT_LIST_HEAD(&stream
->td_list
);
587 INIT_LIST_HEAD(&stream
->free_list
);
588 stream
->next_uframe
= -1;
589 stream
->refcount
= 1;
596 struct ehci_iso_stream
*stream
,
597 struct usb_device
*dev
,
602 static const u8 smask_out
[] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
605 unsigned epnum
, maxp
;
610 * this might be a "high bandwidth" highspeed endpoint,
611 * as encoded in the ep descriptor's wMaxPacket field
613 epnum
= usb_pipeendpoint (pipe
);
614 is_input
= usb_pipein (pipe
) ? USB_DIR_IN
: 0;
616 maxp
= dev
->epmaxpacketin
[epnum
];
619 maxp
= dev
->epmaxpacketout
[epnum
];
623 /* knows about ITD vs SITD */
624 if (dev
->speed
== USB_SPEED_HIGH
) {
625 unsigned multi
= hb_mult(maxp
);
627 stream
->highspeed
= 1;
629 maxp
= max_packet(maxp
);
633 stream
->buf0
= cpu_to_le32 ((epnum
<< 8) | dev
->devnum
);
634 stream
->buf1
= cpu_to_le32 (buf1
);
635 stream
->buf2
= cpu_to_le32 (multi
);
637 /* usbfs wants to report the average usecs per frame tied up
638 * when transfers on this endpoint are scheduled ...
640 stream
->usecs
= HS_USECS_ISO (maxp
);
641 bandwidth
= stream
->usecs
* 8;
642 bandwidth
/= 1 << (interval
- 1);
647 addr
= dev
->ttport
<< 24;
648 addr
|= dev
->tt
->hub
->devnum
<< 16;
651 stream
->usecs
= HS_USECS_ISO (maxp
);
656 stream
->c_usecs
= stream
->usecs
;
657 stream
->usecs
= HS_USECS_ISO (1);
658 stream
->raw_mask
= 1;
660 /* pessimistic c-mask */
661 tmp
= usb_calc_bus_time (USB_SPEED_FULL
, 1, 0, maxp
)
663 stream
->raw_mask
|= 3 << (tmp
+ 9);
665 stream
->raw_mask
= smask_out
[maxp
/ 188];
666 bandwidth
= stream
->usecs
+ stream
->c_usecs
;
667 bandwidth
/= 1 << (interval
+ 2);
669 /* stream->splits gets created from raw_mask later */
670 stream
->address
= cpu_to_le32 (addr
);
672 stream
->bandwidth
= bandwidth
;
676 stream
->bEndpointAddress
= is_input
| epnum
;
677 stream
->interval
= interval
;
682 iso_stream_put(struct ehci_hcd
*ehci
, struct ehci_iso_stream
*stream
)
686 /* free whenever just a dev->ep reference remains.
687 * not like a QH -- no persistent state (toggle, halt)
689 if (stream
->refcount
== 1) {
691 struct hcd_dev
*dev
= stream
->udev
->hcpriv
;
693 // BUG_ON (!list_empty(&stream->td_list));
695 while (!list_empty (&stream
->free_list
)) {
696 struct list_head
*entry
;
698 entry
= stream
->free_list
.next
;
701 /* knows about ITD vs SITD */
702 if (stream
->highspeed
) {
703 struct ehci_itd
*itd
;
705 itd
= list_entry (entry
, struct ehci_itd
,
707 dma_pool_free (ehci
->itd_pool
, itd
,
710 struct ehci_sitd
*sitd
;
712 sitd
= list_entry (entry
, struct ehci_sitd
,
714 dma_pool_free (ehci
->sitd_pool
, sitd
,
719 is_in
= (stream
->bEndpointAddress
& USB_DIR_IN
) ? 0x10 : 0;
720 stream
->bEndpointAddress
&= 0x0f;
721 dev
->ep
[is_in
+ stream
->bEndpointAddress
] = NULL
;
723 if (stream
->rescheduled
) {
724 ehci_info (ehci
, "ep%d%s-iso rescheduled "
725 "%lu times in %lu seconds\n",
726 stream
->bEndpointAddress
, is_in
? "in" : "out",
728 ((jiffies
- stream
->start
)/HZ
)
736 static inline struct ehci_iso_stream
*
737 iso_stream_get (struct ehci_iso_stream
*stream
)
739 if (likely (stream
!= 0))
744 static struct ehci_iso_stream
*
745 iso_stream_find (struct ehci_hcd
*ehci
, struct urb
*urb
)
749 struct ehci_iso_stream
*stream
;
752 epnum
= usb_pipeendpoint (urb
->pipe
);
753 if (usb_pipein(urb
->pipe
))
756 spin_lock_irqsave (&ehci
->lock
, flags
);
758 dev
= (struct hcd_dev
*)urb
->dev
->hcpriv
;
759 stream
= dev
->ep
[epnum
];
761 if (unlikely (stream
== 0)) {
762 stream
= iso_stream_alloc(GFP_ATOMIC
);
763 if (likely (stream
!= 0)) {
764 /* dev->ep owns the initial refcount */
765 dev
->ep
[epnum
] = stream
;
766 iso_stream_init(stream
, urb
->dev
, urb
->pipe
,
770 /* if dev->ep [epnum] is a QH, info1.maxpacket is nonzero */
771 } else if (unlikely (stream
->hw_info1
!= 0)) {
772 ehci_dbg (ehci
, "dev %s ep%d%s, not iso??\n",
773 urb
->dev
->devpath
, epnum
& 0x0f,
774 (epnum
& 0x10) ? "in" : "out");
778 /* caller guarantees an eventual matching iso_stream_put */
779 stream
= iso_stream_get (stream
);
781 spin_unlock_irqrestore (&ehci
->lock
, flags
);
785 /*-------------------------------------------------------------------------*/
787 /* ehci_iso_sched ops can be shared, ITD-only, or SITD-only */
789 static struct ehci_iso_sched
*
790 iso_sched_alloc (unsigned packets
, int mem_flags
)
792 struct ehci_iso_sched
*iso_sched
;
793 int size
= sizeof *iso_sched
;
795 size
+= packets
* sizeof (struct ehci_iso_packet
);
796 iso_sched
= kmalloc (size
, mem_flags
);
797 if (likely (iso_sched
!= 0)) {
798 memset(iso_sched
, 0, size
);
799 INIT_LIST_HEAD (&iso_sched
->td_list
);
806 struct ehci_iso_sched
*iso_sched
,
807 struct ehci_iso_stream
*stream
,
812 dma_addr_t dma
= urb
->transfer_dma
;
814 /* how many uframes are needed for these transfers */
815 iso_sched
->span
= urb
->number_of_packets
* stream
->interval
;
817 /* figure out per-uframe itd fields that we'll need later
818 * when we fit new itds into the schedule.
820 for (i
= 0; i
< urb
->number_of_packets
; i
++) {
821 struct ehci_iso_packet
*uframe
= &iso_sched
->packet
[i
];
826 length
= urb
->iso_frame_desc
[i
].length
;
827 buf
= dma
+ urb
->iso_frame_desc
[i
].offset
;
829 trans
= EHCI_ISOC_ACTIVE
;
830 trans
|= buf
& 0x0fff;
831 if (unlikely (((i
+ 1) == urb
->number_of_packets
))
832 && !(urb
->transfer_flags
& URB_NO_INTERRUPT
))
833 trans
|= EHCI_ITD_IOC
;
834 trans
|= length
<< 16;
835 uframe
->transaction
= cpu_to_le32 (trans
);
837 /* might need to cross a buffer page within a td */
838 uframe
->bufp
= (buf
& ~(u64
)0x0fff);
840 if (unlikely ((uframe
->bufp
!= (buf
& ~(u64
)0x0fff))))
847 struct ehci_iso_stream
*stream
,
848 struct ehci_iso_sched
*iso_sched
853 // caller must hold ehci->lock!
854 list_splice (&iso_sched
->td_list
, &stream
->free_list
);
859 itd_urb_transaction (
860 struct ehci_iso_stream
*stream
,
861 struct ehci_hcd
*ehci
,
866 struct ehci_itd
*itd
;
870 struct ehci_iso_sched
*sched
;
873 sched
= iso_sched_alloc (urb
->number_of_packets
, mem_flags
);
874 if (unlikely (sched
== 0))
877 itd_sched_init (sched
, stream
, urb
);
879 if (urb
->interval
< 8)
880 num_itds
= 1 + (sched
->span
+ 7) / 8;
882 num_itds
= urb
->number_of_packets
;
884 /* allocate/init ITDs */
885 spin_lock_irqsave (&ehci
->lock
, flags
);
886 for (i
= 0; i
< num_itds
; i
++) {
888 /* free_list.next might be cache-hot ... but maybe
889 * the HC caches it too. avoid that issue for now.
892 /* prefer previously-allocated itds */
893 if (likely (!list_empty(&stream
->free_list
))) {
894 itd
= list_entry (stream
->free_list
.prev
,
895 struct ehci_itd
, itd_list
);
896 list_del (&itd
->itd_list
);
897 itd_dma
= itd
->itd_dma
;
902 spin_unlock_irqrestore (&ehci
->lock
, flags
);
903 itd
= dma_pool_alloc (ehci
->itd_pool
, mem_flags
,
905 spin_lock_irqsave (&ehci
->lock
, flags
);
908 if (unlikely (0 == itd
)) {
909 iso_sched_free (stream
, sched
);
912 memset (itd
, 0, sizeof *itd
);
913 itd
->itd_dma
= itd_dma
;
914 list_add (&itd
->itd_list
, &sched
->td_list
);
916 spin_unlock_irqrestore (&ehci
->lock
, flags
);
918 /* temporarily store schedule info in hcpriv */
920 urb
->error_count
= 0;
924 /*-------------------------------------------------------------------------*/
928 struct ehci_hcd
*ehci
,
937 /* can't commit more than 80% periodic == 100 usec */
938 if (periodic_usecs (ehci
, uframe
>> 3, uframe
& 0x7)
942 /* we know urb->interval is 2^N uframes */
944 } while (uframe
< mod
);
950 struct ehci_hcd
*ehci
,
952 struct ehci_iso_stream
*stream
,
954 struct ehci_iso_sched
*sched
,
961 mask
= stream
->raw_mask
<< (uframe
& 7);
963 /* for IN, don't wrap CSPLIT into the next frame */
967 /* this multi-pass logic is simple, but performance may
968 * suffer when the schedule data isn't cached.
971 /* check bandwidth */
972 uframe
%= period_uframes
;
979 /* tt must be idle for start(s), any gap, and csplit.
980 * assume scheduling slop leaves 10+% for control/bulk.
982 if (!tt_no_collision (ehci
, period_uframes
<< 3,
983 stream
->udev
, frame
, mask
))
986 /* check starts (OUT uses more than one) */
987 max_used
= 100 - stream
->usecs
;
988 for (tmp
= stream
->raw_mask
& 0xff; tmp
; tmp
>>= 1, uf
++) {
989 if (periodic_usecs (ehci
, frame
, uf
) > max_used
)
993 /* for IN, check CSPLIT */
994 if (stream
->c_usecs
) {
995 max_used
= 100 - stream
->c_usecs
;
999 if ((stream
->raw_mask
& tmp
) == 0)
1001 if (periodic_usecs (ehci
, frame
, uf
)
1007 /* we know urb->interval is 2^N uframes */
1008 uframe
+= period_uframes
;
1009 } while (uframe
< mod
);
1011 stream
->splits
= cpu_to_le32(stream
->raw_mask
<< (uframe
& 7));
1016 * This scheduler plans almost as far into the future as it has actual
1017 * periodic schedule slots. (Affected by TUNE_FLS, which defaults to
1018 * "as small as possible" to be cache-friendlier.) That limits the size
1019 * transfers you can stream reliably; avoid more than 64 msec per urb.
1020 * Also avoid queue depths of less than ehci's worst irq latency (affected
1021 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
1022 * and other factors); or more than about 230 msec total (for portability,
1023 * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler!
1026 #define SCHEDULE_SLOP 10 /* frames */
1029 iso_stream_schedule (
1030 struct ehci_hcd
*ehci
,
1032 struct ehci_iso_stream
*stream
1035 u32 now
, start
, max
, period
;
1037 unsigned mod
= ehci
->periodic_size
<< 3;
1038 struct ehci_iso_sched
*sched
= urb
->hcpriv
;
1040 if (sched
->span
> (mod
- 8 * SCHEDULE_SLOP
)) {
1041 ehci_dbg (ehci
, "iso request %p too long\n", urb
);
1046 if ((stream
->depth
+ sched
->span
) > mod
) {
1047 ehci_dbg (ehci
, "request %p would overflow (%d+%d>%d)\n",
1048 urb
, stream
->depth
, sched
->span
, mod
);
1053 now
= readl (&ehci
->regs
->frame_index
) % mod
;
1055 /* when's the last uframe this urb could start? */
1058 /* typical case: reuse current schedule. stream is still active,
1059 * and no gaps from host falling behind (irq delays etc)
1061 if (likely (!list_empty (&stream
->td_list
))) {
1062 start
= stream
->next_uframe
;
1065 if (likely ((start
+ sched
->span
) < max
))
1067 /* else fell behind; someday, try to reschedule */
1072 /* need to schedule; when's the next (u)frame we could start?
1073 * this is bigger than ehci->i_thresh allows; scheduling itself
1074 * isn't free, the slop should handle reasonably slow cpus. it
1075 * can also help high bandwidth if the dma and irq loads don't
1076 * jump until after the queue is primed.
1078 start
= SCHEDULE_SLOP
* 8 + (now
& ~0x07);
1080 stream
->next_uframe
= start
;
1082 /* NOTE: assumes URB_ISO_ASAP, to limit complexity/bugs */
1084 period
= urb
->interval
;
1085 if (!stream
->highspeed
)
1088 /* find a uframe slot with enough bandwidth */
1089 for (; start
< (stream
->next_uframe
+ period
); start
++) {
1092 /* check schedule: enough space? */
1093 if (stream
->highspeed
)
1094 enough_space
= itd_slot_ok (ehci
, mod
, start
,
1095 stream
->usecs
, period
);
1097 if ((start
% 8) >= 6)
1099 enough_space
= sitd_slot_ok (ehci
, mod
, stream
,
1100 start
, sched
, period
);
1103 /* schedule it here if there's enough bandwidth */
1105 stream
->next_uframe
= start
% mod
;
1110 /* no room in the schedule */
1111 ehci_dbg (ehci
, "iso %ssched full %p (now %d max %d)\n",
1112 list_empty (&stream
->td_list
) ? "" : "re",
1117 iso_sched_free (stream
, sched
);
1122 urb
->start_frame
= stream
->next_uframe
;
1126 /*-------------------------------------------------------------------------*/
1129 itd_init (struct ehci_iso_stream
*stream
, struct ehci_itd
*itd
)
1133 itd
->hw_next
= EHCI_LIST_END
;
1134 itd
->hw_bufp
[0] = stream
->buf0
;
1135 itd
->hw_bufp
[1] = stream
->buf1
;
1136 itd
->hw_bufp
[2] = stream
->buf2
;
1138 for (i
= 0; i
< 8; i
++)
1141 /* All other fields are filled when scheduling */
1146 struct ehci_itd
*itd
,
1147 struct ehci_iso_sched
*iso_sched
,
1153 struct ehci_iso_packet
*uf
= &iso_sched
->packet
[index
];
1154 unsigned pg
= itd
->pg
;
1156 // BUG_ON (pg == 6 && uf->cross);
1159 itd
->index
[uframe
] = index
;
1161 itd
->hw_transaction
[uframe
] = uf
->transaction
;
1162 itd
->hw_transaction
[uframe
] |= cpu_to_le32 (pg
<< 12);
1163 itd
->hw_bufp
[pg
] |= cpu_to_le32 (uf
->bufp
& ~(u32
)0);
1164 itd
->hw_bufp_hi
[pg
] |= cpu_to_le32 ((u32
)(uf
->bufp
>> 32));
1166 /* iso_frame_desc[].offset must be strictly increasing */
1167 if (unlikely (!first
&& uf
->cross
)) {
1168 u64 bufp
= uf
->bufp
+ 4096;
1170 itd
->hw_bufp
[pg
] |= cpu_to_le32 (bufp
& ~(u32
)0);
1171 itd
->hw_bufp_hi
[pg
] |= cpu_to_le32 ((u32
)(bufp
>> 32));
1176 itd_link (struct ehci_hcd
*ehci
, unsigned frame
, struct ehci_itd
*itd
)
1178 /* always prepend ITD/SITD ... only QH tree is order-sensitive */
1179 itd
->itd_next
= ehci
->pshadow
[frame
];
1180 itd
->hw_next
= ehci
->periodic
[frame
];
1181 ehci
->pshadow
[frame
].itd
= itd
;
1184 ehci
->periodic
[frame
] = cpu_to_le32 (itd
->itd_dma
) | Q_TYPE_ITD
;
1187 /* fit urb's itds into the selected schedule slot; activate as needed */
1190 struct ehci_hcd
*ehci
,
1193 struct ehci_iso_stream
*stream
1196 int packet
, first
= 1;
1197 unsigned next_uframe
, uframe
, frame
;
1198 struct ehci_iso_sched
*iso_sched
= urb
->hcpriv
;
1199 struct ehci_itd
*itd
;
1201 next_uframe
= stream
->next_uframe
% mod
;
1203 if (unlikely (list_empty(&stream
->td_list
))) {
1204 hcd_to_bus (&ehci
->hcd
)->bandwidth_allocated
1205 += stream
->bandwidth
;
1207 "schedule devp %s ep%d%s-iso period %d start %d.%d\n",
1208 urb
->dev
->devpath
, stream
->bEndpointAddress
& 0x0f,
1209 (stream
->bEndpointAddress
& USB_DIR_IN
) ? "in" : "out",
1211 next_uframe
>> 3, next_uframe
& 0x7);
1212 stream
->start
= jiffies
;
1214 hcd_to_bus (&ehci
->hcd
)->bandwidth_isoc_reqs
++;
1216 /* fill iTDs uframe by uframe */
1217 for (packet
= 0, itd
= NULL
; packet
< urb
->number_of_packets
; ) {
1219 /* ASSERT: we have all necessary itds */
1220 // BUG_ON (list_empty (&iso_sched->td_list));
1222 /* ASSERT: no itds for this endpoint in this uframe */
1224 itd
= list_entry (iso_sched
->td_list
.next
,
1225 struct ehci_itd
, itd_list
);
1226 list_move_tail (&itd
->itd_list
, &stream
->td_list
);
1227 itd
->stream
= iso_stream_get (stream
);
1228 itd
->urb
= usb_get_urb (urb
);
1230 itd_init (stream
, itd
);
1233 uframe
= next_uframe
& 0x07;
1234 frame
= next_uframe
>> 3;
1236 itd
->usecs
[uframe
] = stream
->usecs
;
1237 itd_patch (itd
, iso_sched
, packet
, uframe
, first
);
1240 next_uframe
+= stream
->interval
;
1241 stream
->depth
+= stream
->interval
;
1245 /* link completed itds into the schedule */
1246 if (((next_uframe
>> 3) != frame
)
1247 || packet
== urb
->number_of_packets
) {
1248 itd_link (ehci
, frame
% ehci
->periodic_size
, itd
);
1252 stream
->next_uframe
= next_uframe
;
1254 /* don't need that schedule data any more */
1255 iso_sched_free (stream
, iso_sched
);
1258 timer_action (ehci
, TIMER_IO_WATCHDOG
);
1259 if (unlikely (!ehci
->periodic_sched
++))
1260 return enable_periodic (ehci
);
1264 #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1268 struct ehci_hcd
*ehci
,
1269 struct ehci_itd
*itd
,
1270 struct pt_regs
*regs
1272 struct urb
*urb
= itd
->urb
;
1273 struct usb_iso_packet_descriptor
*desc
;
1277 struct ehci_iso_stream
*stream
= itd
->stream
;
1278 struct usb_device
*dev
;
1280 /* for each uframe with a packet */
1281 for (uframe
= 0; uframe
< 8; uframe
++) {
1282 if (likely (itd
->index
[uframe
] == -1))
1284 urb_index
= itd
->index
[uframe
];
1285 desc
= &urb
->iso_frame_desc
[urb_index
];
1287 t
= le32_to_cpup (&itd
->hw_transaction
[uframe
]);
1288 itd
->hw_transaction
[uframe
] = 0;
1289 stream
->depth
-= stream
->interval
;
1291 /* report transfer status */
1292 if (unlikely (t
& ISO_ERRS
)) {
1294 if (t
& EHCI_ISOC_BUF_ERR
)
1295 desc
->status
= usb_pipein (urb
->pipe
)
1296 ? -ENOSR
/* hc couldn't read */
1297 : -ECOMM
; /* hc couldn't write */
1298 else if (t
& EHCI_ISOC_BABBLE
)
1299 desc
->status
= -EOVERFLOW
;
1300 else /* (t & EHCI_ISOC_XACTERR) */
1301 desc
->status
= -EPROTO
;
1303 /* HC need not update length with this error */
1304 if (!(t
& EHCI_ISOC_BABBLE
))
1305 desc
->actual_length
= EHCI_ITD_LENGTH (t
);
1306 } else if (likely ((t
& EHCI_ISOC_ACTIVE
) == 0)) {
1308 desc
->actual_length
= EHCI_ITD_LENGTH (t
);
1315 list_move (&itd
->itd_list
, &stream
->free_list
);
1316 iso_stream_put (ehci
, stream
);
1318 /* handle completion now? */
1319 if (likely ((urb_index
+ 1) != urb
->number_of_packets
))
1322 /* ASSERT: it's really the last itd for this urb
1323 list_for_each_entry (itd, &stream->td_list, itd_list)
1324 BUG_ON (itd->urb == urb);
1327 /* give urb back to the driver ... can be out-of-order */
1328 dev
= usb_get_dev (urb
->dev
);
1329 ehci_urb_done (ehci
, urb
, regs
);
1332 /* defer stopping schedule; completion can submit */
1333 ehci
->periodic_sched
--;
1334 if (unlikely (!ehci
->periodic_sched
))
1335 (void) disable_periodic (ehci
);
1336 hcd_to_bus (&ehci
->hcd
)->bandwidth_isoc_reqs
--;
1338 if (unlikely (list_empty (&stream
->td_list
))) {
1339 hcd_to_bus (&ehci
->hcd
)->bandwidth_allocated
1340 -= stream
->bandwidth
;
1342 "deschedule devp %s ep%d%s-iso\n",
1343 dev
->devpath
, stream
->bEndpointAddress
& 0x0f,
1344 (stream
->bEndpointAddress
& USB_DIR_IN
) ? "in" : "out");
1346 iso_stream_put (ehci
, stream
);
1352 /*-------------------------------------------------------------------------*/
1354 static int itd_submit (struct ehci_hcd
*ehci
, struct urb
*urb
, int mem_flags
)
1356 int status
= -EINVAL
;
1357 unsigned long flags
;
1358 struct ehci_iso_stream
*stream
;
1360 /* Get iso_stream head */
1361 stream
= iso_stream_find (ehci
, urb
);
1362 if (unlikely (stream
== 0)) {
1363 ehci_dbg (ehci
, "can't get iso stream\n");
1366 if (unlikely (urb
->interval
!= stream
->interval
)) {
1367 ehci_dbg (ehci
, "can't change iso interval %d --> %d\n",
1368 stream
->interval
, urb
->interval
);
1372 #ifdef EHCI_URB_TRACE
1374 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1375 __FUNCTION__
, urb
->dev
->devpath
, urb
,
1376 usb_pipeendpoint (urb
->pipe
),
1377 usb_pipein (urb
->pipe
) ? "in" : "out",
1378 urb
->transfer_buffer_length
,
1379 urb
->number_of_packets
, urb
->interval
,
1383 /* allocate ITDs w/o locking anything */
1384 status
= itd_urb_transaction (stream
, ehci
, urb
, mem_flags
);
1385 if (unlikely (status
< 0)) {
1386 ehci_dbg (ehci
, "can't init itds\n");
1390 /* schedule ... need to lock */
1391 spin_lock_irqsave (&ehci
->lock
, flags
);
1392 status
= iso_stream_schedule (ehci
, urb
, stream
);
1393 if (likely (status
== 0))
1394 itd_link_urb (ehci
, urb
, ehci
->periodic_size
<< 3, stream
);
1395 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1398 if (unlikely (status
< 0))
1399 iso_stream_put (ehci
, stream
);
1403 #ifdef CONFIG_USB_EHCI_SPLIT_ISO
1405 /*-------------------------------------------------------------------------*/
1408 * "Split ISO TDs" ... used for USB 1.1 devices going through the
1409 * TTs in USB 2.0 hubs. These need microframe scheduling.
1414 struct ehci_iso_sched
*iso_sched
,
1415 struct ehci_iso_stream
*stream
,
1420 dma_addr_t dma
= urb
->transfer_dma
;
1422 /* how many frames are needed for these transfers */
1423 iso_sched
->span
= urb
->number_of_packets
* stream
->interval
;
1425 /* figure out per-frame sitd fields that we'll need later
1426 * when we fit new sitds into the schedule.
1428 for (i
= 0; i
< urb
->number_of_packets
; i
++) {
1429 struct ehci_iso_packet
*packet
= &iso_sched
->packet
[i
];
1434 length
= urb
->iso_frame_desc
[i
].length
& 0x03ff;
1435 buf
= dma
+ urb
->iso_frame_desc
[i
].offset
;
1437 trans
= SITD_STS_ACTIVE
;
1438 if (((i
+ 1) == urb
->number_of_packets
)
1439 && !(urb
->transfer_flags
& URB_NO_INTERRUPT
))
1441 trans
|= length
<< 16;
1442 packet
->transaction
= cpu_to_le32 (trans
);
1444 /* might need to cross a buffer page within a td */
1447 packet
->buf1
= buf
& ~0x0fff;
1448 if (packet
->buf1
!= (buf
& ~(u64
)0x0fff))
1451 /* OUT uses multiple start-splits */
1452 if (stream
->bEndpointAddress
& USB_DIR_IN
)
1454 length
= 1 + (length
/ 188);
1455 packet
->buf1
|= length
;
1456 if (length
> 1) /* BEGIN vs ALL */
1457 packet
->buf1
|= 1 << 3;
1462 sitd_urb_transaction (
1463 struct ehci_iso_stream
*stream
,
1464 struct ehci_hcd
*ehci
,
1469 struct ehci_sitd
*sitd
;
1470 dma_addr_t sitd_dma
;
1472 struct ehci_iso_sched
*iso_sched
;
1473 unsigned long flags
;
1475 iso_sched
= iso_sched_alloc (urb
->number_of_packets
, mem_flags
);
1479 sitd_sched_init (iso_sched
, stream
, urb
);
1481 /* allocate/init sITDs */
1482 spin_lock_irqsave (&ehci
->lock
, flags
);
1483 for (i
= 0; i
< urb
->number_of_packets
; i
++) {
1485 /* NOTE: for now, we don't try to handle wraparound cases
1486 * for IN (using sitd->hw_backpointer, like a FSTN), which
1487 * means we never need two sitds for full speed packets.
1490 /* free_list.next might be cache-hot ... but maybe
1491 * the HC caches it too. avoid that issue for now.
1494 /* prefer previously-allocated sitds */
1495 if (!list_empty(&stream
->free_list
)) {
1496 sitd
= list_entry (stream
->free_list
.prev
,
1497 struct ehci_sitd
, sitd_list
);
1498 list_del (&sitd
->sitd_list
);
1499 sitd_dma
= sitd
->sitd_dma
;
1504 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1505 sitd
= dma_pool_alloc (ehci
->sitd_pool
, mem_flags
,
1507 spin_lock_irqsave (&ehci
->lock
, flags
);
1511 iso_sched_free (stream
, iso_sched
);
1512 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1515 memset (sitd
, 0, sizeof *sitd
);
1516 sitd
->sitd_dma
= sitd_dma
;
1517 list_add (&sitd
->sitd_list
, &iso_sched
->td_list
);
1520 /* temporarily store schedule info in hcpriv */
1521 urb
->hcpriv
= iso_sched
;
1522 urb
->error_count
= 0;
1524 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1528 /*-------------------------------------------------------------------------*/
1532 struct ehci_iso_stream
*stream
,
1533 struct ehci_sitd
*sitd
,
1534 struct ehci_iso_sched
*iso_sched
,
1538 struct ehci_iso_packet
*uf
= &iso_sched
->packet
[index
];
1539 u64 bufp
= uf
->bufp
;
1541 sitd
->hw_next
= EHCI_LIST_END
;
1542 sitd
->hw_fullspeed_ep
= stream
->address
;
1543 sitd
->hw_uframe
= stream
->splits
;
1544 sitd
->hw_results
= uf
->transaction
;
1545 sitd
->hw_backpointer
= EHCI_LIST_END
;
1548 sitd
->hw_buf
[0] = cpu_to_le32 (bufp
);
1549 sitd
->hw_buf_hi
[0] = cpu_to_le32 (bufp
>> 32);
1551 sitd
->hw_buf
[1] = cpu_to_le32 (uf
->buf1
);
1554 sitd
->hw_buf_hi
[1] = cpu_to_le32 (bufp
>> 32);
1556 sitd
->index
= index
;
1560 sitd_link (struct ehci_hcd
*ehci
, unsigned frame
, struct ehci_sitd
*sitd
)
1562 /* note: sitd ordering could matter (CSPLIT then SSPLIT) */
1563 sitd
->sitd_next
= ehci
->pshadow
[frame
];
1564 sitd
->hw_next
= ehci
->periodic
[frame
];
1565 ehci
->pshadow
[frame
].sitd
= sitd
;
1566 sitd
->frame
= frame
;
1568 ehci
->periodic
[frame
] = cpu_to_le32 (sitd
->sitd_dma
) | Q_TYPE_SITD
;
1571 /* fit urb's sitds into the selected schedule slot; activate as needed */
1574 struct ehci_hcd
*ehci
,
1577 struct ehci_iso_stream
*stream
1581 unsigned next_uframe
;
1582 struct ehci_iso_sched
*sched
= urb
->hcpriv
;
1583 struct ehci_sitd
*sitd
;
1585 next_uframe
= stream
->next_uframe
;
1587 if (list_empty(&stream
->td_list
)) {
1588 /* usbfs ignores TT bandwidth */
1589 hcd_to_bus (&ehci
->hcd
)->bandwidth_allocated
1590 += stream
->bandwidth
;
1592 "sched dev%s ep%d%s-iso [%d] %dms/%04x\n",
1593 urb
->dev
->devpath
, stream
->bEndpointAddress
& 0x0f,
1594 (stream
->bEndpointAddress
& USB_DIR_IN
) ? "in" : "out",
1595 (next_uframe
>> 3) % ehci
->periodic_size
,
1596 stream
->interval
, le32_to_cpu (stream
->splits
));
1597 stream
->start
= jiffies
;
1599 hcd_to_bus (&ehci
->hcd
)->bandwidth_isoc_reqs
++;
1601 /* fill sITDs frame by frame */
1602 for (packet
= 0, sitd
= NULL
;
1603 packet
< urb
->number_of_packets
;
1606 /* ASSERT: we have all necessary sitds */
1607 BUG_ON (list_empty (&sched
->td_list
));
1609 /* ASSERT: no itds for this endpoint in this frame */
1611 sitd
= list_entry (sched
->td_list
.next
,
1612 struct ehci_sitd
, sitd_list
);
1613 list_move_tail (&sitd
->sitd_list
, &stream
->td_list
);
1614 sitd
->stream
= iso_stream_get (stream
);
1615 sitd
->urb
= usb_get_urb (urb
);
1617 sitd_patch (stream
, sitd
, sched
, packet
);
1618 sitd_link (ehci
, (next_uframe
>> 3) % ehci
->periodic_size
,
1621 next_uframe
+= stream
->interval
<< 3;
1622 stream
->depth
+= stream
->interval
<< 3;
1624 stream
->next_uframe
= next_uframe
% mod
;
1626 /* don't need that schedule data any more */
1627 iso_sched_free (stream
, sched
);
1630 timer_action (ehci
, TIMER_IO_WATCHDOG
);
1631 if (!ehci
->periodic_sched
++)
1632 return enable_periodic (ehci
);
1636 /*-------------------------------------------------------------------------*/
1638 #define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
1639 | SITD_STS_XACT | SITD_STS_MMF | SITD_STS_STS)
1643 struct ehci_hcd
*ehci
,
1644 struct ehci_sitd
*sitd
,
1645 struct pt_regs
*regs
1647 struct urb
*urb
= sitd
->urb
;
1648 struct usb_iso_packet_descriptor
*desc
;
1651 struct ehci_iso_stream
*stream
= sitd
->stream
;
1652 struct usb_device
*dev
;
1654 urb_index
= sitd
->index
;
1655 desc
= &urb
->iso_frame_desc
[urb_index
];
1656 t
= le32_to_cpup (&sitd
->hw_results
);
1658 /* report transfer status */
1659 if (t
& SITD_ERRS
) {
1661 if (t
& SITD_STS_DBE
)
1662 desc
->status
= usb_pipein (urb
->pipe
)
1663 ? -ENOSR
/* hc couldn't read */
1664 : -ECOMM
; /* hc couldn't write */
1665 else if (t
& SITD_STS_BABBLE
)
1666 desc
->status
= -EOVERFLOW
;
1667 else /* XACT, MMF, etc */
1668 desc
->status
= -EPROTO
;
1671 desc
->actual_length
= desc
->length
- SITD_LENGTH (t
);
1676 sitd
->stream
= NULL
;
1677 list_move (&sitd
->sitd_list
, &stream
->free_list
);
1678 stream
->depth
-= stream
->interval
<< 3;
1679 iso_stream_put (ehci
, stream
);
1681 /* handle completion now? */
1682 if ((urb_index
+ 1) != urb
->number_of_packets
)
1685 /* ASSERT: it's really the last sitd for this urb
1686 list_for_each_entry (sitd, &stream->td_list, sitd_list)
1687 BUG_ON (sitd->urb == urb);
1690 /* give urb back to the driver */
1691 dev
= usb_get_dev (urb
->dev
);
1692 ehci_urb_done (ehci
, urb
, regs
);
1695 /* defer stopping schedule; completion can submit */
1696 ehci
->periodic_sched
--;
1697 if (!ehci
->periodic_sched
)
1698 (void) disable_periodic (ehci
);
1699 hcd_to_bus (&ehci
->hcd
)->bandwidth_isoc_reqs
--;
1701 if (list_empty (&stream
->td_list
)) {
1702 hcd_to_bus (&ehci
->hcd
)->bandwidth_allocated
1703 -= stream
->bandwidth
;
1705 "deschedule devp %s ep%d%s-iso\n",
1706 dev
->devpath
, stream
->bEndpointAddress
& 0x0f,
1707 (stream
->bEndpointAddress
& USB_DIR_IN
) ? "in" : "out");
1709 iso_stream_put (ehci
, stream
);
1716 static int sitd_submit (struct ehci_hcd
*ehci
, struct urb
*urb
, int mem_flags
)
1718 int status
= -EINVAL
;
1719 unsigned long flags
;
1720 struct ehci_iso_stream
*stream
;
1722 // FIXME remove when csplits behave
1723 if (usb_pipein(urb
->pipe
)) {
1724 ehci_dbg (ehci
, "no iso-IN split transactions yet\n");
1728 /* Get iso_stream head */
1729 stream
= iso_stream_find (ehci
, urb
);
1731 ehci_dbg (ehci
, "can't get iso stream\n");
1734 if (urb
->interval
!= stream
->interval
) {
1735 ehci_dbg (ehci
, "can't change iso interval %d --> %d\n",
1736 stream
->interval
, urb
->interval
);
1740 #ifdef EHCI_URB_TRACE
1742 "submit %p dev%s ep%d%s-iso len %d\n",
1743 urb
, urb
->dev
->devpath
,
1744 usb_pipeendpoint (urb
->pipe
),
1745 usb_pipein (urb
->pipe
) ? "in" : "out",
1746 urb
->transfer_buffer_length
);
1749 /* allocate SITDs */
1750 status
= sitd_urb_transaction (stream
, ehci
, urb
, mem_flags
);
1752 ehci_dbg (ehci
, "can't init sitds\n");
1756 /* schedule ... need to lock */
1757 spin_lock_irqsave (&ehci
->lock
, flags
);
1758 status
= iso_stream_schedule (ehci
, urb
, stream
);
1760 sitd_link_urb (ehci
, urb
, ehci
->periodic_size
<< 3, stream
);
1761 spin_unlock_irqrestore (&ehci
->lock
, flags
);
1765 iso_stream_put (ehci
, stream
);
1772 sitd_submit (struct ehci_hcd
*ehci
, struct urb
*urb
, int mem_flags
)
1774 ehci_dbg (ehci
, "split iso support is disabled\n");
1778 static inline unsigned
1780 struct ehci_hcd
*ehci
,
1781 struct ehci_sitd
*sitd
,
1782 struct pt_regs
*regs
1784 ehci_err (ehci
, "sitd_complete %p?\n", sitd
);
1788 #endif /* USB_EHCI_SPLIT_ISO */
1790 /*-------------------------------------------------------------------------*/
1793 scan_periodic (struct ehci_hcd
*ehci
, struct pt_regs
*regs
)
1795 unsigned frame
, clock
, now_uframe
, mod
;
1798 mod
= ehci
->periodic_size
<< 3;
1801 * When running, scan from last scan point up to "now"
1802 * else clean up by scanning everything that's left.
1803 * Touches as few pages as possible: cache-friendly.
1805 now_uframe
= ehci
->next_uframe
;
1806 if (HCD_IS_RUNNING (ehci
->hcd
.state
))
1807 clock
= readl (&ehci
->regs
->frame_index
);
1809 clock
= now_uframe
+ mod
- 1;
1813 union ehci_shadow q
, *q_p
;
1817 /* don't scan past the live uframe */
1818 frame
= now_uframe
>> 3;
1819 if (frame
== (clock
>> 3))
1820 uframes
= now_uframe
& 0x07;
1822 /* safe to scan the whole frame at once */
1828 /* scan each element in frame's queue for completions */
1829 q_p
= &ehci
->pshadow
[frame
];
1830 hw_p
= &ehci
->periodic
[frame
];
1832 type
= Q_NEXT_TYPE (*hw_p
);
1835 while (q
.ptr
!= 0) {
1837 union ehci_shadow temp
;
1841 /* handle any completions */
1842 temp
.qh
= qh_get (q
.qh
);
1843 type
= Q_NEXT_TYPE (q
.qh
->hw_next
);
1845 modified
= qh_completions (ehci
, temp
.qh
, regs
);
1846 if (unlikely (list_empty (&temp
.qh
->qtd_list
)))
1847 intr_deschedule (ehci
, temp
.qh
, 0);
1851 /* for "save place" FSTNs, look at QH entries
1852 * in the previous frame for completions.
1854 if (q
.fstn
->hw_prev
!= EHCI_LIST_END
) {
1855 dbg ("ignoring completions from FSTNs");
1857 type
= Q_NEXT_TYPE (q
.fstn
->hw_next
);
1858 q
= q
.fstn
->fstn_next
;
1861 /* skip itds for later in the frame */
1863 for (uf
= uframes
; uf
< 8; uf
++) {
1864 if (0 == (q
.itd
->hw_transaction
[uf
]
1867 q_p
= &q
.itd
->itd_next
;
1868 hw_p
= &q
.itd
->hw_next
;
1869 type
= Q_NEXT_TYPE (q
.itd
->hw_next
);
1876 /* this one's ready ... HC won't cache the
1877 * pointer for much longer, if at all.
1879 *q_p
= q
.itd
->itd_next
;
1880 *hw_p
= q
.itd
->hw_next
;
1881 type
= Q_NEXT_TYPE (q
.itd
->hw_next
);
1883 modified
= itd_complete (ehci
, q
.itd
, regs
);
1887 if (q
.sitd
->hw_results
& SITD_ACTIVE
) {
1888 q_p
= &q
.sitd
->sitd_next
;
1889 hw_p
= &q
.sitd
->hw_next
;
1890 type
= Q_NEXT_TYPE (q
.sitd
->hw_next
);
1894 *q_p
= q
.sitd
->sitd_next
;
1895 *hw_p
= q
.sitd
->hw_next
;
1896 type
= Q_NEXT_TYPE (q
.sitd
->hw_next
);
1898 modified
= sitd_complete (ehci
, q
.sitd
, regs
);
1902 dbg ("corrupt type %d frame %d shadow %p",
1903 type
, frame
, q
.ptr
);
1908 /* assume completion callbacks modify the queue */
1909 if (unlikely (modified
))
1913 /* stop when we catch up to the HC */
1915 // FIXME: this assumes we won't get lapped when
1916 // latencies climb; that should be rare, but...
1917 // detect it, and just go all the way around.
1918 // FLR might help detect this case, so long as latencies
1919 // don't exceed periodic_size msec (default 1.024 sec).
1921 // FIXME: likewise assumes HC doesn't halt mid-scan
1923 if (now_uframe
== clock
) {
1926 if (!HCD_IS_RUNNING (ehci
->hcd
.state
))
1928 ehci
->next_uframe
= now_uframe
;
1929 now
= readl (&ehci
->regs
->frame_index
) % mod
;
1930 if (now_uframe
== now
)
1933 /* rescan the rest of this frame, then ... */