?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
USB: EHCI: fix HUB TT scheduling issue with iso transfer
[linux-2.6/linux-acpi-2.6/ibm-acpi-2.6.git] / drivers / usb / host / ehci-sched.c
blobafef3df9757d4fbb40231cc96fc1956bbb3e7af8
1 /*
2 * Copyright (c) 2001-2004 by David Brownell
3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License as published by the
7 * Free Software Foundation; either version 2 of the License, or (at your
8 * option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 * for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software Foundation,
17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
18 */
19
20 /* this file is part of ehci-hcd.c */
21
22 /*-------------------------------------------------------------------------*/
23
24 /*
25 * EHCI scheduled transaction support: interrupt, iso, split iso
26 * These are called "periodic" transactions in the EHCI spec.
27 *
28 * Note that for interrupt transfers, the QH/QTD manipulation is shared
29 * with the "asynchronous" transaction support (control/bulk transfers).
30 * The only real difference is in how interrupt transfers are scheduled.
31 *
32 * For ISO, we make an "iso_stream" head to serve the same role as a QH.
33 * It keeps track of every ITD (or SITD) that's linked, and holds enough
34 * pre-calculated schedule data to make appending to the queue be quick.
35 */
36
37 static int ehci_get_frame (struct usb_hcd *hcd);
38
39 #ifdef CONFIG_PCI
40
41 static unsigned ehci_read_frame_index(struct ehci_hcd *ehci)
42 {
43 unsigned uf;
44
45 /*
46 * The MosChip MCS9990 controller updates its microframe counter
47 * a little before the frame counter, and occasionally we will read
48 * the invalid intermediate value. Avoid problems by checking the
49 * microframe number (the low-order 3 bits); if they are 0 then
50 * re-read the register to get the correct value.
51 */
52 uf = ehci_readl(ehci, &ehci->regs->frame_index);
53 if (unlikely(ehci->frame_index_bug && ((uf & 7) == 0)))
54 uf = ehci_readl(ehci, &ehci->regs->frame_index);
55 return uf;
56 }
57
58 #endif
59
60 /*-------------------------------------------------------------------------*/
61
62 /*
63 * periodic_next_shadow - return "next" pointer on shadow list
64 * @periodic: host pointer to qh/itd/sitd
65 * @tag: hardware tag for type of this record
66 */
67 static union ehci_shadow *
68 periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic,
69 __hc32 tag)
70 {
71 switch (hc32_to_cpu(ehci, tag)) {
72 case Q_TYPE_QH:
73 return &periodic->qh->qh_next;
74 case Q_TYPE_FSTN:
75 return &periodic->fstn->fstn_next;
76 case Q_TYPE_ITD:
77 return &periodic->itd->itd_next;
78 // case Q_TYPE_SITD:
79 default:
80 return &periodic->sitd->sitd_next;
81 }
82 }
83
84 static __hc32 *
85 shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic,
86 __hc32 tag)
87 {
88 switch (hc32_to_cpu(ehci, tag)) {
89 /* our ehci_shadow.qh is actually software part */
90 case Q_TYPE_QH:
91 return &periodic->qh->hw->hw_next;
92 /* others are hw parts */
93 default:
94 return periodic->hw_next;
95 }
96 }
97
98 /* caller must hold ehci->lock */
99 static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr)
100 {
101 union ehci_shadow *prev_p = &ehci->pshadow[frame];
102 __hc32 *hw_p = &ehci->periodic[frame];
103 union ehci_shadow here = *prev_p;
104
105 /* find predecessor of "ptr"; hw and shadow lists are in sync */
106 while (here.ptr && here.ptr != ptr) {
107 prev_p = periodic_next_shadow(ehci, prev_p,
108 Q_NEXT_TYPE(ehci, *hw_p));
109 hw_p = shadow_next_periodic(ehci, &here,
110 Q_NEXT_TYPE(ehci, *hw_p));
111 here = *prev_p;
112 }
113 /* an interrupt entry (at list end) could have been shared */
114 if (!here.ptr)
115 return;
116
117 /* update shadow and hardware lists ... the old "next" pointers
118 * from ptr may still be in use, the caller updates them.
119 */
120 *prev_p = *periodic_next_shadow(ehci, &here,
121 Q_NEXT_TYPE(ehci, *hw_p));
122
123 if (!ehci->use_dummy_qh ||
124 *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p))
125 != EHCI_LIST_END(ehci))
126 *hw_p = *shadow_next_periodic(ehci, &here,
127 Q_NEXT_TYPE(ehci, *hw_p));
128 else
129 *hw_p = ehci->dummy->qh_dma;
130 }
131
132 /* how many of the uframe's 125 usecs are allocated? */
133 static unsigned short
134 periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe)
135 {
136 __hc32 *hw_p = &ehci->periodic [frame];
137 union ehci_shadow *q = &ehci->pshadow [frame];
138 unsigned usecs = 0;
139 struct ehci_qh_hw *hw;
140
141 while (q->ptr) {
142 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
143 case Q_TYPE_QH:
144 hw = q->qh->hw;
145 /* is it in the S-mask? */
146 if (hw->hw_info2 & cpu_to_hc32(ehci, 1 << uframe))
147 usecs += q->qh->usecs;
148 /* ... or C-mask? */
149 if (hw->hw_info2 & cpu_to_hc32(ehci,
150 1 << (8 + uframe)))
151 usecs += q->qh->c_usecs;
152 hw_p = &hw->hw_next;
153 q = &q->qh->qh_next;
154 break;
155 // case Q_TYPE_FSTN:
156 default:
157 /* for "save place" FSTNs, count the relevant INTR
158 * bandwidth from the previous frame
159 */
160 if (q->fstn->hw_prev != EHCI_LIST_END(ehci)) {
161 ehci_dbg (ehci, "ignoring FSTN cost ...\n");
162 }
163 hw_p = &q->fstn->hw_next;
164 q = &q->fstn->fstn_next;
165 break;
166 case Q_TYPE_ITD:
167 if (q->itd->hw_transaction[uframe])
168 usecs += q->itd->stream->usecs;
169 hw_p = &q->itd->hw_next;
170 q = &q->itd->itd_next;
171 break;
172 case Q_TYPE_SITD:
173 /* is it in the S-mask? (count SPLIT, DATA) */
174 if (q->sitd->hw_uframe & cpu_to_hc32(ehci,
175 1 << uframe)) {
176 if (q->sitd->hw_fullspeed_ep &
177 cpu_to_hc32(ehci, 1<<31))
178 usecs += q->sitd->stream->usecs;
179 else /* worst case for OUT start-split */
180 usecs += HS_USECS_ISO (188);
181 }
182
183 /* ... C-mask? (count CSPLIT, DATA) */
184 if (q->sitd->hw_uframe &
185 cpu_to_hc32(ehci, 1 << (8 + uframe))) {
186 /* worst case for IN complete-split */
187 usecs += q->sitd->stream->c_usecs;
188 }
189
190 hw_p = &q->sitd->hw_next;
191 q = &q->sitd->sitd_next;
192 break;
193 }
194 }
195 #ifdef DEBUG
196 if (usecs > ehci->uframe_periodic_max)
197 ehci_err (ehci, "uframe %d sched overrun: %d usecs\n",
198 frame * 8 + uframe, usecs);
199 #endif
200 return usecs;
201 }
202
203 /*-------------------------------------------------------------------------*/
204
205 static int same_tt (struct usb_device *dev1, struct usb_device *dev2)
206 {
207 if (!dev1->tt || !dev2->tt)
208 return 0;
209 if (dev1->tt != dev2->tt)
210 return 0;
211 if (dev1->tt->multi)
212 return dev1->ttport == dev2->ttport;
213 else
214 return 1;
215 }
216
217 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
218
219 /* Which uframe does the low/fullspeed transfer start in?
220 *
221 * The parameter is the mask of ssplits in "H-frame" terms
222 * and this returns the transfer start uframe in "B-frame" terms,
223 * which allows both to match, e.g. a ssplit in "H-frame" uframe 0
224 * will cause a transfer in "B-frame" uframe 0. "B-frames" lag
225 * "H-frames" by 1 uframe. See the EHCI spec sec 4.5 and figure 4.7.
226 */
227 static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask)
228 {
229 unsigned char smask = QH_SMASK & hc32_to_cpu(ehci, mask);
230 if (!smask) {
231 ehci_err(ehci, "invalid empty smask!\n");
232 /* uframe 7 can't have bw so this will indicate failure */
233 return 7;
234 }
235 return ffs(smask) - 1;
236 }
237
238 static const unsigned char
239 max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
240
241 /* carryover low/fullspeed bandwidth that crosses uframe boundries */
242 static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8])
243 {
244 int i;
245 for (i=0; i<7; i++) {
246 if (max_tt_usecs[i] < tt_usecs[i]) {
247 tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i];
248 tt_usecs[i] = max_tt_usecs[i];
249 }
250 }
251 }
252
253 /* How many of the tt's periodic downstream 1000 usecs are allocated?
254 *
255 * While this measures the bandwidth in terms of usecs/uframe,
256 * the low/fullspeed bus has no notion of uframes, so any particular
257 * low/fullspeed transfer can "carry over" from one uframe to the next,
258 * since the TT just performs downstream transfers in sequence.
259 *
260 * For example two separate 100 usec transfers can start in the same uframe,
261 * and the second one would "carry over" 75 usecs into the next uframe.
262 */
263 static void
264 periodic_tt_usecs (
265 struct ehci_hcd *ehci,
266 struct usb_device *dev,
267 unsigned frame,
268 unsigned short tt_usecs[8]
269 )
270 {
271 __hc32 *hw_p = &ehci->periodic [frame];
272 union ehci_shadow *q = &ehci->pshadow [frame];
273 unsigned char uf;
274
275 memset(tt_usecs, 0, 16);
276
277 while (q->ptr) {
278 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
279 case Q_TYPE_ITD:
280 hw_p = &q->itd->hw_next;
281 q = &q->itd->itd_next;
282 continue;
283 case Q_TYPE_QH:
284 if (same_tt(dev, q->qh->dev)) {
285 uf = tt_start_uframe(ehci, q->qh->hw->hw_info2);
286 tt_usecs[uf] += q->qh->tt_usecs;
287 }
288 hw_p = &q->qh->hw->hw_next;
289 q = &q->qh->qh_next;
290 continue;
291 case Q_TYPE_SITD:
292 if (same_tt(dev, q->sitd->urb->dev)) {
293 uf = tt_start_uframe(ehci, q->sitd->hw_uframe);
294 tt_usecs[uf] += q->sitd->stream->tt_usecs;
295 }
296 hw_p = &q->sitd->hw_next;
297 q = &q->sitd->sitd_next;
298 continue;
299 // case Q_TYPE_FSTN:
300 default:
301 ehci_dbg(ehci, "ignoring periodic frame %d FSTN\n",
302 frame);
303 hw_p = &q->fstn->hw_next;
304 q = &q->fstn->fstn_next;
305 }
306 }
307
308 carryover_tt_bandwidth(tt_usecs);
309
310 if (max_tt_usecs[7] < tt_usecs[7])
311 ehci_err(ehci, "frame %d tt sched overrun: %d usecs\n",
312 frame, tt_usecs[7] - max_tt_usecs[7]);
313 }
314
315 /*
316 * Return true if the device's tt's downstream bus is available for a
317 * periodic transfer of the specified length (usecs), starting at the
318 * specified frame/uframe. Note that (as summarized in section 11.19
319 * of the usb 2.0 spec) TTs can buffer multiple transactions for each
320 * uframe.
321 *
322 * The uframe parameter is when the fullspeed/lowspeed transfer
323 * should be executed in "B-frame" terms, which is the same as the
324 * highspeed ssplit's uframe (which is in "H-frame" terms). For example
325 * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0.
326 * See the EHCI spec sec 4.5 and fig 4.7.
327 *
328 * This checks if the full/lowspeed bus, at the specified starting uframe,
329 * has the specified bandwidth available, according to rules listed
330 * in USB 2.0 spec section 11.18.1 fig 11-60.
331 *
332 * This does not check if the transfer would exceed the max ssplit
333 * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4,
334 * since proper scheduling limits ssplits to less than 16 per uframe.
335 */
336 static int tt_available (
337 struct ehci_hcd *ehci,
338 unsigned period,
339 struct usb_device *dev,
340 unsigned frame,
341 unsigned uframe,
342 u16 usecs
343 )
344 {
345 if ((period == 0) || (uframe >= 7)) /* error */
346 return 0;
347
348 for (; frame < ehci->periodic_size; frame += period) {
349 unsigned short tt_usecs[8];
350
351 periodic_tt_usecs (ehci, dev, frame, tt_usecs);
352
353 ehci_vdbg(ehci, "tt frame %d check %d usecs start uframe %d in"
354 " schedule %d/%d/%d/%d/%d/%d/%d/%d\n",
355 frame, usecs, uframe,
356 tt_usecs[0], tt_usecs[1], tt_usecs[2], tt_usecs[3],
357 tt_usecs[4], tt_usecs[5], tt_usecs[6], tt_usecs[7]);
358
359 if (max_tt_usecs[uframe] <= tt_usecs[uframe]) {
360 ehci_vdbg(ehci, "frame %d uframe %d fully scheduled\n",
361 frame, uframe);
362 return 0;
363 }
364
365 /* special case for isoc transfers larger than 125us:
366 * the first and each subsequent fully used uframe
367 * must be empty, so as to not illegally delay
368 * already scheduled transactions
369 */
370 if (125 < usecs) {
371 int ufs = (usecs / 125);
372 int i;
373 for (i = uframe; i < (uframe + ufs) && i < 8; i++)
374 if (0 < tt_usecs[i]) {
375 ehci_vdbg(ehci,
376 "multi-uframe xfer can't fit "
377 "in frame %d uframe %d\n",
378 frame, i);
379 return 0;
380 }
381 }
382
383 tt_usecs[uframe] += usecs;
384
385 carryover_tt_bandwidth(tt_usecs);
386
387 /* fail if the carryover pushed bw past the last uframe's limit */
388 if (max_tt_usecs[7] < tt_usecs[7]) {
389 ehci_vdbg(ehci,
390 "tt unavailable usecs %d frame %d uframe %d\n",
391 usecs, frame, uframe);
392 return 0;
393 }
394 }
395
396 return 1;
397 }
398
399 #else
400
401 /* return true iff the device's transaction translator is available
402 * for a periodic transfer starting at the specified frame, using
403 * all the uframes in the mask.
404 */
405 static int tt_no_collision (
406 struct ehci_hcd *ehci,
407 unsigned period,
408 struct usb_device *dev,
409 unsigned frame,
410 u32 uf_mask
411 )
412 {
413 if (period == 0) /* error */
414 return 0;
415
416 /* note bandwidth wastage: split never follows csplit
417 * (different dev or endpoint) until the next uframe.
418 * calling convention doesn't make that distinction.
419 */
420 for (; frame < ehci->periodic_size; frame += period) {
421 union ehci_shadow here;
422 __hc32 type;
423 struct ehci_qh_hw *hw;
424
425 here = ehci->pshadow [frame];
426 type = Q_NEXT_TYPE(ehci, ehci->periodic [frame]);
427 while (here.ptr) {
428 switch (hc32_to_cpu(ehci, type)) {
429 case Q_TYPE_ITD:
430 type = Q_NEXT_TYPE(ehci, here.itd->hw_next);
431 here = here.itd->itd_next;
432 continue;
433 case Q_TYPE_QH:
434 hw = here.qh->hw;
435 if (same_tt (dev, here.qh->dev)) {
436 u32 mask;
437
438 mask = hc32_to_cpu(ehci,
439 hw->hw_info2);
440 /* "knows" no gap is needed */
441 mask |= mask >> 8;
442 if (mask & uf_mask)
443 break;
444 }
445 type = Q_NEXT_TYPE(ehci, hw->hw_next);
446 here = here.qh->qh_next;
447 continue;
448 case Q_TYPE_SITD:
449 if (same_tt (dev, here.sitd->urb->dev)) {
450 u16 mask;
451
452 mask = hc32_to_cpu(ehci, here.sitd
453 ->hw_uframe);
454 /* FIXME assumes no gap for IN! */
455 mask |= mask >> 8;
456 if (mask & uf_mask)
457 break;
458 }
459 type = Q_NEXT_TYPE(ehci, here.sitd->hw_next);
460 here = here.sitd->sitd_next;
461 continue;
462 // case Q_TYPE_FSTN:
463 default:
464 ehci_dbg (ehci,
465 "periodic frame %d bogus type %d\n",
466 frame, type);
467 }
468
469 /* collision or error */
470 return 0;
471 }
472 }
473
474 /* no collision */
475 return 1;
476 }
477
478 #endif /* CONFIG_USB_EHCI_TT_NEWSCHED */
479
480 /*-------------------------------------------------------------------------*/
481
482 static int enable_periodic (struct ehci_hcd *ehci)
483 {
484 u32 cmd;
485 int status;
486
487 if (ehci->periodic_sched++)
488 return 0;
489
490 /* did clearing PSE did take effect yet?
491 * takes effect only at frame boundaries...
492 */
493 status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
494 STS_PSS, 0, 9 * 125);
495 if (status) {
496 usb_hc_died(ehci_to_hcd(ehci));
497 return status;
498 }
499
500 cmd = ehci_readl(ehci, &ehci->regs->command) | CMD_PSE;
501 ehci_writel(ehci, cmd, &ehci->regs->command);
502 /* posted write ... PSS happens later */
503 ehci_to_hcd(ehci)->state = HC_STATE_RUNNING;
504
505 /* make sure ehci_work scans these */
506 ehci->next_uframe = ehci_read_frame_index(ehci)
507 % (ehci->periodic_size << 3);
508 if (unlikely(ehci->broken_periodic))
509 ehci->last_periodic_enable = ktime_get_real();
510 return 0;
511 }
512
513 static int disable_periodic (struct ehci_hcd *ehci)
514 {
515 u32 cmd;
516 int status;
517
518 if (--ehci->periodic_sched)
519 return 0;
520
521 if (unlikely(ehci->broken_periodic)) {
522 /* delay experimentally determined */
523 ktime_t safe = ktime_add_us(ehci->last_periodic_enable, 1000);
524 ktime_t now = ktime_get_real();
525 s64 delay = ktime_us_delta(safe, now);
526
527 if (unlikely(delay > 0))
528 udelay(delay);
529 }
530
531 /* did setting PSE not take effect yet?
532 * takes effect only at frame boundaries...
533 */
534 status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
535 STS_PSS, STS_PSS, 9 * 125);
536 if (status) {
537 usb_hc_died(ehci_to_hcd(ehci));
538 return status;
539 }
540
541 cmd = ehci_readl(ehci, &ehci->regs->command) & ~CMD_PSE;
542 ehci_writel(ehci, cmd, &ehci->regs->command);
543 /* posted write ... */
544
545 free_cached_lists(ehci);
546
547 ehci->next_uframe = -1;
548 return 0;
549 }
550
551 /*-------------------------------------------------------------------------*/
552
553 /* periodic schedule slots have iso tds (normal or split) first, then a
554 * sparse tree for active interrupt transfers.
555 *
556 * this just links in a qh; caller guarantees uframe masks are set right.
557 * no FSTN support (yet; ehci 0.96+)
558 */
559 static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh)
560 {
561 unsigned i;
562 unsigned period = qh->period;
563
564 dev_dbg (&qh->dev->dev,
565 "link qh%d-%04x/%p start %d [%d/%d us]\n",
566 period, hc32_to_cpup(ehci, &qh->hw->hw_info2)
567 & (QH_CMASK | QH_SMASK),
568 qh, qh->start, qh->usecs, qh->c_usecs);
569
570 /* high bandwidth, or otherwise every microframe */
571 if (period == 0)
572 period = 1;
573
574 for (i = qh->start; i < ehci->periodic_size; i += period) {
575 union ehci_shadow *prev = &ehci->pshadow[i];
576 __hc32 *hw_p = &ehci->periodic[i];
577 union ehci_shadow here = *prev;
578 __hc32 type = 0;
579
580 /* skip the iso nodes at list head */
581 while (here.ptr) {
582 type = Q_NEXT_TYPE(ehci, *hw_p);
583 if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
584 break;
585 prev = periodic_next_shadow(ehci, prev, type);
586 hw_p = shadow_next_periodic(ehci, &here, type);
587 here = *prev;
588 }
589
590 /* sorting each branch by period (slow-->fast)
591 * enables sharing interior tree nodes
592 */
593 while (here.ptr && qh != here.qh) {
594 if (qh->period > here.qh->period)
595 break;
596 prev = &here.qh->qh_next;
597 hw_p = &here.qh->hw->hw_next;
598 here = *prev;
599 }
600 /* link in this qh, unless some earlier pass did that */
601 if (qh != here.qh) {
602 qh->qh_next = here;
603 if (here.qh)
604 qh->hw->hw_next = *hw_p;
605 wmb ();
606 prev->qh = qh;
607 *hw_p = QH_NEXT (ehci, qh->qh_dma);
608 }
609 }
610 qh->qh_state = QH_STATE_LINKED;
611 qh->xacterrs = 0;
612 qh_get (qh);
613
614 /* update per-qh bandwidth for usbfs */
615 ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period
616 ? ((qh->usecs + qh->c_usecs) / qh->period)
617 : (qh->usecs * 8);
618
619 /* maybe enable periodic schedule processing */
620 return enable_periodic(ehci);
621 }
622
623 static int qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
624 {
625 unsigned i;
626 unsigned period;
627
628 // FIXME:
629 // IF this isn't high speed
630 // and this qh is active in the current uframe
631 // (and overlay token SplitXstate is false?)
632 // THEN
633 // qh->hw_info1 |= cpu_to_hc32(1 << 7 /* "ignore" */);
634
635 /* high bandwidth, or otherwise part of every microframe */
636 if ((period = qh->period) == 0)
637 period = 1;
638
639 for (i = qh->start; i < ehci->periodic_size; i += period)
640 periodic_unlink (ehci, i, qh);
641
642 /* update per-qh bandwidth for usbfs */
643 ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period
644 ? ((qh->usecs + qh->c_usecs) / qh->period)
645 : (qh->usecs * 8);
646
647 dev_dbg (&qh->dev->dev,
648 "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
649 qh->period,
650 hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK),
651 qh, qh->start, qh->usecs, qh->c_usecs);
652
653 /* qh->qh_next still "live" to HC */
654 qh->qh_state = QH_STATE_UNLINK;
655 qh->qh_next.ptr = NULL;
656 qh_put (qh);
657
658 /* maybe turn off periodic schedule */
659 return disable_periodic(ehci);
660 }
661
662 static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh)
663 {
664 unsigned wait;
665 struct ehci_qh_hw *hw = qh->hw;
666 int rc;
667
668 /* If the QH isn't linked then there's nothing we can do
669 * unless we were called during a giveback, in which case
670 * qh_completions() has to deal with it.
671 */
672 if (qh->qh_state != QH_STATE_LINKED) {
673 if (qh->qh_state == QH_STATE_COMPLETING)
674 qh->needs_rescan = 1;
675 return;
676 }
677
678 qh_unlink_periodic (ehci, qh);
679
680 /* simple/paranoid: always delay, expecting the HC needs to read
681 * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and
682 * expect khubd to clean up after any CSPLITs we won't issue.
683 * active high speed queues may need bigger delays...
684 */
685 if (list_empty (&qh->qtd_list)
686 || (cpu_to_hc32(ehci, QH_CMASK)
687 & hw->hw_info2) != 0)
688 wait = 2;
689 else
690 wait = 55; /* worst case: 3 * 1024 */
691
692 udelay (wait);
693 qh->qh_state = QH_STATE_IDLE;
694 hw->hw_next = EHCI_LIST_END(ehci);
695 wmb ();
696
697 qh_completions(ehci, qh);
698
699 /* reschedule QH iff another request is queued */
700 if (!list_empty(&qh->qtd_list) &&
701 HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
702 rc = qh_schedule(ehci, qh);
703
704 /* An error here likely indicates handshake failure
705 * or no space left in the schedule. Neither fault
706 * should happen often ...
707 *
708 * FIXME kill the now-dysfunctional queued urbs
709 */
710 if (rc != 0)
711 ehci_err(ehci, "can't reschedule qh %p, err %d\n",
712 qh, rc);
713 }
714 }
715
716 /*-------------------------------------------------------------------------*/
717
718 static int check_period (
719 struct ehci_hcd *ehci,
720 unsigned frame,
721 unsigned uframe,
722 unsigned period,
723 unsigned usecs
724 ) {
725 int claimed;
726
727 /* complete split running into next frame?
728 * given FSTN support, we could sometimes check...
729 */
730 if (uframe >= 8)
731 return 0;
732
733 /* convert "usecs we need" to "max already claimed" */
734 usecs = ehci->uframe_periodic_max - usecs;
735
736 /* we "know" 2 and 4 uframe intervals were rejected; so
737 * for period 0, check _every_ microframe in the schedule.
738 */
739 if (unlikely (period == 0)) {
740 do {
741 for (uframe = 0; uframe < 7; uframe++) {
742 claimed = periodic_usecs (ehci, frame, uframe);
743 if (claimed > usecs)
744 return 0;
745 }
746 } while ((frame += 1) < ehci->periodic_size);
747
748 /* just check the specified uframe, at that period */
749 } else {
750 do {
751 claimed = periodic_usecs (ehci, frame, uframe);
752 if (claimed > usecs)
753 return 0;
754 } while ((frame += period) < ehci->periodic_size);
755 }
756
757 // success!
758 return 1;
759 }
760
761 static int check_intr_schedule (
762 struct ehci_hcd *ehci,
763 unsigned frame,
764 unsigned uframe,
765 const struct ehci_qh *qh,
766 __hc32 *c_maskp
767 )
768 {
769 int retval = -ENOSPC;
770 u8 mask = 0;
771
772 if (qh->c_usecs && uframe >= 6) /* FSTN territory? */
773 goto done;
774
775 if (!check_period (ehci, frame, uframe, qh->period, qh->usecs))
776 goto done;
777 if (!qh->c_usecs) {
778 retval = 0;
779 *c_maskp = 0;
780 goto done;
781 }
782
783 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
784 if (tt_available (ehci, qh->period, qh->dev, frame, uframe,
785 qh->tt_usecs)) {
786 unsigned i;
787
788 /* TODO : this may need FSTN for SSPLIT in uframe 5. */
789 for (i=uframe+1; i<8 && i<uframe+4; i++)
790 if (!check_period (ehci, frame, i,
791 qh->period, qh->c_usecs))
792 goto done;
793 else
794 mask |= 1 << i;
795
796 retval = 0;
797
798 *c_maskp = cpu_to_hc32(ehci, mask << 8);
799 }
800 #else
801 /* Make sure this tt's buffer is also available for CSPLITs.
802 * We pessimize a bit; probably the typical full speed case
803 * doesn't need the second CSPLIT.
804 *
805 * NOTE: both SPLIT and CSPLIT could be checked in just
806 * one smart pass...
807 */
808 mask = 0x03 << (uframe + qh->gap_uf);
809 *c_maskp = cpu_to_hc32(ehci, mask << 8);
810
811 mask |= 1 << uframe;
812 if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) {
813 if (!check_period (ehci, frame, uframe + qh->gap_uf + 1,
814 qh->period, qh->c_usecs))
815 goto done;
816 if (!check_period (ehci, frame, uframe + qh->gap_uf,
817 qh->period, qh->c_usecs))
818 goto done;
819 retval = 0;
820 }
821 #endif
822 done:
823 return retval;
824 }
825
826 /* "first fit" scheduling policy used the first time through,
827 * or when the previous schedule slot can't be re-used.
828 */
829 static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh)
830 {
831 int status;
832 unsigned uframe;
833 __hc32 c_mask;
834 unsigned frame; /* 0..(qh->period - 1), or NO_FRAME */
835 struct ehci_qh_hw *hw = qh->hw;
836
837 qh_refresh(ehci, qh);
838 hw->hw_next = EHCI_LIST_END(ehci);
839 frame = qh->start;
840
841 /* reuse the previous schedule slots, if we can */
842 if (frame < qh->period) {
843 uframe = ffs(hc32_to_cpup(ehci, &hw->hw_info2) & QH_SMASK);
844 status = check_intr_schedule (ehci, frame, --uframe,
845 qh, &c_mask);
846 } else {
847 uframe = 0;
848 c_mask = 0;
849 status = -ENOSPC;
850 }
851
852 /* else scan the schedule to find a group of slots such that all
853 * uframes have enough periodic bandwidth available.
854 */
855 if (status) {
856 /* "normal" case, uframing flexible except with splits */
857 if (qh->period) {
858 int i;
859
860 for (i = qh->period; status && i > 0; --i) {
861 frame = ++ehci->random_frame % qh->period;
862 for (uframe = 0; uframe < 8; uframe++) {
863 status = check_intr_schedule (ehci,
864 frame, uframe, qh,
865 &c_mask);
866 if (status == 0)
867 break;
868 }
869 }
870
871 /* qh->period == 0 means every uframe */
872 } else {
873 frame = 0;
874 status = check_intr_schedule (ehci, 0, 0, qh, &c_mask);
875 }
876 if (status)
877 goto done;
878 qh->start = frame;
879
880 /* reset S-frame and (maybe) C-frame masks */
881 hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK));
882 hw->hw_info2 |= qh->period
883 ? cpu_to_hc32(ehci, 1 << uframe)
884 : cpu_to_hc32(ehci, QH_SMASK);
885 hw->hw_info2 |= c_mask;
886 } else
887 ehci_dbg (ehci, "reused qh %p schedule\n", qh);
888
889 /* stuff into the periodic schedule */
890 status = qh_link_periodic (ehci, qh);
891 done:
892 return status;
893 }
894
895 static int intr_submit (
896 struct ehci_hcd *ehci,
897 struct urb *urb,
898 struct list_head *qtd_list,
899 gfp_t mem_flags
900 ) {
901 unsigned epnum;
902 unsigned long flags;
903 struct ehci_qh *qh;
904 int status;
905 struct list_head empty;
906
907 /* get endpoint and transfer/schedule data */
908 epnum = urb->ep->desc.bEndpointAddress;
909
910 spin_lock_irqsave (&ehci->lock, flags);
911
912 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
913 status = -ESHUTDOWN;
914 goto done_not_linked;
915 }
916 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
917 if (unlikely(status))
918 goto done_not_linked;
919
920 /* get qh and force any scheduling errors */
921 INIT_LIST_HEAD (&empty);
922 qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv);
923 if (qh == NULL) {
924 status = -ENOMEM;
925 goto done;
926 }
927 if (qh->qh_state == QH_STATE_IDLE) {
928 if ((status = qh_schedule (ehci, qh)) != 0)
929 goto done;
930 }
931
932 /* then queue the urb's tds to the qh */
933 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
934 BUG_ON (qh == NULL);
935
936 /* ... update usbfs periodic stats */
937 ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
938
939 done:
940 if (unlikely(status))
941 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
942 done_not_linked:
943 spin_unlock_irqrestore (&ehci->lock, flags);
944 if (status)
945 qtd_list_free (ehci, urb, qtd_list);
946
947 return status;
948 }
949
950 /*-------------------------------------------------------------------------*/
951
952 /* ehci_iso_stream ops work with both ITD and SITD */
953
954 static struct ehci_iso_stream *
955 iso_stream_alloc (gfp_t mem_flags)
956 {
957 struct ehci_iso_stream *stream;
958
959 stream = kzalloc(sizeof *stream, mem_flags);
960 if (likely (stream != NULL)) {
961 INIT_LIST_HEAD(&stream->td_list);
962 INIT_LIST_HEAD(&stream->free_list);
963 stream->next_uframe = -1;
964 stream->refcount = 1;
965 }
966 return stream;
967 }
968
969 static void
970 iso_stream_init (
971 struct ehci_hcd *ehci,
972 struct ehci_iso_stream *stream,
973 struct usb_device *dev,
974 int pipe,
975 unsigned interval
976 )
977 {
978 static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
979
980 u32 buf1;
981 unsigned epnum, maxp;
982 int is_input;
983 long bandwidth;
984
985 /*
986 * this might be a "high bandwidth" highspeed endpoint,
987 * as encoded in the ep descriptor's wMaxPacket field
988 */
989 epnum = usb_pipeendpoint (pipe);
990 is_input = usb_pipein (pipe) ? USB_DIR_IN : 0;
991 maxp = usb_maxpacket(dev, pipe, !is_input);
992 if (is_input) {
993 buf1 = (1 << 11);
994 } else {
995 buf1 = 0;
996 }
997
998 /* knows about ITD vs SITD */
999 if (dev->speed == USB_SPEED_HIGH) {
1000 unsigned multi = hb_mult(maxp);
1001
1002 stream->highspeed = 1;
1003
1004 maxp = max_packet(maxp);
1005 buf1 |= maxp;
1006 maxp *= multi;
1007
1008 stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum);
1009 stream->buf1 = cpu_to_hc32(ehci, buf1);
1010 stream->buf2 = cpu_to_hc32(ehci, multi);
1011
1012 /* usbfs wants to report the average usecs per frame tied up
1013 * when transfers on this endpoint are scheduled ...
1014 */
1015 stream->usecs = HS_USECS_ISO (maxp);
1016 bandwidth = stream->usecs * 8;
1017 bandwidth /= interval;
1018
1019 } else {
1020 u32 addr;
1021 int think_time;
1022 int hs_transfers;
1023
1024 addr = dev->ttport << 24;
1025 if (!ehci_is_TDI(ehci)
1026 || (dev->tt->hub !=
1027 ehci_to_hcd(ehci)->self.root_hub))
1028 addr |= dev->tt->hub->devnum << 16;
1029 addr |= epnum << 8;
1030 addr |= dev->devnum;
1031 stream->usecs = HS_USECS_ISO (maxp);
1032 think_time = dev->tt ? dev->tt->think_time : 0;
1033 stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time (
1034 dev->speed, is_input, 1, maxp));
1035 hs_transfers = max (1u, (maxp + 187) / 188);
1036 if (is_input) {
1037 u32 tmp;
1038
1039 addr |= 1 << 31;
1040 stream->c_usecs = stream->usecs;
1041 stream->usecs = HS_USECS_ISO (1);
1042 stream->raw_mask = 1;
1043
1044 /* c-mask as specified in USB 2.0 11.18.4 3.c */
1045 tmp = (1 << (hs_transfers + 2)) - 1;
1046 stream->raw_mask |= tmp << (8 + 2);
1047 } else
1048 stream->raw_mask = smask_out [hs_transfers - 1];
1049 bandwidth = stream->usecs + stream->c_usecs;
1050 bandwidth /= interval << 3;
1051
1052 /* stream->splits gets created from raw_mask later */
1053 stream->address = cpu_to_hc32(ehci, addr);
1054 }
1055 stream->bandwidth = bandwidth;
1056
1057 stream->udev = dev;
1058
1059 stream->bEndpointAddress = is_input | epnum;
1060 stream->interval = interval;
1061 stream->maxp = maxp;
1062 }
1063
1064 static void
1065 iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream)
1066 {
1067 stream->refcount--;
1068
1069 /* free whenever just a dev->ep reference remains.
1070 * not like a QH -- no persistent state (toggle, halt)
1071 */
1072 if (stream->refcount == 1) {
1073 // BUG_ON (!list_empty(&stream->td_list));
1074
1075 while (!list_empty (&stream->free_list)) {
1076 struct list_head *entry;
1077
1078 entry = stream->free_list.next;
1079 list_del (entry);
1080
1081 /* knows about ITD vs SITD */
1082 if (stream->highspeed) {
1083 struct ehci_itd *itd;
1084
1085 itd = list_entry (entry, struct ehci_itd,
1086 itd_list);
1087 dma_pool_free (ehci->itd_pool, itd,
1088 itd->itd_dma);
1089 } else {
1090 struct ehci_sitd *sitd;
1091
1092 sitd = list_entry (entry, struct ehci_sitd,
1093 sitd_list);
1094 dma_pool_free (ehci->sitd_pool, sitd,
1095 sitd->sitd_dma);
1096 }
1097 }
1098
1099 stream->bEndpointAddress &= 0x0f;
1100 if (stream->ep)
1101 stream->ep->hcpriv = NULL;
1102
1103 kfree(stream);
1104 }
1105 }
1106
1107 static inline struct ehci_iso_stream *
1108 iso_stream_get (struct ehci_iso_stream *stream)
1109 {
1110 if (likely (stream != NULL))
1111 stream->refcount++;
1112 return stream;
1113 }
1114
1115 static struct ehci_iso_stream *
1116 iso_stream_find (struct ehci_hcd *ehci, struct urb *urb)
1117 {
1118 unsigned epnum;
1119 struct ehci_iso_stream *stream;
1120 struct usb_host_endpoint *ep;
1121 unsigned long flags;
1122
1123 epnum = usb_pipeendpoint (urb->pipe);
1124 if (usb_pipein(urb->pipe))
1125 ep = urb->dev->ep_in[epnum];
1126 else
1127 ep = urb->dev->ep_out[epnum];
1128
1129 spin_lock_irqsave (&ehci->lock, flags);
1130 stream = ep->hcpriv;
1131
1132 if (unlikely (stream == NULL)) {
1133 stream = iso_stream_alloc(GFP_ATOMIC);
1134 if (likely (stream != NULL)) {
1135 /* dev->ep owns the initial refcount */
1136 ep->hcpriv = stream;
1137 stream->ep = ep;
1138 iso_stream_init(ehci, stream, urb->dev, urb->pipe,
1139 urb->interval);
1140 }
1141
1142 /* if dev->ep [epnum] is a QH, hw is set */
1143 } else if (unlikely (stream->hw != NULL)) {
1144 ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n",
1145 urb->dev->devpath, epnum,
1146 usb_pipein(urb->pipe) ? "in" : "out");
1147 stream = NULL;
1148 }
1149
1150 /* caller guarantees an eventual matching iso_stream_put */
1151 stream = iso_stream_get (stream);
1152
1153 spin_unlock_irqrestore (&ehci->lock, flags);
1154 return stream;
1155 }
1156
1157 /*-------------------------------------------------------------------------*/
1158
1159 /* ehci_iso_sched ops can be ITD-only or SITD-only */
1160
1161 static struct ehci_iso_sched *
1162 iso_sched_alloc (unsigned packets, gfp_t mem_flags)
1163 {
1164 struct ehci_iso_sched *iso_sched;
1165 int size = sizeof *iso_sched;
1166
1167 size += packets * sizeof (struct ehci_iso_packet);
1168 iso_sched = kzalloc(size, mem_flags);
1169 if (likely (iso_sched != NULL)) {
1170 INIT_LIST_HEAD (&iso_sched->td_list);
1171 }
1172 return iso_sched;
1173 }
1174
1175 static inline void
1176 itd_sched_init(
1177 struct ehci_hcd *ehci,
1178 struct ehci_iso_sched *iso_sched,
1179 struct ehci_iso_stream *stream,
1180 struct urb *urb
1181 )
1182 {
1183 unsigned i;
1184 dma_addr_t dma = urb->transfer_dma;
1185
1186 /* how many uframes are needed for these transfers */
1187 iso_sched->span = urb->number_of_packets * stream->interval;
1188
1189 /* figure out per-uframe itd fields that we'll need later
1190 * when we fit new itds into the schedule.
1191 */
1192 for (i = 0; i < urb->number_of_packets; i++) {
1193 struct ehci_iso_packet *uframe = &iso_sched->packet [i];
1194 unsigned length;
1195 dma_addr_t buf;
1196 u32 trans;
1197
1198 length = urb->iso_frame_desc [i].length;
1199 buf = dma + urb->iso_frame_desc [i].offset;
1200
1201 trans = EHCI_ISOC_ACTIVE;
1202 trans |= buf & 0x0fff;
1203 if (unlikely (((i + 1) == urb->number_of_packets))
1204 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1205 trans |= EHCI_ITD_IOC;
1206 trans |= length << 16;
1207 uframe->transaction = cpu_to_hc32(ehci, trans);
1208
1209 /* might need to cross a buffer page within a uframe */
1210 uframe->bufp = (buf & ~(u64)0x0fff);
1211 buf += length;
1212 if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff))))
1213 uframe->cross = 1;
1214 }
1215 }
1216
1217 static void
1218 iso_sched_free (
1219 struct ehci_iso_stream *stream,
1220 struct ehci_iso_sched *iso_sched
1221 )
1222 {
1223 if (!iso_sched)
1224 return;
1225 // caller must hold ehci->lock!
1226 list_splice (&iso_sched->td_list, &stream->free_list);
1227 kfree (iso_sched);
1228 }
1229
1230 static int
1231 itd_urb_transaction (
1232 struct ehci_iso_stream *stream,
1233 struct ehci_hcd *ehci,
1234 struct urb *urb,
1235 gfp_t mem_flags
1236 )
1237 {
1238 struct ehci_itd *itd;
1239 dma_addr_t itd_dma;
1240 int i;
1241 unsigned num_itds;
1242 struct ehci_iso_sched *sched;
1243 unsigned long flags;
1244
1245 sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1246 if (unlikely (sched == NULL))
1247 return -ENOMEM;
1248
1249 itd_sched_init(ehci, sched, stream, urb);
1250
1251 if (urb->interval < 8)
1252 num_itds = 1 + (sched->span + 7) / 8;
1253 else
1254 num_itds = urb->number_of_packets;
1255
1256 /* allocate/init ITDs */
1257 spin_lock_irqsave (&ehci->lock, flags);
1258 for (i = 0; i < num_itds; i++) {
1259
1260 /* free_list.next might be cache-hot ... but maybe
1261 * the HC caches it too. avoid that issue for now.
1262 */
1263
1264 /* prefer previously-allocated itds */
1265 if (likely (!list_empty(&stream->free_list))) {
1266 itd = list_entry (stream->free_list.prev,
1267 struct ehci_itd, itd_list);
1268 list_del (&itd->itd_list);
1269 itd_dma = itd->itd_dma;
1270 } else {
1271 spin_unlock_irqrestore (&ehci->lock, flags);
1272 itd = dma_pool_alloc (ehci->itd_pool, mem_flags,
1273 &itd_dma);
1274 spin_lock_irqsave (&ehci->lock, flags);
1275 if (!itd) {
1276 iso_sched_free(stream, sched);
1277 spin_unlock_irqrestore(&ehci->lock, flags);
1278 return -ENOMEM;
1279 }
1280 }
1281
1282 memset (itd, 0, sizeof *itd);
1283 itd->itd_dma = itd_dma;
1284 list_add (&itd->itd_list, &sched->td_list);
1285 }
1286 spin_unlock_irqrestore (&ehci->lock, flags);
1287
1288 /* temporarily store schedule info in hcpriv */
1289 urb->hcpriv = sched;
1290 urb->error_count = 0;
1291 return 0;
1292 }
1293
1294 /*-------------------------------------------------------------------------*/
1295
1296 static inline int
1297 itd_slot_ok (
1298 struct ehci_hcd *ehci,
1299 u32 mod,
1300 u32 uframe,
1301 u8 usecs,
1302 u32 period
1303 )
1304 {
1305 uframe %= period;
1306 do {
1307 /* can't commit more than uframe_periodic_max usec */
1308 if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7)
1309 > (ehci->uframe_periodic_max - usecs))
1310 return 0;
1311
1312 /* we know urb->interval is 2^N uframes */
1313 uframe += period;
1314 } while (uframe < mod);
1315 return 1;
1316 }
1317
1318 static inline int
1319 sitd_slot_ok (
1320 struct ehci_hcd *ehci,
1321 u32 mod,
1322 struct ehci_iso_stream *stream,
1323 u32 uframe,
1324 struct ehci_iso_sched *sched,
1325 u32 period_uframes
1326 )
1327 {
1328 u32 mask, tmp;
1329 u32 frame, uf;
1330
1331 mask = stream->raw_mask << (uframe & 7);
1332
1333 /* for IN, don't wrap CSPLIT into the next frame */
1334 if (mask & ~0xffff)
1335 return 0;
1336
1337 /* this multi-pass logic is simple, but performance may
1338 * suffer when the schedule data isn't cached.
1339 */
1340
1341 /* check bandwidth */
1342 uframe %= period_uframes;
1343 do {
1344 u32 max_used;
1345
1346 frame = uframe >> 3;
1347 uf = uframe & 7;
1348
1349 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
1350 /* The tt's fullspeed bus bandwidth must be available.
1351 * tt_available scheduling guarantees 10+% for control/bulk.
1352 */
1353 if (!tt_available (ehci, period_uframes << 3,
1354 stream->udev, frame, uf, stream->tt_usecs))
1355 return 0;
1356 #else
1357 /* tt must be idle for start(s), any gap, and csplit.
1358 * assume scheduling slop leaves 10+% for control/bulk.
1359 */
1360 if (!tt_no_collision (ehci, period_uframes << 3,
1361 stream->udev, frame, mask))
1362 return 0;
1363 #endif
1364
1365 /* check starts (OUT uses more than one) */
1366 max_used = ehci->uframe_periodic_max - stream->usecs;
1367 for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) {
1368 if (periodic_usecs (ehci, frame, uf) > max_used)
1369 return 0;
1370 }
1371
1372 /* for IN, check CSPLIT */
1373 if (stream->c_usecs) {
1374 uf = uframe & 7;
1375 max_used = ehci->uframe_periodic_max - stream->c_usecs;
1376 do {
1377 tmp = 1 << uf;
1378 tmp <<= 8;
1379 if ((stream->raw_mask & tmp) == 0)
1380 continue;
1381 if (periodic_usecs (ehci, frame, uf)
1382 > max_used)
1383 return 0;
1384 } while (++uf < 8);
1385 }
1386
1387 /* we know urb->interval is 2^N uframes */
1388 uframe += period_uframes;
1389 } while (uframe < mod);
1390
1391 stream->splits = cpu_to_hc32(ehci, stream->raw_mask << (uframe & 7));
1392 return 1;
1393 }
1394
1395 /*
1396 * This scheduler plans almost as far into the future as it has actual
1397 * periodic schedule slots. (Affected by TUNE_FLS, which defaults to
1398 * "as small as possible" to be cache-friendlier.) That limits the size
1399 * transfers you can stream reliably; avoid more than 64 msec per urb.
1400 * Also avoid queue depths of less than ehci's worst irq latency (affected
1401 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
1402 * and other factors); or more than about 230 msec total (for portability,
1403 * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler!
1404 */
1405
1406 #define SCHEDULE_SLOP 80 /* microframes */
1407
1408 static int
1409 iso_stream_schedule (
1410 struct ehci_hcd *ehci,
1411 struct urb *urb,
1412 struct ehci_iso_stream *stream
1413 )
1414 {
1415 u32 now, next, start, period, span;
1416 int status;
1417 unsigned mod = ehci->periodic_size << 3;
1418 struct ehci_iso_sched *sched = urb->hcpriv;
1419
1420 period = urb->interval;
1421 span = sched->span;
1422 if (!stream->highspeed) {
1423 period <<= 3;
1424 span <<= 3;
1425 }
1426
1427 if (span > mod - SCHEDULE_SLOP) {
1428 ehci_dbg (ehci, "iso request %p too long\n", urb);
1429 status = -EFBIG;
1430 goto fail;
1431 }
1432
1433 now = ehci_read_frame_index(ehci) & (mod - 1);
1434
1435 /* Typical case: reuse current schedule, stream is still active.
1436 * Hopefully there are no gaps from the host falling behind
1437 * (irq delays etc), but if there are we'll take the next
1438 * slot in the schedule, implicitly assuming URB_ISO_ASAP.
1439 */
1440 if (likely (!list_empty (&stream->td_list))) {
1441 u32 excess;
1442
1443 /* For high speed devices, allow scheduling within the
1444 * isochronous scheduling threshold. For full speed devices
1445 * and Intel PCI-based controllers, don't (work around for
1446 * Intel ICH9 bug).
1447 */
1448 if (!stream->highspeed && ehci->fs_i_thresh)
1449 next = now + ehci->i_thresh;
1450 else
1451 next = now;
1452
1453 /* Fell behind (by up to twice the slop amount)?
1454 * We decide based on the time of the last currently-scheduled
1455 * slot, not the time of the next available slot.
1456 */
1457 excess = (stream->next_uframe - period - next) & (mod - 1);
1458 if (excess >= mod - 2 * SCHEDULE_SLOP)
1459 start = next + excess - mod + period *
1460 DIV_ROUND_UP(mod - excess, period);
1461 else
1462 start = next + excess + period;
1463 if (start - now >= mod) {
1464 ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n",
1465 urb, start - now - period, period,
1466 mod);
1467 status = -EFBIG;
1468 goto fail;
1469 }
1470 }
1471
1472 /* need to schedule; when's the next (u)frame we could start?
1473 * this is bigger than ehci->i_thresh allows; scheduling itself
1474 * isn't free, the slop should handle reasonably slow cpus. it
1475 * can also help high bandwidth if the dma and irq loads don't
1476 * jump until after the queue is primed.
1477 */
1478 else {
1479 start = SCHEDULE_SLOP + (now & ~0x07);
1480
1481 /* NOTE: assumes URB_ISO_ASAP, to limit complexity/bugs */
1482
1483 /* find a uframe slot with enough bandwidth.
1484 * Early uframes are more precious because full-speed
1485 * iso IN transfers can't use late uframes,
1486 * and therefore they should be allocated last.
1487 */
1488 next = start;
1489 start += period;
1490 do {
1491 start--;
1492 /* check schedule: enough space? */
1493 if (stream->highspeed) {
1494 if (itd_slot_ok(ehci, mod, start,
1495 stream->usecs, period))
1496 break;
1497 } else {
1498 if ((start % 8) >= 6)
1499 continue;
1500 if (sitd_slot_ok(ehci, mod, stream,
1501 start, sched, period))
1502 break;
1503 }
1504 } while (start > next);
1505
1506 /* no room in the schedule */
1507 if (start == next) {
1508 ehci_dbg(ehci, "iso resched full %p (now %d max %d)\n",
1509 urb, now, now + mod);
1510 status = -ENOSPC;
1511 goto fail;
1512 }
1513 }
1514
1515 /* Tried to schedule too far into the future? */
1516 if (unlikely(start - now + span - period
1517 >= mod - 2 * SCHEDULE_SLOP)) {
1518 ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n",
1519 urb, start - now, span - period,
1520 mod - 2 * SCHEDULE_SLOP);
1521 status = -EFBIG;
1522 goto fail;
1523 }
1524
1525 stream->next_uframe = start & (mod - 1);
1526
1527 /* report high speed start in uframes; full speed, in frames */
1528 urb->start_frame = stream->next_uframe;
1529 if (!stream->highspeed)
1530 urb->start_frame >>= 3;
1531 return 0;
1532
1533 fail:
1534 iso_sched_free(stream, sched);
1535 urb->hcpriv = NULL;
1536 return status;
1537 }
1538
1539 /*-------------------------------------------------------------------------*/
1540
1541 static inline void
1542 itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream,
1543 struct ehci_itd *itd)
1544 {
1545 int i;
1546
1547 /* it's been recently zeroed */
1548 itd->hw_next = EHCI_LIST_END(ehci);
1549 itd->hw_bufp [0] = stream->buf0;
1550 itd->hw_bufp [1] = stream->buf1;
1551 itd->hw_bufp [2] = stream->buf2;
1552
1553 for (i = 0; i < 8; i++)
1554 itd->index[i] = -1;
1555
1556 /* All other fields are filled when scheduling */
1557 }
1558
1559 static inline void
1560 itd_patch(
1561 struct ehci_hcd *ehci,
1562 struct ehci_itd *itd,
1563 struct ehci_iso_sched *iso_sched,
1564 unsigned index,
1565 u16 uframe
1566 )
1567 {
1568 struct ehci_iso_packet *uf = &iso_sched->packet [index];
1569 unsigned pg = itd->pg;
1570
1571 // BUG_ON (pg == 6 && uf->cross);
1572
1573 uframe &= 0x07;
1574 itd->index [uframe] = index;
1575
1576 itd->hw_transaction[uframe] = uf->transaction;
1577 itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12);
1578 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0);
1579 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32));
1580
1581 /* iso_frame_desc[].offset must be strictly increasing */
1582 if (unlikely (uf->cross)) {
1583 u64 bufp = uf->bufp + 4096;
1584
1585 itd->pg = ++pg;
1586 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0);
1587 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32));
1588 }
1589 }
1590
1591 static inline void
1592 itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
1593 {
1594 union ehci_shadow *prev = &ehci->pshadow[frame];
1595 __hc32 *hw_p = &ehci->periodic[frame];
1596 union ehci_shadow here = *prev;
1597 __hc32 type = 0;
1598
1599 /* skip any iso nodes which might belong to previous microframes */
1600 while (here.ptr) {
1601 type = Q_NEXT_TYPE(ehci, *hw_p);
1602 if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
1603 break;
1604 prev = periodic_next_shadow(ehci, prev, type);
1605 hw_p = shadow_next_periodic(ehci, &here, type);
1606 here = *prev;
1607 }
1608
1609 itd->itd_next = here;
1610 itd->hw_next = *hw_p;
1611 prev->itd = itd;
1612 itd->frame = frame;
1613 wmb ();
1614 *hw_p = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD);
1615 }
1616
1617 /* fit urb's itds into the selected schedule slot; activate as needed */
1618 static int
1619 itd_link_urb (
1620 struct ehci_hcd *ehci,
1621 struct urb *urb,
1622 unsigned mod,
1623 struct ehci_iso_stream *stream
1624 )
1625 {
1626 int packet;
1627 unsigned next_uframe, uframe, frame;
1628 struct ehci_iso_sched *iso_sched = urb->hcpriv;
1629 struct ehci_itd *itd;
1630
1631 next_uframe = stream->next_uframe & (mod - 1);
1632
1633 if (unlikely (list_empty(&stream->td_list))) {
1634 ehci_to_hcd(ehci)->self.bandwidth_allocated
1635 += stream->bandwidth;
1636 ehci_vdbg (ehci,
1637 "schedule devp %s ep%d%s-iso period %d start %d.%d\n",
1638 urb->dev->devpath, stream->bEndpointAddress & 0x0f,
1639 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
1640 urb->interval,
1641 next_uframe >> 3, next_uframe & 0x7);
1642 }
1643
1644 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1645 if (ehci->amd_pll_fix == 1)
1646 usb_amd_quirk_pll_disable();
1647 }
1648
1649 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1650
1651 /* fill iTDs uframe by uframe */
1652 for (packet = 0, itd = NULL; packet < urb->number_of_packets; ) {
1653 if (itd == NULL) {
1654 /* ASSERT: we have all necessary itds */
1655 // BUG_ON (list_empty (&iso_sched->td_list));
1656
1657 /* ASSERT: no itds for this endpoint in this uframe */
1658
1659 itd = list_entry (iso_sched->td_list.next,
1660 struct ehci_itd, itd_list);
1661 list_move_tail (&itd->itd_list, &stream->td_list);
1662 itd->stream = iso_stream_get (stream);
1663 itd->urb = urb;
1664 itd_init (ehci, stream, itd);
1665 }
1666
1667 uframe = next_uframe & 0x07;
1668 frame = next_uframe >> 3;
1669
1670 itd_patch(ehci, itd, iso_sched, packet, uframe);
1671
1672 next_uframe += stream->interval;
1673 next_uframe &= mod - 1;
1674 packet++;
1675
1676 /* link completed itds into the schedule */
1677 if (((next_uframe >> 3) != frame)
1678 || packet == urb->number_of_packets) {
1679 itd_link(ehci, frame & (ehci->periodic_size - 1), itd);
1680 itd = NULL;
1681 }
1682 }
1683 stream->next_uframe = next_uframe;
1684
1685 /* don't need that schedule data any more */
1686 iso_sched_free (stream, iso_sched);
1687 urb->hcpriv = NULL;
1688
1689 timer_action (ehci, TIMER_IO_WATCHDOG);
1690 return enable_periodic(ehci);
1691 }
1692
1693 #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1694
1695 /* Process and recycle a completed ITD. Return true iff its urb completed,
1696 * and hence its completion callback probably added things to the hardware
1697 * schedule.
1698 *
1699 * Note that we carefully avoid recycling this descriptor until after any
1700 * completion callback runs, so that it won't be reused quickly. That is,
1701 * assuming (a) no more than two urbs per frame on this endpoint, and also
1702 * (b) only this endpoint's completions submit URBs. It seems some silicon
1703 * corrupts things if you reuse completed descriptors very quickly...
1704 */
1705 static unsigned
1706 itd_complete (
1707 struct ehci_hcd *ehci,
1708 struct ehci_itd *itd
1709 ) {
1710 struct urb *urb = itd->urb;
1711 struct usb_iso_packet_descriptor *desc;
1712 u32 t;
1713 unsigned uframe;
1714 int urb_index = -1;
1715 struct ehci_iso_stream *stream = itd->stream;
1716 struct usb_device *dev;
1717 unsigned retval = false;
1718
1719 /* for each uframe with a packet */
1720 for (uframe = 0; uframe < 8; uframe++) {
1721 if (likely (itd->index[uframe] == -1))
1722 continue;
1723 urb_index = itd->index[uframe];
1724 desc = &urb->iso_frame_desc [urb_index];
1725
1726 t = hc32_to_cpup(ehci, &itd->hw_transaction [uframe]);
1727 itd->hw_transaction [uframe] = 0;
1728
1729 /* report transfer status */
1730 if (unlikely (t & ISO_ERRS)) {
1731 urb->error_count++;
1732 if (t & EHCI_ISOC_BUF_ERR)
1733 desc->status = usb_pipein (urb->pipe)
1734 ? -ENOSR /* hc couldn't read */
1735 : -ECOMM; /* hc couldn't write */
1736 else if (t & EHCI_ISOC_BABBLE)
1737 desc->status = -EOVERFLOW;
1738 else /* (t & EHCI_ISOC_XACTERR) */
1739 desc->status = -EPROTO;
1740
1741 /* HC need not update length with this error */
1742 if (!(t & EHCI_ISOC_BABBLE)) {
1743 desc->actual_length = EHCI_ITD_LENGTH(t);
1744 urb->actual_length += desc->actual_length;
1745 }
1746 } else if (likely ((t & EHCI_ISOC_ACTIVE) == 0)) {
1747 desc->status = 0;
1748 desc->actual_length = EHCI_ITD_LENGTH(t);
1749 urb->actual_length += desc->actual_length;
1750 } else {
1751 /* URB was too late */
1752 desc->status = -EXDEV;
1753 }
1754 }
1755
1756 /* handle completion now? */
1757 if (likely ((urb_index + 1) != urb->number_of_packets))
1758 goto done;
1759
1760 /* ASSERT: it's really the last itd for this urb
1761 list_for_each_entry (itd, &stream->td_list, itd_list)
1762 BUG_ON (itd->urb == urb);
1763 */
1764
1765 /* give urb back to the driver; completion often (re)submits */
1766 dev = urb->dev;
1767 ehci_urb_done(ehci, urb, 0);
1768 retval = true;
1769 urb = NULL;
1770 (void) disable_periodic(ehci);
1771 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1772
1773 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1774 if (ehci->amd_pll_fix == 1)
1775 usb_amd_quirk_pll_enable();
1776 }
1777
1778 if (unlikely(list_is_singular(&stream->td_list))) {
1779 ehci_to_hcd(ehci)->self.bandwidth_allocated
1780 -= stream->bandwidth;
1781 ehci_vdbg (ehci,
1782 "deschedule devp %s ep%d%s-iso\n",
1783 dev->devpath, stream->bEndpointAddress & 0x0f,
1784 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
1785 }
1786 iso_stream_put (ehci, stream);
1787
1788 done:
1789 itd->urb = NULL;
1790 if (ehci->clock_frame != itd->frame || itd->index[7] != -1) {
1791 /* OK to recycle this ITD now. */
1792 itd->stream = NULL;
1793 list_move(&itd->itd_list, &stream->free_list);
1794 iso_stream_put(ehci, stream);
1795 } else {
1796 /* HW might remember this ITD, so we can't recycle it yet.
1797 * Move it to a safe place until a new frame starts.
1798 */
1799 list_move(&itd->itd_list, &ehci->cached_itd_list);
1800 if (stream->refcount == 2) {
1801 /* If iso_stream_put() were called here, stream
1802 * would be freed. Instead, just prevent reuse.
1803 */
1804 stream->ep->hcpriv = NULL;
1805 stream->ep = NULL;
1806 }
1807 }
1808 return retval;
1809 }
1810
1811 /*-------------------------------------------------------------------------*/
1812
1813 static int itd_submit (struct ehci_hcd *ehci, struct urb *urb,
1814 gfp_t mem_flags)
1815 {
1816 int status = -EINVAL;
1817 unsigned long flags;
1818 struct ehci_iso_stream *stream;
1819
1820 /* Get iso_stream head */
1821 stream = iso_stream_find (ehci, urb);
1822 if (unlikely (stream == NULL)) {
1823 ehci_dbg (ehci, "can't get iso stream\n");
1824 return -ENOMEM;
1825 }
1826 if (unlikely (urb->interval != stream->interval)) {
1827 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
1828 stream->interval, urb->interval);
1829 goto done;
1830 }
1831
1832 #ifdef EHCI_URB_TRACE
1833 ehci_dbg (ehci,
1834 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1835 __func__, urb->dev->devpath, urb,
1836 usb_pipeendpoint (urb->pipe),
1837 usb_pipein (urb->pipe) ? "in" : "out",
1838 urb->transfer_buffer_length,
1839 urb->number_of_packets, urb->interval,
1840 stream);
1841 #endif
1842
1843 /* allocate ITDs w/o locking anything */
1844 status = itd_urb_transaction (stream, ehci, urb, mem_flags);
1845 if (unlikely (status < 0)) {
1846 ehci_dbg (ehci, "can't init itds\n");
1847 goto done;
1848 }
1849
1850 /* schedule ... need to lock */
1851 spin_lock_irqsave (&ehci->lock, flags);
1852 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
1853 status = -ESHUTDOWN;
1854 goto done_not_linked;
1855 }
1856 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1857 if (unlikely(status))
1858 goto done_not_linked;
1859 status = iso_stream_schedule(ehci, urb, stream);
1860 if (likely (status == 0))
1861 itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
1862 else
1863 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1864 done_not_linked:
1865 spin_unlock_irqrestore (&ehci->lock, flags);
1866
1867 done:
1868 if (unlikely (status < 0))
1869 iso_stream_put (ehci, stream);
1870 return status;
1871 }
1872
1873 /*-------------------------------------------------------------------------*/
1874
1875 /*
1876 * "Split ISO TDs" ... used for USB 1.1 devices going through the
1877 * TTs in USB 2.0 hubs. These need microframe scheduling.
1878 */
1879
1880 static inline void
1881 sitd_sched_init(
1882 struct ehci_hcd *ehci,
1883 struct ehci_iso_sched *iso_sched,
1884 struct ehci_iso_stream *stream,
1885 struct urb *urb
1886 )
1887 {
1888 unsigned i;
1889 dma_addr_t dma = urb->transfer_dma;
1890
1891 /* how many frames are needed for these transfers */
1892 iso_sched->span = urb->number_of_packets * stream->interval;
1893
1894 /* figure out per-frame sitd fields that we'll need later
1895 * when we fit new sitds into the schedule.
1896 */
1897 for (i = 0; i < urb->number_of_packets; i++) {
1898 struct ehci_iso_packet *packet = &iso_sched->packet [i];
1899 unsigned length;
1900 dma_addr_t buf;
1901 u32 trans;
1902
1903 length = urb->iso_frame_desc [i].length & 0x03ff;
1904 buf = dma + urb->iso_frame_desc [i].offset;
1905
1906 trans = SITD_STS_ACTIVE;
1907 if (((i + 1) == urb->number_of_packets)
1908 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1909 trans |= SITD_IOC;
1910 trans |= length << 16;
1911 packet->transaction = cpu_to_hc32(ehci, trans);
1912
1913 /* might need to cross a buffer page within a td */
1914 packet->bufp = buf;
1915 packet->buf1 = (buf + length) & ~0x0fff;
1916 if (packet->buf1 != (buf & ~(u64)0x0fff))
1917 packet->cross = 1;
1918
1919 /* OUT uses multiple start-splits */
1920 if (stream->bEndpointAddress & USB_DIR_IN)
1921 continue;
1922 length = (length + 187) / 188;
1923 if (length > 1) /* BEGIN vs ALL */
1924 length |= 1 << 3;
1925 packet->buf1 |= length;
1926 }
1927 }
1928
1929 static int
1930 sitd_urb_transaction (
1931 struct ehci_iso_stream *stream,
1932 struct ehci_hcd *ehci,
1933 struct urb *urb,
1934 gfp_t mem_flags
1935 )
1936 {
1937 struct ehci_sitd *sitd;
1938 dma_addr_t sitd_dma;
1939 int i;
1940 struct ehci_iso_sched *iso_sched;
1941 unsigned long flags;
1942
1943 iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1944 if (iso_sched == NULL)
1945 return -ENOMEM;
1946
1947 sitd_sched_init(ehci, iso_sched, stream, urb);
1948
1949 /* allocate/init sITDs */
1950 spin_lock_irqsave (&ehci->lock, flags);
1951 for (i = 0; i < urb->number_of_packets; i++) {
1952
1953 /* NOTE: for now, we don't try to handle wraparound cases
1954 * for IN (using sitd->hw_backpointer, like a FSTN), which
1955 * means we never need two sitds for full speed packets.
1956 */
1957
1958 /* free_list.next might be cache-hot ... but maybe
1959 * the HC caches it too. avoid that issue for now.
1960 */
1961
1962 /* prefer previously-allocated sitds */
1963 if (!list_empty(&stream->free_list)) {
1964 sitd = list_entry (stream->free_list.prev,
1965 struct ehci_sitd, sitd_list);
1966 list_del (&sitd->sitd_list);
1967 sitd_dma = sitd->sitd_dma;
1968 } else {
1969 spin_unlock_irqrestore (&ehci->lock, flags);
1970 sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags,
1971 &sitd_dma);
1972 spin_lock_irqsave (&ehci->lock, flags);
1973 if (!sitd) {
1974 iso_sched_free(stream, iso_sched);
1975 spin_unlock_irqrestore(&ehci->lock, flags);
1976 return -ENOMEM;
1977 }
1978 }
1979
1980 memset (sitd, 0, sizeof *sitd);
1981 sitd->sitd_dma = sitd_dma;
1982 list_add (&sitd->sitd_list, &iso_sched->td_list);
1983 }
1984
1985 /* temporarily store schedule info in hcpriv */
1986 urb->hcpriv = iso_sched;
1987 urb->error_count = 0;
1988
1989 spin_unlock_irqrestore (&ehci->lock, flags);
1990 return 0;
1991 }
1992
1993 /*-------------------------------------------------------------------------*/
1994
1995 static inline void
1996 sitd_patch(
1997 struct ehci_hcd *ehci,
1998 struct ehci_iso_stream *stream,
1999 struct ehci_sitd *sitd,
2000 struct ehci_iso_sched *iso_sched,
2001 unsigned index
2002 )
2003 {
2004 struct ehci_iso_packet *uf = &iso_sched->packet [index];
2005 u64 bufp = uf->bufp;
2006
2007 sitd->hw_next = EHCI_LIST_END(ehci);
2008 sitd->hw_fullspeed_ep = stream->address;
2009 sitd->hw_uframe = stream->splits;
2010 sitd->hw_results = uf->transaction;
2011 sitd->hw_backpointer = EHCI_LIST_END(ehci);
2012
2013 bufp = uf->bufp;
2014 sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp);
2015 sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32);
2016
2017 sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1);
2018 if (uf->cross)
2019 bufp += 4096;
2020 sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32);
2021 sitd->index = index;
2022 }
2023
2024 static inline void
2025 sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
2026 {
2027 /* note: sitd ordering could matter (CSPLIT then SSPLIT) */
2028 sitd->sitd_next = ehci->pshadow [frame];
2029 sitd->hw_next = ehci->periodic [frame];
2030 ehci->pshadow [frame].sitd = sitd;
2031 sitd->frame = frame;
2032 wmb ();
2033 ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD);
2034 }
2035
2036 /* fit urb's sitds into the selected schedule slot; activate as needed */
2037 static int
2038 sitd_link_urb (
2039 struct ehci_hcd *ehci,
2040 struct urb *urb,
2041 unsigned mod,
2042 struct ehci_iso_stream *stream
2043 )
2044 {
2045 int packet;
2046 unsigned next_uframe;
2047 struct ehci_iso_sched *sched = urb->hcpriv;
2048 struct ehci_sitd *sitd;
2049
2050 next_uframe = stream->next_uframe;
2051
2052 if (list_empty(&stream->td_list)) {
2053 /* usbfs ignores TT bandwidth */
2054 ehci_to_hcd(ehci)->self.bandwidth_allocated
2055 += stream->bandwidth;
2056 ehci_vdbg (ehci,
2057 "sched devp %s ep%d%s-iso [%d] %dms/%04x\n",
2058 urb->dev->devpath, stream->bEndpointAddress & 0x0f,
2059 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
2060 (next_uframe >> 3) & (ehci->periodic_size - 1),
2061 stream->interval, hc32_to_cpu(ehci, stream->splits));
2062 }
2063
2064 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2065 if (ehci->amd_pll_fix == 1)
2066 usb_amd_quirk_pll_disable();
2067 }
2068
2069 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
2070
2071 /* fill sITDs frame by frame */
2072 for (packet = 0, sitd = NULL;
2073 packet < urb->number_of_packets;
2074 packet++) {
2075
2076 /* ASSERT: we have all necessary sitds */
2077 BUG_ON (list_empty (&sched->td_list));
2078
2079 /* ASSERT: no itds for this endpoint in this frame */
2080
2081 sitd = list_entry (sched->td_list.next,
2082 struct ehci_sitd, sitd_list);
2083 list_move_tail (&sitd->sitd_list, &stream->td_list);
2084 sitd->stream = iso_stream_get (stream);
2085 sitd->urb = urb;
2086
2087 sitd_patch(ehci, stream, sitd, sched, packet);
2088 sitd_link(ehci, (next_uframe >> 3) & (ehci->periodic_size - 1),
2089 sitd);
2090
2091 next_uframe += stream->interval << 3;
2092 }
2093 stream->next_uframe = next_uframe & (mod - 1);
2094
2095 /* don't need that schedule data any more */
2096 iso_sched_free (stream, sched);
2097 urb->hcpriv = NULL;
2098
2099 timer_action (ehci, TIMER_IO_WATCHDOG);
2100 return enable_periodic(ehci);
2101 }
2102
2103 /*-------------------------------------------------------------------------*/
2104
2105 #define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
2106 | SITD_STS_XACT | SITD_STS_MMF)
2107
2108 /* Process and recycle a completed SITD. Return true iff its urb completed,
2109 * and hence its completion callback probably added things to the hardware
2110 * schedule.
2111 *
2112 * Note that we carefully avoid recycling this descriptor until after any
2113 * completion callback runs, so that it won't be reused quickly. That is,
2114 * assuming (a) no more than two urbs per frame on this endpoint, and also
2115 * (b) only this endpoint's completions submit URBs. It seems some silicon
2116 * corrupts things if you reuse completed descriptors very quickly...
2117 */
2118 static unsigned
2119 sitd_complete (
2120 struct ehci_hcd *ehci,
2121 struct ehci_sitd *sitd
2122 ) {
2123 struct urb *urb = sitd->urb;
2124 struct usb_iso_packet_descriptor *desc;
2125 u32 t;
2126 int urb_index = -1;
2127 struct ehci_iso_stream *stream = sitd->stream;
2128 struct usb_device *dev;
2129 unsigned retval = false;
2130
2131 urb_index = sitd->index;
2132 desc = &urb->iso_frame_desc [urb_index];
2133 t = hc32_to_cpup(ehci, &sitd->hw_results);
2134
2135 /* report transfer status */
2136 if (t & SITD_ERRS) {
2137 urb->error_count++;
2138 if (t & SITD_STS_DBE)
2139 desc->status = usb_pipein (urb->pipe)
2140 ? -ENOSR /* hc couldn't read */
2141 : -ECOMM; /* hc couldn't write */
2142 else if (t & SITD_STS_BABBLE)
2143 desc->status = -EOVERFLOW;
2144 else /* XACT, MMF, etc */
2145 desc->status = -EPROTO;
2146 } else {
2147 desc->status = 0;
2148 desc->actual_length = desc->length - SITD_LENGTH(t);
2149 urb->actual_length += desc->actual_length;
2150 }
2151
2152 /* handle completion now? */
2153 if ((urb_index + 1) != urb->number_of_packets)
2154 goto done;
2155
2156 /* ASSERT: it's really the last sitd for this urb
2157 list_for_each_entry (sitd, &stream->td_list, sitd_list)
2158 BUG_ON (sitd->urb == urb);
2159 */
2160
2161 /* give urb back to the driver; completion often (re)submits */
2162 dev = urb->dev;
2163 ehci_urb_done(ehci, urb, 0);
2164 retval = true;
2165 urb = NULL;
2166 (void) disable_periodic(ehci);
2167 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
2168
2169 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2170 if (ehci->amd_pll_fix == 1)
2171 usb_amd_quirk_pll_enable();
2172 }
2173
2174 if (list_is_singular(&stream->td_list)) {
2175 ehci_to_hcd(ehci)->self.bandwidth_allocated
2176 -= stream->bandwidth;
2177 ehci_vdbg (ehci,
2178 "deschedule devp %s ep%d%s-iso\n",
2179 dev->devpath, stream->bEndpointAddress & 0x0f,
2180 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
2181 }
2182 iso_stream_put (ehci, stream);
2183
2184 done:
2185 sitd->urb = NULL;
2186 if (ehci->clock_frame != sitd->frame) {
2187 /* OK to recycle this SITD now. */
2188 sitd->stream = NULL;
2189 list_move(&sitd->sitd_list, &stream->free_list);
2190 iso_stream_put(ehci, stream);
2191 } else {
2192 /* HW might remember this SITD, so we can't recycle it yet.
2193 * Move it to a safe place until a new frame starts.
2194 */
2195 list_move(&sitd->sitd_list, &ehci->cached_sitd_list);
2196 if (stream->refcount == 2) {
2197 /* If iso_stream_put() were called here, stream
2198 * would be freed. Instead, just prevent reuse.
2199 */
2200 stream->ep->hcpriv = NULL;
2201 stream->ep = NULL;
2202 }
2203 }
2204 return retval;
2205 }
2206
2207
2208 static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb,
2209 gfp_t mem_flags)
2210 {
2211 int status = -EINVAL;
2212 unsigned long flags;
2213 struct ehci_iso_stream *stream;
2214
2215 /* Get iso_stream head */
2216 stream = iso_stream_find (ehci, urb);
2217 if (stream == NULL) {
2218 ehci_dbg (ehci, "can't get iso stream\n");
2219 return -ENOMEM;
2220 }
2221 if (urb->interval != stream->interval) {
2222 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
2223 stream->interval, urb->interval);
2224 goto done;
2225 }
2226
2227 #ifdef EHCI_URB_TRACE
2228 ehci_dbg (ehci,
2229 "submit %p dev%s ep%d%s-iso len %d\n",
2230 urb, urb->dev->devpath,
2231 usb_pipeendpoint (urb->pipe),
2232 usb_pipein (urb->pipe) ? "in" : "out",
2233 urb->transfer_buffer_length);
2234 #endif
2235
2236 /* allocate SITDs */
2237 status = sitd_urb_transaction (stream, ehci, urb, mem_flags);
2238 if (status < 0) {
2239 ehci_dbg (ehci, "can't init sitds\n");
2240 goto done;
2241 }
2242
2243 /* schedule ... need to lock */
2244 spin_lock_irqsave (&ehci->lock, flags);
2245 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
2246 status = -ESHUTDOWN;
2247 goto done_not_linked;
2248 }
2249 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
2250 if (unlikely(status))
2251 goto done_not_linked;
2252 status = iso_stream_schedule(ehci, urb, stream);
2253 if (status == 0)
2254 sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
2255 else
2256 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
2257 done_not_linked:
2258 spin_unlock_irqrestore (&ehci->lock, flags);
2259
2260 done:
2261 if (status < 0)
2262 iso_stream_put (ehci, stream);
2263 return status;
2264 }
2265
2266 /*-------------------------------------------------------------------------*/
2267
2268 static void free_cached_lists(struct ehci_hcd *ehci)
2269 {
2270 struct ehci_itd *itd, *n;
2271 struct ehci_sitd *sitd, *sn;
2272
2273 list_for_each_entry_safe(itd, n, &ehci->cached_itd_list, itd_list) {
2274 struct ehci_iso_stream *stream = itd->stream;
2275 itd->stream = NULL;
2276 list_move(&itd->itd_list, &stream->free_list);
2277 iso_stream_put(ehci, stream);
2278 }
2279
2280 list_for_each_entry_safe(sitd, sn, &ehci->cached_sitd_list, sitd_list) {
2281 struct ehci_iso_stream *stream = sitd->stream;
2282 sitd->stream = NULL;
2283 list_move(&sitd->sitd_list, &stream->free_list);
2284 iso_stream_put(ehci, stream);
2285 }
2286 }
2287
2288 /*-------------------------------------------------------------------------*/
2289
2290 static void
2291 scan_periodic (struct ehci_hcd *ehci)
2292 {
2293 unsigned now_uframe, frame, clock, clock_frame, mod;
2294 unsigned modified;
2295
2296 mod = ehci->periodic_size << 3;
2297
2298 /*
2299 * When running, scan from last scan point up to "now"
2300 * else clean up by scanning everything that's left.
2301 * Touches as few pages as possible: cache-friendly.
2302 */
2303 now_uframe = ehci->next_uframe;
2304 if (HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
2305 clock = ehci_read_frame_index(ehci);
2306 clock_frame = (clock >> 3) & (ehci->periodic_size - 1);
2307 } else {
2308 clock = now_uframe + mod - 1;
2309 clock_frame = -1;
2310 }
2311 if (ehci->clock_frame != clock_frame) {
2312 free_cached_lists(ehci);
2313 ehci->clock_frame = clock_frame;
2314 }
2315 clock &= mod - 1;
2316 clock_frame = clock >> 3;
2317 ++ehci->periodic_stamp;
2318
2319 for (;;) {
2320 union ehci_shadow q, *q_p;
2321 __hc32 type, *hw_p;
2322 unsigned incomplete = false;
2323
2324 frame = now_uframe >> 3;
2325
2326 restart:
2327 /* scan each element in frame's queue for completions */
2328 q_p = &ehci->pshadow [frame];
2329 hw_p = &ehci->periodic [frame];
2330 q.ptr = q_p->ptr;
2331 type = Q_NEXT_TYPE(ehci, *hw_p);
2332 modified = 0;
2333
2334 while (q.ptr != NULL) {
2335 unsigned uf;
2336 union ehci_shadow temp;
2337 int live;
2338
2339 live = HC_IS_RUNNING (ehci_to_hcd(ehci)->state);
2340 switch (hc32_to_cpu(ehci, type)) {
2341 case Q_TYPE_QH:
2342 /* handle any completions */
2343 temp.qh = qh_get (q.qh);
2344 type = Q_NEXT_TYPE(ehci, q.qh->hw->hw_next);
2345 q = q.qh->qh_next;
2346 if (temp.qh->stamp != ehci->periodic_stamp) {
2347 modified = qh_completions(ehci, temp.qh);
2348 if (!modified)
2349 temp.qh->stamp = ehci->periodic_stamp;
2350 if (unlikely(list_empty(&temp.qh->qtd_list) ||
2351 temp.qh->needs_rescan))
2352 intr_deschedule(ehci, temp.qh);
2353 }
2354 qh_put (temp.qh);
2355 break;
2356 case Q_TYPE_FSTN:
2357 /* for "save place" FSTNs, look at QH entries
2358 * in the previous frame for completions.
2359 */
2360 if (q.fstn->hw_prev != EHCI_LIST_END(ehci)) {
2361 dbg ("ignoring completions from FSTNs");
2362 }
2363 type = Q_NEXT_TYPE(ehci, q.fstn->hw_next);
2364 q = q.fstn->fstn_next;
2365 break;
2366 case Q_TYPE_ITD:
2367 /* If this ITD is still active, leave it for
2368 * later processing ... check the next entry.
2369 * No need to check for activity unless the
2370 * frame is current.
2371 */
2372 if (frame == clock_frame && live) {
2373 rmb();
2374 for (uf = 0; uf < 8; uf++) {
2375 if (q.itd->hw_transaction[uf] &
2376 ITD_ACTIVE(ehci))
2377 break;
2378 }
2379 if (uf < 8) {
2380 incomplete = true;
2381 q_p = &q.itd->itd_next;
2382 hw_p = &q.itd->hw_next;
2383 type = Q_NEXT_TYPE(ehci,
2384 q.itd->hw_next);
2385 q = *q_p;
2386 break;
2387 }
2388 }
2389
2390 /* Take finished ITDs out of the schedule
2391 * and process them: recycle, maybe report
2392 * URB completion. HC won't cache the
2393 * pointer for much longer, if at all.
2394 */
2395 *q_p = q.itd->itd_next;
2396 if (!ehci->use_dummy_qh ||
2397 q.itd->hw_next != EHCI_LIST_END(ehci))
2398 *hw_p = q.itd->hw_next;
2399 else
2400 *hw_p = ehci->dummy->qh_dma;
2401 type = Q_NEXT_TYPE(ehci, q.itd->hw_next);
2402 wmb();
2403 modified = itd_complete (ehci, q.itd);
2404 q = *q_p;
2405 break;
2406 case Q_TYPE_SITD:
2407 /* If this SITD is still active, leave it for
2408 * later processing ... check the next entry.
2409 * No need to check for activity unless the
2410 * frame is current.
2411 */
2412 if (((frame == clock_frame) ||
2413 (((frame + 1) & (ehci->periodic_size - 1))
2414 == clock_frame))
2415 && live
2416 && (q.sitd->hw_results &
2417 SITD_ACTIVE(ehci))) {
2418
2419 incomplete = true;
2420 q_p = &q.sitd->sitd_next;
2421 hw_p = &q.sitd->hw_next;
2422 type = Q_NEXT_TYPE(ehci,
2423 q.sitd->hw_next);
2424 q = *q_p;
2425 break;
2426 }
2427
2428 /* Take finished SITDs out of the schedule
2429 * and process them: recycle, maybe report
2430 * URB completion.
2431 */
2432 *q_p = q.sitd->sitd_next;
2433 if (!ehci->use_dummy_qh ||
2434 q.sitd->hw_next != EHCI_LIST_END(ehci))
2435 *hw_p = q.sitd->hw_next;
2436 else
2437 *hw_p = ehci->dummy->qh_dma;
2438 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2439 wmb();
2440 modified = sitd_complete (ehci, q.sitd);
2441 q = *q_p;
2442 break;
2443 default:
2444 dbg ("corrupt type %d frame %d shadow %p",
2445 type, frame, q.ptr);
2446 // BUG ();
2447 q.ptr = NULL;
2448 }
2449
2450 /* assume completion callbacks modify the queue */
2451 if (unlikely (modified)) {
2452 if (likely(ehci->periodic_sched > 0))
2453 goto restart;
2454 /* short-circuit this scan */
2455 now_uframe = clock;
2456 break;
2457 }
2458 }
2459
2460 /* If we can tell we caught up to the hardware, stop now.
2461 * We can't advance our scan without collecting the ISO
2462 * transfers that are still pending in this frame.
2463 */
2464 if (incomplete && HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
2465 ehci->next_uframe = now_uframe;
2466 break;
2467 }
2468
2469 // FIXME: this assumes we won't get lapped when
2470 // latencies climb; that should be rare, but...
2471 // detect it, and just go all the way around.
2472 // FLR might help detect this case, so long as latencies
2473 // don't exceed periodic_size msec (default 1.024 sec).
2474
2475 // FIXME: likewise assumes HC doesn't halt mid-scan
2476
2477 if (now_uframe == clock) {
2478 unsigned now;
2479
2480 if (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state)
2481 || ehci->periodic_sched == 0)
2482 break;
2483 ehci->next_uframe = now_uframe;
2484 now = ehci_read_frame_index(ehci) & (mod - 1);
2485 if (now_uframe == now)
2486 break;
2487
2488 /* rescan the rest of this frame, then ... */
2489 clock = now;
2490 clock_frame = clock >> 3;
2491 if (ehci->clock_frame != clock_frame) {
2492 free_cached_lists(ehci);
2493 ehci->clock_frame = clock_frame;
2494 ++ehci->periodic_stamp;
2495 }
2496 } else {
2497 now_uframe++;
2498 now_uframe &= mod - 1;
2499 }
2500 }
2501 }
Linux 2.6 ibm-acpi/thinkpad-acpi driver git tree