release/src-rt-6.x.4708/linux/linux-2.6.36/drivers/usb/host/ehci-sched.c

   1 /*
   2  * Copyright (c) 2001-2004 by David Brownell
   3  * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License as published by the
   7  * Free Software Foundation; either version 2 of the License, or (at your
   8  * option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software Foundation,
  17  * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18  */
  19
  20 /* this file is part of ehci-hcd.c */
  21
  22 /*-------------------------------------------------------------------------*/
  23
  24 /*
  25  * EHCI scheduled transaction support:  interrupt, iso, split iso
  26  * These are called "periodic" transactions in the EHCI spec.
  27  *
  28  * Note that for interrupt transfers, the QH/QTD manipulation is shared
  29  * with the "asynchronous" transaction support (control/bulk transfers).
  30  * The only real difference is in how interrupt transfers are scheduled.
  31  *
  32  * For ISO, we make an "iso_stream" head to serve the same role as a QH.
  33  * It keeps track of every ITD (or SITD) that's linked, and holds enough
  34  * pre-calculated schedule data to make appending to the queue be quick.
  35  */
  36
  37 static int ehci_get_frame (struct usb_hcd *hcd);
  38
  39 /*-------------------------------------------------------------------------*/
  40
  41 /*
  42  * periodic_next_shadow - return "next" pointer on shadow list
  43  * @periodic: host pointer to qh/itd/sitd
  44  * @tag: hardware tag for type of this record
  45  */
  46 static union ehci_shadow *
  47 periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic,
  48                 __hc32 tag)
  49 {
  50         switch (hc32_to_cpu(ehci, tag)) {
  51         case Q_TYPE_QH:
  52                 return &periodic->qh->qh_next;
  53         case Q_TYPE_FSTN:
  54                 return &periodic->fstn->fstn_next;
  55         case Q_TYPE_ITD:
  56                 return &periodic->itd->itd_next;
  57         // case Q_TYPE_SITD:
  58         default:
  59                 return &periodic->sitd->sitd_next;
  60         }
  61 }
  62
  63 static __hc32 *
  64 shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic,
  65                 __hc32 tag)
  66 {
  67         switch (hc32_to_cpu(ehci, tag)) {
  68         /* our ehci_shadow.qh is actually software part */
  69         case Q_TYPE_QH:
  70                 return &periodic->qh->hw->hw_next;
  71         /* others are hw parts */
  72         default:
  73                 return periodic->hw_next;
  74         }
  75 }
  76
  77 /* caller must hold ehci->lock */
  78 static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr)
  79 {
  80         union ehci_shadow       *prev_p = &ehci->pshadow[frame];
  81         __hc32                  *hw_p = &ehci->periodic[frame];
  82         union ehci_shadow       here = *prev_p;
  83
  84         /* find predecessor of "ptr"; hw and shadow lists are in sync */
  85         while (here.ptr && here.ptr != ptr) {
  86                 prev_p = periodic_next_shadow(ehci, prev_p,
  87                                 Q_NEXT_TYPE(ehci, *hw_p));
  88                 hw_p = shadow_next_periodic(ehci, &here,
  89                                 Q_NEXT_TYPE(ehci, *hw_p));
  90                 here = *prev_p;
  91         }
  92         /* an interrupt entry (at list end) could have been shared */
  93         if (!here.ptr)
  94                 return;
  95
  96         /* update shadow and hardware lists ... the old "next" pointers
  97          * from ptr may still be in use, the caller updates them.
  98          */
  99         *prev_p = *periodic_next_shadow(ehci, &here,
 100                         Q_NEXT_TYPE(ehci, *hw_p));
 101         *hw_p = *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p));
 102 }
 103
 104 /* how many of the uframe's 125 usecs are allocated? */
 105 static unsigned short
 106 periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe)
 107 {
 108         __hc32                  *hw_p = &ehci->periodic [frame];
 109         union ehci_shadow       *q = &ehci->pshadow [frame];
 110         unsigned                usecs = 0;
 111         struct ehci_qh_hw       *hw;
 112
 113         while (q->ptr) {
 114                 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
 115                 case Q_TYPE_QH:
 116                         hw = q->qh->hw;
 117                         /* is it in the S-mask? */
 118                         if (hw->hw_info2 & cpu_to_hc32(ehci, 1 << uframe))
 119                                 usecs += q->qh->usecs;
 120                         /* ... or C-mask? */
 121                         if (hw->hw_info2 & cpu_to_hc32(ehci,
 122                                         1 << (8 + uframe)))
 123                                 usecs += q->qh->c_usecs;
 124                         hw_p = &hw->hw_next;
 125                         q = &q->qh->qh_next;
 126                         break;
 127                 // case Q_TYPE_FSTN:
 128                 default:
 129                         /* for "save place" FSTNs, count the relevant INTR
 130                          * bandwidth from the previous frame
 131                          */
 132                         if (q->fstn->hw_prev != EHCI_LIST_END(ehci)) {
 133                                 ehci_dbg (ehci, "ignoring FSTN cost ...\n");
 134                         }
 135                         hw_p = &q->fstn->hw_next;
 136                         q = &q->fstn->fstn_next;
 137                         break;
 138                 case Q_TYPE_ITD:
 139                         if (q->itd->hw_transaction[uframe])
 140                                 usecs += q->itd->stream->usecs;
 141                         hw_p = &q->itd->hw_next;
 142                         q = &q->itd->itd_next;
 143                         break;
 144                 case Q_TYPE_SITD:
 145                         /* is it in the S-mask?  (count SPLIT, DATA) */
 146                         if (q->sitd->hw_uframe & cpu_to_hc32(ehci,
 147                                         1 << uframe)) {
 148                                 if (q->sitd->hw_fullspeed_ep &
 149                                                 cpu_to_hc32(ehci, 1<<31))
 150                                         usecs += q->sitd->stream->usecs;
 151                                 else    /* worst case for OUT start-split */
 152                                         usecs += HS_USECS_ISO (188);
 153                         }
 154
 155                         /* ... C-mask?  (count CSPLIT, DATA) */
 156                         if (q->sitd->hw_uframe &
 157                                         cpu_to_hc32(ehci, 1 << (8 + uframe))) {
 158                                 /* worst case for IN complete-split */
 159                                 usecs += q->sitd->stream->c_usecs;
 160                         }
 161
 162                         hw_p = &q->sitd->hw_next;
 163                         q = &q->sitd->sitd_next;
 164                         break;
 165                 }
 166         }
 167 #ifdef  DEBUG
 168         if (usecs > 100)
 169                 ehci_err (ehci, "uframe %d sched overrun: %d usecs\n",
 170                         frame * 8 + uframe, usecs);
 171 #endif
 172         return usecs;
 173 }
 174
 175 /*-------------------------------------------------------------------------*/
 176
 177 static int same_tt (struct usb_device *dev1, struct usb_device *dev2)
 178 {
 179         if (!dev1->tt || !dev2->tt)
 180                 return 0;
 181         if (dev1->tt != dev2->tt)
 182                 return 0;
 183         if (dev1->tt->multi)
 184                 return dev1->ttport == dev2->ttport;
 185         else
 186                 return 1;
 187 }
 188
 189 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
 190
 191 /* Which uframe does the low/fullspeed transfer start in?
 192  *
 193  * The parameter is the mask of ssplits in "H-frame" terms
 194  * and this returns the transfer start uframe in "B-frame" terms,
 195  * which allows both to match, e.g. a ssplit in "H-frame" uframe 0
 196  * will cause a transfer in "B-frame" uframe 0.  "B-frames" lag
 197  * "H-frames" by 1 uframe.  See the EHCI spec sec 4.5 and figure 4.7.
 198  */
 199 static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask)
 200 {
 201         unsigned char smask = QH_SMASK & hc32_to_cpu(ehci, mask);
 202         if (!smask) {
 203                 ehci_err(ehci, "invalid empty smask!\n");
 204                 /* uframe 7 can't have bw so this will indicate failure */
 205                 return 7;
 206         }
 207         return ffs(smask) - 1;
 208 }
 209
 210 static const unsigned char
 211 max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
 212
 213 /* carryover low/fullspeed bandwidth that crosses uframe boundries */
 214 static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8])
 215 {
 216         int i;
 217         for (i=0; i<7; i++) {
 218                 if (max_tt_usecs[i] < tt_usecs[i]) {
 219                         tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i];
 220                         tt_usecs[i] = max_tt_usecs[i];
 221                 }
 222         }
 223 }
 224
 225 /* How many of the tt's periodic downstream 1000 usecs are allocated?
 226  *
 227  * While this measures the bandwidth in terms of usecs/uframe,
 228  * the low/fullspeed bus has no notion of uframes, so any particular
 229  * low/fullspeed transfer can "carry over" from one uframe to the next,
 230  * since the TT just performs downstream transfers in sequence.
 231  *
 232  * For example two separate 100 usec transfers can start in the same uframe,
 233  * and the second one would "carry over" 75 usecs into the next uframe.
 234  */
 235 static void
 236 periodic_tt_usecs (
 237         struct ehci_hcd *ehci,
 238         struct usb_device *dev,
 239         unsigned frame,
 240         unsigned short tt_usecs[8]
 241 )
 242 {
 243         __hc32                  *hw_p = &ehci->periodic [frame];
 244         union ehci_shadow       *q = &ehci->pshadow [frame];
 245         unsigned char           uf;
 246
 247         memset(tt_usecs, 0, 16);
 248
 249         while (q->ptr) {
 250                 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
 251                 case Q_TYPE_ITD:
 252                         hw_p = &q->itd->hw_next;
 253                         q = &q->itd->itd_next;
 254                         continue;
 255                 case Q_TYPE_QH:
 256                         if (same_tt(dev, q->qh->dev)) {
 257                                 uf = tt_start_uframe(ehci, q->qh->hw->hw_info2);
 258                                 tt_usecs[uf] += q->qh->tt_usecs;
 259                         }
 260                         hw_p = &q->qh->hw->hw_next;
 261                         q = &q->qh->qh_next;
 262                         continue;
 263                 case Q_TYPE_SITD:
 264                         if (same_tt(dev, q->sitd->urb->dev)) {
 265                                 uf = tt_start_uframe(ehci, q->sitd->hw_uframe);
 266                                 tt_usecs[uf] += q->sitd->stream->tt_usecs;
 267                         }
 268                         hw_p = &q->sitd->hw_next;
 269                         q = &q->sitd->sitd_next;
 270                         continue;
 271                 // case Q_TYPE_FSTN:
 272                 default:
 273                         ehci_dbg(ehci, "ignoring periodic frame %d FSTN\n",
 274                                         frame);
 275                         hw_p = &q->fstn->hw_next;
 276                         q = &q->fstn->fstn_next;
 277                 }
 278         }
 279
 280         carryover_tt_bandwidth(tt_usecs);
 281
 282         if (max_tt_usecs[7] < tt_usecs[7])
 283                 ehci_err(ehci, "frame %d tt sched overrun: %d usecs\n",
 284                         frame, tt_usecs[7] - max_tt_usecs[7]);
 285 }
 286
 287 /*
 288  * Return true if the device's tt's downstream bus is available for a
 289  * periodic transfer of the specified length (usecs), starting at the
 290  * specified frame/uframe.  Note that (as summarized in section 11.19
 291  * of the usb 2.0 spec) TTs can buffer multiple transactions for each
 292  * uframe.
 293  *
 294  * The uframe parameter is when the fullspeed/lowspeed transfer
 295  * should be executed in "B-frame" terms, which is the same as the
 296  * highspeed ssplit's uframe (which is in "H-frame" terms).  For example
 297  * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0.
 298  * See the EHCI spec sec 4.5 and fig 4.7.
 299  *
 300  * This checks if the full/lowspeed bus, at the specified starting uframe,
 301  * has the specified bandwidth available, according to rules listed
 302  * in USB 2.0 spec section 11.18.1 fig 11-60.
 303  *
 304  * This does not check if the transfer would exceed the max ssplit
 305  * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4,
 306  * since proper scheduling limits ssplits to less than 16 per uframe.
 307  */
 308 static int tt_available (
 309         struct ehci_hcd         *ehci,
 310         unsigned                period,
 311         struct usb_device       *dev,
 312         unsigned                frame,
 313         unsigned                uframe,
 314         u16                     usecs
 315 )
 316 {
 317         if ((period == 0) || (uframe >= 7))     /* error */
 318                 return 0;
 319
 320         for (; frame < ehci->periodic_size; frame += period) {
 321                 unsigned short tt_usecs[8];
 322
 323                 periodic_tt_usecs (ehci, dev, frame, tt_usecs);
 324
 325                 ehci_vdbg(ehci, "tt frame %d check %d usecs start uframe %d in"
 326                         " schedule %d/%d/%d/%d/%d/%d/%d/%d\n",
 327                         frame, usecs, uframe,
 328                         tt_usecs[0], tt_usecs[1], tt_usecs[2], tt_usecs[3],
 329                         tt_usecs[4], tt_usecs[5], tt_usecs[6], tt_usecs[7]);
 330
 331                 if (max_tt_usecs[uframe] <= tt_usecs[uframe]) {
 332                         ehci_vdbg(ehci, "frame %d uframe %d fully scheduled\n",
 333                                 frame, uframe);
 334                         return 0;
 335                 }
 336
 337                 /* special case for isoc transfers larger than 125us:
 338                  * the first and each subsequent fully used uframe
 339                  * must be empty, so as to not illegally delay
 340                  * already scheduled transactions
 341                  */
 342                 if (125 < usecs) {
 343                         int ufs = (usecs / 125);
 344                         int i;
 345                         for (i = uframe; i < (uframe + ufs) && i < 8; i++)
 346                                 if (0 < tt_usecs[i]) {
 347                                         ehci_vdbg(ehci,
 348                                                 "multi-uframe xfer can't fit "
 349                                                 "in frame %d uframe %d\n",
 350                                                 frame, i);
 351                                         return 0;
 352                                 }
 353                 }
 354
 355                 tt_usecs[uframe] += usecs;
 356
 357                 carryover_tt_bandwidth(tt_usecs);
 358
 359                 /* fail if the carryover pushed bw past the last uframe's limit */
 360                 if (max_tt_usecs[7] < tt_usecs[7]) {
 361                         ehci_vdbg(ehci,
 362                                 "tt unavailable usecs %d frame %d uframe %d\n",
 363                                 usecs, frame, uframe);
 364                         return 0;
 365                 }
 366         }
 367
 368         return 1;
 369 }
 370
 371 #else
 372
 373 /* return true iff the device's transaction translator is available
 374  * for a periodic transfer starting at the specified frame, using
 375  * all the uframes in the mask.
 376  */
 377 static int tt_no_collision (
 378         struct ehci_hcd         *ehci,
 379         unsigned                period,
 380         struct usb_device       *dev,
 381         unsigned                frame,
 382         u32                     uf_mask
 383 )
 384 {
 385         if (period == 0)        /* error */
 386                 return 0;
 387
 388         /* note bandwidth wastage:  split never follows csplit
 389          * (different dev or endpoint) until the next uframe.
 390          * calling convention doesn't make that distinction.
 391          */
 392         for (; frame < ehci->periodic_size; frame += period) {
 393                 union ehci_shadow       here;
 394                 __hc32                  type;
 395                 struct ehci_qh_hw       *hw;
 396
 397                 here = ehci->pshadow [frame];
 398                 type = Q_NEXT_TYPE(ehci, ehci->periodic [frame]);
 399                 while (here.ptr) {
 400                         switch (hc32_to_cpu(ehci, type)) {
 401                         case Q_TYPE_ITD:
 402                                 type = Q_NEXT_TYPE(ehci, here.itd->hw_next);
 403                                 here = here.itd->itd_next;
 404                                 continue;
 405                         case Q_TYPE_QH:
 406                                 hw = here.qh->hw;
 407                                 if (same_tt (dev, here.qh->dev)) {
 408                                         u32             mask;
 409
 410                                         mask = hc32_to_cpu(ehci,
 411                                                         hw->hw_info2);
 412                                         /* "knows" no gap is needed */
 413                                         mask |= mask >> 8;
 414                                         if (mask & uf_mask)
 415                                                 break;
 416                                 }
 417                                 type = Q_NEXT_TYPE(ehci, hw->hw_next);
 418                                 here = here.qh->qh_next;
 419                                 continue;
 420                         case Q_TYPE_SITD:
 421                                 if (same_tt (dev, here.sitd->urb->dev)) {
 422                                         u16             mask;
 423
 424                                         mask = hc32_to_cpu(ehci, here.sitd
 425                                                                 ->hw_uframe);
 426                                         mask |= mask >> 8;
 427                                         if (mask & uf_mask)
 428                                                 break;
 429                                 }
 430                                 type = Q_NEXT_TYPE(ehci, here.sitd->hw_next);
 431                                 here = here.sitd->sitd_next;
 432                                 continue;
 433                         // case Q_TYPE_FSTN:
 434                         default:
 435                                 ehci_dbg (ehci,
 436                                         "periodic frame %d bogus type %d\n",
 437                                         frame, type);
 438                         }
 439
 440                         /* collision or error */
 441                         return 0;
 442                 }
 443         }
 444
 445         /* no collision */
 446         return 1;
 447 }
 448
 449 #endif /* CONFIG_USB_EHCI_TT_NEWSCHED */
 450
 451 /*-------------------------------------------------------------------------*/
 452
 453 static int enable_periodic (struct ehci_hcd *ehci)
 454 {
 455         u32     cmd;
 456         int     status;
 457
 458         if (ehci->periodic_sched++)
 459                 return 0;
 460
 461         /* did clearing PSE did take effect yet?
 462          * takes effect only at frame boundaries...
 463          */
 464         status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
 465                                              STS_PSS, 0, 9 * 125);
 466         if (status)
 467                 return status;
 468
 469         cmd = ehci_readl(ehci, &ehci->regs->command) | CMD_PSE;
 470         ehci_writel(ehci, cmd, &ehci->regs->command);
 471         /* posted write ... PSS happens later */
 472         ehci_to_hcd(ehci)->state = HC_STATE_RUNNING;
 473
 474         /* make sure ehci_work scans these */
 475         ehci->next_uframe = ehci_readl(ehci, &ehci->regs->frame_index)
 476                 % (ehci->periodic_size << 3);
 477         if (unlikely(ehci->broken_periodic))
 478                 ehci->last_periodic_enable = ktime_get_real();
 479         return 0;
 480 }
 481
 482 static int disable_periodic (struct ehci_hcd *ehci)
 483 {
 484         u32     cmd;
 485         int     status;
 486
 487         if (--ehci->periodic_sched)
 488                 return 0;
 489
 490         if (unlikely(ehci->broken_periodic)) {
 491                 /* delay experimentally determined */
 492                 ktime_t safe = ktime_add_us(ehci->last_periodic_enable, 1000);
 493                 ktime_t now = ktime_get_real();
 494                 s64 delay = ktime_us_delta(safe, now);
 495
 496                 if (unlikely(delay > 0))
 497                         udelay(delay);
 498         }
 499
 500         /* did setting PSE not take effect yet?
 501          * takes effect only at frame boundaries...
 502          */
 503         status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
 504                                              STS_PSS, STS_PSS, 9 * 125);
 505         if (status)
 506                 return status;
 507
 508         cmd = ehci_readl(ehci, &ehci->regs->command) & ~CMD_PSE;
 509         ehci_writel(ehci, cmd, &ehci->regs->command);
 510         /* posted write ... */
 511
 512         free_cached_lists(ehci);
 513
 514         ehci->next_uframe = -1;
 515         return 0;
 516 }
 517
 518 /*-------------------------------------------------------------------------*/
 519
 520 /* periodic schedule slots have iso tds (normal or split) first, then a
 521  * sparse tree for active interrupt transfers.
 522  *
 523  * this just links in a qh; caller guarantees uframe masks are set right.
 524  * no FSTN support (yet; ehci 0.96+)
 525  */
 526 static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh)
 527 {
 528         unsigned        i;
 529         unsigned        period = qh->period;
 530
 531         dev_dbg (&qh->dev->dev,
 532                 "link qh%d-%04x/%p start %d [%d/%d us]\n",
 533                 period, hc32_to_cpup(ehci, &qh->hw->hw_info2)
 534                         & (QH_CMASK | QH_SMASK),
 535                 qh, qh->start, qh->usecs, qh->c_usecs);
 536
 537         /* high bandwidth, or otherwise every microframe */
 538         if (period == 0)
 539                 period = 1;
 540
 541         for (i = qh->start; i < ehci->periodic_size; i += period) {
 542                 union ehci_shadow       *prev = &ehci->pshadow[i];
 543                 __hc32                  *hw_p = &ehci->periodic[i];
 544                 union ehci_shadow       here = *prev;
 545                 __hc32                  type = 0;
 546
 547                 /* skip the iso nodes at list head */
 548                 while (here.ptr) {
 549                         type = Q_NEXT_TYPE(ehci, *hw_p);
 550                         if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
 551                                 break;
 552                         prev = periodic_next_shadow(ehci, prev, type);
 553                         hw_p = shadow_next_periodic(ehci, &here, type);
 554                         here = *prev;
 555                 }
 556
 557                 /* sorting each branch by period (slow-->fast)
 558                  * enables sharing interior tree nodes
 559                  */
 560                 while (here.ptr && qh != here.qh) {
 561                         if (qh->period > here.qh->period)
 562                                 break;
 563                         prev = &here.qh->qh_next;
 564                         hw_p = &here.qh->hw->hw_next;
 565                         here = *prev;
 566                 }
 567                 /* link in this qh, unless some earlier pass did that */
 568                 if (qh != here.qh) {
 569                         qh->qh_next = here;
 570                         if (here.qh)
 571                                 qh->hw->hw_next = *hw_p;
 572                         wmb ();
 573                         prev->qh = qh;
 574                         *hw_p = QH_NEXT (ehci, qh->qh_dma);
 575                 }
 576         }
 577         qh->qh_state = QH_STATE_LINKED;
 578         qh->xacterrs = 0;
 579         qh_get (qh);
 580
 581         /* update per-qh bandwidth for usbfs */
 582         ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period
 583                 ? ((qh->usecs + qh->c_usecs) / qh->period)
 584                 : (qh->usecs * 8);
 585
 586         /* maybe enable periodic schedule processing */
 587         return enable_periodic(ehci);
 588 }
 589
 590 static int qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
 591 {
 592         unsigned        i;
 593         unsigned        period;
 594
 595         // IF this isn't high speed
 596         //   and this qh is active in the current uframe
 597         //   (and overlay token SplitXstate is false?)
 598         // THEN
 599         //   qh->hw_info1 |= cpu_to_hc32(1 << 7 /* "ignore" */);
 600
 601         /* high bandwidth, or otherwise part of every microframe */
 602         if ((period = qh->period) == 0)
 603                 period = 1;
 604
 605         for (i = qh->start; i < ehci->periodic_size; i += period)
 606                 periodic_unlink (ehci, i, qh);
 607
 608         /* update per-qh bandwidth for usbfs */
 609         ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period
 610                 ? ((qh->usecs + qh->c_usecs) / qh->period)
 611                 : (qh->usecs * 8);
 612
 613         dev_dbg (&qh->dev->dev,
 614                 "unlink qh%d-%04x/%p start %d [%d/%d us]\n",
 615                 qh->period,
 616                 hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK),
 617                 qh, qh->start, qh->usecs, qh->c_usecs);
 618
 619         /* qh->qh_next still "live" to HC */
 620         qh->qh_state = QH_STATE_UNLINK;
 621         qh->qh_next.ptr = NULL;
 622         qh_put (qh);
 623
 624         /* maybe turn off periodic schedule */
 625         return disable_periodic(ehci);
 626 }
 627
 628 static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh)
 629 {
 630         unsigned                wait;
 631         struct ehci_qh_hw       *hw = qh->hw;
 632         int                     rc;
 633
 634         /* If the QH isn't linked then there's nothing we can do
 635          * unless we were called during a giveback, in which case
 636          * qh_completions() has to deal with it.
 637          */
 638         if (qh->qh_state != QH_STATE_LINKED) {
 639                 if (qh->qh_state == QH_STATE_COMPLETING)
 640                         qh->needs_rescan = 1;
 641                 return;
 642         }
 643
 644         qh_unlink_periodic (ehci, qh);
 645
 646         /* simple/paranoid:  always delay, expecting the HC needs to read
 647          * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and
 648          * expect khubd to clean up after any CSPLITs we won't issue.
 649          * active high speed queues may need bigger delays...
 650          */
 651         if (list_empty (&qh->qtd_list)
 652                         || (cpu_to_hc32(ehci, QH_CMASK)
 653                                         & hw->hw_info2) != 0)
 654                 wait = 2;
 655         else
 656                 wait = 55;      /* worst case: 3 * 1024 */
 657
 658         udelay (wait);
 659         qh->qh_state = QH_STATE_IDLE;
 660         hw->hw_next = EHCI_LIST_END(ehci);
 661         wmb ();
 662
 663         qh_completions(ehci, qh);
 664
 665         /* reschedule QH iff another request is queued */
 666         if (!list_empty(&qh->qtd_list) &&
 667                         HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
 668                 rc = qh_schedule(ehci, qh);
 669
 670                 if (rc != 0)
 671                         ehci_err(ehci, "can't reschedule qh %p, err %d\n",
 672                                         qh, rc);
 673         }
 674 }
 675
 676 /*-------------------------------------------------------------------------*/
 677
 678 static int check_period (
 679         struct ehci_hcd *ehci,
 680         unsigned        frame,
 681         unsigned        uframe,
 682         unsigned        period,
 683         unsigned        usecs
 684 ) {
 685         int             claimed;
 686
 687         /* complete split running into next frame?
 688          * given FSTN support, we could sometimes check...
 689          */
 690         if (uframe >= 8)
 691                 return 0;
 692
 693         /*
 694          * 80% periodic == 100 usec/uframe available
 695          * convert "usecs we need" to "max already claimed"
 696          */
 697         usecs = 100 - usecs;
 698
 699         /* we "know" 2 and 4 uframe intervals were rejected; so
 700          * for period 0, check _every_ microframe in the schedule.
 701          */
 702         if (unlikely (period == 0)) {
 703                 do {
 704                         for (uframe = 0; uframe < 7; uframe++) {
 705                                 claimed = periodic_usecs (ehci, frame, uframe);
 706                                 if (claimed > usecs)
 707                                         return 0;
 708                         }
 709                 } while ((frame += 1) < ehci->periodic_size);
 710
 711         /* just check the specified uframe, at that period */
 712         } else {
 713                 do {
 714                         claimed = periodic_usecs (ehci, frame, uframe);
 715                         if (claimed > usecs)
 716                                 return 0;
 717                 } while ((frame += period) < ehci->periodic_size);
 718         }
 719
 720         // success!
 721         return 1;
 722 }
 723
 724 static int check_intr_schedule (
 725         struct ehci_hcd         *ehci,
 726         unsigned                frame,
 727         unsigned                uframe,
 728         const struct ehci_qh    *qh,
 729         __hc32                  *c_maskp
 730 )
 731 {
 732         int             retval = -ENOSPC;
 733         u8              mask = 0;
 734
 735         if (qh->c_usecs && uframe >= 6)         /* FSTN territory? */
 736                 goto done;
 737
 738         if (!check_period (ehci, frame, uframe, qh->period, qh->usecs))
 739                 goto done;
 740         if (!qh->c_usecs) {
 741                 retval = 0;
 742                 *c_maskp = 0;
 743                 goto done;
 744         }
 745
 746 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
 747         if (tt_available (ehci, qh->period, qh->dev, frame, uframe,
 748                                 qh->tt_usecs)) {
 749                 unsigned i;
 750
 751                 /* TODO : this may need FSTN for SSPLIT in uframe 5. */
 752                 for (i=uframe+1; i<8 && i<uframe+4; i++)
 753                         if (!check_period (ehci, frame, i,
 754                                                 qh->period, qh->c_usecs))
 755                                 goto done;
 756                         else
 757                                 mask |= 1 << i;
 758
 759                 retval = 0;
 760
 761                 *c_maskp = cpu_to_hc32(ehci, mask << 8);
 762         }
 763 #else
 764         /* Make sure this tt's buffer is also available for CSPLITs.
 765          * We pessimize a bit; probably the typical full speed case
 766          * doesn't need the second CSPLIT.
 767          *
 768          * NOTE:  both SPLIT and CSPLIT could be checked in just
 769          * one smart pass...
 770          */
 771         mask = 0x03 << (uframe + qh->gap_uf);
 772         *c_maskp = cpu_to_hc32(ehci, mask << 8);
 773
 774         mask |= 1 << uframe;
 775         if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) {
 776                 if (!check_period (ehci, frame, uframe + qh->gap_uf + 1,
 777                                         qh->period, qh->c_usecs))
 778                         goto done;
 779                 if (!check_period (ehci, frame, uframe + qh->gap_uf,
 780                                         qh->period, qh->c_usecs))
 781                         goto done;
 782                 retval = 0;
 783         }
 784 #endif
 785 done:
 786         return retval;
 787 }
 788
 789 /* "first fit" scheduling policy used the first time through,
 790  * or when the previous schedule slot can't be re-used.
 791  */
 792 static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh)
 793 {
 794         int             status;
 795         unsigned        uframe;
 796         __hc32          c_mask;
 797         unsigned        frame;          /* 0..(qh->period - 1), or NO_FRAME */
 798         struct ehci_qh_hw       *hw = qh->hw;
 799
 800         qh_refresh(ehci, qh);
 801         hw->hw_next = EHCI_LIST_END(ehci);
 802         frame = qh->start;
 803
 804         /* reuse the previous schedule slots, if we can */
 805         if (frame < qh->period) {
 806                 uframe = ffs(hc32_to_cpup(ehci, &hw->hw_info2) & QH_SMASK);
 807                 status = check_intr_schedule (ehci, frame, --uframe,
 808                                 qh, &c_mask);
 809         } else {
 810                 uframe = 0;
 811                 c_mask = 0;
 812                 status = -ENOSPC;
 813         }
 814
 815         /* else scan the schedule to find a group of slots such that all
 816          * uframes have enough periodic bandwidth available.
 817          */
 818         if (status) {
 819                 /* "normal" case, uframing flexible except with splits */
 820                 if (qh->period) {
 821                         int             i;
 822
 823                         for (i = qh->period; status && i > 0; --i) {
 824                                 frame = ++ehci->random_frame % qh->period;
 825                                 for (uframe = 0; uframe < 8; uframe++) {
 826                                         status = check_intr_schedule (ehci,
 827                                                         frame, uframe, qh,
 828                                                         &c_mask);
 829                                         if (status == 0)
 830                                                 break;
 831                                 }
 832                         }
 833
 834                 /* qh->period == 0 means every uframe */
 835                 } else {
 836                         frame = 0;
 837                         status = check_intr_schedule (ehci, 0, 0, qh, &c_mask);
 838                 }
 839                 if (status)
 840                         goto done;
 841                 qh->start = frame;
 842
 843                 /* reset S-frame and (maybe) C-frame masks */
 844                 hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK));
 845                 hw->hw_info2 |= qh->period
 846                         ? cpu_to_hc32(ehci, 1 << uframe)
 847                         : cpu_to_hc32(ehci, QH_SMASK);
 848                 hw->hw_info2 |= c_mask;
 849         } else
 850                 ehci_dbg (ehci, "reused qh %p schedule\n", qh);
 851
 852         /* stuff into the periodic schedule */
 853         status = qh_link_periodic (ehci, qh);
 854 done:
 855         return status;
 856 }
 857
 858 static int intr_submit (
 859         struct ehci_hcd         *ehci,
 860         struct urb              *urb,
 861         struct list_head        *qtd_list,
 862         gfp_t                   mem_flags
 863 ) {
 864         unsigned                epnum;
 865         unsigned long           flags;
 866         struct ehci_qh          *qh;
 867         int                     status;
 868         struct list_head        empty;
 869
 870         /* get endpoint and transfer/schedule data */
 871         epnum = urb->ep->desc.bEndpointAddress;
 872
 873         spin_lock_irqsave (&ehci->lock, flags);
 874
 875         if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
 876                 status = -ESHUTDOWN;
 877                 goto done_not_linked;
 878         }
 879         status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
 880         if (unlikely(status))
 881                 goto done_not_linked;
 882
 883         /* get qh and force any scheduling errors */
 884         INIT_LIST_HEAD (&empty);
 885         qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv);
 886         if (qh == NULL) {
 887                 status = -ENOMEM;
 888                 goto done;
 889         }
 890         if (qh->qh_state == QH_STATE_IDLE) {
 891                 if ((status = qh_schedule (ehci, qh)) != 0)
 892                         goto done;
 893         }
 894
 895         /* then queue the urb's tds to the qh */
 896         qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
 897         BUG_ON (qh == NULL);
 898
 899         /* ... update usbfs periodic stats */
 900         ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
 901
 902 done:
 903         if (unlikely(status))
 904                 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
 905 done_not_linked:
 906         spin_unlock_irqrestore (&ehci->lock, flags);
 907         if (status)
 908                 qtd_list_free (ehci, urb, qtd_list);
 909
 910         return status;
 911 }
 912
 913 /*-------------------------------------------------------------------------*/
 914
 915 /* ehci_iso_stream ops work with both ITD and SITD */
 916
 917 static struct ehci_iso_stream *
 918 iso_stream_alloc (gfp_t mem_flags)
 919 {
 920         struct ehci_iso_stream *stream;
 921
 922         stream = kzalloc(sizeof *stream, mem_flags);
 923         if (likely (stream != NULL)) {
 924                 INIT_LIST_HEAD(&stream->td_list);
 925                 INIT_LIST_HEAD(&stream->free_list);
 926                 stream->next_uframe = -1;
 927                 stream->refcount = 1;
 928         }
 929         return stream;
 930 }
 931
 932 static void
 933 iso_stream_init (
 934         struct ehci_hcd         *ehci,
 935         struct ehci_iso_stream  *stream,
 936         struct usb_device       *dev,
 937         int                     pipe,
 938         unsigned                interval
 939 )
 940 {
 941         static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
 942
 943         u32                     buf1;
 944         unsigned                epnum, maxp;
 945         int                     is_input;
 946         long                    bandwidth;
 947
 948         /*
 949          * this might be a "high bandwidth" highspeed endpoint,
 950          * as encoded in the ep descriptor's wMaxPacket field
 951          */
 952         epnum = usb_pipeendpoint (pipe);
 953         is_input = usb_pipein (pipe) ? USB_DIR_IN : 0;
 954         maxp = usb_maxpacket(dev, pipe, !is_input);
 955         if (is_input) {
 956                 buf1 = (1 << 11);
 957         } else {
 958                 buf1 = 0;
 959         }
 960
 961         /* knows about ITD vs SITD */
 962         if (dev->speed == USB_SPEED_HIGH) {
 963                 unsigned multi = hb_mult(maxp);
 964
 965                 stream->highspeed = 1;
 966
 967                 maxp = max_packet(maxp);
 968                 buf1 |= maxp;
 969                 maxp *= multi;
 970
 971                 stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum);
 972                 stream->buf1 = cpu_to_hc32(ehci, buf1);
 973                 stream->buf2 = cpu_to_hc32(ehci, multi);
 974
 975                 /* usbfs wants to report the average usecs per frame tied up
 976                  * when transfers on this endpoint are scheduled ...
 977                  */
 978                 stream->usecs = HS_USECS_ISO (maxp);
 979                 bandwidth = stream->usecs * 8;
 980                 bandwidth /= interval;
 981
 982         } else {
 983                 u32             addr;
 984                 int             think_time;
 985                 int             hs_transfers;
 986
 987                 addr = dev->ttport << 24;
 988                 if (!ehci_is_TDI(ehci)
 989                                 || (dev->tt->hub !=
 990                                         ehci_to_hcd(ehci)->self.root_hub))
 991                         addr |= dev->tt->hub->devnum << 16;
 992                 addr |= epnum << 8;
 993                 addr |= dev->devnum;
 994                 stream->usecs = HS_USECS_ISO (maxp);
 995                 think_time = dev->tt ? dev->tt->think_time : 0;
 996                 stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time (
 997                                 dev->speed, is_input, 1, maxp));
 998                 hs_transfers = max (1u, (maxp + 187) / 188);
 999                 if (is_input) {
1000                         u32     tmp;
1001
1002                         addr |= 1 << 31;
1003                         stream->c_usecs = stream->usecs;
1004                         stream->usecs = HS_USECS_ISO (1);
1005                         stream->raw_mask = 1;
1006
1007                         /* c-mask as specified in USB 2.0 11.18.4 3.c */
1008                         tmp = (1 << (hs_transfers + 2)) - 1;
1009                         stream->raw_mask |= tmp << (8 + 2);
1010                 } else
1011                         stream->raw_mask = smask_out [hs_transfers - 1];
1012                 bandwidth = stream->usecs + stream->c_usecs;
1013                 bandwidth /= interval << 3;
1014
1015                 /* stream->splits gets created from raw_mask later */
1016                 stream->address = cpu_to_hc32(ehci, addr);
1017         }
1018         stream->bandwidth = bandwidth;
1019
1020         stream->udev = dev;
1021
1022         stream->bEndpointAddress = is_input | epnum;
1023         stream->interval = interval;
1024         stream->maxp = maxp;
1025 }
1026
1027 static void
1028 iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream)
1029 {
1030         stream->refcount--;
1031
1032         /* free whenever just a dev->ep reference remains.
1033          * not like a QH -- no persistent state (toggle, halt)
1034          */
1035         if (stream->refcount == 1) {
1036                 int             is_in;
1037
1038                 // BUG_ON (!list_empty(&stream->td_list));
1039
1040                 while (!list_empty (&stream->free_list)) {
1041                         struct list_head        *entry;
1042
1043                         entry = stream->free_list.next;
1044                         list_del (entry);
1045
1046                         /* knows about ITD vs SITD */
1047                         if (stream->highspeed) {
1048                                 struct ehci_itd         *itd;
1049
1050                                 itd = list_entry (entry, struct ehci_itd,
1051                                                 itd_list);
1052                                 dma_pool_free (ehci->itd_pool, itd,
1053                                                 itd->itd_dma);
1054                         } else {
1055                                 struct ehci_sitd        *sitd;
1056
1057                                 sitd = list_entry (entry, struct ehci_sitd,
1058                                                 sitd_list);
1059                                 dma_pool_free (ehci->sitd_pool, sitd,
1060                                                 sitd->sitd_dma);
1061                         }
1062                 }
1063
1064                 is_in = (stream->bEndpointAddress & USB_DIR_IN) ? 0x10 : 0;
1065                 stream->bEndpointAddress &= 0x0f;
1066                 if (stream->ep)
1067                         stream->ep->hcpriv = NULL;
1068
1069                 kfree(stream);
1070         }
1071 }
1072
1073 static inline struct ehci_iso_stream *
1074 iso_stream_get (struct ehci_iso_stream *stream)
1075 {
1076         if (likely (stream != NULL))
1077                 stream->refcount++;
1078         return stream;
1079 }
1080
1081 static struct ehci_iso_stream *
1082 iso_stream_find (struct ehci_hcd *ehci, struct urb *urb)
1083 {
1084         unsigned                epnum;
1085         struct ehci_iso_stream  *stream;
1086         struct usb_host_endpoint *ep;
1087         unsigned long           flags;
1088
1089         epnum = usb_pipeendpoint (urb->pipe);
1090         if (usb_pipein(urb->pipe))
1091                 ep = urb->dev->ep_in[epnum];
1092         else
1093                 ep = urb->dev->ep_out[epnum];
1094
1095         spin_lock_irqsave (&ehci->lock, flags);
1096         stream = ep->hcpriv;
1097
1098         if (unlikely (stream == NULL)) {
1099                 stream = iso_stream_alloc(GFP_ATOMIC);
1100                 if (likely (stream != NULL)) {
1101                         /* dev->ep owns the initial refcount */
1102                         ep->hcpriv = stream;
1103                         stream->ep = ep;
1104                         iso_stream_init(ehci, stream, urb->dev, urb->pipe,
1105                                         urb->interval);
1106                 }
1107
1108         /* if dev->ep [epnum] is a QH, hw is set */
1109         } else if (unlikely (stream->hw != NULL)) {
1110                 ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n",
1111                         urb->dev->devpath, epnum,
1112                         usb_pipein(urb->pipe) ? "in" : "out");
1113                 stream = NULL;
1114         }
1115
1116         /* caller guarantees an eventual matching iso_stream_put */
1117         stream = iso_stream_get (stream);
1118
1119         spin_unlock_irqrestore (&ehci->lock, flags);
1120         return stream;
1121 }
1122
1123 /*-------------------------------------------------------------------------*/
1124
1125 /* ehci_iso_sched ops can be ITD-only or SITD-only */
1126
1127 static struct ehci_iso_sched *
1128 iso_sched_alloc (unsigned packets, gfp_t mem_flags)
1129 {
1130         struct ehci_iso_sched   *iso_sched;
1131         int                     size = sizeof *iso_sched;
1132
1133         size += packets * sizeof (struct ehci_iso_packet);
1134         iso_sched = kzalloc(size, mem_flags);
1135         if (likely (iso_sched != NULL)) {
1136                 INIT_LIST_HEAD (&iso_sched->td_list);
1137         }
1138         return iso_sched;
1139 }
1140
1141 static inline void
1142 itd_sched_init(
1143         struct ehci_hcd         *ehci,
1144         struct ehci_iso_sched   *iso_sched,
1145         struct ehci_iso_stream  *stream,
1146         struct urb              *urb
1147 )
1148 {
1149         unsigned        i;
1150         dma_addr_t      dma = urb->transfer_dma;
1151
1152         /* how many uframes are needed for these transfers */
1153         iso_sched->span = urb->number_of_packets * stream->interval;
1154
1155         /* figure out per-uframe itd fields that we'll need later
1156          * when we fit new itds into the schedule.
1157          */
1158         for (i = 0; i < urb->number_of_packets; i++) {
1159                 struct ehci_iso_packet  *uframe = &iso_sched->packet [i];
1160                 unsigned                length;
1161                 dma_addr_t              buf;
1162                 u32                     trans;
1163
1164                 length = urb->iso_frame_desc [i].length;
1165                 buf = dma + urb->iso_frame_desc [i].offset;
1166
1167                 trans = EHCI_ISOC_ACTIVE;
1168                 trans |= buf & 0x0fff;
1169                 if (unlikely (((i + 1) == urb->number_of_packets))
1170                                 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1171                         trans |= EHCI_ITD_IOC;
1172                 trans |= length << 16;
1173                 uframe->transaction = cpu_to_hc32(ehci, trans);
1174
1175                 /* might need to cross a buffer page within a uframe */
1176                 uframe->bufp = (buf & ~(u64)0x0fff);
1177                 buf += length;
1178                 if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff))))
1179                         uframe->cross = 1;
1180         }
1181 }
1182
1183 static void
1184 iso_sched_free (
1185         struct ehci_iso_stream  *stream,
1186         struct ehci_iso_sched   *iso_sched
1187 )
1188 {
1189         if (!iso_sched)
1190                 return;
1191         // caller must hold ehci->lock!
1192         list_splice (&iso_sched->td_list, &stream->free_list);
1193         kfree (iso_sched);
1194 }
1195
1196 static int
1197 itd_urb_transaction (
1198         struct ehci_iso_stream  *stream,
1199         struct ehci_hcd         *ehci,
1200         struct urb              *urb,
1201         gfp_t                   mem_flags
1202 )
1203 {
1204         struct ehci_itd         *itd;
1205         dma_addr_t              itd_dma;
1206         int                     i;
1207         unsigned                num_itds;
1208         struct ehci_iso_sched   *sched;
1209         unsigned long           flags;
1210
1211         sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1212         if (unlikely (sched == NULL))
1213                 return -ENOMEM;
1214
1215         itd_sched_init(ehci, sched, stream, urb);
1216
1217         if (urb->interval < 8)
1218                 num_itds = 1 + (sched->span + 7) / 8;
1219         else
1220                 num_itds = urb->number_of_packets;
1221
1222         /* allocate/init ITDs */
1223         spin_lock_irqsave (&ehci->lock, flags);
1224         for (i = 0; i < num_itds; i++) {
1225
1226                 /* free_list.next might be cache-hot ... but maybe
1227                  * the HC caches it too. avoid that issue for now.
1228                  */
1229
1230                 /* prefer previously-allocated itds */
1231                 if (likely (!list_empty(&stream->free_list))) {
1232                         itd = list_entry (stream->free_list.prev,
1233                                         struct ehci_itd, itd_list);
1234                         list_del (&itd->itd_list);
1235                         itd_dma = itd->itd_dma;
1236                 } else {
1237                         spin_unlock_irqrestore (&ehci->lock, flags);
1238                         itd = dma_pool_alloc (ehci->itd_pool, mem_flags,
1239                                         &itd_dma);
1240                         spin_lock_irqsave (&ehci->lock, flags);
1241                         if (!itd) {
1242                                 iso_sched_free(stream, sched);
1243                                 spin_unlock_irqrestore(&ehci->lock, flags);
1244                                 return -ENOMEM;
1245                         }
1246                 }
1247
1248                 memset (itd, 0, sizeof *itd);
1249                 itd->itd_dma = itd_dma;
1250                 list_add (&itd->itd_list, &sched->td_list);
1251         }
1252         spin_unlock_irqrestore (&ehci->lock, flags);
1253
1254         /* temporarily store schedule info in hcpriv */
1255         urb->hcpriv = sched;
1256         urb->error_count = 0;
1257         return 0;
1258 }
1259
1260 /*-------------------------------------------------------------------------*/
1261
1262 static inline int
1263 itd_slot_ok (
1264         struct ehci_hcd         *ehci,
1265         u32                     mod,
1266         u32                     uframe,
1267         u8                      usecs,
1268         u32                     period
1269 )
1270 {
1271         uframe %= period;
1272         do {
1273                 /* can't commit more than 80% periodic == 100 usec */
1274                 if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7)
1275                                 > (100 - usecs))
1276                         return 0;
1277
1278                 /* we know urb->interval is 2^N uframes */
1279                 uframe += period;
1280         } while (uframe < mod);
1281         return 1;
1282 }
1283
1284 static inline int
1285 sitd_slot_ok (
1286         struct ehci_hcd         *ehci,
1287         u32                     mod,
1288         struct ehci_iso_stream  *stream,
1289         u32                     uframe,
1290         struct ehci_iso_sched   *sched,
1291         u32                     period_uframes
1292 )
1293 {
1294         u32                     mask, tmp;
1295         u32                     frame, uf;
1296
1297         mask = stream->raw_mask << (uframe & 7);
1298
1299         /* for IN, don't wrap CSPLIT into the next frame */
1300         if (mask & ~0xffff)
1301                 return 0;
1302
1303         /* this multi-pass logic is simple, but performance may
1304          * suffer when the schedule data isn't cached.
1305          */
1306
1307         /* check bandwidth */
1308         uframe %= period_uframes;
1309         do {
1310                 u32             max_used;
1311
1312                 frame = uframe >> 3;
1313                 uf = uframe & 7;
1314
1315 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED
1316                 /* The tt's fullspeed bus bandwidth must be available.
1317                  * tt_available scheduling guarantees 10+% for control/bulk.
1318                  */
1319                 if (!tt_available (ehci, period_uframes << 3,
1320                                 stream->udev, frame, uf, stream->tt_usecs))
1321                         return 0;
1322 #else
1323                 /* tt must be idle for start(s), any gap, and csplit.
1324                  * assume scheduling slop leaves 10+% for control/bulk.
1325                  */
1326                 if (!tt_no_collision (ehci, period_uframes << 3,
1327                                 stream->udev, frame, mask))
1328                         return 0;
1329 #endif
1330
1331                 /* check starts (OUT uses more than one) */
1332                 max_used = 100 - stream->usecs;
1333                 for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) {
1334                         if (periodic_usecs (ehci, frame, uf) > max_used)
1335                                 return 0;
1336                 }
1337
1338                 /* for IN, check CSPLIT */
1339                 if (stream->c_usecs) {
1340                         uf = uframe & 7;
1341                         max_used = 100 - stream->c_usecs;
1342                         do {
1343                                 tmp = 1 << uf;
1344                                 tmp <<= 8;
1345                                 if ((stream->raw_mask & tmp) == 0)
1346                                         continue;
1347                                 if (periodic_usecs (ehci, frame, uf)
1348                                                 > max_used)
1349                                         return 0;
1350                         } while (++uf < 8);
1351                 }
1352
1353                 /* we know urb->interval is 2^N uframes */
1354                 uframe += period_uframes;
1355         } while (uframe < mod);
1356
1357         stream->splits = cpu_to_hc32(ehci, stream->raw_mask << (uframe & 7));
1358         return 1;
1359 }
1360
1361 /*
1362  * This scheduler plans almost as far into the future as it has actual
1363  * periodic schedule slots.  (Affected by TUNE_FLS, which defaults to
1364  * "as small as possible" to be cache-friendlier.)  That limits the size
1365  * transfers you can stream reliably; avoid more than 64 msec per urb.
1366  * Also avoid queue depths of less than ehci's worst irq latency (affected
1367  * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
1368  * and other factors); or more than about 230 msec total (for portability,
1369  * given EHCI_TUNE_FLS and the slop).  Or, write a smarter scheduler!
1370  */
1371
1372 #define SCHEDULE_SLOP   80      /* microframes */
1373
1374 static int
1375 iso_stream_schedule (
1376         struct ehci_hcd         *ehci,
1377         struct urb              *urb,
1378         struct ehci_iso_stream  *stream
1379 )
1380 {
1381         u32                     now, next, start, period, span;
1382         int                     status;
1383         unsigned                mod = ehci->periodic_size << 3;
1384         struct ehci_iso_sched   *sched = urb->hcpriv;
1385
1386         period = urb->interval;
1387         span = sched->span;
1388         if (!stream->highspeed) {
1389                 period <<= 3;
1390                 span <<= 3;
1391         }
1392
1393         if (span > mod - SCHEDULE_SLOP) {
1394                 ehci_dbg (ehci, "iso request %p too long\n", urb);
1395                 status = -EFBIG;
1396                 goto fail;
1397         }
1398
1399         now = ehci_readl(ehci, &ehci->regs->frame_index) & (mod - 1);
1400
1401         /* Typical case: reuse current schedule, stream is still active.
1402          * Hopefully there are no gaps from the host falling behind
1403          * (irq delays etc), but if there are we'll take the next
1404          * slot in the schedule, implicitly assuming URB_ISO_ASAP.
1405          */
1406         if (likely (!list_empty (&stream->td_list))) {
1407                 u32     excess;
1408
1409                 if (!stream->highspeed && ehci->fs_i_thresh)
1410                         next = now + ehci->i_thresh;
1411                 else
1412                         next = now;
1413
1414                 /* Fell behind (by up to twice the slop amount)?
1415                  * We decide based on the time of the last currently-scheduled
1416                  * slot, not the time of the next available slot.
1417                  */
1418                 excess = (stream->next_uframe - period - next) & (mod - 1);
1419                 if (excess >= mod - 2 * SCHEDULE_SLOP)
1420                         start = next + excess - mod + period *
1421                                         DIV_ROUND_UP(mod - excess, period);
1422                 else
1423                         start = next + excess + period;
1424                 if (start - now >= mod) {
1425                         ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n",
1426                                         urb, start - now - period, period,
1427                                         mod);
1428                         status = -EFBIG;
1429                         goto fail;
1430                 }
1431         }
1432
1433         /* need to schedule; when's the next (u)frame we could start?
1434          * this is bigger than ehci->i_thresh allows; scheduling itself
1435          * isn't free, the slop should handle reasonably slow cpus.  it
1436          * can also help high bandwidth if the dma and irq loads don't
1437          * jump until after the queue is primed.
1438          */
1439         else {
1440                 start = SCHEDULE_SLOP + (now & ~0x07);
1441
1442                 /* NOTE:  assumes URB_ISO_ASAP, to limit complexity/bugs */
1443
1444                 /* find a uframe slot with enough bandwidth */
1445                 next = start + period;
1446                 for (; start < next; start++) {
1447
1448                         /* check schedule: enough space? */
1449                         if (stream->highspeed) {
1450                                 if (itd_slot_ok(ehci, mod, start,
1451                                                 stream->usecs, period))
1452                                         break;
1453                         } else {
1454                                 if ((start % 8) >= 6)
1455                                         continue;
1456                                 if (sitd_slot_ok(ehci, mod, stream,
1457                                                 start, sched, period))
1458                                         break;
1459                         }
1460                 }
1461
1462                 /* no room in the schedule */
1463                 if (start == next) {
1464                         ehci_dbg(ehci, "iso resched full %p (now %d max %d)\n",
1465                                 urb, now, now + mod);
1466                         status = -ENOSPC;
1467                         goto fail;
1468                 }
1469         }
1470
1471         /* Tried to schedule too far into the future? */
1472         if (unlikely(start - now + span - period
1473                                 >= mod - 2 * SCHEDULE_SLOP)) {
1474                 ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n",
1475                                 urb, start - now, span - period,
1476                                 mod - 2 * SCHEDULE_SLOP);
1477                 status = -EFBIG;
1478                 goto fail;
1479         }
1480
1481         stream->next_uframe = start & (mod - 1);
1482
1483         /* report high speed start in uframes; full speed, in frames */
1484         urb->start_frame = stream->next_uframe;
1485         if (!stream->highspeed)
1486                 urb->start_frame >>= 3;
1487         return 0;
1488
1489  fail:
1490         iso_sched_free(stream, sched);
1491         urb->hcpriv = NULL;
1492         return status;
1493 }
1494
1495 /*-------------------------------------------------------------------------*/
1496
1497 static inline void
1498 itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream,
1499                 struct ehci_itd *itd)
1500 {
1501         int i;
1502
1503         /* it's been recently zeroed */
1504         itd->hw_next = EHCI_LIST_END(ehci);
1505         itd->hw_bufp [0] = stream->buf0;
1506         itd->hw_bufp [1] = stream->buf1;
1507         itd->hw_bufp [2] = stream->buf2;
1508
1509         for (i = 0; i < 8; i++)
1510                 itd->index[i] = -1;
1511
1512         /* All other fields are filled when scheduling */
1513 }
1514
1515 static inline void
1516 itd_patch(
1517         struct ehci_hcd         *ehci,
1518         struct ehci_itd         *itd,
1519         struct ehci_iso_sched   *iso_sched,
1520         unsigned                index,
1521         u16                     uframe
1522 )
1523 {
1524         struct ehci_iso_packet  *uf = &iso_sched->packet [index];
1525         unsigned                pg = itd->pg;
1526
1527         // BUG_ON (pg == 6 && uf->cross);
1528
1529         uframe &= 0x07;
1530         itd->index [uframe] = index;
1531
1532         itd->hw_transaction[uframe] = uf->transaction;
1533         itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12);
1534         itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0);
1535         itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32));
1536
1537         /* iso_frame_desc[].offset must be strictly increasing */
1538         if (unlikely (uf->cross)) {
1539                 u64     bufp = uf->bufp + 4096;
1540
1541                 itd->pg = ++pg;
1542                 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0);
1543                 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32));
1544         }
1545 }
1546
1547 static inline void
1548 itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
1549 {
1550         union ehci_shadow       *prev = &ehci->pshadow[frame];
1551         __hc32                  *hw_p = &ehci->periodic[frame];
1552         union ehci_shadow       here = *prev;
1553         __hc32                  type = 0;
1554
1555         /* skip any iso nodes which might belong to previous microframes */
1556         while (here.ptr) {
1557                 type = Q_NEXT_TYPE(ehci, *hw_p);
1558                 if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
1559                         break;
1560                 prev = periodic_next_shadow(ehci, prev, type);
1561                 hw_p = shadow_next_periodic(ehci, &here, type);
1562                 here = *prev;
1563         }
1564
1565         itd->itd_next = here;
1566         itd->hw_next = *hw_p;
1567         prev->itd = itd;
1568         itd->frame = frame;
1569         wmb ();
1570         *hw_p = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD);
1571 }
1572
1573 #define AB_REG_BAR_LOW 0xe0
1574 #define AB_REG_BAR_HIGH 0xe1
1575 #define AB_INDX(addr) ((addr) + 0x00)
1576 #define AB_DATA(addr) ((addr) + 0x04)
1577 #define NB_PCIE_INDX_ADDR 0xe0
1578 #define NB_PCIE_INDX_DATA 0xe4
1579 #define NB_PIF0_PWRDOWN_0 0x01100012
1580 #define NB_PIF0_PWRDOWN_1 0x01100013
1581
1582 static void ehci_quirk_amd_L1(struct ehci_hcd *ehci, int disable)
1583 {
1584         u32 addr, addr_low, addr_high, val;
1585
1586         outb_p(AB_REG_BAR_LOW, 0xcd6);
1587         addr_low = inb_p(0xcd7);
1588         outb_p(AB_REG_BAR_HIGH, 0xcd6);
1589         addr_high = inb_p(0xcd7);
1590         addr = addr_high << 8 | addr_low;
1591         outl_p(0x30, AB_INDX(addr));
1592         outl_p(0x40, AB_DATA(addr));
1593         outl_p(0x34, AB_INDX(addr));
1594         val = inl_p(AB_DATA(addr));
1595
1596         if (disable) {
1597                 val &= ~0x8;
1598                 val |= (1 << 4) | (1 << 9);
1599         } else {
1600                 val |= 0x8;
1601                 val &= ~((1 << 4) | (1 << 9));
1602         }
1603         outl_p(val, AB_DATA(addr));
1604
1605         if (amd_nb_dev) {
1606                 addr = NB_PIF0_PWRDOWN_0;
1607                 pci_write_config_dword(amd_nb_dev, NB_PCIE_INDX_ADDR, addr);
1608                 pci_read_config_dword(amd_nb_dev, NB_PCIE_INDX_DATA, &val);
1609                 if (disable)
1610                         val &= ~(0x3f << 7);
1611                 else
1612                         val |= 0x3f << 7;
1613
1614                 pci_write_config_dword(amd_nb_dev, NB_PCIE_INDX_DATA, val);
1615
1616                 addr = NB_PIF0_PWRDOWN_1;
1617                 pci_write_config_dword(amd_nb_dev, NB_PCIE_INDX_ADDR, addr);
1618                 pci_read_config_dword(amd_nb_dev, NB_PCIE_INDX_DATA, &val);
1619                 if (disable)
1620                         val &= ~(0x3f << 7);
1621                 else
1622                         val |= 0x3f << 7;
1623
1624                 pci_write_config_dword(amd_nb_dev, NB_PCIE_INDX_DATA, val);
1625         }
1626
1627         return;
1628 }
1629
1630 /* fit urb's itds into the selected schedule slot; activate as needed */
1631 static int
1632 itd_link_urb (
1633         struct ehci_hcd         *ehci,
1634         struct urb              *urb,
1635         unsigned                mod,
1636         struct ehci_iso_stream  *stream
1637 )
1638 {
1639         int                     packet;
1640         unsigned                next_uframe, uframe, frame;
1641         struct ehci_iso_sched   *iso_sched = urb->hcpriv;
1642         struct ehci_itd         *itd;
1643
1644         next_uframe = stream->next_uframe & (mod - 1);
1645
1646         if (unlikely (list_empty(&stream->td_list))) {
1647                 ehci_to_hcd(ehci)->self.bandwidth_allocated
1648                                 += stream->bandwidth;
1649                 ehci_vdbg (ehci,
1650                         "schedule devp %s ep%d%s-iso period %d start %d.%d\n",
1651                         urb->dev->devpath, stream->bEndpointAddress & 0x0f,
1652                         (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
1653                         urb->interval,
1654                         next_uframe >> 3, next_uframe & 0x7);
1655         }
1656
1657         if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1658                 if (ehci->amd_l1_fix == 1)
1659                         ehci_quirk_amd_L1(ehci, 1);
1660         }
1661
1662         ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1663
1664         /* fill iTDs uframe by uframe */
1665         for (packet = 0, itd = NULL; packet < urb->number_of_packets; ) {
1666                 if (itd == NULL) {
1667                         /* ASSERT:  we have all necessary itds */
1668                         // BUG_ON (list_empty (&iso_sched->td_list));
1669
1670                         /* ASSERT:  no itds for this endpoint in this uframe */
1671
1672                         itd = list_entry (iso_sched->td_list.next,
1673                                         struct ehci_itd, itd_list);
1674                         list_move_tail (&itd->itd_list, &stream->td_list);
1675                         itd->stream = iso_stream_get (stream);
1676                         itd->urb = urb;
1677                         itd_init (ehci, stream, itd);
1678                 }
1679
1680                 uframe = next_uframe & 0x07;
1681                 frame = next_uframe >> 3;
1682
1683                 itd_patch(ehci, itd, iso_sched, packet, uframe);
1684
1685                 next_uframe += stream->interval;
1686                 next_uframe &= mod - 1;
1687                 packet++;
1688
1689                 /* link completed itds into the schedule */
1690                 if (((next_uframe >> 3) != frame)
1691                                 || packet == urb->number_of_packets) {
1692                         itd_link(ehci, frame & (ehci->periodic_size - 1), itd);
1693                         itd = NULL;
1694                 }
1695         }
1696         stream->next_uframe = next_uframe;
1697
1698         /* don't need that schedule data any more */
1699         iso_sched_free (stream, iso_sched);
1700         urb->hcpriv = NULL;
1701
1702         timer_action (ehci, TIMER_IO_WATCHDOG);
1703         return enable_periodic(ehci);
1704 }
1705
1706 #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1707
1708 /* Process and recycle a completed ITD.  Return true iff its urb completed,
1709  * and hence its completion callback probably added things to the hardware
1710  * schedule.
1711  *
1712  * Note that we carefully avoid recycling this descriptor until after any
1713  * completion callback runs, so that it won't be reused quickly.  That is,
1714  * assuming (a) no more than two urbs per frame on this endpoint, and also
1715  * (b) only this endpoint's completions submit URBs.  It seems some silicon
1716  * corrupts things if you reuse completed descriptors very quickly...
1717  */
1718 static unsigned
1719 itd_complete (
1720         struct ehci_hcd *ehci,
1721         struct ehci_itd *itd
1722 ) {
1723         struct urb                              *urb = itd->urb;
1724         struct usb_iso_packet_descriptor        *desc;
1725         u32                                     t;
1726         unsigned                                uframe;
1727         int                                     urb_index = -1;
1728         struct ehci_iso_stream                  *stream = itd->stream;
1729         struct usb_device                       *dev;
1730         unsigned                                retval = false;
1731
1732         /* for each uframe with a packet */
1733         for (uframe = 0; uframe < 8; uframe++) {
1734                 if (likely (itd->index[uframe] == -1))
1735                         continue;
1736                 urb_index = itd->index[uframe];
1737                 desc = &urb->iso_frame_desc [urb_index];
1738
1739                 t = hc32_to_cpup(ehci, &itd->hw_transaction [uframe]);
1740                 itd->hw_transaction [uframe] = 0;
1741
1742                 /* report transfer status */
1743                 if (unlikely (t & ISO_ERRS)) {
1744                         urb->error_count++;
1745                         if (t & EHCI_ISOC_BUF_ERR)
1746                                 desc->status = usb_pipein (urb->pipe)
1747                                         ? -ENOSR  /* hc couldn't read */
1748                                         : -ECOMM; /* hc couldn't write */
1749                         else if (t & EHCI_ISOC_BABBLE)
1750                                 desc->status = -EOVERFLOW;
1751                         else /* (t & EHCI_ISOC_XACTERR) */
1752                                 desc->status = -EPROTO;
1753
1754                         /* HC need not update length with this error */
1755                         if (!(t & EHCI_ISOC_BABBLE)) {
1756                                 desc->actual_length = EHCI_ITD_LENGTH(t);
1757                                 urb->actual_length += desc->actual_length;
1758                         }
1759                 } else if (likely ((t & EHCI_ISOC_ACTIVE) == 0)) {
1760                         desc->status = 0;
1761                         desc->actual_length = EHCI_ITD_LENGTH(t);
1762                         urb->actual_length += desc->actual_length;
1763                 } else {
1764                         /* URB was too late */
1765                         desc->status = -EXDEV;
1766                 }
1767         }
1768
1769         /* handle completion now? */
1770         if (likely ((urb_index + 1) != urb->number_of_packets))
1771                 goto done;
1772
1773         /* ASSERT: it's really the last itd for this urb
1774         list_for_each_entry (itd, &stream->td_list, itd_list)
1775                 BUG_ON (itd->urb == urb);
1776          */
1777
1778         /* give urb back to the driver; completion often (re)submits */
1779         dev = urb->dev;
1780         ehci_urb_done(ehci, urb, 0);
1781         retval = true;
1782         urb = NULL;
1783         (void) disable_periodic(ehci);
1784         ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1785
1786         if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1787                 if (ehci->amd_l1_fix == 1)
1788                         ehci_quirk_amd_L1(ehci, 0);
1789         }
1790
1791         if (unlikely(list_is_singular(&stream->td_list))) {
1792                 ehci_to_hcd(ehci)->self.bandwidth_allocated
1793                                 -= stream->bandwidth;
1794                 ehci_vdbg (ehci,
1795                         "deschedule devp %s ep%d%s-iso\n",
1796                         dev->devpath, stream->bEndpointAddress & 0x0f,
1797                         (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
1798         }
1799         iso_stream_put (ehci, stream);
1800
1801 done:
1802         itd->urb = NULL;
1803         if (ehci->clock_frame != itd->frame || itd->index[7] != -1) {
1804                 /* OK to recycle this ITD now. */
1805                 itd->stream = NULL;
1806                 list_move(&itd->itd_list, &stream->free_list);
1807                 iso_stream_put(ehci, stream);
1808         } else {
1809                 /* HW might remember this ITD, so we can't recycle it yet.
1810                  * Move it to a safe place until a new frame starts.
1811                  */
1812                 list_move(&itd->itd_list, &ehci->cached_itd_list);
1813                 if (stream->refcount == 2) {
1814                         /* If iso_stream_put() were called here, stream
1815                          * would be freed.  Instead, just prevent reuse.
1816                          */
1817                         stream->ep->hcpriv = NULL;
1818                         stream->ep = NULL;
1819                 }
1820         }
1821         return retval;
1822 }
1823
1824 /*-------------------------------------------------------------------------*/
1825
1826 static int itd_submit (struct ehci_hcd *ehci, struct urb *urb,
1827         gfp_t mem_flags)
1828 {
1829         int                     status = -EINVAL;
1830         unsigned long           flags;
1831         struct ehci_iso_stream  *stream;
1832
1833         /* Get iso_stream head */
1834         stream = iso_stream_find (ehci, urb);
1835         if (unlikely (stream == NULL)) {
1836                 ehci_dbg (ehci, "can't get iso stream\n");
1837                 return -ENOMEM;
1838         }
1839         if (unlikely (urb->interval != stream->interval)) {
1840                 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
1841                         stream->interval, urb->interval);
1842                 goto done;
1843         }
1844
1845 #ifdef EHCI_URB_TRACE
1846         ehci_dbg (ehci,
1847                 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1848                 __func__, urb->dev->devpath, urb,
1849                 usb_pipeendpoint (urb->pipe),
1850                 usb_pipein (urb->pipe) ? "in" : "out",
1851                 urb->transfer_buffer_length,
1852                 urb->number_of_packets, urb->interval,
1853                 stream);
1854 #endif
1855
1856         /* allocate ITDs w/o locking anything */
1857         status = itd_urb_transaction (stream, ehci, urb, mem_flags);
1858         if (unlikely (status < 0)) {
1859                 ehci_dbg (ehci, "can't init itds\n");
1860                 goto done;
1861         }
1862
1863         /* schedule ... need to lock */
1864         spin_lock_irqsave (&ehci->lock, flags);
1865         if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
1866                 status = -ESHUTDOWN;
1867                 goto done_not_linked;
1868         }
1869         status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1870         if (unlikely(status))
1871                 goto done_not_linked;
1872         status = iso_stream_schedule(ehci, urb, stream);
1873         if (likely (status == 0))
1874                 itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
1875         else
1876                 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1877 done_not_linked:
1878         spin_unlock_irqrestore (&ehci->lock, flags);
1879
1880 done:
1881         if (unlikely (status < 0))
1882                 iso_stream_put (ehci, stream);
1883         return status;
1884 }
1885
1886 /*-------------------------------------------------------------------------*/
1887
1888 /*
1889  * "Split ISO TDs" ... used for USB 1.1 devices going through the
1890  * TTs in USB 2.0 hubs.  These need microframe scheduling.
1891  */
1892
1893 static inline void
1894 sitd_sched_init(
1895         struct ehci_hcd         *ehci,
1896         struct ehci_iso_sched   *iso_sched,
1897         struct ehci_iso_stream  *stream,
1898         struct urb              *urb
1899 )
1900 {
1901         unsigned        i;
1902         dma_addr_t      dma = urb->transfer_dma;
1903
1904         /* how many frames are needed for these transfers */
1905         iso_sched->span = urb->number_of_packets * stream->interval;
1906
1907         /* figure out per-frame sitd fields that we'll need later
1908          * when we fit new sitds into the schedule.
1909          */
1910         for (i = 0; i < urb->number_of_packets; i++) {
1911                 struct ehci_iso_packet  *packet = &iso_sched->packet [i];
1912                 unsigned                length;
1913                 dma_addr_t              buf;
1914                 u32                     trans;
1915
1916                 length = urb->iso_frame_desc [i].length & 0x03ff;
1917                 buf = dma + urb->iso_frame_desc [i].offset;
1918
1919                 trans = SITD_STS_ACTIVE;
1920                 if (((i + 1) == urb->number_of_packets)
1921                                 && !(urb->transfer_flags & URB_NO_INTERRUPT))
1922                         trans |= SITD_IOC;
1923                 trans |= length << 16;
1924                 packet->transaction = cpu_to_hc32(ehci, trans);
1925
1926                 /* might need to cross a buffer page within a td */
1927                 packet->bufp = buf;
1928                 packet->buf1 = (buf + length) & ~0x0fff;
1929                 if (packet->buf1 != (buf & ~(u64)0x0fff))
1930                         packet->cross = 1;
1931
1932                 /* OUT uses multiple start-splits */
1933                 if (stream->bEndpointAddress & USB_DIR_IN)
1934                         continue;
1935                 length = (length + 187) / 188;
1936                 if (length > 1) /* BEGIN vs ALL */
1937                         length |= 1 << 3;
1938                 packet->buf1 |= length;
1939         }
1940 }
1941
1942 static int
1943 sitd_urb_transaction (
1944         struct ehci_iso_stream  *stream,
1945         struct ehci_hcd         *ehci,
1946         struct urb              *urb,
1947         gfp_t                   mem_flags
1948 )
1949 {
1950         struct ehci_sitd        *sitd;
1951         dma_addr_t              sitd_dma;
1952         int                     i;
1953         struct ehci_iso_sched   *iso_sched;
1954         unsigned long           flags;
1955
1956         iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1957         if (iso_sched == NULL)
1958                 return -ENOMEM;
1959
1960         sitd_sched_init(ehci, iso_sched, stream, urb);
1961
1962         /* allocate/init sITDs */
1963         spin_lock_irqsave (&ehci->lock, flags);
1964         for (i = 0; i < urb->number_of_packets; i++) {
1965
1966                 /* NOTE:  for now, we don't try to handle wraparound cases
1967                  * for IN (using sitd->hw_backpointer, like a FSTN), which
1968                  * means we never need two sitds for full speed packets.
1969                  */
1970
1971                 /* free_list.next might be cache-hot ... but maybe
1972                  * the HC caches it too. avoid that issue for now.
1973                  */
1974
1975                 /* prefer previously-allocated sitds */
1976                 if (!list_empty(&stream->free_list)) {
1977                         sitd = list_entry (stream->free_list.prev,
1978                                          struct ehci_sitd, sitd_list);
1979                         list_del (&sitd->sitd_list);
1980                         sitd_dma = sitd->sitd_dma;
1981                 } else {
1982                         spin_unlock_irqrestore (&ehci->lock, flags);
1983                         sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags,
1984                                         &sitd_dma);
1985                         spin_lock_irqsave (&ehci->lock, flags);
1986                         if (!sitd) {
1987                                 iso_sched_free(stream, iso_sched);
1988                                 spin_unlock_irqrestore(&ehci->lock, flags);
1989                                 return -ENOMEM;
1990                         }
1991                 }
1992
1993                 memset (sitd, 0, sizeof *sitd);
1994                 sitd->sitd_dma = sitd_dma;
1995                 list_add (&sitd->sitd_list, &iso_sched->td_list);
1996         }
1997
1998         /* temporarily store schedule info in hcpriv */
1999         urb->hcpriv = iso_sched;
2000         urb->error_count = 0;
2001
2002         spin_unlock_irqrestore (&ehci->lock, flags);
2003         return 0;
2004 }
2005
2006 /*-------------------------------------------------------------------------*/
2007
2008 static inline void
2009 sitd_patch(
2010         struct ehci_hcd         *ehci,
2011         struct ehci_iso_stream  *stream,
2012         struct ehci_sitd        *sitd,
2013         struct ehci_iso_sched   *iso_sched,
2014         unsigned                index
2015 )
2016 {
2017         struct ehci_iso_packet  *uf = &iso_sched->packet [index];
2018         u64                     bufp = uf->bufp;
2019
2020         sitd->hw_next = EHCI_LIST_END(ehci);
2021         sitd->hw_fullspeed_ep = stream->address;
2022         sitd->hw_uframe = stream->splits;
2023         sitd->hw_results = uf->transaction;
2024         sitd->hw_backpointer = EHCI_LIST_END(ehci);
2025
2026         bufp = uf->bufp;
2027         sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp);
2028         sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32);
2029
2030         sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1);
2031         if (uf->cross)
2032                 bufp += 4096;
2033         sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32);
2034         sitd->index = index;
2035 }
2036
2037 static inline void
2038 sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
2039 {
2040         /* note: sitd ordering could matter (CSPLIT then SSPLIT) */
2041         sitd->sitd_next = ehci->pshadow [frame];
2042         sitd->hw_next = ehci->periodic [frame];
2043         ehci->pshadow [frame].sitd = sitd;
2044         sitd->frame = frame;
2045         wmb ();
2046         ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD);
2047 }
2048
2049 /* fit urb's sitds into the selected schedule slot; activate as needed */
2050 static int
2051 sitd_link_urb (
2052         struct ehci_hcd         *ehci,
2053         struct urb              *urb,
2054         unsigned                mod,
2055         struct ehci_iso_stream  *stream
2056 )
2057 {
2058         int                     packet;
2059         unsigned                next_uframe;
2060         struct ehci_iso_sched   *sched = urb->hcpriv;
2061         struct ehci_sitd        *sitd;
2062
2063         next_uframe = stream->next_uframe;
2064
2065         if (list_empty(&stream->td_list)) {
2066                 /* usbfs ignores TT bandwidth */
2067                 ehci_to_hcd(ehci)->self.bandwidth_allocated
2068                                 += stream->bandwidth;
2069                 ehci_vdbg (ehci,
2070                         "sched devp %s ep%d%s-iso [%d] %dms/%04x\n",
2071                         urb->dev->devpath, stream->bEndpointAddress & 0x0f,
2072                         (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
2073                         (next_uframe >> 3) & (ehci->periodic_size - 1),
2074                         stream->interval, hc32_to_cpu(ehci, stream->splits));
2075         }
2076
2077         if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2078                 if (ehci->amd_l1_fix == 1)
2079                         ehci_quirk_amd_L1(ehci, 1);
2080         }
2081
2082         ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
2083
2084         /* fill sITDs frame by frame */
2085         for (packet = 0, sitd = NULL;
2086                         packet < urb->number_of_packets;
2087                         packet++) {
2088
2089                 /* ASSERT:  we have all necessary sitds */
2090                 BUG_ON (list_empty (&sched->td_list));
2091
2092                 /* ASSERT:  no itds for this endpoint in this frame */
2093
2094                 sitd = list_entry (sched->td_list.next,
2095                                 struct ehci_sitd, sitd_list);
2096                 list_move_tail (&sitd->sitd_list, &stream->td_list);
2097                 sitd->stream = iso_stream_get (stream);
2098                 sitd->urb = urb;
2099
2100                 sitd_patch(ehci, stream, sitd, sched, packet);
2101                 sitd_link(ehci, (next_uframe >> 3) & (ehci->periodic_size - 1),
2102                                 sitd);
2103
2104                 next_uframe += stream->interval << 3;
2105         }
2106         stream->next_uframe = next_uframe & (mod - 1);
2107
2108         /* don't need that schedule data any more */
2109         iso_sched_free (stream, sched);
2110         urb->hcpriv = NULL;
2111
2112         timer_action (ehci, TIMER_IO_WATCHDOG);
2113         return enable_periodic(ehci);
2114 }
2115
2116 /*-------------------------------------------------------------------------*/
2117
2118 #define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
2119                                 | SITD_STS_XACT | SITD_STS_MMF)
2120
2121 /* Process and recycle a completed SITD.  Return true iff its urb completed,
2122  * and hence its completion callback probably added things to the hardware
2123  * schedule.
2124  *
2125  * Note that we carefully avoid recycling this descriptor until after any
2126  * completion callback runs, so that it won't be reused quickly.  That is,
2127  * assuming (a) no more than two urbs per frame on this endpoint, and also
2128  * (b) only this endpoint's completions submit URBs.  It seems some silicon
2129  * corrupts things if you reuse completed descriptors very quickly...
2130  */
2131 static unsigned
2132 sitd_complete (
2133         struct ehci_hcd         *ehci,
2134         struct ehci_sitd        *sitd
2135 ) {
2136         struct urb                              *urb = sitd->urb;
2137         struct usb_iso_packet_descriptor        *desc;
2138         u32                                     t;
2139         int                                     urb_index = -1;
2140         struct ehci_iso_stream                  *stream = sitd->stream;
2141         struct usb_device                       *dev;
2142         unsigned                                retval = false;
2143
2144         urb_index = sitd->index;
2145         desc = &urb->iso_frame_desc [urb_index];
2146         t = hc32_to_cpup(ehci, &sitd->hw_results);
2147
2148         /* report transfer status */
2149         if (t & SITD_ERRS) {
2150                 urb->error_count++;
2151                 if (t & SITD_STS_DBE)
2152                         desc->status = usb_pipein (urb->pipe)
2153                                 ? -ENOSR  /* hc couldn't read */
2154                                 : -ECOMM; /* hc couldn't write */
2155                 else if (t & SITD_STS_BABBLE)
2156                         desc->status = -EOVERFLOW;
2157                 else /* XACT, MMF, etc */
2158                         desc->status = -EPROTO;
2159         } else {
2160                 desc->status = 0;
2161                 desc->actual_length = desc->length - SITD_LENGTH(t);
2162                 urb->actual_length += desc->actual_length;
2163         }
2164
2165         /* handle completion now? */
2166         if ((urb_index + 1) != urb->number_of_packets)
2167                 goto done;
2168
2169         /* ASSERT: it's really the last sitd for this urb
2170         list_for_each_entry (sitd, &stream->td_list, sitd_list)
2171                 BUG_ON (sitd->urb == urb);
2172          */
2173
2174         /* give urb back to the driver; completion often (re)submits */
2175         dev = urb->dev;
2176         ehci_urb_done(ehci, urb, 0);
2177         retval = true;
2178         urb = NULL;
2179         (void) disable_periodic(ehci);
2180         ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
2181
2182         if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2183                 if (ehci->amd_l1_fix == 1)
2184                         ehci_quirk_amd_L1(ehci, 0);
2185         }
2186
2187         if (list_is_singular(&stream->td_list)) {
2188                 ehci_to_hcd(ehci)->self.bandwidth_allocated
2189                                 -= stream->bandwidth;
2190                 ehci_vdbg (ehci,
2191                         "deschedule devp %s ep%d%s-iso\n",
2192                         dev->devpath, stream->bEndpointAddress & 0x0f,
2193                         (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
2194         }
2195         iso_stream_put (ehci, stream);
2196
2197 done:
2198         sitd->urb = NULL;
2199         if (ehci->clock_frame != sitd->frame) {
2200                 /* OK to recycle this SITD now. */
2201                 sitd->stream = NULL;
2202                 list_move(&sitd->sitd_list, &stream->free_list);
2203                 iso_stream_put(ehci, stream);
2204         } else {
2205                 /* HW might remember this SITD, so we can't recycle it yet.
2206                  * Move it to a safe place until a new frame starts.
2207                  */
2208                 list_move(&sitd->sitd_list, &ehci->cached_sitd_list);
2209                 if (stream->refcount == 2) {
2210                         /* If iso_stream_put() were called here, stream
2211                          * would be freed.  Instead, just prevent reuse.
2212                          */
2213                         stream->ep->hcpriv = NULL;
2214                         stream->ep = NULL;
2215                 }
2216         }
2217         return retval;
2218 }
2219
2220
2221 static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb,
2222         gfp_t mem_flags)
2223 {
2224         int                     status = -EINVAL;
2225         unsigned long           flags;
2226         struct ehci_iso_stream  *stream;
2227
2228         /* Get iso_stream head */
2229         stream = iso_stream_find (ehci, urb);
2230         if (stream == NULL) {
2231                 ehci_dbg (ehci, "can't get iso stream\n");
2232                 return -ENOMEM;
2233         }
2234         if (urb->interval != stream->interval) {
2235                 ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
2236                         stream->interval, urb->interval);
2237                 goto done;
2238         }
2239
2240 #ifdef EHCI_URB_TRACE
2241         ehci_dbg (ehci,
2242                 "submit %p dev%s ep%d%s-iso len %d\n",
2243                 urb, urb->dev->devpath,
2244                 usb_pipeendpoint (urb->pipe),
2245                 usb_pipein (urb->pipe) ? "in" : "out",
2246                 urb->transfer_buffer_length);
2247 #endif
2248
2249         /* allocate SITDs */
2250         status = sitd_urb_transaction (stream, ehci, urb, mem_flags);
2251         if (status < 0) {
2252                 ehci_dbg (ehci, "can't init sitds\n");
2253                 goto done;
2254         }
2255
2256         /* schedule ... need to lock */
2257         spin_lock_irqsave (&ehci->lock, flags);
2258         if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
2259                 status = -ESHUTDOWN;
2260                 goto done_not_linked;
2261         }
2262         status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
2263         if (unlikely(status))
2264                 goto done_not_linked;
2265         status = iso_stream_schedule(ehci, urb, stream);
2266         if (status == 0)
2267                 sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
2268         else
2269                 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
2270 done_not_linked:
2271         spin_unlock_irqrestore (&ehci->lock, flags);
2272
2273 done:
2274         if (status < 0)
2275                 iso_stream_put (ehci, stream);
2276         return status;
2277 }
2278
2279 /*-------------------------------------------------------------------------*/
2280
2281 static void free_cached_lists(struct ehci_hcd *ehci)
2282 {
2283         struct ehci_itd *itd, *n;
2284         struct ehci_sitd *sitd, *sn;
2285
2286         list_for_each_entry_safe(itd, n, &ehci->cached_itd_list, itd_list) {
2287                 struct ehci_iso_stream  *stream = itd->stream;
2288                 itd->stream = NULL;
2289                 list_move(&itd->itd_list, &stream->free_list);
2290                 iso_stream_put(ehci, stream);
2291         }
2292
2293         list_for_each_entry_safe(sitd, sn, &ehci->cached_sitd_list, sitd_list) {
2294                 struct ehci_iso_stream  *stream = sitd->stream;
2295                 sitd->stream = NULL;
2296                 list_move(&sitd->sitd_list, &stream->free_list);
2297                 iso_stream_put(ehci, stream);
2298         }
2299 }
2300
2301 /*-------------------------------------------------------------------------*/
2302
2303 static void
2304 scan_periodic (struct ehci_hcd *ehci)
2305 {
2306         unsigned        now_uframe, frame, clock, clock_frame, mod;
2307         unsigned        modified;
2308
2309         mod = ehci->periodic_size << 3;
2310
2311         /*
2312          * When running, scan from last scan point up to "now"
2313          * else clean up by scanning everything that's left.
2314          * Touches as few pages as possible:  cache-friendly.
2315          */
2316         now_uframe = ehci->next_uframe;
2317         if (HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
2318                 clock = ehci_readl(ehci, &ehci->regs->frame_index);
2319                 clock_frame = (clock >> 3) & (ehci->periodic_size - 1);
2320         } else  {
2321                 clock = now_uframe + mod - 1;
2322                 clock_frame = -1;
2323         }
2324         if (ehci->clock_frame != clock_frame) {
2325                 free_cached_lists(ehci);
2326                 ehci->clock_frame = clock_frame;
2327         }
2328         clock &= mod - 1;
2329         clock_frame = clock >> 3;
2330
2331         for (;;) {
2332                 union ehci_shadow       q, *q_p;
2333                 __hc32                  type, *hw_p;
2334                 unsigned                incomplete = false;
2335
2336                 frame = now_uframe >> 3;
2337
2338 restart:
2339                 /* scan each element in frame's queue for completions */
2340                 q_p = &ehci->pshadow [frame];
2341                 hw_p = &ehci->periodic [frame];
2342                 q.ptr = q_p->ptr;
2343                 type = Q_NEXT_TYPE(ehci, *hw_p);
2344                 modified = 0;
2345
2346                 while (q.ptr != NULL) {
2347                         unsigned                uf;
2348                         union ehci_shadow       temp;
2349                         int                     live;
2350
2351                         live = HC_IS_RUNNING (ehci_to_hcd(ehci)->state);
2352                         switch (hc32_to_cpu(ehci, type)) {
2353                         case Q_TYPE_QH:
2354                                 /* handle any completions */
2355                                 temp.qh = qh_get (q.qh);
2356                                 type = Q_NEXT_TYPE(ehci, q.qh->hw->hw_next);
2357                                 q = q.qh->qh_next;
2358                                 modified = qh_completions (ehci, temp.qh);
2359                                 if (unlikely(list_empty(&temp.qh->qtd_list) ||
2360                                                 temp.qh->needs_rescan))
2361                                         intr_deschedule (ehci, temp.qh);
2362                                 qh_put (temp.qh);
2363                                 break;
2364                         case Q_TYPE_FSTN:
2365                                 /* for "save place" FSTNs, look at QH entries
2366                                  * in the previous frame for completions.
2367                                  */
2368                                 if (q.fstn->hw_prev != EHCI_LIST_END(ehci)) {
2369                                         dbg ("ignoring completions from FSTNs");
2370                                 }
2371                                 type = Q_NEXT_TYPE(ehci, q.fstn->hw_next);
2372                                 q = q.fstn->fstn_next;
2373                                 break;
2374                         case Q_TYPE_ITD:
2375                                 /* If this ITD is still active, leave it for
2376                                  * later processing ... check the next entry.
2377                                  * No need to check for activity unless the
2378                                  * frame is current.
2379                                  */
2380                                 if (frame == clock_frame && live) {
2381                                         rmb();
2382                                         for (uf = 0; uf < 8; uf++) {
2383                                                 if (q.itd->hw_transaction[uf] &
2384                                                             ITD_ACTIVE(ehci))
2385                                                         break;
2386                                         }
2387                                         if (uf < 8) {
2388                                                 incomplete = true;
2389                                                 q_p = &q.itd->itd_next;
2390                                                 hw_p = &q.itd->hw_next;
2391                                                 type = Q_NEXT_TYPE(ehci,
2392                                                         q.itd->hw_next);
2393                                                 q = *q_p;
2394                                                 break;
2395                                         }
2396                                 }
2397
2398                                 /* Take finished ITDs out of the schedule
2399                                  * and process them:  recycle, maybe report
2400                                  * URB completion.  HC won't cache the
2401                                  * pointer for much longer, if at all.
2402                                  */
2403                                 *q_p = q.itd->itd_next;
2404                                 *hw_p = q.itd->hw_next;
2405                                 type = Q_NEXT_TYPE(ehci, q.itd->hw_next);
2406                                 wmb();
2407                                 modified = itd_complete (ehci, q.itd);
2408                                 q = *q_p;
2409                                 break;
2410                         case Q_TYPE_SITD:
2411                                 /* If this SITD is still active, leave it for
2412                                  * later processing ... check the next entry.
2413                                  * No need to check for activity unless the
2414                                  * frame is current.
2415                                  */
2416                                 if (((frame == clock_frame) ||
2417                                      (((frame + 1) & (ehci->periodic_size - 1))
2418                                       == clock_frame))
2419                                     && live
2420                                     && (q.sitd->hw_results &
2421                                         SITD_ACTIVE(ehci))) {
2422
2423                                         incomplete = true;
2424                                         q_p = &q.sitd->sitd_next;
2425                                         hw_p = &q.sitd->hw_next;
2426                                         type = Q_NEXT_TYPE(ehci,
2427                                                         q.sitd->hw_next);
2428                                         q = *q_p;
2429                                         break;
2430                                 }
2431
2432                                 /* Take finished SITDs out of the schedule
2433                                  * and process them:  recycle, maybe report
2434                                  * URB completion.
2435                                  */
2436                                 *q_p = q.sitd->sitd_next;
2437                                 *hw_p = q.sitd->hw_next;
2438                                 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2439                                 wmb();
2440                                 modified = sitd_complete (ehci, q.sitd);
2441                                 q = *q_p;
2442                                 break;
2443                         default:
2444                                 dbg ("corrupt type %d frame %d shadow %p",
2445                                         type, frame, q.ptr);
2446                                 // BUG ();
2447                                 q.ptr = NULL;
2448                         }
2449
2450                         /* assume completion callbacks modify the queue */
2451                         if (unlikely (modified)) {
2452                                 if (likely(ehci->periodic_sched > 0))
2453                                         goto restart;
2454                                 /* short-circuit this scan */
2455                                 now_uframe = clock;
2456                                 break;
2457                         }
2458                 }
2459
2460                 /* If we can tell we caught up to the hardware, stop now.
2461                  * We can't advance our scan without collecting the ISO
2462                  * transfers that are still pending in this frame.
2463                  */
2464                 if (incomplete && HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
2465                         ehci->next_uframe = now_uframe;
2466                         break;
2467                 }
2468
2469                 // latencies climb; that should be rare, but...
2470                 // detect it, and just go all the way around.
2471                 // FLR might help detect this case, so long as latencies
2472                 // don't exceed periodic_size msec (default 1.024 sec).
2473
2474
2475                 if (now_uframe == clock) {
2476                         unsigned        now;
2477
2478                         if (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state)
2479                                         || ehci->periodic_sched == 0)
2480                                 break;
2481                         ehci->next_uframe = now_uframe;
2482                         now = ehci_readl(ehci, &ehci->regs->frame_index) &
2483                                         (mod - 1);
2484                         if (now_uframe == now)
2485                                 break;
2486
2487                         /* rescan the rest of this frame, then ... */
2488                         clock = now;
2489                         clock_frame = clock >> 3;
2490                         if (ehci->clock_frame != clock_frame) {
2491                                 free_cached_lists(ehci);
2492                                 ehci->clock_frame = clock_frame;
2493                         }
2494                 } else {
2495                         now_uframe++;
2496                         now_uframe &= mod - 1;
2497                 }
2498         }
2499 }