1 /* 2 * Copyright (c) 2001-2004 by David Brownell 3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License as published by the 7 * Free Software Foundation; either version 2 of the License, or (at your 8 * option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software Foundation, 17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 18 */ 19 20 /* this file is part of ehci-hcd.c */ 21 22 /*-------------------------------------------------------------------------*/ 23 24 /* 25 * EHCI scheduled transaction support: interrupt, iso, split iso 26 * These are called "periodic" transactions in the EHCI spec. 27 * 28 * Note that for interrupt transfers, the QH/QTD manipulation is shared 29 * with the "asynchronous" transaction support (control/bulk transfers). 30 * The only real difference is in how interrupt transfers are scheduled. 31 * 32 * For ISO, we make an "iso_stream" head to serve the same role as a QH. 33 * It keeps track of every ITD (or SITD) that's linked, and holds enough 34 * pre-calculated schedule data to make appending to the queue be quick. 35 */ 36 37 static int ehci_get_frame (struct usb_hcd *hcd); 38 39 #ifdef CONFIG_PCI 40 41 static unsigned ehci_read_frame_index(struct ehci_hcd *ehci) 42 { 43 unsigned uf; 44 45 /* 46 * The MosChip MCS9990 controller updates its microframe counter 47 * a little before the frame counter, and occasionally we will read 48 * the invalid intermediate value. Avoid problems by checking the 49 * microframe number (the low-order 3 bits); if they are 0 then 50 * re-read the register to get the correct value. 51 */ 52 uf = ehci_readl(ehci, &ehci->regs->frame_index); 53 if (unlikely(ehci->frame_index_bug && ((uf & 7) == 0))) 54 uf = ehci_readl(ehci, &ehci->regs->frame_index); 55 return uf; 56 } 57 58 #endif 59 60 /*-------------------------------------------------------------------------*/ 61 62 /* 63 * periodic_next_shadow - return "next" pointer on shadow list 64 * @periodic: host pointer to qh/itd/sitd 65 * @tag: hardware tag for type of this record 66 */ 67 static union ehci_shadow * 68 periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic, 69 __hc32 tag) 70 { 71 switch (hc32_to_cpu(ehci, tag)) { 72 case Q_TYPE_QH: 73 return &periodic->qh->qh_next; 74 case Q_TYPE_FSTN: 75 return &periodic->fstn->fstn_next; 76 case Q_TYPE_ITD: 77 return &periodic->itd->itd_next; 78 // case Q_TYPE_SITD: 79 default: 80 return &periodic->sitd->sitd_next; 81 } 82 } 83 84 static __hc32 * 85 shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic, 86 __hc32 tag) 87 { 88 switch (hc32_to_cpu(ehci, tag)) { 89 /* our ehci_shadow.qh is actually software part */ 90 case Q_TYPE_QH: 91 return &periodic->qh->hw->hw_next; 92 /* others are hw parts */ 93 default: 94 return periodic->hw_next; 95 } 96 } 97 98 /* caller must hold ehci->lock */ 99 static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr) 100 { 101 union ehci_shadow *prev_p = &ehci->pshadow[frame]; 102 __hc32 *hw_p = &ehci->periodic[frame]; 103 union ehci_shadow here = *prev_p; 104 105 /* find predecessor of "ptr"; hw and shadow lists are in sync */ 106 while (here.ptr && here.ptr != ptr) { 107 prev_p = periodic_next_shadow(ehci, prev_p, 108 Q_NEXT_TYPE(ehci, *hw_p)); 109 hw_p = shadow_next_periodic(ehci, &here, 110 Q_NEXT_TYPE(ehci, *hw_p)); 111 here = *prev_p; 112 } 113 /* an interrupt entry (at list end) could have been shared */ 114 if (!here.ptr) 115 return; 116 117 /* update shadow and hardware lists ... the old "next" pointers 118 * from ptr may still be in use, the caller updates them. 119 */ 120 *prev_p = *periodic_next_shadow(ehci, &here, 121 Q_NEXT_TYPE(ehci, *hw_p)); 122 123 if (!ehci->use_dummy_qh || 124 *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p)) 125 != EHCI_LIST_END(ehci)) 126 *hw_p = *shadow_next_periodic(ehci, &here, 127 Q_NEXT_TYPE(ehci, *hw_p)); 128 else 129 *hw_p = ehci->dummy->qh_dma; 130 } 131 132 /* how many of the uframe's 125 usecs are allocated? */ 133 static unsigned short 134 periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe) 135 { 136 __hc32 *hw_p = &ehci->periodic [frame]; 137 union ehci_shadow *q = &ehci->pshadow [frame]; 138 unsigned usecs = 0; 139 struct ehci_qh_hw *hw; 140 141 while (q->ptr) { 142 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) { 143 case Q_TYPE_QH: 144 hw = q->qh->hw; 145 /* is it in the S-mask? */ 146 if (hw->hw_info2 & cpu_to_hc32(ehci, 1 << uframe)) 147 usecs += q->qh->usecs; 148 /* ... or C-mask? */ 149 if (hw->hw_info2 & cpu_to_hc32(ehci, 150 1 << (8 + uframe))) 151 usecs += q->qh->c_usecs; 152 hw_p = &hw->hw_next; 153 q = &q->qh->qh_next; 154 break; 155 // case Q_TYPE_FSTN: 156 default: 157 /* for "save place" FSTNs, count the relevant INTR 158 * bandwidth from the previous frame 159 */ 160 if (q->fstn->hw_prev != EHCI_LIST_END(ehci)) { 161 ehci_dbg (ehci, "ignoring FSTN cost ...\n"); 162 } 163 hw_p = &q->fstn->hw_next; 164 q = &q->fstn->fstn_next; 165 break; 166 case Q_TYPE_ITD: 167 if (q->itd->hw_transaction[uframe]) 168 usecs += q->itd->stream->usecs; 169 hw_p = &q->itd->hw_next; 170 q = &q->itd->itd_next; 171 break; 172 case Q_TYPE_SITD: 173 /* is it in the S-mask? (count SPLIT, DATA) */ 174 if (q->sitd->hw_uframe & cpu_to_hc32(ehci, 175 1 << uframe)) { 176 if (q->sitd->hw_fullspeed_ep & 177 cpu_to_hc32(ehci, 1<<31)) 178 usecs += q->sitd->stream->usecs; 179 else /* worst case for OUT start-split */ 180 usecs += HS_USECS_ISO (188); 181 } 182 183 /* ... C-mask? (count CSPLIT, DATA) */ 184 if (q->sitd->hw_uframe & 185 cpu_to_hc32(ehci, 1 << (8 + uframe))) { 186 /* worst case for IN complete-split */ 187 usecs += q->sitd->stream->c_usecs; 188 } 189 190 hw_p = &q->sitd->hw_next; 191 q = &q->sitd->sitd_next; 192 break; 193 } 194 } 195 #ifdef DEBUG 196 if (usecs > ehci->uframe_periodic_max) 197 ehci_err (ehci, "uframe %d sched overrun: %d usecs\n", 198 frame * 8 + uframe, usecs); 199 #endif 200 return usecs; 201 } 202 203 /*-------------------------------------------------------------------------*/ 204 205 static int same_tt (struct usb_device *dev1, struct usb_device *dev2) 206 { 207 if (!dev1->tt || !dev2->tt) 208 return 0; 209 if (dev1->tt != dev2->tt) 210 return 0; 211 if (dev1->tt->multi) 212 return dev1->ttport == dev2->ttport; 213 else 214 return 1; 215 } 216 217 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED 218 219 /* Which uframe does the low/fullspeed transfer start in? 220 * 221 * The parameter is the mask of ssplits in "H-frame" terms 222 * and this returns the transfer start uframe in "B-frame" terms, 223 * which allows both to match, e.g. a ssplit in "H-frame" uframe 0 224 * will cause a transfer in "B-frame" uframe 0. "B-frames" lag 225 * "H-frames" by 1 uframe. See the EHCI spec sec 4.5 and figure 4.7. 226 */ 227 static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask) 228 { 229 unsigned char smask = QH_SMASK & hc32_to_cpu(ehci, mask); 230 if (!smask) { 231 ehci_err(ehci, "invalid empty smask!\n"); 232 /* uframe 7 can't have bw so this will indicate failure */ 233 return 7; 234 } 235 return ffs(smask) - 1; 236 } 237 238 static const unsigned char 239 max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 }; 240 241 /* carryover low/fullspeed bandwidth that crosses uframe boundries */ 242 static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8]) 243 { 244 int i; 245 for (i=0; i<7; i++) { 246 if (max_tt_usecs[i] < tt_usecs[i]) { 247 tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i]; 248 tt_usecs[i] = max_tt_usecs[i]; 249 } 250 } 251 } 252 253 /* How many of the tt's periodic downstream 1000 usecs are allocated? 254 * 255 * While this measures the bandwidth in terms of usecs/uframe, 256 * the low/fullspeed bus has no notion of uframes, so any particular 257 * low/fullspeed transfer can "carry over" from one uframe to the next, 258 * since the TT just performs downstream transfers in sequence. 259 * 260 * For example two separate 100 usec transfers can start in the same uframe, 261 * and the second one would "carry over" 75 usecs into the next uframe. 262 */ 263 static void 264 periodic_tt_usecs ( 265 struct ehci_hcd *ehci, 266 struct usb_device *dev, 267 unsigned frame, 268 unsigned short tt_usecs[8] 269 ) 270 { 271 __hc32 *hw_p = &ehci->periodic [frame]; 272 union ehci_shadow *q = &ehci->pshadow [frame]; 273 unsigned char uf; 274 275 memset(tt_usecs, 0, 16); 276 277 while (q->ptr) { 278 switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) { 279 case Q_TYPE_ITD: 280 hw_p = &q->itd->hw_next; 281 q = &q->itd->itd_next; 282 continue; 283 case Q_TYPE_QH: 284 if (same_tt(dev, q->qh->dev)) { 285 uf = tt_start_uframe(ehci, q->qh->hw->hw_info2); 286 tt_usecs[uf] += q->qh->tt_usecs; 287 } 288 hw_p = &q->qh->hw->hw_next; 289 q = &q->qh->qh_next; 290 continue; 291 case Q_TYPE_SITD: 292 if (same_tt(dev, q->sitd->urb->dev)) { 293 uf = tt_start_uframe(ehci, q->sitd->hw_uframe); 294 tt_usecs[uf] += q->sitd->stream->tt_usecs; 295 } 296 hw_p = &q->sitd->hw_next; 297 q = &q->sitd->sitd_next; 298 continue; 299 // case Q_TYPE_FSTN: 300 default: 301 ehci_dbg(ehci, "ignoring periodic frame %d FSTN\n", 302 frame); 303 hw_p = &q->fstn->hw_next; 304 q = &q->fstn->fstn_next; 305 } 306 } 307 308 carryover_tt_bandwidth(tt_usecs); 309 310 if (max_tt_usecs[7] < tt_usecs[7]) 311 ehci_err(ehci, "frame %d tt sched overrun: %d usecs\n", 312 frame, tt_usecs[7] - max_tt_usecs[7]); 313 } 314 315 /* 316 * Return true if the device's tt's downstream bus is available for a 317 * periodic transfer of the specified length (usecs), starting at the 318 * specified frame/uframe. Note that (as summarized in section 11.19 319 * of the usb 2.0 spec) TTs can buffer multiple transactions for each 320 * uframe. 321 * 322 * The uframe parameter is when the fullspeed/lowspeed transfer 323 * should be executed in "B-frame" terms, which is the same as the 324 * highspeed ssplit's uframe (which is in "H-frame" terms). For example 325 * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0. 326 * See the EHCI spec sec 4.5 and fig 4.7. 327 * 328 * This checks if the full/lowspeed bus, at the specified starting uframe, 329 * has the specified bandwidth available, according to rules listed 330 * in USB 2.0 spec section 11.18.1 fig 11-60. 331 * 332 * This does not check if the transfer would exceed the max ssplit 333 * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4, 334 * since proper scheduling limits ssplits to less than 16 per uframe. 335 */ 336 static int tt_available ( 337 struct ehci_hcd *ehci, 338 unsigned period, 339 struct usb_device *dev, 340 unsigned frame, 341 unsigned uframe, 342 u16 usecs 343 ) 344 { 345 if ((period == 0) || (uframe >= 7)) /* error */ 346 return 0; 347 348 for (; frame < ehci->periodic_size; frame += period) { 349 unsigned short tt_usecs[8]; 350 351 periodic_tt_usecs (ehci, dev, frame, tt_usecs); 352 353 ehci_vdbg(ehci, "tt frame %d check %d usecs start uframe %d in" 354 " schedule %d/%d/%d/%d/%d/%d/%d/%d\n", 355 frame, usecs, uframe, 356 tt_usecs[0], tt_usecs[1], tt_usecs[2], tt_usecs[3], 357 tt_usecs[4], tt_usecs[5], tt_usecs[6], tt_usecs[7]); 358 359 if (max_tt_usecs[uframe] <= tt_usecs[uframe]) { 360 ehci_vdbg(ehci, "frame %d uframe %d fully scheduled\n", 361 frame, uframe); 362 return 0; 363 } 364 365 /* special case for isoc transfers larger than 125us: 366 * the first and each subsequent fully used uframe 367 * must be empty, so as to not illegally delay 368 * already scheduled transactions 369 */ 370 if (125 < usecs) { 371 int ufs = (usecs / 125); 372 int i; 373 for (i = uframe; i < (uframe + ufs) && i < 8; i++) 374 if (0 < tt_usecs[i]) { 375 ehci_vdbg(ehci, 376 "multi-uframe xfer can't fit " 377 "in frame %d uframe %d\n", 378 frame, i); 379 return 0; 380 } 381 } 382 383 tt_usecs[uframe] += usecs; 384 385 carryover_tt_bandwidth(tt_usecs); 386 387 /* fail if the carryover pushed bw past the last uframe's limit */ 388 if (max_tt_usecs[7] < tt_usecs[7]) { 389 ehci_vdbg(ehci, 390 "tt unavailable usecs %d frame %d uframe %d\n", 391 usecs, frame, uframe); 392 return 0; 393 } 394 } 395 396 return 1; 397 } 398 399 #else 400 401 /* return true iff the device's transaction translator is available 402 * for a periodic transfer starting at the specified frame, using 403 * all the uframes in the mask. 404 */ 405 static int tt_no_collision ( 406 struct ehci_hcd *ehci, 407 unsigned period, 408 struct usb_device *dev, 409 unsigned frame, 410 u32 uf_mask 411 ) 412 { 413 if (period == 0) /* error */ 414 return 0; 415 416 /* note bandwidth wastage: split never follows csplit 417 * (different dev or endpoint) until the next uframe. 418 * calling convention doesn't make that distinction. 419 */ 420 for (; frame < ehci->periodic_size; frame += period) { 421 union ehci_shadow here; 422 __hc32 type; 423 struct ehci_qh_hw *hw; 424 425 here = ehci->pshadow [frame]; 426 type = Q_NEXT_TYPE(ehci, ehci->periodic [frame]); 427 while (here.ptr) { 428 switch (hc32_to_cpu(ehci, type)) { 429 case Q_TYPE_ITD: 430 type = Q_NEXT_TYPE(ehci, here.itd->hw_next); 431 here = here.itd->itd_next; 432 continue; 433 case Q_TYPE_QH: 434 hw = here.qh->hw; 435 if (same_tt (dev, here.qh->dev)) { 436 u32 mask; 437 438 mask = hc32_to_cpu(ehci, 439 hw->hw_info2); 440 /* "knows" no gap is needed */ 441 mask |= mask >> 8; 442 if (mask & uf_mask) 443 break; 444 } 445 type = Q_NEXT_TYPE(ehci, hw->hw_next); 446 here = here.qh->qh_next; 447 continue; 448 case Q_TYPE_SITD: 449 if (same_tt (dev, here.sitd->urb->dev)) { 450 u16 mask; 451 452 mask = hc32_to_cpu(ehci, here.sitd 453 ->hw_uframe); 454 /* FIXME assumes no gap for IN! */ 455 mask |= mask >> 8; 456 if (mask & uf_mask) 457 break; 458 } 459 type = Q_NEXT_TYPE(ehci, here.sitd->hw_next); 460 here = here.sitd->sitd_next; 461 continue; 462 // case Q_TYPE_FSTN: 463 default: 464 ehci_dbg (ehci, 465 "periodic frame %d bogus type %d\n", 466 frame, type); 467 } 468 469 /* collision or error */ 470 return 0; 471 } 472 } 473 474 /* no collision */ 475 return 1; 476 } 477 478 #endif /* CONFIG_USB_EHCI_TT_NEWSCHED */ 479 480 /*-------------------------------------------------------------------------*/ 481 482 static int enable_periodic (struct ehci_hcd *ehci) 483 { 484 u32 cmd; 485 int status; 486 487 if (ehci->periodic_sched++) 488 return 0; 489 490 /* did clearing PSE did take effect yet? 491 * takes effect only at frame boundaries... 492 */ 493 status = handshake_on_error_set_halt(ehci, &ehci->regs->status, 494 STS_PSS, 0, 9 * 125); 495 if (status) { 496 usb_hc_died(ehci_to_hcd(ehci)); 497 return status; 498 } 499 500 cmd = ehci_readl(ehci, &ehci->regs->command) | CMD_PSE; 501 ehci_writel(ehci, cmd, &ehci->regs->command); 502 /* posted write ... PSS happens later */ 503 504 /* make sure ehci_work scans these */ 505 ehci->next_uframe = ehci_read_frame_index(ehci) 506 % (ehci->periodic_size << 3); 507 if (unlikely(ehci->broken_periodic)) 508 ehci->last_periodic_enable = ktime_get_real(); 509 return 0; 510 } 511 512 static int disable_periodic (struct ehci_hcd *ehci) 513 { 514 u32 cmd; 515 int status; 516 517 if (--ehci->periodic_sched) 518 return 0; 519 520 if (unlikely(ehci->broken_periodic)) { 521 /* delay experimentally determined */ 522 ktime_t safe = ktime_add_us(ehci->last_periodic_enable, 1000); 523 ktime_t now = ktime_get_real(); 524 s64 delay = ktime_us_delta(safe, now); 525 526 if (unlikely(delay > 0)) 527 udelay(delay); 528 } 529 530 /* did setting PSE not take effect yet? 531 * takes effect only at frame boundaries... 532 */ 533 status = handshake_on_error_set_halt(ehci, &ehci->regs->status, 534 STS_PSS, STS_PSS, 9 * 125); 535 if (status) { 536 usb_hc_died(ehci_to_hcd(ehci)); 537 return status; 538 } 539 540 cmd = ehci_readl(ehci, &ehci->regs->command) & ~CMD_PSE; 541 ehci_writel(ehci, cmd, &ehci->regs->command); 542 /* posted write ... */ 543 544 free_cached_lists(ehci); 545 546 ehci->next_uframe = -1; 547 return 0; 548 } 549 550 /*-------------------------------------------------------------------------*/ 551 552 /* periodic schedule slots have iso tds (normal or split) first, then a 553 * sparse tree for active interrupt transfers. 554 * 555 * this just links in a qh; caller guarantees uframe masks are set right. 556 * no FSTN support (yet; ehci 0.96+) 557 */ 558 static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh) 559 { 560 unsigned i; 561 unsigned period = qh->period; 562 563 dev_dbg (&qh->dev->dev, 564 "link qh%d-%04x/%p start %d [%d/%d us]\n", 565 period, hc32_to_cpup(ehci, &qh->hw->hw_info2) 566 & (QH_CMASK | QH_SMASK), 567 qh, qh->start, qh->usecs, qh->c_usecs); 568 569 /* high bandwidth, or otherwise every microframe */ 570 if (period == 0) 571 period = 1; 572 573 for (i = qh->start; i < ehci->periodic_size; i += period) { 574 union ehci_shadow *prev = &ehci->pshadow[i]; 575 __hc32 *hw_p = &ehci->periodic[i]; 576 union ehci_shadow here = *prev; 577 __hc32 type = 0; 578 579 /* skip the iso nodes at list head */ 580 while (here.ptr) { 581 type = Q_NEXT_TYPE(ehci, *hw_p); 582 if (type == cpu_to_hc32(ehci, Q_TYPE_QH)) 583 break; 584 prev = periodic_next_shadow(ehci, prev, type); 585 hw_p = shadow_next_periodic(ehci, &here, type); 586 here = *prev; 587 } 588 589 /* sorting each branch by period (slow-->fast) 590 * enables sharing interior tree nodes 591 */ 592 while (here.ptr && qh != here.qh) { 593 if (qh->period > here.qh->period) 594 break; 595 prev = &here.qh->qh_next; 596 hw_p = &here.qh->hw->hw_next; 597 here = *prev; 598 } 599 /* link in this qh, unless some earlier pass did that */ 600 if (qh != here.qh) { 601 qh->qh_next = here; 602 if (here.qh) 603 qh->hw->hw_next = *hw_p; 604 wmb (); 605 prev->qh = qh; 606 *hw_p = QH_NEXT (ehci, qh->qh_dma); 607 } 608 } 609 qh->qh_state = QH_STATE_LINKED; 610 qh->xacterrs = 0; 611 qh_get (qh); 612 613 /* update per-qh bandwidth for usbfs */ 614 ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period 615 ? ((qh->usecs + qh->c_usecs) / qh->period) 616 : (qh->usecs * 8); 617 618 /* maybe enable periodic schedule processing */ 619 return enable_periodic(ehci); 620 } 621 622 static int qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh) 623 { 624 unsigned i; 625 unsigned period; 626 627 // FIXME: 628 // IF this isn't high speed 629 // and this qh is active in the current uframe 630 // (and overlay token SplitXstate is false?) 631 // THEN 632 // qh->hw_info1 |= cpu_to_hc32(1 << 7 /* "ignore" */); 633 634 /* high bandwidth, or otherwise part of every microframe */ 635 if ((period = qh->period) == 0) 636 period = 1; 637 638 for (i = qh->start; i < ehci->periodic_size; i += period) 639 periodic_unlink (ehci, i, qh); 640 641 /* update per-qh bandwidth for usbfs */ 642 ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period 643 ? ((qh->usecs + qh->c_usecs) / qh->period) 644 : (qh->usecs * 8); 645 646 dev_dbg (&qh->dev->dev, 647 "unlink qh%d-%04x/%p start %d [%d/%d us]\n", 648 qh->period, 649 hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK), 650 qh, qh->start, qh->usecs, qh->c_usecs); 651 652 /* qh->qh_next still "live" to HC */ 653 qh->qh_state = QH_STATE_UNLINK; 654 qh->qh_next.ptr = NULL; 655 qh_put (qh); 656 657 /* maybe turn off periodic schedule */ 658 return disable_periodic(ehci); 659 } 660 661 static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh) 662 { 663 unsigned wait; 664 struct ehci_qh_hw *hw = qh->hw; 665 int rc; 666 667 /* If the QH isn't linked then there's nothing we can do 668 * unless we were called during a giveback, in which case 669 * qh_completions() has to deal with it. 670 */ 671 if (qh->qh_state != QH_STATE_LINKED) { 672 if (qh->qh_state == QH_STATE_COMPLETING) 673 qh->needs_rescan = 1; 674 return; 675 } 676 677 qh_unlink_periodic (ehci, qh); 678 679 /* simple/paranoid: always delay, expecting the HC needs to read 680 * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and 681 * expect khubd to clean up after any CSPLITs we won't issue. 682 * active high speed queues may need bigger delays... 683 */ 684 if (list_empty (&qh->qtd_list) 685 || (cpu_to_hc32(ehci, QH_CMASK) 686 & hw->hw_info2) != 0) 687 wait = 2; 688 else 689 wait = 55; /* worst case: 3 * 1024 */ 690 691 udelay (wait); 692 qh->qh_state = QH_STATE_IDLE; 693 hw->hw_next = EHCI_LIST_END(ehci); 694 wmb (); 695 696 qh_completions(ehci, qh); 697 698 /* reschedule QH iff another request is queued */ 699 if (!list_empty(&qh->qtd_list) && 700 ehci->rh_state == EHCI_RH_RUNNING) { 701 rc = qh_schedule(ehci, qh); 702 703 /* An error here likely indicates handshake failure 704 * or no space left in the schedule. Neither fault 705 * should happen often ... 706 * 707 * FIXME kill the now-dysfunctional queued urbs 708 */ 709 if (rc != 0) 710 ehci_err(ehci, "can't reschedule qh %p, err %d\n", 711 qh, rc); 712 } 713 } 714 715 /*-------------------------------------------------------------------------*/ 716 717 static int check_period ( 718 struct ehci_hcd *ehci, 719 unsigned frame, 720 unsigned uframe, 721 unsigned period, 722 unsigned usecs 723 ) { 724 int claimed; 725 726 /* complete split running into next frame? 727 * given FSTN support, we could sometimes check... 728 */ 729 if (uframe >= 8) 730 return 0; 731 732 /* convert "usecs we need" to "max already claimed" */ 733 usecs = ehci->uframe_periodic_max - usecs; 734 735 /* we "know" 2 and 4 uframe intervals were rejected; so 736 * for period 0, check _every_ microframe in the schedule. 737 */ 738 if (unlikely (period == 0)) { 739 do { 740 for (uframe = 0; uframe < 7; uframe++) { 741 claimed = periodic_usecs (ehci, frame, uframe); 742 if (claimed > usecs) 743 return 0; 744 } 745 } while ((frame += 1) < ehci->periodic_size); 746 747 /* just check the specified uframe, at that period */ 748 } else { 749 do { 750 claimed = periodic_usecs (ehci, frame, uframe); 751 if (claimed > usecs) 752 return 0; 753 } while ((frame += period) < ehci->periodic_size); 754 } 755 756 // success! 757 return 1; 758 } 759 760 static int check_intr_schedule ( 761 struct ehci_hcd *ehci, 762 unsigned frame, 763 unsigned uframe, 764 const struct ehci_qh *qh, 765 __hc32 *c_maskp 766 ) 767 { 768 int retval = -ENOSPC; 769 u8 mask = 0; 770 771 if (qh->c_usecs && uframe >= 6) /* FSTN territory? */ 772 goto done; 773 774 if (!check_period (ehci, frame, uframe, qh->period, qh->usecs)) 775 goto done; 776 if (!qh->c_usecs) { 777 retval = 0; 778 *c_maskp = 0; 779 goto done; 780 } 781 782 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED 783 if (tt_available (ehci, qh->period, qh->dev, frame, uframe, 784 qh->tt_usecs)) { 785 unsigned i; 786 787 /* TODO : this may need FSTN for SSPLIT in uframe 5. */ 788 for (i=uframe+1; i<8 && i<uframe+4; i++) 789 if (!check_period (ehci, frame, i, 790 qh->period, qh->c_usecs)) 791 goto done; 792 else 793 mask |= 1 << i; 794 795 retval = 0; 796 797 *c_maskp = cpu_to_hc32(ehci, mask << 8); 798 } 799 #else 800 /* Make sure this tt's buffer is also available for CSPLITs. 801 * We pessimize a bit; probably the typical full speed case 802 * doesn't need the second CSPLIT. 803 * 804 * NOTE: both SPLIT and CSPLIT could be checked in just 805 * one smart pass... 806 */ 807 mask = 0x03 << (uframe + qh->gap_uf); 808 *c_maskp = cpu_to_hc32(ehci, mask << 8); 809 810 mask |= 1 << uframe; 811 if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) { 812 if (!check_period (ehci, frame, uframe + qh->gap_uf + 1, 813 qh->period, qh->c_usecs)) 814 goto done; 815 if (!check_period (ehci, frame, uframe + qh->gap_uf, 816 qh->period, qh->c_usecs)) 817 goto done; 818 retval = 0; 819 } 820 #endif 821 done: 822 return retval; 823 } 824 825 /* "first fit" scheduling policy used the first time through, 826 * or when the previous schedule slot can't be re-used. 827 */ 828 static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh) 829 { 830 int status; 831 unsigned uframe; 832 __hc32 c_mask; 833 unsigned frame; /* 0..(qh->period - 1), or NO_FRAME */ 834 struct ehci_qh_hw *hw = qh->hw; 835 836 qh_refresh(ehci, qh); 837 hw->hw_next = EHCI_LIST_END(ehci); 838 frame = qh->start; 839 840 /* reuse the previous schedule slots, if we can */ 841 if (frame < qh->period) { 842 uframe = ffs(hc32_to_cpup(ehci, &hw->hw_info2) & QH_SMASK); 843 status = check_intr_schedule (ehci, frame, --uframe, 844 qh, &c_mask); 845 } else { 846 uframe = 0; 847 c_mask = 0; 848 status = -ENOSPC; 849 } 850 851 /* else scan the schedule to find a group of slots such that all 852 * uframes have enough periodic bandwidth available. 853 */ 854 if (status) { 855 /* "normal" case, uframing flexible except with splits */ 856 if (qh->period) { 857 int i; 858 859 for (i = qh->period; status && i > 0; --i) { 860 frame = ++ehci->random_frame % qh->period; 861 for (uframe = 0; uframe < 8; uframe++) { 862 status = check_intr_schedule (ehci, 863 frame, uframe, qh, 864 &c_mask); 865 if (status == 0) 866 break; 867 } 868 } 869 870 /* qh->period == 0 means every uframe */ 871 } else { 872 frame = 0; 873 status = check_intr_schedule (ehci, 0, 0, qh, &c_mask); 874 } 875 if (status) 876 goto done; 877 qh->start = frame; 878 879 /* reset S-frame and (maybe) C-frame masks */ 880 hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK)); 881 hw->hw_info2 |= qh->period 882 ? cpu_to_hc32(ehci, 1 << uframe) 883 : cpu_to_hc32(ehci, QH_SMASK); 884 hw->hw_info2 |= c_mask; 885 } else 886 ehci_dbg (ehci, "reused qh %p schedule\n", qh); 887 888 /* stuff into the periodic schedule */ 889 status = qh_link_periodic (ehci, qh); 890 done: 891 return status; 892 } 893 894 static int intr_submit ( 895 struct ehci_hcd *ehci, 896 struct urb *urb, 897 struct list_head *qtd_list, 898 gfp_t mem_flags 899 ) { 900 unsigned epnum; 901 unsigned long flags; 902 struct ehci_qh *qh; 903 int status; 904 struct list_head empty; 905 906 /* get endpoint and transfer/schedule data */ 907 epnum = urb->ep->desc.bEndpointAddress; 908 909 spin_lock_irqsave (&ehci->lock, flags); 910 911 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) { 912 status = -ESHUTDOWN; 913 goto done_not_linked; 914 } 915 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb); 916 if (unlikely(status)) 917 goto done_not_linked; 918 919 /* get qh and force any scheduling errors */ 920 INIT_LIST_HEAD (&empty); 921 qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv); 922 if (qh == NULL) { 923 status = -ENOMEM; 924 goto done; 925 } 926 if (qh->qh_state == QH_STATE_IDLE) { 927 if ((status = qh_schedule (ehci, qh)) != 0) 928 goto done; 929 } 930 931 /* then queue the urb's tds to the qh */ 932 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv); 933 BUG_ON (qh == NULL); 934 935 /* ... update usbfs periodic stats */ 936 ehci_to_hcd(ehci)->self.bandwidth_int_reqs++; 937 938 done: 939 if (unlikely(status)) 940 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 941 done_not_linked: 942 spin_unlock_irqrestore (&ehci->lock, flags); 943 if (status) 944 qtd_list_free (ehci, urb, qtd_list); 945 946 return status; 947 } 948 949 /*-------------------------------------------------------------------------*/ 950 951 /* ehci_iso_stream ops work with both ITD and SITD */ 952 953 static struct ehci_iso_stream * 954 iso_stream_alloc (gfp_t mem_flags) 955 { 956 struct ehci_iso_stream *stream; 957 958 stream = kzalloc(sizeof *stream, mem_flags); 959 if (likely (stream != NULL)) { 960 INIT_LIST_HEAD(&stream->td_list); 961 INIT_LIST_HEAD(&stream->free_list); 962 stream->next_uframe = -1; 963 stream->refcount = 1; 964 } 965 return stream; 966 } 967 968 static void 969 iso_stream_init ( 970 struct ehci_hcd *ehci, 971 struct ehci_iso_stream *stream, 972 struct usb_device *dev, 973 int pipe, 974 unsigned interval 975 ) 976 { 977 static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f }; 978 979 u32 buf1; 980 unsigned epnum, maxp; 981 int is_input; 982 long bandwidth; 983 984 /* 985 * this might be a "high bandwidth" highspeed endpoint, 986 * as encoded in the ep descriptor's wMaxPacket field 987 */ 988 epnum = usb_pipeendpoint (pipe); 989 is_input = usb_pipein (pipe) ? USB_DIR_IN : 0; 990 maxp = usb_maxpacket(dev, pipe, !is_input); 991 if (is_input) { 992 buf1 = (1 << 11); 993 } else { 994 buf1 = 0; 995 } 996 997 /* knows about ITD vs SITD */ 998 if (dev->speed == USB_SPEED_HIGH) { 999 unsigned multi = hb_mult(maxp); 1000 1001 stream->highspeed = 1; 1002 1003 maxp = max_packet(maxp); 1004 buf1 |= maxp; 1005 maxp *= multi; 1006 1007 stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum); 1008 stream->buf1 = cpu_to_hc32(ehci, buf1); 1009 stream->buf2 = cpu_to_hc32(ehci, multi); 1010 1011 /* usbfs wants to report the average usecs per frame tied up 1012 * when transfers on this endpoint are scheduled ... 1013 */ 1014 stream->usecs = HS_USECS_ISO (maxp); 1015 bandwidth = stream->usecs * 8; 1016 bandwidth /= interval; 1017 1018 } else { 1019 u32 addr; 1020 int think_time; 1021 int hs_transfers; 1022 1023 addr = dev->ttport << 24; 1024 if (!ehci_is_TDI(ehci) 1025 || (dev->tt->hub != 1026 ehci_to_hcd(ehci)->self.root_hub)) 1027 addr |= dev->tt->hub->devnum << 16; 1028 addr |= epnum << 8; 1029 addr |= dev->devnum; 1030 stream->usecs = HS_USECS_ISO (maxp); 1031 think_time = dev->tt ? dev->tt->think_time : 0; 1032 stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time ( 1033 dev->speed, is_input, 1, maxp)); 1034 hs_transfers = max (1u, (maxp + 187) / 188); 1035 if (is_input) { 1036 u32 tmp; 1037 1038 addr |= 1 << 31; 1039 stream->c_usecs = stream->usecs; 1040 stream->usecs = HS_USECS_ISO (1); 1041 stream->raw_mask = 1; 1042 1043 /* c-mask as specified in USB 2.0 11.18.4 3.c */ 1044 tmp = (1 << (hs_transfers + 2)) - 1; 1045 stream->raw_mask |= tmp << (8 + 2); 1046 } else 1047 stream->raw_mask = smask_out [hs_transfers - 1]; 1048 bandwidth = stream->usecs + stream->c_usecs; 1049 bandwidth /= interval << 3; 1050 1051 /* stream->splits gets created from raw_mask later */ 1052 stream->address = cpu_to_hc32(ehci, addr); 1053 } 1054 stream->bandwidth = bandwidth; 1055 1056 stream->udev = dev; 1057 1058 stream->bEndpointAddress = is_input | epnum; 1059 stream->interval = interval; 1060 stream->maxp = maxp; 1061 } 1062 1063 static void 1064 iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream) 1065 { 1066 stream->refcount--; 1067 1068 /* free whenever just a dev->ep reference remains. 1069 * not like a QH -- no persistent state (toggle, halt) 1070 */ 1071 if (stream->refcount == 1) { 1072 // BUG_ON (!list_empty(&stream->td_list)); 1073 1074 while (!list_empty (&stream->free_list)) { 1075 struct list_head *entry; 1076 1077 entry = stream->free_list.next; 1078 list_del (entry); 1079 1080 /* knows about ITD vs SITD */ 1081 if (stream->highspeed) { 1082 struct ehci_itd *itd; 1083 1084 itd = list_entry (entry, struct ehci_itd, 1085 itd_list); 1086 dma_pool_free (ehci->itd_pool, itd, 1087 itd->itd_dma); 1088 } else { 1089 struct ehci_sitd *sitd; 1090 1091 sitd = list_entry (entry, struct ehci_sitd, 1092 sitd_list); 1093 dma_pool_free (ehci->sitd_pool, sitd, 1094 sitd->sitd_dma); 1095 } 1096 } 1097 1098 stream->bEndpointAddress &= 0x0f; 1099 if (stream->ep) 1100 stream->ep->hcpriv = NULL; 1101 1102 kfree(stream); 1103 } 1104 } 1105 1106 static inline struct ehci_iso_stream * 1107 iso_stream_get (struct ehci_iso_stream *stream) 1108 { 1109 if (likely (stream != NULL)) 1110 stream->refcount++; 1111 return stream; 1112 } 1113 1114 static struct ehci_iso_stream * 1115 iso_stream_find (struct ehci_hcd *ehci, struct urb *urb) 1116 { 1117 unsigned epnum; 1118 struct ehci_iso_stream *stream; 1119 struct usb_host_endpoint *ep; 1120 unsigned long flags; 1121 1122 epnum = usb_pipeendpoint (urb->pipe); 1123 if (usb_pipein(urb->pipe)) 1124 ep = urb->dev->ep_in[epnum]; 1125 else 1126 ep = urb->dev->ep_out[epnum]; 1127 1128 spin_lock_irqsave (&ehci->lock, flags); 1129 stream = ep->hcpriv; 1130 1131 if (unlikely (stream == NULL)) { 1132 stream = iso_stream_alloc(GFP_ATOMIC); 1133 if (likely (stream != NULL)) { 1134 /* dev->ep owns the initial refcount */ 1135 ep->hcpriv = stream; 1136 stream->ep = ep; 1137 iso_stream_init(ehci, stream, urb->dev, urb->pipe, 1138 urb->interval); 1139 } 1140 1141 /* if dev->ep [epnum] is a QH, hw is set */ 1142 } else if (unlikely (stream->hw != NULL)) { 1143 ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n", 1144 urb->dev->devpath, epnum, 1145 usb_pipein(urb->pipe) ? "in" : "out"); 1146 stream = NULL; 1147 } 1148 1149 /* caller guarantees an eventual matching iso_stream_put */ 1150 stream = iso_stream_get (stream); 1151 1152 spin_unlock_irqrestore (&ehci->lock, flags); 1153 return stream; 1154 } 1155 1156 /*-------------------------------------------------------------------------*/ 1157 1158 /* ehci_iso_sched ops can be ITD-only or SITD-only */ 1159 1160 static struct ehci_iso_sched * 1161 iso_sched_alloc (unsigned packets, gfp_t mem_flags) 1162 { 1163 struct ehci_iso_sched *iso_sched; 1164 int size = sizeof *iso_sched; 1165 1166 size += packets * sizeof (struct ehci_iso_packet); 1167 iso_sched = kzalloc(size, mem_flags); 1168 if (likely (iso_sched != NULL)) { 1169 INIT_LIST_HEAD (&iso_sched->td_list); 1170 } 1171 return iso_sched; 1172 } 1173 1174 static inline void 1175 itd_sched_init( 1176 struct ehci_hcd *ehci, 1177 struct ehci_iso_sched *iso_sched, 1178 struct ehci_iso_stream *stream, 1179 struct urb *urb 1180 ) 1181 { 1182 unsigned i; 1183 dma_addr_t dma = urb->transfer_dma; 1184 1185 /* how many uframes are needed for these transfers */ 1186 iso_sched->span = urb->number_of_packets * stream->interval; 1187 1188 /* figure out per-uframe itd fields that we'll need later 1189 * when we fit new itds into the schedule. 1190 */ 1191 for (i = 0; i < urb->number_of_packets; i++) { 1192 struct ehci_iso_packet *uframe = &iso_sched->packet [i]; 1193 unsigned length; 1194 dma_addr_t buf; 1195 u32 trans; 1196 1197 length = urb->iso_frame_desc [i].length; 1198 buf = dma + urb->iso_frame_desc [i].offset; 1199 1200 trans = EHCI_ISOC_ACTIVE; 1201 trans |= buf & 0x0fff; 1202 if (unlikely (((i + 1) == urb->number_of_packets)) 1203 && !(urb->transfer_flags & URB_NO_INTERRUPT)) 1204 trans |= EHCI_ITD_IOC; 1205 trans |= length << 16; 1206 uframe->transaction = cpu_to_hc32(ehci, trans); 1207 1208 /* might need to cross a buffer page within a uframe */ 1209 uframe->bufp = (buf & ~(u64)0x0fff); 1210 buf += length; 1211 if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff)))) 1212 uframe->cross = 1; 1213 } 1214 } 1215 1216 static void 1217 iso_sched_free ( 1218 struct ehci_iso_stream *stream, 1219 struct ehci_iso_sched *iso_sched 1220 ) 1221 { 1222 if (!iso_sched) 1223 return; 1224 // caller must hold ehci->lock! 1225 list_splice (&iso_sched->td_list, &stream->free_list); 1226 kfree (iso_sched); 1227 } 1228 1229 static int 1230 itd_urb_transaction ( 1231 struct ehci_iso_stream *stream, 1232 struct ehci_hcd *ehci, 1233 struct urb *urb, 1234 gfp_t mem_flags 1235 ) 1236 { 1237 struct ehci_itd *itd; 1238 dma_addr_t itd_dma; 1239 int i; 1240 unsigned num_itds; 1241 struct ehci_iso_sched *sched; 1242 unsigned long flags; 1243 1244 sched = iso_sched_alloc (urb->number_of_packets, mem_flags); 1245 if (unlikely (sched == NULL)) 1246 return -ENOMEM; 1247 1248 itd_sched_init(ehci, sched, stream, urb); 1249 1250 if (urb->interval < 8) 1251 num_itds = 1 + (sched->span + 7) / 8; 1252 else 1253 num_itds = urb->number_of_packets; 1254 1255 /* allocate/init ITDs */ 1256 spin_lock_irqsave (&ehci->lock, flags); 1257 for (i = 0; i < num_itds; i++) { 1258 1259 /* free_list.next might be cache-hot ... but maybe 1260 * the HC caches it too. avoid that issue for now. 1261 */ 1262 1263 /* prefer previously-allocated itds */ 1264 if (likely (!list_empty(&stream->free_list))) { 1265 itd = list_entry (stream->free_list.prev, 1266 struct ehci_itd, itd_list); 1267 list_del (&itd->itd_list); 1268 itd_dma = itd->itd_dma; 1269 } else { 1270 spin_unlock_irqrestore (&ehci->lock, flags); 1271 itd = dma_pool_alloc (ehci->itd_pool, mem_flags, 1272 &itd_dma); 1273 spin_lock_irqsave (&ehci->lock, flags); 1274 if (!itd) { 1275 iso_sched_free(stream, sched); 1276 spin_unlock_irqrestore(&ehci->lock, flags); 1277 return -ENOMEM; 1278 } 1279 } 1280 1281 memset (itd, 0, sizeof *itd); 1282 itd->itd_dma = itd_dma; 1283 list_add (&itd->itd_list, &sched->td_list); 1284 } 1285 spin_unlock_irqrestore (&ehci->lock, flags); 1286 1287 /* temporarily store schedule info in hcpriv */ 1288 urb->hcpriv = sched; 1289 urb->error_count = 0; 1290 return 0; 1291 } 1292 1293 /*-------------------------------------------------------------------------*/ 1294 1295 static inline int 1296 itd_slot_ok ( 1297 struct ehci_hcd *ehci, 1298 u32 mod, 1299 u32 uframe, 1300 u8 usecs, 1301 u32 period 1302 ) 1303 { 1304 uframe %= period; 1305 do { 1306 /* can't commit more than uframe_periodic_max usec */ 1307 if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7) 1308 > (ehci->uframe_periodic_max - usecs)) 1309 return 0; 1310 1311 /* we know urb->interval is 2^N uframes */ 1312 uframe += period; 1313 } while (uframe < mod); 1314 return 1; 1315 } 1316 1317 static inline int 1318 sitd_slot_ok ( 1319 struct ehci_hcd *ehci, 1320 u32 mod, 1321 struct ehci_iso_stream *stream, 1322 u32 uframe, 1323 struct ehci_iso_sched *sched, 1324 u32 period_uframes 1325 ) 1326 { 1327 u32 mask, tmp; 1328 u32 frame, uf; 1329 1330 mask = stream->raw_mask << (uframe & 7); 1331 1332 /* for IN, don't wrap CSPLIT into the next frame */ 1333 if (mask & ~0xffff) 1334 return 0; 1335 1336 /* this multi-pass logic is simple, but performance may 1337 * suffer when the schedule data isn't cached. 1338 */ 1339 1340 /* check bandwidth */ 1341 uframe %= period_uframes; 1342 do { 1343 u32 max_used; 1344 1345 frame = uframe >> 3; 1346 uf = uframe & 7; 1347 1348 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED 1349 /* The tt's fullspeed bus bandwidth must be available. 1350 * tt_available scheduling guarantees 10+% for control/bulk. 1351 */ 1352 if (!tt_available (ehci, period_uframes << 3, 1353 stream->udev, frame, uf, stream->tt_usecs)) 1354 return 0; 1355 #else 1356 /* tt must be idle for start(s), any gap, and csplit. 1357 * assume scheduling slop leaves 10+% for control/bulk. 1358 */ 1359 if (!tt_no_collision (ehci, period_uframes << 3, 1360 stream->udev, frame, mask)) 1361 return 0; 1362 #endif 1363 1364 /* check starts (OUT uses more than one) */ 1365 max_used = ehci->uframe_periodic_max - stream->usecs; 1366 for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) { 1367 if (periodic_usecs (ehci, frame, uf) > max_used) 1368 return 0; 1369 } 1370 1371 /* for IN, check CSPLIT */ 1372 if (stream->c_usecs) { 1373 uf = uframe & 7; 1374 max_used = ehci->uframe_periodic_max - stream->c_usecs; 1375 do { 1376 tmp = 1 << uf; 1377 tmp <<= 8; 1378 if ((stream->raw_mask & tmp) == 0) 1379 continue; 1380 if (periodic_usecs (ehci, frame, uf) 1381 > max_used) 1382 return 0; 1383 } while (++uf < 8); 1384 } 1385 1386 /* we know urb->interval is 2^N uframes */ 1387 uframe += period_uframes; 1388 } while (uframe < mod); 1389 1390 stream->splits = cpu_to_hc32(ehci, stream->raw_mask << (uframe & 7)); 1391 return 1; 1392 } 1393 1394 /* 1395 * This scheduler plans almost as far into the future as it has actual 1396 * periodic schedule slots. (Affected by TUNE_FLS, which defaults to 1397 * "as small as possible" to be cache-friendlier.) That limits the size 1398 * transfers you can stream reliably; avoid more than 64 msec per urb. 1399 * Also avoid queue depths of less than ehci's worst irq latency (affected 1400 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter, 1401 * and other factors); or more than about 230 msec total (for portability, 1402 * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler! 1403 */ 1404 1405 #define SCHEDULE_SLOP 80 /* microframes */ 1406 1407 static int 1408 iso_stream_schedule ( 1409 struct ehci_hcd *ehci, 1410 struct urb *urb, 1411 struct ehci_iso_stream *stream 1412 ) 1413 { 1414 u32 now, next, start, period, span; 1415 int status; 1416 unsigned mod = ehci->periodic_size << 3; 1417 struct ehci_iso_sched *sched = urb->hcpriv; 1418 1419 period = urb->interval; 1420 span = sched->span; 1421 if (!stream->highspeed) { 1422 period <<= 3; 1423 span <<= 3; 1424 } 1425 1426 if (span > mod - SCHEDULE_SLOP) { 1427 ehci_dbg (ehci, "iso request %p too long\n", urb); 1428 status = -EFBIG; 1429 goto fail; 1430 } 1431 1432 now = ehci_read_frame_index(ehci) & (mod - 1); 1433 1434 /* Typical case: reuse current schedule, stream is still active. 1435 * Hopefully there are no gaps from the host falling behind 1436 * (irq delays etc), but if there are we'll take the next 1437 * slot in the schedule, implicitly assuming URB_ISO_ASAP. 1438 */ 1439 if (likely (!list_empty (&stream->td_list))) { 1440 u32 excess; 1441 1442 /* For high speed devices, allow scheduling within the 1443 * isochronous scheduling threshold. For full speed devices 1444 * and Intel PCI-based controllers, don't (work around for 1445 * Intel ICH9 bug). 1446 */ 1447 if (!stream->highspeed && ehci->fs_i_thresh) 1448 next = now + ehci->i_thresh; 1449 else 1450 next = now; 1451 1452 /* Fell behind (by up to twice the slop amount)? 1453 * We decide based on the time of the last currently-scheduled 1454 * slot, not the time of the next available slot. 1455 */ 1456 excess = (stream->next_uframe - period - next) & (mod - 1); 1457 if (excess >= mod - 2 * SCHEDULE_SLOP) 1458 start = next + excess - mod + period * 1459 DIV_ROUND_UP(mod - excess, period); 1460 else 1461 start = next + excess + period; 1462 if (start - now >= mod) { 1463 ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n", 1464 urb, start - now - period, period, 1465 mod); 1466 status = -EFBIG; 1467 goto fail; 1468 } 1469 } 1470 1471 /* need to schedule; when's the next (u)frame we could start? 1472 * this is bigger than ehci->i_thresh allows; scheduling itself 1473 * isn't free, the slop should handle reasonably slow cpus. it 1474 * can also help high bandwidth if the dma and irq loads don't 1475 * jump until after the queue is primed. 1476 */ 1477 else { 1478 start = SCHEDULE_SLOP + (now & ~0x07); 1479 1480 /* NOTE: assumes URB_ISO_ASAP, to limit complexity/bugs */ 1481 1482 /* find a uframe slot with enough bandwidth */ 1483 next = start + period; 1484 for (; start < next; start++) { 1485 1486 /* check schedule: enough space? */ 1487 if (stream->highspeed) { 1488 if (itd_slot_ok(ehci, mod, start, 1489 stream->usecs, period)) 1490 break; 1491 } else { 1492 if ((start % 8) >= 6) 1493 continue; 1494 if (sitd_slot_ok(ehci, mod, stream, 1495 start, sched, period)) 1496 break; 1497 } 1498 } 1499 1500 /* no room in the schedule */ 1501 if (start == next) { 1502 ehci_dbg(ehci, "iso resched full %p (now %d max %d)\n", 1503 urb, now, now + mod); 1504 status = -ENOSPC; 1505 goto fail; 1506 } 1507 } 1508 1509 /* Tried to schedule too far into the future? */ 1510 if (unlikely(start - now + span - period 1511 >= mod - 2 * SCHEDULE_SLOP)) { 1512 ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n", 1513 urb, start - now, span - period, 1514 mod - 2 * SCHEDULE_SLOP); 1515 status = -EFBIG; 1516 goto fail; 1517 } 1518 1519 stream->next_uframe = start & (mod - 1); 1520 1521 /* report high speed start in uframes; full speed, in frames */ 1522 urb->start_frame = stream->next_uframe; 1523 if (!stream->highspeed) 1524 urb->start_frame >>= 3; 1525 return 0; 1526 1527 fail: 1528 iso_sched_free(stream, sched); 1529 urb->hcpriv = NULL; 1530 return status; 1531 } 1532 1533 /*-------------------------------------------------------------------------*/ 1534 1535 static inline void 1536 itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream, 1537 struct ehci_itd *itd) 1538 { 1539 int i; 1540 1541 /* it's been recently zeroed */ 1542 itd->hw_next = EHCI_LIST_END(ehci); 1543 itd->hw_bufp [0] = stream->buf0; 1544 itd->hw_bufp [1] = stream->buf1; 1545 itd->hw_bufp [2] = stream->buf2; 1546 1547 for (i = 0; i < 8; i++) 1548 itd->index[i] = -1; 1549 1550 /* All other fields are filled when scheduling */ 1551 } 1552 1553 static inline void 1554 itd_patch( 1555 struct ehci_hcd *ehci, 1556 struct ehci_itd *itd, 1557 struct ehci_iso_sched *iso_sched, 1558 unsigned index, 1559 u16 uframe 1560 ) 1561 { 1562 struct ehci_iso_packet *uf = &iso_sched->packet [index]; 1563 unsigned pg = itd->pg; 1564 1565 // BUG_ON (pg == 6 && uf->cross); 1566 1567 uframe &= 0x07; 1568 itd->index [uframe] = index; 1569 1570 itd->hw_transaction[uframe] = uf->transaction; 1571 itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12); 1572 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0); 1573 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32)); 1574 1575 /* iso_frame_desc[].offset must be strictly increasing */ 1576 if (unlikely (uf->cross)) { 1577 u64 bufp = uf->bufp + 4096; 1578 1579 itd->pg = ++pg; 1580 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0); 1581 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32)); 1582 } 1583 } 1584 1585 static inline void 1586 itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd) 1587 { 1588 union ehci_shadow *prev = &ehci->pshadow[frame]; 1589 __hc32 *hw_p = &ehci->periodic[frame]; 1590 union ehci_shadow here = *prev; 1591 __hc32 type = 0; 1592 1593 /* skip any iso nodes which might belong to previous microframes */ 1594 while (here.ptr) { 1595 type = Q_NEXT_TYPE(ehci, *hw_p); 1596 if (type == cpu_to_hc32(ehci, Q_TYPE_QH)) 1597 break; 1598 prev = periodic_next_shadow(ehci, prev, type); 1599 hw_p = shadow_next_periodic(ehci, &here, type); 1600 here = *prev; 1601 } 1602 1603 itd->itd_next = here; 1604 itd->hw_next = *hw_p; 1605 prev->itd = itd; 1606 itd->frame = frame; 1607 wmb (); 1608 *hw_p = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD); 1609 } 1610 1611 /* fit urb's itds into the selected schedule slot; activate as needed */ 1612 static int 1613 itd_link_urb ( 1614 struct ehci_hcd *ehci, 1615 struct urb *urb, 1616 unsigned mod, 1617 struct ehci_iso_stream *stream 1618 ) 1619 { 1620 int packet; 1621 unsigned next_uframe, uframe, frame; 1622 struct ehci_iso_sched *iso_sched = urb->hcpriv; 1623 struct ehci_itd *itd; 1624 1625 next_uframe = stream->next_uframe & (mod - 1); 1626 1627 if (unlikely (list_empty(&stream->td_list))) { 1628 ehci_to_hcd(ehci)->self.bandwidth_allocated 1629 += stream->bandwidth; 1630 ehci_vdbg (ehci, 1631 "schedule devp %s ep%d%s-iso period %d start %d.%d\n", 1632 urb->dev->devpath, stream->bEndpointAddress & 0x0f, 1633 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out", 1634 urb->interval, 1635 next_uframe >> 3, next_uframe & 0x7); 1636 } 1637 1638 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 1639 if (ehci->amd_pll_fix == 1) 1640 usb_amd_quirk_pll_disable(); 1641 } 1642 1643 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++; 1644 1645 /* fill iTDs uframe by uframe */ 1646 for (packet = 0, itd = NULL; packet < urb->number_of_packets; ) { 1647 if (itd == NULL) { 1648 /* ASSERT: we have all necessary itds */ 1649 // BUG_ON (list_empty (&iso_sched->td_list)); 1650 1651 /* ASSERT: no itds for this endpoint in this uframe */ 1652 1653 itd = list_entry (iso_sched->td_list.next, 1654 struct ehci_itd, itd_list); 1655 list_move_tail (&itd->itd_list, &stream->td_list); 1656 itd->stream = iso_stream_get (stream); 1657 itd->urb = urb; 1658 itd_init (ehci, stream, itd); 1659 } 1660 1661 uframe = next_uframe & 0x07; 1662 frame = next_uframe >> 3; 1663 1664 itd_patch(ehci, itd, iso_sched, packet, uframe); 1665 1666 next_uframe += stream->interval; 1667 next_uframe &= mod - 1; 1668 packet++; 1669 1670 /* link completed itds into the schedule */ 1671 if (((next_uframe >> 3) != frame) 1672 || packet == urb->number_of_packets) { 1673 itd_link(ehci, frame & (ehci->periodic_size - 1), itd); 1674 itd = NULL; 1675 } 1676 } 1677 stream->next_uframe = next_uframe; 1678 1679 /* don't need that schedule data any more */ 1680 iso_sched_free (stream, iso_sched); 1681 urb->hcpriv = NULL; 1682 1683 timer_action (ehci, TIMER_IO_WATCHDOG); 1684 return enable_periodic(ehci); 1685 } 1686 1687 #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR) 1688 1689 /* Process and recycle a completed ITD. Return true iff its urb completed, 1690 * and hence its completion callback probably added things to the hardware 1691 * schedule. 1692 * 1693 * Note that we carefully avoid recycling this descriptor until after any 1694 * completion callback runs, so that it won't be reused quickly. That is, 1695 * assuming (a) no more than two urbs per frame on this endpoint, and also 1696 * (b) only this endpoint's completions submit URBs. It seems some silicon 1697 * corrupts things if you reuse completed descriptors very quickly... 1698 */ 1699 static unsigned 1700 itd_complete ( 1701 struct ehci_hcd *ehci, 1702 struct ehci_itd *itd 1703 ) { 1704 struct urb *urb = itd->urb; 1705 struct usb_iso_packet_descriptor *desc; 1706 u32 t; 1707 unsigned uframe; 1708 int urb_index = -1; 1709 struct ehci_iso_stream *stream = itd->stream; 1710 struct usb_device *dev; 1711 unsigned retval = false; 1712 1713 /* for each uframe with a packet */ 1714 for (uframe = 0; uframe < 8; uframe++) { 1715 if (likely (itd->index[uframe] == -1)) 1716 continue; 1717 urb_index = itd->index[uframe]; 1718 desc = &urb->iso_frame_desc [urb_index]; 1719 1720 t = hc32_to_cpup(ehci, &itd->hw_transaction [uframe]); 1721 itd->hw_transaction [uframe] = 0; 1722 1723 /* report transfer status */ 1724 if (unlikely (t & ISO_ERRS)) { 1725 urb->error_count++; 1726 if (t & EHCI_ISOC_BUF_ERR) 1727 desc->status = usb_pipein (urb->pipe) 1728 ? -ENOSR /* hc couldn't read */ 1729 : -ECOMM; /* hc couldn't write */ 1730 else if (t & EHCI_ISOC_BABBLE) 1731 desc->status = -EOVERFLOW; 1732 else /* (t & EHCI_ISOC_XACTERR) */ 1733 desc->status = -EPROTO; 1734 1735 /* HC need not update length with this error */ 1736 if (!(t & EHCI_ISOC_BABBLE)) { 1737 desc->actual_length = EHCI_ITD_LENGTH(t); 1738 urb->actual_length += desc->actual_length; 1739 } 1740 } else if (likely ((t & EHCI_ISOC_ACTIVE) == 0)) { 1741 desc->status = 0; 1742 desc->actual_length = EHCI_ITD_LENGTH(t); 1743 urb->actual_length += desc->actual_length; 1744 } else { 1745 /* URB was too late */ 1746 desc->status = -EXDEV; 1747 } 1748 } 1749 1750 /* handle completion now? */ 1751 if (likely ((urb_index + 1) != urb->number_of_packets)) 1752 goto done; 1753 1754 /* ASSERT: it's really the last itd for this urb 1755 list_for_each_entry (itd, &stream->td_list, itd_list) 1756 BUG_ON (itd->urb == urb); 1757 */ 1758 1759 /* give urb back to the driver; completion often (re)submits */ 1760 dev = urb->dev; 1761 ehci_urb_done(ehci, urb, 0); 1762 retval = true; 1763 urb = NULL; 1764 (void) disable_periodic(ehci); 1765 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--; 1766 1767 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 1768 if (ehci->amd_pll_fix == 1) 1769 usb_amd_quirk_pll_enable(); 1770 } 1771 1772 if (unlikely(list_is_singular(&stream->td_list))) { 1773 ehci_to_hcd(ehci)->self.bandwidth_allocated 1774 -= stream->bandwidth; 1775 ehci_vdbg (ehci, 1776 "deschedule devp %s ep%d%s-iso\n", 1777 dev->devpath, stream->bEndpointAddress & 0x0f, 1778 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out"); 1779 } 1780 iso_stream_put (ehci, stream); 1781 1782 done: 1783 itd->urb = NULL; 1784 if (ehci->clock_frame != itd->frame || itd->index[7] != -1) { 1785 /* OK to recycle this ITD now. */ 1786 itd->stream = NULL; 1787 list_move(&itd->itd_list, &stream->free_list); 1788 iso_stream_put(ehci, stream); 1789 } else { 1790 /* HW might remember this ITD, so we can't recycle it yet. 1791 * Move it to a safe place until a new frame starts. 1792 */ 1793 list_move(&itd->itd_list, &ehci->cached_itd_list); 1794 if (stream->refcount == 2) { 1795 /* If iso_stream_put() were called here, stream 1796 * would be freed. Instead, just prevent reuse. 1797 */ 1798 stream->ep->hcpriv = NULL; 1799 stream->ep = NULL; 1800 } 1801 } 1802 return retval; 1803 } 1804 1805 /*-------------------------------------------------------------------------*/ 1806 1807 static int itd_submit (struct ehci_hcd *ehci, struct urb *urb, 1808 gfp_t mem_flags) 1809 { 1810 int status = -EINVAL; 1811 unsigned long flags; 1812 struct ehci_iso_stream *stream; 1813 1814 /* Get iso_stream head */ 1815 stream = iso_stream_find (ehci, urb); 1816 if (unlikely (stream == NULL)) { 1817 ehci_dbg (ehci, "can't get iso stream\n"); 1818 return -ENOMEM; 1819 } 1820 if (unlikely (urb->interval != stream->interval)) { 1821 ehci_dbg (ehci, "can't change iso interval %d --> %d\n", 1822 stream->interval, urb->interval); 1823 goto done; 1824 } 1825 1826 #ifdef EHCI_URB_TRACE 1827 ehci_dbg (ehci, 1828 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n", 1829 __func__, urb->dev->devpath, urb, 1830 usb_pipeendpoint (urb->pipe), 1831 usb_pipein (urb->pipe) ? "in" : "out", 1832 urb->transfer_buffer_length, 1833 urb->number_of_packets, urb->interval, 1834 stream); 1835 #endif 1836 1837 /* allocate ITDs w/o locking anything */ 1838 status = itd_urb_transaction (stream, ehci, urb, mem_flags); 1839 if (unlikely (status < 0)) { 1840 ehci_dbg (ehci, "can't init itds\n"); 1841 goto done; 1842 } 1843 1844 /* schedule ... need to lock */ 1845 spin_lock_irqsave (&ehci->lock, flags); 1846 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) { 1847 status = -ESHUTDOWN; 1848 goto done_not_linked; 1849 } 1850 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb); 1851 if (unlikely(status)) 1852 goto done_not_linked; 1853 status = iso_stream_schedule(ehci, urb, stream); 1854 if (likely (status == 0)) 1855 itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream); 1856 else 1857 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 1858 done_not_linked: 1859 spin_unlock_irqrestore (&ehci->lock, flags); 1860 1861 done: 1862 if (unlikely (status < 0)) 1863 iso_stream_put (ehci, stream); 1864 return status; 1865 } 1866 1867 /*-------------------------------------------------------------------------*/ 1868 1869 /* 1870 * "Split ISO TDs" ... used for USB 1.1 devices going through the 1871 * TTs in USB 2.0 hubs. These need microframe scheduling. 1872 */ 1873 1874 static inline void 1875 sitd_sched_init( 1876 struct ehci_hcd *ehci, 1877 struct ehci_iso_sched *iso_sched, 1878 struct ehci_iso_stream *stream, 1879 struct urb *urb 1880 ) 1881 { 1882 unsigned i; 1883 dma_addr_t dma = urb->transfer_dma; 1884 1885 /* how many frames are needed for these transfers */ 1886 iso_sched->span = urb->number_of_packets * stream->interval; 1887 1888 /* figure out per-frame sitd fields that we'll need later 1889 * when we fit new sitds into the schedule. 1890 */ 1891 for (i = 0; i < urb->number_of_packets; i++) { 1892 struct ehci_iso_packet *packet = &iso_sched->packet [i]; 1893 unsigned length; 1894 dma_addr_t buf; 1895 u32 trans; 1896 1897 length = urb->iso_frame_desc [i].length & 0x03ff; 1898 buf = dma + urb->iso_frame_desc [i].offset; 1899 1900 trans = SITD_STS_ACTIVE; 1901 if (((i + 1) == urb->number_of_packets) 1902 && !(urb->transfer_flags & URB_NO_INTERRUPT)) 1903 trans |= SITD_IOC; 1904 trans |= length << 16; 1905 packet->transaction = cpu_to_hc32(ehci, trans); 1906 1907 /* might need to cross a buffer page within a td */ 1908 packet->bufp = buf; 1909 packet->buf1 = (buf + length) & ~0x0fff; 1910 if (packet->buf1 != (buf & ~(u64)0x0fff)) 1911 packet->cross = 1; 1912 1913 /* OUT uses multiple start-splits */ 1914 if (stream->bEndpointAddress & USB_DIR_IN) 1915 continue; 1916 length = (length + 187) / 188; 1917 if (length > 1) /* BEGIN vs ALL */ 1918 length |= 1 << 3; 1919 packet->buf1 |= length; 1920 } 1921 } 1922 1923 static int 1924 sitd_urb_transaction ( 1925 struct ehci_iso_stream *stream, 1926 struct ehci_hcd *ehci, 1927 struct urb *urb, 1928 gfp_t mem_flags 1929 ) 1930 { 1931 struct ehci_sitd *sitd; 1932 dma_addr_t sitd_dma; 1933 int i; 1934 struct ehci_iso_sched *iso_sched; 1935 unsigned long flags; 1936 1937 iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags); 1938 if (iso_sched == NULL) 1939 return -ENOMEM; 1940 1941 sitd_sched_init(ehci, iso_sched, stream, urb); 1942 1943 /* allocate/init sITDs */ 1944 spin_lock_irqsave (&ehci->lock, flags); 1945 for (i = 0; i < urb->number_of_packets; i++) { 1946 1947 /* NOTE: for now, we don't try to handle wraparound cases 1948 * for IN (using sitd->hw_backpointer, like a FSTN), which 1949 * means we never need two sitds for full speed packets. 1950 */ 1951 1952 /* free_list.next might be cache-hot ... but maybe 1953 * the HC caches it too. avoid that issue for now. 1954 */ 1955 1956 /* prefer previously-allocated sitds */ 1957 if (!list_empty(&stream->free_list)) { 1958 sitd = list_entry (stream->free_list.prev, 1959 struct ehci_sitd, sitd_list); 1960 list_del (&sitd->sitd_list); 1961 sitd_dma = sitd->sitd_dma; 1962 } else { 1963 spin_unlock_irqrestore (&ehci->lock, flags); 1964 sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags, 1965 &sitd_dma); 1966 spin_lock_irqsave (&ehci->lock, flags); 1967 if (!sitd) { 1968 iso_sched_free(stream, iso_sched); 1969 spin_unlock_irqrestore(&ehci->lock, flags); 1970 return -ENOMEM; 1971 } 1972 } 1973 1974 memset (sitd, 0, sizeof *sitd); 1975 sitd->sitd_dma = sitd_dma; 1976 list_add (&sitd->sitd_list, &iso_sched->td_list); 1977 } 1978 1979 /* temporarily store schedule info in hcpriv */ 1980 urb->hcpriv = iso_sched; 1981 urb->error_count = 0; 1982 1983 spin_unlock_irqrestore (&ehci->lock, flags); 1984 return 0; 1985 } 1986 1987 /*-------------------------------------------------------------------------*/ 1988 1989 static inline void 1990 sitd_patch( 1991 struct ehci_hcd *ehci, 1992 struct ehci_iso_stream *stream, 1993 struct ehci_sitd *sitd, 1994 struct ehci_iso_sched *iso_sched, 1995 unsigned index 1996 ) 1997 { 1998 struct ehci_iso_packet *uf = &iso_sched->packet [index]; 1999 u64 bufp = uf->bufp; 2000 2001 sitd->hw_next = EHCI_LIST_END(ehci); 2002 sitd->hw_fullspeed_ep = stream->address; 2003 sitd->hw_uframe = stream->splits; 2004 sitd->hw_results = uf->transaction; 2005 sitd->hw_backpointer = EHCI_LIST_END(ehci); 2006 2007 bufp = uf->bufp; 2008 sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp); 2009 sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32); 2010 2011 sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1); 2012 if (uf->cross) 2013 bufp += 4096; 2014 sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32); 2015 sitd->index = index; 2016 } 2017 2018 static inline void 2019 sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd) 2020 { 2021 /* note: sitd ordering could matter (CSPLIT then SSPLIT) */ 2022 sitd->sitd_next = ehci->pshadow [frame]; 2023 sitd->hw_next = ehci->periodic [frame]; 2024 ehci->pshadow [frame].sitd = sitd; 2025 sitd->frame = frame; 2026 wmb (); 2027 ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD); 2028 } 2029 2030 /* fit urb's sitds into the selected schedule slot; activate as needed */ 2031 static int 2032 sitd_link_urb ( 2033 struct ehci_hcd *ehci, 2034 struct urb *urb, 2035 unsigned mod, 2036 struct ehci_iso_stream *stream 2037 ) 2038 { 2039 int packet; 2040 unsigned next_uframe; 2041 struct ehci_iso_sched *sched = urb->hcpriv; 2042 struct ehci_sitd *sitd; 2043 2044 next_uframe = stream->next_uframe; 2045 2046 if (list_empty(&stream->td_list)) { 2047 /* usbfs ignores TT bandwidth */ 2048 ehci_to_hcd(ehci)->self.bandwidth_allocated 2049 += stream->bandwidth; 2050 ehci_vdbg (ehci, 2051 "sched devp %s ep%d%s-iso [%d] %dms/%04x\n", 2052 urb->dev->devpath, stream->bEndpointAddress & 0x0f, 2053 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out", 2054 (next_uframe >> 3) & (ehci->periodic_size - 1), 2055 stream->interval, hc32_to_cpu(ehci, stream->splits)); 2056 } 2057 2058 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 2059 if (ehci->amd_pll_fix == 1) 2060 usb_amd_quirk_pll_disable(); 2061 } 2062 2063 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++; 2064 2065 /* fill sITDs frame by frame */ 2066 for (packet = 0, sitd = NULL; 2067 packet < urb->number_of_packets; 2068 packet++) { 2069 2070 /* ASSERT: we have all necessary sitds */ 2071 BUG_ON (list_empty (&sched->td_list)); 2072 2073 /* ASSERT: no itds for this endpoint in this frame */ 2074 2075 sitd = list_entry (sched->td_list.next, 2076 struct ehci_sitd, sitd_list); 2077 list_move_tail (&sitd->sitd_list, &stream->td_list); 2078 sitd->stream = iso_stream_get (stream); 2079 sitd->urb = urb; 2080 2081 sitd_patch(ehci, stream, sitd, sched, packet); 2082 sitd_link(ehci, (next_uframe >> 3) & (ehci->periodic_size - 1), 2083 sitd); 2084 2085 next_uframe += stream->interval << 3; 2086 } 2087 stream->next_uframe = next_uframe & (mod - 1); 2088 2089 /* don't need that schedule data any more */ 2090 iso_sched_free (stream, sched); 2091 urb->hcpriv = NULL; 2092 2093 timer_action (ehci, TIMER_IO_WATCHDOG); 2094 return enable_periodic(ehci); 2095 } 2096 2097 /*-------------------------------------------------------------------------*/ 2098 2099 #define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \ 2100 | SITD_STS_XACT | SITD_STS_MMF) 2101 2102 /* Process and recycle a completed SITD. Return true iff its urb completed, 2103 * and hence its completion callback probably added things to the hardware 2104 * schedule. 2105 * 2106 * Note that we carefully avoid recycling this descriptor until after any 2107 * completion callback runs, so that it won't be reused quickly. That is, 2108 * assuming (a) no more than two urbs per frame on this endpoint, and also 2109 * (b) only this endpoint's completions submit URBs. It seems some silicon 2110 * corrupts things if you reuse completed descriptors very quickly... 2111 */ 2112 static unsigned 2113 sitd_complete ( 2114 struct ehci_hcd *ehci, 2115 struct ehci_sitd *sitd 2116 ) { 2117 struct urb *urb = sitd->urb; 2118 struct usb_iso_packet_descriptor *desc; 2119 u32 t; 2120 int urb_index = -1; 2121 struct ehci_iso_stream *stream = sitd->stream; 2122 struct usb_device *dev; 2123 unsigned retval = false; 2124 2125 urb_index = sitd->index; 2126 desc = &urb->iso_frame_desc [urb_index]; 2127 t = hc32_to_cpup(ehci, &sitd->hw_results); 2128 2129 /* report transfer status */ 2130 if (t & SITD_ERRS) { 2131 urb->error_count++; 2132 if (t & SITD_STS_DBE) 2133 desc->status = usb_pipein (urb->pipe) 2134 ? -ENOSR /* hc couldn't read */ 2135 : -ECOMM; /* hc couldn't write */ 2136 else if (t & SITD_STS_BABBLE) 2137 desc->status = -EOVERFLOW; 2138 else /* XACT, MMF, etc */ 2139 desc->status = -EPROTO; 2140 } else { 2141 desc->status = 0; 2142 desc->actual_length = desc->length - SITD_LENGTH(t); 2143 urb->actual_length += desc->actual_length; 2144 } 2145 2146 /* handle completion now? */ 2147 if ((urb_index + 1) != urb->number_of_packets) 2148 goto done; 2149 2150 /* ASSERT: it's really the last sitd for this urb 2151 list_for_each_entry (sitd, &stream->td_list, sitd_list) 2152 BUG_ON (sitd->urb == urb); 2153 */ 2154 2155 /* give urb back to the driver; completion often (re)submits */ 2156 dev = urb->dev; 2157 ehci_urb_done(ehci, urb, 0); 2158 retval = true; 2159 urb = NULL; 2160 (void) disable_periodic(ehci); 2161 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--; 2162 2163 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 2164 if (ehci->amd_pll_fix == 1) 2165 usb_amd_quirk_pll_enable(); 2166 } 2167 2168 if (list_is_singular(&stream->td_list)) { 2169 ehci_to_hcd(ehci)->self.bandwidth_allocated 2170 -= stream->bandwidth; 2171 ehci_vdbg (ehci, 2172 "deschedule devp %s ep%d%s-iso\n", 2173 dev->devpath, stream->bEndpointAddress & 0x0f, 2174 (stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out"); 2175 } 2176 iso_stream_put (ehci, stream); 2177 2178 done: 2179 sitd->urb = NULL; 2180 if (ehci->clock_frame != sitd->frame) { 2181 /* OK to recycle this SITD now. */ 2182 sitd->stream = NULL; 2183 list_move(&sitd->sitd_list, &stream->free_list); 2184 iso_stream_put(ehci, stream); 2185 } else { 2186 /* HW might remember this SITD, so we can't recycle it yet. 2187 * Move it to a safe place until a new frame starts. 2188 */ 2189 list_move(&sitd->sitd_list, &ehci->cached_sitd_list); 2190 if (stream->refcount == 2) { 2191 /* If iso_stream_put() were called here, stream 2192 * would be freed. Instead, just prevent reuse. 2193 */ 2194 stream->ep->hcpriv = NULL; 2195 stream->ep = NULL; 2196 } 2197 } 2198 return retval; 2199 } 2200 2201 2202 static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb, 2203 gfp_t mem_flags) 2204 { 2205 int status = -EINVAL; 2206 unsigned long flags; 2207 struct ehci_iso_stream *stream; 2208 2209 /* Get iso_stream head */ 2210 stream = iso_stream_find (ehci, urb); 2211 if (stream == NULL) { 2212 ehci_dbg (ehci, "can't get iso stream\n"); 2213 return -ENOMEM; 2214 } 2215 if (urb->interval != stream->interval) { 2216 ehci_dbg (ehci, "can't change iso interval %d --> %d\n", 2217 stream->interval, urb->interval); 2218 goto done; 2219 } 2220 2221 #ifdef EHCI_URB_TRACE 2222 ehci_dbg (ehci, 2223 "submit %p dev%s ep%d%s-iso len %d\n", 2224 urb, urb->dev->devpath, 2225 usb_pipeendpoint (urb->pipe), 2226 usb_pipein (urb->pipe) ? "in" : "out", 2227 urb->transfer_buffer_length); 2228 #endif 2229 2230 /* allocate SITDs */ 2231 status = sitd_urb_transaction (stream, ehci, urb, mem_flags); 2232 if (status < 0) { 2233 ehci_dbg (ehci, "can't init sitds\n"); 2234 goto done; 2235 } 2236 2237 /* schedule ... need to lock */ 2238 spin_lock_irqsave (&ehci->lock, flags); 2239 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) { 2240 status = -ESHUTDOWN; 2241 goto done_not_linked; 2242 } 2243 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb); 2244 if (unlikely(status)) 2245 goto done_not_linked; 2246 status = iso_stream_schedule(ehci, urb, stream); 2247 if (status == 0) 2248 sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream); 2249 else 2250 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 2251 done_not_linked: 2252 spin_unlock_irqrestore (&ehci->lock, flags); 2253 2254 done: 2255 if (status < 0) 2256 iso_stream_put (ehci, stream); 2257 return status; 2258 } 2259 2260 /*-------------------------------------------------------------------------*/ 2261 2262 static void free_cached_lists(struct ehci_hcd *ehci) 2263 { 2264 struct ehci_itd *itd, *n; 2265 struct ehci_sitd *sitd, *sn; 2266 2267 list_for_each_entry_safe(itd, n, &ehci->cached_itd_list, itd_list) { 2268 struct ehci_iso_stream *stream = itd->stream; 2269 itd->stream = NULL; 2270 list_move(&itd->itd_list, &stream->free_list); 2271 iso_stream_put(ehci, stream); 2272 } 2273 2274 list_for_each_entry_safe(sitd, sn, &ehci->cached_sitd_list, sitd_list) { 2275 struct ehci_iso_stream *stream = sitd->stream; 2276 sitd->stream = NULL; 2277 list_move(&sitd->sitd_list, &stream->free_list); 2278 iso_stream_put(ehci, stream); 2279 } 2280 } 2281 2282 /*-------------------------------------------------------------------------*/ 2283 2284 static void 2285 scan_periodic (struct ehci_hcd *ehci) 2286 { 2287 unsigned now_uframe, frame, clock, clock_frame, mod; 2288 unsigned modified; 2289 2290 mod = ehci->periodic_size << 3; 2291 2292 /* 2293 * When running, scan from last scan point up to "now" 2294 * else clean up by scanning everything that's left. 2295 * Touches as few pages as possible: cache-friendly. 2296 */ 2297 now_uframe = ehci->next_uframe; 2298 if (ehci->rh_state == EHCI_RH_RUNNING) { 2299 clock = ehci_read_frame_index(ehci); 2300 clock_frame = (clock >> 3) & (ehci->periodic_size - 1); 2301 } else { 2302 clock = now_uframe + mod - 1; 2303 clock_frame = -1; 2304 } 2305 if (ehci->clock_frame != clock_frame) { 2306 free_cached_lists(ehci); 2307 ehci->clock_frame = clock_frame; 2308 } 2309 clock &= mod - 1; 2310 clock_frame = clock >> 3; 2311 ++ehci->periodic_stamp; 2312 2313 for (;;) { 2314 union ehci_shadow q, *q_p; 2315 __hc32 type, *hw_p; 2316 unsigned incomplete = false; 2317 2318 frame = now_uframe >> 3; 2319 2320 restart: 2321 /* scan each element in frame's queue for completions */ 2322 q_p = &ehci->pshadow [frame]; 2323 hw_p = &ehci->periodic [frame]; 2324 q.ptr = q_p->ptr; 2325 type = Q_NEXT_TYPE(ehci, *hw_p); 2326 modified = 0; 2327 2328 while (q.ptr != NULL) { 2329 unsigned uf; 2330 union ehci_shadow temp; 2331 int live; 2332 2333 live = (ehci->rh_state == EHCI_RH_RUNNING); 2334 switch (hc32_to_cpu(ehci, type)) { 2335 case Q_TYPE_QH: 2336 /* handle any completions */ 2337 temp.qh = qh_get (q.qh); 2338 type = Q_NEXT_TYPE(ehci, q.qh->hw->hw_next); 2339 q = q.qh->qh_next; 2340 if (temp.qh->stamp != ehci->periodic_stamp) { 2341 modified = qh_completions(ehci, temp.qh); 2342 if (!modified) 2343 temp.qh->stamp = ehci->periodic_stamp; 2344 if (unlikely(list_empty(&temp.qh->qtd_list) || 2345 temp.qh->needs_rescan)) 2346 intr_deschedule(ehci, temp.qh); 2347 } 2348 qh_put (temp.qh); 2349 break; 2350 case Q_TYPE_FSTN: 2351 /* for "save place" FSTNs, look at QH entries 2352 * in the previous frame for completions. 2353 */ 2354 if (q.fstn->hw_prev != EHCI_LIST_END(ehci)) { 2355 dbg ("ignoring completions from FSTNs"); 2356 } 2357 type = Q_NEXT_TYPE(ehci, q.fstn->hw_next); 2358 q = q.fstn->fstn_next; 2359 break; 2360 case Q_TYPE_ITD: 2361 /* If this ITD is still active, leave it for 2362 * later processing ... check the next entry. 2363 * No need to check for activity unless the 2364 * frame is current. 2365 */ 2366 if (frame == clock_frame && live) { 2367 rmb(); 2368 for (uf = 0; uf < 8; uf++) { 2369 if (q.itd->hw_transaction[uf] & 2370 ITD_ACTIVE(ehci)) 2371 break; 2372 } 2373 if (uf < 8) { 2374 incomplete = true; 2375 q_p = &q.itd->itd_next; 2376 hw_p = &q.itd->hw_next; 2377 type = Q_NEXT_TYPE(ehci, 2378 q.itd->hw_next); 2379 q = *q_p; 2380 break; 2381 } 2382 } 2383 2384 /* Take finished ITDs out of the schedule 2385 * and process them: recycle, maybe report 2386 * URB completion. HC won't cache the 2387 * pointer for much longer, if at all. 2388 */ 2389 *q_p = q.itd->itd_next; 2390 if (!ehci->use_dummy_qh || 2391 q.itd->hw_next != EHCI_LIST_END(ehci)) 2392 *hw_p = q.itd->hw_next; 2393 else 2394 *hw_p = ehci->dummy->qh_dma; 2395 type = Q_NEXT_TYPE(ehci, q.itd->hw_next); 2396 wmb(); 2397 modified = itd_complete (ehci, q.itd); 2398 q = *q_p; 2399 break; 2400 case Q_TYPE_SITD: 2401 /* If this SITD is still active, leave it for 2402 * later processing ... check the next entry. 2403 * No need to check for activity unless the 2404 * frame is current. 2405 */ 2406 if (((frame == clock_frame) || 2407 (((frame + 1) & (ehci->periodic_size - 1)) 2408 == clock_frame)) 2409 && live 2410 && (q.sitd->hw_results & 2411 SITD_ACTIVE(ehci))) { 2412 2413 incomplete = true; 2414 q_p = &q.sitd->sitd_next; 2415 hw_p = &q.sitd->hw_next; 2416 type = Q_NEXT_TYPE(ehci, 2417 q.sitd->hw_next); 2418 q = *q_p; 2419 break; 2420 } 2421 2422 /* Take finished SITDs out of the schedule 2423 * and process them: recycle, maybe report 2424 * URB completion. 2425 */ 2426 *q_p = q.sitd->sitd_next; 2427 if (!ehci->use_dummy_qh || 2428 q.sitd->hw_next != EHCI_LIST_END(ehci)) 2429 *hw_p = q.sitd->hw_next; 2430 else 2431 *hw_p = ehci->dummy->qh_dma; 2432 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next); 2433 wmb(); 2434 modified = sitd_complete (ehci, q.sitd); 2435 q = *q_p; 2436 break; 2437 default: 2438 dbg ("corrupt type %d frame %d shadow %p", 2439 type, frame, q.ptr); 2440 // BUG (); 2441 q.ptr = NULL; 2442 } 2443 2444 /* assume completion callbacks modify the queue */ 2445 if (unlikely (modified)) { 2446 if (likely(ehci->periodic_sched > 0)) 2447 goto restart; 2448 /* short-circuit this scan */ 2449 now_uframe = clock; 2450 break; 2451 } 2452 } 2453 2454 /* If we can tell we caught up to the hardware, stop now. 2455 * We can't advance our scan without collecting the ISO 2456 * transfers that are still pending in this frame. 2457 */ 2458 if (incomplete && ehci->rh_state == EHCI_RH_RUNNING) { 2459 ehci->next_uframe = now_uframe; 2460 break; 2461 } 2462 2463 // FIXME: this assumes we won't get lapped when 2464 // latencies climb; that should be rare, but... 2465 // detect it, and just go all the way around. 2466 // FLR might help detect this case, so long as latencies 2467 // don't exceed periodic_size msec (default 1.024 sec). 2468 2469 // FIXME: likewise assumes HC doesn't halt mid-scan 2470 2471 if (now_uframe == clock) { 2472 unsigned now; 2473 2474 if (ehci->rh_state != EHCI_RH_RUNNING 2475 || ehci->periodic_sched == 0) 2476 break; 2477 ehci->next_uframe = now_uframe; 2478 now = ehci_read_frame_index(ehci) & (mod - 1); 2479 if (now_uframe == now) 2480 break; 2481 2482 /* rescan the rest of this frame, then ... */ 2483 clock = now; 2484 clock_frame = clock >> 3; 2485 if (ehci->clock_frame != clock_frame) { 2486 free_cached_lists(ehci); 2487 ehci->clock_frame = clock_frame; 2488 ++ehci->periodic_stamp; 2489 } 2490 } else { 2491 now_uframe++; 2492 now_uframe &= mod - 1; 2493 } 2494 } 2495 } 2496