1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * This file contains functions that are called via interrupts. 27 */ 28 29 #include <sys/scsi/adapters/pmcs/pmcs.h> 30 31 #ifdef DEBUG 32 #define VALID_IOMB_CHECK(p, w, m, b, c) \ 33 if (!(w & PMCS_IOMB_VALID)) { \ 34 char l[64]; \ 35 (void) snprintf(l, sizeof (l), \ 36 "%s: INVALID IOMB (oq_ci=%u oq_pi=%u)", __func__, b, c); \ 37 pmcs_print_entry(pwp, PMCS_PRT_DEBUG, l, m); \ 38 STEP_OQ_ENTRY(pwp, PMCS_OQ_EVENTS, b, 1); \ 39 continue; \ 40 } 41 #define WRONG_OBID_CHECK(pwp, w, q) \ 42 if (((w & PMCS_IOMB_OBID_MASK) >> PMCS_IOMB_OBID_SHIFT) != q) { \ 43 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, \ 44 "%s: COMPLETION WITH WRONG OBID (0x%x)", __func__, \ 45 (w & PMCS_IOMB_OBID_MASK) >> PMCS_IOMB_OBID_SHIFT); \ 46 } 47 #else 48 #define VALID_IOMB_CHECK(a, b, c, d, e) 49 #define WRONG_OBID_CHECK(a, b, c) 50 #endif 51 52 #define OQLIM_CHECK(p, l) \ 53 if (++l == (p)->ioq_depth) { \ 54 pmcs_prt(p, PMCS_PRT_DEBUG, NULL, NULL, \ 55 "%s: possible ob queue overflow", \ 56 __func__); \ 57 break; \ 58 } 59 60 #define COPY_OUTBOUND(p, w, l, n, a, x, q, c) \ 61 n = ((w & PMCS_IOMB_BC_MASK) >> PMCS_IOMB_BC_SHIFT); \ 62 a = PMCS_QENTRY_SIZE; \ 63 (void) memcpy(l, x, PMCS_QENTRY_SIZE); \ 64 if (n > 1) { \ 65 a <<= 1; \ 66 (void) memcpy(&l[PMCS_QENTRY_SIZE], \ 67 GET_OQ_ENTRY(p, q, c, 1), PMCS_QENTRY_SIZE); \ 68 } \ 69 pmcs_prt(p, PMCS_PRT_DEBUG3, NULL, NULL, \ 70 "%s: ptr %p ci %d w0 %x nbuf %d", \ 71 __func__, (void *)x, ci, w0, n) 72 73 #define EVT_PRT(hwp, msg, phy) \ 74 pmcs_prt(hwp, PMCS_PRT_DEBUG, NULL, NULL, "Phy 0x%x: %s", phy, # msg) 75 76 77 /* 78 * Map the link rate reported in the event to the SAS link rate value 79 */ 80 static uint8_t 81 pmcs_link_rate(uint32_t event_link_rate) 82 { 83 uint8_t sas_link_rate = 0; 84 85 switch (event_link_rate) { 86 case 1: 87 sas_link_rate = SAS_LINK_RATE_1_5GBIT; 88 break; 89 case 2: 90 sas_link_rate = SAS_LINK_RATE_3GBIT; 91 break; 92 case 4: 93 sas_link_rate = SAS_LINK_RATE_6GBIT; 94 break; 95 } 96 97 return (sas_link_rate); 98 } 99 100 /* 101 * Called with pwrk lock 102 */ 103 static void 104 pmcs_complete_work(pmcs_hw_t *pwp, pmcwork_t *pwrk, uint32_t *iomb, size_t amt) 105 { 106 #ifdef DEBUG 107 pwp->ltime[pwp->lti] = gethrtime(); 108 pwp->ltags[pwp->lti++] = pwrk->htag; 109 #endif 110 pwrk->htag |= PMCS_TAG_DONE; 111 112 /* 113 * If the command has timed out, leave it in that state. 114 */ 115 if (pwrk->state != PMCS_WORK_STATE_TIMED_OUT) { 116 pwrk->state = PMCS_WORK_STATE_INTR; 117 } 118 119 pmcs_complete_work_impl(pwp, pwrk, iomb, amt); 120 } 121 122 static void 123 pmcs_work_not_found(pmcs_hw_t *pwp, uint32_t htag, uint32_t *iomb) 124 { 125 #ifdef DEBUG 126 int i; 127 hrtime_t now; 128 char buf[64]; 129 130 (void) snprintf(buf, sizeof (buf), 131 "unable to find work structure for tag 0x%x", htag); 132 133 pmcs_print_entry(pwp, PMCS_PRT_DEBUG1, buf, iomb); 134 if (htag == 0) { 135 return; 136 } 137 now = gethrtime(); 138 for (i = 0; i < 256; i++) { 139 mutex_enter(&pwp->dbglock); 140 if (pwp->ltags[i] == htag) { 141 pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, NULL, 142 "same tag already completed (%llu us ago)", 143 (unsigned long long) (now - pwp->ltime[i])); 144 } 145 if (pwp->ftags[i] == htag) { 146 pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, NULL, 147 "same tag started (line %d) (%llu ns ago)", 148 pwp->ftag_lines[i], (unsigned long long) 149 (now - pwp->ftime[i])); 150 } 151 mutex_exit(&pwp->dbglock); 152 } 153 #else 154 char buf[64]; 155 (void) snprintf(buf, sizeof (buf), 156 "unable to find work structure for tag 0x%x", htag); 157 pmcs_print_entry(pwp, PMCS_PRT_DEBUG1, buf, iomb); 158 #endif 159 } 160 161 162 static void 163 pmcs_process_io_completion(pmcs_hw_t *pwp, pmcs_iocomp_cb_t *ioccb, size_t amt) 164 { 165 pmcwork_t *pwrk; 166 uint32_t tag_type; 167 uint32_t htag = LE_32(((uint32_t *)((void *)ioccb->iomb))[1]); 168 169 pwrk = pmcs_tag2wp(pwp, htag); 170 if (pwrk == NULL) { 171 pmcs_work_not_found(pwp, htag, (void *)&ioccb->iomb); 172 kmem_cache_free(pwp->iocomp_cb_cache, ioccb); 173 return; 174 } 175 176 pwrk->htag |= PMCS_TAG_DONE; 177 178 /* 179 * If the command has timed out, leave it in that state. 180 */ 181 if (pwrk->state != PMCS_WORK_STATE_TIMED_OUT) { 182 pwrk->state = PMCS_WORK_STATE_INTR; 183 } 184 185 /* 186 * Some SATA and SAS commands are run in "WAIT" mode. 187 * We can tell this from the tag type. In this case, 188 * we just do a wakeup (not a callback). 189 */ 190 tag_type = PMCS_TAG_TYPE(pwrk->htag); 191 if (tag_type == PMCS_TAG_TYPE_WAIT) { 192 ASSERT(PMCS_TAG_TYPE(pwrk->htag) == PMCS_TAG_TYPE_WAIT); 193 if (pwrk->arg && amt) { 194 (void) memcpy(pwrk->arg, ioccb->iomb, amt); 195 } 196 cv_signal(&pwrk->sleep_cv); 197 mutex_exit(&pwrk->lock); 198 kmem_cache_free(pwp->iocomp_cb_cache, ioccb); 199 return; 200 } 201 ASSERT(tag_type == PMCS_TAG_TYPE_CBACK); 202 203 #ifdef DEBUG 204 pwp->ltime[pwp->lti] = gethrtime(); 205 pwp->ltags[pwp->lti++] = pwrk->htag; 206 #endif 207 208 ioccb->pwrk = pwrk; 209 210 /* 211 * Only update state to IOCOMPQ if we were in the INTR state. 212 * Any other state (e.g. TIMED_OUT, ABORTED) needs to remain. 213 */ 214 if (pwrk->state == PMCS_WORK_STATE_INTR) { 215 pwrk->state = PMCS_WORK_STATE_IOCOMPQ; 216 } 217 218 mutex_enter(&pwp->cq_lock); 219 if (pwp->iocomp_cb_tail) { 220 pwp->iocomp_cb_tail->next = ioccb; 221 pwp->iocomp_cb_tail = ioccb; 222 } else { 223 pwp->iocomp_cb_head = ioccb; 224 pwp->iocomp_cb_tail = ioccb; 225 } 226 ioccb->next = NULL; 227 mutex_exit(&pwp->cq_lock); 228 229 mutex_exit(&pwrk->lock); 230 /* Completion queue will be run at end of pmcs_iodone_intr */ 231 } 232 233 234 static void 235 pmcs_process_completion(pmcs_hw_t *pwp, void *iomb, size_t amt) 236 { 237 pmcwork_t *pwrk; 238 uint32_t htag = LE_32(((uint32_t *)iomb)[1]); 239 240 pwrk = pmcs_tag2wp(pwp, htag); 241 if (pwrk == NULL) { 242 pmcs_work_not_found(pwp, htag, iomb); 243 return; 244 } 245 246 pmcs_complete_work(pwp, pwrk, iomb, amt); 247 /* 248 * The pwrk lock is now released 249 */ 250 } 251 252 static void 253 pmcs_kill_port(pmcs_hw_t *pwp, int portid) 254 { 255 pmcs_phy_t *pptr = pwp->ports[portid]; 256 257 if (pptr == NULL) { 258 return; 259 } 260 261 /* 262 * Clear any subsidiary phys 263 */ 264 mutex_enter(&pwp->lock); 265 266 for (pptr = pwp->root_phys; pptr; pptr = pptr->sibling) { 267 pmcs_lock_phy(pptr); 268 if (pptr->link_rate && pptr->portid == portid && 269 pptr->subsidiary) { 270 pmcs_clear_phy(pwp, pptr); 271 } 272 pmcs_unlock_phy(pptr); 273 } 274 275 pptr = pwp->ports[portid]; 276 pwp->ports[portid] = NULL; 277 mutex_exit(&pwp->lock); 278 279 pmcs_lock_phy(pptr); 280 pmcs_kill_changed(pwp, pptr, 0); 281 pmcs_unlock_phy(pptr); 282 283 RESTART_DISCOVERY(pwp); 284 pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "PortID 0x%x Cleared", portid); 285 } 286 287 void 288 pmcs_process_sas_hw_event(pmcs_hw_t *pwp, void *iomb, size_t amt) 289 { 290 uint32_t w1 = LE_32(((uint32_t *)iomb)[1]); 291 uint32_t w3 = LE_32(((uint32_t *)iomb)[3]); 292 char buf[32]; 293 uint8_t phynum = IOP_EVENT_PHYNUM(w1); 294 uint8_t portid = IOP_EVENT_PORTID(w1); 295 pmcs_iport_t *iport; 296 pmcs_phy_t *pptr, *subphy, *tphyp; 297 int need_ack = 0; 298 int primary; 299 300 switch (IOP_EVENT_EVENT(w1)) { 301 case IOP_EVENT_PHY_STOP_STATUS: 302 if (IOP_EVENT_STATUS(w1)) { 303 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, 304 "PORT %d failed to stop (0x%x)", 305 phynum, IOP_EVENT_STATUS(w1)); 306 } else { 307 pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL, 308 "PHY 0x%x Stopped", phynum); 309 mutex_enter(&pwp->lock); 310 pptr = pwp->root_phys + phynum; 311 pmcs_lock_phy(pptr); 312 mutex_exit(&pwp->lock); 313 if (pptr->configured) { 314 pmcs_kill_changed(pwp, pptr, 0); 315 } else { 316 pmcs_set_changed(pwp, pptr, B_TRUE, 0); 317 } 318 pmcs_unlock_phy(pptr); 319 RESTART_DISCOVERY(pwp); 320 } 321 /* Reposition htag to the 'expected' position. */ 322 ((uint32_t *)iomb)[1] = ((uint32_t *)iomb)[2]; 323 pmcs_process_completion(pwp, iomb, amt); 324 break; 325 case IOP_EVENT_SAS_PHY_UP: 326 { 327 static const uint8_t sas_identify_af_endian_xfvec[] = { 328 0x5c, 0x5a, 0x56, 0x00 329 }; 330 pmcs_phy_t *rp; 331 sas_identify_af_t af; 332 uint64_t phy_id, wwn; 333 334 /* 335 * If we're not at running state, don't do anything 336 */ 337 mutex_enter(&pwp->lock); 338 if (pwp->state != STATE_RUNNING) { 339 mutex_exit(&pwp->lock); 340 break; 341 } 342 pptr = pwp->root_phys + phynum; 343 pmcs_lock_phy(pptr); 344 345 /* 346 * No need to lock the primary root PHY. It can never go 347 * away, and we're only concerned with the port width and 348 * the portid, both of which only ever change in this function. 349 */ 350 rp = pwp->ports[portid]; 351 352 mutex_exit(&pwp->lock); 353 354 pmcs_endian_transform(pwp, &af, &((uint32_t *)iomb)[4], 355 sas_identify_af_endian_xfvec); 356 357 /* Copy the remote address into our phy handle */ 358 (void) memcpy(pptr->sas_address, af.sas_address, 8); 359 wwn = pmcs_barray2wwn(pptr->sas_address); 360 phy_id = (uint64_t)af.phy_identifier; 361 362 /* 363 * Check to see if there is a PortID already active. 364 */ 365 if (rp) { 366 if (rp->portid != portid) { 367 pmcs_unlock_phy(pptr); 368 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 369 "PortID 0x%x: PHY 0x%x SAS LINK UP IS FOR " 370 "A DIFFERENT PORTID 0x%x", rp->portid, 371 phynum, portid); 372 break; 373 } 374 375 /* 376 * If the dtype isn't NOTHING, then this is actually 377 * the primary PHY for this port. It probably went 378 * down and came back up, so be sure not to mark it 379 * as a subsidiary. 380 */ 381 if (pptr->dtype == NOTHING) { 382 pptr->subsidiary = 1; 383 } 384 pptr->link_rate = 385 pmcs_link_rate(IOP_EVENT_LINK_RATE(w1)); 386 pptr->portid = portid; 387 pptr->dead = 0; 388 pmcs_unlock_phy(pptr); 389 390 rp->width = IOP_EVENT_NPIP(w3); 391 392 /* Add this PHY to the phymap */ 393 if (sas_phymap_phy_add(pwp->hss_phymap, phynum, 394 pwp->sas_wwns[0], wwn) != DDI_SUCCESS) { 395 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 396 "Unable to add phy %u for 0x%" PRIx64 ".0x%" 397 PRIx64, phynum, pwp->sas_wwns[rp->phynum], 398 wwn); 399 } 400 401 /* 402 * Get our iport, if attached, and set it up. Update 403 * the PHY's phymask props while we're locked. 404 */ 405 pmcs_lock_phy(pptr); 406 pmcs_update_phy_pm_props(pptr, (1ULL << phynum), 407 (1ULL << phy_id), B_TRUE); 408 pmcs_unlock_phy(pptr); 409 iport = pmcs_get_iport_by_wwn(pwp, wwn); 410 if (iport) { 411 primary = !pptr->subsidiary; 412 413 mutex_enter(&iport->lock); 414 if (primary) { 415 iport->pptr = pptr; 416 } 417 if (iport->ua_state == UA_ACTIVE) { 418 pmcs_add_phy_to_iport(iport, pptr); 419 pptr->iport = iport; 420 } 421 mutex_exit(&iport->lock); 422 pmcs_rele_iport(iport); 423 } 424 425 pmcs_update_phy_pm_props(rp, (1ULL << phynum), 426 (1ULL << phy_id), B_TRUE); 427 pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL, 428 "PortID 0x%x: PHY 0x%x SAS LINK UP WIDENS PORT " 429 "TO %d PHYS", portid, phynum, rp->width); 430 431 break; 432 } 433 434 /* 435 * Check to see if anything is here already 436 */ 437 if (pptr->dtype != NOTHING && pptr->configured) { 438 pmcs_unlock_phy(pptr); 439 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 440 "PortID 0x%x: SAS PHY 0x%x UP HITS EXISTING " 441 "CONFIGURED TREE", portid, phynum); 442 break; 443 } 444 445 if (af.address_frame_type != SAS_AF_IDENTIFY) { 446 pmcs_unlock_phy(pptr); 447 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 448 "SAS link up on phy 0x%x, " 449 "but unexpected frame type 0x%x found", phynum, 450 af.address_frame_type); 451 break; 452 } 453 pptr->width = IOP_EVENT_NPIP(w3); 454 pptr->portid = portid; 455 pptr->dead = 0; 456 pptr->link_rate = pmcs_link_rate(IOP_EVENT_LINK_RATE(w1)); 457 458 /* 459 * Check to see whether this is an expander or an endpoint 460 */ 461 switch (af.device_type) { 462 case SAS_IF_DTYPE_ENDPOINT: 463 pptr->pend_dtype = SAS; 464 pptr->dtype = SAS; 465 break; 466 case SAS_IF_DTYPE_EDGE: 467 case SAS_IF_DTYPE_FANOUT: 468 pptr->pend_dtype = EXPANDER; 469 pptr->dtype = EXPANDER; 470 break; 471 default: 472 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 473 "unknown device type 0x%x", af.device_type); 474 pptr->pend_dtype = NOTHING; 475 pptr->dtype = NOTHING; 476 break; 477 } 478 479 /* 480 * If this is a direct-attached SAS drive, do the spinup 481 * release now. 482 */ 483 if (pptr->dtype == SAS) { 484 pptr->spinup_hold = 1; 485 pmcs_spinup_release(pwp, pptr); 486 pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL, 487 "Release spinup hold on PHY 0x%x", phynum); 488 } 489 490 pmcs_set_changed(pwp, pptr, B_TRUE, 0); 491 if (pptr->width > 1) { 492 pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL, 493 "PortID 0x%x: PHY 0x%x SAS" 494 " LINK UP @ %s Gb with %d phys/s", portid, phynum, 495 pmcs_get_rate(pptr->link_rate), pptr->width); 496 } else { 497 pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL, 498 "PortID 0x%x: PHY 0x%x SAS" 499 " LINK UP @ %s Gb/s", portid, phynum, 500 pmcs_get_rate(pptr->link_rate)); 501 } 502 pmcs_unlock_phy(pptr); 503 504 /* Add this PHY to the phymap */ 505 if (sas_phymap_phy_add(pwp->hss_phymap, phynum, 506 pwp->sas_wwns[0], wwn) != DDI_SUCCESS) { 507 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 508 "Unable to add phy %u for 0x%" PRIx64 ".0x%" 509 PRIx64, phynum, pwp->sas_wwns[pptr->phynum], wwn); 510 } 511 512 /* Get a pointer to our iport and set it up if attached */ 513 iport = pmcs_get_iport_by_wwn(pwp, wwn); 514 if (iport) { 515 primary = !pptr->subsidiary; 516 517 mutex_enter(&iport->lock); 518 if (primary) { 519 iport->pptr = pptr; 520 } 521 if (iport->ua_state == UA_ACTIVE) { 522 pmcs_add_phy_to_iport(iport, pptr); 523 pptr->iport = iport; 524 } 525 mutex_exit(&iport->lock); 526 pmcs_rele_iport(iport); 527 } 528 529 pmcs_lock_phy(pptr); 530 pmcs_update_phy_pm_props(pptr, (1ULL << phynum), 531 (1ULL << phy_id), B_TRUE); 532 pmcs_smhba_log_sysevent(pwp, ESC_SAS_PHY_EVENT, 533 SAS_PHY_ONLINE, pptr); 534 pmcs_unlock_phy(pptr); 535 536 mutex_enter(&pwp->lock); 537 pwp->ports[portid] = pptr; 538 mutex_exit(&pwp->lock); 539 RESTART_DISCOVERY(pwp); 540 541 break; 542 } 543 case IOP_EVENT_SATA_PHY_UP: { 544 uint64_t wwn; 545 /* 546 * If we're not at running state, don't do anything 547 */ 548 mutex_enter(&pwp->lock); 549 if (pwp->state != STATE_RUNNING) { 550 mutex_exit(&pwp->lock); 551 break; 552 } 553 554 /* 555 * Check to see if anything is here already 556 */ 557 pmcs_lock_phy(pwp->root_phys + phynum); 558 pptr = pwp->root_phys + phynum; 559 mutex_exit(&pwp->lock); 560 561 if (pptr->dtype != NOTHING && pptr->configured) { 562 pmcs_unlock_phy(pptr); 563 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 564 "PortID 0x%x: SATA PHY 0x%x" 565 " UP HITS EXISTING CONFIGURED TREE", 566 portid, phynum); 567 break; 568 } 569 570 pptr->width = 1; 571 pptr->dead = 0; 572 573 /* 574 * Install the PHY number in the least significant byte 575 * with a NAA=3 (locally assigned address) in the most 576 * significant nubble. 577 * 578 * Later, we'll either use that or dig a 579 * WWN out of words 108..111. 580 */ 581 pptr->sas_address[0] = 0x30; 582 pptr->sas_address[1] = 0; 583 pptr->sas_address[2] = 0; 584 pptr->sas_address[3] = 0; 585 pptr->sas_address[4] = 0; 586 pptr->sas_address[5] = 0; 587 pptr->sas_address[6] = 0; 588 pptr->sas_address[7] = phynum; 589 pptr->portid = portid; 590 pptr->link_rate = pmcs_link_rate(IOP_EVENT_LINK_RATE(w1)); 591 pptr->dtype = SATA; 592 pmcs_set_changed(pwp, pptr, B_TRUE, 0); 593 pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL, 594 "PortID 0x%x: PHY 0x%x SATA LINK UP @ %s Gb/s", 595 pptr->portid, phynum, pmcs_get_rate(pptr->link_rate)); 596 wwn = pmcs_barray2wwn(pptr->sas_address); 597 pmcs_unlock_phy(pptr); 598 599 /* Add this PHY to the phymap */ 600 if (sas_phymap_phy_add(pwp->hss_phymap, phynum, 601 pwp->sas_wwns[0], wwn) != DDI_SUCCESS) { 602 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 603 "Unable to add phy %u for 0x%" PRIx64 ".0x%" 604 PRIx64, phynum, pwp->sas_wwns[pptr->phynum], 605 wwn); 606 } 607 608 /* Get our iport, if attached, and set it up */ 609 iport = pmcs_get_iport_by_wwn(pwp, wwn); 610 if (iport) { 611 mutex_enter(&iport->lock); 612 iport->pptr = pptr; 613 if (iport->ua_state == UA_ACTIVE) { 614 pmcs_add_phy_to_iport(iport, pptr); 615 pptr->iport = iport; 616 ASSERT(iport->nphy == 1); 617 iport->nphy = 1; 618 } 619 mutex_exit(&iport->lock); 620 pmcs_rele_iport(iport); 621 } 622 623 pmcs_lock_phy(pptr); 624 pmcs_update_phy_pm_props(pptr, (1ULL << phynum), 1ULL, B_TRUE); 625 pmcs_smhba_log_sysevent(pwp, ESC_SAS_PHY_EVENT, 626 SAS_PHY_ONLINE, pptr); 627 pmcs_unlock_phy(pptr); 628 629 mutex_enter(&pwp->lock); 630 pwp->ports[pptr->portid] = pptr; 631 mutex_exit(&pwp->lock); 632 RESTART_DISCOVERY(pwp); 633 break; 634 } 635 636 case IOP_EVENT_SATA_SPINUP_HOLD: 637 tphyp = (pmcs_phy_t *)(pwp->root_phys + phynum); 638 /* 639 * No need to lock the entire tree for this 640 */ 641 mutex_enter(&tphyp->phy_lock); 642 tphyp->spinup_hold = 1; 643 pmcs_spinup_release(pwp, tphyp); 644 mutex_exit(&tphyp->phy_lock); 645 break; 646 case IOP_EVENT_PHY_DOWN: { 647 uint64_t wwn; 648 649 /* 650 * If we're not at running state, don't do anything 651 */ 652 mutex_enter(&pwp->lock); 653 if (pwp->state != STATE_RUNNING) { 654 mutex_exit(&pwp->lock); 655 break; 656 } 657 pptr = pwp->ports[portid]; 658 659 subphy = pwp->root_phys + phynum; 660 /* 661 * subphy is a pointer to the PHY corresponding to the incoming 662 * event. pptr points to the primary PHY for the corresponding 663 * port. So, subphy and pptr may or may not be the same PHY, 664 * but that doesn't change what we need to do with each. 665 */ 666 ASSERT(subphy); 667 mutex_exit(&pwp->lock); 668 669 if (pptr == NULL) { 670 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, 671 "PortID 0x%x: PHY 0x%x LINK DOWN- no portid ptr", 672 portid, phynum); 673 break; 674 } 675 if (IOP_EVENT_PORT_STATE(w3) == IOP_EVENT_PS_NIL) { 676 pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, 677 "PortID 0x%x: PHY 0x%x NOT VALID YET", 678 portid, phynum); 679 need_ack = 1; 680 break; 681 } 682 if (IOP_EVENT_PORT_STATE(w3) == IOP_EVENT_PS_IN_RESET) { 683 pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, 684 "PortID 0x%x: PHY 0x%x IN RESET", 685 portid, phynum); 686 /* Entire port is down due to a host-initiated reset */ 687 mutex_enter(&pptr->phy_lock); 688 /* Clear the phymask props in pptr */ 689 pmcs_update_phy_pm_props(pptr, pptr->att_port_pm_tmp, 690 pptr->tgt_port_pm_tmp, B_FALSE); 691 iport = pptr->iport; 692 mutex_exit(&pptr->phy_lock); 693 if (iport) { 694 mutex_enter(&iport->lock); 695 pmcs_iport_teardown_phys(iport); 696 mutex_exit(&iport->lock); 697 } 698 699 /* Clear down all PHYs in the port */ 700 for (pptr = pwp->root_phys; pptr; 701 pptr = pptr->sibling) { 702 pmcs_lock_phy(pptr); 703 if (pptr->portid == portid) { 704 pptr->dtype = NOTHING; 705 pptr->portid = 706 PMCS_IPORT_INVALID_PORT_ID; 707 if (pptr->valid_device_id) { 708 pptr->deregister_wait = 1; 709 } 710 } 711 pmcs_unlock_phy(pptr); 712 SCHEDULE_WORK(pwp, PMCS_WORK_DEREGISTER_DEV); 713 (void) ddi_taskq_dispatch(pwp->tq, pmcs_worker, 714 pwp, DDI_NOSLEEP); 715 } 716 717 break; 718 } 719 720 if (IOP_EVENT_PORT_STATE(w3) == IOP_EVENT_PS_LOSTCOMM) { 721 pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, 722 "PortID 0x%x: PHY 0x%x TEMPORARILY DOWN", 723 portid, phynum); 724 need_ack = 1; 725 break; 726 } 727 728 if (IOP_EVENT_PORT_STATE(w3) == IOP_EVENT_PS_VALID) { 729 730 /* 731 * This is not the last phy in the port, so if this 732 * is the primary PHY, promote another PHY to primary. 733 */ 734 if (pptr == subphy) { 735 primary = !subphy->subsidiary; 736 ASSERT(primary); 737 738 tphyp = pptr; 739 pptr = pmcs_promote_next_phy(tphyp); 740 741 if (pptr) { 742 /* Update primary pptr in ports */ 743 pwp->ports[portid] = pptr; 744 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, 745 NULL, "PortID 0x%x: PHY 0x%x " 746 "promoted to primary", portid, 747 pptr->phynum); 748 } else { 749 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, 750 NULL, "PortID 0x%x: PHY 0x%x: " 751 "unable to promote phy", portid, 752 phynum); 753 } 754 } 755 756 /* 757 * Drop port width on the primary phy handle 758 * No need to lock the entire tree for this 759 */ 760 mutex_enter(&pptr->phy_lock); 761 pmcs_update_phy_pm_props(pptr, subphy->att_port_pm_tmp, 762 subphy->tgt_port_pm_tmp, B_FALSE); 763 pptr->width = IOP_EVENT_NPIP(w3); 764 mutex_exit(&pptr->phy_lock); 765 766 /* Clear the iport reference and portid on the subphy */ 767 mutex_enter(&subphy->phy_lock); 768 iport = subphy->iport; 769 subphy->iport = NULL; 770 subphy->portid = PMCS_PHY_INVALID_PORT_ID; 771 subphy->dtype = NOTHING; 772 mutex_exit(&subphy->phy_lock); 773 774 /* 775 * If the iport was set on this phy, decrement its 776 * nphy count and remove this phy from the phys list. 777 */ 778 if (iport) { 779 mutex_enter(&iport->lock); 780 if (iport->ua_state == UA_ACTIVE) { 781 pmcs_remove_phy_from_iport(iport, 782 subphy); 783 } 784 mutex_exit(&iport->lock); 785 } 786 787 pmcs_lock_phy(subphy); 788 wwn = pmcs_barray2wwn(pptr->sas_address); 789 if (subphy->subsidiary) 790 pmcs_clear_phy(pwp, subphy); 791 pmcs_unlock_phy(subphy); 792 793 /* Remove this PHY from the phymap */ 794 if (sas_phymap_phy_rem(pwp->hss_phymap, phynum) != 795 DDI_SUCCESS) { 796 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 797 "Unable to remove phy %u for 0x%" PRIx64 798 ".0x%" PRIx64, phynum, 799 pwp->sas_wwns[pptr->phynum], wwn); 800 } 801 802 pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL, 803 "PortID 0x%x: PHY 0x%x LINK DOWN NARROWS PORT " 804 "TO %d PHYS", portid, phynum, pptr->width); 805 break; 806 } 807 if (IOP_EVENT_PORT_STATE(w3) != IOP_EVENT_PS_INVALID) { 808 pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, 809 "PortID 0x%x: PHY 0x%x LINK DOWN NOT HANDLED " 810 "(state 0x%x)", portid, phynum, 811 IOP_EVENT_PORT_STATE(w3)); 812 need_ack = 1; 813 break; 814 } 815 /* Remove this PHY from the phymap */ 816 if (sas_phymap_phy_rem(pwp->hss_phymap, phynum) != 817 DDI_SUCCESS) { 818 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 819 "Unable to remove phy %u for 0x%" PRIx64 820 ".0x%" PRIx64, phynum, 821 pwp->sas_wwns[pptr->phynum], wwn); 822 } 823 824 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, 825 "PortID 0x%x: PHY 0x%x LINK DOWN (port invalid)", 826 portid, phynum); 827 828 /* 829 * Last PHY on the port. 830 * Assumption: pptr and subphy are both "valid". In fact, 831 * they should be one and the same. 832 * 833 * Drop port width on the primary phy handle 834 * Report the event and clear its PHY pm props while we've 835 * got the lock 836 */ 837 ASSERT(pptr == subphy); 838 mutex_enter(&pptr->phy_lock); 839 pptr->width = 0; 840 pmcs_update_phy_pm_props(pptr, pptr->att_port_pm_tmp, 841 pptr->tgt_port_pm_tmp, B_FALSE); 842 pmcs_smhba_log_sysevent(pwp, ESC_SAS_PHY_EVENT, 843 SAS_PHY_OFFLINE, pptr); 844 mutex_exit(&pptr->phy_lock); 845 846 /* Clear the iport reference and portid on the subphy */ 847 pmcs_lock_phy(subphy); 848 iport = subphy->iport; 849 subphy->deregister_wait = 1; 850 subphy->iport = NULL; 851 subphy->portid = PMCS_PHY_INVALID_PORT_ID; 852 subphy->dtype = NOTHING; 853 pmcs_unlock_phy(subphy); 854 SCHEDULE_WORK(pwp, PMCS_WORK_DEREGISTER_DEV); 855 (void) ddi_taskq_dispatch(pwp->tq, pmcs_worker, 856 pwp, DDI_NOSLEEP); 857 858 /* 859 * If the iport was set on this phy, decrement its 860 * nphy count and remove this phy from the phys list. 861 * Also, clear the iport's pptr as this port is now 862 * down. 863 */ 864 if (iport) { 865 mutex_enter(&iport->lock); 866 if (iport->ua_state == UA_ACTIVE) { 867 pmcs_remove_phy_from_iport(iport, subphy); 868 iport->pptr = NULL; 869 iport->ua_state = UA_PEND_DEACTIVATE; 870 } 871 mutex_exit(&iport->lock); 872 } 873 874 pmcs_lock_phy(subphy); 875 if (subphy->subsidiary) 876 pmcs_clear_phy(pwp, subphy); 877 pmcs_unlock_phy(subphy); 878 879 /* 880 * Since we're now really dead, it's time to clean up. 881 */ 882 pmcs_kill_port(pwp, portid); 883 need_ack = 1; 884 885 break; 886 } 887 case IOP_EVENT_BROADCAST_CHANGE: 888 pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL, 889 "PortID 0x%x: PHY 0x%x Broadcast Change", portid, phynum); 890 need_ack = 1; 891 mutex_enter(&pwp->lock); 892 pptr = pwp->ports[portid]; 893 if (pptr) { 894 pmcs_lock_phy(pptr); 895 if (pptr->phynum == phynum) { 896 pmcs_set_changed(pwp, pptr, B_TRUE, 0); 897 } 898 pmcs_smhba_log_sysevent(pwp, ESC_SAS_HBA_PORT_BROADCAST, 899 SAS_PORT_BROADCAST_CHANGE, pptr); 900 pmcs_unlock_phy(pptr); 901 } 902 mutex_exit(&pwp->lock); 903 RESTART_DISCOVERY(pwp); 904 break; 905 case IOP_EVENT_BROADCAST_SES: 906 EVT_PRT(pwp, IOP_EVENT_BROADCAST_SES, phynum); 907 mutex_enter(&pwp->lock); 908 pptr = pwp->ports[portid]; 909 mutex_exit(&pwp->lock); 910 if (pptr) { 911 pmcs_lock_phy(pptr); 912 pmcs_smhba_log_sysevent(pwp, ESC_SAS_HBA_PORT_BROADCAST, 913 SAS_PORT_BROADCAST_SES, pptr); 914 pmcs_unlock_phy(pptr); 915 } 916 break; 917 case IOP_EVENT_PHY_ERR_INBOUND_CRC: 918 { 919 char buf[32]; 920 (void) snprintf(buf, sizeof (buf), "Inbound PHY CRC error"); 921 need_ack = 1; 922 break; 923 } 924 case IOP_EVENT_HARD_RESET_RECEIVED: 925 EVT_PRT(pwp, IOP_EVENT_HARD_RESET_RECEIVED, phynum); 926 break; 927 case IOP_EVENT_EVENT_ID_FRAME_TIMO: 928 EVT_PRT(pwp, IOP_EVENT_EVENT_ID_FRAME_TIMO, phynum); 929 break; 930 case IOP_EVENT_BROADCAST_EXP: 931 pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, 932 "PortID 0x%x: PHY 0x%x Broadcast Exp Change", 933 portid, phynum); 934 /* 935 * Comparing Section 6.8.1.4 of SMHBA (rev 7) spec and Section 936 * 7.2.3 of SAS2 (Rev 15) spec, 937 * _BROADCAST_EXPANDER event corresponds to _D01_4 primitive 938 */ 939 mutex_enter(&pwp->lock); 940 pptr = pwp->ports[portid]; 941 mutex_exit(&pwp->lock); 942 if (pptr) { 943 pmcs_lock_phy(pptr); 944 pmcs_smhba_log_sysevent(pwp, ESC_SAS_HBA_PORT_BROADCAST, 945 SAS_PORT_BROADCAST_D01_4, pptr); 946 pmcs_unlock_phy(pptr); 947 } 948 break; 949 case IOP_EVENT_PHY_START_STATUS: 950 switch (IOP_EVENT_STATUS(w1)) { 951 case IOP_PHY_START_OK: 952 pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL, 953 "PHY 0x%x Started", phynum); 954 break; 955 case IOP_PHY_START_ALREADY: 956 pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, 957 "PHY 0x%x Started (Already)", phynum); 958 break; 959 case IOP_PHY_START_INVALID: 960 pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL, 961 "PHY 0x%x failed to start (invalid phy)", phynum); 962 break; 963 case IOP_PHY_START_ERROR: 964 pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL, 965 "PHY 0x%x Start Error", phynum); 966 break; 967 default: 968 pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL, 969 "PHY 0x%x failed to start (0x%x)", phynum, 970 IOP_EVENT_STATUS(w1)); 971 break; 972 } 973 /* Reposition htag to the 'expected' position. */ 974 ((uint32_t *)iomb)[1] = ((uint32_t *)iomb)[2]; 975 pmcs_process_completion(pwp, iomb, amt); 976 break; 977 case IOP_EVENT_PHY_ERR_INVALID_DWORD: 978 need_ack = 1; 979 EVT_PRT(pwp, IOP_EVENT_PHY_ERR_INVALID_DWORD, phynum); 980 break; 981 case IOP_EVENT_PHY_ERR_DISPARITY_ERROR: 982 need_ack = 1; 983 EVT_PRT(pwp, IOP_EVENT_PHY_ERR_DISPARITY_ERROR, phynum); 984 break; 985 case IOP_EVENT_PHY_ERR_CODE_VIOLATION: 986 need_ack = 1; 987 EVT_PRT(pwp, IOP_EVENT_PHY_ERR_CODE_VIOLATION, phynum); 988 break; 989 case IOP_EVENT_PHY_ERR_LOSS_OF_DWORD_SYN: 990 need_ack = 1; 991 EVT_PRT(pwp, IOP_EVENT_PHY_ERR_LOSS_OF_DWORD_SYN, phynum); 992 break; 993 case IOP_EVENT_PHY_ERR_PHY_RESET_FAILD: 994 need_ack = 1; 995 EVT_PRT(pwp, IOP_EVENT_PHY_ERR_PHY_RESET_FAILD, phynum); 996 break; 997 case IOP_EVENT_PORT_RECOVERY_TIMER_TMO: 998 EVT_PRT(pwp, IOP_EVENT_PORT_RECOVERY_TIMER_TMO, phynum); 999 break; 1000 case IOP_EVENT_PORT_RECOVER: 1001 EVT_PRT(pwp, IOP_EVENT_PORT_RECOVER, phynum); 1002 break; 1003 case IOP_EVENT_PORT_INVALID: 1004 mutex_enter(&pwp->lock); 1005 if (pwp->state != STATE_RUNNING) { 1006 mutex_exit(&pwp->lock); 1007 break; 1008 } 1009 mutex_exit(&pwp->lock); 1010 pmcs_kill_port(pwp, portid); 1011 pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, 1012 "PortID 0x%x: PORT Now Invalid", portid); 1013 break; 1014 case IOP_EVENT_PORT_RESET_TIMER_TMO: 1015 EVT_PRT(pwp, IOP_EVENT_PORT_RESET_TIMER_TMO, phynum); 1016 break; 1017 case IOP_EVENT_PORT_RESET_COMPLETE: 1018 EVT_PRT(pwp, IOP_EVENT_PORT_RESET_COMPLETE, phynum); 1019 break; 1020 case IOP_EVENT_BROADCAST_ASYNC_EVENT: 1021 EVT_PRT(pwp, IOP_EVENT_BROADCAST_ASYNC_EVENT, phynum); 1022 /* 1023 * Comparing Section 6.8.1.4 of SMHBA (rev 7) spec and Section 1024 * 7.2.3 of SAS2 (Rev 15) spec, 1025 * _BROADCAST_ASYNC event corresponds to _D04_7 primitive 1026 */ 1027 mutex_enter(&pwp->lock); 1028 pptr = pwp->ports[portid]; 1029 mutex_exit(&pwp->lock); 1030 if (pptr) { 1031 pmcs_lock_phy(pptr); 1032 pmcs_smhba_log_sysevent(pwp, ESC_SAS_HBA_PORT_BROADCAST, 1033 SAS_PORT_BROADCAST_D04_7, pptr); 1034 pmcs_unlock_phy(pptr); 1035 } 1036 break; 1037 default: 1038 (void) snprintf(buf, sizeof (buf), 1039 "unknown SAS H/W Event PHY 0x%x", phynum); 1040 pmcs_print_entry(pwp, PMCS_PRT_DEBUG, buf, iomb); 1041 break; 1042 } 1043 if (need_ack) { 1044 mutex_enter(&pwp->lock); 1045 /* 1046 * Don't lock the entire tree for this. Just grab the mutex 1047 * on the root PHY. 1048 */ 1049 tphyp = pwp->root_phys + phynum; 1050 mutex_enter(&tphyp->phy_lock); 1051 tphyp->hw_event_ack = w1; 1052 mutex_exit(&tphyp->phy_lock); 1053 mutex_exit(&pwp->lock); 1054 pmcs_ack_events(pwp); 1055 } 1056 } 1057 1058 static void 1059 pmcs_process_echo_completion(pmcs_hw_t *pwp, void *iomb, size_t amt) 1060 { 1061 echo_test_t fred; 1062 pmcwork_t *pwrk; 1063 uint32_t *msg = iomb, htag = LE_32(msg[1]); 1064 pwrk = pmcs_tag2wp(pwp, htag); 1065 if (pwrk) { 1066 (void) memcpy(&fred, &((uint32_t *)iomb)[2], sizeof (fred)); 1067 fred.ptr[0]++; 1068 msg[2] = LE_32(PMCOUT_STATUS_OK); 1069 pmcs_complete_work(pwp, pwrk, msg, amt); 1070 } else { 1071 pmcs_print_entry(pwp, PMCS_PRT_DEBUG, 1072 "ECHO completion with no work structure", iomb); 1073 } 1074 } 1075 1076 static void 1077 pmcs_process_ssp_event(pmcs_hw_t *pwp, void *iomb, size_t amt) 1078 { 1079 _NOTE(ARGUNUSED(amt)); 1080 uint32_t status, htag, *w; 1081 pmcwork_t *pwrk; 1082 pmcs_phy_t *phyp = NULL; 1083 char *path; 1084 1085 w = iomb; 1086 htag = LE_32(w[1]); 1087 status = LE_32(w[2]); 1088 1089 1090 pwrk = pmcs_tag2wp(pwp, htag); 1091 if (pwrk == NULL) { 1092 path = "????"; 1093 } else { 1094 phyp = pwrk->phy; 1095 path = pwrk->phy->path; 1096 } 1097 1098 if (status != PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED) { 1099 char buf[20]; 1100 const char *emsg = pmcs_status_str(status); 1101 1102 if (emsg == NULL) { 1103 (void) snprintf(buf, sizeof (buf), "Status 0x%x", 1104 status); 1105 emsg = buf; 1106 } 1107 pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, NULL, "%s: Bad SAS Status " 1108 "(tag 0x%x) %s on %s", __func__, htag, emsg, path); 1109 if (pwrk != NULL) { 1110 /* 1111 * There may be pending command on a target device. 1112 * Or, it may be a double fault. 1113 */ 1114 pmcs_start_ssp_event_recovery(pwp, pwrk, iomb, amt); 1115 } 1116 } else { 1117 pmcs_prt(pwp, PMCS_PRT_DEBUG2, phyp, NULL, 1118 "%s: tag %x put onto the wire for %s", 1119 __func__, htag, path); 1120 if (pwrk) { 1121 pwrk->onwire = 1; 1122 mutex_exit(&pwrk->lock); 1123 } 1124 } 1125 } 1126 1127 static void 1128 pmcs_process_sata_event(pmcs_hw_t *pwp, void *iomb, size_t amt) 1129 { 1130 _NOTE(ARGUNUSED(amt)); 1131 pmcwork_t *pwrk = NULL; 1132 pmcs_phy_t *pptr; 1133 uint32_t status, htag, *w; 1134 char *path; 1135 1136 w = iomb; 1137 htag = LE_32(w[1]); 1138 status = LE_32(w[2]); 1139 1140 /* 1141 * If the status is PMCOUT_STATUS_XFER_ERROR_ABORTED_NCQ_MODE, 1142 * we have to issue a READ LOG EXT ATA (page 0x10) command 1143 * to the device. In this case, htag is not valid. 1144 * 1145 * If the status is PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED, we're 1146 * just noting that an I/O got put onto the wire. 1147 * 1148 * Othewise, other errors are indicative that things need to 1149 * be aborted. 1150 */ 1151 path = NULL; 1152 if (htag) { 1153 pwrk = pmcs_tag2wp(pwp, htag); 1154 if (pwrk) { 1155 pmcs_lock_phy(pwrk->phy); 1156 pptr = pwrk->phy; 1157 path = pptr->path; 1158 } 1159 } 1160 if (path == NULL) { 1161 mutex_enter(&pwp->lock); 1162 pptr = pmcs_find_phy_by_devid(pwp, LE_32(w[4])); 1163 /* This PHY is now locked */ 1164 mutex_exit(&pwp->lock); 1165 if (pptr) { 1166 path = pptr->path; 1167 } else { 1168 path = "????"; 1169 } 1170 } 1171 1172 if (status != PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED) { 1173 char buf[20]; 1174 const char *emsg = pmcs_status_str(status); 1175 1176 ASSERT(pptr != NULL); 1177 if (emsg == NULL) { 1178 (void) snprintf(buf, sizeof (buf), "Status 0x%x", 1179 status); 1180 emsg = buf; 1181 } 1182 if (status == PMCOUT_STATUS_XFER_ERROR_ABORTED_NCQ_MODE) { 1183 ASSERT(pptr != NULL); 1184 pptr->need_rl_ext = 1; 1185 htag = 0; 1186 } else { 1187 pptr->abort_pending = 1; 1188 } 1189 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 1190 "%s: Bad SATA Status (tag 0x%x) %s on %s", 1191 __func__, htag, emsg, path); 1192 SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE); 1193 /* 1194 * Unlike SSP devices, we let the abort we 1195 * schedule above force the completion of 1196 * problem commands. 1197 */ 1198 if (pwrk) { 1199 mutex_exit(&pwrk->lock); 1200 } 1201 } else if (status == PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED) { 1202 pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, NULL, 1203 "%s: tag %x put onto the wire for %s", 1204 __func__, htag, path); 1205 if (pwrk) { 1206 pwrk->onwire = 1; 1207 mutex_exit(&pwrk->lock); 1208 } 1209 } 1210 1211 if (pptr) { 1212 pmcs_unlock_phy(pptr); 1213 } 1214 } 1215 1216 static void 1217 pmcs_process_abort_completion(pmcs_hw_t *pwp, void *iomb, size_t amt) 1218 { 1219 pmcs_phy_t *pptr; 1220 struct pmcwork *pwrk; 1221 uint32_t htag = LE_32(((uint32_t *)iomb)[1]); 1222 uint32_t status = LE_32(((uint32_t *)iomb)[2]); 1223 uint32_t scp = LE_32(((uint32_t *)iomb)[3]) & 0x1; 1224 char *path; 1225 1226 pwrk = pmcs_tag2wp(pwp, htag); 1227 if (pwrk == NULL) { 1228 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, 1229 "%s: cannot find work structure for ABORT", __func__); 1230 return; 1231 } 1232 1233 pptr = pwrk->phy; 1234 if (pptr) { 1235 /* 1236 * Don't use pmcs_lock_phy here since it could potentially lock 1237 * other PHYs beneath, which is unnecessary in this context. 1238 */ 1239 mutex_enter(&pptr->phy_lock); 1240 pptr->abort_pending = 0; 1241 pptr->abort_sent = 0; 1242 1243 /* 1244 * Don't do this if the status was ABORT_IN_PROGRESS and 1245 * the scope bit was set 1246 */ 1247 if ((status != PMCOUT_STATUS_IO_ABORT_IN_PROGRESS) || !scp) { 1248 pptr->abort_all_start = 0; 1249 cv_signal(&pptr->abort_all_cv); 1250 } 1251 path = pptr->path; 1252 mutex_exit(&pptr->phy_lock); 1253 } else { 1254 path = "(no phy)"; 1255 } 1256 1257 switch (status) { 1258 case PMCOUT_STATUS_OK: 1259 if (scp) { 1260 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 1261 "%s: abort all succeeded for %s. (htag=0x%x)", 1262 __func__, path, htag); 1263 } else { 1264 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 1265 "%s: abort tag 0x%x succeeded for %s. (htag=0x%x)", 1266 __func__, pwrk->abt_htag, path, htag); 1267 } 1268 break; 1269 1270 case PMCOUT_STATUS_IO_NOT_VALID: 1271 if (scp) { 1272 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 1273 "%s: ABORT %s failed (DEV NOT VALID) for %s. " 1274 "(htag=0x%x)", __func__, scp ? "all" : "tag", 1275 path, htag); 1276 } else { 1277 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, 1278 "%s: ABORT %s failed (I/O NOT VALID) for %s. " 1279 "(htag=0x%x)", __func__, scp ? "all" : "tag", 1280 path, htag); 1281 } 1282 break; 1283 1284 case PMCOUT_STATUS_IO_ABORT_IN_PROGRESS: 1285 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, "%s: ABORT %s failed " 1286 "for %s, htag 0x%x (ABORT IN PROGRESS)", __func__, 1287 scp ? "all" : "tag", path, htag); 1288 break; 1289 1290 default: 1291 pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, "%s: Unknown status " 1292 "%d for ABORT %s, htag 0x%x, PHY %s", __func__, status, 1293 scp ? "all" : "tag", htag, path); 1294 break; 1295 } 1296 1297 pmcs_complete_work(pwp, pwrk, iomb, amt); 1298 } 1299 1300 static void 1301 pmcs_process_general_event(pmcs_hw_t *pwp, uint32_t *iomb) 1302 { 1303 uint32_t htag; 1304 char local[60]; 1305 struct pmcwork *pwrk; 1306 int i; 1307 1308 if (LE_32(iomb[1]) == INBOUND_IOMB_V_BIT_NOT_SET) { 1309 (void) snprintf(local, sizeof (local), 1310 "VALID bit not set on INBOUND IOMB"); 1311 } else if (LE_32(iomb[1]) == 1312 INBOUND_IOMB_OPC_NOT_SUPPORTED) { 1313 (void) snprintf(local, sizeof (local), 1314 "opcode not set on inbound IOMB"); 1315 } else { 1316 (void) snprintf(local, sizeof (local), 1317 "unknown GENERAL EVENT status (0x%x)", 1318 LE_32(iomb[1])); 1319 } 1320 /* Pull up bad IOMB into usual position */ 1321 for (i = 0; i < PMCS_MSG_SIZE - 2; i++) { 1322 iomb[i] = iomb[i+2]; 1323 } 1324 /* overwrite status with an error */ 1325 iomb[2] = LE_32(PMCOUT_STATUS_PROG_ERROR); 1326 iomb[PMCS_MSG_SIZE - 2] = 0; 1327 iomb[PMCS_MSG_SIZE - 1] = 0; 1328 htag = LE_32(iomb[1]); 1329 pmcs_print_entry(pwp, PMCS_PRT_DEBUG, local, iomb); 1330 pwrk = pmcs_tag2wp(pwp, htag); 1331 if (pwrk) { 1332 pmcs_complete_work(pwp, pwrk, iomb, PMCS_QENTRY_SIZE); 1333 } 1334 } 1335 1336 void 1337 pmcs_general_intr(pmcs_hw_t *pwp) 1338 { 1339 char local[PMCS_QENTRY_SIZE << 1]; 1340 uint32_t w0, pi, ci; 1341 uint32_t *ptr, nbuf, lim = 0; 1342 size_t amt; 1343 1344 ci = pmcs_rd_oqci(pwp, PMCS_OQ_GENERAL); 1345 pi = pmcs_rd_oqpi(pwp, PMCS_OQ_GENERAL); 1346 1347 while (ci != pi) { 1348 OQLIM_CHECK(pwp, lim); 1349 ptr = GET_OQ_ENTRY(pwp, PMCS_OQ_GENERAL, ci, 0); 1350 w0 = LE_32(ptr[0]); 1351 VALID_IOMB_CHECK(pwp, w0, ptr, ci, pi); 1352 WRONG_OBID_CHECK(pwp, w0, PMCS_OQ_GENERAL); 1353 COPY_OUTBOUND(pwp, w0, local, nbuf, amt, ptr, 1354 PMCS_OQ_GENERAL, ci); 1355 1356 switch (w0 & PMCS_IOMB_OPCODE_MASK) { 1357 case PMCOUT_SSP_COMPLETION: 1358 /* 1359 * We only get SSP completion here for Task Management 1360 * completions. 1361 */ 1362 case PMCOUT_SMP_COMPLETION: 1363 case PMCOUT_LOCAL_PHY_CONTROL: 1364 case PMCOUT_DEVICE_REGISTRATION: 1365 case PMCOUT_DEREGISTER_DEVICE_HANDLE: 1366 case PMCOUT_GET_NVMD_DATA: 1367 case PMCOUT_SET_NVMD_DATA: 1368 case PMCOUT_GET_DEVICE_STATE: 1369 case PMCOUT_SET_DEVICE_STATE: 1370 pmcs_process_completion(pwp, local, amt); 1371 break; 1372 case PMCOUT_SSP_ABORT: 1373 case PMCOUT_SATA_ABORT: 1374 case PMCOUT_SMP_ABORT: 1375 pmcs_process_abort_completion(pwp, local, amt); 1376 break; 1377 case PMCOUT_SSP_EVENT: 1378 pmcs_process_ssp_event(pwp, local, amt); 1379 break; 1380 case PMCOUT_ECHO: 1381 pmcs_process_echo_completion(pwp, local, amt); 1382 break; 1383 case PMCOUT_SAS_HW_EVENT_ACK_ACK: 1384 if (LE_32(ptr[2]) != SAS_HW_EVENT_ACK_OK) { 1385 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, 1386 "SAS H/W EVENT ACK/ACK Status=0x%b", 1387 LE_32(ptr[2]), "\020\4InvParm\3" 1388 "InvPort\2InvPhy\1InvSEA"); 1389 } 1390 pmcs_process_completion(pwp, local, amt); 1391 break; 1392 case PMCOUT_SKIP_ENTRIES: 1393 pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL, 1394 "%s: skip %d entries", __func__, nbuf); 1395 break; 1396 default: 1397 (void) snprintf(local, sizeof (local), 1398 "%s: unhandled message", __func__); 1399 pmcs_print_entry(pwp, PMCS_PRT_DEBUG, local, ptr); 1400 break; 1401 } 1402 STEP_OQ_ENTRY(pwp, PMCS_OQ_GENERAL, ci, nbuf); 1403 } 1404 if (lim) { 1405 SYNC_OQ_ENTRY(pwp, PMCS_OQ_GENERAL, ci, pi); 1406 } 1407 } 1408 1409 /* 1410 * pmcs_check_intr_coal 1411 * 1412 * This function makes a determination on the dynamic value of the 1413 * interrupt coalescing timer register. We only use this for I/O 1414 * completions. 1415 * 1416 * The basic algorithm is as follows: 1417 * 1418 * PMCS_MAX_IO_COMPS_PER_INTR: The maximum number of I/O completions per 1419 * I/O completion interrupt. We won't increase the interrupt coalescing 1420 * timer if we're already processing this many completions per interrupt 1421 * beyond the threshold. 1422 * 1423 * Values in io_intr_coal structure: 1424 * 1425 * intr_latency: The average number of nsecs between interrupts during 1426 * the echo test. Used to help determine whether to increase the coalescing 1427 * timer. 1428 * 1429 * intr_threshold: Calculated number of interrupts beyond which we may 1430 * increase the timer. This value is calculated based on the calculated 1431 * interrupt latency during the ECHO test and the current value of the 1432 * coalescing timer. 1433 * 1434 * nsecs_between_intrs: Total number of nsecs between all the interrupts 1435 * in the current timeslice. 1436 * 1437 * last_io_comp: Time of the last I/O interrupt. 1438 * 1439 * num_io_completions: Number of I/O completions during the slice 1440 * 1441 * num_intrs: Number of I/O completion interrupts during the slice 1442 * 1443 * max_io_completions: Number of times we hit >= PMCS_MAX_IO_COMPS_PER_INTR 1444 * during interrupt processing. 1445 * 1446 * PMCS_MAX_IO_COMPS_LOWAT_SHIFT/HIWAT_SHIFT 1447 * Low and high marks used to determine whether we processed enough interrupts 1448 * that contained the maximum number of I/O completions to warrant increasing 1449 * the timer 1450 * 1451 * intr_coal_timer: The current value of the register (in usecs) 1452 * 1453 * timer_on: B_TRUE means we are using the timer 1454 * 1455 * The timer is increased if we processed more than intr_threshold interrupts 1456 * during the quantum and the number of interrupts containing the maximum 1457 * number of I/O completions is between PMCS_MAX_IO_COMPS_LOWAT_SHIFT and 1458 * _HIWAT_SHIFT 1459 * 1460 * If the average time between completions is greater than twice 1461 * the current timer value, the timer value is decreased. 1462 * 1463 * If we did not take any interrupts during a quantum, we turn the timer off. 1464 */ 1465 void 1466 pmcs_check_intr_coal(void *arg) 1467 { 1468 pmcs_hw_t *pwp = (pmcs_hw_t *)arg; 1469 uint32_t avg_nsecs; 1470 clock_t lbolt, ret; 1471 pmcs_io_intr_coal_t *ici; 1472 1473 ici = &pwp->io_intr_coal; 1474 mutex_enter(&pwp->ict_lock); 1475 while (ici->stop_thread == B_FALSE) { 1476 /* 1477 * Wait for next time quantum... collect stats 1478 */ 1479 lbolt = ddi_get_lbolt(); 1480 while (ici->stop_thread == B_FALSE) { 1481 ret = cv_timedwait(&pwp->ict_cv, &pwp->ict_lock, 1482 lbolt + ici->quantum); 1483 if (ret == -1) { 1484 break; 1485 } 1486 } 1487 1488 if (ici->stop_thread == B_TRUE) { 1489 continue; 1490 } 1491 1492 DTRACE_PROBE1(pmcs__check__intr__coal, pmcs_io_intr_coal_t *, 1493 &pwp->io_intr_coal); 1494 1495 /* 1496 * Determine whether to adjust timer 1497 */ 1498 if (ici->num_intrs == 0) { 1499 /* 1500 * If timer is off, nothing more to do. 1501 */ 1502 if (!pwp->io_intr_coal.timer_on) { 1503 continue; 1504 } 1505 1506 /* 1507 * No interrupts. Turn off the timer. 1508 */ 1509 pmcs_wr_topunit(pwp, PMCS_INT_COALESCING_CONTROL, 0); 1510 1511 if (pwp->odb_auto_clear & (1 << PMCS_MSIX_IODONE)) { 1512 pmcs_wr_topunit(pwp, PMCS_OBDB_AUTO_CLR, 1513 pwp->odb_auto_clear); 1514 } 1515 1516 ici->timer_on = B_FALSE; 1517 ici->max_io_completions = 0; 1518 ici->num_intrs = 0; 1519 ici->int_cleared = B_FALSE; 1520 ici->num_io_completions = 0; 1521 DTRACE_PROBE1(pmcs__intr__coalesce__timer__off, 1522 pmcs_io_intr_coal_t *, ici); 1523 continue; 1524 } 1525 1526 avg_nsecs = ici->nsecs_between_intrs / ici->num_intrs; 1527 1528 if ((ici->num_intrs > ici->intr_threshold) && 1529 (ici->max_io_completions > (ici->num_intrs >> 1530 PMCS_MAX_IO_COMPS_LOWAT_SHIFT)) && 1531 (ici->max_io_completions < (ici->num_intrs >> 1532 PMCS_MAX_IO_COMPS_HIWAT_SHIFT))) { 1533 pmcs_set_intr_coal_timer(pwp, INCREASE_TIMER); 1534 } else if (avg_nsecs > 1535 (ici->intr_coal_timer * 1000 * 2)) { 1536 pmcs_set_intr_coal_timer(pwp, DECREASE_TIMER); 1537 } 1538 1539 /* 1540 * Reset values for new sampling period. 1541 */ 1542 ici->max_io_completions = 0; 1543 ici->nsecs_between_intrs = 0; 1544 ici->num_intrs = 0; 1545 ici->num_io_completions = 0; 1546 1547 /* 1548 * If a firmware event log file is configured, check to see 1549 * if it needs to be written to the file. We do this here 1550 * because writing to a file from a callout thread (i.e. 1551 * from the watchdog timer) can cause livelocks. 1552 */ 1553 if (pwp->fwlog_file) { 1554 mutex_exit(&pwp->ict_lock); 1555 pmcs_gather_fwlog(pwp); 1556 mutex_enter(&pwp->ict_lock); 1557 } 1558 } 1559 1560 mutex_exit(&pwp->ict_lock); 1561 thread_exit(); 1562 } 1563 1564 void 1565 pmcs_iodone_intr(pmcs_hw_t *pwp) 1566 { 1567 char local[PMCS_QENTRY_SIZE << 1]; 1568 pmcs_iocomp_cb_t *ioccb; 1569 uint32_t w0, ci, pi, nbuf, lim = 0, niodone = 0, iomb_opcode; 1570 size_t amt; 1571 uint32_t *ptr; 1572 hrtime_t curtime = gethrtime(); 1573 1574 ci = pmcs_rd_oqci(pwp, PMCS_OQ_IODONE); 1575 pi = pmcs_rd_oqpi(pwp, PMCS_OQ_IODONE); 1576 1577 while (ci != pi) { 1578 OQLIM_CHECK(pwp, lim); 1579 ptr = GET_OQ_ENTRY(pwp, PMCS_OQ_IODONE, ci, 0); 1580 w0 = LE_32(ptr[0]); 1581 VALID_IOMB_CHECK(pwp, w0, ptr, ci, pi); 1582 WRONG_OBID_CHECK(pwp, w0, PMCS_OQ_IODONE); 1583 iomb_opcode = (w0 & PMCS_IOMB_OPCODE_MASK); 1584 1585 if ((iomb_opcode == PMCOUT_SSP_COMPLETION) || 1586 (iomb_opcode == PMCOUT_SATA_COMPLETION)) { 1587 ioccb = 1588 kmem_cache_alloc(pwp->iocomp_cb_cache, KM_NOSLEEP); 1589 if (ioccb == NULL) { 1590 pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL, 1591 "%s: kmem_cache_alloc failed", __func__); 1592 break; 1593 } 1594 1595 COPY_OUTBOUND(pwp, w0, ioccb->iomb, nbuf, amt, ptr, 1596 PMCS_OQ_IODONE, ci); 1597 1598 niodone++; 1599 pmcs_process_io_completion(pwp, ioccb, amt); 1600 } else { 1601 COPY_OUTBOUND(pwp, w0, local, nbuf, amt, ptr, 1602 PMCS_OQ_IODONE, ci); 1603 1604 switch (iomb_opcode) { 1605 case PMCOUT_ECHO: 1606 pmcs_process_echo_completion(pwp, local, amt); 1607 break; 1608 case PMCOUT_SATA_EVENT: 1609 pmcs_process_sata_event(pwp, local, amt); 1610 break; 1611 case PMCOUT_SSP_EVENT: 1612 pmcs_process_ssp_event(pwp, local, amt); 1613 break; 1614 case PMCOUT_SKIP_ENTRIES: 1615 pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL, 1616 "%s: skip %d entries", __func__, nbuf); 1617 break; 1618 default: 1619 (void) snprintf(local, sizeof (local), 1620 "%s: unhandled message", __func__); 1621 pmcs_print_entry(pwp, PMCS_PRT_DEBUG, local, 1622 ptr); 1623 break; 1624 } 1625 } 1626 1627 STEP_OQ_ENTRY(pwp, PMCS_OQ_IODONE, ci, nbuf); 1628 } 1629 1630 if (lim != 0) { 1631 SYNC_OQ_ENTRY(pwp, PMCS_OQ_IODONE, ci, pi); 1632 } 1633 1634 /* 1635 * Update the interrupt coalescing timer check stats and run 1636 * completions for queued up commands. 1637 */ 1638 1639 if (niodone > 0) { 1640 /* 1641 * If we can't get the lock, then completions are either 1642 * already running or will be scheduled to do so shortly. 1643 */ 1644 if (mutex_tryenter(&pwp->cq_lock) != 0) { 1645 PMCS_CQ_RUN_LOCKED(pwp); 1646 mutex_exit(&pwp->cq_lock); 1647 } 1648 1649 mutex_enter(&pwp->ict_lock); 1650 pwp->io_intr_coal.nsecs_between_intrs += 1651 curtime - pwp->io_intr_coal.last_io_comp; 1652 pwp->io_intr_coal.num_intrs++; 1653 pwp->io_intr_coal.num_io_completions += niodone; 1654 if (niodone >= PMCS_MAX_IO_COMPS_PER_INTR) { 1655 pwp->io_intr_coal.max_io_completions++; 1656 } 1657 pwp->io_intr_coal.last_io_comp = gethrtime(); 1658 mutex_exit(&pwp->ict_lock); 1659 } 1660 } 1661 1662 void 1663 pmcs_event_intr(pmcs_hw_t *pwp) 1664 { 1665 char local[PMCS_QENTRY_SIZE << 1]; 1666 uint32_t w0, ci, pi, nbuf, lim = 0; 1667 size_t amt; 1668 uint32_t *ptr; 1669 1670 ci = pmcs_rd_oqci(pwp, PMCS_OQ_EVENTS); 1671 pi = pmcs_rd_oqpi(pwp, PMCS_OQ_EVENTS); 1672 1673 while (ci != pi) { 1674 OQLIM_CHECK(pwp, lim); 1675 ptr = GET_OQ_ENTRY(pwp, PMCS_OQ_EVENTS, ci, 0); 1676 w0 = LE_32(ptr[0]); 1677 VALID_IOMB_CHECK(pwp, w0, ptr, ci, pi); 1678 WRONG_OBID_CHECK(pwp, w0, PMCS_OQ_EVENTS); 1679 COPY_OUTBOUND(pwp, w0, local, nbuf, amt, ptr, 1680 PMCS_OQ_EVENTS, ci); 1681 1682 switch (w0 & PMCS_IOMB_OPCODE_MASK) { 1683 case PMCOUT_ECHO: 1684 pmcs_process_echo_completion(pwp, local, amt); 1685 break; 1686 case PMCOUT_SATA_EVENT: 1687 pmcs_process_sata_event(pwp, local, amt); 1688 break; 1689 case PMCOUT_SSP_EVENT: 1690 pmcs_process_ssp_event(pwp, local, amt); 1691 break; 1692 case PMCOUT_GENERAL_EVENT: 1693 pmcs_process_general_event(pwp, ptr); 1694 break; 1695 case PMCOUT_DEVICE_HANDLE_REMOVED: 1696 { 1697 uint32_t port = IOP_EVENT_PORTID(LE_32(ptr[1])); 1698 uint32_t did = LE_32(ptr[2]); 1699 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, 1700 "PortID 0x%x device_id 0x%x removed", port, did); 1701 break; 1702 } 1703 case PMCOUT_SAS_HW_EVENT: 1704 if (nbuf > 1) { 1705 pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, 1706 "multiple SAS HW_EVENT (%d) responses " 1707 "in EVENT OQ", nbuf); 1708 } 1709 pmcs_process_sas_hw_event(pwp, local, PMCS_QENTRY_SIZE); 1710 break; 1711 case PMCOUT_FW_FLASH_UPDATE: 1712 case PMCOUT_GET_TIME_STAMP: 1713 case PMCOUT_GET_DEVICE_STATE: 1714 case PMCOUT_SET_DEVICE_STATE: 1715 case PMCOUT_SAS_DIAG_EXECUTE: 1716 pmcs_process_completion(pwp, local, amt); 1717 break; 1718 case PMCOUT_SKIP_ENTRIES: 1719 pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL, 1720 "%s: skip %d entries", __func__, nbuf); 1721 break; 1722 default: 1723 (void) snprintf(local, sizeof (local), 1724 "%s: unhandled message", __func__); 1725 pmcs_print_entry(pwp, PMCS_PRT_DEBUG, local, ptr); 1726 break; 1727 } 1728 STEP_OQ_ENTRY(pwp, PMCS_OQ_EVENTS, ci, nbuf); 1729 } 1730 if (lim) { 1731 SYNC_OQ_ENTRY(pwp, PMCS_OQ_EVENTS, ci, pi); 1732 } 1733 } 1734 1735 void 1736 pmcs_timed_out(pmcs_hw_t *pwp, uint32_t htag, const char *func) 1737 { 1738 #ifdef DEBUG 1739 hrtime_t now = gethrtime(); 1740 int i; 1741 1742 for (i = 0; i < 256; i++) { 1743 if (pwp->ftags[i] == htag) { 1744 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, 1745 "Inbound msg (tag 0x%8x) timed out - " 1746 "was started %llu ns ago in %s:%d", 1747 htag, (unsigned long long) (now - pwp->ftime[i]), 1748 func, pwp->ftag_lines[i]); 1749 return; 1750 } 1751 } 1752 #endif 1753 pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, 1754 "Inbound Message (tag 0x%08x) timed out- was started in %s", 1755 htag, func); 1756 } 1757