1 /*- 2 * Common functions for CAM "type" (peripheral) drivers. 3 * 4 * SPDX-License-Identifier: BSD-2-Clause 5 * 6 * Copyright (c) 1997, 1998 Justin T. Gibbs. 7 * Copyright (c) 1997, 1998, 1999, 2000 Kenneth D. Merry. 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions, and the following disclaimer, 15 * without modification, immediately at the beginning of the file. 16 * 2. The name of the author may not be used to endorse or promote products 17 * derived from this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 23 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/types.h> 35 #include <sys/malloc.h> 36 #include <sys/kernel.h> 37 #include <sys/bio.h> 38 #include <sys/conf.h> 39 #include <sys/devctl.h> 40 #include <sys/lock.h> 41 #include <sys/mutex.h> 42 #include <sys/buf.h> 43 #include <sys/proc.h> 44 #include <sys/devicestat.h> 45 #include <sys/sbuf.h> 46 #include <sys/sysctl.h> 47 #include <vm/vm.h> 48 #include <vm/vm_extern.h> 49 50 #include <cam/cam.h> 51 #include <cam/cam_ccb.h> 52 #include <cam/cam_compat.h> 53 #include <cam/cam_queue.h> 54 #include <cam/cam_xpt_periph.h> 55 #include <cam/cam_xpt_internal.h> 56 #include <cam/cam_periph.h> 57 #include <cam/cam_debug.h> 58 #include <cam/cam_sim.h> 59 60 #include <cam/scsi/scsi_all.h> 61 #include <cam/scsi/scsi_message.h> 62 #include <cam/scsi/scsi_pass.h> 63 64 static u_int camperiphnextunit(struct periph_driver *p_drv, 65 u_int newunit, bool wired, 66 path_id_t pathid, target_id_t target, 67 lun_id_t lun); 68 static u_int camperiphunit(struct periph_driver *p_drv, 69 path_id_t pathid, target_id_t target, 70 lun_id_t lun, 71 const char *sn); 72 static void camperiphdone(struct cam_periph *periph, 73 union ccb *done_ccb); 74 static void camperiphfree(struct cam_periph *periph); 75 static int camperiphscsistatuserror(union ccb *ccb, 76 union ccb **orig_ccb, 77 cam_flags camflags, 78 uint32_t sense_flags, 79 int *openings, 80 uint32_t *relsim_flags, 81 uint32_t *timeout, 82 uint32_t *action, 83 const char **action_string); 84 static int camperiphscsisenseerror(union ccb *ccb, 85 union ccb **orig_ccb, 86 cam_flags camflags, 87 uint32_t sense_flags, 88 int *openings, 89 uint32_t *relsim_flags, 90 uint32_t *timeout, 91 uint32_t *action, 92 const char **action_string); 93 static void cam_periph_devctl_notify(union ccb *ccb); 94 95 static int nperiph_drivers; 96 static int initialized = 0; 97 struct periph_driver **periph_drivers; 98 99 static MALLOC_DEFINE(M_CAMPERIPH, "CAM periph", "CAM peripheral buffers"); 100 101 static int periph_selto_delay = 1000; 102 TUNABLE_INT("kern.cam.periph_selto_delay", &periph_selto_delay); 103 static int periph_noresrc_delay = 500; 104 TUNABLE_INT("kern.cam.periph_noresrc_delay", &periph_noresrc_delay); 105 static int periph_busy_delay = 500; 106 TUNABLE_INT("kern.cam.periph_busy_delay", &periph_busy_delay); 107 108 static u_int periph_mapmem_thresh = 65536; 109 SYSCTL_UINT(_kern_cam, OID_AUTO, mapmem_thresh, CTLFLAG_RWTUN, 110 &periph_mapmem_thresh, 0, "Threshold for user-space buffer mapping"); 111 112 void 113 periphdriver_register(void *data) 114 { 115 struct periph_driver *drv = (struct periph_driver *)data; 116 struct periph_driver **newdrivers, **old; 117 int ndrivers; 118 119 again: 120 ndrivers = nperiph_drivers + 2; 121 newdrivers = malloc(sizeof(*newdrivers) * ndrivers, M_CAMPERIPH, 122 M_WAITOK); 123 xpt_lock_buses(); 124 if (ndrivers != nperiph_drivers + 2) { 125 /* 126 * Lost race against itself; go around. 127 */ 128 xpt_unlock_buses(); 129 free(newdrivers, M_CAMPERIPH); 130 goto again; 131 } 132 if (periph_drivers) 133 bcopy(periph_drivers, newdrivers, 134 sizeof(*newdrivers) * nperiph_drivers); 135 newdrivers[nperiph_drivers] = drv; 136 newdrivers[nperiph_drivers + 1] = NULL; 137 old = periph_drivers; 138 periph_drivers = newdrivers; 139 nperiph_drivers++; 140 xpt_unlock_buses(); 141 if (old) 142 free(old, M_CAMPERIPH); 143 /* If driver marked as early or it is late now, initialize it. */ 144 if (((drv->flags & CAM_PERIPH_DRV_EARLY) != 0 && initialized > 0) || 145 initialized > 1) 146 (*drv->init)(); 147 } 148 149 int 150 periphdriver_unregister(void *data) 151 { 152 struct periph_driver *drv = (struct periph_driver *)data; 153 int error, n; 154 155 /* If driver marked as early or it is late now, deinitialize it. */ 156 if (((drv->flags & CAM_PERIPH_DRV_EARLY) != 0 && initialized > 0) || 157 initialized > 1) { 158 if (drv->deinit == NULL) { 159 printf("CAM periph driver '%s' doesn't have deinit.\n", 160 drv->driver_name); 161 return (EOPNOTSUPP); 162 } 163 error = drv->deinit(); 164 if (error != 0) 165 return (error); 166 } 167 168 xpt_lock_buses(); 169 for (n = 0; n < nperiph_drivers && periph_drivers[n] != drv; n++) 170 ; 171 KASSERT(n < nperiph_drivers, 172 ("Periph driver '%s' was not registered", drv->driver_name)); 173 for (; n + 1 < nperiph_drivers; n++) 174 periph_drivers[n] = periph_drivers[n + 1]; 175 periph_drivers[n + 1] = NULL; 176 nperiph_drivers--; 177 xpt_unlock_buses(); 178 return (0); 179 } 180 181 void 182 periphdriver_init(int level) 183 { 184 int i, early; 185 186 initialized = max(initialized, level); 187 for (i = 0; periph_drivers[i] != NULL; i++) { 188 early = (periph_drivers[i]->flags & CAM_PERIPH_DRV_EARLY) ? 1 : 2; 189 if (early == initialized) 190 (*periph_drivers[i]->init)(); 191 } 192 } 193 194 cam_status 195 cam_periph_alloc(periph_ctor_t *periph_ctor, 196 periph_oninv_t *periph_oninvalidate, 197 periph_dtor_t *periph_dtor, periph_start_t *periph_start, 198 char *name, cam_periph_type type, struct cam_path *path, 199 ac_callback_t *ac_callback, ac_code code, void *arg) 200 { 201 struct periph_driver **p_drv; 202 struct cam_sim *sim; 203 struct cam_periph *periph; 204 struct cam_periph *cur_periph; 205 path_id_t path_id; 206 target_id_t target_id; 207 lun_id_t lun_id; 208 cam_status status; 209 u_int init_level; 210 211 init_level = 0; 212 /* 213 * Handle Hot-Plug scenarios. If there is already a peripheral 214 * of our type assigned to this path, we are likely waiting for 215 * final close on an old, invalidated, peripheral. If this is 216 * the case, queue up a deferred call to the peripheral's async 217 * handler. If it looks like a mistaken re-allocation, complain. 218 */ 219 if ((periph = cam_periph_find(path, name)) != NULL) { 220 if ((periph->flags & CAM_PERIPH_INVALID) != 0 221 && (periph->flags & CAM_PERIPH_NEW_DEV_FOUND) == 0) { 222 periph->flags |= CAM_PERIPH_NEW_DEV_FOUND; 223 periph->deferred_callback = ac_callback; 224 periph->deferred_ac = code; 225 return (CAM_REQ_INPROG); 226 } else { 227 printf("cam_periph_alloc: attempt to re-allocate " 228 "valid device %s%d rejected flags %#x " 229 "refcount %d\n", periph->periph_name, 230 periph->unit_number, periph->flags, 231 periph->refcount); 232 } 233 return (CAM_REQ_INVALID); 234 } 235 236 periph = (struct cam_periph *)malloc(sizeof(*periph), M_CAMPERIPH, 237 M_NOWAIT|M_ZERO); 238 239 if (periph == NULL) 240 return (CAM_RESRC_UNAVAIL); 241 242 init_level++; 243 244 sim = xpt_path_sim(path); 245 path_id = xpt_path_path_id(path); 246 target_id = xpt_path_target_id(path); 247 lun_id = xpt_path_lun_id(path); 248 periph->periph_start = periph_start; 249 periph->periph_dtor = periph_dtor; 250 periph->periph_oninval = periph_oninvalidate; 251 periph->type = type; 252 periph->periph_name = name; 253 periph->scheduled_priority = CAM_PRIORITY_NONE; 254 periph->immediate_priority = CAM_PRIORITY_NONE; 255 periph->refcount = 1; /* Dropped by invalidation. */ 256 periph->sim = sim; 257 SLIST_INIT(&periph->ccb_list); 258 status = xpt_create_path(&path, periph, path_id, target_id, lun_id); 259 if (status != CAM_REQ_CMP) 260 goto failure; 261 periph->path = path; 262 263 xpt_lock_buses(); 264 for (p_drv = periph_drivers; *p_drv != NULL; p_drv++) { 265 if (strcmp((*p_drv)->driver_name, name) == 0) 266 break; 267 } 268 if (*p_drv == NULL) { 269 printf("cam_periph_alloc: invalid periph name '%s'\n", name); 270 xpt_unlock_buses(); 271 xpt_free_path(periph->path); 272 free(periph, M_CAMPERIPH); 273 return (CAM_REQ_INVALID); 274 } 275 periph->unit_number = camperiphunit(*p_drv, path_id, target_id, lun_id, 276 path->device->serial_num); 277 cur_periph = TAILQ_FIRST(&(*p_drv)->units); 278 while (cur_periph != NULL 279 && cur_periph->unit_number < periph->unit_number) 280 cur_periph = TAILQ_NEXT(cur_periph, unit_links); 281 if (cur_periph != NULL) { 282 KASSERT(cur_periph->unit_number != periph->unit_number, 283 ("duplicate units on periph list")); 284 TAILQ_INSERT_BEFORE(cur_periph, periph, unit_links); 285 } else { 286 TAILQ_INSERT_TAIL(&(*p_drv)->units, periph, unit_links); 287 (*p_drv)->generation++; 288 } 289 xpt_unlock_buses(); 290 291 init_level++; 292 293 status = xpt_add_periph(periph); 294 if (status != CAM_REQ_CMP) 295 goto failure; 296 297 init_level++; 298 CAM_DEBUG(periph->path, CAM_DEBUG_INFO, ("Periph created\n")); 299 300 status = periph_ctor(periph, arg); 301 302 if (status == CAM_REQ_CMP) 303 init_level++; 304 305 failure: 306 switch (init_level) { 307 case 4: 308 /* Initialized successfully */ 309 break; 310 case 3: 311 CAM_DEBUG(periph->path, CAM_DEBUG_INFO, ("Periph destroyed\n")); 312 xpt_remove_periph(periph); 313 /* FALLTHROUGH */ 314 case 2: 315 xpt_lock_buses(); 316 TAILQ_REMOVE(&(*p_drv)->units, periph, unit_links); 317 xpt_unlock_buses(); 318 xpt_free_path(periph->path); 319 /* FALLTHROUGH */ 320 case 1: 321 free(periph, M_CAMPERIPH); 322 /* FALLTHROUGH */ 323 case 0: 324 /* No cleanup to perform. */ 325 break; 326 default: 327 panic("%s: Unknown init level", __func__); 328 } 329 return(status); 330 } 331 332 /* 333 * Find a peripheral structure with the specified path, target, lun, 334 * and (optionally) type. If the name is NULL, this function will return 335 * the first peripheral driver that matches the specified path. 336 */ 337 struct cam_periph * 338 cam_periph_find(struct cam_path *path, char *name) 339 { 340 struct periph_driver **p_drv; 341 struct cam_periph *periph; 342 343 xpt_lock_buses(); 344 for (p_drv = periph_drivers; *p_drv != NULL; p_drv++) { 345 if (name != NULL && (strcmp((*p_drv)->driver_name, name) != 0)) 346 continue; 347 348 TAILQ_FOREACH(periph, &(*p_drv)->units, unit_links) { 349 if (xpt_path_comp(periph->path, path) == 0) { 350 xpt_unlock_buses(); 351 cam_periph_assert(periph, MA_OWNED); 352 return(periph); 353 } 354 } 355 if (name != NULL) { 356 xpt_unlock_buses(); 357 return(NULL); 358 } 359 } 360 xpt_unlock_buses(); 361 return(NULL); 362 } 363 364 /* 365 * Find peripheral driver instances attached to the specified path. 366 */ 367 int 368 cam_periph_list(struct cam_path *path, struct sbuf *sb) 369 { 370 struct sbuf local_sb; 371 struct periph_driver **p_drv; 372 struct cam_periph *periph; 373 int count; 374 int sbuf_alloc_len; 375 376 sbuf_alloc_len = 16; 377 retry: 378 sbuf_new(&local_sb, NULL, sbuf_alloc_len, SBUF_FIXEDLEN); 379 count = 0; 380 xpt_lock_buses(); 381 for (p_drv = periph_drivers; *p_drv != NULL; p_drv++) { 382 TAILQ_FOREACH(periph, &(*p_drv)->units, unit_links) { 383 if (xpt_path_comp(periph->path, path) != 0) 384 continue; 385 386 if (sbuf_len(&local_sb) != 0) 387 sbuf_cat(&local_sb, ","); 388 389 sbuf_printf(&local_sb, "%s%d", periph->periph_name, 390 periph->unit_number); 391 392 if (sbuf_error(&local_sb) == ENOMEM) { 393 sbuf_alloc_len *= 2; 394 xpt_unlock_buses(); 395 sbuf_delete(&local_sb); 396 goto retry; 397 } 398 count++; 399 } 400 } 401 xpt_unlock_buses(); 402 sbuf_finish(&local_sb); 403 if (sbuf_len(sb) != 0) 404 sbuf_cat(sb, ","); 405 sbuf_cat(sb, sbuf_data(&local_sb)); 406 sbuf_delete(&local_sb); 407 return (count); 408 } 409 410 int 411 cam_periph_acquire(struct cam_periph *periph) 412 { 413 int status; 414 415 if (periph == NULL) 416 return (EINVAL); 417 418 status = ENOENT; 419 xpt_lock_buses(); 420 if ((periph->flags & CAM_PERIPH_INVALID) == 0) { 421 periph->refcount++; 422 status = 0; 423 } 424 xpt_unlock_buses(); 425 426 return (status); 427 } 428 429 void 430 cam_periph_doacquire(struct cam_periph *periph) 431 { 432 433 xpt_lock_buses(); 434 KASSERT(periph->refcount >= 1, 435 ("cam_periph_doacquire() with refcount == %d", periph->refcount)); 436 periph->refcount++; 437 xpt_unlock_buses(); 438 } 439 440 void 441 cam_periph_release_locked_buses(struct cam_periph *periph) 442 { 443 444 cam_periph_assert(periph, MA_OWNED); 445 KASSERT(periph->refcount >= 1, ("periph->refcount >= 1")); 446 if (--periph->refcount == 0) 447 camperiphfree(periph); 448 } 449 450 void 451 cam_periph_release_locked(struct cam_periph *periph) 452 { 453 454 if (periph == NULL) 455 return; 456 457 xpt_lock_buses(); 458 cam_periph_release_locked_buses(periph); 459 xpt_unlock_buses(); 460 } 461 462 void 463 cam_periph_release(struct cam_periph *periph) 464 { 465 struct mtx *mtx; 466 467 if (periph == NULL) 468 return; 469 470 cam_periph_assert(periph, MA_NOTOWNED); 471 mtx = cam_periph_mtx(periph); 472 mtx_lock(mtx); 473 cam_periph_release_locked(periph); 474 mtx_unlock(mtx); 475 } 476 477 /* 478 * hold/unhold act as mutual exclusion for sections of the code that 479 * need to sleep and want to make sure that other sections that 480 * will interfere are held off. This only protects exclusive sections 481 * from each other. 482 */ 483 int 484 cam_periph_hold(struct cam_periph *periph, int priority) 485 { 486 int error; 487 488 /* 489 * Increment the reference count on the peripheral 490 * while we wait for our lock attempt to succeed 491 * to ensure the peripheral doesn't disappear out 492 * from user us while we sleep. 493 */ 494 495 if (cam_periph_acquire(periph) != 0) 496 return (ENXIO); 497 498 cam_periph_assert(periph, MA_OWNED); 499 while ((periph->flags & CAM_PERIPH_LOCKED) != 0) { 500 periph->flags |= CAM_PERIPH_LOCK_WANTED; 501 if ((error = cam_periph_sleep(periph, periph, priority, 502 "caplck", 0)) != 0) { 503 cam_periph_release_locked(periph); 504 return (error); 505 } 506 if (periph->flags & CAM_PERIPH_INVALID) { 507 cam_periph_release_locked(periph); 508 return (ENXIO); 509 } 510 } 511 512 periph->flags |= CAM_PERIPH_LOCKED; 513 return (0); 514 } 515 516 void 517 cam_periph_unhold(struct cam_periph *periph) 518 { 519 520 cam_periph_assert(periph, MA_OWNED); 521 522 periph->flags &= ~CAM_PERIPH_LOCKED; 523 if ((periph->flags & CAM_PERIPH_LOCK_WANTED) != 0) { 524 periph->flags &= ~CAM_PERIPH_LOCK_WANTED; 525 wakeup(periph); 526 } 527 528 cam_periph_release_locked(periph); 529 } 530 531 void 532 cam_periph_hold_boot(struct cam_periph *periph) 533 { 534 535 root_mount_hold_token(periph->periph_name, &periph->periph_rootmount); 536 } 537 538 void 539 cam_periph_release_boot(struct cam_periph *periph) 540 { 541 542 root_mount_rel(&periph->periph_rootmount); 543 } 544 545 /* 546 * Look for the next unit number that is not currently in use for this 547 * peripheral type starting at "newunit". Also exclude unit numbers that 548 * are reserved by for future "hardwiring" unless we already know that this 549 * is a potential wired device. Only assume that the device is "wired" the 550 * first time through the loop since after that we'll be looking at unit 551 * numbers that did not match a wiring entry. 552 */ 553 static u_int 554 camperiphnextunit(struct periph_driver *p_drv, u_int newunit, bool wired, 555 path_id_t pathid, target_id_t target, lun_id_t lun) 556 { 557 struct cam_periph *periph; 558 char *periph_name; 559 int i, val, dunit, r; 560 const char *dname, *strval; 561 562 periph_name = p_drv->driver_name; 563 for (;;newunit++) { 564 for (periph = TAILQ_FIRST(&p_drv->units); 565 periph != NULL && periph->unit_number != newunit; 566 periph = TAILQ_NEXT(periph, unit_links)) 567 ; 568 569 if (periph != NULL && periph->unit_number == newunit) { 570 if (wired) { 571 xpt_print(periph->path, "Duplicate Wired " 572 "Device entry!\n"); 573 xpt_print(periph->path, "Second device (%s " 574 "device at scbus%d target %d lun %d) will " 575 "not be wired\n", periph_name, pathid, 576 target, lun); 577 wired = false; 578 } 579 continue; 580 } 581 if (wired) 582 break; 583 584 /* 585 * Don't allow the mere presence of any attributes of a device 586 * means that it is for a wired down entry. Instead, insist that 587 * one of the matching criteria from camperiphunit be present 588 * for the device. 589 */ 590 i = 0; 591 dname = periph_name; 592 for (;;) { 593 r = resource_find_dev(&i, dname, &dunit, NULL, NULL); 594 if (r != 0) 595 break; 596 597 if (newunit != dunit) 598 continue; 599 if (resource_string_value(dname, dunit, "sn", &strval) == 0 || 600 resource_int_value(dname, dunit, "lun", &val) == 0 || 601 resource_int_value(dname, dunit, "target", &val) == 0 || 602 resource_string_value(dname, dunit, "at", &strval) == 0) 603 break; 604 } 605 if (r != 0) 606 break; 607 } 608 return (newunit); 609 } 610 611 static u_int 612 camperiphunit(struct periph_driver *p_drv, path_id_t pathid, 613 target_id_t target, lun_id_t lun, const char *sn) 614 { 615 bool wired = false; 616 u_int unit; 617 int i, val, dunit; 618 const char *dname, *strval; 619 char pathbuf[32], *periph_name; 620 621 periph_name = p_drv->driver_name; 622 snprintf(pathbuf, sizeof(pathbuf), "scbus%d", pathid); 623 unit = 0; 624 i = 0; 625 dname = periph_name; 626 627 for (wired = false; resource_find_dev(&i, dname, &dunit, NULL, NULL) == 0; 628 wired = false) { 629 if (resource_string_value(dname, dunit, "at", &strval) == 0) { 630 if (strcmp(strval, pathbuf) != 0) 631 continue; 632 wired = true; 633 } 634 if (resource_int_value(dname, dunit, "target", &val) == 0) { 635 if (val != target) 636 continue; 637 wired = true; 638 } 639 if (resource_int_value(dname, dunit, "lun", &val) == 0) { 640 if (val != lun) 641 continue; 642 wired = true; 643 } 644 if (resource_string_value(dname, dunit, "sn", &strval) == 0) { 645 if (sn == NULL || strcmp(strval, sn) != 0) 646 continue; 647 wired = true; 648 } 649 if (wired) { 650 unit = dunit; 651 break; 652 } 653 } 654 655 /* 656 * Either start from 0 looking for the next unit or from 657 * the unit number given in the resource config. This way, 658 * if we have wildcard matches, we don't return the same 659 * unit number twice. 660 */ 661 unit = camperiphnextunit(p_drv, unit, wired, pathid, target, lun); 662 663 return (unit); 664 } 665 666 void 667 cam_periph_invalidate(struct cam_periph *periph) 668 { 669 670 cam_periph_assert(periph, MA_OWNED); 671 /* 672 * We only tear down the device the first time a peripheral is 673 * invalidated. 674 */ 675 if ((periph->flags & CAM_PERIPH_INVALID) != 0) 676 return; 677 678 CAM_DEBUG(periph->path, CAM_DEBUG_INFO, ("Periph invalidated\n")); 679 if ((periph->flags & CAM_PERIPH_ANNOUNCED) && !rebooting) { 680 struct sbuf sb; 681 char buffer[160]; 682 683 sbuf_new(&sb, buffer, 160, SBUF_FIXEDLEN); 684 xpt_denounce_periph_sbuf(periph, &sb); 685 sbuf_finish(&sb); 686 sbuf_putbuf(&sb); 687 } 688 periph->flags |= CAM_PERIPH_INVALID; 689 periph->flags &= ~CAM_PERIPH_NEW_DEV_FOUND; 690 if (periph->periph_oninval != NULL) 691 periph->periph_oninval(periph); 692 cam_periph_release_locked(periph); 693 } 694 695 static void 696 camperiphfree(struct cam_periph *periph) 697 { 698 struct periph_driver **p_drv; 699 struct periph_driver *drv; 700 701 cam_periph_assert(periph, MA_OWNED); 702 KASSERT(periph->periph_allocating == 0, ("%s%d: freed while allocating", 703 periph->periph_name, periph->unit_number)); 704 for (p_drv = periph_drivers; *p_drv != NULL; p_drv++) { 705 if (strcmp((*p_drv)->driver_name, periph->periph_name) == 0) 706 break; 707 } 708 if (*p_drv == NULL) { 709 printf("camperiphfree: attempt to free non-existant periph\n"); 710 return; 711 } 712 /* 713 * Cache a pointer to the periph_driver structure. If a 714 * periph_driver is added or removed from the array (see 715 * periphdriver_register()) while we drop the toplogy lock 716 * below, p_drv may change. This doesn't protect against this 717 * particular periph_driver going away. That will require full 718 * reference counting in the periph_driver infrastructure. 719 */ 720 drv = *p_drv; 721 722 /* 723 * We need to set this flag before dropping the topology lock, to 724 * let anyone who is traversing the list that this peripheral is 725 * about to be freed, and there will be no more reference count 726 * checks. 727 */ 728 periph->flags |= CAM_PERIPH_FREE; 729 730 /* 731 * The peripheral destructor semantics dictate calling with only the 732 * SIM mutex held. Since it might sleep, it should not be called 733 * with the topology lock held. 734 */ 735 xpt_unlock_buses(); 736 737 /* 738 * We need to call the peripheral destructor prior to removing the 739 * peripheral from the list. Otherwise, we risk running into a 740 * scenario where the peripheral unit number may get reused 741 * (because it has been removed from the list), but some resources 742 * used by the peripheral are still hanging around. In particular, 743 * the devfs nodes used by some peripherals like the pass(4) driver 744 * aren't fully cleaned up until the destructor is run. If the 745 * unit number is reused before the devfs instance is fully gone, 746 * devfs will panic. 747 */ 748 if (periph->periph_dtor != NULL) 749 periph->periph_dtor(periph); 750 751 /* 752 * The peripheral list is protected by the topology lock. We have to 753 * remove the periph from the drv list before we call deferred_ac. The 754 * AC_FOUND_DEVICE callback won't create a new periph if it's still there. 755 */ 756 xpt_lock_buses(); 757 758 TAILQ_REMOVE(&drv->units, periph, unit_links); 759 drv->generation++; 760 761 xpt_remove_periph(periph); 762 763 xpt_unlock_buses(); 764 if ((periph->flags & CAM_PERIPH_ANNOUNCED) && !rebooting) 765 xpt_print(periph->path, "Periph destroyed\n"); 766 else 767 CAM_DEBUG(periph->path, CAM_DEBUG_INFO, ("Periph destroyed\n")); 768 769 if (periph->flags & CAM_PERIPH_NEW_DEV_FOUND) { 770 union ccb ccb; 771 void *arg; 772 773 memset(&ccb, 0, sizeof(ccb)); 774 switch (periph->deferred_ac) { 775 case AC_FOUND_DEVICE: 776 ccb.ccb_h.func_code = XPT_GDEV_TYPE; 777 xpt_setup_ccb(&ccb.ccb_h, periph->path, CAM_PRIORITY_NORMAL); 778 xpt_action(&ccb); 779 arg = &ccb; 780 break; 781 case AC_PATH_REGISTERED: 782 xpt_path_inq(&ccb.cpi, periph->path); 783 arg = &ccb; 784 break; 785 default: 786 arg = NULL; 787 break; 788 } 789 periph->deferred_callback(NULL, periph->deferred_ac, 790 periph->path, arg); 791 } 792 xpt_free_path(periph->path); 793 free(periph, M_CAMPERIPH); 794 xpt_lock_buses(); 795 } 796 797 /* 798 * Map user virtual pointers into kernel virtual address space, so we can 799 * access the memory. This is now a generic function that centralizes most 800 * of the sanity checks on the data flags, if any. 801 * This also only works for up to maxphys memory. Since we use 802 * buffers to map stuff in and out, we're limited to the buffer size. 803 */ 804 int 805 cam_periph_mapmem(union ccb *ccb, struct cam_periph_map_info *mapinfo, 806 u_int maxmap) 807 { 808 int numbufs, i; 809 uint8_t **data_ptrs[CAM_PERIPH_MAXMAPS]; 810 uint32_t lengths[CAM_PERIPH_MAXMAPS]; 811 uint32_t dirs[CAM_PERIPH_MAXMAPS]; 812 813 bzero(mapinfo, sizeof(*mapinfo)); 814 if (maxmap == 0) 815 maxmap = DFLTPHYS; /* traditional default */ 816 else if (maxmap > maxphys) 817 maxmap = maxphys; /* for safety */ 818 switch(ccb->ccb_h.func_code) { 819 case XPT_DEV_MATCH: 820 if (ccb->cdm.match_buf_len == 0) { 821 printf("cam_periph_mapmem: invalid match buffer " 822 "length 0\n"); 823 return(EINVAL); 824 } 825 if (ccb->cdm.pattern_buf_len > 0) { 826 data_ptrs[0] = (uint8_t **)&ccb->cdm.patterns; 827 lengths[0] = ccb->cdm.pattern_buf_len; 828 dirs[0] = CAM_DIR_OUT; 829 data_ptrs[1] = (uint8_t **)&ccb->cdm.matches; 830 lengths[1] = ccb->cdm.match_buf_len; 831 dirs[1] = CAM_DIR_IN; 832 numbufs = 2; 833 } else { 834 data_ptrs[0] = (uint8_t **)&ccb->cdm.matches; 835 lengths[0] = ccb->cdm.match_buf_len; 836 dirs[0] = CAM_DIR_IN; 837 numbufs = 1; 838 } 839 /* 840 * This request will not go to the hardware, no reason 841 * to be so strict. vmapbuf() is able to map up to maxphys. 842 */ 843 maxmap = maxphys; 844 break; 845 case XPT_SCSI_IO: 846 case XPT_CONT_TARGET_IO: 847 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_NONE) 848 return(0); 849 if ((ccb->ccb_h.flags & CAM_DATA_MASK) != CAM_DATA_VADDR) 850 return (EINVAL); 851 data_ptrs[0] = &ccb->csio.data_ptr; 852 lengths[0] = ccb->csio.dxfer_len; 853 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 854 numbufs = 1; 855 break; 856 case XPT_ATA_IO: 857 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_NONE) 858 return(0); 859 if ((ccb->ccb_h.flags & CAM_DATA_MASK) != CAM_DATA_VADDR) 860 return (EINVAL); 861 data_ptrs[0] = &ccb->ataio.data_ptr; 862 lengths[0] = ccb->ataio.dxfer_len; 863 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 864 numbufs = 1; 865 break; 866 case XPT_MMC_IO: 867 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_NONE) 868 return(0); 869 /* Two mappings: one for cmd->data and one for cmd->data->data */ 870 data_ptrs[0] = (unsigned char **)&ccb->mmcio.cmd.data; 871 lengths[0] = sizeof(struct mmc_data *); 872 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 873 data_ptrs[1] = (unsigned char **)&ccb->mmcio.cmd.data->data; 874 lengths[1] = ccb->mmcio.cmd.data->len; 875 dirs[1] = ccb->ccb_h.flags & CAM_DIR_MASK; 876 numbufs = 2; 877 break; 878 case XPT_SMP_IO: 879 data_ptrs[0] = &ccb->smpio.smp_request; 880 lengths[0] = ccb->smpio.smp_request_len; 881 dirs[0] = CAM_DIR_OUT; 882 data_ptrs[1] = &ccb->smpio.smp_response; 883 lengths[1] = ccb->smpio.smp_response_len; 884 dirs[1] = CAM_DIR_IN; 885 numbufs = 2; 886 break; 887 case XPT_NVME_IO: 888 case XPT_NVME_ADMIN: 889 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_NONE) 890 return (0); 891 if ((ccb->ccb_h.flags & CAM_DATA_MASK) != CAM_DATA_VADDR) 892 return (EINVAL); 893 data_ptrs[0] = &ccb->nvmeio.data_ptr; 894 lengths[0] = ccb->nvmeio.dxfer_len; 895 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 896 numbufs = 1; 897 break; 898 case XPT_DEV_ADVINFO: 899 if (ccb->cdai.bufsiz == 0) 900 return (0); 901 902 data_ptrs[0] = (uint8_t **)&ccb->cdai.buf; 903 lengths[0] = ccb->cdai.bufsiz; 904 dirs[0] = CAM_DIR_IN; 905 numbufs = 1; 906 907 /* 908 * This request will not go to the hardware, no reason 909 * to be so strict. vmapbuf() is able to map up to maxphys. 910 */ 911 maxmap = maxphys; 912 break; 913 default: 914 return(EINVAL); 915 break; /* NOTREACHED */ 916 } 917 918 /* 919 * Check the transfer length and permissions first, so we don't 920 * have to unmap any previously mapped buffers. 921 */ 922 for (i = 0; i < numbufs; i++) { 923 if (lengths[i] > maxmap) { 924 printf("cam_periph_mapmem: attempt to map %lu bytes, " 925 "which is greater than %lu\n", 926 (long)(lengths[i]), (u_long)maxmap); 927 return (E2BIG); 928 } 929 } 930 931 /* 932 * This keeps the kernel stack of current thread from getting 933 * swapped. In low-memory situations where the kernel stack might 934 * otherwise get swapped out, this holds it and allows the thread 935 * to make progress and release the kernel mapped pages sooner. 936 * 937 * XXX KDM should I use P_NOSWAP instead? 938 */ 939 PHOLD(curproc); 940 941 for (i = 0; i < numbufs; i++) { 942 /* Save the user's data address. */ 943 mapinfo->orig[i] = *data_ptrs[i]; 944 945 /* 946 * For small buffers use malloc+copyin/copyout instead of 947 * mapping to KVA to avoid expensive TLB shootdowns. For 948 * small allocations malloc is backed by UMA, and so much 949 * cheaper on SMP systems. 950 */ 951 if (lengths[i] <= periph_mapmem_thresh && 952 ccb->ccb_h.func_code != XPT_MMC_IO) { 953 *data_ptrs[i] = malloc(lengths[i], M_CAMPERIPH, 954 M_WAITOK); 955 if (dirs[i] != CAM_DIR_IN) { 956 if (copyin(mapinfo->orig[i], *data_ptrs[i], 957 lengths[i]) != 0) { 958 free(*data_ptrs[i], M_CAMPERIPH); 959 *data_ptrs[i] = mapinfo->orig[i]; 960 goto fail; 961 } 962 } else 963 bzero(*data_ptrs[i], lengths[i]); 964 continue; 965 } 966 967 /* 968 * Get the buffer. 969 */ 970 mapinfo->bp[i] = uma_zalloc(pbuf_zone, M_WAITOK); 971 972 /* set the direction */ 973 mapinfo->bp[i]->b_iocmd = (dirs[i] == CAM_DIR_OUT) ? 974 BIO_WRITE : BIO_READ; 975 976 /* Map the buffer into kernel memory. */ 977 if (vmapbuf(mapinfo->bp[i], *data_ptrs[i], lengths[i], 1) < 0) { 978 uma_zfree(pbuf_zone, mapinfo->bp[i]); 979 goto fail; 980 } 981 982 /* set our pointer to the new mapped area */ 983 *data_ptrs[i] = mapinfo->bp[i]->b_data; 984 } 985 986 /* 987 * Now that we've gotten this far, change ownership to the kernel 988 * of the buffers so that we don't run afoul of returning to user 989 * space with locks (on the buffer) held. 990 */ 991 for (i = 0; i < numbufs; i++) { 992 if (mapinfo->bp[i]) 993 BUF_KERNPROC(mapinfo->bp[i]); 994 } 995 996 mapinfo->num_bufs_used = numbufs; 997 return(0); 998 999 fail: 1000 for (i--; i >= 0; i--) { 1001 if (mapinfo->bp[i]) { 1002 vunmapbuf(mapinfo->bp[i]); 1003 uma_zfree(pbuf_zone, mapinfo->bp[i]); 1004 } else 1005 free(*data_ptrs[i], M_CAMPERIPH); 1006 *data_ptrs[i] = mapinfo->orig[i]; 1007 } 1008 PRELE(curproc); 1009 return(EACCES); 1010 } 1011 1012 /* 1013 * Unmap memory segments mapped into kernel virtual address space by 1014 * cam_periph_mapmem(). 1015 */ 1016 int 1017 cam_periph_unmapmem(union ccb *ccb, struct cam_periph_map_info *mapinfo) 1018 { 1019 int error, numbufs, i; 1020 uint8_t **data_ptrs[CAM_PERIPH_MAXMAPS]; 1021 uint32_t lengths[CAM_PERIPH_MAXMAPS]; 1022 uint32_t dirs[CAM_PERIPH_MAXMAPS]; 1023 1024 if (mapinfo->num_bufs_used <= 0) { 1025 /* nothing to free and the process wasn't held. */ 1026 return (0); 1027 } 1028 1029 switch (ccb->ccb_h.func_code) { 1030 case XPT_DEV_MATCH: 1031 if (ccb->cdm.pattern_buf_len > 0) { 1032 data_ptrs[0] = (uint8_t **)&ccb->cdm.patterns; 1033 lengths[0] = ccb->cdm.pattern_buf_len; 1034 dirs[0] = CAM_DIR_OUT; 1035 data_ptrs[1] = (uint8_t **)&ccb->cdm.matches; 1036 lengths[1] = ccb->cdm.match_buf_len; 1037 dirs[1] = CAM_DIR_IN; 1038 numbufs = 2; 1039 } else { 1040 data_ptrs[0] = (uint8_t **)&ccb->cdm.matches; 1041 lengths[0] = ccb->cdm.match_buf_len; 1042 dirs[0] = CAM_DIR_IN; 1043 numbufs = 1; 1044 } 1045 break; 1046 case XPT_SCSI_IO: 1047 case XPT_CONT_TARGET_IO: 1048 data_ptrs[0] = &ccb->csio.data_ptr; 1049 lengths[0] = ccb->csio.dxfer_len; 1050 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 1051 numbufs = 1; 1052 break; 1053 case XPT_ATA_IO: 1054 data_ptrs[0] = &ccb->ataio.data_ptr; 1055 lengths[0] = ccb->ataio.dxfer_len; 1056 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 1057 numbufs = 1; 1058 break; 1059 case XPT_MMC_IO: 1060 data_ptrs[0] = (uint8_t **)&ccb->mmcio.cmd.data; 1061 lengths[0] = sizeof(struct mmc_data *); 1062 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 1063 data_ptrs[1] = (uint8_t **)&ccb->mmcio.cmd.data->data; 1064 lengths[1] = ccb->mmcio.cmd.data->len; 1065 dirs[1] = ccb->ccb_h.flags & CAM_DIR_MASK; 1066 numbufs = 2; 1067 break; 1068 case XPT_SMP_IO: 1069 data_ptrs[0] = &ccb->smpio.smp_request; 1070 lengths[0] = ccb->smpio.smp_request_len; 1071 dirs[0] = CAM_DIR_OUT; 1072 data_ptrs[1] = &ccb->smpio.smp_response; 1073 lengths[1] = ccb->smpio.smp_response_len; 1074 dirs[1] = CAM_DIR_IN; 1075 numbufs = 2; 1076 break; 1077 case XPT_NVME_IO: 1078 case XPT_NVME_ADMIN: 1079 data_ptrs[0] = &ccb->nvmeio.data_ptr; 1080 lengths[0] = ccb->nvmeio.dxfer_len; 1081 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 1082 numbufs = 1; 1083 break; 1084 case XPT_DEV_ADVINFO: 1085 data_ptrs[0] = (uint8_t **)&ccb->cdai.buf; 1086 lengths[0] = ccb->cdai.bufsiz; 1087 dirs[0] = CAM_DIR_IN; 1088 numbufs = 1; 1089 break; 1090 default: 1091 numbufs = 0; 1092 break; 1093 } 1094 1095 error = 0; 1096 for (i = 0; i < numbufs; i++) { 1097 if (mapinfo->bp[i]) { 1098 /* unmap the buffer */ 1099 vunmapbuf(mapinfo->bp[i]); 1100 1101 /* release the buffer */ 1102 uma_zfree(pbuf_zone, mapinfo->bp[i]); 1103 } else { 1104 if (dirs[i] != CAM_DIR_OUT) { 1105 int error1; 1106 1107 error1 = copyout(*data_ptrs[i], mapinfo->orig[i], 1108 lengths[i]); 1109 if (error == 0) 1110 error = error1; 1111 } 1112 free(*data_ptrs[i], M_CAMPERIPH); 1113 } 1114 1115 /* Set the user's pointer back to the original value */ 1116 *data_ptrs[i] = mapinfo->orig[i]; 1117 } 1118 1119 /* allow ourselves to be swapped once again */ 1120 PRELE(curproc); 1121 1122 return (error); 1123 } 1124 1125 static int 1126 cam_periph_ioctl_compat(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td) 1127 { 1128 struct cam_periph *periph; 1129 1130 /* 1131 * For compat, we need to cast struct periph * into struct cdev *dev and 1132 * then back again. 1133 */ 1134 periph = (struct cam_periph *)(void *)dev; 1135 cam_periph_assert(periph, MA_OWNED); 1136 return (cam_periph_ioctl(periph, cmd, addr, cderror)); 1137 } 1138 1139 int 1140 cam_periph_ioctl(struct cam_periph *periph, u_long cmd, caddr_t addr, 1141 int (*error_routine)(union ccb *ccb, 1142 cam_flags camflags, 1143 uint32_t sense_flags)) 1144 { 1145 union ccb *ccb; 1146 int error; 1147 int found; 1148 1149 error = found = 0; 1150 1151 switch(cmd){ 1152 case CAMGETPASSTHRU_0x19: 1153 case CAMGETPASSTHRU: 1154 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL); 1155 xpt_setup_ccb(&ccb->ccb_h, 1156 ccb->ccb_h.path, 1157 CAM_PRIORITY_NORMAL); 1158 ccb->ccb_h.func_code = XPT_GDEVLIST; 1159 1160 /* 1161 * Basically, the point of this is that we go through 1162 * getting the list of devices, until we find a passthrough 1163 * device. In the current version of the CAM code, the 1164 * only way to determine what type of device we're dealing 1165 * with is by its name. 1166 */ 1167 while (found == 0) { 1168 ccb->cgdl.index = 0; 1169 ccb->cgdl.status = CAM_GDEVLIST_MORE_DEVS; 1170 while (ccb->cgdl.status == CAM_GDEVLIST_MORE_DEVS) { 1171 /* we want the next device in the list */ 1172 xpt_action(ccb); 1173 if (strncmp(ccb->cgdl.periph_name, 1174 "pass", 4) == 0){ 1175 found = 1; 1176 break; 1177 } 1178 } 1179 if ((ccb->cgdl.status == CAM_GDEVLIST_LAST_DEVICE) && 1180 (found == 0)) { 1181 ccb->cgdl.periph_name[0] = '\0'; 1182 ccb->cgdl.unit_number = 0; 1183 break; 1184 } 1185 } 1186 1187 /* copy the result back out */ 1188 bcopy(ccb, addr, sizeof(union ccb)); 1189 1190 /* and release the ccb */ 1191 xpt_release_ccb(ccb); 1192 1193 break; 1194 default: 1195 /* 1196 * We assume that the compat layer doesn't care about 1197 * the dev parameter. It just passes it through, so 1198 * cheat a little. 1199 */ 1200 error = cam_compat_ioctl((struct cdev *)(void *)periph, 1201 cmd, addr, flag, td, cam_periph_ioctl_compat); 1202 break; 1203 } 1204 return(error); 1205 } 1206 1207 static void 1208 cam_periph_done_panic(struct cam_periph *periph, union ccb *done_ccb) 1209 { 1210 1211 panic("%s: already done with ccb %p", __func__, done_ccb); 1212 } 1213 1214 static void 1215 cam_periph_done(struct cam_periph *periph, union ccb *done_ccb) 1216 { 1217 1218 /* Caller will release the CCB */ 1219 xpt_path_assert(done_ccb->ccb_h.path, MA_OWNED); 1220 done_ccb->ccb_h.cbfcnp = cam_periph_done_panic; 1221 wakeup(&done_ccb->ccb_h.cbfcnp); 1222 } 1223 1224 static void 1225 cam_periph_ccbwait(union ccb *ccb) 1226 { 1227 1228 if ((ccb->ccb_h.func_code & XPT_FC_QUEUED) != 0) { 1229 while (ccb->ccb_h.cbfcnp != cam_periph_done_panic) 1230 xpt_path_sleep(ccb->ccb_h.path, &ccb->ccb_h.cbfcnp, 1231 PRIBIO, "cbwait", 0); 1232 } 1233 KASSERT(ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX && 1234 (ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_INPROG, 1235 ("%s: proceeding with incomplete ccb: ccb=%p, func_code=%#x, " 1236 "status=%#x, index=%d", __func__, ccb, ccb->ccb_h.func_code, 1237 ccb->ccb_h.status, ccb->ccb_h.pinfo.index)); 1238 } 1239 1240 /* 1241 * Dispatch a CCB and wait for it to complete. If the CCB has set a 1242 * callback function (ccb->ccb_h.cbfcnp), it will be overwritten and lost. 1243 */ 1244 int 1245 cam_periph_runccb(union ccb *ccb, 1246 int (*error_routine)(union ccb *ccb, 1247 cam_flags camflags, 1248 uint32_t sense_flags), 1249 cam_flags camflags, uint32_t sense_flags, 1250 struct devstat *ds) 1251 { 1252 struct bintime *starttime; 1253 struct bintime ltime; 1254 int error; 1255 bool must_poll; 1256 uint32_t timeout = 1; 1257 1258 starttime = NULL; 1259 xpt_path_assert(ccb->ccb_h.path, MA_OWNED); 1260 KASSERT((ccb->ccb_h.flags & CAM_UNLOCKED) == 0, 1261 ("%s: ccb=%p, func_code=%#x, flags=%#x", __func__, ccb, 1262 ccb->ccb_h.func_code, ccb->ccb_h.flags)); 1263 1264 /* 1265 * If the user has supplied a stats structure, and if we understand 1266 * this particular type of ccb, record the transaction start. 1267 */ 1268 if (ds != NULL && 1269 (ccb->ccb_h.func_code == XPT_SCSI_IO || 1270 ccb->ccb_h.func_code == XPT_ATA_IO || 1271 ccb->ccb_h.func_code == XPT_NVME_IO)) { 1272 starttime = <ime; 1273 binuptime(starttime); 1274 devstat_start_transaction(ds, starttime); 1275 } 1276 1277 /* 1278 * We must poll the I/O while we're dumping. The scheduler is normally 1279 * stopped for dumping, except when we call doadump from ddb. While the 1280 * scheduler is running in this case, we still need to poll the I/O to 1281 * avoid sleeping waiting for the ccb to complete. 1282 * 1283 * A panic triggered dump stops the scheduler, any callback from the 1284 * shutdown_post_sync event will run with the scheduler stopped, but 1285 * before we're officially dumping. To avoid hanging in adashutdown 1286 * initiated commands (or other similar situations), we have to test for 1287 * either dumping or SCHEDULER_STOPPED() here. 1288 * 1289 * To avoid locking problems, dumping/polling callers must call 1290 * without a periph lock held. 1291 */ 1292 must_poll = dumping || SCHEDULER_STOPPED(); 1293 ccb->ccb_h.cbfcnp = cam_periph_done; 1294 1295 /* 1296 * If we're polling, then we need to ensure that we have ample resources 1297 * in the periph. cam_periph_error can reschedule the ccb by calling 1298 * xpt_action and returning ERESTART, so we have to effect the polling 1299 * in the do loop below. 1300 */ 1301 if (must_poll) { 1302 if (cam_sim_pollable(ccb->ccb_h.path->bus->sim)) 1303 timeout = xpt_poll_setup(ccb); 1304 else 1305 timeout = 0; 1306 } 1307 1308 if (timeout == 0) { 1309 ccb->ccb_h.status = CAM_RESRC_UNAVAIL; 1310 error = EBUSY; 1311 } else { 1312 xpt_action(ccb); 1313 do { 1314 if (must_poll) { 1315 xpt_pollwait(ccb, timeout); 1316 timeout = ccb->ccb_h.timeout * 10; 1317 } else { 1318 cam_periph_ccbwait(ccb); 1319 } 1320 if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) 1321 error = 0; 1322 else if (error_routine != NULL) { 1323 /* 1324 * cbfcnp is modified by cam_periph_ccbwait so 1325 * reset it before we call the error routine 1326 * which may call xpt_done. 1327 */ 1328 ccb->ccb_h.cbfcnp = cam_periph_done; 1329 error = (*error_routine)(ccb, camflags, sense_flags); 1330 } else 1331 error = 0; 1332 } while (error == ERESTART); 1333 } 1334 1335 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { 1336 cam_release_devq(ccb->ccb_h.path, 1337 /* relsim_flags */0, 1338 /* openings */0, 1339 /* timeout */0, 1340 /* getcount_only */ FALSE); 1341 ccb->ccb_h.status &= ~CAM_DEV_QFRZN; 1342 } 1343 1344 if (ds != NULL) { 1345 uint32_t bytes; 1346 devstat_tag_type tag; 1347 bool valid = true; 1348 1349 if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 1350 bytes = ccb->csio.dxfer_len - ccb->csio.resid; 1351 tag = (devstat_tag_type)(ccb->csio.tag_action & 0x3); 1352 } else if (ccb->ccb_h.func_code == XPT_ATA_IO) { 1353 bytes = ccb->ataio.dxfer_len - ccb->ataio.resid; 1354 tag = (devstat_tag_type)0; 1355 } else if (ccb->ccb_h.func_code == XPT_NVME_IO) { 1356 bytes = ccb->nvmeio.dxfer_len; /* NB: resid no possible */ 1357 tag = (devstat_tag_type)0; 1358 } else { 1359 valid = false; 1360 } 1361 if (valid) 1362 devstat_end_transaction(ds, bytes, tag, 1363 ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_NONE) ? 1364 DEVSTAT_NO_DATA : (ccb->ccb_h.flags & CAM_DIR_OUT) ? 1365 DEVSTAT_WRITE : DEVSTAT_READ, NULL, starttime); 1366 } 1367 1368 return(error); 1369 } 1370 1371 void 1372 cam_freeze_devq(struct cam_path *path) 1373 { 1374 struct ccb_hdr ccb_h; 1375 1376 CAM_DEBUG(path, CAM_DEBUG_TRACE, ("cam_freeze_devq\n")); 1377 memset(&ccb_h, 0, sizeof(ccb_h)); 1378 xpt_setup_ccb(&ccb_h, path, /*priority*/1); 1379 ccb_h.func_code = XPT_NOOP; 1380 ccb_h.flags = CAM_DEV_QFREEZE; 1381 xpt_action((union ccb *)&ccb_h); 1382 } 1383 1384 uint32_t 1385 cam_release_devq(struct cam_path *path, uint32_t relsim_flags, 1386 uint32_t openings, uint32_t arg, 1387 int getcount_only) 1388 { 1389 struct ccb_relsim crs; 1390 1391 CAM_DEBUG(path, CAM_DEBUG_TRACE, ("cam_release_devq(%u, %u, %u, %d)\n", 1392 relsim_flags, openings, arg, getcount_only)); 1393 memset(&crs, 0, sizeof(crs)); 1394 xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL); 1395 crs.ccb_h.func_code = XPT_REL_SIMQ; 1396 crs.ccb_h.flags = getcount_only ? CAM_DEV_QFREEZE : 0; 1397 crs.release_flags = relsim_flags; 1398 crs.openings = openings; 1399 crs.release_timeout = arg; 1400 xpt_action((union ccb *)&crs); 1401 return (crs.qfrozen_cnt); 1402 } 1403 1404 #define saved_ccb_ptr ppriv_ptr0 1405 static void 1406 camperiphdone(struct cam_periph *periph, union ccb *done_ccb) 1407 { 1408 union ccb *saved_ccb; 1409 cam_status status; 1410 struct scsi_start_stop_unit *scsi_cmd; 1411 int error = 0, error_code, sense_key, asc, ascq; 1412 uint16_t done_flags; 1413 1414 scsi_cmd = (struct scsi_start_stop_unit *) 1415 &done_ccb->csio.cdb_io.cdb_bytes; 1416 status = done_ccb->ccb_h.status; 1417 1418 if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) { 1419 if (scsi_extract_sense_ccb(done_ccb, 1420 &error_code, &sense_key, &asc, &ascq)) { 1421 /* 1422 * If the error is "invalid field in CDB", 1423 * and the load/eject flag is set, turn the 1424 * flag off and try again. This is just in 1425 * case the drive in question barfs on the 1426 * load eject flag. The CAM code should set 1427 * the load/eject flag by default for 1428 * removable media. 1429 */ 1430 if ((scsi_cmd->opcode == START_STOP_UNIT) && 1431 ((scsi_cmd->how & SSS_LOEJ) != 0) && 1432 (asc == 0x24) && (ascq == 0x00)) { 1433 scsi_cmd->how &= ~SSS_LOEJ; 1434 if (status & CAM_DEV_QFRZN) { 1435 cam_release_devq(done_ccb->ccb_h.path, 1436 0, 0, 0, 0); 1437 done_ccb->ccb_h.status &= 1438 ~CAM_DEV_QFRZN; 1439 } 1440 xpt_action(done_ccb); 1441 goto out; 1442 } 1443 } 1444 error = cam_periph_error(done_ccb, 0, 1445 SF_RETRY_UA | SF_NO_PRINT); 1446 if (error == ERESTART) 1447 goto out; 1448 if (done_ccb->ccb_h.status & CAM_DEV_QFRZN) { 1449 cam_release_devq(done_ccb->ccb_h.path, 0, 0, 0, 0); 1450 done_ccb->ccb_h.status &= ~CAM_DEV_QFRZN; 1451 } 1452 } else { 1453 /* 1454 * If we have successfully taken a device from the not 1455 * ready to ready state, re-scan the device and re-get 1456 * the inquiry information. Many devices (mostly disks) 1457 * don't properly report their inquiry information unless 1458 * they are spun up. 1459 */ 1460 if (scsi_cmd->opcode == START_STOP_UNIT) 1461 xpt_async(AC_INQ_CHANGED, done_ccb->ccb_h.path, NULL); 1462 } 1463 1464 /* If we tried long wait and still failed, remember that. */ 1465 if ((periph->flags & CAM_PERIPH_RECOVERY_WAIT) && 1466 (done_ccb->csio.cdb_io.cdb_bytes[0] == TEST_UNIT_READY)) { 1467 periph->flags &= ~CAM_PERIPH_RECOVERY_WAIT; 1468 if (error != 0 && done_ccb->ccb_h.retry_count == 0) 1469 periph->flags |= CAM_PERIPH_RECOVERY_WAIT_FAILED; 1470 } 1471 1472 /* 1473 * After recovery action(s) completed, return to the original CCB. 1474 * If the recovery CCB has failed, considering its own possible 1475 * retries and recovery, assume we are back in state where we have 1476 * been originally, but without recovery hopes left. In such case, 1477 * after the final attempt below, we cancel any further retries, 1478 * blocking by that also any new recovery attempts for this CCB, 1479 * and the result will be the final one returned to the CCB owher. 1480 */ 1481 saved_ccb = (union ccb *)done_ccb->ccb_h.saved_ccb_ptr; 1482 KASSERT(saved_ccb->ccb_h.func_code == XPT_SCSI_IO, 1483 ("%s: saved_ccb func_code %#x != XPT_SCSI_IO", 1484 __func__, saved_ccb->ccb_h.func_code)); 1485 KASSERT(done_ccb->ccb_h.func_code == XPT_SCSI_IO, 1486 ("%s: done_ccb func_code %#x != XPT_SCSI_IO", 1487 __func__, done_ccb->ccb_h.func_code)); 1488 saved_ccb->ccb_h.periph_links = done_ccb->ccb_h.periph_links; 1489 done_flags = done_ccb->ccb_h.alloc_flags; 1490 bcopy(saved_ccb, done_ccb, sizeof(struct ccb_scsiio)); 1491 done_ccb->ccb_h.alloc_flags = done_flags; 1492 xpt_free_ccb(saved_ccb); 1493 if (done_ccb->ccb_h.cbfcnp != camperiphdone) 1494 periph->flags &= ~CAM_PERIPH_RECOVERY_INPROG; 1495 if (error != 0) 1496 done_ccb->ccb_h.retry_count = 0; 1497 xpt_action(done_ccb); 1498 1499 out: 1500 /* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */ 1501 cam_release_devq(done_ccb->ccb_h.path, 0, 0, 0, 0); 1502 } 1503 1504 /* 1505 * Generic Async Event handler. Peripheral drivers usually 1506 * filter out the events that require personal attention, 1507 * and leave the rest to this function. 1508 */ 1509 void 1510 cam_periph_async(struct cam_periph *periph, uint32_t code, 1511 struct cam_path *path, void *arg) 1512 { 1513 switch (code) { 1514 case AC_LOST_DEVICE: 1515 cam_periph_invalidate(periph); 1516 break; 1517 default: 1518 break; 1519 } 1520 } 1521 1522 void 1523 cam_periph_bus_settle(struct cam_periph *periph, u_int bus_settle) 1524 { 1525 struct ccb_getdevstats cgds; 1526 1527 memset(&cgds, 0, sizeof(cgds)); 1528 xpt_setup_ccb(&cgds.ccb_h, periph->path, CAM_PRIORITY_NORMAL); 1529 cgds.ccb_h.func_code = XPT_GDEV_STATS; 1530 xpt_action((union ccb *)&cgds); 1531 cam_periph_freeze_after_event(periph, &cgds.last_reset, bus_settle); 1532 } 1533 1534 void 1535 cam_periph_freeze_after_event(struct cam_periph *periph, 1536 struct timeval* event_time, u_int duration_ms) 1537 { 1538 struct timeval delta; 1539 struct timeval duration_tv; 1540 1541 if (!timevalisset(event_time)) 1542 return; 1543 1544 microtime(&delta); 1545 timevalsub(&delta, event_time); 1546 duration_tv.tv_sec = duration_ms / 1000; 1547 duration_tv.tv_usec = (duration_ms % 1000) * 1000; 1548 if (timevalcmp(&delta, &duration_tv, <)) { 1549 timevalsub(&duration_tv, &delta); 1550 1551 duration_ms = duration_tv.tv_sec * 1000; 1552 duration_ms += duration_tv.tv_usec / 1000; 1553 cam_freeze_devq(periph->path); 1554 cam_release_devq(periph->path, 1555 RELSIM_RELEASE_AFTER_TIMEOUT, 1556 /*reduction*/0, 1557 /*timeout*/duration_ms, 1558 /*getcount_only*/0); 1559 } 1560 1561 } 1562 1563 static int 1564 camperiphscsistatuserror(union ccb *ccb, union ccb **orig_ccb, 1565 cam_flags camflags, uint32_t sense_flags, 1566 int *openings, uint32_t *relsim_flags, 1567 uint32_t *timeout, uint32_t *action, const char **action_string) 1568 { 1569 struct cam_periph *periph; 1570 int error; 1571 1572 switch (ccb->csio.scsi_status) { 1573 case SCSI_STATUS_OK: 1574 case SCSI_STATUS_COND_MET: 1575 case SCSI_STATUS_INTERMED: 1576 case SCSI_STATUS_INTERMED_COND_MET: 1577 error = 0; 1578 break; 1579 case SCSI_STATUS_CMD_TERMINATED: 1580 case SCSI_STATUS_CHECK_COND: 1581 error = camperiphscsisenseerror(ccb, orig_ccb, 1582 camflags, 1583 sense_flags, 1584 openings, 1585 relsim_flags, 1586 timeout, 1587 action, 1588 action_string); 1589 break; 1590 case SCSI_STATUS_QUEUE_FULL: 1591 { 1592 /* no decrement */ 1593 struct ccb_getdevstats cgds; 1594 1595 /* 1596 * First off, find out what the current 1597 * transaction counts are. 1598 */ 1599 memset(&cgds, 0, sizeof(cgds)); 1600 xpt_setup_ccb(&cgds.ccb_h, 1601 ccb->ccb_h.path, 1602 CAM_PRIORITY_NORMAL); 1603 cgds.ccb_h.func_code = XPT_GDEV_STATS; 1604 xpt_action((union ccb *)&cgds); 1605 1606 /* 1607 * If we were the only transaction active, treat 1608 * the QUEUE FULL as if it were a BUSY condition. 1609 */ 1610 if (cgds.dev_active != 0) { 1611 int total_openings; 1612 1613 /* 1614 * Reduce the number of openings to 1615 * be 1 less than the amount it took 1616 * to get a queue full bounded by the 1617 * minimum allowed tag count for this 1618 * device. 1619 */ 1620 total_openings = cgds.dev_active + cgds.dev_openings; 1621 *openings = cgds.dev_active; 1622 if (*openings < cgds.mintags) 1623 *openings = cgds.mintags; 1624 if (*openings < total_openings) 1625 *relsim_flags = RELSIM_ADJUST_OPENINGS; 1626 else { 1627 /* 1628 * Some devices report queue full for 1629 * temporary resource shortages. For 1630 * this reason, we allow a minimum 1631 * tag count to be entered via a 1632 * quirk entry to prevent the queue 1633 * count on these devices from falling 1634 * to a pessimisticly low value. We 1635 * still wait for the next successful 1636 * completion, however, before queueing 1637 * more transactions to the device. 1638 */ 1639 *relsim_flags = RELSIM_RELEASE_AFTER_CMDCMPLT; 1640 } 1641 *timeout = 0; 1642 error = ERESTART; 1643 *action &= ~SSQ_PRINT_SENSE; 1644 break; 1645 } 1646 /* FALLTHROUGH */ 1647 } 1648 case SCSI_STATUS_BUSY: 1649 /* 1650 * Restart the queue after either another 1651 * command completes or a 1 second timeout. 1652 */ 1653 periph = xpt_path_periph(ccb->ccb_h.path); 1654 if (periph->flags & CAM_PERIPH_INVALID) { 1655 error = ENXIO; 1656 *action_string = "Periph was invalidated"; 1657 } else if ((sense_flags & SF_RETRY_BUSY) != 0 || 1658 ccb->ccb_h.retry_count > 0) { 1659 if ((sense_flags & SF_RETRY_BUSY) == 0) 1660 ccb->ccb_h.retry_count--; 1661 error = ERESTART; 1662 *relsim_flags = RELSIM_RELEASE_AFTER_TIMEOUT 1663 | RELSIM_RELEASE_AFTER_CMDCMPLT; 1664 *timeout = 1000; 1665 } else { 1666 error = EIO; 1667 *action_string = "Retries exhausted"; 1668 } 1669 break; 1670 case SCSI_STATUS_RESERV_CONFLICT: 1671 default: 1672 error = EIO; 1673 break; 1674 } 1675 return (error); 1676 } 1677 1678 static int 1679 camperiphscsisenseerror(union ccb *ccb, union ccb **orig, 1680 cam_flags camflags, uint32_t sense_flags, 1681 int *openings, uint32_t *relsim_flags, 1682 uint32_t *timeout, uint32_t *action, const char **action_string) 1683 { 1684 struct cam_periph *periph; 1685 union ccb *orig_ccb = ccb; 1686 int error, recoveryccb; 1687 uint16_t flags; 1688 1689 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING) 1690 if (ccb->ccb_h.func_code == XPT_SCSI_IO && ccb->csio.bio != NULL) 1691 biotrack(ccb->csio.bio, __func__); 1692 #endif 1693 1694 periph = xpt_path_periph(ccb->ccb_h.path); 1695 recoveryccb = (ccb->ccb_h.cbfcnp == camperiphdone); 1696 if ((periph->flags & CAM_PERIPH_RECOVERY_INPROG) && !recoveryccb) { 1697 /* 1698 * If error recovery is already in progress, don't attempt 1699 * to process this error, but requeue it unconditionally 1700 * and attempt to process it once error recovery has 1701 * completed. This failed command is probably related to 1702 * the error that caused the currently active error recovery 1703 * action so our current recovery efforts should also 1704 * address this command. Be aware that the error recovery 1705 * code assumes that only one recovery action is in progress 1706 * on a particular peripheral instance at any given time 1707 * (e.g. only one saved CCB for error recovery) so it is 1708 * imperitive that we don't violate this assumption. 1709 */ 1710 error = ERESTART; 1711 *action &= ~SSQ_PRINT_SENSE; 1712 } else { 1713 scsi_sense_action err_action; 1714 struct ccb_getdev cgd; 1715 1716 /* 1717 * Grab the inquiry data for this device. 1718 */ 1719 memset(&cgd, 0, sizeof(cgd)); 1720 xpt_setup_ccb(&cgd.ccb_h, ccb->ccb_h.path, CAM_PRIORITY_NORMAL); 1721 cgd.ccb_h.func_code = XPT_GDEV_TYPE; 1722 xpt_action((union ccb *)&cgd); 1723 1724 err_action = scsi_error_action(&ccb->csio, &cgd.inq_data, 1725 sense_flags); 1726 error = err_action & SS_ERRMASK; 1727 1728 /* 1729 * Do not autostart sequential access devices 1730 * to avoid unexpected tape loading. 1731 */ 1732 if ((err_action & SS_MASK) == SS_START && 1733 SID_TYPE(&cgd.inq_data) == T_SEQUENTIAL) { 1734 *action_string = "Will not autostart a " 1735 "sequential access device"; 1736 goto sense_error_done; 1737 } 1738 1739 /* 1740 * Avoid recovery recursion if recovery action is the same. 1741 */ 1742 if ((err_action & SS_MASK) >= SS_START && recoveryccb) { 1743 if (((err_action & SS_MASK) == SS_START && 1744 ccb->csio.cdb_io.cdb_bytes[0] == START_STOP_UNIT) || 1745 ((err_action & SS_MASK) == SS_TUR && 1746 (ccb->csio.cdb_io.cdb_bytes[0] == TEST_UNIT_READY))) { 1747 err_action = SS_RETRY|SSQ_DECREMENT_COUNT|EIO; 1748 *relsim_flags = RELSIM_RELEASE_AFTER_TIMEOUT; 1749 *timeout = 500; 1750 } 1751 } 1752 1753 /* 1754 * If the recovery action will consume a retry, 1755 * make sure we actually have retries available. 1756 */ 1757 if ((err_action & SSQ_DECREMENT_COUNT) != 0) { 1758 if (ccb->ccb_h.retry_count > 0 && 1759 (periph->flags & CAM_PERIPH_INVALID) == 0) 1760 ccb->ccb_h.retry_count--; 1761 else { 1762 *action_string = "Retries exhausted"; 1763 goto sense_error_done; 1764 } 1765 } 1766 1767 if ((err_action & SS_MASK) >= SS_START) { 1768 /* 1769 * Do common portions of commands that 1770 * use recovery CCBs. 1771 */ 1772 orig_ccb = xpt_alloc_ccb_nowait(); 1773 if (orig_ccb == NULL) { 1774 *action_string = "Can't allocate recovery CCB"; 1775 goto sense_error_done; 1776 } 1777 /* 1778 * Clear freeze flag for original request here, as 1779 * this freeze will be dropped as part of ERESTART. 1780 */ 1781 ccb->ccb_h.status &= ~CAM_DEV_QFRZN; 1782 1783 KASSERT(ccb->ccb_h.func_code == XPT_SCSI_IO, 1784 ("%s: ccb func_code %#x != XPT_SCSI_IO", 1785 __func__, ccb->ccb_h.func_code)); 1786 flags = orig_ccb->ccb_h.alloc_flags; 1787 bcopy(ccb, orig_ccb, sizeof(struct ccb_scsiio)); 1788 orig_ccb->ccb_h.alloc_flags = flags; 1789 } 1790 1791 switch (err_action & SS_MASK) { 1792 case SS_NOP: 1793 *action_string = "No recovery action needed"; 1794 error = 0; 1795 break; 1796 case SS_RETRY: 1797 *action_string = "Retrying command (per sense data)"; 1798 error = ERESTART; 1799 break; 1800 case SS_FAIL: 1801 *action_string = "Unretryable error"; 1802 break; 1803 case SS_START: 1804 { 1805 int le; 1806 1807 /* 1808 * Send a start unit command to the device, and 1809 * then retry the command. 1810 */ 1811 *action_string = "Attempting to start unit"; 1812 periph->flags |= CAM_PERIPH_RECOVERY_INPROG; 1813 1814 /* 1815 * Check for removable media and set 1816 * load/eject flag appropriately. 1817 */ 1818 if (SID_IS_REMOVABLE(&cgd.inq_data)) 1819 le = TRUE; 1820 else 1821 le = FALSE; 1822 1823 scsi_start_stop(&ccb->csio, 1824 /*retries*/1, 1825 camperiphdone, 1826 MSG_SIMPLE_Q_TAG, 1827 /*start*/TRUE, 1828 /*load/eject*/le, 1829 /*immediate*/FALSE, 1830 SSD_FULL_SIZE, 1831 /*timeout*/50000); 1832 break; 1833 } 1834 case SS_TUR: 1835 { 1836 /* 1837 * Send a Test Unit Ready to the device. 1838 * If the 'many' flag is set, we send 120 1839 * test unit ready commands, one every half 1840 * second. Otherwise, we just send one TUR. 1841 * We only want to do this if the retry 1842 * count has not been exhausted. 1843 */ 1844 int retries; 1845 1846 if ((err_action & SSQ_MANY) != 0 && (periph->flags & 1847 CAM_PERIPH_RECOVERY_WAIT_FAILED) == 0) { 1848 periph->flags |= CAM_PERIPH_RECOVERY_WAIT; 1849 *action_string = "Polling device for readiness"; 1850 retries = 120; 1851 } else { 1852 *action_string = "Testing device for readiness"; 1853 retries = 1; 1854 } 1855 periph->flags |= CAM_PERIPH_RECOVERY_INPROG; 1856 scsi_test_unit_ready(&ccb->csio, 1857 retries, 1858 camperiphdone, 1859 MSG_SIMPLE_Q_TAG, 1860 SSD_FULL_SIZE, 1861 /*timeout*/5000); 1862 1863 /* 1864 * Accomplish our 500ms delay by deferring 1865 * the release of our device queue appropriately. 1866 */ 1867 *relsim_flags = RELSIM_RELEASE_AFTER_TIMEOUT; 1868 *timeout = 500; 1869 break; 1870 } 1871 default: 1872 panic("Unhandled error action %x", err_action); 1873 } 1874 1875 if ((err_action & SS_MASK) >= SS_START) { 1876 /* 1877 * Drop the priority, so that the recovery 1878 * CCB is the first to execute. Freeze the queue 1879 * after this command is sent so that we can 1880 * restore the old csio and have it queued in 1881 * the proper order before we release normal 1882 * transactions to the device. 1883 */ 1884 ccb->ccb_h.pinfo.priority--; 1885 ccb->ccb_h.flags |= CAM_DEV_QFREEZE; 1886 ccb->ccb_h.saved_ccb_ptr = orig_ccb; 1887 error = ERESTART; 1888 *orig = orig_ccb; 1889 } 1890 1891 sense_error_done: 1892 *action = err_action; 1893 } 1894 return (error); 1895 } 1896 1897 /* 1898 * Generic error handler. Peripheral drivers usually filter 1899 * out the errors that they handle in a unique manner, then 1900 * call this function. 1901 */ 1902 int 1903 cam_periph_error(union ccb *ccb, cam_flags camflags, 1904 uint32_t sense_flags) 1905 { 1906 struct cam_path *newpath; 1907 union ccb *orig_ccb, *scan_ccb; 1908 struct cam_periph *periph; 1909 const char *action_string; 1910 cam_status status; 1911 int frozen, error, openings, devctl_err; 1912 uint32_t action, relsim_flags, timeout; 1913 1914 action = SSQ_PRINT_SENSE; 1915 periph = xpt_path_periph(ccb->ccb_h.path); 1916 action_string = NULL; 1917 status = ccb->ccb_h.status; 1918 frozen = (status & CAM_DEV_QFRZN) != 0; 1919 status &= CAM_STATUS_MASK; 1920 devctl_err = openings = relsim_flags = timeout = 0; 1921 orig_ccb = ccb; 1922 1923 /* Filter the errors that should be reported via devctl */ 1924 switch (ccb->ccb_h.status & CAM_STATUS_MASK) { 1925 case CAM_CMD_TIMEOUT: 1926 case CAM_REQ_ABORTED: 1927 case CAM_REQ_CMP_ERR: 1928 case CAM_REQ_TERMIO: 1929 case CAM_UNREC_HBA_ERROR: 1930 case CAM_DATA_RUN_ERR: 1931 case CAM_SCSI_STATUS_ERROR: 1932 case CAM_ATA_STATUS_ERROR: 1933 case CAM_SMP_STATUS_ERROR: 1934 case CAM_DEV_NOT_THERE: 1935 case CAM_NVME_STATUS_ERROR: 1936 devctl_err++; 1937 break; 1938 default: 1939 break; 1940 } 1941 1942 switch (status) { 1943 case CAM_REQ_CMP: 1944 error = 0; 1945 action &= ~SSQ_PRINT_SENSE; 1946 break; 1947 case CAM_SCSI_STATUS_ERROR: 1948 error = camperiphscsistatuserror(ccb, &orig_ccb, 1949 camflags, sense_flags, &openings, &relsim_flags, 1950 &timeout, &action, &action_string); 1951 break; 1952 case CAM_AUTOSENSE_FAIL: 1953 error = EIO; /* we have to kill the command */ 1954 break; 1955 case CAM_UA_ABORT: 1956 case CAM_UA_TERMIO: 1957 case CAM_MSG_REJECT_REC: 1958 /* XXX Don't know that these are correct */ 1959 error = EIO; 1960 break; 1961 case CAM_SEL_TIMEOUT: 1962 if ((camflags & CAM_RETRY_SELTO) != 0) { 1963 if (ccb->ccb_h.retry_count > 0 && 1964 (periph->flags & CAM_PERIPH_INVALID) == 0) { 1965 ccb->ccb_h.retry_count--; 1966 error = ERESTART; 1967 1968 /* 1969 * Wait a bit to give the device 1970 * time to recover before we try again. 1971 */ 1972 relsim_flags = RELSIM_RELEASE_AFTER_TIMEOUT; 1973 timeout = periph_selto_delay; 1974 break; 1975 } 1976 action_string = "Retries exhausted"; 1977 } 1978 /* FALLTHROUGH */ 1979 case CAM_DEV_NOT_THERE: 1980 error = ENXIO; 1981 action = SSQ_LOST; 1982 break; 1983 case CAM_REQ_INVALID: 1984 case CAM_PATH_INVALID: 1985 case CAM_NO_HBA: 1986 case CAM_PROVIDE_FAIL: 1987 case CAM_REQ_TOO_BIG: 1988 case CAM_LUN_INVALID: 1989 case CAM_TID_INVALID: 1990 case CAM_FUNC_NOTAVAIL: 1991 error = EINVAL; 1992 break; 1993 case CAM_SCSI_BUS_RESET: 1994 case CAM_BDR_SENT: 1995 /* 1996 * Commands that repeatedly timeout and cause these 1997 * kinds of error recovery actions, should return 1998 * CAM_CMD_TIMEOUT, which allows us to safely assume 1999 * that this command was an innocent bystander to 2000 * these events and should be unconditionally 2001 * retried. 2002 */ 2003 case CAM_REQUEUE_REQ: 2004 /* Unconditional requeue if device is still there */ 2005 if (periph->flags & CAM_PERIPH_INVALID) { 2006 action_string = "Periph was invalidated"; 2007 error = ENXIO; 2008 } else if (sense_flags & SF_NO_RETRY) { 2009 error = EIO; 2010 action_string = "Retry was blocked"; 2011 } else { 2012 error = ERESTART; 2013 action &= ~SSQ_PRINT_SENSE; 2014 } 2015 break; 2016 case CAM_RESRC_UNAVAIL: 2017 /* Wait a bit for the resource shortage to abate. */ 2018 timeout = periph_noresrc_delay; 2019 /* FALLTHROUGH */ 2020 case CAM_BUSY: 2021 if (timeout == 0) { 2022 /* Wait a bit for the busy condition to abate. */ 2023 timeout = periph_busy_delay; 2024 } 2025 relsim_flags = RELSIM_RELEASE_AFTER_TIMEOUT; 2026 /* FALLTHROUGH */ 2027 case CAM_ATA_STATUS_ERROR: 2028 case CAM_NVME_STATUS_ERROR: 2029 case CAM_SMP_STATUS_ERROR: 2030 case CAM_REQ_CMP_ERR: 2031 case CAM_CMD_TIMEOUT: 2032 case CAM_UNEXP_BUSFREE: 2033 case CAM_UNCOR_PARITY: 2034 case CAM_DATA_RUN_ERR: 2035 default: 2036 if (periph->flags & CAM_PERIPH_INVALID) { 2037 error = ENXIO; 2038 action_string = "Periph was invalidated"; 2039 } else if (ccb->ccb_h.retry_count == 0) { 2040 error = EIO; 2041 action_string = "Retries exhausted"; 2042 } else if (sense_flags & SF_NO_RETRY) { 2043 error = EIO; 2044 action_string = "Retry was blocked"; 2045 } else { 2046 ccb->ccb_h.retry_count--; 2047 error = ERESTART; 2048 } 2049 break; 2050 } 2051 2052 if ((sense_flags & SF_PRINT_ALWAYS) || 2053 CAM_DEBUGGED(ccb->ccb_h.path, CAM_DEBUG_INFO)) 2054 action |= SSQ_PRINT_SENSE; 2055 else if (sense_flags & SF_NO_PRINT) 2056 action &= ~SSQ_PRINT_SENSE; 2057 if ((action & SSQ_PRINT_SENSE) != 0) 2058 cam_error_print(orig_ccb, CAM_ESF_ALL, CAM_EPF_ALL); 2059 if (error != 0 && (action & SSQ_PRINT_SENSE) != 0) { 2060 if (error != ERESTART) { 2061 if (action_string == NULL) 2062 action_string = "Unretryable error"; 2063 xpt_print(ccb->ccb_h.path, "Error %d, %s\n", 2064 error, action_string); 2065 } else if (action_string != NULL) 2066 xpt_print(ccb->ccb_h.path, "%s\n", action_string); 2067 else { 2068 xpt_print(ccb->ccb_h.path, 2069 "Retrying command, %d more tries remain\n", 2070 ccb->ccb_h.retry_count); 2071 } 2072 } 2073 2074 if (devctl_err && (error != 0 || (action & SSQ_PRINT_SENSE) != 0)) 2075 cam_periph_devctl_notify(orig_ccb); 2076 2077 if ((action & SSQ_LOST) != 0) { 2078 lun_id_t lun_id; 2079 2080 /* 2081 * For a selection timeout, we consider all of the LUNs on 2082 * the target to be gone. If the status is CAM_DEV_NOT_THERE, 2083 * then we only get rid of the device(s) specified by the 2084 * path in the original CCB. 2085 */ 2086 if (status == CAM_SEL_TIMEOUT) 2087 lun_id = CAM_LUN_WILDCARD; 2088 else 2089 lun_id = xpt_path_lun_id(ccb->ccb_h.path); 2090 2091 /* Should we do more if we can't create the path?? */ 2092 if (xpt_create_path(&newpath, periph, 2093 xpt_path_path_id(ccb->ccb_h.path), 2094 xpt_path_target_id(ccb->ccb_h.path), 2095 lun_id) == CAM_REQ_CMP) { 2096 /* 2097 * Let peripheral drivers know that this 2098 * device has gone away. 2099 */ 2100 xpt_async(AC_LOST_DEVICE, newpath, NULL); 2101 xpt_free_path(newpath); 2102 } 2103 } 2104 2105 /* Broadcast UNIT ATTENTIONs to all periphs. */ 2106 if ((action & SSQ_UA) != 0) 2107 xpt_async(AC_UNIT_ATTENTION, orig_ccb->ccb_h.path, orig_ccb); 2108 2109 /* Rescan target on "Reported LUNs data has changed" */ 2110 if ((action & SSQ_RESCAN) != 0) { 2111 if (xpt_create_path(&newpath, NULL, 2112 xpt_path_path_id(ccb->ccb_h.path), 2113 xpt_path_target_id(ccb->ccb_h.path), 2114 CAM_LUN_WILDCARD) == CAM_REQ_CMP) { 2115 scan_ccb = xpt_alloc_ccb_nowait(); 2116 if (scan_ccb != NULL) { 2117 scan_ccb->ccb_h.path = newpath; 2118 scan_ccb->ccb_h.func_code = XPT_SCAN_TGT; 2119 scan_ccb->crcn.flags = 0; 2120 xpt_rescan(scan_ccb); 2121 } else { 2122 xpt_print(newpath, 2123 "Can't allocate CCB to rescan target\n"); 2124 xpt_free_path(newpath); 2125 } 2126 } 2127 } 2128 2129 /* Attempt a retry */ 2130 if (error == ERESTART || error == 0) { 2131 if (frozen != 0) 2132 ccb->ccb_h.status &= ~CAM_DEV_QFRZN; 2133 if (error == ERESTART) 2134 xpt_action(ccb); 2135 if (frozen != 0) 2136 cam_release_devq(ccb->ccb_h.path, 2137 relsim_flags, 2138 openings, 2139 timeout, 2140 /*getcount_only*/0); 2141 } 2142 2143 return (error); 2144 } 2145 2146 #define CAM_PERIPH_DEVD_MSG_SIZE 256 2147 2148 static void 2149 cam_periph_devctl_notify(union ccb *ccb) 2150 { 2151 struct cam_periph *periph; 2152 struct ccb_getdev *cgd; 2153 struct sbuf sb; 2154 int serr, sk, asc, ascq; 2155 char *sbmsg, *type; 2156 2157 sbmsg = malloc(CAM_PERIPH_DEVD_MSG_SIZE, M_CAMPERIPH, M_NOWAIT); 2158 if (sbmsg == NULL) 2159 return; 2160 2161 sbuf_new(&sb, sbmsg, CAM_PERIPH_DEVD_MSG_SIZE, SBUF_FIXEDLEN); 2162 2163 periph = xpt_path_periph(ccb->ccb_h.path); 2164 sbuf_printf(&sb, "device=%s%d ", periph->periph_name, 2165 periph->unit_number); 2166 2167 sbuf_cat(&sb, "serial=\""); 2168 if ((cgd = (struct ccb_getdev *)xpt_alloc_ccb_nowait()) != NULL) { 2169 xpt_setup_ccb(&cgd->ccb_h, ccb->ccb_h.path, 2170 CAM_PRIORITY_NORMAL); 2171 cgd->ccb_h.func_code = XPT_GDEV_TYPE; 2172 xpt_action((union ccb *)cgd); 2173 2174 if (cgd->ccb_h.status == CAM_REQ_CMP) 2175 sbuf_bcat(&sb, cgd->serial_num, cgd->serial_num_len); 2176 xpt_free_ccb((union ccb *)cgd); 2177 } 2178 sbuf_cat(&sb, "\" "); 2179 sbuf_printf(&sb, "cam_status=\"0x%x\" ", ccb->ccb_h.status); 2180 2181 switch (ccb->ccb_h.status & CAM_STATUS_MASK) { 2182 case CAM_CMD_TIMEOUT: 2183 sbuf_printf(&sb, "timeout=%d ", ccb->ccb_h.timeout); 2184 type = "timeout"; 2185 break; 2186 case CAM_SCSI_STATUS_ERROR: 2187 sbuf_printf(&sb, "scsi_status=%d ", ccb->csio.scsi_status); 2188 if (scsi_extract_sense_ccb(ccb, &serr, &sk, &asc, &ascq)) 2189 sbuf_printf(&sb, "scsi_sense=\"%02x %02x %02x %02x\" ", 2190 serr, sk, asc, ascq); 2191 type = "error"; 2192 break; 2193 case CAM_ATA_STATUS_ERROR: 2194 sbuf_cat(&sb, "RES=\""); 2195 ata_res_sbuf(&ccb->ataio.res, &sb); 2196 sbuf_cat(&sb, "\" "); 2197 type = "error"; 2198 break; 2199 case CAM_NVME_STATUS_ERROR: 2200 { 2201 struct ccb_nvmeio *n = &ccb->nvmeio; 2202 2203 sbuf_printf(&sb, "sc=\"%02x\" sct=\"%02x\" cdw0=\"%08x\" ", 2204 NVME_STATUS_GET_SC(n->cpl.status), 2205 NVME_STATUS_GET_SCT(n->cpl.status), n->cpl.cdw0); 2206 type = "error"; 2207 break; 2208 } 2209 default: 2210 type = "error"; 2211 break; 2212 } 2213 2214 2215 switch (ccb->ccb_h.func_code) { 2216 case XPT_SCSI_IO: 2217 sbuf_cat(&sb, "CDB=\""); 2218 scsi_cdb_sbuf(scsiio_cdb_ptr(&ccb->csio), &sb); 2219 sbuf_cat(&sb, "\" "); 2220 break; 2221 case XPT_ATA_IO: 2222 sbuf_cat(&sb, "ACB=\""); 2223 ata_cmd_sbuf(&ccb->ataio.cmd, &sb); 2224 sbuf_cat(&sb, "\" "); 2225 break; 2226 case XPT_NVME_IO: 2227 case XPT_NVME_ADMIN: 2228 { 2229 struct ccb_nvmeio *n = &ccb->nvmeio; 2230 struct nvme_command *cmd = &n->cmd; 2231 2232 // XXX Likely should be nvme_cmd_sbuf 2233 sbuf_printf(&sb, "opc=\"%02x\" fuse=\"%02x\" cid=\"%04x\" " 2234 "nsid=\"%08x\" cdw10=\"%08x\" cdw11=\"%08x\" cdw12=\"%08x\" " 2235 "cdw13=\"%08x\" cdw14=\"%08x\" cdw15=\"%08x\" ", 2236 cmd->opc, cmd->fuse, cmd->cid, cmd->nsid, cmd->cdw10, 2237 cmd->cdw11, cmd->cdw12, cmd->cdw13, cmd->cdw14, cmd->cdw15); 2238 break; 2239 } 2240 default: 2241 break; 2242 } 2243 2244 if (sbuf_finish(&sb) == 0) 2245 devctl_notify("CAM", "periph", type, sbuf_data(&sb)); 2246 sbuf_delete(&sb); 2247 free(sbmsg, M_CAMPERIPH); 2248 } 2249 2250 /* 2251 * Sysctl to force an invalidation of the drive right now. Can be 2252 * called with CTLFLAG_MPSAFE since we take periph lock. 2253 */ 2254 int 2255 cam_periph_invalidate_sysctl(SYSCTL_HANDLER_ARGS) 2256 { 2257 struct cam_periph *periph; 2258 int error, value; 2259 2260 periph = arg1; 2261 value = 0; 2262 error = sysctl_handle_int(oidp, &value, 0, req); 2263 if (error != 0 || req->newptr == NULL || value != 1) 2264 return (error); 2265 2266 cam_periph_lock(periph); 2267 cam_periph_invalidate(periph); 2268 cam_periph_unlock(periph); 2269 2270 return (0); 2271 } 2272