1 /*- 2 * Common functions for CAM "type" (peripheral) drivers. 3 * 4 * SPDX-License-Identifier: BSD-2-Clause 5 * 6 * Copyright (c) 1997, 1998 Justin T. Gibbs. 7 * Copyright (c) 1997, 1998, 1999, 2000 Kenneth D. Merry. 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions, and the following disclaimer, 15 * without modification, immediately at the beginning of the file. 16 * 2. The name of the author may not be used to endorse or promote products 17 * derived from this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 23 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/types.h> 35 #include <sys/malloc.h> 36 #include <sys/kernel.h> 37 #include <sys/bio.h> 38 #include <sys/conf.h> 39 #include <sys/devctl.h> 40 #include <sys/lock.h> 41 #include <sys/mutex.h> 42 #include <sys/buf.h> 43 #include <sys/proc.h> 44 #include <sys/devicestat.h> 45 #include <sys/sbuf.h> 46 #include <sys/sysctl.h> 47 #include <vm/vm.h> 48 #include <vm/vm_extern.h> 49 50 #include <cam/cam.h> 51 #include <cam/cam_ccb.h> 52 #include <cam/cam_compat.h> 53 #include <cam/cam_queue.h> 54 #include <cam/cam_xpt_periph.h> 55 #include <cam/cam_xpt_internal.h> 56 #include <cam/cam_periph.h> 57 #include <cam/cam_debug.h> 58 #include <cam/cam_sim.h> 59 60 #include <cam/scsi/scsi_all.h> 61 #include <cam/scsi/scsi_message.h> 62 #include <cam/scsi/scsi_pass.h> 63 64 /* SDT Probes */ 65 SDT_PROBE_DEFINE3(cam, , periph, error, "union ccb *", "cam_flags", 66 "uint32_t"); 67 SDT_PROBE_DEFINE2(cam, , periph, recovery, "union ccb *", "int"); 68 SDT_PROBE_DEFINE1(cam, , periph, invalidate, "struct cam_periph *"); 69 SDT_PROBE_DEFINE1(cam, , periph, hold__boot, "struct cam_periph *"); 70 SDT_PROBE_DEFINE1(cam, , periph, release__boot, "struct cam_periph *"); 71 72 static u_int camperiphnextunit(struct periph_driver *p_drv, 73 u_int newunit, bool wired, 74 path_id_t pathid, target_id_t target, 75 lun_id_t lun); 76 static u_int camperiphunit(struct periph_driver *p_drv, 77 path_id_t pathid, target_id_t target, 78 lun_id_t lun, 79 const char *sn); 80 static void camperiphdone(struct cam_periph *periph, 81 union ccb *done_ccb); 82 static void camperiphfree(struct cam_periph *periph); 83 static int camperiphscsistatuserror(union ccb *ccb, 84 union ccb **orig_ccb, 85 cam_flags camflags, 86 uint32_t sense_flags, 87 int *openings, 88 uint32_t *relsim_flags, 89 uint32_t *timeout, 90 uint32_t *action, 91 const char **action_string); 92 static int camperiphscsisenseerror(union ccb *ccb, 93 union ccb **orig_ccb, 94 cam_flags camflags, 95 uint32_t sense_flags, 96 int *openings, 97 uint32_t *relsim_flags, 98 uint32_t *timeout, 99 uint32_t *action, 100 const char **action_string); 101 static void cam_periph_devctl_notify(union ccb *ccb); 102 static char *cam_periph_devctl_sb_init(struct sbuf *sb, 103 struct cam_periph *periph); 104 static void cam_periph_devctl_sb_fini(struct sbuf *sb, char *sbmsg, 105 const char *type); 106 107 static int nperiph_drivers; 108 static int initialized = 0; 109 struct periph_driver **periph_drivers; 110 111 static MALLOC_DEFINE(M_CAMPERIPH, "CAM periph", "CAM peripheral buffers"); 112 113 static int periph_selto_delay = 1000; 114 TUNABLE_INT("kern.cam.periph_selto_delay", &periph_selto_delay); 115 static int periph_noresrc_delay = 500; 116 TUNABLE_INT("kern.cam.periph_noresrc_delay", &periph_noresrc_delay); 117 static int periph_busy_delay = 500; 118 TUNABLE_INT("kern.cam.periph_busy_delay", &periph_busy_delay); 119 120 static u_int periph_mapmem_thresh = 65536; 121 SYSCTL_UINT(_kern_cam, OID_AUTO, mapmem_thresh, CTLFLAG_RWTUN, 122 &periph_mapmem_thresh, 0, "Threshold for user-space buffer mapping"); 123 124 void 125 periphdriver_register(void *data) 126 { 127 struct periph_driver *drv = (struct periph_driver *)data; 128 struct periph_driver **newdrivers, **old; 129 int ndrivers; 130 131 again: 132 ndrivers = nperiph_drivers + 2; 133 newdrivers = malloc(sizeof(*newdrivers) * ndrivers, M_CAMPERIPH, 134 M_WAITOK); 135 xpt_lock_buses(); 136 if (ndrivers != nperiph_drivers + 2) { 137 /* 138 * Lost race against itself; go around. 139 */ 140 xpt_unlock_buses(); 141 free(newdrivers, M_CAMPERIPH); 142 goto again; 143 } 144 if (periph_drivers) 145 bcopy(periph_drivers, newdrivers, 146 sizeof(*newdrivers) * nperiph_drivers); 147 newdrivers[nperiph_drivers] = drv; 148 newdrivers[nperiph_drivers + 1] = NULL; 149 old = periph_drivers; 150 periph_drivers = newdrivers; 151 nperiph_drivers++; 152 xpt_unlock_buses(); 153 if (old) 154 free(old, M_CAMPERIPH); 155 /* If driver marked as early or it is late now, initialize it. */ 156 if (((drv->flags & CAM_PERIPH_DRV_EARLY) != 0 && initialized > 0) || 157 initialized > 1) 158 (*drv->init)(); 159 } 160 161 int 162 periphdriver_unregister(void *data) 163 { 164 struct periph_driver *drv = (struct periph_driver *)data; 165 int error, n; 166 167 /* If driver marked as early or it is late now, deinitialize it. */ 168 if (((drv->flags & CAM_PERIPH_DRV_EARLY) != 0 && initialized > 0) || 169 initialized > 1) { 170 if (drv->deinit == NULL) { 171 printf("CAM periph driver '%s' doesn't have deinit.\n", 172 drv->driver_name); 173 return (EOPNOTSUPP); 174 } 175 error = drv->deinit(); 176 if (error != 0) 177 return (error); 178 } 179 180 xpt_lock_buses(); 181 for (n = 0; n < nperiph_drivers && periph_drivers[n] != drv; n++) 182 ; 183 KASSERT(n < nperiph_drivers, 184 ("Periph driver '%s' was not registered", drv->driver_name)); 185 for (; n + 1 < nperiph_drivers; n++) 186 periph_drivers[n] = periph_drivers[n + 1]; 187 periph_drivers[n + 1] = NULL; 188 nperiph_drivers--; 189 xpt_unlock_buses(); 190 return (0); 191 } 192 193 void 194 periphdriver_init(int level) 195 { 196 int i, early; 197 198 initialized = max(initialized, level); 199 for (i = 0; periph_drivers[i] != NULL; i++) { 200 early = (periph_drivers[i]->flags & CAM_PERIPH_DRV_EARLY) ? 1 : 2; 201 if (early == initialized) 202 (*periph_drivers[i]->init)(); 203 } 204 } 205 206 cam_status 207 cam_periph_alloc(periph_ctor_t *periph_ctor, 208 periph_oninv_t *periph_oninvalidate, 209 periph_dtor_t *periph_dtor, periph_start_t *periph_start, 210 char *name, cam_periph_type type, struct cam_path *path, 211 ac_callback_t *ac_callback, ac_code code, void *arg) 212 { 213 struct periph_driver **p_drv; 214 struct cam_sim *sim; 215 struct cam_periph *periph; 216 struct cam_periph *cur_periph; 217 path_id_t path_id; 218 target_id_t target_id; 219 lun_id_t lun_id; 220 cam_status status; 221 u_int init_level; 222 223 init_level = 0; 224 /* 225 * Handle Hot-Plug scenarios. If there is already a peripheral 226 * of our type assigned to this path, we are likely waiting for 227 * final close on an old, invalidated, peripheral. If this is 228 * the case, queue up a deferred call to the peripheral's async 229 * handler. If it looks like a mistaken re-allocation, complain. 230 */ 231 if ((periph = cam_periph_find(path, name)) != NULL) { 232 if ((periph->flags & CAM_PERIPH_INVALID) != 0 233 && (periph->flags & CAM_PERIPH_NEW_DEV_FOUND) == 0) { 234 periph->flags |= CAM_PERIPH_NEW_DEV_FOUND; 235 periph->deferred_callback = ac_callback; 236 periph->deferred_ac = code; 237 return (CAM_REQ_INPROG); 238 } else { 239 printf("cam_periph_alloc: attempt to re-allocate " 240 "valid device %s%d rejected flags %#x " 241 "refcount %d\n", periph->periph_name, 242 periph->unit_number, periph->flags, 243 periph->refcount); 244 } 245 return (CAM_REQ_INVALID); 246 } 247 248 periph = (struct cam_periph *)malloc(sizeof(*periph), M_CAMPERIPH, 249 M_NOWAIT|M_ZERO); 250 251 if (periph == NULL) 252 return (CAM_RESRC_UNAVAIL); 253 254 init_level++; 255 256 sim = xpt_path_sim(path); 257 path_id = xpt_path_path_id(path); 258 target_id = xpt_path_target_id(path); 259 lun_id = xpt_path_lun_id(path); 260 periph->periph_start = periph_start; 261 periph->periph_dtor = periph_dtor; 262 periph->periph_oninval = periph_oninvalidate; 263 periph->type = type; 264 periph->periph_name = name; 265 periph->scheduled_priority = CAM_PRIORITY_NONE; 266 periph->immediate_priority = CAM_PRIORITY_NONE; 267 periph->refcount = 1; /* Dropped by invalidation. */ 268 periph->sim = sim; 269 SLIST_INIT(&periph->ccb_list); 270 status = xpt_create_path(&path, periph, path_id, target_id, lun_id); 271 if (status != CAM_REQ_CMP) 272 goto failure; 273 periph->path = path; 274 275 xpt_lock_buses(); 276 for (p_drv = periph_drivers; *p_drv != NULL; p_drv++) { 277 if (strcmp((*p_drv)->driver_name, name) == 0) 278 break; 279 } 280 if (*p_drv == NULL) { 281 printf("cam_periph_alloc: invalid periph name '%s'\n", name); 282 xpt_unlock_buses(); 283 xpt_free_path(periph->path); 284 free(periph, M_CAMPERIPH); 285 return (CAM_REQ_INVALID); 286 } 287 periph->unit_number = camperiphunit(*p_drv, path_id, target_id, lun_id, 288 path->device->serial_num); 289 cur_periph = TAILQ_FIRST(&(*p_drv)->units); 290 while (cur_periph != NULL 291 && cur_periph->unit_number < periph->unit_number) 292 cur_periph = TAILQ_NEXT(cur_periph, unit_links); 293 if (cur_periph != NULL) { 294 KASSERT(cur_periph->unit_number != periph->unit_number, 295 ("duplicate units on periph list")); 296 TAILQ_INSERT_BEFORE(cur_periph, periph, unit_links); 297 } else { 298 TAILQ_INSERT_TAIL(&(*p_drv)->units, periph, unit_links); 299 (*p_drv)->generation++; 300 } 301 xpt_unlock_buses(); 302 303 init_level++; 304 305 status = xpt_add_periph(periph); 306 if (status != CAM_REQ_CMP) 307 goto failure; 308 309 init_level++; 310 CAM_DEBUG(periph->path, CAM_DEBUG_INFO, ("Periph created\n")); 311 312 status = periph_ctor(periph, arg); 313 314 if (status == CAM_REQ_CMP) 315 init_level++; 316 317 failure: 318 switch (init_level) { 319 case 4: 320 /* Initialized successfully */ 321 break; 322 case 3: 323 CAM_DEBUG(periph->path, CAM_DEBUG_INFO, ("Periph destroyed\n")); 324 xpt_remove_periph(periph); 325 /* FALLTHROUGH */ 326 case 2: 327 xpt_lock_buses(); 328 TAILQ_REMOVE(&(*p_drv)->units, periph, unit_links); 329 xpt_unlock_buses(); 330 xpt_free_path(periph->path); 331 /* FALLTHROUGH */ 332 case 1: 333 free(periph, M_CAMPERIPH); 334 /* FALLTHROUGH */ 335 case 0: 336 /* No cleanup to perform. */ 337 break; 338 default: 339 panic("%s: Unknown init level", __func__); 340 } 341 return(status); 342 } 343 344 /* 345 * Find a peripheral structure with the specified path, target, lun, 346 * and (optionally) type. If the name is NULL, this function will return 347 * the first peripheral driver that matches the specified path. 348 */ 349 struct cam_periph * 350 cam_periph_find(struct cam_path *path, char *name) 351 { 352 struct periph_driver **p_drv; 353 struct cam_periph *periph; 354 355 xpt_lock_buses(); 356 for (p_drv = periph_drivers; *p_drv != NULL; p_drv++) { 357 if (name != NULL && (strcmp((*p_drv)->driver_name, name) != 0)) 358 continue; 359 360 TAILQ_FOREACH(periph, &(*p_drv)->units, unit_links) { 361 if (xpt_path_comp(periph->path, path) == 0) { 362 xpt_unlock_buses(); 363 cam_periph_assert(periph, MA_OWNED); 364 return(periph); 365 } 366 } 367 if (name != NULL) { 368 xpt_unlock_buses(); 369 return(NULL); 370 } 371 } 372 xpt_unlock_buses(); 373 return(NULL); 374 } 375 376 /* 377 * Find peripheral driver instances attached to the specified path. 378 */ 379 int 380 cam_periph_list(struct cam_path *path, struct sbuf *sb) 381 { 382 struct sbuf local_sb; 383 struct periph_driver **p_drv; 384 struct cam_periph *periph; 385 int count; 386 int sbuf_alloc_len; 387 388 sbuf_alloc_len = 16; 389 retry: 390 sbuf_new(&local_sb, NULL, sbuf_alloc_len, SBUF_FIXEDLEN); 391 count = 0; 392 xpt_lock_buses(); 393 for (p_drv = periph_drivers; *p_drv != NULL; p_drv++) { 394 TAILQ_FOREACH(periph, &(*p_drv)->units, unit_links) { 395 if (xpt_path_comp(periph->path, path) != 0) 396 continue; 397 398 if (sbuf_len(&local_sb) != 0) 399 sbuf_cat(&local_sb, ","); 400 401 sbuf_printf(&local_sb, "%s%d", periph->periph_name, 402 periph->unit_number); 403 404 if (sbuf_error(&local_sb) == ENOMEM) { 405 sbuf_alloc_len *= 2; 406 xpt_unlock_buses(); 407 sbuf_delete(&local_sb); 408 goto retry; 409 } 410 count++; 411 } 412 } 413 xpt_unlock_buses(); 414 sbuf_finish(&local_sb); 415 if (sbuf_len(sb) != 0) 416 sbuf_cat(sb, ","); 417 sbuf_cat(sb, sbuf_data(&local_sb)); 418 sbuf_delete(&local_sb); 419 return (count); 420 } 421 422 int 423 cam_periph_acquire(struct cam_periph *periph) 424 { 425 int status; 426 427 if (periph == NULL) 428 return (EINVAL); 429 430 status = ENOENT; 431 xpt_lock_buses(); 432 if ((periph->flags & CAM_PERIPH_INVALID) == 0) { 433 periph->refcount++; 434 status = 0; 435 } 436 xpt_unlock_buses(); 437 438 return (status); 439 } 440 441 void 442 cam_periph_doacquire(struct cam_periph *periph) 443 { 444 445 xpt_lock_buses(); 446 KASSERT(periph->refcount >= 1, 447 ("cam_periph_doacquire() with refcount == %d", periph->refcount)); 448 periph->refcount++; 449 xpt_unlock_buses(); 450 } 451 452 void 453 cam_periph_release_locked_buses(struct cam_periph *periph) 454 { 455 456 cam_periph_assert(periph, MA_OWNED); 457 KASSERT(periph->refcount >= 1, ("periph->refcount >= 1")); 458 if (--periph->refcount == 0) 459 camperiphfree(periph); 460 } 461 462 void 463 cam_periph_release_locked(struct cam_periph *periph) 464 { 465 466 if (periph == NULL) 467 return; 468 469 xpt_lock_buses(); 470 cam_periph_release_locked_buses(periph); 471 xpt_unlock_buses(); 472 } 473 474 void 475 cam_periph_release(struct cam_periph *periph) 476 { 477 struct mtx *mtx; 478 479 if (periph == NULL) 480 return; 481 482 cam_periph_assert(periph, MA_NOTOWNED); 483 mtx = cam_periph_mtx(periph); 484 mtx_lock(mtx); 485 cam_periph_release_locked(periph); 486 mtx_unlock(mtx); 487 } 488 489 /* 490 * hold/unhold act as mutual exclusion for sections of the code that 491 * need to sleep and want to make sure that other sections that 492 * will interfere are held off. This only protects exclusive sections 493 * from each other. 494 */ 495 int 496 cam_periph_hold(struct cam_periph *periph, int priority) 497 { 498 int error; 499 500 /* 501 * Increment the reference count on the peripheral 502 * while we wait for our lock attempt to succeed 503 * to ensure the peripheral doesn't disappear out 504 * from user us while we sleep. 505 */ 506 507 if (cam_periph_acquire(periph) != 0) 508 return (ENXIO); 509 510 cam_periph_assert(periph, MA_OWNED); 511 while ((periph->flags & CAM_PERIPH_LOCKED) != 0) { 512 periph->flags |= CAM_PERIPH_LOCK_WANTED; 513 if ((error = cam_periph_sleep(periph, periph, priority, 514 "caplck", 0)) != 0) { 515 cam_periph_release_locked(periph); 516 return (error); 517 } 518 if (periph->flags & CAM_PERIPH_INVALID) { 519 cam_periph_release_locked(periph); 520 return (ENXIO); 521 } 522 } 523 524 periph->flags |= CAM_PERIPH_LOCKED; 525 return (0); 526 } 527 528 void 529 cam_periph_unhold(struct cam_periph *periph) 530 { 531 532 cam_periph_assert(periph, MA_OWNED); 533 534 periph->flags &= ~CAM_PERIPH_LOCKED; 535 if ((periph->flags & CAM_PERIPH_LOCK_WANTED) != 0) { 536 periph->flags &= ~CAM_PERIPH_LOCK_WANTED; 537 wakeup(periph); 538 } 539 540 cam_periph_release_locked(periph); 541 } 542 543 void 544 cam_periph_hold_boot(struct cam_periph *periph) 545 { 546 547 CAM_PROBE1(periph, hold__boot, periph); 548 root_mount_hold_token(periph->periph_name, &periph->periph_rootmount); 549 } 550 551 void 552 cam_periph_release_boot(struct cam_periph *periph) 553 { 554 555 CAM_PROBE1(periph, release__boot, periph); 556 root_mount_rel(&periph->periph_rootmount); 557 } 558 559 /* 560 * Look for the next unit number that is not currently in use for this 561 * peripheral type starting at "newunit". Also exclude unit numbers that 562 * are reserved by for future "hardwiring" unless we already know that this 563 * is a potential wired device. Only assume that the device is "wired" the 564 * first time through the loop since after that we'll be looking at unit 565 * numbers that did not match a wiring entry. 566 */ 567 static u_int 568 camperiphnextunit(struct periph_driver *p_drv, u_int newunit, bool wired, 569 path_id_t pathid, target_id_t target, lun_id_t lun) 570 { 571 struct cam_periph *periph; 572 char *periph_name; 573 int i, val, dunit, r; 574 const char *dname, *strval; 575 576 periph_name = p_drv->driver_name; 577 for (;;newunit++) { 578 for (periph = TAILQ_FIRST(&p_drv->units); 579 periph != NULL && periph->unit_number != newunit; 580 periph = TAILQ_NEXT(periph, unit_links)) 581 ; 582 583 if (periph != NULL && periph->unit_number == newunit) { 584 if (wired) { 585 xpt_print(periph->path, "Duplicate Wired " 586 "Device entry!\n"); 587 xpt_print(periph->path, "Second device (%s " 588 "device at scbus%d target %d lun %d) will " 589 "not be wired\n", periph_name, pathid, 590 target, lun); 591 wired = false; 592 } 593 continue; 594 } 595 if (wired) 596 break; 597 598 /* 599 * Don't allow the mere presence of any attributes of a device 600 * means that it is for a wired down entry. Instead, insist that 601 * one of the matching criteria from camperiphunit be present 602 * for the device. 603 */ 604 i = 0; 605 dname = periph_name; 606 for (;;) { 607 r = resource_find_dev(&i, dname, &dunit, NULL, NULL); 608 if (r != 0) 609 break; 610 611 if (newunit != dunit) 612 continue; 613 if (resource_string_value(dname, dunit, "sn", &strval) == 0 || 614 resource_int_value(dname, dunit, "lun", &val) == 0 || 615 resource_int_value(dname, dunit, "target", &val) == 0 || 616 resource_string_value(dname, dunit, "at", &strval) == 0) 617 break; 618 } 619 if (r != 0) 620 break; 621 } 622 return (newunit); 623 } 624 625 static u_int 626 camperiphunit(struct periph_driver *p_drv, path_id_t pathid, 627 target_id_t target, lun_id_t lun, const char *sn) 628 { 629 bool wired = false; 630 u_int unit; 631 int i, val, dunit; 632 const char *dname, *strval; 633 char pathbuf[32], *periph_name; 634 635 periph_name = p_drv->driver_name; 636 snprintf(pathbuf, sizeof(pathbuf), "scbus%d", pathid); 637 unit = 0; 638 i = 0; 639 dname = periph_name; 640 641 for (wired = false; resource_find_dev(&i, dname, &dunit, NULL, NULL) == 0; 642 wired = false) { 643 if (resource_string_value(dname, dunit, "at", &strval) == 0) { 644 if (strcmp(strval, pathbuf) != 0) 645 continue; 646 wired = true; 647 } 648 if (resource_int_value(dname, dunit, "target", &val) == 0) { 649 if (val != target) 650 continue; 651 wired = true; 652 } 653 if (resource_int_value(dname, dunit, "lun", &val) == 0) { 654 if (val != lun) 655 continue; 656 wired = true; 657 } 658 if (resource_string_value(dname, dunit, "sn", &strval) == 0) { 659 if (sn == NULL || strcmp(strval, sn) != 0) 660 continue; 661 wired = true; 662 } 663 if (wired) { 664 unit = dunit; 665 break; 666 } 667 } 668 669 /* 670 * Either start from 0 looking for the next unit or from 671 * the unit number given in the resource config. This way, 672 * if we have wildcard matches, we don't return the same 673 * unit number twice. 674 */ 675 unit = camperiphnextunit(p_drv, unit, wired, pathid, target, lun); 676 677 return (unit); 678 } 679 680 static void 681 cam_periph_invalidate_devctl(struct cam_periph *periph) 682 { 683 struct sbuf sb; 684 char *sbmsg; 685 686 sbmsg = cam_periph_devctl_sb_init(&sb, periph); 687 if (sbmsg != NULL) 688 cam_periph_devctl_sb_fini(&sb, sbmsg, "invalidate"); 689 } 690 691 void 692 cam_periph_invalidate(struct cam_periph *periph) 693 { 694 695 cam_periph_assert(periph, MA_OWNED); 696 /* 697 * We only tear down the device the first time a peripheral is 698 * invalidated. 699 */ 700 if ((periph->flags & CAM_PERIPH_INVALID) != 0) 701 return; 702 703 CAM_PROBE1(periph, invalidate, periph); 704 CAM_DEBUG(periph->path, CAM_DEBUG_INFO, ("Periph invalidated\n")); 705 if (!rebooting) 706 cam_periph_invalidate_devctl(periph); 707 if ((periph->flags & CAM_PERIPH_ANNOUNCED) && !rebooting) { 708 struct sbuf sb; 709 char buffer[160]; 710 711 sbuf_new(&sb, buffer, 160, SBUF_FIXEDLEN); 712 xpt_denounce_periph_sbuf(periph, &sb); 713 sbuf_finish(&sb); 714 sbuf_putbuf(&sb); 715 } 716 periph->flags |= CAM_PERIPH_INVALID; 717 periph->flags &= ~CAM_PERIPH_NEW_DEV_FOUND; 718 if (periph->periph_oninval != NULL) 719 periph->periph_oninval(periph); 720 cam_periph_release_locked(periph); 721 } 722 723 static void 724 camperiphfree(struct cam_periph *periph) 725 { 726 struct periph_driver **p_drv; 727 struct periph_driver *drv; 728 729 cam_periph_assert(periph, MA_OWNED); 730 KASSERT(periph->periph_allocating == 0, ("%s%d: freed while allocating", 731 periph->periph_name, periph->unit_number)); 732 for (p_drv = periph_drivers; *p_drv != NULL; p_drv++) { 733 if (strcmp((*p_drv)->driver_name, periph->periph_name) == 0) 734 break; 735 } 736 if (*p_drv == NULL) { 737 printf("camperiphfree: attempt to free non-existant periph\n"); 738 return; 739 } 740 /* 741 * Cache a pointer to the periph_driver structure. If a 742 * periph_driver is added or removed from the array (see 743 * periphdriver_register()) while we drop the toplogy lock 744 * below, p_drv may change. This doesn't protect against this 745 * particular periph_driver going away. That will require full 746 * reference counting in the periph_driver infrastructure. 747 */ 748 drv = *p_drv; 749 750 /* 751 * We need to set this flag before dropping the topology lock, to 752 * let anyone who is traversing the list that this peripheral is 753 * about to be freed, and there will be no more reference count 754 * checks. 755 */ 756 periph->flags |= CAM_PERIPH_FREE; 757 758 /* 759 * The peripheral destructor semantics dictate calling with only the 760 * SIM mutex held. Since it might sleep, it should not be called 761 * with the topology lock held. 762 */ 763 xpt_unlock_buses(); 764 765 /* 766 * We need to call the peripheral destructor prior to removing the 767 * peripheral from the list. Otherwise, we risk running into a 768 * scenario where the peripheral unit number may get reused 769 * (because it has been removed from the list), but some resources 770 * used by the peripheral are still hanging around. In particular, 771 * the devfs nodes used by some peripherals like the pass(4) driver 772 * aren't fully cleaned up until the destructor is run. If the 773 * unit number is reused before the devfs instance is fully gone, 774 * devfs will panic. 775 */ 776 if (periph->periph_dtor != NULL) 777 periph->periph_dtor(periph); 778 779 /* 780 * The peripheral list is protected by the topology lock. We have to 781 * remove the periph from the drv list before we call deferred_ac. The 782 * AC_FOUND_DEVICE callback won't create a new periph if it's still there. 783 */ 784 xpt_lock_buses(); 785 786 TAILQ_REMOVE(&drv->units, periph, unit_links); 787 drv->generation++; 788 789 xpt_remove_periph(periph); 790 791 xpt_unlock_buses(); 792 if ((periph->flags & CAM_PERIPH_ANNOUNCED) && !rebooting) 793 xpt_print(periph->path, "Periph destroyed\n"); 794 else 795 CAM_DEBUG(periph->path, CAM_DEBUG_INFO, ("Periph destroyed\n")); 796 797 if (periph->flags & CAM_PERIPH_NEW_DEV_FOUND) { 798 switch (periph->deferred_ac) { 799 case AC_FOUND_DEVICE: { 800 struct ccb_getdev cgd; 801 802 xpt_gdev_type(&cgd, periph->path); 803 periph->deferred_callback(NULL, periph->deferred_ac, 804 periph->path, &cgd); 805 break; 806 } 807 case AC_PATH_REGISTERED: { 808 struct ccb_pathinq cpi; 809 810 xpt_path_inq(&cpi, periph->path); 811 periph->deferred_callback(NULL, periph->deferred_ac, 812 periph->path, &cpi); 813 break; 814 } 815 default: 816 periph->deferred_callback(NULL, periph->deferred_ac, 817 periph->path, NULL); 818 break; 819 } 820 } 821 xpt_free_path(periph->path); 822 free(periph, M_CAMPERIPH); 823 xpt_lock_buses(); 824 } 825 826 /* 827 * Map user virtual pointers into kernel virtual address space, so we can 828 * access the memory. This is now a generic function that centralizes most 829 * of the sanity checks on the data flags, if any. 830 * This also only works for up to maxphys memory. Since we use 831 * buffers to map stuff in and out, we're limited to the buffer size. 832 */ 833 int 834 cam_periph_mapmem(union ccb *ccb, struct cam_periph_map_info *mapinfo, 835 u_int maxmap) 836 { 837 int numbufs, i; 838 uint8_t **data_ptrs[CAM_PERIPH_MAXMAPS]; 839 uint32_t lengths[CAM_PERIPH_MAXMAPS]; 840 uint32_t dirs[CAM_PERIPH_MAXMAPS]; 841 842 bzero(mapinfo, sizeof(*mapinfo)); 843 if (maxmap == 0) 844 maxmap = DFLTPHYS; /* traditional default */ 845 else if (maxmap > maxphys) 846 maxmap = maxphys; /* for safety */ 847 switch(ccb->ccb_h.func_code) { 848 case XPT_DEV_MATCH: 849 if (ccb->cdm.match_buf_len == 0) { 850 printf("cam_periph_mapmem: invalid match buffer " 851 "length 0\n"); 852 return(EINVAL); 853 } 854 if (ccb->cdm.pattern_buf_len > 0) { 855 data_ptrs[0] = (uint8_t **)&ccb->cdm.patterns; 856 lengths[0] = ccb->cdm.pattern_buf_len; 857 dirs[0] = CAM_DIR_OUT; 858 data_ptrs[1] = (uint8_t **)&ccb->cdm.matches; 859 lengths[1] = ccb->cdm.match_buf_len; 860 dirs[1] = CAM_DIR_IN; 861 numbufs = 2; 862 } else { 863 data_ptrs[0] = (uint8_t **)&ccb->cdm.matches; 864 lengths[0] = ccb->cdm.match_buf_len; 865 dirs[0] = CAM_DIR_IN; 866 numbufs = 1; 867 } 868 /* 869 * This request will not go to the hardware, no reason 870 * to be so strict. vmapbuf() is able to map up to maxphys. 871 */ 872 maxmap = maxphys; 873 break; 874 case XPT_SCSI_IO: 875 case XPT_CONT_TARGET_IO: 876 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_NONE) 877 return(0); 878 if ((ccb->ccb_h.flags & CAM_DATA_MASK) != CAM_DATA_VADDR) 879 return (EINVAL); 880 data_ptrs[0] = &ccb->csio.data_ptr; 881 lengths[0] = ccb->csio.dxfer_len; 882 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 883 numbufs = 1; 884 break; 885 case XPT_ATA_IO: 886 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_NONE) 887 return(0); 888 if ((ccb->ccb_h.flags & CAM_DATA_MASK) != CAM_DATA_VADDR) 889 return (EINVAL); 890 data_ptrs[0] = &ccb->ataio.data_ptr; 891 lengths[0] = ccb->ataio.dxfer_len; 892 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 893 numbufs = 1; 894 break; 895 case XPT_MMC_IO: 896 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_NONE) 897 return(0); 898 /* Two mappings: one for cmd->data and one for cmd->data->data */ 899 data_ptrs[0] = (unsigned char **)&ccb->mmcio.cmd.data; 900 lengths[0] = sizeof(struct mmc_data *); 901 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 902 data_ptrs[1] = (unsigned char **)&ccb->mmcio.cmd.data->data; 903 lengths[1] = ccb->mmcio.cmd.data->len; 904 dirs[1] = ccb->ccb_h.flags & CAM_DIR_MASK; 905 numbufs = 2; 906 break; 907 case XPT_SMP_IO: 908 data_ptrs[0] = &ccb->smpio.smp_request; 909 lengths[0] = ccb->smpio.smp_request_len; 910 dirs[0] = CAM_DIR_OUT; 911 data_ptrs[1] = &ccb->smpio.smp_response; 912 lengths[1] = ccb->smpio.smp_response_len; 913 dirs[1] = CAM_DIR_IN; 914 numbufs = 2; 915 break; 916 case XPT_NVME_IO: 917 case XPT_NVME_ADMIN: 918 if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_NONE) 919 return (0); 920 if ((ccb->ccb_h.flags & CAM_DATA_MASK) != CAM_DATA_VADDR) 921 return (EINVAL); 922 data_ptrs[0] = &ccb->nvmeio.data_ptr; 923 lengths[0] = ccb->nvmeio.dxfer_len; 924 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 925 numbufs = 1; 926 break; 927 case XPT_DEV_ADVINFO: 928 if (ccb->cdai.bufsiz == 0) 929 return (0); 930 931 data_ptrs[0] = (uint8_t **)&ccb->cdai.buf; 932 lengths[0] = ccb->cdai.bufsiz; 933 dirs[0] = CAM_DIR_IN; 934 numbufs = 1; 935 936 /* 937 * This request will not go to the hardware, no reason 938 * to be so strict. vmapbuf() is able to map up to maxphys. 939 */ 940 maxmap = maxphys; 941 break; 942 default: 943 return(EINVAL); 944 break; /* NOTREACHED */ 945 } 946 947 /* 948 * Check the transfer length and permissions first, so we don't 949 * have to unmap any previously mapped buffers. 950 */ 951 for (i = 0; i < numbufs; i++) { 952 if (lengths[i] > maxmap) { 953 printf("cam_periph_mapmem: attempt to map %lu bytes, " 954 "which is greater than %lu\n", 955 (long)(lengths[i]), (u_long)maxmap); 956 return (E2BIG); 957 } 958 } 959 960 for (i = 0; i < numbufs; i++) { 961 /* Save the user's data address. */ 962 mapinfo->orig[i] = *data_ptrs[i]; 963 964 /* 965 * For small buffers use malloc+copyin/copyout instead of 966 * mapping to KVA to avoid expensive TLB shootdowns. For 967 * small allocations malloc is backed by UMA, and so much 968 * cheaper on SMP systems. 969 */ 970 if (lengths[i] <= periph_mapmem_thresh && 971 ccb->ccb_h.func_code != XPT_MMC_IO) { 972 *data_ptrs[i] = malloc(lengths[i], M_CAMPERIPH, 973 M_WAITOK); 974 if (dirs[i] != CAM_DIR_IN) { 975 if (copyin(mapinfo->orig[i], *data_ptrs[i], 976 lengths[i]) != 0) { 977 free(*data_ptrs[i], M_CAMPERIPH); 978 *data_ptrs[i] = mapinfo->orig[i]; 979 goto fail; 980 } 981 } else 982 bzero(*data_ptrs[i], lengths[i]); 983 continue; 984 } 985 986 /* 987 * Get the buffer. 988 */ 989 mapinfo->bp[i] = uma_zalloc(pbuf_zone, M_WAITOK); 990 991 /* set the direction */ 992 mapinfo->bp[i]->b_iocmd = (dirs[i] == CAM_DIR_OUT) ? 993 BIO_WRITE : BIO_READ; 994 995 /* Map the buffer into kernel memory. */ 996 if (vmapbuf(mapinfo->bp[i], *data_ptrs[i], lengths[i], 1) < 0) { 997 uma_zfree(pbuf_zone, mapinfo->bp[i]); 998 goto fail; 999 } 1000 1001 /* set our pointer to the new mapped area */ 1002 *data_ptrs[i] = mapinfo->bp[i]->b_data; 1003 } 1004 1005 /* 1006 * Now that we've gotten this far, change ownership to the kernel 1007 * of the buffers so that we don't run afoul of returning to user 1008 * space with locks (on the buffer) held. 1009 */ 1010 for (i = 0; i < numbufs; i++) { 1011 if (mapinfo->bp[i]) 1012 BUF_KERNPROC(mapinfo->bp[i]); 1013 } 1014 1015 mapinfo->num_bufs_used = numbufs; 1016 return(0); 1017 1018 fail: 1019 for (i--; i >= 0; i--) { 1020 if (mapinfo->bp[i]) { 1021 vunmapbuf(mapinfo->bp[i]); 1022 uma_zfree(pbuf_zone, mapinfo->bp[i]); 1023 } else 1024 free(*data_ptrs[i], M_CAMPERIPH); 1025 *data_ptrs[i] = mapinfo->orig[i]; 1026 } 1027 return(EACCES); 1028 } 1029 1030 /* 1031 * Unmap memory segments mapped into kernel virtual address space by 1032 * cam_periph_mapmem(). 1033 */ 1034 int 1035 cam_periph_unmapmem(union ccb *ccb, struct cam_periph_map_info *mapinfo) 1036 { 1037 int error, numbufs, i; 1038 uint8_t **data_ptrs[CAM_PERIPH_MAXMAPS]; 1039 uint32_t lengths[CAM_PERIPH_MAXMAPS]; 1040 uint32_t dirs[CAM_PERIPH_MAXMAPS]; 1041 1042 if (mapinfo->num_bufs_used <= 0) { 1043 /* nothing to free and the process wasn't held. */ 1044 return (0); 1045 } 1046 1047 switch (ccb->ccb_h.func_code) { 1048 case XPT_DEV_MATCH: 1049 if (ccb->cdm.pattern_buf_len > 0) { 1050 data_ptrs[0] = (uint8_t **)&ccb->cdm.patterns; 1051 lengths[0] = ccb->cdm.pattern_buf_len; 1052 dirs[0] = CAM_DIR_OUT; 1053 data_ptrs[1] = (uint8_t **)&ccb->cdm.matches; 1054 lengths[1] = ccb->cdm.match_buf_len; 1055 dirs[1] = CAM_DIR_IN; 1056 numbufs = 2; 1057 } else { 1058 data_ptrs[0] = (uint8_t **)&ccb->cdm.matches; 1059 lengths[0] = ccb->cdm.match_buf_len; 1060 dirs[0] = CAM_DIR_IN; 1061 numbufs = 1; 1062 } 1063 break; 1064 case XPT_SCSI_IO: 1065 case XPT_CONT_TARGET_IO: 1066 data_ptrs[0] = &ccb->csio.data_ptr; 1067 lengths[0] = ccb->csio.dxfer_len; 1068 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 1069 numbufs = 1; 1070 break; 1071 case XPT_ATA_IO: 1072 data_ptrs[0] = &ccb->ataio.data_ptr; 1073 lengths[0] = ccb->ataio.dxfer_len; 1074 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 1075 numbufs = 1; 1076 break; 1077 case XPT_MMC_IO: 1078 data_ptrs[0] = (uint8_t **)&ccb->mmcio.cmd.data; 1079 lengths[0] = sizeof(struct mmc_data *); 1080 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 1081 data_ptrs[1] = (uint8_t **)&ccb->mmcio.cmd.data->data; 1082 lengths[1] = ccb->mmcio.cmd.data->len; 1083 dirs[1] = ccb->ccb_h.flags & CAM_DIR_MASK; 1084 numbufs = 2; 1085 break; 1086 case XPT_SMP_IO: 1087 data_ptrs[0] = &ccb->smpio.smp_request; 1088 lengths[0] = ccb->smpio.smp_request_len; 1089 dirs[0] = CAM_DIR_OUT; 1090 data_ptrs[1] = &ccb->smpio.smp_response; 1091 lengths[1] = ccb->smpio.smp_response_len; 1092 dirs[1] = CAM_DIR_IN; 1093 numbufs = 2; 1094 break; 1095 case XPT_NVME_IO: 1096 case XPT_NVME_ADMIN: 1097 data_ptrs[0] = &ccb->nvmeio.data_ptr; 1098 lengths[0] = ccb->nvmeio.dxfer_len; 1099 dirs[0] = ccb->ccb_h.flags & CAM_DIR_MASK; 1100 numbufs = 1; 1101 break; 1102 case XPT_DEV_ADVINFO: 1103 data_ptrs[0] = (uint8_t **)&ccb->cdai.buf; 1104 lengths[0] = ccb->cdai.bufsiz; 1105 dirs[0] = CAM_DIR_IN; 1106 numbufs = 1; 1107 break; 1108 default: 1109 numbufs = 0; 1110 break; 1111 } 1112 1113 error = 0; 1114 for (i = 0; i < numbufs; i++) { 1115 if (mapinfo->bp[i]) { 1116 /* unmap the buffer */ 1117 vunmapbuf(mapinfo->bp[i]); 1118 1119 /* release the buffer */ 1120 uma_zfree(pbuf_zone, mapinfo->bp[i]); 1121 } else { 1122 if (dirs[i] != CAM_DIR_OUT) { 1123 int error1; 1124 1125 error1 = copyout(*data_ptrs[i], mapinfo->orig[i], 1126 lengths[i]); 1127 if (error == 0) 1128 error = error1; 1129 } 1130 free(*data_ptrs[i], M_CAMPERIPH); 1131 } 1132 1133 /* Set the user's pointer back to the original value */ 1134 *data_ptrs[i] = mapinfo->orig[i]; 1135 } 1136 1137 return (error); 1138 } 1139 1140 int 1141 cam_periph_ioctl(struct cam_periph *periph, u_long cmd, caddr_t addr, 1142 int (*error_routine)(union ccb *ccb, 1143 cam_flags camflags, 1144 uint32_t sense_flags)) 1145 { 1146 union ccb *ccb; 1147 int error; 1148 int found; 1149 1150 error = found = 0; 1151 1152 switch(cmd){ 1153 case CAMGETPASSTHRU_0x19: 1154 case CAMGETPASSTHRU: 1155 ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL); 1156 xpt_setup_ccb(&ccb->ccb_h, 1157 ccb->ccb_h.path, 1158 CAM_PRIORITY_NORMAL); 1159 ccb->ccb_h.func_code = XPT_GDEVLIST; 1160 1161 /* 1162 * Basically, the point of this is that we go through 1163 * getting the list of devices, until we find a passthrough 1164 * device. In the current version of the CAM code, the 1165 * only way to determine what type of device we're dealing 1166 * with is by its name. 1167 */ 1168 while (found == 0) { 1169 ccb->cgdl.index = 0; 1170 ccb->cgdl.status = CAM_GDEVLIST_MORE_DEVS; 1171 while (ccb->cgdl.status == CAM_GDEVLIST_MORE_DEVS) { 1172 /* we want the next device in the list */ 1173 xpt_action(ccb); 1174 if (strncmp(ccb->cgdl.periph_name, 1175 "pass", 4) == 0){ 1176 found = 1; 1177 break; 1178 } 1179 } 1180 if ((ccb->cgdl.status == CAM_GDEVLIST_LAST_DEVICE) && 1181 (found == 0)) { 1182 ccb->cgdl.periph_name[0] = '\0'; 1183 ccb->cgdl.unit_number = 0; 1184 break; 1185 } 1186 } 1187 1188 /* copy the result back out */ 1189 bcopy(ccb, addr, sizeof(union ccb)); 1190 1191 /* and release the ccb */ 1192 xpt_release_ccb(ccb); 1193 1194 break; 1195 default: 1196 error = ENOTTY; 1197 break; 1198 } 1199 return(error); 1200 } 1201 1202 static void 1203 cam_periph_done_panic(struct cam_periph *periph, union ccb *done_ccb) 1204 { 1205 1206 panic("%s: already done with ccb %p", __func__, done_ccb); 1207 } 1208 1209 static void 1210 cam_periph_done(struct cam_periph *periph, union ccb *done_ccb) 1211 { 1212 1213 /* Caller will release the CCB */ 1214 xpt_path_assert(done_ccb->ccb_h.path, MA_OWNED); 1215 done_ccb->ccb_h.cbfcnp = cam_periph_done_panic; 1216 wakeup(&done_ccb->ccb_h.cbfcnp); 1217 } 1218 1219 static void 1220 cam_periph_ccbwait(union ccb *ccb) 1221 { 1222 1223 if ((ccb->ccb_h.func_code & XPT_FC_QUEUED) != 0) { 1224 while (ccb->ccb_h.cbfcnp != cam_periph_done_panic) 1225 xpt_path_sleep(ccb->ccb_h.path, &ccb->ccb_h.cbfcnp, 1226 PRIBIO, "cbwait", 0); 1227 } 1228 KASSERT(ccb->ccb_h.pinfo.index == CAM_UNQUEUED_INDEX && 1229 (ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_INPROG, 1230 ("%s: proceeding with incomplete ccb: ccb=%p, func_code=%#x, " 1231 "status=%#x, index=%d", __func__, ccb, ccb->ccb_h.func_code, 1232 ccb->ccb_h.status, ccb->ccb_h.pinfo.index)); 1233 } 1234 1235 /* 1236 * Dispatch a CCB and wait for it to complete. If the CCB has set a 1237 * callback function (ccb->ccb_h.cbfcnp), it will be overwritten and lost. 1238 */ 1239 int 1240 cam_periph_runccb(union ccb *ccb, 1241 int (*error_routine)(union ccb *ccb, 1242 cam_flags camflags, 1243 uint32_t sense_flags), 1244 cam_flags camflags, uint32_t sense_flags, 1245 struct devstat *ds) 1246 { 1247 struct bintime *starttime; 1248 struct bintime ltime; 1249 int error; 1250 bool must_poll; 1251 uint32_t timeout = 1; 1252 1253 starttime = NULL; 1254 xpt_path_assert(ccb->ccb_h.path, MA_OWNED); 1255 KASSERT((ccb->ccb_h.flags & CAM_UNLOCKED) == 0, 1256 ("%s: ccb=%p, func_code=%#x, flags=%#x", __func__, ccb, 1257 ccb->ccb_h.func_code, ccb->ccb_h.flags)); 1258 1259 /* 1260 * If the user has supplied a stats structure, and if we understand 1261 * this particular type of ccb, record the transaction start. 1262 */ 1263 if (ds != NULL && 1264 (ccb->ccb_h.func_code == XPT_SCSI_IO || 1265 ccb->ccb_h.func_code == XPT_ATA_IO || 1266 ccb->ccb_h.func_code == XPT_NVME_IO)) { 1267 starttime = <ime; 1268 binuptime(starttime); 1269 devstat_start_transaction(ds, starttime); 1270 } 1271 1272 /* 1273 * We must poll the I/O while we're dumping. The scheduler is normally 1274 * stopped for dumping, except when we call doadump from ddb. While the 1275 * scheduler is running in this case, we still need to poll the I/O to 1276 * avoid sleeping waiting for the ccb to complete. 1277 * 1278 * A panic triggered dump stops the scheduler, any callback from the 1279 * shutdown_post_sync event will run with the scheduler stopped, but 1280 * before we're officially dumping. To avoid hanging in adashutdown 1281 * initiated commands (or other similar situations), we have to test for 1282 * either dumping or SCHEDULER_STOPPED() here. 1283 * 1284 * To avoid locking problems, dumping/polling callers must call 1285 * without a periph lock held. 1286 */ 1287 must_poll = dumping || SCHEDULER_STOPPED(); 1288 ccb->ccb_h.cbfcnp = cam_periph_done; 1289 1290 /* 1291 * If we're polling, then we need to ensure that we have ample resources 1292 * in the periph. cam_periph_error can reschedule the ccb by calling 1293 * xpt_action and returning ERESTART, so we have to effect the polling 1294 * in the do loop below. 1295 */ 1296 if (must_poll) { 1297 if (cam_sim_pollable(ccb->ccb_h.path->bus->sim)) 1298 timeout = xpt_poll_setup(ccb); 1299 else 1300 timeout = 0; 1301 } 1302 1303 if (timeout == 0) { 1304 ccb->ccb_h.status = CAM_RESRC_UNAVAIL; 1305 error = EBUSY; 1306 } else { 1307 xpt_action(ccb); 1308 do { 1309 if (must_poll) { 1310 xpt_pollwait(ccb, timeout); 1311 timeout = ccb->ccb_h.timeout * 10; 1312 } else { 1313 cam_periph_ccbwait(ccb); 1314 } 1315 if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) 1316 error = 0; 1317 else if (error_routine != NULL) { 1318 /* 1319 * cbfcnp is modified by cam_periph_ccbwait so 1320 * reset it before we call the error routine 1321 * which may call xpt_done. 1322 */ 1323 ccb->ccb_h.cbfcnp = cam_periph_done; 1324 error = (*error_routine)(ccb, camflags, sense_flags); 1325 } else 1326 error = 0; 1327 } while (error == ERESTART); 1328 } 1329 1330 if ((ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) { 1331 cam_release_devq(ccb->ccb_h.path, 1332 /* relsim_flags */0, 1333 /* openings */0, 1334 /* timeout */0, 1335 /* getcount_only */ FALSE); 1336 ccb->ccb_h.status &= ~CAM_DEV_QFRZN; 1337 } 1338 1339 if (ds != NULL) { 1340 uint32_t bytes; 1341 devstat_tag_type tag; 1342 bool valid = true; 1343 1344 if (ccb->ccb_h.func_code == XPT_SCSI_IO) { 1345 bytes = ccb->csio.dxfer_len - ccb->csio.resid; 1346 tag = (devstat_tag_type)(ccb->csio.tag_action & 0x3); 1347 } else if (ccb->ccb_h.func_code == XPT_ATA_IO) { 1348 bytes = ccb->ataio.dxfer_len - ccb->ataio.resid; 1349 tag = (devstat_tag_type)0; 1350 } else if (ccb->ccb_h.func_code == XPT_NVME_IO) { 1351 bytes = ccb->nvmeio.dxfer_len; /* NB: resid no possible */ 1352 tag = (devstat_tag_type)0; 1353 } else { 1354 valid = false; 1355 } 1356 if (valid) 1357 devstat_end_transaction(ds, bytes, tag, 1358 ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_NONE) ? 1359 DEVSTAT_NO_DATA : (ccb->ccb_h.flags & CAM_DIR_OUT) ? 1360 DEVSTAT_WRITE : DEVSTAT_READ, NULL, starttime); 1361 } 1362 1363 return(error); 1364 } 1365 1366 void 1367 cam_freeze_devq(struct cam_path *path) 1368 { 1369 struct ccb_hdr ccb_h; 1370 1371 CAM_DEBUG(path, CAM_DEBUG_TRACE, ("cam_freeze_devq\n")); 1372 memset(&ccb_h, 0, sizeof(ccb_h)); 1373 xpt_setup_ccb(&ccb_h, path, /*priority*/1); 1374 ccb_h.func_code = XPT_NOOP; 1375 ccb_h.flags = CAM_DEV_QFREEZE; 1376 xpt_action((union ccb *)&ccb_h); 1377 } 1378 1379 uint32_t 1380 cam_release_devq(struct cam_path *path, uint32_t relsim_flags, 1381 uint32_t openings, uint32_t arg, 1382 int getcount_only) 1383 { 1384 struct ccb_relsim crs; 1385 1386 CAM_DEBUG(path, CAM_DEBUG_TRACE, ("cam_release_devq(%u, %u, %u, %d)\n", 1387 relsim_flags, openings, arg, getcount_only)); 1388 memset(&crs, 0, sizeof(crs)); 1389 xpt_setup_ccb(&crs.ccb_h, path, CAM_PRIORITY_NORMAL); 1390 crs.ccb_h.func_code = XPT_REL_SIMQ; 1391 crs.ccb_h.flags = getcount_only ? CAM_DEV_QFREEZE : 0; 1392 crs.release_flags = relsim_flags; 1393 crs.openings = openings; 1394 crs.release_timeout = arg; 1395 xpt_action((union ccb *)&crs); 1396 return (crs.qfrozen_cnt); 1397 } 1398 1399 #define saved_ccb_ptr ppriv_ptr0 1400 static void 1401 camperiphdone(struct cam_periph *periph, union ccb *done_ccb) 1402 { 1403 union ccb *saved_ccb; 1404 cam_status status; 1405 struct scsi_start_stop_unit *scsi_cmd; 1406 int error = 0, error_code, sense_key, asc, ascq; 1407 uint16_t done_flags; 1408 1409 scsi_cmd = (struct scsi_start_stop_unit *) 1410 &done_ccb->csio.cdb_io.cdb_bytes; 1411 status = done_ccb->ccb_h.status; 1412 1413 if ((status & CAM_STATUS_MASK) != CAM_REQ_CMP) { 1414 if (scsi_extract_sense_ccb(done_ccb, 1415 &error_code, &sense_key, &asc, &ascq)) { 1416 /* 1417 * If the error is "invalid field in CDB", 1418 * and the load/eject flag is set, turn the 1419 * flag off and try again. This is just in 1420 * case the drive in question barfs on the 1421 * load eject flag. The CAM code should set 1422 * the load/eject flag by default for 1423 * removable media. 1424 */ 1425 if ((scsi_cmd->opcode == START_STOP_UNIT) && 1426 ((scsi_cmd->how & SSS_LOEJ) != 0) && 1427 (asc == 0x24) && (ascq == 0x00)) { 1428 scsi_cmd->how &= ~SSS_LOEJ; 1429 if (status & CAM_DEV_QFRZN) { 1430 cam_release_devq(done_ccb->ccb_h.path, 1431 0, 0, 0, 0); 1432 done_ccb->ccb_h.status &= 1433 ~CAM_DEV_QFRZN; 1434 } 1435 xpt_action(done_ccb); 1436 goto out; 1437 } 1438 } 1439 error = cam_periph_error(done_ccb, 0, 1440 SF_RETRY_UA | SF_NO_PRINT); 1441 if (error == ERESTART) 1442 goto out; 1443 if (done_ccb->ccb_h.status & CAM_DEV_QFRZN) { 1444 cam_release_devq(done_ccb->ccb_h.path, 0, 0, 0, 0); 1445 done_ccb->ccb_h.status &= ~CAM_DEV_QFRZN; 1446 } 1447 } else { 1448 /* 1449 * If we have successfully taken a device from the not 1450 * ready to ready state, re-scan the device and re-get 1451 * the inquiry information. Many devices (mostly disks) 1452 * don't properly report their inquiry information unless 1453 * they are spun up. 1454 */ 1455 if (scsi_cmd->opcode == START_STOP_UNIT) 1456 xpt_async(AC_INQ_CHANGED, done_ccb->ccb_h.path, NULL); 1457 } 1458 1459 /* If we tried long wait and still failed, remember that. */ 1460 if ((periph->flags & CAM_PERIPH_RECOVERY_WAIT) && 1461 (done_ccb->csio.cdb_io.cdb_bytes[0] == TEST_UNIT_READY)) { 1462 periph->flags &= ~CAM_PERIPH_RECOVERY_WAIT; 1463 if (error != 0 && done_ccb->ccb_h.retry_count == 0) 1464 periph->flags |= CAM_PERIPH_RECOVERY_WAIT_FAILED; 1465 } 1466 1467 /* 1468 * After recovery action(s) completed, return to the original CCB. 1469 * If the recovery CCB has failed, considering its own possible 1470 * retries and recovery, assume we are back in state where we have 1471 * been originally, but without recovery hopes left. In such case, 1472 * after the final attempt below, we cancel any further retries, 1473 * blocking by that also any new recovery attempts for this CCB, 1474 * and the result will be the final one returned to the CCB owher. 1475 */ 1476 saved_ccb = (union ccb *)done_ccb->ccb_h.saved_ccb_ptr; 1477 KASSERT(saved_ccb->ccb_h.func_code == XPT_SCSI_IO, 1478 ("%s: saved_ccb func_code %#x != XPT_SCSI_IO", 1479 __func__, saved_ccb->ccb_h.func_code)); 1480 KASSERT(done_ccb->ccb_h.func_code == XPT_SCSI_IO, 1481 ("%s: done_ccb func_code %#x != XPT_SCSI_IO", 1482 __func__, done_ccb->ccb_h.func_code)); 1483 saved_ccb->ccb_h.periph_links = done_ccb->ccb_h.periph_links; 1484 done_flags = done_ccb->ccb_h.alloc_flags; 1485 bcopy(saved_ccb, done_ccb, sizeof(struct ccb_scsiio)); 1486 done_ccb->ccb_h.alloc_flags = done_flags; 1487 xpt_free_ccb(saved_ccb); 1488 if (done_ccb->ccb_h.cbfcnp != camperiphdone) 1489 periph->flags &= ~CAM_PERIPH_RECOVERY_INPROG; 1490 if (error != 0) 1491 done_ccb->ccb_h.retry_count = 0; 1492 xpt_action(done_ccb); 1493 1494 out: 1495 /* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */ 1496 cam_release_devq(done_ccb->ccb_h.path, 0, 0, 0, 0); 1497 } 1498 1499 /* 1500 * Generic Async Event handler. Peripheral drivers usually 1501 * filter out the events that require personal attention, 1502 * and leave the rest to this function. 1503 */ 1504 void 1505 cam_periph_async(struct cam_periph *periph, uint32_t code, 1506 struct cam_path *path, void *arg) 1507 { 1508 switch (code) { 1509 case AC_LOST_DEVICE: 1510 cam_periph_invalidate(periph); 1511 break; 1512 default: 1513 break; 1514 } 1515 } 1516 1517 void 1518 cam_periph_bus_settle(struct cam_periph *periph, u_int bus_settle) 1519 { 1520 struct ccb_getdevstats cgds; 1521 1522 memset(&cgds, 0, sizeof(cgds)); 1523 xpt_setup_ccb(&cgds.ccb_h, periph->path, CAM_PRIORITY_NORMAL); 1524 cgds.ccb_h.func_code = XPT_GDEV_STATS; 1525 xpt_action((union ccb *)&cgds); 1526 cam_periph_freeze_after_event(periph, &cgds.last_reset, bus_settle); 1527 } 1528 1529 void 1530 cam_periph_freeze_after_event(struct cam_periph *periph, 1531 struct timeval* event_time, u_int duration_ms) 1532 { 1533 struct timeval delta; 1534 struct timeval duration_tv; 1535 1536 if (!timevalisset(event_time)) 1537 return; 1538 1539 microtime(&delta); 1540 timevalsub(&delta, event_time); 1541 duration_tv.tv_sec = duration_ms / 1000; 1542 duration_tv.tv_usec = (duration_ms % 1000) * 1000; 1543 if (timevalcmp(&delta, &duration_tv, <)) { 1544 timevalsub(&duration_tv, &delta); 1545 1546 duration_ms = duration_tv.tv_sec * 1000; 1547 duration_ms += duration_tv.tv_usec / 1000; 1548 cam_freeze_devq(periph->path); 1549 cam_release_devq(periph->path, 1550 RELSIM_RELEASE_AFTER_TIMEOUT, 1551 /*reduction*/0, 1552 /*timeout*/duration_ms, 1553 /*getcount_only*/0); 1554 } 1555 1556 } 1557 1558 static int 1559 camperiphscsistatuserror(union ccb *ccb, union ccb **orig_ccb, 1560 cam_flags camflags, uint32_t sense_flags, 1561 int *openings, uint32_t *relsim_flags, 1562 uint32_t *timeout, uint32_t *action, const char **action_string) 1563 { 1564 struct cam_periph *periph; 1565 int error; 1566 1567 switch (ccb->csio.scsi_status) { 1568 case SCSI_STATUS_OK: 1569 case SCSI_STATUS_COND_MET: 1570 case SCSI_STATUS_INTERMED: 1571 case SCSI_STATUS_INTERMED_COND_MET: 1572 error = 0; 1573 break; 1574 case SCSI_STATUS_CMD_TERMINATED: 1575 case SCSI_STATUS_CHECK_COND: 1576 error = camperiphscsisenseerror(ccb, orig_ccb, 1577 camflags, 1578 sense_flags, 1579 openings, 1580 relsim_flags, 1581 timeout, 1582 action, 1583 action_string); 1584 break; 1585 case SCSI_STATUS_QUEUE_FULL: 1586 { 1587 /* no decrement */ 1588 struct ccb_getdevstats cgds; 1589 1590 /* 1591 * First off, find out what the current 1592 * transaction counts are. 1593 */ 1594 memset(&cgds, 0, sizeof(cgds)); 1595 xpt_setup_ccb(&cgds.ccb_h, 1596 ccb->ccb_h.path, 1597 CAM_PRIORITY_NORMAL); 1598 cgds.ccb_h.func_code = XPT_GDEV_STATS; 1599 xpt_action((union ccb *)&cgds); 1600 1601 /* 1602 * If we were the only transaction active, treat 1603 * the QUEUE FULL as if it were a BUSY condition. 1604 */ 1605 if (cgds.dev_active != 0) { 1606 int total_openings; 1607 1608 /* 1609 * Reduce the number of openings to 1610 * be 1 less than the amount it took 1611 * to get a queue full bounded by the 1612 * minimum allowed tag count for this 1613 * device. 1614 */ 1615 total_openings = cgds.dev_active + cgds.dev_openings; 1616 *openings = cgds.dev_active; 1617 if (*openings < cgds.mintags) 1618 *openings = cgds.mintags; 1619 if (*openings < total_openings) 1620 *relsim_flags = RELSIM_ADJUST_OPENINGS; 1621 else { 1622 /* 1623 * Some devices report queue full for 1624 * temporary resource shortages. For 1625 * this reason, we allow a minimum 1626 * tag count to be entered via a 1627 * quirk entry to prevent the queue 1628 * count on these devices from falling 1629 * to a pessimisticly low value. We 1630 * still wait for the next successful 1631 * completion, however, before queueing 1632 * more transactions to the device. 1633 */ 1634 *relsim_flags = RELSIM_RELEASE_AFTER_CMDCMPLT; 1635 } 1636 *timeout = 0; 1637 error = ERESTART; 1638 *action &= ~SSQ_PRINT_SENSE; 1639 break; 1640 } 1641 /* FALLTHROUGH */ 1642 } 1643 case SCSI_STATUS_BUSY: 1644 /* 1645 * Restart the queue after either another 1646 * command completes or a 1 second timeout. 1647 */ 1648 periph = xpt_path_periph(ccb->ccb_h.path); 1649 if (periph->flags & CAM_PERIPH_INVALID) { 1650 error = ENXIO; 1651 *action_string = "Periph was invalidated"; 1652 } else if ((sense_flags & SF_RETRY_BUSY) != 0 || 1653 ccb->ccb_h.retry_count > 0) { 1654 if ((sense_flags & SF_RETRY_BUSY) == 0) 1655 ccb->ccb_h.retry_count--; 1656 error = ERESTART; 1657 *relsim_flags = RELSIM_RELEASE_AFTER_TIMEOUT 1658 | RELSIM_RELEASE_AFTER_CMDCMPLT; 1659 *timeout = 1000; 1660 } else { 1661 error = EIO; 1662 *action_string = "Retries exhausted"; 1663 } 1664 break; 1665 case SCSI_STATUS_RESERV_CONFLICT: 1666 default: 1667 error = EIO; 1668 break; 1669 } 1670 return (error); 1671 } 1672 1673 static int 1674 camperiphscsisenseerror(union ccb *ccb, union ccb **orig, 1675 cam_flags camflags, uint32_t sense_flags, 1676 int *openings, uint32_t *relsim_flags, 1677 uint32_t *timeout, uint32_t *action, const char **action_string) 1678 { 1679 struct cam_periph *periph; 1680 union ccb *orig_ccb = ccb; 1681 int error, recoveryccb; 1682 uint16_t flags; 1683 1684 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING) 1685 if (ccb->ccb_h.func_code == XPT_SCSI_IO && ccb->csio.bio != NULL) 1686 biotrack(ccb->csio.bio, __func__); 1687 #endif 1688 1689 periph = xpt_path_periph(ccb->ccb_h.path); 1690 recoveryccb = (ccb->ccb_h.cbfcnp == camperiphdone); 1691 if ((periph->flags & CAM_PERIPH_RECOVERY_INPROG) && !recoveryccb) { 1692 /* 1693 * If error recovery is already in progress, don't attempt 1694 * to process this error, but requeue it unconditionally 1695 * and attempt to process it once error recovery has 1696 * completed. This failed command is probably related to 1697 * the error that caused the currently active error recovery 1698 * action so our current recovery efforts should also 1699 * address this command. Be aware that the error recovery 1700 * code assumes that only one recovery action is in progress 1701 * on a particular peripheral instance at any given time 1702 * (e.g. only one saved CCB for error recovery) so it is 1703 * imperitive that we don't violate this assumption. 1704 */ 1705 error = ERESTART; 1706 *action &= ~SSQ_PRINT_SENSE; 1707 } else { 1708 scsi_sense_action err_action; 1709 struct ccb_getdev cgd; 1710 1711 /* 1712 * Grab the inquiry data for this device. 1713 */ 1714 xpt_gdev_type(&cgd, ccb->ccb_h.path); 1715 1716 err_action = scsi_error_action(&ccb->csio, &cgd.inq_data, 1717 sense_flags); 1718 error = err_action & SS_ERRMASK; 1719 1720 /* 1721 * Do not autostart sequential access devices 1722 * to avoid unexpected tape loading. 1723 */ 1724 if ((err_action & SS_MASK) == SS_START && 1725 SID_TYPE(&cgd.inq_data) == T_SEQUENTIAL) { 1726 *action_string = "Will not autostart a " 1727 "sequential access device"; 1728 goto sense_error_done; 1729 } 1730 1731 /* 1732 * Avoid recovery recursion if recovery action is the same. 1733 */ 1734 if ((err_action & SS_MASK) >= SS_START && recoveryccb) { 1735 if (((err_action & SS_MASK) == SS_START && 1736 ccb->csio.cdb_io.cdb_bytes[0] == START_STOP_UNIT) || 1737 ((err_action & SS_MASK) == SS_TUR && 1738 (ccb->csio.cdb_io.cdb_bytes[0] == TEST_UNIT_READY))) { 1739 err_action = SS_RETRY|SSQ_DECREMENT_COUNT|EIO; 1740 *relsim_flags = RELSIM_RELEASE_AFTER_TIMEOUT; 1741 *timeout = 500; 1742 } 1743 } 1744 1745 /* 1746 * If the recovery action will consume a retry, 1747 * make sure we actually have retries available. 1748 */ 1749 if ((err_action & SSQ_DECREMENT_COUNT) != 0) { 1750 if (ccb->ccb_h.retry_count > 0 && 1751 (periph->flags & CAM_PERIPH_INVALID) == 0) 1752 ccb->ccb_h.retry_count--; 1753 else { 1754 *action_string = "Retries exhausted"; 1755 goto sense_error_done; 1756 } 1757 } 1758 1759 if ((err_action & SS_MASK) >= SS_START) { 1760 /* 1761 * Do common portions of commands that 1762 * use recovery CCBs. 1763 */ 1764 orig_ccb = xpt_alloc_ccb_nowait(); 1765 if (orig_ccb == NULL) { 1766 *action_string = "Can't allocate recovery CCB"; 1767 goto sense_error_done; 1768 } 1769 /* 1770 * Clear freeze flag for original request here, as 1771 * this freeze will be dropped as part of ERESTART. 1772 */ 1773 ccb->ccb_h.status &= ~CAM_DEV_QFRZN; 1774 1775 KASSERT(ccb->ccb_h.func_code == XPT_SCSI_IO, 1776 ("%s: ccb func_code %#x != XPT_SCSI_IO", 1777 __func__, ccb->ccb_h.func_code)); 1778 flags = orig_ccb->ccb_h.alloc_flags; 1779 bcopy(ccb, orig_ccb, sizeof(struct ccb_scsiio)); 1780 orig_ccb->ccb_h.alloc_flags = flags; 1781 } 1782 1783 switch (err_action & SS_MASK) { 1784 case SS_NOP: 1785 *action_string = "No recovery action needed"; 1786 error = 0; 1787 break; 1788 case SS_RETRY: 1789 *action_string = "Retrying command (per sense data)"; 1790 error = ERESTART; 1791 break; 1792 case SS_FAIL: 1793 *action_string = "Unretryable error"; 1794 break; 1795 case SS_START: 1796 { 1797 int le; 1798 1799 /* 1800 * Send a start unit command to the device, and 1801 * then retry the command. 1802 */ 1803 *action_string = "Attempting to start unit"; 1804 periph->flags |= CAM_PERIPH_RECOVERY_INPROG; 1805 1806 /* 1807 * Check for removable media and set 1808 * load/eject flag appropriately. 1809 */ 1810 if (SID_IS_REMOVABLE(&cgd.inq_data)) 1811 le = TRUE; 1812 else 1813 le = FALSE; 1814 1815 scsi_start_stop(&ccb->csio, 1816 /*retries*/1, 1817 camperiphdone, 1818 MSG_SIMPLE_Q_TAG, 1819 /*start*/TRUE, 1820 /*load/eject*/le, 1821 /*immediate*/FALSE, 1822 SSD_FULL_SIZE, 1823 /*timeout*/50000); 1824 break; 1825 } 1826 case SS_TUR: 1827 { 1828 /* 1829 * Send a Test Unit Ready to the device. 1830 * If the 'many' flag is set, we send 120 1831 * test unit ready commands, one every half 1832 * second. Otherwise, we just send one TUR. 1833 * We only want to do this if the retry 1834 * count has not been exhausted. 1835 */ 1836 int retries; 1837 1838 if ((err_action & SSQ_MANY) != 0 && (periph->flags & 1839 CAM_PERIPH_RECOVERY_WAIT_FAILED) == 0) { 1840 periph->flags |= CAM_PERIPH_RECOVERY_WAIT; 1841 *action_string = "Polling device for readiness"; 1842 retries = 120; 1843 } else { 1844 *action_string = "Testing device for readiness"; 1845 retries = 1; 1846 } 1847 periph->flags |= CAM_PERIPH_RECOVERY_INPROG; 1848 scsi_test_unit_ready(&ccb->csio, 1849 retries, 1850 camperiphdone, 1851 MSG_SIMPLE_Q_TAG, 1852 SSD_FULL_SIZE, 1853 /*timeout*/5000); 1854 1855 /* 1856 * Accomplish our 500ms delay by deferring 1857 * the release of our device queue appropriately. 1858 */ 1859 *relsim_flags = RELSIM_RELEASE_AFTER_TIMEOUT; 1860 *timeout = 500; 1861 break; 1862 } 1863 default: 1864 panic("Unhandled error action %x", err_action); 1865 } 1866 1867 if ((err_action & SS_MASK) >= SS_START) { 1868 /* 1869 * Drop the priority, so that the recovery 1870 * CCB is the first to execute. Freeze the queue 1871 * after this command is sent so that we can 1872 * restore the old csio and have it queued in 1873 * the proper order before we release normal 1874 * transactions to the device. 1875 */ 1876 ccb->ccb_h.pinfo.priority--; 1877 ccb->ccb_h.flags |= CAM_DEV_QFREEZE; 1878 ccb->ccb_h.saved_ccb_ptr = orig_ccb; 1879 error = ERESTART; 1880 *orig = orig_ccb; 1881 } 1882 1883 sense_error_done: 1884 *action = err_action; 1885 } 1886 return (error); 1887 } 1888 1889 /* 1890 * Generic error handler. Peripheral drivers usually filter 1891 * out the errors that they handle in a unique manner, then 1892 * call this function. 1893 */ 1894 int 1895 cam_periph_error(union ccb *ccb, cam_flags camflags, 1896 uint32_t sense_flags) 1897 { 1898 struct cam_path *newpath; 1899 union ccb *orig_ccb, *scan_ccb; 1900 struct cam_periph *periph; 1901 const char *action_string; 1902 cam_status status; 1903 bool frozen; 1904 int error, openings, devctl_err; 1905 uint32_t action, relsim_flags, timeout; 1906 1907 CAM_PROBE3(periph, error, ccb, camflags, sense_flags); 1908 1909 action = SSQ_PRINT_SENSE; 1910 periph = xpt_path_periph(ccb->ccb_h.path); 1911 action_string = NULL; 1912 status = ccb->ccb_h.status; 1913 frozen = (status & CAM_DEV_QFRZN) != 0; 1914 status &= CAM_STATUS_MASK; 1915 devctl_err = openings = relsim_flags = timeout = 0; 1916 orig_ccb = ccb; 1917 1918 /* Filter the errors that should be reported via devctl */ 1919 switch (ccb->ccb_h.status & CAM_STATUS_MASK) { 1920 case CAM_CMD_TIMEOUT: 1921 case CAM_REQ_ABORTED: 1922 case CAM_REQ_CMP_ERR: 1923 case CAM_REQ_TERMIO: 1924 case CAM_UNREC_HBA_ERROR: 1925 case CAM_DATA_RUN_ERR: 1926 case CAM_SCSI_STATUS_ERROR: 1927 case CAM_ATA_STATUS_ERROR: 1928 case CAM_SMP_STATUS_ERROR: 1929 case CAM_DEV_NOT_THERE: 1930 case CAM_NVME_STATUS_ERROR: 1931 devctl_err++; 1932 break; 1933 default: 1934 break; 1935 } 1936 1937 switch (status) { 1938 case CAM_REQ_CMP: 1939 error = 0; 1940 action &= ~SSQ_PRINT_SENSE; 1941 break; 1942 case CAM_SCSI_STATUS_ERROR: 1943 error = camperiphscsistatuserror(ccb, &orig_ccb, 1944 camflags, sense_flags, &openings, &relsim_flags, 1945 &timeout, &action, &action_string); 1946 break; 1947 case CAM_AUTOSENSE_FAIL: 1948 error = EIO; /* we have to kill the command */ 1949 break; 1950 case CAM_UA_ABORT: 1951 case CAM_UA_TERMIO: 1952 case CAM_MSG_REJECT_REC: 1953 /* XXX Don't know that these are correct */ 1954 error = EIO; 1955 break; 1956 case CAM_SEL_TIMEOUT: 1957 if ((camflags & CAM_RETRY_SELTO) != 0) { 1958 if (ccb->ccb_h.retry_count > 0 && 1959 (periph->flags & CAM_PERIPH_INVALID) == 0) { 1960 ccb->ccb_h.retry_count--; 1961 error = ERESTART; 1962 1963 /* 1964 * Wait a bit to give the device 1965 * time to recover before we try again. 1966 */ 1967 relsim_flags = RELSIM_RELEASE_AFTER_TIMEOUT; 1968 timeout = periph_selto_delay; 1969 break; 1970 } 1971 action_string = "Retries exhausted"; 1972 } 1973 /* FALLTHROUGH */ 1974 case CAM_DEV_NOT_THERE: 1975 error = ENXIO; 1976 action = SSQ_LOST; 1977 break; 1978 case CAM_REQ_INVALID: 1979 case CAM_PATH_INVALID: 1980 case CAM_NO_HBA: 1981 case CAM_PROVIDE_FAIL: 1982 case CAM_REQ_TOO_BIG: 1983 case CAM_LUN_INVALID: 1984 case CAM_TID_INVALID: 1985 case CAM_FUNC_NOTAVAIL: 1986 error = EINVAL; 1987 break; 1988 case CAM_SCSI_BUS_RESET: 1989 case CAM_BDR_SENT: 1990 /* 1991 * Commands that repeatedly timeout and cause these 1992 * kinds of error recovery actions, should return 1993 * CAM_CMD_TIMEOUT, which allows us to safely assume 1994 * that this command was an innocent bystander to 1995 * these events and should be unconditionally 1996 * retried. 1997 */ 1998 case CAM_REQUEUE_REQ: 1999 /* Unconditional requeue if device is still there */ 2000 if (periph->flags & CAM_PERIPH_INVALID) { 2001 action_string = "Periph was invalidated"; 2002 error = ENXIO; 2003 } else if (sense_flags & SF_NO_RETRY) { 2004 error = EIO; 2005 action_string = "Retry was blocked"; 2006 } else { 2007 error = ERESTART; 2008 action &= ~SSQ_PRINT_SENSE; 2009 } 2010 break; 2011 case CAM_RESRC_UNAVAIL: 2012 /* Wait a bit for the resource shortage to abate. */ 2013 timeout = periph_noresrc_delay; 2014 /* FALLTHROUGH */ 2015 case CAM_BUSY: 2016 if (timeout == 0) { 2017 /* Wait a bit for the busy condition to abate. */ 2018 timeout = periph_busy_delay; 2019 } 2020 relsim_flags = RELSIM_RELEASE_AFTER_TIMEOUT; 2021 /* FALLTHROUGH */ 2022 case CAM_ATA_STATUS_ERROR: 2023 case CAM_NVME_STATUS_ERROR: 2024 case CAM_SMP_STATUS_ERROR: 2025 case CAM_REQ_CMP_ERR: 2026 case CAM_CMD_TIMEOUT: 2027 case CAM_UNEXP_BUSFREE: 2028 case CAM_UNCOR_PARITY: 2029 case CAM_DATA_RUN_ERR: 2030 default: 2031 if (periph->flags & CAM_PERIPH_INVALID) { 2032 error = ENXIO; 2033 action_string = "Periph was invalidated"; 2034 } else if (ccb->ccb_h.retry_count == 0) { 2035 error = EIO; 2036 action_string = "Retries exhausted"; 2037 } else if (sense_flags & SF_NO_RETRY) { 2038 error = EIO; 2039 action_string = "Retry was blocked"; 2040 } else { 2041 ccb->ccb_h.retry_count--; 2042 error = ERESTART; 2043 } 2044 break; 2045 } 2046 2047 if ((sense_flags & SF_PRINT_ALWAYS) || 2048 CAM_DEBUGGED(ccb->ccb_h.path, CAM_DEBUG_INFO)) 2049 action |= SSQ_PRINT_SENSE; 2050 else if (sense_flags & SF_NO_PRINT) 2051 action &= ~SSQ_PRINT_SENSE; 2052 if ((action & SSQ_PRINT_SENSE) != 0) 2053 cam_error_print(orig_ccb, CAM_ESF_ALL, CAM_EPF_ALL); 2054 if (error != 0 && (action & SSQ_PRINT_SENSE) != 0) { 2055 if (error != ERESTART) { 2056 if (action_string == NULL) 2057 action_string = "Unretryable error"; 2058 xpt_print(ccb->ccb_h.path, "Error %d, %s\n", 2059 error, action_string); 2060 } else if (action_string != NULL) 2061 xpt_print(ccb->ccb_h.path, "%s\n", action_string); 2062 else { 2063 xpt_print(ccb->ccb_h.path, 2064 "Retrying command, %d more tries remain\n", 2065 ccb->ccb_h.retry_count); 2066 } 2067 } 2068 2069 if (devctl_err && (error != 0 || (action & SSQ_PRINT_SENSE) != 0)) 2070 cam_periph_devctl_notify(orig_ccb); 2071 2072 if ((action & SSQ_LOST) != 0) { 2073 lun_id_t lun_id; 2074 2075 /* 2076 * For a selection timeout, we consider all of the LUNs on 2077 * the target to be gone. If the status is CAM_DEV_NOT_THERE, 2078 * then we only get rid of the device(s) specified by the 2079 * path in the original CCB. 2080 */ 2081 if (status == CAM_SEL_TIMEOUT) 2082 lun_id = CAM_LUN_WILDCARD; 2083 else 2084 lun_id = xpt_path_lun_id(ccb->ccb_h.path); 2085 2086 /* Should we do more if we can't create the path?? */ 2087 if (xpt_create_path(&newpath, periph, 2088 xpt_path_path_id(ccb->ccb_h.path), 2089 xpt_path_target_id(ccb->ccb_h.path), 2090 lun_id) == CAM_REQ_CMP) { 2091 /* 2092 * Let peripheral drivers know that this 2093 * device has gone away. 2094 */ 2095 xpt_async(AC_LOST_DEVICE, newpath, NULL); 2096 xpt_free_path(newpath); 2097 } 2098 } 2099 2100 /* Broadcast UNIT ATTENTIONs to all periphs. */ 2101 if ((action & SSQ_UA) != 0) 2102 xpt_async(AC_UNIT_ATTENTION, orig_ccb->ccb_h.path, orig_ccb); 2103 2104 /* Rescan target on "Reported LUNs data has changed" */ 2105 if ((action & SSQ_RESCAN) != 0) { 2106 if (xpt_create_path(&newpath, NULL, 2107 xpt_path_path_id(ccb->ccb_h.path), 2108 xpt_path_target_id(ccb->ccb_h.path), 2109 CAM_LUN_WILDCARD) == CAM_REQ_CMP) { 2110 scan_ccb = xpt_alloc_ccb_nowait(); 2111 if (scan_ccb != NULL) { 2112 scan_ccb->ccb_h.path = newpath; 2113 scan_ccb->ccb_h.func_code = XPT_SCAN_TGT; 2114 scan_ccb->crcn.flags = 0; 2115 xpt_rescan(scan_ccb); 2116 } else { 2117 xpt_print(newpath, 2118 "Can't allocate CCB to rescan target\n"); 2119 xpt_free_path(newpath); 2120 } 2121 } 2122 } 2123 2124 /* Attempt a retry */ 2125 if (error == ERESTART || error == 0) { 2126 if (frozen) 2127 ccb->ccb_h.status &= ~CAM_DEV_QFRZN; 2128 if (error == ERESTART) 2129 xpt_action(ccb); 2130 if (frozen) 2131 cam_release_devq(ccb->ccb_h.path, 2132 relsim_flags, 2133 openings, 2134 timeout, 2135 /*getcount_only*/0); 2136 } 2137 2138 CAM_PROBE2(periph, recovery, ccb, error); 2139 return (error); 2140 } 2141 2142 #define CAM_PERIPH_DEVD_MSG_SIZE 1024 2143 2144 /* 2145 * Allocate and initialize an sbuf for a devctl notification, populating it 2146 * with the device name and serial number. Returns the malloc'd backing 2147 * buffer, or NULL on allocation failure. On success, the caller can append 2148 * additional fields to sb before calling cam_periph_devctl_sb_fini(). 2149 */ 2150 static char * 2151 cam_periph_devctl_sb_init(struct sbuf *sb, struct cam_periph *periph) 2152 { 2153 struct ccb_getdev *cgd; 2154 char *sbmsg; 2155 2156 sbmsg = malloc(CAM_PERIPH_DEVD_MSG_SIZE, M_CAMPERIPH, M_NOWAIT); 2157 if (sbmsg == NULL) 2158 return (NULL); 2159 2160 sbuf_new(sb, sbmsg, CAM_PERIPH_DEVD_MSG_SIZE, SBUF_FIXEDLEN); 2161 2162 sbuf_printf(sb, "device=%s%d ", periph->periph_name, 2163 periph->unit_number); 2164 2165 if ((cgd = (struct ccb_getdev *)xpt_alloc_ccb_nowait()) != NULL) { 2166 xpt_gdev_type(cgd, periph->path); 2167 if (cgd->ccb_h.status == CAM_REQ_CMP && 2168 cgd->serial_num_len > 0) { 2169 sbuf_cat(sb, "serial=\""); 2170 sbuf_bcat(sb, cgd->serial_num, cgd->serial_num_len); 2171 sbuf_cat(sb, "\" "); 2172 } else { 2173 sbuf_cat(sb, "path=\""); 2174 xpt_path_sbuf(periph->path, sb); 2175 sbuf_cat(sb, "\" "); 2176 } 2177 xpt_free_ccb((union ccb *)cgd); 2178 } 2179 2180 return (sbmsg); 2181 } 2182 2183 /* 2184 * Finish and send a devctl notification, then clean up the sbuf and its 2185 * backing buffer. 2186 */ 2187 static void 2188 cam_periph_devctl_sb_fini(struct sbuf *sb, char *sbmsg, const char *type) 2189 { 2190 2191 if (sbuf_finish(sb) == 0) 2192 devctl_notify("CAM", "periph", type, sbuf_data(sb)); 2193 sbuf_delete(sb); 2194 free(sbmsg, M_CAMPERIPH); 2195 } 2196 2197 static void 2198 cam_periph_devctl_notify(union ccb *ccb) 2199 { 2200 struct cam_periph *periph; 2201 struct sbuf sb; 2202 char *sbmsg, *type; 2203 2204 periph = xpt_path_periph(ccb->ccb_h.path); 2205 sbmsg = cam_periph_devctl_sb_init(&sb, periph); 2206 if (sbmsg == NULL) 2207 return; 2208 2209 sbuf_printf(&sb, "cam_status=\"0x%x\" ", ccb->ccb_h.status); 2210 2211 switch (ccb->ccb_h.status & CAM_STATUS_MASK) { 2212 case CAM_CMD_TIMEOUT: 2213 sbuf_printf(&sb, "timeout=%d ", ccb->ccb_h.timeout); 2214 type = "timeout"; 2215 break; 2216 case CAM_SCSI_STATUS_ERROR: 2217 scsi_format_sense_devd(&ccb->csio, &sb); 2218 type = "error"; 2219 break; 2220 case CAM_ATA_STATUS_ERROR: 2221 sbuf_cat(&sb, "RES=\""); 2222 ata_res_sbuf(&ccb->ataio.res, &sb); 2223 sbuf_cat(&sb, "\" "); 2224 type = "error"; 2225 break; 2226 case CAM_NVME_STATUS_ERROR: 2227 { 2228 struct ccb_nvmeio *n = &ccb->nvmeio; 2229 2230 sbuf_printf(&sb, "sct=\"%02x\" sc=\"%02x\" cdw0=\"%08x\" ", 2231 NVME_STATUS_GET_SCT(n->cpl.status), 2232 NVME_STATUS_GET_SC(n->cpl.status), n->cpl.cdw0); 2233 type = "error"; 2234 break; 2235 } 2236 default: 2237 type = "error"; 2238 break; 2239 } 2240 2241 2242 switch (ccb->ccb_h.func_code) { 2243 case XPT_SCSI_IO: 2244 sbuf_cat(&sb, "CDB=\""); 2245 scsi_cdb_sbuf(scsiio_cdb_ptr(&ccb->csio), &sb); 2246 sbuf_cat(&sb, "\" "); 2247 break; 2248 case XPT_ATA_IO: 2249 sbuf_cat(&sb, "ACB=\""); 2250 ata_cmd_sbuf(&ccb->ataio.cmd, &sb); 2251 sbuf_cat(&sb, "\" "); 2252 break; 2253 case XPT_NVME_IO: 2254 case XPT_NVME_ADMIN: 2255 { 2256 struct ccb_nvmeio *n = &ccb->nvmeio; 2257 struct nvme_command *cmd = &n->cmd; 2258 2259 // XXX Likely should be nvme_cmd_sbuf 2260 sbuf_printf(&sb, "cmdset=\"%s\" opc=\"%02x\" fuse=\"%02x\" cid=\"%04x\" " 2261 "nsid=\"%08x\" cdw10=\"%08x\" cdw11=\"%08x\" cdw12=\"%08x\" " 2262 "cdw13=\"%08x\" cdw14=\"%08x\" cdw15=\"%08x\" ", 2263 ccb->ccb_h.func_code == XPT_NVME_ADMIN ? "admin" : "io", 2264 cmd->opc, cmd->fuse, cmd->cid, cmd->nsid, cmd->cdw10, 2265 cmd->cdw11, cmd->cdw12, cmd->cdw13, cmd->cdw14, cmd->cdw15); 2266 break; 2267 } 2268 default: 2269 break; 2270 } 2271 2272 cam_periph_devctl_sb_fini(&sb, sbmsg, type); 2273 } 2274 2275 /* 2276 * Sysctl to force an invalidation of the drive right now. Can be 2277 * called with CTLFLAG_MPSAFE since we take periph lock. 2278 */ 2279 int 2280 cam_periph_invalidate_sysctl(SYSCTL_HANDLER_ARGS) 2281 { 2282 struct cam_periph *periph; 2283 int error, value; 2284 2285 periph = arg1; 2286 value = 0; 2287 error = sysctl_handle_int(oidp, &value, 0, req); 2288 if (error != 0 || req->newptr == NULL || value != 1) 2289 return (error); 2290 2291 cam_periph_lock(periph); 2292 cam_periph_invalidate(periph); 2293 cam_periph_unlock(periph); 2294 2295 return (0); 2296 } 2297