1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright (c) 2017 Joyent, Inc. 14 */ 15 16 #include <sys/scsi/adapters/smrt/smrt.h> 17 18 static void 19 smrt_physical_free(smrt_physical_t *smpt) 20 { 21 VERIFY(list_is_empty(&smpt->smpt_targets)); 22 VERIFY(smpt->smpt_info != NULL); 23 24 kmem_free(smpt->smpt_info, sizeof (*smpt->smpt_info)); 25 list_destroy(&smpt->smpt_targets); 26 kmem_free(smpt, sizeof (*smpt)); 27 } 28 29 /* 30 * Determine if a physical device enumerated should be shown to the world. There 31 * are three conditions to satisfy for this to be true. 32 * 33 * 1. The device (SAS, SATA, SES, etc.) must not have a masked CISS address. A 34 * masked CISS address indicates a device that we should not be performing I/O 35 * to. 36 * 2. The drive (SAS or SATA device) must not be marked as a member of a logical 37 * volume. 38 * 3. The drive (SAS or SATA device) must not be marked as a spare. 39 */ 40 static boolean_t 41 smrt_physical_visible(PhysDevAddr_t *addr, smrt_identify_physical_drive_t *info) 42 { 43 if (addr->Mode == SMRT_CISS_MODE_MASKED) { 44 return (B_FALSE); 45 } 46 47 if ((info->sipd_more_flags & (SMRT_MORE_FLAGS_LOGVOL | 48 SMRT_MORE_FLAGS_SPARE)) != 0) { 49 return (B_FALSE); 50 } 51 52 return (B_TRUE); 53 } 54 55 /* 56 * Note, the caller is responsible for making sure that the unit-address form of 57 * the WWN is pased in. Any additional information to target a specific LUN 58 * will be ignored. 59 */ 60 smrt_physical_t * 61 smrt_phys_lookup_by_ua(smrt_t *smrt, const char *ua) 62 { 63 VERIFY(MUTEX_HELD(&smrt->smrt_mutex)); 64 65 /* 66 * Sanity check that the caller has provided us enough bytes for a 67 * properly formed unit-address form of a WWN. 68 */ 69 if (strlen(ua) < SCSI_WWN_UA_STRLEN) 70 return (NULL); 71 72 for (smrt_physical_t *smpt = list_head(&smrt->smrt_physicals); 73 smpt != NULL; smpt = list_next(&smrt->smrt_physicals, smpt)) { 74 char wwnstr[SCSI_WWN_BUFLEN]; 75 76 (void) scsi_wwn_to_wwnstr(smpt->smpt_wwn, 1, wwnstr); 77 if (strncmp(wwnstr, ua, SCSI_WWN_UA_STRLEN) != 0) 78 continue; 79 80 /* 81 * Verify that the UA string is either a comma or null there. 82 * We accept the comma in case it's being used as part of a 83 * normal UA with a LUN. 84 */ 85 if (ua[SCSI_WWN_UA_STRLEN] != '\0' && 86 ua[SCSI_WWN_UA_STRLEN] != ',') { 87 continue; 88 } 89 90 return (smpt); 91 } 92 93 return (NULL); 94 } 95 96 static smrt_physical_t * 97 smrt_phys_lookup_by_wwn(smrt_t *smrt, uint64_t wwn) 98 { 99 VERIFY(MUTEX_HELD(&smrt->smrt_mutex)); 100 101 for (smrt_physical_t *smpt = list_head(&smrt->smrt_physicals); 102 smpt != NULL; smpt = list_next(&smrt->smrt_physicals, smpt)) { 103 if (wwn == smpt->smpt_wwn) 104 return (smpt); 105 } 106 107 return (NULL); 108 } 109 110 static int 111 smrt_phys_identify(smrt_t *smrt, smrt_identify_physical_drive_t *info, 112 uint16_t bmic, uint16_t timeout) 113 { 114 smrt_command_t *smcm = NULL; 115 smrt_identify_physical_drive_t *sipd; 116 smrt_identify_physical_drive_req_t sipdr; 117 int ret; 118 size_t sz, copysz; 119 120 sz = sizeof (smrt_identify_physical_drive_t); 121 sz = P2ROUNDUP_TYPED(sz, 512, size_t); 122 if ((smcm = smrt_command_alloc(smrt, SMRT_CMDTYPE_INTERNAL, 123 KM_NOSLEEP)) == NULL || smrt_command_attach_internal(smrt, smcm, 124 sizeof (*sipd), KM_NOSLEEP) != 0) { 125 ret = ENOMEM; 126 goto out; 127 } 128 129 sipd = smcm->smcm_internal->smcmi_va; 130 131 smrt_write_controller_lun_addr(&smcm->smcm_va_cmd->Header.LUN); 132 133 smcm->smcm_va_cmd->Request.CDBLen = sizeof (sipdr); 134 smcm->smcm_va_cmd->Request.Timeout = LE_16(timeout); 135 smcm->smcm_va_cmd->Request.Type.Type = CISS_TYPE_CMD; 136 smcm->smcm_va_cmd->Request.Type.Attribute = CISS_ATTR_SIMPLE; 137 smcm->smcm_va_cmd->Request.Type.Direction = CISS_XFER_READ; 138 139 /* 140 * Construct the IDENTIFY PHYSICAL DEVICE request CDB. Note that any 141 * reserved fields in the request must be filled with zeroes. 142 */ 143 bzero(&sipdr, sizeof (sipdr)); 144 sipdr.sipdr_opcode = CISS_SCMD_BMIC_READ; 145 sipdr.sipdr_lun = 0; 146 sipdr.sipdr_bmic_index1 = bmic & 0x00ff; 147 sipdr.sipdr_command = CISS_BMIC_IDENTIFY_PHYSICAL_DEVICE; 148 sipdr.sipdr_bmic_index2 = (bmic & 0xff00) >> 8; 149 bcopy(&sipdr, &smcm->smcm_va_cmd->Request.CDB[0], 150 MIN(CISS_CDBLEN, sizeof (sipdr))); 151 152 mutex_enter(&smrt->smrt_mutex); 153 154 /* 155 * Send the command to the device. 156 */ 157 smcm->smcm_status |= SMRT_CMD_STATUS_POLLED; 158 if ((ret = smrt_submit(smrt, smcm)) != 0) { 159 mutex_exit(&smrt->smrt_mutex); 160 goto out; 161 } 162 163 /* 164 * Poll for completion. 165 */ 166 smcm->smcm_expiry = gethrtime() + timeout * NANOSEC; 167 if ((ret = smrt_poll_for(smrt, smcm)) != 0) { 168 VERIFY3S(ret, ==, ETIMEDOUT); 169 VERIFY0(smcm->smcm_status & SMRT_CMD_STATUS_POLL_COMPLETE); 170 171 /* 172 * The command timed out; abandon it now. Remove the POLLED 173 * flag so that the periodic routine will send an abort to 174 * clean it up next time around. 175 */ 176 smcm->smcm_status |= SMRT_CMD_STATUS_ABANDONED; 177 smcm->smcm_status &= ~SMRT_CMD_STATUS_POLLED; 178 smcm = NULL; 179 mutex_exit(&smrt->smrt_mutex); 180 goto out; 181 } 182 mutex_exit(&smrt->smrt_mutex); 183 184 if (smcm->smcm_status & SMRT_CMD_STATUS_RESET_SENT) { 185 /* 186 * The controller was reset while we were trying to discover 187 * physical volumes. Report failure. 188 */ 189 ret = EIO; 190 goto out; 191 } 192 193 if (smcm->smcm_status & SMRT_CMD_STATUS_ERROR) { 194 ErrorInfo_t *ei = smcm->smcm_va_err; 195 196 if (ei->CommandStatus != CISS_CMD_DATA_UNDERRUN) { 197 dev_err(smrt->smrt_dip, CE_WARN, "identify physical " 198 "device error: status 0x%x", ei->CommandStatus); 199 ret = EIO; 200 goto out; 201 } 202 203 copysz = MIN(sizeof (*sipd), sz - ei->ResidualCnt); 204 } else { 205 copysz = sizeof (*sipd); 206 } 207 208 209 sz = MIN(sizeof (*sipd), copysz); 210 bcopy(sipd, info, sizeof (*sipd)); 211 212 ret = 0; 213 out: 214 if (smcm != NULL) { 215 smrt_command_free(smcm); 216 } 217 218 return (ret); 219 } 220 221 static int 222 smrt_read_phys_ext(smrt_t *smrt, smrt_report_physical_lun_t *smrpl, 223 uint16_t timeout, uint64_t gen) 224 { 225 smrt_report_physical_lun_extent_t *extents = smrpl->smrpl_data.extents; 226 uint32_t count = BE_32(smrpl->smrpl_datasize) / 227 sizeof (smrt_report_physical_lun_extent_t); 228 uint32_t i; 229 230 VERIFY(MUTEX_HELD(&smrt->smrt_mutex)); 231 232 if (count > SMRT_MAX_PHYSDEV) { 233 count = SMRT_MAX_PHYSDEV; 234 } 235 236 for (i = 0; i < count; i++) { 237 int ret; 238 smrt_physical_t *smpt; 239 smrt_identify_physical_drive_t *info; 240 smrt_report_physical_opdi_t *opdi; 241 uint16_t bmic; 242 uint64_t wwn, satawwn; 243 char name[SCSI_MAXNAMELEN]; 244 245 opdi = &extents[i].srple_extdata.srple_opdi; 246 247 mutex_exit(&smrt->smrt_mutex); 248 249 /* 250 * Get the extended information about this device. 251 */ 252 info = kmem_zalloc(sizeof (*info), KM_NOSLEEP); 253 if (info == NULL) { 254 mutex_enter(&smrt->smrt_mutex); 255 return (ENOMEM); 256 } 257 258 bmic = smrt_lun_addr_to_bmic(&extents[i].srple_addr); 259 ret = smrt_phys_identify(smrt, info, bmic, timeout); 260 if (ret != 0) { 261 mutex_enter(&smrt->smrt_mutex); 262 kmem_free(info, sizeof (*info)); 263 return (ret); 264 } 265 266 wwn = *(uint64_t *)opdi->srpo_wwid; 267 wwn = BE_64(wwn); 268 269 /* 270 * SATA devices may not have a proper WWN returned from firmware 271 * based on the SATL specification. Try to fetch the proper id 272 * for SATA devices, if the drive has one. If the drive doesn't 273 * have one or the SATL refuses to give us one, we use whatever 274 * the controller told us. 275 */ 276 if (opdi->srpo_dtype == SMRT_DTYPE_SATA && 277 smrt_sata_determine_wwn(smrt, &extents[i].srple_addr, 278 &satawwn, timeout) == 0) { 279 wwn = satawwn; 280 } 281 282 mutex_enter(&smrt->smrt_mutex); 283 smpt = smrt_phys_lookup_by_wwn(smrt, wwn); 284 if (smpt != NULL) { 285 /* 286 * Sanity check that the model and serial number of this 287 * device is the same for this WWN. If it's not, the 288 * controller is probably lying about something. 289 */ 290 if (bcmp(smpt->smpt_info->sipd_model, info->sipd_model, 291 sizeof (info->sipd_model)) != 0 || 292 bcmp(smpt->smpt_info->sipd_serial, 293 info->sipd_serial, sizeof (info->sipd_serial)) != 294 0 || smpt->smpt_dtype != opdi->srpo_dtype) { 295 dev_err(smrt->smrt_dip, CE_PANIC, "physical " 296 "target with wwn 0x%" PRIx64 " changed " 297 "model, serial, or type unexpectedly: " 298 "smrt_physical_t %p, phys info: %p", wwn, 299 smpt, info); 300 } 301 302 /* 303 * When panicking, we don't allow a device's visibility 304 * to change to being invisible and be able to actually 305 * panic. We only worry about devices which are used 306 * for I/O. We purposefully ignore SES devices. 307 */ 308 if (ddi_in_panic() && 309 (opdi->srpo_dtype == SMRT_DTYPE_SATA || 310 opdi->srpo_dtype == SMRT_DTYPE_SAS)) { 311 boolean_t visible; 312 313 visible = smrt_physical_visible( 314 &smpt->smpt_addr.PhysDev, smpt->smpt_info); 315 316 if (visible != smpt->smpt_visible) { 317 dev_err(smrt->smrt_dip, CE_PANIC, 318 "physical target with wwn 0x%" 319 PRIx64 " changed visibility status " 320 "unexpectedly", wwn); 321 } 322 } 323 324 kmem_free(smpt->smpt_info, sizeof (*smpt->smpt_info)); 325 smpt->smpt_info = NULL; 326 } else { 327 smpt = kmem_zalloc(sizeof (smrt_physical_t), 328 KM_NOSLEEP); 329 if (smpt == NULL) { 330 kmem_free(info, sizeof (*info)); 331 return (ENOMEM); 332 } 333 334 smpt->smpt_wwn = wwn; 335 smpt->smpt_dtype = opdi->srpo_dtype; 336 list_create(&smpt->smpt_targets, sizeof (smrt_target_t), 337 offsetof(smrt_target_t, smtg_link_lun)); 338 smpt->smpt_ctlr = smrt; 339 list_insert_tail(&smrt->smrt_physicals, smpt); 340 } 341 342 VERIFY3P(smpt->smpt_info, ==, NULL); 343 344 /* 345 * Determine if this device is supported and if it's visible to 346 * the system. Some devices may not be visible to the system 347 * because they're used in logical volumes or spares. 348 * Unsupported devices are also not visible. 349 */ 350 switch (smpt->smpt_dtype) { 351 case SMRT_DTYPE_SATA: 352 case SMRT_DTYPE_SAS: 353 smpt->smpt_supported = B_TRUE; 354 smpt->smpt_visible = 355 smrt_physical_visible(&extents[i].srple_addr, info); 356 break; 357 case SMRT_DTYPE_SES: 358 smpt->smpt_supported = B_TRUE; 359 smpt->smpt_visible = 360 smrt_physical_visible(&extents[i].srple_addr, info); 361 break; 362 default: 363 smpt->smpt_visible = B_FALSE; 364 smpt->smpt_supported = B_FALSE; 365 } 366 367 smpt->smpt_info = info; 368 smpt->smpt_addr.PhysDev = extents[i].srple_addr; 369 smpt->smpt_bmic = bmic; 370 smpt->smpt_gen = gen; 371 (void) scsi_wwn_to_wwnstr(smpt->smpt_wwn, 1, name); 372 if (!ddi_in_panic() && smpt->smpt_visible && 373 scsi_hba_tgtmap_set_add(smrt->smrt_phys_tgtmap, 374 SCSI_TGT_SCSI_DEVICE, name, NULL) != DDI_SUCCESS) { 375 return (EIO); 376 } 377 } 378 379 return (0); 380 } 381 382 int 383 smrt_phys_discover(smrt_t *smrt, uint16_t timeout, uint64_t gen) 384 { 385 smrt_command_t *smcm; 386 smrt_report_physical_lun_t *smrpl; 387 smrt_report_physical_lun_req_t smrplr; 388 int r; 389 390 /* 391 * Allocate the command to send to the device, including buffer space 392 * for the returned list of Physical Volumes. 393 */ 394 if ((smcm = smrt_command_alloc(smrt, SMRT_CMDTYPE_INTERNAL, 395 KM_NOSLEEP)) == NULL || smrt_command_attach_internal(smrt, smcm, 396 sizeof (*smrpl), KM_NOSLEEP) != 0) { 397 r = ENOMEM; 398 mutex_enter(&smrt->smrt_mutex); 399 goto out; 400 } 401 402 smrpl = smcm->smcm_internal->smcmi_va; 403 404 smrt_write_controller_lun_addr(&smcm->smcm_va_cmd->Header.LUN); 405 406 smcm->smcm_va_cmd->Request.CDBLen = sizeof (smrplr); 407 smcm->smcm_va_cmd->Request.Timeout = LE_16(timeout); 408 smcm->smcm_va_cmd->Request.Type.Type = CISS_TYPE_CMD; 409 smcm->smcm_va_cmd->Request.Type.Attribute = CISS_ATTR_SIMPLE; 410 smcm->smcm_va_cmd->Request.Type.Direction = CISS_XFER_READ; 411 412 /* 413 * The Report Physical LUNs command is essentially a vendor-specific 414 * SCSI command, which we assemble into the CDB region of the command 415 * block. 416 */ 417 bzero(&smrplr, sizeof (smrplr)); 418 smrplr.smrplr_opcode = CISS_SCMD_REPORT_PHYSICAL_LUNS; 419 smrplr.smrplr_extflag = SMRT_REPORT_PHYSICAL_LUN_EXT_OPDI; 420 smrplr.smrplr_datasize = BE_32(sizeof (smrt_report_physical_lun_t)); 421 bcopy(&smrplr, &smcm->smcm_va_cmd->Request.CDB[0], 422 MIN(CISS_CDBLEN, sizeof (smrplr))); 423 424 mutex_enter(&smrt->smrt_mutex); 425 426 /* 427 * Send the command to the device. 428 */ 429 smcm->smcm_status |= SMRT_CMD_STATUS_POLLED; 430 if ((r = smrt_submit(smrt, smcm)) != 0) { 431 goto out; 432 } 433 434 /* 435 * Poll for completion. 436 */ 437 smcm->smcm_expiry = gethrtime() + timeout * NANOSEC; 438 if ((r = smrt_poll_for(smrt, smcm)) != 0) { 439 VERIFY3S(r, ==, ETIMEDOUT); 440 VERIFY0(smcm->smcm_status & SMRT_CMD_STATUS_POLL_COMPLETE); 441 442 /* 443 * The command timed out; abandon it now. Remove the POLLED 444 * flag so that the periodic routine will send an abort to 445 * clean it up next time around. 446 */ 447 smcm->smcm_status |= SMRT_CMD_STATUS_ABANDONED; 448 smcm->smcm_status &= ~SMRT_CMD_STATUS_POLLED; 449 smcm = NULL; 450 goto out; 451 } 452 453 if (smcm->smcm_status & SMRT_CMD_STATUS_RESET_SENT) { 454 /* 455 * 456 * The controller was reset while we were trying to discover 457 * logical volumes. Report failure. 458 */ 459 r = EIO; 460 goto out; 461 } 462 463 if (smcm->smcm_status & SMRT_CMD_STATUS_ERROR) { 464 ErrorInfo_t *ei = smcm->smcm_va_err; 465 466 if (ei->CommandStatus != CISS_CMD_DATA_UNDERRUN) { 467 dev_err(smrt->smrt_dip, CE_WARN, "physical target " 468 "discovery error: status 0x%x", ei->CommandStatus); 469 r = EIO; 470 goto out; 471 } 472 } 473 474 /* 475 * If the controller doesn't support extended physical reporting, it 476 * likely doesn't even support physical devices that we'd care about 477 * exposing. As such, we treat this as an OK case. 478 */ 479 if ((smrpl->smrpl_extflag & SMRT_REPORT_PHYSICAL_LUN_EXT_MASK) != 480 SMRT_REPORT_PHYSICAL_LUN_EXT_OPDI) { 481 r = 0; 482 goto out; 483 } 484 485 if (!ddi_in_panic() && 486 scsi_hba_tgtmap_set_begin(smrt->smrt_phys_tgtmap) != DDI_SUCCESS) { 487 dev_err(smrt->smrt_dip, CE_WARN, "failed to begin target map " 488 "observation on %s", SMRT_IPORT_PHYS); 489 r = EIO; 490 goto out; 491 } 492 493 r = smrt_read_phys_ext(smrt, smrpl, timeout, gen); 494 495 if (r == 0 && !ddi_in_panic()) { 496 if (scsi_hba_tgtmap_set_end(smrt->smrt_phys_tgtmap, 0) != 497 DDI_SUCCESS) { 498 dev_err(smrt->smrt_dip, CE_WARN, "failed to end target " 499 "map observation on %s", SMRT_IPORT_PHYS); 500 r = EIO; 501 } 502 } else if (r != 0 && !ddi_in_panic()) { 503 if (scsi_hba_tgtmap_set_flush(smrt->smrt_phys_tgtmap) != 504 DDI_SUCCESS) { 505 dev_err(smrt->smrt_dip, CE_WARN, "failed to end target " 506 "map observation on %s", SMRT_IPORT_PHYS); 507 r = EIO; 508 } 509 } 510 511 if (r == 0) { 512 smrt_physical_t *smpt, *next; 513 514 /* 515 * Prune physical devices that do not match the current 516 * generation and are not marked as visible devices. Visible 517 * devices will be dealt with as part of the target map work. 518 */ 519 for (smpt = list_head(&smrt->smrt_physicals), next = NULL; 520 smpt != NULL; smpt = next) { 521 next = list_next(&smrt->smrt_physicals, smpt); 522 if (smpt->smpt_visible || smpt->smpt_gen == gen) 523 continue; 524 list_remove(&smrt->smrt_physicals, smpt); 525 smrt_physical_free(smpt); 526 } 527 528 /* 529 * Update the time of the last successful Physical Volume 530 * discovery: 531 */ 532 smrt->smrt_last_phys_discovery = gethrtime(); 533 534 /* 535 * Now, for each unsupported device that we haven't warned about 536 * encountering, try and give the administrator some hope of 537 * knowing about this. 538 */ 539 for (smpt = list_head(&smrt->smrt_physicals), next = NULL; 540 smpt != NULL; smpt = next) { 541 if (smpt->smpt_supported || smpt->smpt_unsup_warn) 542 continue; 543 smpt->smpt_unsup_warn = B_TRUE; 544 dev_err(smrt->smrt_dip, CE_WARN, "encountered " 545 "unsupported device with device type %d", 546 smpt->smpt_dtype); 547 } 548 } 549 550 out: 551 mutex_exit(&smrt->smrt_mutex); 552 553 if (smcm != NULL) { 554 smrt_command_free(smcm); 555 } 556 return (r); 557 } 558 559 void 560 smrt_phys_tgtmap_activate(void *arg, char *addr, scsi_tgtmap_tgt_type_t type, 561 void **privpp) 562 { 563 smrt_t *smrt = arg; 564 smrt_physical_t *smpt; 565 566 VERIFY3S(type, ==, SCSI_TGT_SCSI_DEVICE); 567 mutex_enter(&smrt->smrt_mutex); 568 smpt = smrt_phys_lookup_by_ua(smrt, addr); 569 VERIFY(smpt != NULL); 570 VERIFY(smpt->smpt_supported); 571 VERIFY(smpt->smpt_visible); 572 *privpp = NULL; 573 mutex_exit(&smrt->smrt_mutex); 574 } 575 576 boolean_t 577 smrt_phys_tgtmap_deactivate(void *arg, char *addr, scsi_tgtmap_tgt_type_t type, 578 void *priv, scsi_tgtmap_deact_rsn_t reason) 579 { 580 smrt_t *smrt = arg; 581 smrt_physical_t *smpt; 582 583 VERIFY3S(type, ==, SCSI_TGT_SCSI_DEVICE); 584 VERIFY3P(priv, ==, NULL); 585 586 mutex_enter(&smrt->smrt_mutex); 587 smpt = smrt_phys_lookup_by_ua(smrt, addr); 588 589 /* 590 * If the device disappeared or became invisible, then it may have 591 * already been removed. 592 */ 593 if (smpt == NULL || !smpt->smpt_visible) { 594 mutex_exit(&smrt->smrt_mutex); 595 return (B_FALSE); 596 } 597 598 list_remove(&smrt->smrt_physicals, smpt); 599 smrt_physical_free(smpt); 600 mutex_exit(&smrt->smrt_mutex); 601 return (B_FALSE); 602 } 603 604 void 605 smrt_phys_teardown(smrt_t *smrt) 606 { 607 smrt_physical_t *smpt; 608 609 while ((smpt = list_remove_head(&smrt->smrt_physicals)) != NULL) { 610 smrt_physical_free(smpt); 611 } 612 } 613