1 /* 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2020-2025, Broadcom Inc. All rights reserved. 5 * Support: <fbsd-storage-driver.pdl@broadcom.com> 6 * 7 * Authors: Sumit Saxena <sumit.saxena@broadcom.com> 8 * Chandrakanth Patil <chandrakanth.patil@broadcom.com> 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are 12 * met: 13 * 14 * 1. Redistributions of source code must retain the above copyright notice, 15 * this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright notice, 17 * this list of conditions and the following disclaimer in the documentation and/or other 18 * materials provided with the distribution. 19 * 3. Neither the name of the Broadcom Inc. nor the names of its contributors 20 * may be used to endorse or promote products derived from this software without 21 * specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 * POSSIBILITY OF SUCH DAMAGE. 34 * 35 * The views and conclusions contained in the software and documentation are 36 * those of the authors and should not be interpreted as representing 37 * official policies,either expressed or implied, of the FreeBSD Project. 38 * 39 * Mail to: Broadcom Inc 1320 Ridder Park Dr, San Jose, CA 95131 40 * 41 * Broadcom Inc. (Broadcom) MPI3MR Adapter FreeBSD 42 */ 43 44 #include <sys/types.h> 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/kernel.h> 48 #include <sys/selinfo.h> 49 #include <sys/module.h> 50 #include <sys/bus.h> 51 #include <sys/conf.h> 52 #include <sys/bio.h> 53 #include <sys/malloc.h> 54 #include <sys/uio.h> 55 #include <sys/sysctl.h> 56 #include <sys/endian.h> 57 #include <sys/queue.h> 58 #include <sys/kthread.h> 59 #include <sys/taskqueue.h> 60 #include <sys/sbuf.h> 61 62 #include <machine/bus.h> 63 #include <machine/resource.h> 64 #include <sys/rman.h> 65 66 #include <machine/stdarg.h> 67 68 #include <cam/cam.h> 69 #include <cam/cam_ccb.h> 70 #include <cam/cam_debug.h> 71 #include <cam/cam_sim.h> 72 #include <cam/cam_xpt_sim.h> 73 #include <cam/cam_xpt_periph.h> 74 #include <cam/cam_periph.h> 75 #include <cam/scsi/scsi_all.h> 76 #include <cam/scsi/scsi_message.h> 77 #include <cam/scsi/smp_all.h> 78 79 #include <dev/nvme/nvme.h> 80 #include "mpi/mpi30_api.h" 81 #include "mpi3mr_cam.h" 82 #include "mpi3mr.h" 83 #include <sys/time.h> /* XXX for pcpu.h */ 84 #include <sys/pcpu.h> /* XXX for PCPU_GET */ 85 #include <asm/unaligned.h> 86 87 #define smp_processor_id() PCPU_GET(cpuid) 88 89 static void 90 mpi3mr_enqueue_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cm); 91 static void 92 mpi3mr_map_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cm); 93 void 94 mpi3mr_release_simq_reinit(struct mpi3mr_cam_softc *cam_sc); 95 static void 96 mpi3mr_freeup_events(struct mpi3mr_softc *sc); 97 98 extern int 99 mpi3mr_register_events(struct mpi3mr_softc *sc); 100 extern void mpi3mr_add_sg_single(void *paddr, U8 flags, U32 length, 101 bus_addr_t dma_addr); 102 103 static U32 event_count; 104 105 static 106 inline void mpi3mr_divert_ws(Mpi3SCSIIORequest_t *req, 107 struct ccb_scsiio *csio, 108 U16 ws_len) 109 { 110 U8 unmap = 0, ndob = 0; 111 U32 num_blocks = 0; 112 U8 opcode = scsiio_cdb_ptr(csio)[0]; 113 U16 service_action = ((scsiio_cdb_ptr(csio)[8] << 8) | scsiio_cdb_ptr(csio)[9]); 114 115 116 if (opcode == WRITE_SAME_16 || 117 (opcode == VARIABLE_LEN_CDB && 118 service_action == WRITE_SAME_32)) { 119 120 int unmap_ndob_index = (opcode == WRITE_SAME_16) ? 1 : 10; 121 122 unmap = scsiio_cdb_ptr(csio)[unmap_ndob_index] & 0x08; 123 ndob = scsiio_cdb_ptr(csio)[unmap_ndob_index] & 0x01; 124 num_blocks = get_unaligned_be32(scsiio_cdb_ptr(csio) + 125 ((opcode == WRITE_SAME_16) ? 10 : 28)); 126 127 /* Check conditions for diversion to firmware */ 128 if (unmap && ndob && num_blocks > ws_len) { 129 req->MsgFlags |= MPI3_SCSIIO_MSGFLAGS_DIVERT_TO_FIRMWARE; 130 req->Flags = htole32(le32toh(req->Flags) | 131 MPI3_SCSIIO_FLAGS_DIVERT_REASON_WRITE_SAME_TOO_LARGE); 132 } 133 } 134 } 135 136 static void mpi3mr_prepare_sgls(void *arg, 137 bus_dma_segment_t *segs, int nsegs, int error) 138 { 139 struct mpi3mr_softc *sc; 140 struct mpi3mr_cmd *cm; 141 u_int i; 142 bus_addr_t chain_dma; 143 void *chain; 144 U8 *sg_local; 145 U32 chain_length; 146 int sges_left; 147 U32 sges_in_segment; 148 U8 simple_sgl_flags; 149 U8 simple_sgl_flags_last; 150 U8 last_chain_sgl_flags; 151 struct mpi3mr_chain *chain_req; 152 Mpi3SCSIIORequest_t *scsiio_req; 153 union ccb *ccb; 154 155 cm = (struct mpi3mr_cmd *)arg; 156 sc = cm->sc; 157 scsiio_req = (Mpi3SCSIIORequest_t *) &cm->io_request; 158 ccb = cm->ccb; 159 160 if (error) { 161 device_printf(sc->mpi3mr_dev, "%s: error=%d\n",__func__, error); 162 if (error == EFBIG) { 163 mpi3mr_set_ccbstatus(ccb, CAM_REQ_TOO_BIG); 164 } else { 165 mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP_ERR); 166 } 167 mpi3mr_release_command(cm); 168 xpt_done(ccb); 169 return; 170 } 171 172 if (cm->data_dir == MPI3MR_READ) 173 bus_dmamap_sync(sc->buffer_dmat, cm->dmamap, 174 BUS_DMASYNC_PREREAD); 175 if (cm->data_dir == MPI3MR_WRITE) 176 bus_dmamap_sync(sc->buffer_dmat, cm->dmamap, 177 BUS_DMASYNC_PREWRITE); 178 179 KASSERT(nsegs <= sc->max_sgl_entries && nsegs > 0, 180 ("%s: bad SGE count: %d\n", device_get_nameunit(sc->mpi3mr_dev), nsegs)); 181 KASSERT(scsiio_req->DataLength != 0, 182 ("%s: Data segments (%d), but DataLength == 0\n", 183 device_get_nameunit(sc->mpi3mr_dev), nsegs)); 184 185 simple_sgl_flags = MPI3_SGE_FLAGS_ELEMENT_TYPE_SIMPLE | 186 MPI3_SGE_FLAGS_DLAS_SYSTEM; 187 simple_sgl_flags_last = simple_sgl_flags | 188 MPI3_SGE_FLAGS_END_OF_LIST; 189 last_chain_sgl_flags = MPI3_SGE_FLAGS_ELEMENT_TYPE_LAST_CHAIN | 190 MPI3_SGE_FLAGS_DLAS_SYSTEM; 191 192 sg_local = (U8 *)&scsiio_req->SGL; 193 194 sges_left = nsegs; 195 196 sges_in_segment = (sc->facts.op_req_sz - 197 offsetof(Mpi3SCSIIORequest_t, SGL))/sizeof(Mpi3SGESimple_t); 198 199 i = 0; 200 201 mpi3mr_dprint(sc, MPI3MR_TRACE, "SGE count: %d IO size: %d\n", 202 nsegs, scsiio_req->DataLength); 203 204 if (sges_left <= sges_in_segment) 205 goto fill_in_last_segment; 206 207 /* fill in main message segment when there is a chain following */ 208 while (sges_in_segment > 1) { 209 mpi3mr_add_sg_single(sg_local, simple_sgl_flags, 210 segs[i].ds_len, segs[i].ds_addr); 211 sg_local += sizeof(Mpi3SGESimple_t); 212 sges_left--; 213 sges_in_segment--; 214 i++; 215 } 216 217 chain_req = &sc->chain_sgl_list[cm->hosttag]; 218 219 chain = chain_req->buf; 220 chain_dma = chain_req->buf_phys; 221 memset(chain_req->buf, 0, sc->max_sgl_entries * sizeof(Mpi3SGESimple_t)); 222 sges_in_segment = sges_left; 223 chain_length = sges_in_segment * sizeof(Mpi3SGESimple_t); 224 225 mpi3mr_add_sg_single(sg_local, last_chain_sgl_flags, 226 chain_length, chain_dma); 227 228 sg_local = chain; 229 230 fill_in_last_segment: 231 while (sges_left > 0) { 232 if (sges_left == 1) 233 mpi3mr_add_sg_single(sg_local, 234 simple_sgl_flags_last, segs[i].ds_len, 235 segs[i].ds_addr); 236 else 237 mpi3mr_add_sg_single(sg_local, simple_sgl_flags, 238 segs[i].ds_len, segs[i].ds_addr); 239 sg_local += sizeof(Mpi3SGESimple_t); 240 sges_left--; 241 i++; 242 } 243 244 /* 245 * Now that we've created the sgls, we send the request to the device. 246 * Unlike in Linux, dmaload isn't guaranteed to load every time, but 247 * this function is always called when the resources are available, so 248 * we can send the request to hardware here always. mpi3mr_map_request 249 * knows about this quirk and will only take evasive action when an 250 * error other than EINPROGRESS is returned from dmaload. 251 */ 252 mpi3mr_enqueue_request(sc, cm); 253 254 return; 255 } 256 257 static void 258 mpi3mr_map_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cm) 259 { 260 u_int32_t retcode = 0; 261 union ccb *ccb; 262 263 ccb = cm->ccb; 264 if (cm->data != NULL) { 265 mtx_lock(&sc->io_lock); 266 /* Map data buffer into bus space */ 267 retcode = bus_dmamap_load_ccb(sc->buffer_dmat, cm->dmamap, 268 ccb, mpi3mr_prepare_sgls, cm, 0); 269 mtx_unlock(&sc->io_lock); 270 if (retcode != 0 && retcode != EINPROGRESS) { 271 device_printf(sc->mpi3mr_dev, 272 "bus_dmamap_load(): retcode = %d\n", retcode); 273 /* 274 * Any other error means prepare_sgls wasn't called, and 275 * will never be called, so we have to mop up. This error 276 * should never happen, though. 277 */ 278 mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP_ERR); 279 mpi3mr_release_command(cm); 280 xpt_done(ccb); 281 } 282 } else { 283 /* 284 * No data, we enqueue it directly here. 285 */ 286 mpi3mr_enqueue_request(sc, cm); 287 } 288 } 289 290 void 291 mpi3mr_unmap_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cmd) 292 { 293 if (cmd->data != NULL) { 294 if (cmd->data_dir == MPI3MR_READ) 295 bus_dmamap_sync(sc->buffer_dmat, cmd->dmamap, BUS_DMASYNC_POSTREAD); 296 if (cmd->data_dir == MPI3MR_WRITE) 297 bus_dmamap_sync(sc->buffer_dmat, cmd->dmamap, BUS_DMASYNC_POSTWRITE); 298 mtx_lock(&sc->io_lock); 299 bus_dmamap_unload(sc->buffer_dmat, cmd->dmamap); 300 mtx_unlock(&sc->io_lock); 301 } 302 } 303 304 /** 305 * mpi3mr_allow_unmap_to_fw - Whether an unmap is allowed to fw 306 * @sc: Adapter instance reference 307 * @ccb: SCSI Command reference 308 * 309 * The controller hardware cannot handle certain unmap commands 310 * for NVMe drives, this routine checks those and return true 311 * and completes the SCSI command with proper status and sense 312 * data. 313 * 314 * Return: TRUE for allowed unmap, FALSE otherwise. 315 */ 316 static bool mpi3mr_allow_unmap_to_fw(struct mpi3mr_softc *sc, 317 union ccb *ccb) 318 { 319 struct ccb_scsiio *csio; 320 uint16_t param_list_len, block_desc_len, trunc_param_len = 0; 321 322 csio = &ccb->csio; 323 param_list_len = (uint16_t) ((scsiio_cdb_ptr(csio)[7] << 8) | scsiio_cdb_ptr(csio)[8]); 324 325 switch(pci_get_revid(sc->mpi3mr_dev)) { 326 case SAS4116_CHIP_REV_A0: 327 if (!param_list_len) { 328 mpi3mr_dprint(sc, MPI3MR_ERROR, 329 "%s: CDB received with zero parameter length\n", 330 __func__); 331 mpi3mr_print_cdb(ccb); 332 mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); 333 xpt_done(ccb); 334 return false; 335 } 336 337 if (param_list_len < 24) { 338 mpi3mr_dprint(sc, MPI3MR_ERROR, 339 "%s: CDB received with invalid param_list_len: %d\n", 340 __func__, param_list_len); 341 mpi3mr_print_cdb(ccb); 342 scsi_set_sense_data(&ccb->csio.sense_data, 343 /*sense_format*/ SSD_TYPE_FIXED, 344 /*current_error*/ 1, 345 /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST, 346 /*asc*/ 0x1A, 347 /*ascq*/ 0x00, 348 /*extra args*/ SSD_ELEM_NONE); 349 ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; 350 ccb->ccb_h.status = 351 CAM_SCSI_STATUS_ERROR | 352 CAM_AUTOSNS_VALID; 353 return false; 354 } 355 356 if (param_list_len != csio->dxfer_len) { 357 mpi3mr_dprint(sc, MPI3MR_ERROR, 358 "%s: CDB received with param_list_len: %d bufflen: %d\n", 359 __func__, param_list_len, csio->dxfer_len); 360 mpi3mr_print_cdb(ccb); 361 scsi_set_sense_data(&ccb->csio.sense_data, 362 /*sense_format*/ SSD_TYPE_FIXED, 363 /*current_error*/ 1, 364 /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST, 365 /*asc*/ 0x1A, 366 /*ascq*/ 0x00, 367 /*extra args*/ SSD_ELEM_NONE); 368 ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; 369 ccb->ccb_h.status = 370 CAM_SCSI_STATUS_ERROR | 371 CAM_AUTOSNS_VALID; 372 xpt_done(ccb); 373 return false; 374 } 375 376 block_desc_len = (uint16_t) (csio->data_ptr[2] << 8 | csio->data_ptr[3]); 377 378 if (block_desc_len < 16) { 379 mpi3mr_dprint(sc, MPI3MR_ERROR, 380 "%s: Invalid descriptor length in param list: %d\n", 381 __func__, block_desc_len); 382 mpi3mr_print_cdb(ccb); 383 scsi_set_sense_data(&ccb->csio.sense_data, 384 /*sense_format*/ SSD_TYPE_FIXED, 385 /*current_error*/ 1, 386 /*sense_key*/ SSD_KEY_ILLEGAL_REQUEST, 387 /*asc*/ 0x26, 388 /*ascq*/ 0x00, 389 /*extra args*/ SSD_ELEM_NONE); 390 ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND; 391 ccb->ccb_h.status = 392 CAM_SCSI_STATUS_ERROR | 393 CAM_AUTOSNS_VALID; 394 xpt_done(ccb); 395 return false; 396 } 397 398 if (param_list_len > (block_desc_len + 8)) { 399 mpi3mr_print_cdb(ccb); 400 mpi3mr_dprint(sc, MPI3MR_INFO, 401 "%s: Truncating param_list_len(%d) to block_desc_len+8(%d)\n", 402 __func__, param_list_len, (block_desc_len + 8)); 403 param_list_len = block_desc_len + 8; 404 scsiio_cdb_ptr(csio)[7] = (param_list_len >> 8) | 0xff; 405 scsiio_cdb_ptr(csio)[8] = param_list_len | 0xff; 406 mpi3mr_print_cdb(ccb); 407 } 408 break; 409 410 case SAS4116_CHIP_REV_B0: 411 if ((param_list_len > 24) && ((param_list_len - 8) & 0xF)) { 412 trunc_param_len -= (param_list_len - 8) & 0xF; 413 mpi3mr_print_cdb(ccb); 414 mpi3mr_dprint(sc, MPI3MR_INFO, 415 "%s: Truncating param_list_len from (%d) to (%d)\n", 416 __func__, param_list_len, trunc_param_len); 417 scsiio_cdb_ptr(csio)[7] = (param_list_len >> 8) | 0xff; 418 scsiio_cdb_ptr(csio)[8] = param_list_len | 0xff; 419 mpi3mr_print_cdb(ccb); 420 } 421 break; 422 } 423 424 return true; 425 } 426 427 /** 428 * mpi3mr_tm_response_name - get TM response as a string 429 * @resp_code: TM response code 430 * 431 * Convert known task management response code as a readable 432 * string. 433 * 434 * Return: response code string. 435 */ 436 static const char* mpi3mr_tm_response_name(U8 resp_code) 437 { 438 char *desc; 439 440 switch (resp_code) { 441 case MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE: 442 desc = "task management request completed"; 443 break; 444 case MPI3_SCSITASKMGMT_RSPCODE_INVALID_FRAME: 445 desc = "invalid frame"; 446 break; 447 case MPI3_SCSITASKMGMT_RSPCODE_TM_FUNCTION_NOT_SUPPORTED: 448 desc = "task management request not supported"; 449 break; 450 case MPI3_SCSITASKMGMT_RSPCODE_TM_FAILED: 451 desc = "task management request failed"; 452 break; 453 case MPI3_SCSITASKMGMT_RSPCODE_TM_SUCCEEDED: 454 desc = "task management request succeeded"; 455 break; 456 case MPI3_SCSITASKMGMT_RSPCODE_TM_INVALID_LUN: 457 desc = "invalid LUN"; 458 break; 459 case MPI3_SCSITASKMGMT_RSPCODE_TM_OVERLAPPED_TAG: 460 desc = "overlapped tag attempted"; 461 break; 462 case MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC: 463 desc = "task queued, however not sent to target"; 464 break; 465 case MPI3_SCSITASKMGMT_RSPCODE_TM_NVME_DENIED: 466 desc = "task management request denied by NVMe device"; 467 break; 468 default: 469 desc = "unknown"; 470 break; 471 } 472 473 return desc; 474 } 475 476 void mpi3mr_poll_pend_io_completions(struct mpi3mr_softc *sc) 477 { 478 int i; 479 int num_of_reply_queues = sc->num_queues; 480 struct mpi3mr_irq_context *irq_ctx; 481 482 for (i = 0; i < num_of_reply_queues; i++) { 483 irq_ctx = &sc->irq_ctx[i]; 484 mpi3mr_complete_io_cmd(sc, irq_ctx); 485 } 486 } 487 488 void 489 trigger_reset_from_watchdog(struct mpi3mr_softc *sc, U8 reset_type, U16 reset_reason) 490 { 491 if (sc->reset_in_progress) { 492 mpi3mr_dprint(sc, MPI3MR_INFO, "Another reset is in progress, no need to trigger the reset\n"); 493 return; 494 } 495 sc->reset.type = reset_type; 496 sc->reset.reason = reset_reason; 497 498 return; 499 } 500 501 /** 502 * mpi3mr_issue_tm - Issue Task Management request 503 * @sc: Adapter instance reference 504 * @tm_type: Task Management type 505 * @handle: Device handle 506 * @lun: lun ID 507 * @htag: Host tag of the TM request 508 * @timeout: TM timeout value 509 * @drv_cmd: Internal command tracker 510 * @resp_code: Response code place holder 511 * @cmd: Timed out command reference 512 * 513 * Issues a Task Management Request to the controller for a 514 * specified target, lun and command and wait for its completion 515 * and check TM response. Recover the TM if it timed out by 516 * issuing controller reset. 517 * 518 * Return: 0 on success, non-zero on errors 519 */ 520 static int 521 mpi3mr_issue_tm(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cmd, 522 U8 tm_type, unsigned long timeout) 523 { 524 int retval = 0; 525 MPI3_SCSI_TASK_MGMT_REQUEST tm_req; 526 MPI3_SCSI_TASK_MGMT_REPLY *tm_reply = NULL; 527 struct mpi3mr_drvr_cmd *drv_cmd = NULL; 528 struct mpi3mr_target *tgtdev = NULL; 529 struct mpi3mr_op_req_queue *op_req_q = NULL; 530 union ccb *ccb; 531 U8 resp_code; 532 533 534 if (sc->unrecoverable) { 535 mpi3mr_dprint(sc, MPI3MR_INFO, 536 "Controller is in unrecoverable state!! TM not required\n"); 537 return retval; 538 } 539 if (sc->reset_in_progress) { 540 mpi3mr_dprint(sc, MPI3MR_INFO, 541 "controller reset in progress!! TM not required\n"); 542 return retval; 543 } 544 545 if (!cmd->ccb) { 546 mpi3mr_dprint(sc, MPI3MR_ERROR, "SCSIIO command timed-out with NULL ccb\n"); 547 return retval; 548 } 549 ccb = cmd->ccb; 550 551 tgtdev = cmd->targ; 552 if (tgtdev == NULL) { 553 mpi3mr_dprint(sc, MPI3MR_ERROR, "Device does not exist target ID:0x%x," 554 "TM is not required\n", ccb->ccb_h.target_id); 555 return retval; 556 } 557 if (tgtdev->dev_removed == 1) { 558 mpi3mr_dprint(sc, MPI3MR_ERROR, "Device(0x%x) is removed, TM is not required\n", 559 ccb->ccb_h.target_id); 560 return retval; 561 } 562 563 drv_cmd = &sc->host_tm_cmds; 564 mtx_lock(&drv_cmd->lock); 565 566 memset(&tm_req, 0, sizeof(tm_req)); 567 tm_req.DevHandle = htole16(tgtdev->dev_handle); 568 tm_req.TaskType = tm_type; 569 tm_req.HostTag = htole16(MPI3MR_HOSTTAG_TMS); 570 int_to_lun(ccb->ccb_h.target_lun, tm_req.LUN); 571 tm_req.Function = MPI3_FUNCTION_SCSI_TASK_MGMT; 572 drv_cmd->state = MPI3MR_CMD_PENDING; 573 drv_cmd->is_waiting = 1; 574 drv_cmd->callback = NULL; 575 576 if (ccb) { 577 if (tm_type == MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK) { 578 op_req_q = &sc->op_req_q[cmd->req_qidx]; 579 tm_req.TaskHostTag = htole16(cmd->hosttag); 580 tm_req.TaskRequestQueueID = htole16(op_req_q->qid); 581 } 582 } 583 584 if (tgtdev) 585 mpi3mr_atomic_inc(&tgtdev->block_io); 586 587 if (tgtdev && (tgtdev->dev_type == MPI3_DEVICE_DEVFORM_PCIE)) { 588 if ((tm_type == MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK) 589 && tgtdev->dev_spec.pcie_inf.abort_to) 590 timeout = tgtdev->dev_spec.pcie_inf.abort_to; 591 else if ((tm_type == MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET) 592 && tgtdev->dev_spec.pcie_inf.reset_to) 593 timeout = tgtdev->dev_spec.pcie_inf.reset_to; 594 } 595 596 sc->tm_chan = (void *)&drv_cmd; 597 598 mpi3mr_dprint(sc, MPI3MR_DEBUG_TM, 599 "posting task management request: type(%d), handle(0x%04x)\n", 600 tm_type, tgtdev->dev_handle); 601 602 init_completion(&drv_cmd->completion); 603 retval = mpi3mr_submit_admin_cmd(sc, &tm_req, sizeof(tm_req)); 604 if (retval) { 605 mpi3mr_dprint(sc, MPI3MR_ERROR, 606 "posting task management request is failed\n"); 607 retval = -1; 608 goto out_unlock; 609 } 610 wait_for_completion_timeout_tm(&drv_cmd->completion, timeout, sc); 611 612 if (!(drv_cmd->state & MPI3MR_CMD_COMPLETE)) { 613 drv_cmd->is_waiting = 0; 614 retval = -1; 615 if (!(drv_cmd->state & MPI3MR_CMD_RESET)) { 616 mpi3mr_dprint(sc, MPI3MR_ERROR, 617 "task management request timed out after %ld seconds\n", timeout); 618 if (sc->mpi3mr_debug & MPI3MR_DEBUG_TM) { 619 mpi3mr_dprint(sc, MPI3MR_INFO, "tm_request dump\n"); 620 mpi3mr_hexdump(&tm_req, sizeof(tm_req), 8); 621 } 622 trigger_reset_from_watchdog(sc, MPI3MR_TRIGGER_SOFT_RESET, MPI3MR_RESET_FROM_TM_TIMEOUT); 623 retval = ETIMEDOUT; 624 } 625 goto out_unlock; 626 } 627 628 if (!(drv_cmd->state & MPI3MR_CMD_REPLYVALID)) { 629 mpi3mr_dprint(sc, MPI3MR_ERROR, 630 "invalid task management reply message\n"); 631 retval = -1; 632 goto out_unlock; 633 } 634 tm_reply = (MPI3_SCSI_TASK_MGMT_REPLY *)drv_cmd->reply; 635 636 switch (drv_cmd->ioc_status) { 637 case MPI3_IOCSTATUS_SUCCESS: 638 resp_code = tm_reply->ResponseData & MPI3MR_RI_MASK_RESPCODE; 639 break; 640 case MPI3_IOCSTATUS_SCSI_IOC_TERMINATED: 641 resp_code = MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE; 642 break; 643 default: 644 mpi3mr_dprint(sc, MPI3MR_ERROR, 645 "task management request to handle(0x%04x) is failed with ioc_status(0x%04x) log_info(0x%08x)\n", 646 tgtdev->dev_handle, drv_cmd->ioc_status, drv_cmd->ioc_loginfo); 647 retval = -1; 648 goto out_unlock; 649 } 650 651 switch (resp_code) { 652 case MPI3_SCSITASKMGMT_RSPCODE_TM_SUCCEEDED: 653 case MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE: 654 break; 655 case MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC: 656 if (tm_type != MPI3_SCSITASKMGMT_TASKTYPE_QUERY_TASK) 657 retval = -1; 658 break; 659 default: 660 retval = -1; 661 break; 662 } 663 664 mpi3mr_dprint(sc, MPI3MR_DEBUG_TM, 665 "task management request type(%d) completed for handle(0x%04x) with ioc_status(0x%04x), log_info(0x%08x)" 666 "termination_count(%u), response:%s(0x%x)\n", tm_type, tgtdev->dev_handle, drv_cmd->ioc_status, drv_cmd->ioc_loginfo, 667 tm_reply->TerminationCount, mpi3mr_tm_response_name(resp_code), resp_code); 668 669 if (retval) 670 goto out_unlock; 671 672 mpi3mr_disable_interrupts(sc); 673 mpi3mr_poll_pend_io_completions(sc); 674 mpi3mr_enable_interrupts(sc); 675 mpi3mr_poll_pend_io_completions(sc); 676 677 switch (tm_type) { 678 case MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK: 679 if (cmd->state == MPI3MR_CMD_STATE_IN_TM) { 680 mpi3mr_dprint(sc, MPI3MR_ERROR, 681 "%s: task abort returned success from firmware but corresponding CCB (%p) was not terminated" 682 "marking task abort failed!\n", sc->name, cmd->ccb); 683 retval = -1; 684 } 685 break; 686 case MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET: 687 if (mpi3mr_atomic_read(&tgtdev->outstanding)) { 688 mpi3mr_dprint(sc, MPI3MR_ERROR, 689 "%s: target reset returned success from firmware but IOs are still pending on the target (%p)" 690 "marking target reset failed!\n", 691 sc->name, tgtdev); 692 retval = -1; 693 } 694 break; 695 default: 696 break; 697 } 698 699 out_unlock: 700 drv_cmd->state = MPI3MR_CMD_NOTUSED; 701 mtx_unlock(&drv_cmd->lock); 702 if (tgtdev && mpi3mr_atomic_read(&tgtdev->block_io) > 0) 703 mpi3mr_atomic_dec(&tgtdev->block_io); 704 705 return retval; 706 } 707 708 /** 709 * mpi3mr_task_abort- Abort error handling callback 710 * @cmd: Timed out command reference 711 * 712 * Issue Abort Task Management if the command is in LLD scope 713 * and verify if it is aborted successfully and return status 714 * accordingly. 715 * 716 * Return: SUCCESS of successful abort the SCSI command else FAILED 717 */ 718 static int mpi3mr_task_abort(struct mpi3mr_cmd *cmd) 719 { 720 int retval = 0; 721 struct mpi3mr_softc *sc; 722 union ccb *ccb; 723 724 sc = cmd->sc; 725 726 if (!cmd->ccb) { 727 mpi3mr_dprint(sc, MPI3MR_ERROR, "SCSIIO command timed-out with NULL ccb\n"); 728 return retval; 729 } 730 ccb = cmd->ccb; 731 732 mpi3mr_dprint(sc, MPI3MR_INFO, 733 "attempting abort task for ccb(%p)\n", ccb); 734 735 mpi3mr_print_cdb(ccb); 736 737 if (cmd->state != MPI3MR_CMD_STATE_BUSY) { 738 mpi3mr_dprint(sc, MPI3MR_INFO, 739 "%s: ccb is not in driver scope, abort task is not required\n", 740 sc->name); 741 return retval; 742 } 743 cmd->state = MPI3MR_CMD_STATE_IN_TM; 744 745 retval = mpi3mr_issue_tm(sc, cmd, MPI3_SCSITASKMGMT_TASKTYPE_ABORT_TASK, MPI3MR_ABORTTM_TIMEOUT); 746 747 mpi3mr_dprint(sc, MPI3MR_INFO, 748 "abort task is %s for ccb(%p)\n", ((retval == 0) ? "SUCCESS" : "FAILED"), ccb); 749 750 return retval; 751 } 752 753 /** 754 * mpi3mr_target_reset - Target reset error handling callback 755 * @cmd: Timed out command reference 756 * 757 * Issue Target reset Task Management and verify the SCSI commands are 758 * terminated successfully and return status accordingly. 759 * 760 * Return: SUCCESS of successful termination of the SCSI commands else 761 * FAILED 762 */ 763 static int mpi3mr_target_reset(struct mpi3mr_cmd *cmd) 764 { 765 int retval = 0; 766 struct mpi3mr_softc *sc; 767 struct mpi3mr_target *target; 768 769 sc = cmd->sc; 770 771 target = cmd->targ; 772 if (target == NULL) { 773 mpi3mr_dprint(sc, MPI3MR_XINFO, "Device does not exist for target:0x%p," 774 "target reset is not required\n", target); 775 return retval; 776 } 777 778 mpi3mr_dprint(sc, MPI3MR_INFO, 779 "attempting target reset on target(%d)\n", target->per_id); 780 781 782 if (mpi3mr_atomic_read(&target->outstanding)) { 783 mpi3mr_dprint(sc, MPI3MR_INFO, 784 "no outstanding IOs on the target(%d)," 785 " target reset not required.\n", target->per_id); 786 return retval; 787 } 788 789 retval = mpi3mr_issue_tm(sc, cmd, MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET, MPI3MR_RESETTM_TIMEOUT); 790 791 mpi3mr_dprint(sc, MPI3MR_INFO, 792 "target reset is %s for target(%d)\n", ((retval == 0) ? "SUCCESS" : "FAILED"), 793 target->per_id); 794 795 return retval; 796 } 797 798 /** 799 * mpi3mr_get_fw_pending_ios - Calculate pending I/O count 800 * @sc: Adapter instance reference 801 * 802 * Calculate the pending I/Os for the controller and return. 803 * 804 * Return: Number of pending I/Os 805 */ 806 static inline int mpi3mr_get_fw_pending_ios(struct mpi3mr_softc *sc) 807 { 808 U16 i, pend_ios = 0; 809 810 for (i = 0; i < sc->num_queues; i++) 811 pend_ios += mpi3mr_atomic_read(&sc->op_reply_q[i].pend_ios); 812 return pend_ios; 813 } 814 815 /** 816 * mpi3mr_wait_for_host_io - block for I/Os to complete 817 * @sc: Adapter instance reference 818 * @timeout: time out in seconds 819 * 820 * Waits for pending I/Os for the given adapter to complete or 821 * to hit the timeout. 822 * 823 * Return: Nothing 824 */ 825 static int mpi3mr_wait_for_host_io(struct mpi3mr_softc *sc, U32 timeout) 826 { 827 enum mpi3mr_iocstate iocstate; 828 829 iocstate = mpi3mr_get_iocstate(sc); 830 if (iocstate != MRIOC_STATE_READY) { 831 mpi3mr_dprint(sc, MPI3MR_XINFO, "%s :Controller is in NON-READY state! Proceed with Reset\n", __func__); 832 return -1; 833 } 834 835 if (!mpi3mr_get_fw_pending_ios(sc)) 836 return 0; 837 838 mpi3mr_dprint(sc, MPI3MR_INFO, 839 "%s :Waiting for %d seconds prior to reset for %d pending I/Os to complete\n", 840 __func__, timeout, mpi3mr_get_fw_pending_ios(sc)); 841 842 int i; 843 for (i = 0; i < timeout; i++) { 844 if (!mpi3mr_get_fw_pending_ios(sc)) { 845 mpi3mr_dprint(sc, MPI3MR_INFO, "%s :All pending I/Os got completed while waiting! Reset not required\n", __func__); 846 return 0; 847 848 } 849 iocstate = mpi3mr_get_iocstate(sc); 850 if (iocstate != MRIOC_STATE_READY) { 851 mpi3mr_dprint(sc, MPI3MR_XINFO, "%s :Controller state becomes NON-READY while waiting! dont wait further" 852 "Proceed with Reset\n", __func__); 853 return -1; 854 } 855 DELAY(1000 * 1000); 856 } 857 858 mpi3mr_dprint(sc, MPI3MR_INFO, "%s :Pending I/Os after wait exaust is %d! Proceed with Reset\n", __func__, 859 mpi3mr_get_fw_pending_ios(sc)); 860 861 return -1; 862 } 863 864 static void 865 mpi3mr_scsiio_timeout(void *data) 866 { 867 int retval = 0; 868 struct mpi3mr_softc *sc; 869 struct mpi3mr_cmd *cmd; 870 struct mpi3mr_target *targ_dev = NULL; 871 872 if (!data) 873 return; 874 875 cmd = (struct mpi3mr_cmd *)data; 876 sc = cmd->sc; 877 878 if (cmd->ccb == NULL) { 879 mpi3mr_dprint(sc, MPI3MR_ERROR, "SCSIIO command timed-out with NULL ccb\n"); 880 return; 881 } 882 883 /* 884 * TMs are not supported for IO timeouts on VD/LD, so directly issue controller reset 885 * with max timeout for outstanding IOs to complete is 180sec. 886 */ 887 targ_dev = cmd->targ; 888 if (targ_dev && (targ_dev->dev_type == MPI3_DEVICE_DEVFORM_VD)) { 889 if (mpi3mr_wait_for_host_io(sc, MPI3MR_RAID_ERRREC_RESET_TIMEOUT)) 890 trigger_reset_from_watchdog(sc, MPI3MR_TRIGGER_SOFT_RESET, MPI3MR_RESET_FROM_SCSIIO_TIMEOUT); 891 return; 892 } 893 894 /* Issue task abort to recover the timed out IO */ 895 retval = mpi3mr_task_abort(cmd); 896 if (!retval || (retval == ETIMEDOUT)) 897 return; 898 899 /* 900 * task abort has failed to recover the timed out IO, 901 * try with the target reset 902 */ 903 retval = mpi3mr_target_reset(cmd); 904 if (!retval || (retval == ETIMEDOUT)) 905 return; 906 907 /* 908 * task abort and target reset has failed. So issue Controller reset(soft reset) 909 * through OCR thread context 910 */ 911 trigger_reset_from_watchdog(sc, MPI3MR_TRIGGER_SOFT_RESET, MPI3MR_RESET_FROM_SCSIIO_TIMEOUT); 912 913 return; 914 } 915 916 void int_to_lun(unsigned int lun, U8 *req_lun) 917 { 918 int i; 919 920 memset(req_lun, 0, sizeof(*req_lun)); 921 922 for (i = 0; i < sizeof(lun); i += 2) { 923 req_lun[i] = (lun >> 8) & 0xFF; 924 req_lun[i+1] = lun & 0xFF; 925 lun = lun >> 16; 926 } 927 928 } 929 930 static U16 get_req_queue_index(struct mpi3mr_softc *sc) 931 { 932 U16 i = 0, reply_q_index = 0, reply_q_pend_ios = 0; 933 934 reply_q_pend_ios = mpi3mr_atomic_read(&sc->op_reply_q[0].pend_ios); 935 for (i = 0; i < sc->num_queues; i++) { 936 if (reply_q_pend_ios > mpi3mr_atomic_read(&sc->op_reply_q[i].pend_ios)) { 937 reply_q_pend_ios = mpi3mr_atomic_read(&sc->op_reply_q[i].pend_ios); 938 reply_q_index = i; 939 } 940 } 941 942 return reply_q_index; 943 } 944 945 static void 946 mpi3mr_action_scsiio(struct mpi3mr_cam_softc *cam_sc, union ccb *ccb) 947 { 948 Mpi3SCSIIORequest_t *req = NULL; 949 struct ccb_scsiio *csio; 950 struct mpi3mr_softc *sc; 951 struct mpi3mr_target *targ; 952 struct mpi3mr_cmd *cm; 953 uint8_t scsi_opcode, queue_idx; 954 uint32_t mpi_control; 955 956 sc = cam_sc->sc; 957 mtx_assert(&sc->mpi3mr_mtx, MA_OWNED); 958 959 if (sc->unrecoverable) { 960 mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); 961 xpt_done(ccb); 962 return; 963 } 964 965 csio = &ccb->csio; 966 KASSERT(csio->ccb_h.target_id < cam_sc->maxtargets, 967 ("Target %d out of bounds in XPT_SCSI_IO\n", 968 csio->ccb_h.target_id)); 969 970 scsi_opcode = scsiio_cdb_ptr(csio)[0]; 971 972 if ((sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN) && 973 !((scsi_opcode == SYNCHRONIZE_CACHE) || 974 (scsi_opcode == START_STOP_UNIT))) { 975 mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); 976 xpt_done(ccb); 977 return; 978 } 979 980 targ = mpi3mr_find_target_by_per_id(cam_sc, csio->ccb_h.target_id); 981 if (targ == NULL) { 982 mpi3mr_dprint(sc, MPI3MR_XINFO, "Device with target ID: 0x%x does not exist\n", 983 csio->ccb_h.target_id); 984 mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); 985 xpt_done(ccb); 986 return; 987 } 988 989 if (targ && targ->is_hidden) { 990 mpi3mr_dprint(sc, MPI3MR_XINFO, "Device with target ID: 0x%x is hidden\n", 991 csio->ccb_h.target_id); 992 mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); 993 xpt_done(ccb); 994 return; 995 } 996 997 if (targ->dev_removed == 1) { 998 mpi3mr_dprint(sc, MPI3MR_XINFO, "Device with target ID: 0x%x is removed\n", csio->ccb_h.target_id); 999 mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); 1000 xpt_done(ccb); 1001 return; 1002 } 1003 1004 if (targ->dev_handle == 0x0) { 1005 mpi3mr_dprint(sc, MPI3MR_ERROR, "%s NULL handle for target 0x%x\n", 1006 __func__, csio->ccb_h.target_id); 1007 mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); 1008 xpt_done(ccb); 1009 return; 1010 } 1011 1012 if (mpi3mr_atomic_read(&targ->block_io) || 1013 (sc->reset_in_progress == 1) || (sc->prepare_for_reset == 1)) { 1014 mpi3mr_dprint(sc, MPI3MR_TRACE, "%s target is busy target_id: 0x%x\n", 1015 __func__, csio->ccb_h.target_id); 1016 mpi3mr_set_ccbstatus(ccb, CAM_REQUEUE_REQ); 1017 xpt_done(ccb); 1018 return; 1019 } 1020 1021 /* 1022 * Sometimes, it is possible to get a command that is not "In 1023 * Progress" and was actually aborted by the upper layer. Check for 1024 * this here and complete the command without error. 1025 */ 1026 if (mpi3mr_get_ccbstatus(ccb) != CAM_REQ_INPROG) { 1027 mpi3mr_dprint(sc, MPI3MR_TRACE, "%s Command is not in progress for " 1028 "target %u\n", __func__, csio->ccb_h.target_id); 1029 xpt_done(ccb); 1030 return; 1031 } 1032 /* 1033 * If devinfo is 0 this will be a volume. In that case don't tell CAM 1034 * that the volume has timed out. We want volumes to be enumerated 1035 * until they are deleted/removed, not just failed. 1036 */ 1037 if (targ->flags & MPI3MRSAS_TARGET_INREMOVAL) { 1038 if (targ->devinfo == 0) 1039 mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); 1040 else 1041 mpi3mr_set_ccbstatus(ccb, CAM_SEL_TIMEOUT); 1042 xpt_done(ccb); 1043 return; 1044 } 1045 1046 if ((scsi_opcode == UNMAP) && 1047 (pci_get_device(sc->mpi3mr_dev) == MPI3_MFGPAGE_DEVID_SAS4116) && 1048 (targ->dev_type == MPI3_DEVICE_DEVFORM_PCIE) && 1049 (mpi3mr_allow_unmap_to_fw(sc, ccb) == false)) 1050 return; 1051 1052 cm = mpi3mr_get_command(sc); 1053 if (cm == NULL || (sc->mpi3mr_flags & MPI3MR_FLAGS_DIAGRESET)) { 1054 if (cm != NULL) { 1055 mpi3mr_release_command(cm); 1056 } 1057 if ((cam_sc->flags & MPI3MRSAS_QUEUE_FROZEN) == 0) { 1058 xpt_freeze_simq(cam_sc->sim, 1); 1059 cam_sc->flags |= MPI3MRSAS_QUEUE_FROZEN; 1060 } 1061 ccb->ccb_h.status &= ~CAM_SIM_QUEUED; 1062 mpi3mr_set_ccbstatus(ccb, CAM_REQUEUE_REQ); 1063 xpt_done(ccb); 1064 return; 1065 } 1066 1067 switch (csio->ccb_h.flags & CAM_DIR_MASK) { 1068 case CAM_DIR_IN: 1069 mpi_control = MPI3_SCSIIO_FLAGS_DATADIRECTION_READ; 1070 cm->data_dir = MPI3MR_READ; 1071 break; 1072 case CAM_DIR_OUT: 1073 mpi_control = MPI3_SCSIIO_FLAGS_DATADIRECTION_WRITE; 1074 cm->data_dir = MPI3MR_WRITE; 1075 break; 1076 case CAM_DIR_NONE: 1077 default: 1078 mpi_control = MPI3_SCSIIO_FLAGS_DATADIRECTION_NO_DATA_TRANSFER; 1079 break; 1080 } 1081 1082 if (csio->cdb_len > 16) 1083 mpi_control |= MPI3_SCSIIO_FLAGS_CDB_GREATER_THAN_16; 1084 1085 req = (Mpi3SCSIIORequest_t *)&cm->io_request; 1086 bzero(req, sizeof(*req)); 1087 req->Function = MPI3_FUNCTION_SCSI_IO; 1088 req->HostTag = cm->hosttag; 1089 req->DataLength = htole32(csio->dxfer_len); 1090 req->DevHandle = htole16(targ->dev_handle); 1091 1092 /* 1093 * It looks like the hardware doesn't require an explicit tag 1094 * number for each transaction. SAM Task Management not supported 1095 * at the moment. 1096 */ 1097 switch (csio->tag_action) { 1098 case MSG_HEAD_OF_Q_TAG: 1099 mpi_control |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_HEADOFQ; 1100 break; 1101 case MSG_ORDERED_Q_TAG: 1102 mpi_control |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_ORDEREDQ; 1103 break; 1104 case MSG_ACA_TASK: 1105 mpi_control |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_ACAQ; 1106 break; 1107 case CAM_TAG_ACTION_NONE: 1108 case MSG_SIMPLE_Q_TAG: 1109 default: 1110 mpi_control |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_SIMPLEQ; 1111 break; 1112 } 1113 1114 if (targ->ws_len) 1115 mpi3mr_divert_ws(req, csio, targ->ws_len); 1116 1117 req->Flags = htole32(mpi_control); 1118 1119 if (csio->ccb_h.flags & CAM_CDB_POINTER) 1120 bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len); 1121 else { 1122 KASSERT(csio->cdb_len <= IOCDBLEN, 1123 ("cdb_len %d is greater than IOCDBLEN but CAM_CDB_POINTER " 1124 "is not set", csio->cdb_len)); 1125 bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len); 1126 } 1127 1128 cm->length = csio->dxfer_len; 1129 cm->targ = targ; 1130 int_to_lun(csio->ccb_h.target_lun, req->LUN); 1131 cm->ccb = ccb; 1132 csio->ccb_h.qos.sim_data = sbinuptime(); 1133 queue_idx = get_req_queue_index(sc); 1134 cm->req_qidx = queue_idx; 1135 1136 mpi3mr_dprint(sc, MPI3MR_TRACE, "[QID:%d]: func: %s line:%d CDB: 0x%x targetid: %x SMID: 0x%x\n", 1137 (queue_idx + 1), __func__, __LINE__, scsi_opcode, csio->ccb_h.target_id, cm->hosttag); 1138 1139 switch ((ccb->ccb_h.flags & CAM_DATA_MASK)) { 1140 case CAM_DATA_PADDR: 1141 case CAM_DATA_SG_PADDR: 1142 device_printf(sc->mpi3mr_dev, "%s: physical addresses not supported\n", 1143 __func__); 1144 mpi3mr_set_ccbstatus(ccb, CAM_REQ_INVALID); 1145 mpi3mr_release_command(cm); 1146 xpt_done(ccb); 1147 return; 1148 case CAM_DATA_SG: 1149 device_printf(sc->mpi3mr_dev, "%s: scatter gather is not supported\n", 1150 __func__); 1151 mpi3mr_set_ccbstatus(ccb, CAM_REQ_INVALID); 1152 mpi3mr_release_command(cm); 1153 xpt_done(ccb); 1154 return; 1155 case CAM_DATA_VADDR: 1156 case CAM_DATA_BIO: 1157 if (csio->dxfer_len > (sc->max_sgl_entries * PAGE_SIZE)) { 1158 mpi3mr_set_ccbstatus(ccb, CAM_REQ_TOO_BIG); 1159 mpi3mr_release_command(cm); 1160 xpt_done(ccb); 1161 return; 1162 } 1163 ccb->ccb_h.status |= CAM_SIM_QUEUED; 1164 cm->length = csio->dxfer_len; 1165 if (cm->length) 1166 cm->data = csio->data_ptr; 1167 break; 1168 default: 1169 mpi3mr_set_ccbstatus(ccb, CAM_REQ_INVALID); 1170 mpi3mr_release_command(cm); 1171 xpt_done(ccb); 1172 return; 1173 } 1174 1175 /* Prepare SGEs and queue to hardware */ 1176 mpi3mr_map_request(sc, cm); 1177 } 1178 1179 static void 1180 mpi3mr_enqueue_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cm) 1181 { 1182 static int ratelimit; 1183 struct mpi3mr_op_req_queue *opreqq = &sc->op_req_q[cm->req_qidx]; 1184 struct mpi3mr_throttle_group_info *tg = NULL; 1185 uint32_t data_len_blks = 0; 1186 uint32_t tracked_io_sz = 0; 1187 uint32_t ioc_pend_data_len = 0, tg_pend_data_len = 0; 1188 struct mpi3mr_target *targ = cm->targ; 1189 union ccb *ccb = cm->ccb; 1190 Mpi3SCSIIORequest_t *req = (Mpi3SCSIIORequest_t *)&cm->io_request; 1191 1192 if (sc->iot_enable) { 1193 data_len_blks = ccb->csio.dxfer_len >> 9; 1194 1195 if ((data_len_blks >= sc->io_throttle_data_length) && 1196 targ->io_throttle_enabled) { 1197 1198 tracked_io_sz = data_len_blks; 1199 tg = targ->throttle_group; 1200 if (tg) { 1201 mpi3mr_atomic_add(&sc->pend_large_data_sz, data_len_blks); 1202 mpi3mr_atomic_add(&tg->pend_large_data_sz, data_len_blks); 1203 1204 ioc_pend_data_len = mpi3mr_atomic_read(&sc->pend_large_data_sz); 1205 tg_pend_data_len = mpi3mr_atomic_read(&tg->pend_large_data_sz); 1206 1207 if (ratelimit % 1000) { 1208 mpi3mr_dprint(sc, MPI3MR_IOT, 1209 "large vd_io persist_id(%d), handle(0x%04x), data_len(%d)," 1210 "ioc_pending(%d), tg_pending(%d), ioc_high(%d), tg_high(%d)\n", 1211 targ->per_id, targ->dev_handle, 1212 data_len_blks, ioc_pend_data_len, 1213 tg_pend_data_len, sc->io_throttle_high, 1214 tg->high); 1215 ratelimit++; 1216 } 1217 1218 if (!tg->io_divert && ((ioc_pend_data_len >= 1219 sc->io_throttle_high) || 1220 (tg_pend_data_len >= tg->high))) { 1221 tg->io_divert = 1; 1222 mpi3mr_dprint(sc, MPI3MR_IOT, 1223 "VD: Setting divert flag for tg_id(%d), persist_id(%d)\n", 1224 tg->id, targ->per_id); 1225 if (sc->mpi3mr_debug & MPI3MR_IOT) 1226 mpi3mr_print_cdb(ccb); 1227 mpi3mr_set_io_divert_for_all_vd_in_tg(sc, 1228 tg, 1); 1229 } 1230 } else { 1231 mpi3mr_atomic_add(&sc->pend_large_data_sz, data_len_blks); 1232 ioc_pend_data_len = mpi3mr_atomic_read(&sc->pend_large_data_sz); 1233 if (ratelimit % 1000) { 1234 mpi3mr_dprint(sc, MPI3MR_IOT, 1235 "large pd_io persist_id(%d), handle(0x%04x), data_len(%d), ioc_pending(%d), ioc_high(%d)\n", 1236 targ->per_id, targ->dev_handle, 1237 data_len_blks, ioc_pend_data_len, 1238 sc->io_throttle_high); 1239 ratelimit++; 1240 } 1241 1242 if (ioc_pend_data_len >= sc->io_throttle_high) { 1243 targ->io_divert = 1; 1244 mpi3mr_dprint(sc, MPI3MR_IOT, 1245 "PD: Setting divert flag for persist_id(%d)\n", 1246 targ->per_id); 1247 if (sc->mpi3mr_debug & MPI3MR_IOT) 1248 mpi3mr_print_cdb(ccb); 1249 } 1250 } 1251 } 1252 1253 if (targ->io_divert) { 1254 req->MsgFlags |= MPI3_SCSIIO_MSGFLAGS_DIVERT_TO_FIRMWARE; 1255 req->Flags = htole32(le32toh(req->Flags) | MPI3_SCSIIO_FLAGS_DIVERT_REASON_IO_THROTTLING); 1256 } 1257 } 1258 1259 if (mpi3mr_submit_io(sc, opreqq, (U8 *)&cm->io_request)) { 1260 if (tracked_io_sz) { 1261 mpi3mr_atomic_sub(&sc->pend_large_data_sz, tracked_io_sz); 1262 if (tg) 1263 mpi3mr_atomic_sub(&tg->pend_large_data_sz, tracked_io_sz); 1264 } 1265 mpi3mr_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL); 1266 mpi3mr_release_command(cm); 1267 xpt_done(ccb); 1268 } else { 1269 callout_reset_sbt(&cm->callout, mstosbt(ccb->ccb_h.timeout), 0, 1270 mpi3mr_scsiio_timeout, cm, 0); 1271 cm->callout_owner = true; 1272 mpi3mr_atomic_inc(&sc->fw_outstanding); 1273 mpi3mr_atomic_inc(&targ->outstanding); 1274 if (mpi3mr_atomic_read(&sc->fw_outstanding) > sc->io_cmds_highwater) 1275 sc->io_cmds_highwater++; 1276 } 1277 1278 return; 1279 } 1280 1281 static void 1282 mpi3mr_cam_poll(struct cam_sim *sim) 1283 { 1284 struct mpi3mr_cam_softc *cam_sc; 1285 struct mpi3mr_irq_context *irq_ctx; 1286 struct mpi3mr_softc *sc; 1287 int i; 1288 1289 cam_sc = cam_sim_softc(sim); 1290 sc = cam_sc->sc; 1291 1292 mpi3mr_dprint(cam_sc->sc, MPI3MR_TRACE, "func: %s line: %d is called\n", 1293 __func__, __LINE__); 1294 1295 for (i = 0; i < sc->num_queues; i++) { 1296 irq_ctx = sc->irq_ctx + i; 1297 if (irq_ctx->op_reply_q->qid) { 1298 mpi3mr_complete_io_cmd(sc, irq_ctx); 1299 } 1300 } 1301 } 1302 1303 static void 1304 mpi3mr_cam_action(struct cam_sim *sim, union ccb *ccb) 1305 { 1306 struct mpi3mr_cam_softc *cam_sc; 1307 struct mpi3mr_target *targ; 1308 struct mpi3mr_softc *sc; 1309 1310 cam_sc = cam_sim_softc(sim); 1311 sc = cam_sc->sc; 1312 1313 mpi3mr_dprint(cam_sc->sc, MPI3MR_TRACE, "ccb func_code 0x%x target id: 0x%x\n", 1314 ccb->ccb_h.func_code, ccb->ccb_h.target_id); 1315 1316 mtx_assert(&cam_sc->sc->mpi3mr_mtx, MA_OWNED); 1317 1318 switch (ccb->ccb_h.func_code) { 1319 case XPT_PATH_INQ: 1320 { 1321 struct ccb_pathinq *cpi = &ccb->cpi; 1322 1323 cpi->version_num = 1; 1324 cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16; 1325 cpi->target_sprt = 0; 1326 cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED | PIM_NOSCAN; 1327 cpi->hba_eng_cnt = 0; 1328 cpi->max_target = cam_sc->maxtargets - 1; 1329 cpi->max_lun = 0; 1330 1331 /* 1332 * initiator_id is set here to an ID outside the set of valid 1333 * target IDs (including volumes). 1334 */ 1335 cpi->initiator_id = cam_sc->maxtargets; 1336 strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN); 1337 strlcpy(cpi->hba_vid, "Broadcom", HBA_IDLEN); 1338 strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN); 1339 cpi->unit_number = cam_sim_unit(sim); 1340 cpi->bus_id = cam_sim_bus(sim); 1341 /* 1342 * XXXSLM-I think this needs to change based on config page or 1343 * something instead of hardcoded to 150000. 1344 */ 1345 cpi->base_transfer_speed = 150000; 1346 cpi->transport = XPORT_SAS; 1347 cpi->transport_version = 0; 1348 cpi->protocol = PROTO_SCSI; 1349 cpi->protocol_version = SCSI_REV_SPC; 1350 1351 targ = mpi3mr_find_target_by_per_id(cam_sc, ccb->ccb_h.target_id); 1352 1353 if (targ && (targ->dev_type == MPI3_DEVICE_DEVFORM_PCIE) && 1354 ((targ->dev_spec.pcie_inf.dev_info & 1355 MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) == 1356 MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_NVME_DEVICE)) { 1357 cpi->maxio = targ->dev_spec.pcie_inf.mdts; 1358 mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO, 1359 "PCI device target_id: %u max io size: %u\n", 1360 ccb->ccb_h.target_id, cpi->maxio); 1361 } else { 1362 cpi->maxio = PAGE_SIZE * (sc->max_sgl_entries - 1); 1363 } 1364 mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); 1365 break; 1366 } 1367 case XPT_GET_TRAN_SETTINGS: 1368 { 1369 struct ccb_trans_settings *cts; 1370 struct ccb_trans_settings_sas *sas; 1371 struct ccb_trans_settings_scsi *scsi; 1372 1373 cts = &ccb->cts; 1374 sas = &cts->xport_specific.sas; 1375 scsi = &cts->proto_specific.scsi; 1376 1377 KASSERT(cts->ccb_h.target_id < cam_sc->maxtargets, 1378 ("Target %d out of bounds in XPT_GET_TRAN_SETTINGS\n", 1379 cts->ccb_h.target_id)); 1380 targ = mpi3mr_find_target_by_per_id(cam_sc, cts->ccb_h.target_id); 1381 1382 if (targ == NULL) { 1383 mpi3mr_dprint(cam_sc->sc, MPI3MR_TRACE, "Device with target ID: 0x%x does not exist\n", 1384 cts->ccb_h.target_id); 1385 mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); 1386 break; 1387 } 1388 1389 if ((targ->dev_handle == 0x0) || (targ->dev_removed == 1)) { 1390 mpi3mr_set_ccbstatus(ccb, CAM_DEV_NOT_THERE); 1391 break; 1392 } 1393 1394 cts->protocol_version = SCSI_REV_SPC2; 1395 cts->transport = XPORT_SAS; 1396 cts->transport_version = 0; 1397 1398 sas->valid = CTS_SAS_VALID_SPEED; 1399 1400 switch (targ->link_rate) { 1401 case 0x08: 1402 sas->bitrate = 150000; 1403 break; 1404 case 0x09: 1405 sas->bitrate = 300000; 1406 break; 1407 case 0x0a: 1408 sas->bitrate = 600000; 1409 break; 1410 case 0x0b: 1411 sas->bitrate = 1200000; 1412 break; 1413 default: 1414 sas->valid = 0; 1415 } 1416 1417 cts->protocol = PROTO_SCSI; 1418 scsi->valid = CTS_SCSI_VALID_TQ; 1419 scsi->flags = CTS_SCSI_FLAGS_TAG_ENB; 1420 1421 mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); 1422 break; 1423 } 1424 case XPT_CALC_GEOMETRY: 1425 cam_calc_geometry(&ccb->ccg, /*extended*/1); 1426 mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); 1427 break; 1428 case XPT_RESET_DEV: 1429 mpi3mr_dprint(cam_sc->sc, MPI3MR_INFO, "mpi3mr_action " 1430 "XPT_RESET_DEV\n"); 1431 return; 1432 case XPT_RESET_BUS: 1433 case XPT_ABORT: 1434 case XPT_TERM_IO: 1435 mpi3mr_dprint(cam_sc->sc, MPI3MR_INFO, "mpi3mr_action faking success " 1436 "for abort or reset\n"); 1437 mpi3mr_set_ccbstatus(ccb, CAM_REQ_CMP); 1438 break; 1439 case XPT_SCSI_IO: 1440 mpi3mr_action_scsiio(cam_sc, ccb); 1441 return; 1442 default: 1443 mpi3mr_set_ccbstatus(ccb, CAM_FUNC_NOTAVAIL); 1444 break; 1445 } 1446 xpt_done(ccb); 1447 } 1448 1449 void 1450 mpi3mr_startup_increment(struct mpi3mr_cam_softc *cam_sc) 1451 { 1452 if ((cam_sc->flags & MPI3MRSAS_IN_STARTUP) != 0) { 1453 if (cam_sc->startup_refcount++ == 0) { 1454 /* just starting, freeze the simq */ 1455 mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO, 1456 "%s freezing simq\n", __func__); 1457 xpt_hold_boot(); 1458 } 1459 mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO, "%s refcount %u\n", __func__, 1460 cam_sc->startup_refcount); 1461 } 1462 } 1463 1464 void 1465 mpi3mr_release_simq_reinit(struct mpi3mr_cam_softc *cam_sc) 1466 { 1467 if (cam_sc->flags & MPI3MRSAS_QUEUE_FROZEN) { 1468 cam_sc->flags &= ~MPI3MRSAS_QUEUE_FROZEN; 1469 xpt_release_simq(cam_sc->sim, 1); 1470 mpi3mr_dprint(cam_sc->sc, MPI3MR_INFO, "Unfreezing SIM queue\n"); 1471 } 1472 } 1473 1474 void 1475 mpi3mr_rescan_target(struct mpi3mr_softc *sc, struct mpi3mr_target *targ) 1476 { 1477 struct mpi3mr_cam_softc *cam_sc = sc->cam_sc; 1478 path_id_t pathid; 1479 target_id_t targetid; 1480 union ccb *ccb; 1481 1482 pathid = cam_sim_path(cam_sc->sim); 1483 if (targ == NULL) 1484 targetid = CAM_TARGET_WILDCARD; 1485 else 1486 targetid = targ->per_id; 1487 1488 /* 1489 * Allocate a CCB and schedule a rescan. 1490 */ 1491 ccb = xpt_alloc_ccb_nowait(); 1492 if (ccb == NULL) { 1493 mpi3mr_dprint(sc, MPI3MR_ERROR, "unable to alloc CCB for rescan\n"); 1494 return; 1495 } 1496 1497 if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid, targetid, 1498 CAM_LUN_WILDCARD) != CAM_REQ_CMP) { 1499 mpi3mr_dprint(sc, MPI3MR_ERROR, "unable to create path for rescan\n"); 1500 xpt_free_ccb(ccb); 1501 return; 1502 } 1503 1504 if (targetid == CAM_TARGET_WILDCARD) 1505 ccb->ccb_h.func_code = XPT_SCAN_BUS; 1506 else 1507 ccb->ccb_h.func_code = XPT_SCAN_TGT; 1508 1509 mpi3mr_dprint(sc, MPI3MR_EVENT, "%s target id 0x%x\n", __func__, targetid); 1510 xpt_rescan(ccb); 1511 } 1512 1513 void 1514 mpi3mr_startup_decrement(struct mpi3mr_cam_softc *cam_sc) 1515 { 1516 if ((cam_sc->flags & MPI3MRSAS_IN_STARTUP) != 0) { 1517 if (--cam_sc->startup_refcount == 0) { 1518 /* finished all discovery-related actions, release 1519 * the simq and rescan for the latest topology. 1520 */ 1521 mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO, 1522 "%s releasing simq\n", __func__); 1523 cam_sc->flags &= ~MPI3MRSAS_IN_STARTUP; 1524 xpt_release_simq(cam_sc->sim, 1); 1525 xpt_release_boot(); 1526 } 1527 mpi3mr_dprint(cam_sc->sc, MPI3MR_XINFO, "%s refcount %u\n", __func__, 1528 cam_sc->startup_refcount); 1529 } 1530 } 1531 1532 static void 1533 mpi3mr_fw_event_free(struct mpi3mr_softc *sc, struct mpi3mr_fw_event_work *fw_event) 1534 { 1535 if (!fw_event) 1536 return; 1537 1538 if (fw_event->event_data != NULL) { 1539 free(fw_event->event_data, M_MPI3MR); 1540 fw_event->event_data = NULL; 1541 } 1542 1543 free(fw_event, M_MPI3MR); 1544 fw_event = NULL; 1545 } 1546 1547 static void 1548 mpi3mr_freeup_events(struct mpi3mr_softc *sc) 1549 { 1550 struct mpi3mr_fw_event_work *fw_event = NULL; 1551 mtx_lock(&sc->mpi3mr_mtx); 1552 while ((fw_event = TAILQ_FIRST(&sc->cam_sc->ev_queue)) != NULL) { 1553 TAILQ_REMOVE(&sc->cam_sc->ev_queue, fw_event, ev_link); 1554 mpi3mr_fw_event_free(sc, fw_event); 1555 } 1556 mtx_unlock(&sc->mpi3mr_mtx); 1557 } 1558 1559 static void 1560 mpi3mr_sastopochg_evt_debug(struct mpi3mr_softc *sc, 1561 Mpi3EventDataSasTopologyChangeList_t *event_data) 1562 { 1563 int i; 1564 U16 handle; 1565 U8 reason_code, phy_number; 1566 char *status_str = NULL; 1567 U8 link_rate, prev_link_rate; 1568 1569 switch (event_data->ExpStatus) { 1570 case MPI3_EVENT_SAS_TOPO_ES_NOT_RESPONDING: 1571 status_str = "remove"; 1572 break; 1573 case MPI3_EVENT_SAS_TOPO_ES_RESPONDING: 1574 status_str = "responding"; 1575 break; 1576 case MPI3_EVENT_SAS_TOPO_ES_DELAY_NOT_RESPONDING: 1577 status_str = "remove delay"; 1578 break; 1579 case MPI3_EVENT_SAS_TOPO_ES_NO_EXPANDER: 1580 status_str = "direct attached"; 1581 break; 1582 default: 1583 status_str = "unknown status"; 1584 break; 1585 } 1586 1587 mpi3mr_dprint(sc, MPI3MR_INFO, "%s :sas topology change: (%s)\n", 1588 __func__, status_str); 1589 mpi3mr_dprint(sc, MPI3MR_INFO, 1590 "%s :\texpander_handle(0x%04x), enclosure_handle(0x%04x) " 1591 "start_phy(%02d), num_entries(%d)\n", __func__, 1592 (event_data->ExpanderDevHandle), 1593 (event_data->EnclosureHandle), 1594 event_data->StartPhyNum, event_data->NumEntries); 1595 for (i = 0; i < event_data->NumEntries; i++) { 1596 handle = (event_data->PhyEntry[i].AttachedDevHandle); 1597 if (!handle) 1598 continue; 1599 phy_number = event_data->StartPhyNum + i; 1600 reason_code = event_data->PhyEntry[i].PhyStatus & 1601 MPI3_EVENT_SAS_TOPO_PHY_RC_MASK; 1602 switch (reason_code) { 1603 case MPI3_EVENT_SAS_TOPO_PHY_RC_TARG_NOT_RESPONDING: 1604 status_str = "target remove"; 1605 break; 1606 case MPI3_EVENT_SAS_TOPO_PHY_RC_DELAY_NOT_RESPONDING: 1607 status_str = "delay target remove"; 1608 break; 1609 case MPI3_EVENT_SAS_TOPO_PHY_RC_PHY_CHANGED: 1610 status_str = "link rate change"; 1611 break; 1612 case MPI3_EVENT_SAS_TOPO_PHY_RC_NO_CHANGE: 1613 status_str = "target responding"; 1614 break; 1615 default: 1616 status_str = "unknown"; 1617 break; 1618 } 1619 link_rate = event_data->PhyEntry[i].LinkRate >> 4; 1620 prev_link_rate = event_data->PhyEntry[i].LinkRate & 0xF; 1621 mpi3mr_dprint(sc, MPI3MR_INFO, "%s :\tphy(%02d), attached_handle(0x%04x): %s:" 1622 " link rate: new(0x%02x), old(0x%02x)\n", __func__, 1623 phy_number, handle, status_str, link_rate, prev_link_rate); 1624 } 1625 } 1626 1627 static void 1628 mpi3mr_process_sastopochg_evt(struct mpi3mr_softc *sc, struct mpi3mr_fw_event_work *fwevt) 1629 { 1630 1631 Mpi3EventDataSasTopologyChangeList_t *event_data = 1632 (Mpi3EventDataSasTopologyChangeList_t *)fwevt->event_data; 1633 int i; 1634 U16 handle; 1635 U8 reason_code, link_rate; 1636 struct mpi3mr_target *target = NULL; 1637 1638 1639 mpi3mr_sastopochg_evt_debug(sc, event_data); 1640 1641 for (i = 0; i < event_data->NumEntries; i++) { 1642 handle = le16toh(event_data->PhyEntry[i].AttachedDevHandle); 1643 link_rate = event_data->PhyEntry[i].LinkRate >> 4; 1644 1645 if (!handle) 1646 continue; 1647 target = mpi3mr_find_target_by_dev_handle(sc->cam_sc, handle); 1648 1649 if (!target) 1650 continue; 1651 1652 target->link_rate = link_rate; 1653 reason_code = event_data->PhyEntry[i].PhyStatus & 1654 MPI3_EVENT_SAS_TOPO_PHY_RC_MASK; 1655 1656 switch (reason_code) { 1657 case MPI3_EVENT_SAS_TOPO_PHY_RC_TARG_NOT_RESPONDING: 1658 if (target->exposed_to_os) 1659 mpi3mr_remove_device_from_os(sc, target->dev_handle); 1660 mpi3mr_remove_device_from_list(sc, target, false); 1661 break; 1662 case MPI3_EVENT_SAS_TOPO_PHY_RC_PHY_CHANGED: 1663 break; 1664 default: 1665 break; 1666 } 1667 } 1668 1669 /* 1670 * refcount was incremented for this event in 1671 * mpi3mr_evt_handler. Decrement it here because the event has 1672 * been processed. 1673 */ 1674 mpi3mr_startup_decrement(sc->cam_sc); 1675 return; 1676 } 1677 1678 static void 1679 mpi3mr_pcietopochg_evt_debug(struct mpi3mr_softc *sc, 1680 Mpi3EventDataPcieTopologyChangeList_t *event_data) 1681 { 1682 int i; 1683 U16 handle; 1684 U16 reason_code; 1685 U8 port_number; 1686 char *status_str = NULL; 1687 U8 link_rate, prev_link_rate; 1688 1689 switch (event_data->SwitchStatus) { 1690 case MPI3_EVENT_PCIE_TOPO_SS_NOT_RESPONDING: 1691 status_str = "remove"; 1692 break; 1693 case MPI3_EVENT_PCIE_TOPO_SS_RESPONDING: 1694 status_str = "responding"; 1695 break; 1696 case MPI3_EVENT_PCIE_TOPO_SS_DELAY_NOT_RESPONDING: 1697 status_str = "remove delay"; 1698 break; 1699 case MPI3_EVENT_PCIE_TOPO_SS_NO_PCIE_SWITCH: 1700 status_str = "direct attached"; 1701 break; 1702 default: 1703 status_str = "unknown status"; 1704 break; 1705 } 1706 mpi3mr_dprint(sc, MPI3MR_INFO, "%s :pcie topology change: (%s)\n", 1707 __func__, status_str); 1708 mpi3mr_dprint(sc, MPI3MR_INFO, 1709 "%s :\tswitch_handle(0x%04x), enclosure_handle(0x%04x)" 1710 "start_port(%02d), num_entries(%d)\n", __func__, 1711 le16toh(event_data->SwitchDevHandle), 1712 le16toh(event_data->EnclosureHandle), 1713 event_data->StartPortNum, event_data->NumEntries); 1714 for (i = 0; i < event_data->NumEntries; i++) { 1715 handle = 1716 le16toh(event_data->PortEntry[i].AttachedDevHandle); 1717 if (!handle) 1718 continue; 1719 port_number = event_data->StartPortNum + i; 1720 reason_code = event_data->PortEntry[i].PortStatus; 1721 switch (reason_code) { 1722 case MPI3_EVENT_PCIE_TOPO_PS_NOT_RESPONDING: 1723 status_str = "target remove"; 1724 break; 1725 case MPI3_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING: 1726 status_str = "delay target remove"; 1727 break; 1728 case MPI3_EVENT_PCIE_TOPO_PS_PORT_CHANGED: 1729 status_str = "link rate change"; 1730 break; 1731 case MPI3_EVENT_PCIE_TOPO_PS_NO_CHANGE: 1732 status_str = "target responding"; 1733 break; 1734 default: 1735 status_str = "unknown"; 1736 break; 1737 } 1738 link_rate = event_data->PortEntry[i].CurrentPortInfo & 1739 MPI3_EVENT_PCIE_TOPO_PI_RATE_MASK; 1740 prev_link_rate = event_data->PortEntry[i].PreviousPortInfo & 1741 MPI3_EVENT_PCIE_TOPO_PI_RATE_MASK; 1742 mpi3mr_dprint(sc, MPI3MR_INFO, "%s :\tport(%02d), attached_handle(0x%04x): %s:" 1743 " link rate: new(0x%02x), old(0x%02x)\n", __func__, 1744 port_number, handle, status_str, link_rate, prev_link_rate); 1745 } 1746 } 1747 1748 static void mpi3mr_process_pcietopochg_evt(struct mpi3mr_softc *sc, 1749 struct mpi3mr_fw_event_work *fwevt) 1750 { 1751 Mpi3EventDataPcieTopologyChangeList_t *event_data = 1752 (Mpi3EventDataPcieTopologyChangeList_t *)fwevt->event_data; 1753 int i; 1754 U16 handle; 1755 U8 reason_code, link_rate; 1756 struct mpi3mr_target *target = NULL; 1757 1758 1759 mpi3mr_pcietopochg_evt_debug(sc, event_data); 1760 1761 for (i = 0; i < event_data->NumEntries; i++) { 1762 handle = 1763 le16toh(event_data->PortEntry[i].AttachedDevHandle); 1764 if (!handle) 1765 continue; 1766 target = mpi3mr_find_target_by_dev_handle(sc->cam_sc, handle); 1767 if (!target) 1768 continue; 1769 1770 link_rate = event_data->PortEntry[i].CurrentPortInfo & 1771 MPI3_EVENT_PCIE_TOPO_PI_RATE_MASK; 1772 target->link_rate = link_rate; 1773 1774 reason_code = event_data->PortEntry[i].PortStatus; 1775 1776 switch (reason_code) { 1777 case MPI3_EVENT_PCIE_TOPO_PS_NOT_RESPONDING: 1778 if (target->exposed_to_os) 1779 mpi3mr_remove_device_from_os(sc, target->dev_handle); 1780 mpi3mr_remove_device_from_list(sc, target, false); 1781 break; 1782 case MPI3_EVENT_PCIE_TOPO_PS_PORT_CHANGED: 1783 break; 1784 default: 1785 break; 1786 } 1787 } 1788 1789 /* 1790 * refcount was incremented for this event in 1791 * mpi3mr_evt_handler. Decrement it here because the event has 1792 * been processed. 1793 */ 1794 mpi3mr_startup_decrement(sc->cam_sc); 1795 return; 1796 } 1797 1798 void mpi3mr_add_device(struct mpi3mr_softc *sc, U16 per_id) 1799 { 1800 struct mpi3mr_target *target; 1801 1802 mpi3mr_dprint(sc, MPI3MR_EVENT, 1803 "Adding device(persistent id: 0x%x)\n", per_id); 1804 1805 mpi3mr_startup_increment(sc->cam_sc); 1806 target = mpi3mr_find_target_by_per_id(sc->cam_sc, per_id); 1807 1808 if (!target) { 1809 mpi3mr_dprint(sc, MPI3MR_INFO, "Not available in driver's" 1810 "internal target list, persistent_id: %d\n", 1811 per_id); 1812 goto out; 1813 } 1814 1815 if (target->is_hidden) { 1816 mpi3mr_dprint(sc, MPI3MR_EVENT, "Target is hidden, persistent_id: %d\n", 1817 per_id); 1818 goto out; 1819 } 1820 1821 if (!target->exposed_to_os && !sc->reset_in_progress) { 1822 mpi3mr_rescan_target(sc, target); 1823 mpi3mr_dprint(sc, MPI3MR_INFO, 1824 "Added device persistent_id: %d dev_handle: %d\n", per_id, target->dev_handle); 1825 target->exposed_to_os = 1; 1826 } 1827 1828 out: 1829 mpi3mr_startup_decrement(sc->cam_sc); 1830 } 1831 1832 int mpi3mr_remove_device_from_os(struct mpi3mr_softc *sc, U16 handle) 1833 { 1834 int retval = 0; 1835 struct mpi3mr_target *target; 1836 unsigned int target_outstanding; 1837 1838 mpi3mr_dprint(sc, MPI3MR_EVENT, 1839 "Removing Device (dev_handle: %d)\n", handle); 1840 1841 target = mpi3mr_find_target_by_dev_handle(sc->cam_sc, handle); 1842 1843 if (!target) { 1844 mpi3mr_dprint(sc, MPI3MR_INFO, 1845 "Device (persistent_id: %d dev_handle: %d) is already removed from driver's list\n", 1846 target->per_id, handle); 1847 mpi3mr_rescan_target(sc, NULL); 1848 retval = -1; 1849 goto out; 1850 } 1851 1852 target->flags |= MPI3MRSAS_TARGET_INREMOVAL; 1853 1854 target_outstanding = mpi3mr_atomic_read(&target->outstanding); 1855 if (target_outstanding) { 1856 mpi3mr_dprint(sc, MPI3MR_ERROR, "there are [%2d] outstanding IOs on target: %d " 1857 "Poll reply queue once\n", target_outstanding, target->per_id); 1858 mpi3mr_poll_pend_io_completions(sc); 1859 target_outstanding = mpi3mr_atomic_read(&target->outstanding); 1860 if (target_outstanding) 1861 target_outstanding = mpi3mr_atomic_read(&target->outstanding); 1862 mpi3mr_dprint(sc, MPI3MR_ERROR, "[%2d] outstanding IOs present on target: %d " 1863 "despite poll\n", target_outstanding, target->per_id); 1864 } 1865 1866 if (target->exposed_to_os && !sc->reset_in_progress) { 1867 mpi3mr_rescan_target(sc, target); 1868 mpi3mr_dprint(sc, MPI3MR_INFO, 1869 "Removed device(persistent_id: %d dev_handle: %d)\n", target->per_id, handle); 1870 target->exposed_to_os = 0; 1871 } 1872 1873 target->flags &= ~MPI3MRSAS_TARGET_INREMOVAL; 1874 out: 1875 return retval; 1876 } 1877 1878 void mpi3mr_remove_device_from_list(struct mpi3mr_softc *sc, 1879 struct mpi3mr_target *target, bool must_delete) 1880 { 1881 if ((must_delete == false) && 1882 (target->state != MPI3MR_DEV_REMOVE_HS_COMPLETED)) 1883 return; 1884 1885 mtx_lock_spin(&sc->target_lock); 1886 TAILQ_REMOVE(&sc->cam_sc->tgt_list, target, tgt_next); 1887 mtx_unlock_spin(&sc->target_lock); 1888 1889 free(target, M_MPI3MR); 1890 target = NULL; 1891 1892 return; 1893 } 1894 1895 /** 1896 * mpi3mr_devstatuschg_evt_bh - DevStatusChange evt bottomhalf 1897 * @sc: Adapter instance reference 1898 * @fwevt: Firmware event 1899 * 1900 * Process Device Status Change event and based on device's new 1901 * information, either expose the device to the upper layers, or 1902 * remove the device from upper layers. 1903 * 1904 * Return: Nothing. 1905 */ 1906 static void mpi3mr_devstatuschg_evt_bh(struct mpi3mr_softc *sc, 1907 struct mpi3mr_fw_event_work *fwevt) 1908 { 1909 U16 dev_handle = 0; 1910 U8 uhide = 0, delete = 0, cleanup = 0; 1911 struct mpi3mr_target *tgtdev = NULL; 1912 Mpi3EventDataDeviceStatusChange_t *evtdata = 1913 (Mpi3EventDataDeviceStatusChange_t *)fwevt->event_data; 1914 1915 1916 1917 dev_handle = le16toh(evtdata->DevHandle); 1918 mpi3mr_dprint(sc, MPI3MR_INFO, 1919 "%s :device status change: handle(0x%04x): reason code(0x%x)\n", 1920 __func__, dev_handle, evtdata->ReasonCode); 1921 switch (evtdata->ReasonCode) { 1922 case MPI3_EVENT_DEV_STAT_RC_HIDDEN: 1923 delete = 1; 1924 break; 1925 case MPI3_EVENT_DEV_STAT_RC_NOT_HIDDEN: 1926 uhide = 1; 1927 break; 1928 case MPI3_EVENT_DEV_STAT_RC_VD_NOT_RESPONDING: 1929 delete = 1; 1930 cleanup = 1; 1931 break; 1932 default: 1933 mpi3mr_dprint(sc, MPI3MR_INFO, "%s :Unhandled reason code(0x%x)\n", __func__, 1934 evtdata->ReasonCode); 1935 break; 1936 } 1937 1938 tgtdev = mpi3mr_find_target_by_dev_handle(sc->cam_sc, dev_handle); 1939 if (!tgtdev) 1940 return; 1941 1942 if (uhide) { 1943 if (!tgtdev->exposed_to_os) 1944 mpi3mr_add_device(sc, tgtdev->per_id); 1945 } 1946 1947 if (delete) 1948 mpi3mr_remove_device_from_os(sc, dev_handle); 1949 1950 if (cleanup) 1951 mpi3mr_remove_device_from_list(sc, tgtdev, false); 1952 } 1953 1954 /** 1955 * mpi3mr_devinfochg_evt_bh - DeviceInfoChange evt bottomhalf 1956 * @sc: Adapter instance reference 1957 * @dev_pg0: New device page0 1958 * 1959 * Process Device Info Change event and based on device's new 1960 * information, either expose the device to the upper layers, or 1961 * remove the device from upper layers or update the details of 1962 * the device. 1963 * 1964 * Return: Nothing. 1965 */ 1966 static void mpi3mr_devinfochg_evt_bh(struct mpi3mr_softc *sc, 1967 Mpi3DevicePage0_t *dev_pg0) 1968 { 1969 struct mpi3mr_target *tgtdev = NULL; 1970 U16 dev_handle = 0, perst_id = 0; 1971 1972 perst_id = le16toh(dev_pg0->PersistentID); 1973 dev_handle = le16toh(dev_pg0->DevHandle); 1974 mpi3mr_dprint(sc, MPI3MR_INFO, 1975 "%s :Device info change: handle(0x%04x): persist_id(0x%x)\n", 1976 __func__, dev_handle, perst_id); 1977 tgtdev = mpi3mr_find_target_by_dev_handle(sc->cam_sc, dev_handle); 1978 if (!tgtdev) 1979 return; 1980 1981 mpi3mr_update_device(sc, tgtdev, dev_pg0, false); 1982 if (!tgtdev->is_hidden && !tgtdev->exposed_to_os) 1983 mpi3mr_add_device(sc, perst_id); 1984 1985 if (tgtdev->is_hidden && tgtdev->exposed_to_os) 1986 mpi3mr_remove_device_from_os(sc, tgtdev->dev_handle); 1987 } 1988 1989 static void 1990 mpi3mr_fw_work(struct mpi3mr_softc *sc, struct mpi3mr_fw_event_work *fw_event) 1991 { 1992 if (sc->mpi3mr_flags & MPI3MR_FLAGS_SHUTDOWN) 1993 goto out; 1994 1995 if (!fw_event->process_event) 1996 goto evt_ack; 1997 1998 mpi3mr_dprint(sc, MPI3MR_EVENT, "(%d)->(%s) Working on Event: [%x]\n", 1999 event_count++, __func__, fw_event->event); 2000 2001 switch (fw_event->event) { 2002 case MPI3_EVENT_DEVICE_ADDED: 2003 { 2004 Mpi3DevicePage0_t *dev_pg0 = 2005 (Mpi3DevicePage0_t *) fw_event->event_data; 2006 mpi3mr_add_device(sc, dev_pg0->PersistentID); 2007 break; 2008 } 2009 case MPI3_EVENT_DEVICE_INFO_CHANGED: 2010 { 2011 mpi3mr_devinfochg_evt_bh(sc, 2012 (Mpi3DevicePage0_t *) fw_event->event_data); 2013 break; 2014 } 2015 case MPI3_EVENT_DEVICE_STATUS_CHANGE: 2016 { 2017 mpi3mr_devstatuschg_evt_bh(sc, fw_event); 2018 break; 2019 } 2020 case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST: 2021 { 2022 mpi3mr_process_sastopochg_evt(sc, fw_event); 2023 break; 2024 } 2025 case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST: 2026 { 2027 mpi3mr_process_pcietopochg_evt(sc, fw_event); 2028 break; 2029 } 2030 default: 2031 mpi3mr_dprint(sc, MPI3MR_TRACE,"Unhandled event 0x%0X\n", 2032 fw_event->event); 2033 break; 2034 2035 } 2036 2037 evt_ack: 2038 if (fw_event->send_ack) { 2039 mpi3mr_dprint(sc, MPI3MR_EVENT,"Process event ACK for event 0x%0X\n", 2040 fw_event->event); 2041 mpi3mr_process_event_ack(sc, fw_event->event, 2042 fw_event->event_context); 2043 } 2044 2045 out: 2046 mpi3mr_dprint(sc, MPI3MR_EVENT, "(%d)->(%s) Event Free: [%x]\n", event_count, 2047 __func__, fw_event->event); 2048 2049 mpi3mr_fw_event_free(sc, fw_event); 2050 } 2051 2052 void 2053 mpi3mr_firmware_event_work(void *arg, int pending) 2054 { 2055 struct mpi3mr_fw_event_work *fw_event; 2056 struct mpi3mr_softc *sc; 2057 2058 sc = (struct mpi3mr_softc *)arg; 2059 2060 mtx_lock(&sc->fwevt_lock); 2061 while ((fw_event = TAILQ_FIRST(&sc->cam_sc->ev_queue)) != NULL) { 2062 TAILQ_REMOVE(&sc->cam_sc->ev_queue, fw_event, ev_link); 2063 mtx_unlock(&sc->fwevt_lock); 2064 mpi3mr_fw_work(sc, fw_event); 2065 mtx_lock(&sc->fwevt_lock); 2066 } 2067 mtx_unlock(&sc->fwevt_lock); 2068 } 2069 2070 2071 /* 2072 * mpi3mr_cam_attach - CAM layer registration 2073 * @sc: Adapter reference 2074 * 2075 * This function does simq allocation, cam registration, xpt_bus registration, 2076 * event taskqueue initialization and async event handler registration. 2077 * 2078 * Return: 0 on success and proper error codes on failure 2079 */ 2080 int 2081 mpi3mr_cam_attach(struct mpi3mr_softc *sc) 2082 { 2083 struct mpi3mr_cam_softc *cam_sc; 2084 cam_status status; 2085 int unit, error = 0, reqs; 2086 2087 mpi3mr_dprint(sc, MPI3MR_XINFO, "Starting CAM Attach\n"); 2088 2089 cam_sc = malloc(sizeof(struct mpi3mr_cam_softc), M_MPI3MR, M_WAITOK|M_ZERO); 2090 cam_sc->maxtargets = sc->facts.max_perids + 1; 2091 2092 TAILQ_INIT(&cam_sc->tgt_list); 2093 2094 sc->cam_sc = cam_sc; 2095 cam_sc->sc = sc; 2096 2097 reqs = sc->max_host_ios; 2098 2099 if ((cam_sc->devq = cam_simq_alloc(reqs)) == NULL) { 2100 mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocate SIMQ\n"); 2101 error = ENOMEM; 2102 goto out; 2103 } 2104 2105 unit = device_get_unit(sc->mpi3mr_dev); 2106 cam_sc->sim = cam_sim_alloc(mpi3mr_cam_action, mpi3mr_cam_poll, "mpi3mr", cam_sc, 2107 unit, &sc->mpi3mr_mtx, reqs, reqs, cam_sc->devq); 2108 if (cam_sc->sim == NULL) { 2109 mpi3mr_dprint(sc, MPI3MR_ERROR, "Failed to allocate SIM\n"); 2110 error = EINVAL; 2111 goto out; 2112 } 2113 2114 TAILQ_INIT(&cam_sc->ev_queue); 2115 2116 /* Initialize taskqueue for Event Handling */ 2117 TASK_INIT(&cam_sc->ev_task, 0, mpi3mr_firmware_event_work, sc); 2118 cam_sc->ev_tq = taskqueue_create("mpi3mr_taskq", M_NOWAIT | M_ZERO, 2119 taskqueue_thread_enqueue, &cam_sc->ev_tq); 2120 taskqueue_start_threads(&cam_sc->ev_tq, 1, PRIBIO, "%s taskq", 2121 device_get_nameunit(sc->mpi3mr_dev)); 2122 2123 mtx_lock(&sc->mpi3mr_mtx); 2124 2125 /* 2126 * XXX There should be a bus for every port on the adapter, but since 2127 * we're just going to fake the topology for now, we'll pretend that 2128 * everything is just a target on a single bus. 2129 */ 2130 if ((error = xpt_bus_register(cam_sc->sim, sc->mpi3mr_dev, 0)) != 0) { 2131 mpi3mr_dprint(sc, MPI3MR_ERROR, 2132 "Error 0x%x registering SCSI bus\n", error); 2133 mtx_unlock(&sc->mpi3mr_mtx); 2134 goto out; 2135 } 2136 2137 /* 2138 * Assume that discovery events will start right away. 2139 * 2140 * Hold off boot until discovery is complete. 2141 */ 2142 cam_sc->flags |= MPI3MRSAS_IN_STARTUP | MPI3MRSAS_IN_DISCOVERY; 2143 sc->cam_sc->startup_refcount = 0; 2144 mpi3mr_startup_increment(cam_sc); 2145 2146 callout_init(&cam_sc->discovery_callout, 1 /*mpsafe*/); 2147 2148 /* 2149 * Register for async events so we can determine the EEDP 2150 * capabilities of devices. 2151 */ 2152 status = xpt_create_path(&cam_sc->path, /*periph*/NULL, 2153 cam_sim_path(sc->cam_sc->sim), CAM_TARGET_WILDCARD, 2154 CAM_LUN_WILDCARD); 2155 if (status != CAM_REQ_CMP) { 2156 mpi3mr_dprint(sc, MPI3MR_ERROR, 2157 "Error 0x%x creating sim path\n", status); 2158 cam_sc->path = NULL; 2159 } 2160 2161 if (status != CAM_REQ_CMP) { 2162 /* 2163 * EEDP use is the exception, not the rule. 2164 * Warn the user, but do not fail to attach. 2165 */ 2166 mpi3mr_dprint(sc, MPI3MR_INFO, "EEDP capabilities disabled.\n"); 2167 } 2168 2169 mtx_unlock(&sc->mpi3mr_mtx); 2170 2171 error = mpi3mr_register_events(sc); 2172 2173 out: 2174 mpi3mr_dprint(sc, MPI3MR_XINFO, "%s Exiting CAM attach, error: 0x%x n", __func__, error); 2175 return (error); 2176 } 2177 2178 int 2179 mpi3mr_cam_detach(struct mpi3mr_softc *sc) 2180 { 2181 struct mpi3mr_cam_softc *cam_sc; 2182 struct mpi3mr_target *target; 2183 2184 mpi3mr_dprint(sc, MPI3MR_XINFO, "%s, Starting CAM detach\n", __func__); 2185 if (sc->cam_sc == NULL) 2186 return (0); 2187 2188 cam_sc = sc->cam_sc; 2189 2190 mpi3mr_freeup_events(sc); 2191 2192 /* 2193 * Drain and free the event handling taskqueue with the lock 2194 * unheld so that any parallel processing tasks drain properly 2195 * without deadlocking. 2196 */ 2197 if (cam_sc->ev_tq != NULL) 2198 taskqueue_free(cam_sc->ev_tq); 2199 2200 mtx_lock(&sc->mpi3mr_mtx); 2201 2202 while (cam_sc->startup_refcount != 0) 2203 mpi3mr_startup_decrement(cam_sc); 2204 2205 /* Deregister our async handler */ 2206 if (cam_sc->path != NULL) { 2207 xpt_free_path(cam_sc->path); 2208 cam_sc->path = NULL; 2209 } 2210 2211 if (cam_sc->flags & MPI3MRSAS_IN_STARTUP) 2212 xpt_release_simq(cam_sc->sim, 1); 2213 2214 if (cam_sc->sim != NULL) { 2215 xpt_bus_deregister(cam_sim_path(cam_sc->sim)); 2216 cam_sim_free(cam_sc->sim, FALSE); 2217 } 2218 2219 mtx_unlock(&sc->mpi3mr_mtx); 2220 2221 if (cam_sc->devq != NULL) 2222 cam_simq_free(cam_sc->devq); 2223 2224 get_target: 2225 mtx_lock_spin(&sc->target_lock); 2226 TAILQ_FOREACH(target, &cam_sc->tgt_list, tgt_next) { 2227 TAILQ_REMOVE(&sc->cam_sc->tgt_list, target, tgt_next); 2228 mtx_unlock_spin(&sc->target_lock); 2229 goto out_tgt_free; 2230 } 2231 mtx_unlock_spin(&sc->target_lock); 2232 out_tgt_free: 2233 if (target) { 2234 free(target, M_MPI3MR); 2235 target = NULL; 2236 goto get_target; 2237 } 2238 2239 free(cam_sc, M_MPI3MR); 2240 sc->cam_sc = NULL; 2241 2242 mpi3mr_dprint(sc, MPI3MR_XINFO, "%s, Exiting CAM detach\n", __func__); 2243 return (0); 2244 } 2245