1 /* 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2020-2025, Broadcom Inc. All rights reserved. 5 * Support: <fbsd-storage-driver.pdl@broadcom.com> 6 * 7 * Authors: Sumit Saxena <sumit.saxena@broadcom.com> 8 * Chandrakanth Patil <chandrakanth.patil@broadcom.com> 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are 12 * met: 13 * 14 * 1. Redistributions of source code must retain the above copyright notice, 15 * this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright notice, 17 * this list of conditions and the following disclaimer in the documentation and/or other 18 * materials provided with the distribution. 19 * 3. Neither the name of the Broadcom Inc. nor the names of its contributors 20 * may be used to endorse or promote products derived from this software without 21 * specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 * POSSIBILITY OF SUCH DAMAGE. 34 * 35 * The views and conclusions contained in the software and documentation are 36 * those of the authors and should not be interpreted as representing 37 * official policies,either expressed or implied, of the FreeBSD Project. 38 * 39 * Mail to: Broadcom Inc 1320 Ridder Park Dr, San Jose, CA 95131 40 * 41 * Broadcom Inc. (Broadcom) MPI3MR Adapter FreeBSD 42 */ 43 44 #ifndef _MPI3MRVAR_H 45 #define _MPI3MRVAR_H 46 47 #include <sys/types.h> 48 #include <sys/param.h> 49 #include <sys/systm.h> 50 #include <sys/kernel.h> 51 #include <sys/module.h> 52 #include <sys/bus.h> 53 #include <sys/conf.h> 54 #include <sys/malloc.h> 55 #include <sys/sysctl.h> 56 #include <sys/uio.h> 57 #include <sys/selinfo.h> 58 #include <sys/poll.h> 59 60 #include <sys/lock.h> 61 #include <sys/mutex.h> 62 #include <sys/endian.h> 63 #include <sys/sysent.h> 64 #include <sys/taskqueue.h> 65 #include <sys/smp.h> 66 67 #include <machine/bus.h> 68 #include <machine/resource.h> 69 #include <sys/rman.h> 70 71 #include <dev/pci/pcireg.h> 72 #include <dev/pci/pcivar.h> 73 #include <dev/pci/pci_private.h> 74 75 #include <cam/cam.h> 76 #include <cam/cam_ccb.h> 77 #include <cam/cam_debug.h> 78 #include <cam/cam_sim.h> 79 #include <cam/cam_xpt_sim.h> 80 #include <cam/cam_xpt_periph.h> 81 #include <cam/cam_periph.h> 82 #include <cam/scsi/scsi_all.h> 83 #include <cam/scsi/scsi_message.h> 84 85 #include <cam/scsi/smp_all.h> 86 #include <sys/queue.h> 87 #include <sys/kthread.h> 88 #include "mpi/mpi30_api.h" 89 90 #define MPI3MR_DRIVER_VERSION "8.14.0.2.0" 91 #define MPI3MR_DRIVER_RELDATE "9th Apr 2025" 92 93 #define MPI3MR_DRIVER_NAME "mpi3mr" 94 95 #define MPI3MR_NAME_LENGTH 32 96 #define IOCNAME "%s: " 97 98 #define MPI3MR_DEFAULT_MAX_IO_SIZE (1 * 1024 * 1024) 99 100 #define SAS4116_CHIP_REV_A0 0 101 #define SAS4116_CHIP_REV_B0 1 102 103 #define MPI3MR_MAX_SECTORS 2048 104 #define MPI3MR_MAX_CMDS_LUN 7 105 #define MPI3MR_MAX_CDB_LENGTH 16 106 #define MPI3MR_MAX_LUN 16895 107 108 #define MPI3MR_SATA_QDEPTH 32 109 #define MPI3MR_SAS_QDEPTH 64 110 #define MPI3MR_RAID_QDEPTH 128 111 #define MPI3MR_NVME_QDEPTH 128 112 113 /* Definitions for internal SGL and Chain SGL buffers */ 114 #define MPI3MR_4K_PGSZ 4096 115 #define MPI3MR_PAGE_SIZE_4K 4096 116 #define MPI3MR_DEFAULT_SGL_ENTRIES 256 117 #define MPI3MR_MAX_SGL_ENTRIES 2048 118 119 #define MPI3MR_AREQQ_SIZE (2 * MPI3MR_4K_PGSZ) 120 #define MPI3MR_AREPQ_SIZE (4 * MPI3MR_4K_PGSZ) 121 #define MPI3MR_AREQ_FRAME_SZ 128 122 #define MPI3MR_AREP_FRAME_SZ 16 123 124 #define MPI3MR_OPREQQ_SIZE (8 * MPI3MR_4K_PGSZ) 125 #define MPI3MR_OPREPQ_SIZE (4 * MPI3MR_4K_PGSZ) 126 127 /* Operational queue management definitions */ 128 #define MPI3MR_OP_REQ_Q_QD 512 129 #define MPI3MR_OP_REP_Q_QD 1024 130 #define MPI3MR_OP_REP_Q_QD_A0 4096 131 132 #define MPI3MR_THRESHOLD_REPLY_COUNT 100 133 134 #define MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST \ 135 (MPI3_SGE_FLAGS_ELEMENT_TYPE_SIMPLE | MPI3_SGE_FLAGS_DLAS_SYSTEM | \ 136 MPI3_SGE_FLAGS_END_OF_LIST) 137 138 #define MPI3MR_HOSTTAG_INVALID 0xFFFF 139 #define MPI3MR_HOSTTAG_INITCMDS 1 140 #define MPI3MR_HOSTTAG_IOCTLCMDS 2 141 #define MPI3MR_HOSTTAG_PELABORT 3 142 #define MPI3MR_HOSTTAG_PELWAIT 4 143 #define MPI3MR_HOSTTAG_TMS 5 144 #define MPI3MR_HOSTTAG_CFGCMDS 6 145 146 #define MAX_MGMT_ADAPTERS 8 147 #define MPI3MR_WAIT_BEFORE_CTRL_RESET 5 148 149 #define MPI3MR_RESET_REASON_OSTYPE_FREEBSD 0x4 150 #define MPI3MR_RESET_REASON_OSTYPE_SHIFT 28 151 #define MPI3MR_RESET_REASON_IOCNUM_SHIFT 20 152 153 struct mpi3mr_mgmt_info { 154 uint16_t count; 155 struct mpi3mr_softc *sc_ptr[MAX_MGMT_ADAPTERS]; 156 int max_index; 157 }; 158 159 extern char fmt_os_ver[16]; 160 161 #define MPI3MR_OS_VERSION(raw_os_ver, fmt_os_ver) sprintf(raw_os_ver, "%d", __FreeBSD_version); \ 162 sprintf(fmt_os_ver, "%c%c.%c%c.%c%c%c",\ 163 raw_os_ver[0], raw_os_ver[1], raw_os_ver[2],\ 164 raw_os_ver[3], raw_os_ver[4], raw_os_ver[5],\ 165 raw_os_ver[6]); 166 #define MPI3MR_NUM_DEVRMCMD 1 167 #define MPI3MR_HOSTTAG_DEVRMCMD_MIN (MPI3MR_HOSTTAG_CFGCMDS + 1) 168 #define MPI3MR_HOSTTAG_DEVRMCMD_MAX (MPI3MR_HOSTTAG_DEVRMCMD_MIN + \ 169 MPI3MR_NUM_DEVRMCMD - 1) 170 #define MPI3MR_INTERNALCMDS_RESVD MPI3MR_HOSTTAG_DEVRMCMD_MAX 171 172 #define MPI3MR_NUM_EVTACKCMD 4 173 #define MPI3MR_HOSTTAG_EVTACKCMD_MIN (MPI3MR_HOSTTAG_DEVRMCMD_MAX + 1) 174 #define MPI3MR_HOSTTAG_EVTACKCMD_MAX (MPI3MR_HOSTTAG_EVTACKCMD_MIN + \ 175 MPI3MR_NUM_EVTACKCMD - 1) 176 177 /* command/controller interaction timeout definitions in seconds */ 178 #define MPI3MR_INTADMCMD_TIMEOUT 60 179 #define MPI3MR_PORTENABLE_TIMEOUT 300 180 #define MPI3MR_ABORTTM_TIMEOUT 60 181 #define MPI3MR_RESETTM_TIMEOUT 60 182 #define MPI3MR_TSUPDATE_INTERVAL 900 183 #define MPI3MR_DEFAULT_SHUTDOWN_TIME 120 184 #define MPI3MR_RAID_ERRREC_RESET_TIMEOUT 180 185 #define MPI3MR_RESET_HOST_IOWAIT_TIMEOUT 5 186 #define MPI3MR_PREPARE_FOR_RESET_TIMEOUT 180 187 #define MPI3MR_RESET_ACK_TIMEOUT 30 188 #define MPI3MR_MUR_TIMEOUT 120 189 190 #define MPI3MR_CMD_NOTUSED 0x8000 191 #define MPI3MR_CMD_COMPLETE 0x0001 192 #define MPI3MR_CMD_PENDING 0x0002 193 #define MPI3MR_CMD_REPLYVALID 0x0004 194 #define MPI3MR_CMD_RESET 0x0008 195 196 #define MPI3MR_NUM_EVTREPLIES 64 197 #define MPI3MR_SENSEBUF_SZ 256 198 #define MPI3MR_SENSEBUF_FACTOR 3 199 #define MPI3MR_CHAINBUF_FACTOR 3 200 201 #define MPT3SAS_HOSTPGSZ_4KEXP 12 202 203 #define MPI3MR_INVALID_DEV_HANDLE 0xFFFF 204 205 /* Controller Reset related definitions */ 206 #define MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT 5 207 #define MPI3MR_MAX_SHUTDOWN_RETRY_COUNT 2 208 209 /* ResponseCode values */ 210 #define MPI3MR_RI_MASK_RESPCODE (0x000000FF) 211 #define MPI3MR_RSP_TM_COMPLETE 0x00 212 #define MPI3MR_RSP_INVALID_FRAME 0x02 213 #define MPI3MR_RSP_TM_NOT_SUPPORTED 0x04 214 #define MPI3MR_RSP_TM_FAILED 0x05 215 #define MPI3MR_RSP_TM_SUCCEEDED 0x08 216 #define MPI3MR_RSP_TM_INVALID_LUN 0x09 217 #define MPI3MR_RSP_TM_OVERLAPPED_TAG 0x0A 218 #define MPI3MR_RSP_IO_QUEUED_ON_IOC \ 219 MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC 220 221 /* Definitions for the controller security status*/ 222 #define MPI3MR_CTLR_SECURITY_STATUS_MASK 0x0C 223 #define MPI3MR_CTLR_SECURE_DBG_STATUS_MASK 0x02 224 225 #define MPI3MR_INVALID_DEVICE 0x00 226 #define MPI3MR_CONFIG_SECURE_DEVICE 0x04 227 #define MPI3MR_HARD_SECURE_DEVICE 0x08 228 #define MPI3MR_TAMPERED_DEVICE 0x0C 229 230 #define MPI3MR_DEFAULT_MDTS (128 * 1024) 231 #define MPI3MR_DEFAULT_PGSZEXP (12) 232 #define MPI3MR_MAX_IOCTL_TRANSFER_SIZE (1024 * 1024) 233 234 #define MPI3MR_DEVRMHS_RETRYCOUNT 3 235 #define MPI3MR_PELCMDS_RETRYCOUNT 3 236 237 #define MPI3MR_PERIODIC_DELAY 1 /* 1 second heartbeat/watchdog check */ 238 239 #define WRITE_SAME_32 0x0d 240 241 #define MPI3MR_TSUPDATE_INTERVAL 900 242 243 struct completion { 244 unsigned int done; 245 struct mtx lock; 246 }; 247 248 typedef union { 249 volatile unsigned int val; 250 unsigned int val_rdonly; 251 } mpi3mr_atomic_t; 252 253 #define mpi3mr_atomic_read(v) atomic_load_acq_int(&(v)->val) 254 #define mpi3mr_atomic_set(v,i) atomic_store_rel_int(&(v)->val, i) 255 #define mpi3mr_atomic_dec(v) atomic_subtract_int(&(v)->val, 1) 256 #define mpi3mr_atomic_inc(v) atomic_add_int(&(v)->val, 1) 257 #define mpi3mr_atomic_add(v, u) atomic_add_int(&(v)->val, u) 258 #define mpi3mr_atomic_sub(v, u) atomic_subtract_int(&(v)->val, u) 259 260 /* IOCTL data transfer sge*/ 261 #define MPI3MR_NUM_IOCTL_SGE 256 262 #define MPI3MR_IOCTL_SGE_SIZE (8 * 1024) 263 264 struct dma_memory_desc { 265 U32 size; 266 void *addr; 267 bus_dma_tag_t tag; 268 bus_dmamap_t dmamap; 269 bus_addr_t dma_addr; 270 }; 271 272 enum mpi3mr_iocstate { 273 MRIOC_STATE_READY = 1, 274 MRIOC_STATE_RESET, 275 MRIOC_STATE_FAULT, 276 MRIOC_STATE_BECOMING_READY, 277 MRIOC_STATE_RESET_REQUESTED, 278 MRIOC_STATE_UNRECOVERABLE, 279 MRIOC_STATE_COUNT, 280 }; 281 282 /* Init type definitions */ 283 enum mpi3mr_init_type { 284 MPI3MR_INIT_TYPE_INIT = 0, 285 MPI3MR_INIT_TYPE_RESET, 286 MPI3MR_INIT_TYPE_RESUME, 287 }; 288 289 /* Reset reason code definitions*/ 290 enum mpi3mr_reset_reason { 291 MPI3MR_RESET_FROM_BRINGUP = 1, 292 MPI3MR_RESET_FROM_FAULT_WATCH = 2, 293 MPI3MR_RESET_FROM_IOCTL = 3, 294 MPI3MR_RESET_FROM_EH_HOS = 4, 295 MPI3MR_RESET_FROM_TM_TIMEOUT = 5, 296 MPI3MR_RESET_FROM_IOCTL_TIMEOUT = 6, 297 MPI3MR_RESET_FROM_MUR_FAILURE = 7, 298 MPI3MR_RESET_FROM_CTLR_CLEANUP = 8, 299 MPI3MR_RESET_FROM_CIACTIV_FAULT = 9, 300 MPI3MR_RESET_FROM_PE_TIMEOUT = 10, 301 MPI3MR_RESET_FROM_TSU_TIMEOUT = 11, 302 MPI3MR_RESET_FROM_DELREQQ_TIMEOUT = 12, 303 MPI3MR_RESET_FROM_DELREPQ_TIMEOUT = 13, 304 MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT = 14, 305 MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT = 15, 306 MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT = 16, 307 MPI3MR_RESET_FROM_IOCINIT_TIMEOUT = 17, 308 MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT = 18, 309 MPI3MR_RESET_FROM_EVTACK_TIMEOUT = 19, 310 MPI3MR_RESET_FROM_CIACTVRST_TIMER = 20, 311 MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT = 21, 312 MPI3MR_RESET_FROM_PELABORT_TIMEOUT = 22, 313 MPI3MR_RESET_FROM_SYSFS = 23, 314 MPI3MR_RESET_FROM_SYSFS_TIMEOUT = 24, 315 MPI3MR_RESET_FROM_DIAG_BUFFER_POST_TIMEOUT = 25, 316 MPI3MR_RESET_FROM_SCSIIO_TIMEOUT = 26, 317 MPI3MR_RESET_FROM_FIRMWARE = 27, 318 MPI3MR_DEFAULT_RESET_REASON = 28, 319 MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT = 29, 320 MPI3MR_RESET_REASON_COUNT, 321 }; 322 323 struct mpi3mr_compimg_ver 324 { 325 U16 build_num; 326 U16 cust_id; 327 U8 ph_minor; 328 U8 ph_major; 329 U8 gen_minor; 330 U8 gen_major; 331 }; 332 333 struct mpi3mr_ioc_facts 334 { 335 U32 ioc_capabilities; 336 struct mpi3mr_compimg_ver fw_ver; 337 U32 mpi_version; 338 U16 max_reqs; 339 U16 product_id; 340 U16 op_req_sz; 341 U16 reply_sz; 342 U16 exceptions; 343 U16 max_perids; 344 U16 max_pds; 345 U16 max_sasexpanders; 346 U32 max_data_length; 347 U16 max_sasinitiators; 348 U16 max_enclosures; 349 U16 max_pcieswitches; 350 U16 max_nvme; 351 U16 max_vds; 352 U16 max_hpds; 353 U16 max_advhpds; 354 U16 max_raidpds; 355 U16 min_devhandle; 356 U16 max_devhandle; 357 U16 max_op_req_q; 358 U16 max_op_reply_q; 359 U16 shutdown_timeout; 360 U8 ioc_num; 361 U8 who_init; 362 U16 max_msix_vectors; 363 U8 personality; 364 U8 dma_mask; 365 U8 protocol_flags; 366 U8 sge_mod_mask; 367 U8 sge_mod_value; 368 U8 sge_mod_shift; 369 U8 max_dev_per_tg; 370 U16 max_io_throttle_group; 371 U16 io_throttle_data_length; 372 U16 io_throttle_low; 373 U16 io_throttle_high; 374 }; 375 376 struct mpi3mr_op_req_queue { 377 U16 ci; 378 U16 pi; 379 U16 num_reqs; 380 U8 qid; 381 U8 reply_qid; 382 U32 qsz; 383 void *q_base; 384 bus_dma_tag_t q_base_tag; 385 bus_dmamap_t q_base_dmamap; 386 bus_addr_t q_base_phys; 387 struct mtx q_lock; 388 }; 389 390 struct mpi3mr_op_reply_queue { 391 U16 ci; 392 U8 ephase; 393 U8 qid; 394 U16 num_replies; 395 U32 qsz; 396 bus_dma_tag_t q_base_tag; 397 bus_dmamap_t q_base_dmamap; 398 void *q_base; 399 bus_addr_t q_base_phys; 400 mpi3mr_atomic_t pend_ios; 401 bool in_use; 402 struct mtx q_lock; 403 }; 404 405 struct irq_info { 406 MPI3_REPLY_DESCRIPTORS_UNION *post_queue; 407 bus_dma_tag_t buffer_dmat; 408 struct resource *irq; 409 void *intrhand; 410 int irq_rid; 411 }; 412 413 struct mpi3mr_irq_context { 414 struct mpi3mr_softc *sc; 415 U16 msix_index; 416 struct mpi3mr_op_reply_queue *op_reply_q; 417 char name[MPI3MR_NAME_LENGTH]; 418 struct irq_info irq_info; 419 }; 420 421 MALLOC_DECLARE(M_MPI3MR); 422 SYSCTL_DECL(_hw_mpi3mr); 423 424 typedef struct mpi3mr_drvr_cmd DRVR_CMD; 425 typedef void (*DRVR_CMD_CALLBACK)(struct mpi3mr_softc *mrioc, DRVR_CMD *drvrcmd); 426 struct mpi3mr_drvr_cmd { 427 struct mtx lock; 428 struct completion completion; 429 void *reply; 430 U8 *sensebuf; 431 U8 iou_rc; 432 U16 state; 433 U16 dev_handle; 434 U16 ioc_status; 435 U32 ioc_loginfo; 436 U8 is_waiting; 437 U8 is_senseprst; 438 U8 retry_count; 439 U16 host_tag; 440 DRVR_CMD_CALLBACK callback; 441 }; 442 443 struct mpi3mr_cmd; 444 typedef void mpi3mr_evt_callback_t(struct mpi3mr_softc *, uintptr_t, 445 Mpi3EventNotificationReply_t *reply); 446 typedef void mpi3mr_cmd_callback_t(struct mpi3mr_softc *, 447 struct mpi3mr_cmd *cmd); 448 449 #define MPI3MR_IOVEC_COUNT 2 450 451 enum mpi3mr_data_xfer_direction { 452 MPI3MR_READ = 1, 453 MPI3MR_WRITE, 454 }; 455 456 enum mpi3mr_cmd_state { 457 MPI3MR_CMD_STATE_FREE = 1, 458 MPI3MR_CMD_STATE_BUSY, 459 MPI3MR_CMD_STATE_IN_QUEUE, 460 MPI3MR_CMD_STATE_IN_TM, 461 }; 462 463 enum mpi3mr_target_state { 464 MPI3MR_DEV_CREATED = 1, 465 MPI3MR_DEV_REMOVE_HS_COMPLETED = 2, 466 }; 467 468 struct mpi3mr_cmd { 469 TAILQ_ENTRY(mpi3mr_cmd) next; 470 struct mpi3mr_softc *sc; 471 union ccb *ccb; 472 void *data; 473 u_int length; 474 struct mpi3mr_target *targ; 475 u_int data_dir; 476 u_int state; 477 bus_dmamap_t dmamap; 478 struct scsi_sense_data *sense; 479 struct callout callout; 480 bool callout_owner; 481 U16 hosttag; 482 U8 req_qidx; 483 Mpi3SCSIIORequest_t io_request; 484 }; 485 486 struct mpi3mr_chain { 487 bus_dmamap_t buf_dmamap; 488 void *buf; 489 bus_addr_t buf_phys; 490 }; 491 492 struct mpi3mr_event_handle { 493 TAILQ_ENTRY(mpi3mr_event_handle) eh_list; 494 mpi3mr_evt_callback_t *callback; 495 void *data; 496 uint8_t mask[16]; 497 }; 498 499 struct mpi3mr_fw_event_work { 500 U16 event; 501 void *event_data; 502 TAILQ_ENTRY(mpi3mr_fw_event_work) ev_link; 503 U8 send_ack; 504 U8 process_event; 505 U32 event_context; 506 U16 event_data_size; 507 }; 508 509 /** 510 * struct delayed_dev_rmhs_node - Delayed device removal node 511 * 512 * @list: list head 513 * @handle: Device handle 514 * @iou_rc: IO Unit Control Reason Code 515 */ 516 struct delayed_dev_rmhs_node { 517 TAILQ_ENTRY(delayed_dev_rmhs_node) list; 518 U16 handle; 519 U8 iou_rc; 520 }; 521 522 /** 523 * struct delayed_evtack_node - Delayed event ack node 524 * 525 * @list: list head 526 * @event: MPI3 event ID 527 * @event_ctx: Event context 528 */ 529 struct delayed_evtack_node { 530 TAILQ_ENTRY(delayed_evtack_node) list; 531 U8 event; 532 U32 event_ctx; 533 }; 534 535 /* Reset types */ 536 enum reset_type { 537 MPI3MR_NO_RESET, 538 MPI3MR_TRIGGER_SOFT_RESET, 539 }; 540 541 struct mpi3mr_reset { 542 u_int type; 543 U32 reason; 544 int status; 545 bool ioctl_reset_snapdump; 546 }; 547 548 struct mpi3mr_softc { 549 device_t mpi3mr_dev; 550 struct cdev *mpi3mr_cdev; 551 u_int mpi3mr_flags; 552 #define MPI3MR_FLAGS_SHUTDOWN (1 << 0) 553 #define MPI3MR_FLAGS_DIAGRESET (1 << 1) 554 #define MPI3MR_FLAGS_ATTACH_DONE (1 << 2) 555 #define MPI3MR_FLAGS_PORT_ENABLE_DONE (1 << 3) 556 U8 id; 557 int cpu_count; 558 char name[MPI3MR_NAME_LENGTH]; 559 char driver_name[MPI3MR_NAME_LENGTH]; 560 int bars; 561 bus_addr_t dma_loaddr; 562 bus_addr_t dma_hiaddr; 563 u_int mpi3mr_debug; 564 struct mpi3mr_reset reset; 565 int max_msix_vectors; 566 int msix_count; 567 bool msix_enable; 568 int io_cmds_highwater; 569 int max_chains; 570 uint32_t chain_frame_size; 571 struct sysctl_ctx_list sysctl_ctx; 572 struct sysctl_oid *sysctl_tree; 573 char fw_version[32]; 574 struct mpi3mr_chain *chains; 575 struct callout periodic; 576 struct callout device_check_callout; 577 578 struct mpi3mr_cam_softc *cam_sc; 579 struct mpi3mr_cmd **cmd_list; 580 TAILQ_HEAD(, mpi3mr_cmd) cmd_list_head; 581 struct mtx cmd_pool_lock; 582 583 struct resource *mpi3mr_regs_resource; 584 bus_space_handle_t mpi3mr_bhandle; 585 bus_space_tag_t mpi3mr_btag; 586 int mpi3mr_regs_rid; 587 588 bus_dma_tag_t mpi3mr_parent_dmat; 589 bus_dma_tag_t buffer_dmat; 590 591 int num_reqs; 592 int num_replies; 593 int num_chains; 594 595 TAILQ_HEAD(, mpi3mr_event_handle) event_list; 596 struct mpi3mr_event_handle *mpi3mr_log_eh; 597 struct intr_config_hook mpi3mr_ich; 598 599 struct mtx mpi3mr_mtx; 600 struct mtx io_lock; 601 U8 intr_enabled; 602 TAILQ_HEAD(, delayed_dev_rmhs_node) delayed_rmhs_list; 603 TAILQ_HEAD(, delayed_evtack_node) delayed_evtack_cmds_list; 604 605 U16 num_admin_reqs; 606 U32 admin_req_q_sz; 607 U16 admin_req_pi; 608 U16 admin_req_ci; 609 bus_dma_tag_t admin_req_tag; 610 bus_dmamap_t admin_req_dmamap; 611 bus_addr_t admin_req_phys; 612 U8 *admin_req; 613 struct mtx admin_req_lock; 614 615 U16 num_admin_replies; 616 U32 admin_reply_q_sz; 617 U16 admin_reply_ci; 618 U8 admin_reply_ephase; 619 bus_dma_tag_t admin_reply_tag; 620 bus_dmamap_t admin_reply_dmamap; 621 bus_addr_t admin_reply_phys; 622 U8 *admin_reply; 623 struct mtx admin_reply_lock; 624 bool admin_in_use; 625 626 U32 num_reply_bufs; 627 bus_dma_tag_t reply_buf_tag; 628 bus_dmamap_t reply_buf_dmamap; 629 bus_addr_t reply_buf_phys; 630 U8 *reply_buf; 631 bus_addr_t reply_buf_dma_max_address; 632 bus_addr_t reply_buf_dma_min_address; 633 634 U16 reply_free_q_sz; 635 bus_dma_tag_t reply_free_q_tag; 636 bus_dmamap_t reply_free_q_dmamap; 637 bus_addr_t reply_free_q_phys; 638 U64 *reply_free_q; 639 struct mtx reply_free_q_lock; 640 U32 reply_free_q_host_index; 641 642 U32 num_sense_bufs; 643 bus_dma_tag_t sense_buf_tag; 644 bus_dmamap_t sense_buf_dmamap; 645 bus_addr_t sense_buf_phys; 646 U8 *sense_buf; 647 648 U16 sense_buf_q_sz; 649 bus_dma_tag_t sense_buf_q_tag; 650 bus_dmamap_t sense_buf_q_dmamap; 651 bus_addr_t sense_buf_q_phys; 652 U64 *sense_buf_q; 653 struct mtx sense_buf_q_lock; 654 U32 sense_buf_q_host_index; 655 656 void *nvme_encap_prp_list; 657 bus_addr_t nvme_encap_prp_list_dma; 658 bus_dma_tag_t nvme_encap_prp_list_dmatag; 659 bus_dmamap_t nvme_encap_prp_list_dma_dmamap; 660 U32 nvme_encap_prp_sz; 661 662 U32 ready_timeout; 663 664 struct mpi3mr_irq_context *irq_ctx; 665 666 U16 num_queues; /* Number of request/reply queues */ 667 struct mpi3mr_op_req_queue *op_req_q; 668 struct mpi3mr_op_reply_queue *op_reply_q; 669 U16 num_hosttag_op_req_q; 670 671 struct mpi3mr_drvr_cmd init_cmds; 672 struct mpi3mr_ioc_facts facts; 673 U16 reply_sz; 674 U16 op_reply_sz; 675 676 U32 event_masks[MPI3_EVENT_NOTIFY_EVENTMASK_WORDS]; 677 678 char fwevt_worker_name[MPI3MR_NAME_LENGTH]; 679 struct workqueue_struct *fwevt_worker_thread; 680 struct mtx fwevt_lock; 681 struct mtx target_lock; 682 683 U16 max_host_ios; 684 U32 max_sgl_entries; 685 bus_dma_tag_t chain_sgl_list_tag; 686 struct mpi3mr_chain *chain_sgl_list; 687 U16 chain_bitmap_sz; 688 void *chain_bitmap; 689 struct mtx chain_buf_lock; 690 U16 chain_buf_count; 691 692 struct mpi3mr_drvr_cmd ioctl_cmds; 693 struct mpi3mr_drvr_cmd host_tm_cmds; 694 struct mpi3mr_drvr_cmd dev_rmhs_cmds[MPI3MR_NUM_DEVRMCMD]; 695 struct mpi3mr_drvr_cmd evtack_cmds[MPI3MR_NUM_EVTACKCMD]; 696 struct mpi3mr_drvr_cmd cfg_cmds; 697 698 U16 devrem_bitmap_sz; 699 void *devrem_bitmap; 700 701 U16 dev_handle_bitmap_sz; 702 void *removepend_bitmap; 703 704 U16 evtack_cmds_bitmap_sz; 705 void *evtack_cmds_bitmap; 706 707 U32 ts_update_counter; 708 U8 reset_in_progress; 709 U8 unrecoverable; 710 U8 block_ioctls; 711 U8 in_prep_ciactv_rst; 712 U16 prep_ciactv_rst_counter; 713 struct mtx reset_mutex; 714 715 U8 prepare_for_reset; 716 U16 prepare_for_reset_timeout_counter; 717 718 U16 diagsave_timeout; 719 int logging_level; 720 U16 flush_io_count; 721 722 Mpi3DriverInfoLayout_t driver_info; 723 724 U16 change_count; 725 726 U8 *log_data_buffer; 727 U16 log_data_buffer_index; 728 U16 log_data_entry_size; 729 730 U8 pel_wait_pend; 731 U8 pel_abort_requested; 732 U8 pel_class; 733 U16 pel_locale; 734 735 struct mpi3mr_drvr_cmd pel_cmds; 736 struct mpi3mr_drvr_cmd pel_abort_cmd; 737 U32 newest_seqnum; 738 void *pel_seq_number; 739 bus_addr_t pel_seq_number_dma; 740 bus_dma_tag_t pel_seq_num_dmatag; 741 bus_dmamap_t pel_seq_num_dmamap; 742 U32 pel_seq_number_sz; 743 744 struct selinfo mpi3mr_select; 745 U32 mpi3mr_poll_waiting; 746 U32 mpi3mr_aen_triggered; 747 748 U16 wait_for_port_enable; 749 U16 track_mapping_events; 750 U16 pending_map_events; 751 mpi3mr_atomic_t fw_outstanding; 752 mpi3mr_atomic_t pend_ioctls; 753 struct proc *watchdog_thread; 754 void *watchdog_chan; 755 void *tm_chan; 756 u_int8_t remove_in_progress; 757 u_int8_t watchdog_thread_active; 758 u_int8_t do_timedout_reset; 759 bool allow_ios; 760 bool secure_ctrl; 761 mpi3mr_atomic_t pend_large_data_sz; 762 763 u_int32_t io_throttle_data_length; 764 u_int32_t io_throttle_high; 765 u_int32_t io_throttle_low; 766 u_int16_t num_io_throttle_group; 767 u_int iot_enable; 768 struct mpi3mr_throttle_group_info *throttle_groups; 769 770 struct dma_memory_desc ioctl_sge[MPI3MR_NUM_IOCTL_SGE]; 771 struct dma_memory_desc ioctl_chain_sge; 772 struct dma_memory_desc ioctl_resp_sge; 773 bool ioctl_sges_allocated; 774 struct proc *timestamp_thread_proc; 775 void *timestamp_chan; 776 u_int8_t timestamp_thread_active; 777 U32 ts_update_interval; 778 }; 779 780 static __inline uint64_t 781 mpi3mr_regread64(struct mpi3mr_softc *sc, uint32_t offset) 782 { 783 return bus_space_read_8(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset); 784 } 785 786 static __inline void 787 mpi3mr_regwrite64(struct mpi3mr_softc *sc, uint32_t offset, uint64_t val) 788 { 789 bus_space_write_8(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset, val); 790 } 791 792 static __inline uint32_t 793 mpi3mr_regread(struct mpi3mr_softc *sc, uint32_t offset) 794 { 795 return bus_space_read_4(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset); 796 } 797 798 static __inline void 799 mpi3mr_regwrite(struct mpi3mr_softc *sc, uint32_t offset, uint32_t val) 800 { 801 bus_space_write_4(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset, val); 802 } 803 804 #define MPI3MR_INFO (1 << 0) /* Basic info */ 805 #define MPI3MR_FAULT (1 << 1) /* Hardware faults */ 806 #define MPI3MR_EVENT (1 << 2) /* Event data from the controller */ 807 #define MPI3MR_LOG (1 << 3) /* Log data from the controller */ 808 #define MPI3MR_RECOVERY (1 << 4) /* Command error recovery tracing */ 809 #define MPI3MR_ERROR (1 << 5) /* Fatal driver/OS APIs failure */ 810 #define MPI3MR_XINFO (1 << 6) /* Additional info logs*/ 811 #define MPI3MR_TRACE (1 << 7) /* Trace functions */ 812 #define MPI3MR_IOT (1 << 8) /* IO throttling related debugs */ 813 #define MPI3MR_DEBUG_TM (1 << 9) /* Task management related debugs */ 814 #define MPI3MR_DEBUG_IOCTL (1 << 10) /* IOCTL related debugs */ 815 816 #define mpi3mr_printf(sc, args...) \ 817 device_printf((sc)->mpi3mr_dev, ##args) 818 819 #define mpi3mr_print_field(sc, msg, args...) \ 820 printf("\t" msg, ##args) 821 822 #define mpi3mr_vprintf(sc, args...) \ 823 do { \ 824 if (bootverbose) \ 825 mpi3mr_printf(sc, ##args); \ 826 } while (0) 827 828 #define mpi3mr_dprint(sc, level, msg, args...) \ 829 do { \ 830 if ((sc)->mpi3mr_debug & (level)) \ 831 device_printf((sc)->mpi3mr_dev, msg, ##args); \ 832 } while (0) 833 834 #define MPI3MR_PRINTFIELD_START(sc, tag...) \ 835 mpi3mr_printf((sc), ##tag); \ 836 mpi3mr_print_field((sc), ":\n") 837 #define MPI3MR_PRINTFIELD_END(sc, tag) \ 838 mpi3mr_printf((sc), tag "\n") 839 #define MPI3MR_PRINTFIELD(sc, facts, attr, fmt) \ 840 mpi3mr_print_field((sc), #attr ": " #fmt "\n", (facts)->attr) 841 842 #define mpi3mr_kproc_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg) \ 843 kproc_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg) 844 #define mpi3mr_kproc_exit(arg) kproc_exit(arg) 845 846 #if defined(CAM_PRIORITY_XPT) 847 #define MPI3MR_PRIORITY_XPT CAM_PRIORITY_XPT 848 #else 849 #define MPI3MR_PRIORITY_XPT 5 850 #endif 851 852 static __inline void 853 mpi3mr_clear_bit(int b, volatile void *p) 854 { 855 atomic_clear_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f)); 856 } 857 858 static __inline void 859 mpi3mr_set_bit(int b, volatile void *p) 860 { 861 atomic_set_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f)); 862 } 863 864 static __inline int 865 mpi3mr_test_bit(int b, volatile void *p) 866 { 867 return ((volatile int *)p)[b >> 5] & (1 << (b & 0x1f)); 868 } 869 870 static __inline int 871 mpi3mr_test_and_set_bit(int b, volatile void *p) 872 { 873 int ret = ((volatile int *)p)[b >> 5] & (1 << (b & 0x1f)); 874 875 atomic_set_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f)); 876 return ret; 877 } 878 879 static __inline int 880 mpi3mr_find_first_zero_bit(void *p, int bit_count) 881 { 882 int i, sz, j=0; 883 U8 *loc; 884 885 sz = bit_count % 8 ? (bit_count / 8 + 1) : (bit_count / 8); 886 loc = malloc(sz, M_MPI3MR, M_NOWAIT | M_ZERO); 887 888 memcpy(loc, p, sz); 889 890 for (i = 0; i < sz; i++) { 891 j = 0; 892 while (j < 8) { 893 if (!((loc[i] >> j) & 0x1)) 894 goto out; 895 j++; 896 } 897 } 898 out: 899 free(loc, M_MPI3MR); 900 return (i + j); 901 } 902 903 #define MPI3MR_DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d)) 904 905 void 906 init_completion(struct completion *completion); 907 908 void 909 complete(struct completion *completion); 910 911 void wait_for_completion_timeout(struct completion *completion, 912 U32 timeout); 913 void wait_for_completion_timeout_tm(struct completion *completion, 914 U32 timeout, struct mpi3mr_softc *sc); 915 void mpi3mr_add_sg_single(void *paddr, U8 flags, U32 length, 916 bus_addr_t dma_addr); 917 void mpi3mr_enable_interrupts(struct mpi3mr_softc *sc); 918 void mpi3mr_disable_interrupts(struct mpi3mr_softc *sc); 919 void mpi3mr_memaddr_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error); 920 int mpi3mr_submit_admin_cmd(struct mpi3mr_softc *mrioc, void *admin_req, 921 U16 admin_req_sz); 922 int mpi3mr_submit_io(struct mpi3mr_softc *mrioc, 923 struct mpi3mr_op_req_queue *op_req_q, U8 *req); 924 int 925 mpi3mr_alloc_interrupts(struct mpi3mr_softc *sc, U16 setup_one); 926 927 void mpi3mr_cleanup_ioc(struct mpi3mr_softc *sc); 928 int mpi3mr_initialize_ioc(struct mpi3mr_softc *sc, U8 reason); 929 void mpi3mr_build_zero_len_sge(void *paddr); 930 int mpi3mr_issue_event_notification(struct mpi3mr_softc *sc); 931 int 932 mpi3mr_register_events(struct mpi3mr_softc *sc); 933 void mpi3mr_process_op_reply_desc(struct mpi3mr_softc *sc, 934 Mpi3DefaultReplyDescriptor_t *reply_desc, U64 *reply_dma); 935 struct mpi3mr_cmd * 936 mpi3mr_get_command(struct mpi3mr_softc *sc); 937 void 938 mpi3mr_release_command(struct mpi3mr_cmd *cmd); 939 int 940 mpi3mr_complete_io_cmd(struct mpi3mr_softc *sc, 941 struct mpi3mr_irq_context *irq_context); 942 int 943 mpi3mr_cam_detach(struct mpi3mr_softc *sc); 944 int 945 mpi3mr_cam_attach(struct mpi3mr_softc *sc); 946 struct mpi3mr_target * 947 mpi3mr_find_target_by_per_id(struct mpi3mr_cam_softc *cam_sc, 948 uint16_t per_id); 949 struct mpi3mr_target * 950 mpi3mr_find_target_by_dev_handle(struct mpi3mr_cam_softc *cam_sc, 951 uint16_t dev_handle); 952 int mpi3mr_create_device(struct mpi3mr_softc *sc, 953 Mpi3DevicePage0_t *dev_pg0); 954 void 955 mpi3mr_unmap_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cmd); 956 void 957 init_completion(struct completion *completion); 958 void 959 complete(struct completion *completion); 960 void wait_for_completion_timeout(struct completion *completion, 961 U32 timeout); 962 void 963 poll_for_command_completion(struct mpi3mr_softc *sc, 964 struct mpi3mr_drvr_cmd *cmd, U16 wait); 965 int 966 mpi3mr_alloc_requests(struct mpi3mr_softc *sc); 967 void 968 mpi3mr_watchdog(void *arg); 969 int mpi3mr_issue_port_enable(struct mpi3mr_softc *mrioc, U8 async); 970 void 971 mpi3mr_isr(void *privdata); 972 int 973 mpi3mr_alloc_msix_queues(struct mpi3mr_softc *sc); 974 void 975 mpi3mr_destory_mtx(struct mpi3mr_softc *sc); 976 void 977 mpi3mr_free_mem(struct mpi3mr_softc *sc); 978 void 979 mpi3mr_cleanup_interrupts(struct mpi3mr_softc *sc); 980 int mpi3mr_setup_irqs(struct mpi3mr_softc *sc); 981 void mpi3mr_cleanup_event_taskq(struct mpi3mr_softc *sc); 982 void 983 mpi3mr_hexdump(void *buf, int sz, int format); 984 int mpi3mr_soft_reset_handler(struct mpi3mr_softc *sc, 985 U16 reset_reason, bool snapdump); 986 void 987 mpi3mrsas_release_simq_reinit(struct mpi3mr_cam_softc *cam_sc); 988 void 989 mpi3mr_watchdog_thread(void *arg); 990 void mpi3mr_timestamp_thread(void *arg); 991 void mpi3mr_add_device(struct mpi3mr_softc *sc, U16 per_id); 992 int mpi3mr_remove_device(struct mpi3mr_softc *sc, U16 handle); 993 int 994 mpi3mrsas_register_events(struct mpi3mr_softc *sc); 995 int mpi3mr_process_event_ack(struct mpi3mr_softc *sc, U8 event, 996 U32 event_ctx); 997 int mpi3mr_remove_device_from_os(struct mpi3mr_softc *sc, U16 handle); 998 void mpi3mr_remove_device_from_list(struct mpi3mr_softc *sc, struct mpi3mr_target *target, 999 bool must_delete); 1000 void mpi3mr_update_device(struct mpi3mr_softc *mrioc, 1001 struct mpi3mr_target *tgtdev, Mpi3DevicePage0_t *dev_pg0, bool is_added); 1002 void mpi3mr_app_save_logdata(struct mpi3mr_softc *sc, char *event_data, U16 event_data_size); 1003 void mpi3mr_set_io_divert_for_all_vd_in_tg(struct mpi3mr_softc *sc, 1004 struct mpi3mr_throttle_group_info *tg, U8 divert_value); 1005 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_softc *sc); 1006 void mpi3mr_poll_pend_io_completions(struct mpi3mr_softc *sc); 1007 void int_to_lun(unsigned int lun, U8 *req_lun); 1008 void trigger_reset_from_watchdog(struct mpi3mr_softc *sc, U8 reset_type, U16 reset_reason); 1009 void mpi3mr_alloc_ioctl_dma_memory(struct mpi3mr_softc *sc); 1010 int mpi3mr_cfg_get_driver_pg1(struct mpi3mr_softc *sc); 1011 #endif /*MPI3MR_H_INCLUDED*/ 1012