1 /*
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2020-2025, Broadcom Inc. All rights reserved.
5 * Support: <fbsd-storage-driver.pdl@broadcom.com>
6 *
7 * Authors: Sumit Saxena <sumit.saxena@broadcom.com>
8 * Chandrakanth Patil <chandrakanth.patil@broadcom.com>
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are
12 * met:
13 *
14 * 1. Redistributions of source code must retain the above copyright notice,
15 * this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright notice,
17 * this list of conditions and the following disclaimer in the documentation and/or other
18 * materials provided with the distribution.
19 * 3. Neither the name of the Broadcom Inc. nor the names of its contributors
20 * may be used to endorse or promote products derived from this software without
21 * specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33 * POSSIBILITY OF SUCH DAMAGE.
34 *
35 * The views and conclusions contained in the software and documentation are
36 * those of the authors and should not be interpreted as representing
37 * official policies,either expressed or implied, of the FreeBSD Project.
38 *
39 * Mail to: Broadcom Inc 1320 Ridder Park Dr, San Jose, CA 95131
40 *
41 * Broadcom Inc. (Broadcom) MPI3MR Adapter FreeBSD
42 */
43
44 #ifndef _MPI3MRVAR_H
45 #define _MPI3MRVAR_H
46
47 #include <sys/types.h>
48 #include <sys/param.h>
49 #include <sys/systm.h>
50 #include <sys/kernel.h>
51 #include <sys/module.h>
52 #include <sys/bus.h>
53 #include <sys/conf.h>
54 #include <sys/malloc.h>
55 #include <sys/sysctl.h>
56 #include <sys/uio.h>
57 #include <sys/selinfo.h>
58 #include <sys/poll.h>
59
60 #include <sys/lock.h>
61 #include <sys/mutex.h>
62 #include <sys/endian.h>
63 #include <sys/sysent.h>
64 #include <sys/taskqueue.h>
65 #include <sys/smp.h>
66
67 #include <machine/bus.h>
68 #include <machine/resource.h>
69 #include <sys/rman.h>
70
71 #include <dev/pci/pcireg.h>
72 #include <dev/pci/pcivar.h>
73 #include <dev/pci/pci_private.h>
74
75 #include <cam/cam.h>
76 #include <cam/cam_ccb.h>
77 #include <cam/cam_debug.h>
78 #include <cam/cam_sim.h>
79 #include <cam/cam_xpt_sim.h>
80 #include <cam/cam_xpt_periph.h>
81 #include <cam/cam_periph.h>
82 #include <cam/scsi/scsi_all.h>
83 #include <cam/scsi/scsi_message.h>
84
85 #include <cam/scsi/smp_all.h>
86 #include <sys/queue.h>
87 #include <sys/kthread.h>
88 #include "mpi/mpi30_api.h"
89
90 #define MPI3MR_DRIVER_VERSION "8.14.0.2.0"
91 #define MPI3MR_DRIVER_RELDATE "9th Apr 2025"
92
93 #define MPI3MR_DRIVER_NAME "mpi3mr"
94
95 #define MPI3MR_NAME_LENGTH 32
96 #define IOCNAME "%s: "
97
98 #define MPI3MR_DEFAULT_MAX_IO_SIZE (1 * 1024 * 1024)
99
100 #define SAS4116_CHIP_REV_A0 0
101 #define SAS4116_CHIP_REV_B0 1
102
103 #define MPI3MR_MAX_SECTORS 2048
104 #define MPI3MR_MAX_CMDS_LUN 7
105 #define MPI3MR_MAX_CDB_LENGTH 16
106 #define MPI3MR_MAX_LUN 16895
107
108 #define MPI3MR_SATA_QDEPTH 32
109 #define MPI3MR_SAS_QDEPTH 64
110 #define MPI3MR_RAID_QDEPTH 128
111 #define MPI3MR_NVME_QDEPTH 128
112
113 /* Definitions for internal SGL and Chain SGL buffers */
114 #define MPI3MR_4K_PGSZ 4096
115 #define MPI3MR_PAGE_SIZE_4K 4096
116 #define MPI3MR_DEFAULT_SGL_ENTRIES 256
117 #define MPI3MR_MAX_SGL_ENTRIES 2048
118
119 #define MPI3MR_AREQQ_SIZE (2 * MPI3MR_4K_PGSZ)
120 #define MPI3MR_AREPQ_SIZE (4 * MPI3MR_4K_PGSZ)
121 #define MPI3MR_AREQ_FRAME_SZ 128
122 #define MPI3MR_AREP_FRAME_SZ 16
123
124 #define MPI3MR_OPREQQ_SIZE (8 * MPI3MR_4K_PGSZ)
125 #define MPI3MR_OPREPQ_SIZE (4 * MPI3MR_4K_PGSZ)
126
127 /* Operational queue management definitions */
128 #define MPI3MR_OP_REQ_Q_QD 512
129 #define MPI3MR_OP_REP_Q_QD 1024
130 #define MPI3MR_OP_REP_Q_QD_A0 4096
131
132 #define MPI3MR_THRESHOLD_REPLY_COUNT 100
133
134 #define MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST \
135 (MPI3_SGE_FLAGS_ELEMENT_TYPE_SIMPLE | MPI3_SGE_FLAGS_DLAS_SYSTEM | \
136 MPI3_SGE_FLAGS_END_OF_LIST)
137
138 #define MPI3MR_HOSTTAG_INVALID 0xFFFF
139 #define MPI3MR_HOSTTAG_INITCMDS 1
140 #define MPI3MR_HOSTTAG_IOCTLCMDS 2
141 #define MPI3MR_HOSTTAG_PELABORT 3
142 #define MPI3MR_HOSTTAG_PELWAIT 4
143 #define MPI3MR_HOSTTAG_TMS 5
144 #define MPI3MR_HOSTTAG_CFGCMDS 6
145
146 #define MAX_MGMT_ADAPTERS 8
147 #define MPI3MR_WAIT_BEFORE_CTRL_RESET 5
148
149 #define MPI3MR_RESET_REASON_OSTYPE_FREEBSD 0x4
150 #define MPI3MR_RESET_REASON_OSTYPE_SHIFT 28
151 #define MPI3MR_RESET_REASON_IOCNUM_SHIFT 20
152
153 struct mpi3mr_mgmt_info {
154 uint16_t count;
155 struct mpi3mr_softc *sc_ptr[MAX_MGMT_ADAPTERS];
156 int max_index;
157 };
158
159 extern char fmt_os_ver[16];
160
161 #define MPI3MR_OS_VERSION(raw_os_ver, fmt_os_ver) sprintf(raw_os_ver, "%d", __FreeBSD_version); \
162 sprintf(fmt_os_ver, "%c%c.%c%c.%c%c%c",\
163 raw_os_ver[0], raw_os_ver[1], raw_os_ver[2],\
164 raw_os_ver[3], raw_os_ver[4], raw_os_ver[5],\
165 raw_os_ver[6]);
166 #define MPI3MR_NUM_DEVRMCMD 1
167 #define MPI3MR_HOSTTAG_DEVRMCMD_MIN (MPI3MR_HOSTTAG_CFGCMDS + 1)
168 #define MPI3MR_HOSTTAG_DEVRMCMD_MAX (MPI3MR_HOSTTAG_DEVRMCMD_MIN + \
169 MPI3MR_NUM_DEVRMCMD - 1)
170 #define MPI3MR_INTERNALCMDS_RESVD MPI3MR_HOSTTAG_DEVRMCMD_MAX
171
172 #define MPI3MR_NUM_EVTACKCMD 4
173 #define MPI3MR_HOSTTAG_EVTACKCMD_MIN (MPI3MR_HOSTTAG_DEVRMCMD_MAX + 1)
174 #define MPI3MR_HOSTTAG_EVTACKCMD_MAX (MPI3MR_HOSTTAG_EVTACKCMD_MIN + \
175 MPI3MR_NUM_EVTACKCMD - 1)
176
177 /* command/controller interaction timeout definitions in seconds */
178 #define MPI3MR_INTADMCMD_TIMEOUT 60
179 #define MPI3MR_PORTENABLE_TIMEOUT 300
180 #define MPI3MR_ABORTTM_TIMEOUT 60
181 #define MPI3MR_RESETTM_TIMEOUT 60
182 #define MPI3MR_TSUPDATE_INTERVAL 900
183 #define MPI3MR_DEFAULT_SHUTDOWN_TIME 120
184 #define MPI3MR_RAID_ERRREC_RESET_TIMEOUT 180
185 #define MPI3MR_RESET_HOST_IOWAIT_TIMEOUT 5
186 #define MPI3MR_PREPARE_FOR_RESET_TIMEOUT 180
187 #define MPI3MR_RESET_ACK_TIMEOUT 30
188 #define MPI3MR_MUR_TIMEOUT 120
189
190 #define MPI3MR_CMD_NOTUSED 0x8000
191 #define MPI3MR_CMD_COMPLETE 0x0001
192 #define MPI3MR_CMD_PENDING 0x0002
193 #define MPI3MR_CMD_REPLYVALID 0x0004
194 #define MPI3MR_CMD_RESET 0x0008
195
196 #define MPI3MR_NUM_EVTREPLIES 64
197 #define MPI3MR_SENSEBUF_SZ 256
198 #define MPI3MR_SENSEBUF_FACTOR 3
199 #define MPI3MR_CHAINBUF_FACTOR 3
200
201 #define MPT3SAS_HOSTPGSZ_4KEXP 12
202
203 #define MPI3MR_INVALID_DEV_HANDLE 0xFFFF
204
205 /* Controller Reset related definitions */
206 #define MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT 5
207 #define MPI3MR_MAX_SHUTDOWN_RETRY_COUNT 2
208
209 /* ResponseCode values */
210 #define MPI3MR_RI_MASK_RESPCODE (0x000000FF)
211 #define MPI3MR_RSP_TM_COMPLETE 0x00
212 #define MPI3MR_RSP_INVALID_FRAME 0x02
213 #define MPI3MR_RSP_TM_NOT_SUPPORTED 0x04
214 #define MPI3MR_RSP_TM_FAILED 0x05
215 #define MPI3MR_RSP_TM_SUCCEEDED 0x08
216 #define MPI3MR_RSP_TM_INVALID_LUN 0x09
217 #define MPI3MR_RSP_TM_OVERLAPPED_TAG 0x0A
218 #define MPI3MR_RSP_IO_QUEUED_ON_IOC \
219 MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC
220
221 /* Definitions for the controller security status*/
222 #define MPI3MR_CTLR_SECURITY_STATUS_MASK 0x0C
223 #define MPI3MR_CTLR_SECURE_DBG_STATUS_MASK 0x02
224
225 #define MPI3MR_INVALID_DEVICE 0x00
226 #define MPI3MR_CONFIG_SECURE_DEVICE 0x04
227 #define MPI3MR_HARD_SECURE_DEVICE 0x08
228 #define MPI3MR_TAMPERED_DEVICE 0x0C
229
230 #define MPI3MR_DEFAULT_MDTS (128 * 1024)
231 #define MPI3MR_DEFAULT_PGSZEXP (12)
232 #define MPI3MR_MAX_IOCTL_TRANSFER_SIZE (1024 * 1024)
233
234 #define MPI3MR_DEVRMHS_RETRYCOUNT 3
235 #define MPI3MR_PELCMDS_RETRYCOUNT 3
236
237 #define MPI3MR_PERIODIC_DELAY 1 /* 1 second heartbeat/watchdog check */
238
239 #define WRITE_SAME_32 0x0d
240
241 #define MPI3MR_TSUPDATE_INTERVAL 900
242
243 struct completion {
244 unsigned int done;
245 struct mtx lock;
246 };
247
248 typedef union {
249 volatile unsigned int val;
250 unsigned int val_rdonly;
251 } mpi3mr_atomic_t;
252
253 #define mpi3mr_atomic_read(v) atomic_load_acq_int(&(v)->val)
254 #define mpi3mr_atomic_set(v,i) atomic_store_rel_int(&(v)->val, i)
255 #define mpi3mr_atomic_dec(v) atomic_subtract_int(&(v)->val, 1)
256 #define mpi3mr_atomic_inc(v) atomic_add_int(&(v)->val, 1)
257 #define mpi3mr_atomic_add(v, u) atomic_add_int(&(v)->val, u)
258 #define mpi3mr_atomic_sub(v, u) atomic_subtract_int(&(v)->val, u)
259
260 /* IOCTL data transfer sge*/
261 #define MPI3MR_NUM_IOCTL_SGE 256
262 #define MPI3MR_IOCTL_SGE_SIZE (8 * 1024)
263
264 struct dma_memory_desc {
265 U32 size;
266 void *addr;
267 bus_dma_tag_t tag;
268 bus_dmamap_t dmamap;
269 bus_addr_t dma_addr;
270 };
271
272 enum mpi3mr_iocstate {
273 MRIOC_STATE_READY = 1,
274 MRIOC_STATE_RESET,
275 MRIOC_STATE_FAULT,
276 MRIOC_STATE_BECOMING_READY,
277 MRIOC_STATE_RESET_REQUESTED,
278 MRIOC_STATE_UNRECOVERABLE,
279 MRIOC_STATE_COUNT,
280 };
281
282 /* Init type definitions */
283 enum mpi3mr_init_type {
284 MPI3MR_INIT_TYPE_INIT = 0,
285 MPI3MR_INIT_TYPE_RESET,
286 MPI3MR_INIT_TYPE_RESUME,
287 };
288
289 /* Reset reason code definitions*/
290 enum mpi3mr_reset_reason {
291 MPI3MR_RESET_FROM_BRINGUP = 1,
292 MPI3MR_RESET_FROM_FAULT_WATCH = 2,
293 MPI3MR_RESET_FROM_IOCTL = 3,
294 MPI3MR_RESET_FROM_EH_HOS = 4,
295 MPI3MR_RESET_FROM_TM_TIMEOUT = 5,
296 MPI3MR_RESET_FROM_IOCTL_TIMEOUT = 6,
297 MPI3MR_RESET_FROM_MUR_FAILURE = 7,
298 MPI3MR_RESET_FROM_CTLR_CLEANUP = 8,
299 MPI3MR_RESET_FROM_CIACTIV_FAULT = 9,
300 MPI3MR_RESET_FROM_PE_TIMEOUT = 10,
301 MPI3MR_RESET_FROM_TSU_TIMEOUT = 11,
302 MPI3MR_RESET_FROM_DELREQQ_TIMEOUT = 12,
303 MPI3MR_RESET_FROM_DELREPQ_TIMEOUT = 13,
304 MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT = 14,
305 MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT = 15,
306 MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT = 16,
307 MPI3MR_RESET_FROM_IOCINIT_TIMEOUT = 17,
308 MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT = 18,
309 MPI3MR_RESET_FROM_EVTACK_TIMEOUT = 19,
310 MPI3MR_RESET_FROM_CIACTVRST_TIMER = 20,
311 MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT = 21,
312 MPI3MR_RESET_FROM_PELABORT_TIMEOUT = 22,
313 MPI3MR_RESET_FROM_SYSFS = 23,
314 MPI3MR_RESET_FROM_SYSFS_TIMEOUT = 24,
315 MPI3MR_RESET_FROM_DIAG_BUFFER_POST_TIMEOUT = 25,
316 MPI3MR_RESET_FROM_SCSIIO_TIMEOUT = 26,
317 MPI3MR_RESET_FROM_FIRMWARE = 27,
318 MPI3MR_DEFAULT_RESET_REASON = 28,
319 MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT = 29,
320 MPI3MR_RESET_REASON_COUNT,
321 };
322
323 struct mpi3mr_compimg_ver
324 {
325 U16 build_num;
326 U16 cust_id;
327 U8 ph_minor;
328 U8 ph_major;
329 U8 gen_minor;
330 U8 gen_major;
331 };
332
333 struct mpi3mr_ioc_facts
334 {
335 U32 ioc_capabilities;
336 struct mpi3mr_compimg_ver fw_ver;
337 U32 mpi_version;
338 U16 max_reqs;
339 U16 product_id;
340 U16 op_req_sz;
341 U16 reply_sz;
342 U16 exceptions;
343 U16 max_perids;
344 U16 max_pds;
345 U16 max_sasexpanders;
346 U32 max_data_length;
347 U16 max_sasinitiators;
348 U16 max_enclosures;
349 U16 max_pcieswitches;
350 U16 max_nvme;
351 U16 max_vds;
352 U16 max_hpds;
353 U16 max_advhpds;
354 U16 max_raidpds;
355 U16 min_devhandle;
356 U16 max_devhandle;
357 U16 max_op_req_q;
358 U16 max_op_reply_q;
359 U16 shutdown_timeout;
360 U8 ioc_num;
361 U8 who_init;
362 U16 max_msix_vectors;
363 U8 personality;
364 U8 dma_mask;
365 U8 protocol_flags;
366 U8 sge_mod_mask;
367 U8 sge_mod_value;
368 U8 sge_mod_shift;
369 U8 max_dev_per_tg;
370 U16 max_io_throttle_group;
371 U16 io_throttle_data_length;
372 U16 io_throttle_low;
373 U16 io_throttle_high;
374 };
375
376 struct mpi3mr_op_req_queue {
377 U16 ci;
378 U16 pi;
379 U16 num_reqs;
380 U8 qid;
381 U8 reply_qid;
382 U32 qsz;
383 void *q_base;
384 bus_dma_tag_t q_base_tag;
385 bus_dmamap_t q_base_dmamap;
386 bus_addr_t q_base_phys;
387 struct mtx q_lock;
388 };
389
390 struct mpi3mr_op_reply_queue {
391 U16 ci;
392 U8 ephase;
393 U8 qid;
394 U16 num_replies;
395 U32 qsz;
396 bus_dma_tag_t q_base_tag;
397 bus_dmamap_t q_base_dmamap;
398 void *q_base;
399 bus_addr_t q_base_phys;
400 mpi3mr_atomic_t pend_ios;
401 bool in_use;
402 struct mtx q_lock;
403 };
404
405 struct irq_info {
406 MPI3_REPLY_DESCRIPTORS_UNION *post_queue;
407 bus_dma_tag_t buffer_dmat;
408 struct resource *irq;
409 void *intrhand;
410 int irq_rid;
411 };
412
413 struct mpi3mr_irq_context {
414 struct mpi3mr_softc *sc;
415 U16 msix_index;
416 struct mpi3mr_op_reply_queue *op_reply_q;
417 char name[MPI3MR_NAME_LENGTH];
418 struct irq_info irq_info;
419 };
420
421 MALLOC_DECLARE(M_MPI3MR);
422 SYSCTL_DECL(_hw_mpi3mr);
423
424 typedef struct mpi3mr_drvr_cmd DRVR_CMD;
425 typedef void (*DRVR_CMD_CALLBACK)(struct mpi3mr_softc *mrioc, DRVR_CMD *drvrcmd);
426 struct mpi3mr_drvr_cmd {
427 struct mtx lock;
428 struct completion completion;
429 void *reply;
430 U8 *sensebuf;
431 U8 iou_rc;
432 U16 state;
433 U16 dev_handle;
434 U16 ioc_status;
435 U32 ioc_loginfo;
436 U8 is_waiting;
437 U8 is_senseprst;
438 U8 retry_count;
439 U16 host_tag;
440 DRVR_CMD_CALLBACK callback;
441 };
442
443 struct mpi3mr_cmd;
444 typedef void mpi3mr_evt_callback_t(struct mpi3mr_softc *, uintptr_t,
445 Mpi3EventNotificationReply_t *reply);
446 typedef void mpi3mr_cmd_callback_t(struct mpi3mr_softc *,
447 struct mpi3mr_cmd *cmd);
448
449 #define MPI3MR_IOVEC_COUNT 2
450
451 enum mpi3mr_data_xfer_direction {
452 MPI3MR_READ = 1,
453 MPI3MR_WRITE,
454 };
455
456 enum mpi3mr_cmd_state {
457 MPI3MR_CMD_STATE_FREE = 1,
458 MPI3MR_CMD_STATE_BUSY,
459 MPI3MR_CMD_STATE_IN_QUEUE,
460 MPI3MR_CMD_STATE_IN_TM,
461 };
462
463 enum mpi3mr_target_state {
464 MPI3MR_DEV_CREATED = 1,
465 MPI3MR_DEV_REMOVE_HS_COMPLETED = 2,
466 };
467
468 struct mpi3mr_cmd {
469 TAILQ_ENTRY(mpi3mr_cmd) next;
470 struct mpi3mr_softc *sc;
471 union ccb *ccb;
472 void *data;
473 u_int length;
474 struct mpi3mr_target *targ;
475 u_int data_dir;
476 u_int state;
477 bus_dmamap_t dmamap;
478 struct scsi_sense_data *sense;
479 struct callout callout;
480 bool callout_owner;
481 U16 hosttag;
482 U8 req_qidx;
483 Mpi3SCSIIORequest_t io_request;
484 };
485
486 struct mpi3mr_chain {
487 bus_dmamap_t buf_dmamap;
488 void *buf;
489 bus_addr_t buf_phys;
490 };
491
492 struct mpi3mr_event_handle {
493 TAILQ_ENTRY(mpi3mr_event_handle) eh_list;
494 mpi3mr_evt_callback_t *callback;
495 void *data;
496 uint8_t mask[16];
497 };
498
499 struct mpi3mr_fw_event_work {
500 U16 event;
501 void *event_data;
502 TAILQ_ENTRY(mpi3mr_fw_event_work) ev_link;
503 U8 send_ack;
504 U8 process_event;
505 U32 event_context;
506 U16 event_data_size;
507 };
508
509 /**
510 * struct delayed_dev_rmhs_node - Delayed device removal node
511 *
512 * @list: list head
513 * @handle: Device handle
514 * @iou_rc: IO Unit Control Reason Code
515 */
516 struct delayed_dev_rmhs_node {
517 TAILQ_ENTRY(delayed_dev_rmhs_node) list;
518 U16 handle;
519 U8 iou_rc;
520 };
521
522 /**
523 * struct delayed_evtack_node - Delayed event ack node
524 *
525 * @list: list head
526 * @event: MPI3 event ID
527 * @event_ctx: Event context
528 */
529 struct delayed_evtack_node {
530 TAILQ_ENTRY(delayed_evtack_node) list;
531 U8 event;
532 U32 event_ctx;
533 };
534
535 /* Reset types */
536 enum reset_type {
537 MPI3MR_NO_RESET,
538 MPI3MR_TRIGGER_SOFT_RESET,
539 };
540
541 struct mpi3mr_reset {
542 u_int type;
543 U32 reason;
544 int status;
545 bool ioctl_reset_snapdump;
546 };
547
548 struct mpi3mr_softc {
549 device_t mpi3mr_dev;
550 struct cdev *mpi3mr_cdev;
551 u_int mpi3mr_flags;
552 #define MPI3MR_FLAGS_SHUTDOWN (1 << 0)
553 #define MPI3MR_FLAGS_DIAGRESET (1 << 1)
554 #define MPI3MR_FLAGS_ATTACH_DONE (1 << 2)
555 #define MPI3MR_FLAGS_PORT_ENABLE_DONE (1 << 3)
556 U8 id;
557 int cpu_count;
558 char name[MPI3MR_NAME_LENGTH];
559 char driver_name[MPI3MR_NAME_LENGTH];
560 int bars;
561 bus_addr_t dma_loaddr;
562 bus_addr_t dma_hiaddr;
563 u_int mpi3mr_debug;
564 struct mpi3mr_reset reset;
565 int max_msix_vectors;
566 int msix_count;
567 bool msix_enable;
568 int io_cmds_highwater;
569 int max_chains;
570 uint32_t chain_frame_size;
571 struct sysctl_ctx_list sysctl_ctx;
572 struct sysctl_oid *sysctl_tree;
573 char fw_version[32];
574 struct mpi3mr_chain *chains;
575 struct callout periodic;
576 struct callout device_check_callout;
577
578 struct mpi3mr_cam_softc *cam_sc;
579 struct mpi3mr_cmd **cmd_list;
580 TAILQ_HEAD(, mpi3mr_cmd) cmd_list_head;
581 struct mtx cmd_pool_lock;
582
583 struct resource *mpi3mr_regs_resource;
584 bus_space_handle_t mpi3mr_bhandle;
585 bus_space_tag_t mpi3mr_btag;
586 int mpi3mr_regs_rid;
587
588 bus_dma_tag_t mpi3mr_parent_dmat;
589 bus_dma_tag_t buffer_dmat;
590
591 int num_reqs;
592 int num_replies;
593 int num_chains;
594
595 TAILQ_HEAD(, mpi3mr_event_handle) event_list;
596 struct mpi3mr_event_handle *mpi3mr_log_eh;
597 struct intr_config_hook mpi3mr_ich;
598
599 struct mtx mpi3mr_mtx;
600 struct mtx io_lock;
601 U8 intr_enabled;
602 TAILQ_HEAD(, delayed_dev_rmhs_node) delayed_rmhs_list;
603 TAILQ_HEAD(, delayed_evtack_node) delayed_evtack_cmds_list;
604
605 U16 num_admin_reqs;
606 U32 admin_req_q_sz;
607 U16 admin_req_pi;
608 U16 admin_req_ci;
609 bus_dma_tag_t admin_req_tag;
610 bus_dmamap_t admin_req_dmamap;
611 bus_addr_t admin_req_phys;
612 U8 *admin_req;
613 struct mtx admin_req_lock;
614
615 U16 num_admin_replies;
616 U32 admin_reply_q_sz;
617 U16 admin_reply_ci;
618 U8 admin_reply_ephase;
619 bus_dma_tag_t admin_reply_tag;
620 bus_dmamap_t admin_reply_dmamap;
621 bus_addr_t admin_reply_phys;
622 U8 *admin_reply;
623 struct mtx admin_reply_lock;
624 bool admin_in_use;
625
626 U32 num_reply_bufs;
627 bus_dma_tag_t reply_buf_tag;
628 bus_dmamap_t reply_buf_dmamap;
629 bus_addr_t reply_buf_phys;
630 U8 *reply_buf;
631 bus_addr_t reply_buf_dma_max_address;
632 bus_addr_t reply_buf_dma_min_address;
633
634 U16 reply_free_q_sz;
635 bus_dma_tag_t reply_free_q_tag;
636 bus_dmamap_t reply_free_q_dmamap;
637 bus_addr_t reply_free_q_phys;
638 U64 *reply_free_q;
639 struct mtx reply_free_q_lock;
640 U32 reply_free_q_host_index;
641
642 U32 num_sense_bufs;
643 bus_dma_tag_t sense_buf_tag;
644 bus_dmamap_t sense_buf_dmamap;
645 bus_addr_t sense_buf_phys;
646 U8 *sense_buf;
647
648 U16 sense_buf_q_sz;
649 bus_dma_tag_t sense_buf_q_tag;
650 bus_dmamap_t sense_buf_q_dmamap;
651 bus_addr_t sense_buf_q_phys;
652 U64 *sense_buf_q;
653 struct mtx sense_buf_q_lock;
654 U32 sense_buf_q_host_index;
655
656 void *nvme_encap_prp_list;
657 bus_addr_t nvme_encap_prp_list_dma;
658 bus_dma_tag_t nvme_encap_prp_list_dmatag;
659 bus_dmamap_t nvme_encap_prp_list_dma_dmamap;
660 U32 nvme_encap_prp_sz;
661
662 U32 ready_timeout;
663
664 struct mpi3mr_irq_context *irq_ctx;
665
666 U16 num_queues; /* Number of request/reply queues */
667 struct mpi3mr_op_req_queue *op_req_q;
668 struct mpi3mr_op_reply_queue *op_reply_q;
669 U16 num_hosttag_op_req_q;
670
671 struct mpi3mr_drvr_cmd init_cmds;
672 struct mpi3mr_ioc_facts facts;
673 U16 reply_sz;
674 U16 op_reply_sz;
675
676 U32 event_masks[MPI3_EVENT_NOTIFY_EVENTMASK_WORDS];
677
678 char fwevt_worker_name[MPI3MR_NAME_LENGTH];
679 struct workqueue_struct *fwevt_worker_thread;
680 struct mtx fwevt_lock;
681 struct mtx target_lock;
682
683 U16 max_host_ios;
684 U32 max_sgl_entries;
685 bus_dma_tag_t chain_sgl_list_tag;
686 struct mpi3mr_chain *chain_sgl_list;
687 U16 chain_bitmap_sz;
688 void *chain_bitmap;
689 struct mtx chain_buf_lock;
690 U16 chain_buf_count;
691
692 struct mpi3mr_drvr_cmd ioctl_cmds;
693 struct mpi3mr_drvr_cmd host_tm_cmds;
694 struct mpi3mr_drvr_cmd dev_rmhs_cmds[MPI3MR_NUM_DEVRMCMD];
695 struct mpi3mr_drvr_cmd evtack_cmds[MPI3MR_NUM_EVTACKCMD];
696 struct mpi3mr_drvr_cmd cfg_cmds;
697
698 U16 devrem_bitmap_sz;
699 void *devrem_bitmap;
700
701 U16 dev_handle_bitmap_sz;
702 void *removepend_bitmap;
703
704 U16 evtack_cmds_bitmap_sz;
705 void *evtack_cmds_bitmap;
706
707 U32 ts_update_counter;
708 U8 reset_in_progress;
709 U8 unrecoverable;
710 U8 block_ioctls;
711 U8 in_prep_ciactv_rst;
712 U16 prep_ciactv_rst_counter;
713 struct mtx reset_mutex;
714
715 U8 prepare_for_reset;
716 U16 prepare_for_reset_timeout_counter;
717
718 U16 diagsave_timeout;
719 int logging_level;
720 U16 flush_io_count;
721
722 Mpi3DriverInfoLayout_t driver_info;
723
724 U16 change_count;
725
726 U8 *log_data_buffer;
727 U16 log_data_buffer_index;
728 U16 log_data_entry_size;
729
730 U8 pel_wait_pend;
731 U8 pel_abort_requested;
732 U8 pel_class;
733 U16 pel_locale;
734
735 struct mpi3mr_drvr_cmd pel_cmds;
736 struct mpi3mr_drvr_cmd pel_abort_cmd;
737 U32 newest_seqnum;
738 void *pel_seq_number;
739 bus_addr_t pel_seq_number_dma;
740 bus_dma_tag_t pel_seq_num_dmatag;
741 bus_dmamap_t pel_seq_num_dmamap;
742 U32 pel_seq_number_sz;
743
744 struct selinfo mpi3mr_select;
745 U32 mpi3mr_poll_waiting;
746 U32 mpi3mr_aen_triggered;
747
748 U16 wait_for_port_enable;
749 U16 track_mapping_events;
750 U16 pending_map_events;
751 mpi3mr_atomic_t fw_outstanding;
752 mpi3mr_atomic_t pend_ioctls;
753 struct proc *watchdog_thread;
754 void *watchdog_chan;
755 void *tm_chan;
756 u_int8_t remove_in_progress;
757 u_int8_t watchdog_thread_active;
758 u_int8_t do_timedout_reset;
759 bool allow_ios;
760 bool secure_ctrl;
761 mpi3mr_atomic_t pend_large_data_sz;
762
763 u_int32_t io_throttle_data_length;
764 u_int32_t io_throttle_high;
765 u_int32_t io_throttle_low;
766 u_int16_t num_io_throttle_group;
767 u_int iot_enable;
768 struct mpi3mr_throttle_group_info *throttle_groups;
769
770 struct dma_memory_desc ioctl_sge[MPI3MR_NUM_IOCTL_SGE];
771 struct dma_memory_desc ioctl_chain_sge;
772 struct dma_memory_desc ioctl_resp_sge;
773 bool ioctl_sges_allocated;
774 struct proc *timestamp_thread_proc;
775 void *timestamp_chan;
776 u_int8_t timestamp_thread_active;
777 U32 ts_update_interval;
778 };
779
780 static __inline uint64_t
mpi3mr_regread64(struct mpi3mr_softc * sc,uint32_t offset)781 mpi3mr_regread64(struct mpi3mr_softc *sc, uint32_t offset)
782 {
783 return bus_space_read_8(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset);
784 }
785
786 static __inline void
mpi3mr_regwrite64(struct mpi3mr_softc * sc,uint32_t offset,uint64_t val)787 mpi3mr_regwrite64(struct mpi3mr_softc *sc, uint32_t offset, uint64_t val)
788 {
789 bus_space_write_8(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset, val);
790 }
791
792 static __inline uint32_t
mpi3mr_regread(struct mpi3mr_softc * sc,uint32_t offset)793 mpi3mr_regread(struct mpi3mr_softc *sc, uint32_t offset)
794 {
795 return bus_space_read_4(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset);
796 }
797
798 static __inline void
mpi3mr_regwrite(struct mpi3mr_softc * sc,uint32_t offset,uint32_t val)799 mpi3mr_regwrite(struct mpi3mr_softc *sc, uint32_t offset, uint32_t val)
800 {
801 bus_space_write_4(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset, val);
802 }
803
804 #define MPI3MR_INFO (1 << 0) /* Basic info */
805 #define MPI3MR_FAULT (1 << 1) /* Hardware faults */
806 #define MPI3MR_EVENT (1 << 2) /* Event data from the controller */
807 #define MPI3MR_LOG (1 << 3) /* Log data from the controller */
808 #define MPI3MR_RECOVERY (1 << 4) /* Command error recovery tracing */
809 #define MPI3MR_ERROR (1 << 5) /* Fatal driver/OS APIs failure */
810 #define MPI3MR_XINFO (1 << 6) /* Additional info logs*/
811 #define MPI3MR_TRACE (1 << 7) /* Trace functions */
812 #define MPI3MR_IOT (1 << 8) /* IO throttling related debugs */
813 #define MPI3MR_DEBUG_TM (1 << 9) /* Task management related debugs */
814 #define MPI3MR_DEBUG_IOCTL (1 << 10) /* IOCTL related debugs */
815
816 #define mpi3mr_printf(sc, args...) \
817 device_printf((sc)->mpi3mr_dev, ##args)
818
819 #define mpi3mr_print_field(sc, msg, args...) \
820 printf("\t" msg, ##args)
821
822 #define mpi3mr_vprintf(sc, args...) \
823 do { \
824 if (bootverbose) \
825 mpi3mr_printf(sc, ##args); \
826 } while (0)
827
828 #define mpi3mr_dprint(sc, level, msg, args...) \
829 do { \
830 if ((sc)->mpi3mr_debug & (level)) \
831 device_printf((sc)->mpi3mr_dev, msg, ##args); \
832 } while (0)
833
834 #define MPI3MR_PRINTFIELD_START(sc, tag...) \
835 mpi3mr_printf((sc), ##tag); \
836 mpi3mr_print_field((sc), ":\n")
837 #define MPI3MR_PRINTFIELD_END(sc, tag) \
838 mpi3mr_printf((sc), tag "\n")
839 #define MPI3MR_PRINTFIELD(sc, facts, attr, fmt) \
840 mpi3mr_print_field((sc), #attr ": " #fmt "\n", (facts)->attr)
841
842 #define mpi3mr_kproc_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg) \
843 kproc_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg)
844 #define mpi3mr_kproc_exit(arg) kproc_exit(arg)
845
846 #if defined(CAM_PRIORITY_XPT)
847 #define MPI3MR_PRIORITY_XPT CAM_PRIORITY_XPT
848 #else
849 #define MPI3MR_PRIORITY_XPT 5
850 #endif
851
852 static __inline void
mpi3mr_clear_bit(int b,volatile void * p)853 mpi3mr_clear_bit(int b, volatile void *p)
854 {
855 atomic_clear_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f));
856 }
857
858 static __inline void
mpi3mr_set_bit(int b,volatile void * p)859 mpi3mr_set_bit(int b, volatile void *p)
860 {
861 atomic_set_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f));
862 }
863
864 static __inline int
mpi3mr_test_bit(int b,volatile void * p)865 mpi3mr_test_bit(int b, volatile void *p)
866 {
867 return ((volatile int *)p)[b >> 5] & (1 << (b & 0x1f));
868 }
869
870 static __inline int
mpi3mr_test_and_set_bit(int b,volatile void * p)871 mpi3mr_test_and_set_bit(int b, volatile void *p)
872 {
873 int ret = ((volatile int *)p)[b >> 5] & (1 << (b & 0x1f));
874
875 atomic_set_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f));
876 return ret;
877 }
878
879 static __inline int
mpi3mr_find_first_zero_bit(void * p,int bit_count)880 mpi3mr_find_first_zero_bit(void *p, int bit_count)
881 {
882 int i, sz, j=0;
883 U8 *loc;
884
885 sz = bit_count % 8 ? (bit_count / 8 + 1) : (bit_count / 8);
886 loc = malloc(sz, M_MPI3MR, M_NOWAIT | M_ZERO);
887
888 memcpy(loc, p, sz);
889
890 for (i = 0; i < sz; i++) {
891 j = 0;
892 while (j < 8) {
893 if (!((loc[i] >> j) & 0x1))
894 goto out;
895 j++;
896 }
897 }
898 out:
899 free(loc, M_MPI3MR);
900 return (i + j);
901 }
902
903 #define MPI3MR_DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
904
905 void
906 init_completion(struct completion *completion);
907
908 void
909 complete(struct completion *completion);
910
911 void wait_for_completion_timeout(struct completion *completion,
912 U32 timeout);
913 void wait_for_completion_timeout_tm(struct completion *completion,
914 U32 timeout, struct mpi3mr_softc *sc);
915 void mpi3mr_add_sg_single(void *paddr, U8 flags, U32 length,
916 bus_addr_t dma_addr);
917 void mpi3mr_enable_interrupts(struct mpi3mr_softc *sc);
918 void mpi3mr_disable_interrupts(struct mpi3mr_softc *sc);
919 void mpi3mr_memaddr_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error);
920 int mpi3mr_submit_admin_cmd(struct mpi3mr_softc *mrioc, void *admin_req,
921 U16 admin_req_sz);
922 int mpi3mr_submit_io(struct mpi3mr_softc *mrioc,
923 struct mpi3mr_op_req_queue *op_req_q, U8 *req);
924 int
925 mpi3mr_alloc_interrupts(struct mpi3mr_softc *sc, U16 setup_one);
926
927 void mpi3mr_cleanup_ioc(struct mpi3mr_softc *sc);
928 int mpi3mr_initialize_ioc(struct mpi3mr_softc *sc, U8 reason);
929 void mpi3mr_build_zero_len_sge(void *paddr);
930 int mpi3mr_issue_event_notification(struct mpi3mr_softc *sc);
931 int
932 mpi3mr_register_events(struct mpi3mr_softc *sc);
933 void mpi3mr_process_op_reply_desc(struct mpi3mr_softc *sc,
934 Mpi3DefaultReplyDescriptor_t *reply_desc, U64 *reply_dma);
935 struct mpi3mr_cmd *
936 mpi3mr_get_command(struct mpi3mr_softc *sc);
937 void
938 mpi3mr_release_command(struct mpi3mr_cmd *cmd);
939 int
940 mpi3mr_complete_io_cmd(struct mpi3mr_softc *sc,
941 struct mpi3mr_irq_context *irq_context);
942 int
943 mpi3mr_cam_detach(struct mpi3mr_softc *sc);
944 int
945 mpi3mr_cam_attach(struct mpi3mr_softc *sc);
946 struct mpi3mr_target *
947 mpi3mr_find_target_by_per_id(struct mpi3mr_cam_softc *cam_sc,
948 uint16_t per_id);
949 struct mpi3mr_target *
950 mpi3mr_find_target_by_dev_handle(struct mpi3mr_cam_softc *cam_sc,
951 uint16_t dev_handle);
952 int mpi3mr_create_device(struct mpi3mr_softc *sc,
953 Mpi3DevicePage0_t *dev_pg0);
954 void
955 mpi3mr_unmap_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cmd);
956 void
957 init_completion(struct completion *completion);
958 void
959 complete(struct completion *completion);
960 void wait_for_completion_timeout(struct completion *completion,
961 U32 timeout);
962 void
963 poll_for_command_completion(struct mpi3mr_softc *sc,
964 struct mpi3mr_drvr_cmd *cmd, U16 wait);
965 int
966 mpi3mr_alloc_requests(struct mpi3mr_softc *sc);
967 void
968 mpi3mr_watchdog(void *arg);
969 int mpi3mr_issue_port_enable(struct mpi3mr_softc *mrioc, U8 async);
970 void
971 mpi3mr_isr(void *privdata);
972 int
973 mpi3mr_alloc_msix_queues(struct mpi3mr_softc *sc);
974 void
975 mpi3mr_destory_mtx(struct mpi3mr_softc *sc);
976 void
977 mpi3mr_free_mem(struct mpi3mr_softc *sc);
978 void
979 mpi3mr_cleanup_interrupts(struct mpi3mr_softc *sc);
980 int mpi3mr_setup_irqs(struct mpi3mr_softc *sc);
981 void mpi3mr_cleanup_event_taskq(struct mpi3mr_softc *sc);
982 void
983 mpi3mr_hexdump(void *buf, int sz, int format);
984 int mpi3mr_soft_reset_handler(struct mpi3mr_softc *sc,
985 U16 reset_reason, bool snapdump);
986 void
987 mpi3mrsas_release_simq_reinit(struct mpi3mr_cam_softc *cam_sc);
988 void
989 mpi3mr_watchdog_thread(void *arg);
990 void mpi3mr_timestamp_thread(void *arg);
991 void mpi3mr_add_device(struct mpi3mr_softc *sc, U16 per_id);
992 int mpi3mr_remove_device(struct mpi3mr_softc *sc, U16 handle);
993 int
994 mpi3mrsas_register_events(struct mpi3mr_softc *sc);
995 int mpi3mr_process_event_ack(struct mpi3mr_softc *sc, U8 event,
996 U32 event_ctx);
997 int mpi3mr_remove_device_from_os(struct mpi3mr_softc *sc, U16 handle);
998 void mpi3mr_remove_device_from_list(struct mpi3mr_softc *sc, struct mpi3mr_target *target,
999 bool must_delete);
1000 void mpi3mr_update_device(struct mpi3mr_softc *mrioc,
1001 struct mpi3mr_target *tgtdev, Mpi3DevicePage0_t *dev_pg0, bool is_added);
1002 void mpi3mr_app_save_logdata(struct mpi3mr_softc *sc, char *event_data, U16 event_data_size);
1003 void mpi3mr_set_io_divert_for_all_vd_in_tg(struct mpi3mr_softc *sc,
1004 struct mpi3mr_throttle_group_info *tg, U8 divert_value);
1005 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_softc *sc);
1006 void mpi3mr_poll_pend_io_completions(struct mpi3mr_softc *sc);
1007 void int_to_lun(unsigned int lun, U8 *req_lun);
1008 void trigger_reset_from_watchdog(struct mpi3mr_softc *sc, U8 reset_type, U16 reset_reason);
1009 void mpi3mr_alloc_ioctl_dma_memory(struct mpi3mr_softc *sc);
1010 int mpi3mr_cfg_get_driver_pg1(struct mpi3mr_softc *sc);
1011 #endif /*MPI3MR_H_INCLUDED*/
1012