1 /*
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2020-2024, Broadcom Inc. All rights reserved.
5 * Support: <fbsd-storage-driver.pdl@broadcom.com>
6 *
7 * Authors: Sumit Saxena <sumit.saxena@broadcom.com>
8 * Chandrakanth Patil <chandrakanth.patil@broadcom.com>
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are
12 * met:
13 *
14 * 1. Redistributions of source code must retain the above copyright notice,
15 * this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright notice,
17 * this list of conditions and the following disclaimer in the documentation and/or other
18 * materials provided with the distribution.
19 * 3. Neither the name of the Broadcom Inc. nor the names of its contributors
20 * may be used to endorse or promote products derived from this software without
21 * specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
33 * POSSIBILITY OF SUCH DAMAGE.
34 *
35 * The views and conclusions contained in the software and documentation are
36 * those of the authors and should not be interpreted as representing
37 * official policies,either expressed or implied, of the FreeBSD Project.
38 *
39 * Mail to: Broadcom Inc 1320 Ridder Park Dr, San Jose, CA 95131
40 *
41 * Broadcom Inc. (Broadcom) MPI3MR Adapter FreeBSD
42 */
43
44 #ifndef _MPI3MRVAR_H
45 #define _MPI3MRVAR_H
46
47 #include <sys/types.h>
48 #include <sys/param.h>
49 #include <sys/systm.h>
50 #include <sys/kernel.h>
51 #include <sys/module.h>
52 #include <sys/bus.h>
53 #include <sys/conf.h>
54 #include <sys/malloc.h>
55 #include <sys/sysctl.h>
56 #include <sys/uio.h>
57 #include <sys/selinfo.h>
58 #include <sys/poll.h>
59
60 #include <sys/lock.h>
61 #include <sys/mutex.h>
62 #include <sys/endian.h>
63 #include <sys/sysent.h>
64 #include <sys/taskqueue.h>
65 #include <sys/smp.h>
66
67 #include <machine/bus.h>
68 #include <machine/resource.h>
69 #include <sys/rman.h>
70
71 #include <dev/pci/pcireg.h>
72 #include <dev/pci/pcivar.h>
73 #include <dev/pci/pci_private.h>
74
75 #include <cam/cam.h>
76 #include <cam/cam_ccb.h>
77 #include <cam/cam_debug.h>
78 #include <cam/cam_sim.h>
79 #include <cam/cam_xpt_sim.h>
80 #include <cam/cam_xpt_periph.h>
81 #include <cam/cam_periph.h>
82 #include <cam/scsi/scsi_all.h>
83 #include <cam/scsi/scsi_message.h>
84
85 #include <cam/scsi/smp_all.h>
86 #include <sys/queue.h>
87 #include <sys/kthread.h>
88 #include "mpi/mpi30_api.h"
89
90 #define MPI3MR_DRIVER_VERSION "8.10.0.1.0"
91 #define MPI3MR_DRIVER_RELDATE "19th Mar 2024"
92
93 #define MPI3MR_DRIVER_NAME "mpi3mr"
94
95 #define MPI3MR_NAME_LENGTH 32
96 #define IOCNAME "%s: "
97
98 #define SAS4116_CHIP_REV_A0 0
99 #define SAS4116_CHIP_REV_B0 1
100
101 #define MPI3MR_SG_DEPTH (MPI3MR_4K_PGSZ/sizeof(Mpi3SGESimple_t))
102 #define MPI3MR_MAX_SECTORS 2048
103 #define MPI3MR_MAX_CMDS_LUN 7
104 #define MPI3MR_MAX_CDB_LENGTH 16
105 #define MPI3MR_MAX_LUN 16895
106
107 #define MPI3MR_SATA_QDEPTH 32
108 #define MPI3MR_SAS_QDEPTH 64
109 #define MPI3MR_RAID_QDEPTH 128
110 #define MPI3MR_NVME_QDEPTH 128
111
112 #define MPI3MR_4K_PGSZ 4096
113 #define MPI3MR_AREQQ_SIZE (2 * MPI3MR_4K_PGSZ)
114 #define MPI3MR_AREPQ_SIZE (4 * MPI3MR_4K_PGSZ)
115 #define MPI3MR_AREQ_FRAME_SZ 128
116 #define MPI3MR_AREP_FRAME_SZ 16
117
118 #define MPI3MR_OPREQQ_SIZE (8 * MPI3MR_4K_PGSZ)
119 #define MPI3MR_OPREPQ_SIZE (4 * MPI3MR_4K_PGSZ)
120
121 /* Operational queue management definitions */
122 #define MPI3MR_OP_REQ_Q_QD 512
123 #define MPI3MR_OP_REP_Q_QD 1024
124 #define MPI3MR_OP_REP_Q_QD_A0 4096
125
126 #define MPI3MR_THRESHOLD_REPLY_COUNT 100
127
128 #define MPI3MR_CHAINSGE_SIZE MPI3MR_4K_PGSZ
129
130 #define MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST \
131 (MPI3_SGE_FLAGS_ELEMENT_TYPE_SIMPLE | MPI3_SGE_FLAGS_DLAS_SYSTEM | \
132 MPI3_SGE_FLAGS_END_OF_LIST)
133
134 #define MPI3MR_HOSTTAG_INVALID 0xFFFF
135 #define MPI3MR_HOSTTAG_INITCMDS 1
136 #define MPI3MR_HOSTTAG_IOCTLCMDS 2
137 #define MPI3MR_HOSTTAG_PELABORT 3
138 #define MPI3MR_HOSTTAG_PELWAIT 4
139 #define MPI3MR_HOSTTAG_TMS 5
140
141 #define MAX_MGMT_ADAPTERS 8
142 #define MPI3MR_WAIT_BEFORE_CTRL_RESET 5
143
144 #define MPI3MR_RESET_REASON_OSTYPE_FREEBSD 0x4
145 #define MPI3MR_RESET_REASON_OSTYPE_SHIFT 28
146 #define MPI3MR_RESET_REASON_IOCNUM_SHIFT 20
147
148 struct mpi3mr_mgmt_info {
149 uint16_t count;
150 struct mpi3mr_softc *sc_ptr[MAX_MGMT_ADAPTERS];
151 int max_index;
152 };
153
154 extern char fmt_os_ver[16];
155
156 #define MPI3MR_OS_VERSION(raw_os_ver, fmt_os_ver) sprintf(raw_os_ver, "%d", __FreeBSD_version); \
157 sprintf(fmt_os_ver, "%c%c.%c%c.%c%c%c",\
158 raw_os_ver[0], raw_os_ver[1], raw_os_ver[2],\
159 raw_os_ver[3], raw_os_ver[4], raw_os_ver[5],\
160 raw_os_ver[6]);
161 #define MPI3MR_NUM_DEVRMCMD 1
162 #define MPI3MR_HOSTTAG_DEVRMCMD_MIN (MPI3MR_HOSTTAG_TMS + 1)
163 #define MPI3MR_HOSTTAG_DEVRMCMD_MAX (MPI3MR_HOSTTAG_DEVRMCMD_MIN + \
164 MPI3MR_NUM_DEVRMCMD - 1)
165 #define MPI3MR_INTERNALCMDS_RESVD MPI3MR_HOSTTAG_DEVRMCMD_MAX
166
167 #define MPI3MR_NUM_EVTACKCMD 4
168 #define MPI3MR_HOSTTAG_EVTACKCMD_MIN (MPI3MR_HOSTTAG_DEVRMCMD_MAX + 1)
169 #define MPI3MR_HOSTTAG_EVTACKCMD_MAX (MPI3MR_HOSTTAG_EVTACKCMD_MIN + \
170 MPI3MR_NUM_EVTACKCMD - 1)
171
172 /* command/controller interaction timeout definitions in seconds */
173 #define MPI3MR_INTADMCMD_TIMEOUT 60
174 #define MPI3MR_PORTENABLE_TIMEOUT 300
175 #define MPI3MR_ABORTTM_TIMEOUT 60
176 #define MPI3MR_RESETTM_TIMEOUT 60
177 #define MPI3MR_TSUPDATE_INTERVAL 900
178 #define MPI3MR_DEFAULT_SHUTDOWN_TIME 120
179 #define MPI3MR_RAID_ERRREC_RESET_TIMEOUT 180
180 #define MPI3MR_RESET_HOST_IOWAIT_TIMEOUT 5
181 #define MPI3MR_PREPARE_FOR_RESET_TIMEOUT 180
182 #define MPI3MR_RESET_ACK_TIMEOUT 30
183 #define MPI3MR_MUR_TIMEOUT 120
184
185 #define MPI3MR_CMD_NOTUSED 0x8000
186 #define MPI3MR_CMD_COMPLETE 0x0001
187 #define MPI3MR_CMD_PENDING 0x0002
188 #define MPI3MR_CMD_REPLYVALID 0x0004
189 #define MPI3MR_CMD_RESET 0x0008
190
191 #define MPI3MR_NUM_EVTREPLIES 64
192 #define MPI3MR_SENSEBUF_SZ 256
193 #define MPI3MR_SENSEBUF_FACTOR 3
194 #define MPI3MR_CHAINBUF_FACTOR 3
195
196 #define MPT3SAS_HOSTPGSZ_4KEXP 12
197
198 #define MPI3MR_INVALID_DEV_HANDLE 0xFFFF
199
200 /* Controller Reset related definitions */
201 #define MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT 5
202 #define MPI3MR_MAX_SHUTDOWN_RETRY_COUNT 2
203
204 /* ResponseCode values */
205 #define MPI3MR_RI_MASK_RESPCODE (0x000000FF)
206 #define MPI3MR_RSP_TM_COMPLETE 0x00
207 #define MPI3MR_RSP_INVALID_FRAME 0x02
208 #define MPI3MR_RSP_TM_NOT_SUPPORTED 0x04
209 #define MPI3MR_RSP_TM_FAILED 0x05
210 #define MPI3MR_RSP_TM_SUCCEEDED 0x08
211 #define MPI3MR_RSP_TM_INVALID_LUN 0x09
212 #define MPI3MR_RSP_TM_OVERLAPPED_TAG 0x0A
213 #define MPI3MR_RSP_IO_QUEUED_ON_IOC \
214 MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC
215
216 /* Definitions for the controller security status*/
217 #define MPI3MR_CTLR_SECURITY_STATUS_MASK 0x0C
218 #define MPI3MR_CTLR_SECURE_DBG_STATUS_MASK 0x02
219
220 #define MPI3MR_INVALID_DEVICE 0x00
221 #define MPI3MR_CONFIG_SECURE_DEVICE 0x04
222 #define MPI3MR_HARD_SECURE_DEVICE 0x08
223 #define MPI3MR_TAMPERED_DEVICE 0x0C
224
225 #define MPI3MR_DEFAULT_MDTS (128 * 1024)
226 #define MPI3MR_DEFAULT_PGSZEXP (12)
227 #define MPI3MR_MAX_IOCTL_TRANSFER_SIZE (1024 * 1024)
228
229 #define MPI3MR_DEVRMHS_RETRYCOUNT 3
230 #define MPI3MR_PELCMDS_RETRYCOUNT 3
231
232 #define MPI3MR_PERIODIC_DELAY 1 /* 1 second heartbeat/watchdog check */
233
234 #define WRITE_SAME_32 0x0d
235
236 struct completion {
237 unsigned int done;
238 struct mtx lock;
239 };
240
241 typedef union {
242 volatile unsigned int val;
243 unsigned int val_rdonly;
244 } mpi3mr_atomic_t;
245
246 #define mpi3mr_atomic_read(v) atomic_load_acq_int(&(v)->val)
247 #define mpi3mr_atomic_set(v,i) atomic_store_rel_int(&(v)->val, i)
248 #define mpi3mr_atomic_dec(v) atomic_subtract_int(&(v)->val, 1)
249 #define mpi3mr_atomic_inc(v) atomic_add_int(&(v)->val, 1)
250 #define mpi3mr_atomic_add(v, u) atomic_add_int(&(v)->val, u)
251 #define mpi3mr_atomic_sub(v, u) atomic_subtract_int(&(v)->val, u)
252
253 /* IOCTL data transfer sge*/
254 #define MPI3MR_NUM_IOCTL_SGE 256
255 #define MPI3MR_IOCTL_SGE_SIZE (8 * 1024)
256
257 struct dma_memory_desc {
258 U32 size;
259 void *addr;
260 bus_dma_tag_t tag;
261 bus_dmamap_t dmamap;
262 bus_addr_t dma_addr;
263 };
264
265 enum mpi3mr_iocstate {
266 MRIOC_STATE_READY = 1,
267 MRIOC_STATE_RESET,
268 MRIOC_STATE_FAULT,
269 MRIOC_STATE_BECOMING_READY,
270 MRIOC_STATE_RESET_REQUESTED,
271 MRIOC_STATE_UNRECOVERABLE,
272 MRIOC_STATE_COUNT,
273 };
274
275 /* Init type definitions */
276 enum mpi3mr_init_type {
277 MPI3MR_INIT_TYPE_INIT = 0,
278 MPI3MR_INIT_TYPE_RESET,
279 MPI3MR_INIT_TYPE_RESUME,
280 };
281
282 /* Reset reason code definitions*/
283 enum mpi3mr_reset_reason {
284 MPI3MR_RESET_FROM_BRINGUP = 1,
285 MPI3MR_RESET_FROM_FAULT_WATCH = 2,
286 MPI3MR_RESET_FROM_IOCTL = 3,
287 MPI3MR_RESET_FROM_EH_HOS = 4,
288 MPI3MR_RESET_FROM_TM_TIMEOUT = 5,
289 MPI3MR_RESET_FROM_IOCTL_TIMEOUT = 6,
290 MPI3MR_RESET_FROM_MUR_FAILURE = 7,
291 MPI3MR_RESET_FROM_CTLR_CLEANUP = 8,
292 MPI3MR_RESET_FROM_CIACTIV_FAULT = 9,
293 MPI3MR_RESET_FROM_PE_TIMEOUT = 10,
294 MPI3MR_RESET_FROM_TSU_TIMEOUT = 11,
295 MPI3MR_RESET_FROM_DELREQQ_TIMEOUT = 12,
296 MPI3MR_RESET_FROM_DELREPQ_TIMEOUT = 13,
297 MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT = 14,
298 MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT = 15,
299 MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT = 16,
300 MPI3MR_RESET_FROM_IOCINIT_TIMEOUT = 17,
301 MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT = 18,
302 MPI3MR_RESET_FROM_EVTACK_TIMEOUT = 19,
303 MPI3MR_RESET_FROM_CIACTVRST_TIMER = 20,
304 MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT = 21,
305 MPI3MR_RESET_FROM_PELABORT_TIMEOUT = 22,
306 MPI3MR_RESET_FROM_SYSFS = 23,
307 MPI3MR_RESET_FROM_SYSFS_TIMEOUT = 24,
308 MPI3MR_RESET_FROM_DIAG_BUFFER_POST_TIMEOUT = 25,
309 MPI3MR_RESET_FROM_SCSIIO_TIMEOUT = 26,
310 MPI3MR_RESET_FROM_FIRMWARE = 27,
311 MPI3MR_DEFAULT_RESET_REASON = 28,
312 MPI3MR_RESET_REASON_COUNT,
313 };
314
315 struct mpi3mr_compimg_ver
316 {
317 U16 build_num;
318 U16 cust_id;
319 U8 ph_minor;
320 U8 ph_major;
321 U8 gen_minor;
322 U8 gen_major;
323 };
324
325 struct mpi3mr_ioc_facts
326 {
327 U32 ioc_capabilities;
328 struct mpi3mr_compimg_ver fw_ver;
329 U32 mpi_version;
330 U16 max_reqs;
331 U16 product_id;
332 U16 op_req_sz;
333 U16 reply_sz;
334 U16 exceptions;
335 U16 max_perids;
336 U16 max_pds;
337 U16 max_sasexpanders;
338 U16 max_sasinitiators;
339 U16 max_enclosures;
340 U16 max_pcieswitches;
341 U16 max_nvme;
342 U16 max_vds;
343 U16 max_hpds;
344 U16 max_advhpds;
345 U16 max_raidpds;
346 U16 min_devhandle;
347 U16 max_devhandle;
348 U16 max_op_req_q;
349 U16 max_op_reply_q;
350 U16 shutdown_timeout;
351 U8 ioc_num;
352 U8 who_init;
353 U16 max_msix_vectors;
354 U8 personality;
355 U8 dma_mask;
356 U8 protocol_flags;
357 U8 sge_mod_mask;
358 U8 sge_mod_value;
359 U8 sge_mod_shift;
360 U8 max_dev_per_tg;
361 U16 max_io_throttle_group;
362 U16 io_throttle_data_length;
363 U16 io_throttle_low;
364 U16 io_throttle_high;
365 };
366
367 struct mpi3mr_op_req_queue {
368 U16 ci;
369 U16 pi;
370 U16 num_reqs;
371 U8 qid;
372 U8 reply_qid;
373 U32 qsz;
374 void *q_base;
375 bus_dma_tag_t q_base_tag;
376 bus_dmamap_t q_base_dmamap;
377 bus_addr_t q_base_phys;
378 struct mtx q_lock;
379 };
380
381 struct mpi3mr_op_reply_queue {
382 U16 ci;
383 U8 ephase;
384 U8 qid;
385 U16 num_replies;
386 U32 qsz;
387 bus_dma_tag_t q_base_tag;
388 bus_dmamap_t q_base_dmamap;
389 void *q_base;
390 bus_addr_t q_base_phys;
391 mpi3mr_atomic_t pend_ios;
392 bool in_use;
393 struct mtx q_lock;
394 };
395
396 struct irq_info {
397 MPI3_REPLY_DESCRIPTORS_UNION *post_queue;
398 bus_dma_tag_t buffer_dmat;
399 struct resource *irq;
400 void *intrhand;
401 int irq_rid;
402 };
403
404 struct mpi3mr_irq_context {
405 struct mpi3mr_softc *sc;
406 U16 msix_index;
407 struct mpi3mr_op_reply_queue *op_reply_q;
408 char name[MPI3MR_NAME_LENGTH];
409 struct irq_info irq_info;
410 };
411
412 MALLOC_DECLARE(M_MPI3MR);
413 SYSCTL_DECL(_hw_mpi3mr);
414
415 typedef struct mpi3mr_drvr_cmd DRVR_CMD;
416 typedef void (*DRVR_CMD_CALLBACK)(struct mpi3mr_softc *mrioc, DRVR_CMD *drvrcmd);
417 struct mpi3mr_drvr_cmd {
418 struct mtx lock;
419 struct completion completion;
420 void *reply;
421 U8 *sensebuf;
422 U8 iou_rc;
423 U16 state;
424 U16 dev_handle;
425 U16 ioc_status;
426 U32 ioc_loginfo;
427 U8 is_waiting;
428 U8 is_senseprst;
429 U8 retry_count;
430 U16 host_tag;
431 DRVR_CMD_CALLBACK callback;
432 };
433
434 struct mpi3mr_cmd;
435 typedef void mpi3mr_evt_callback_t(struct mpi3mr_softc *, uintptr_t,
436 Mpi3EventNotificationReply_t *reply);
437 typedef void mpi3mr_cmd_callback_t(struct mpi3mr_softc *,
438 struct mpi3mr_cmd *cmd);
439
440 #define MPI3MR_IOVEC_COUNT 2
441
442 enum mpi3mr_data_xfer_direction {
443 MPI3MR_READ = 1,
444 MPI3MR_WRITE,
445 };
446
447 enum mpi3mr_cmd_state {
448 MPI3MR_CMD_STATE_FREE = 1,
449 MPI3MR_CMD_STATE_BUSY,
450 MPI3MR_CMD_STATE_IN_QUEUE,
451 MPI3MR_CMD_STATE_IN_TM,
452 };
453
454 enum mpi3mr_target_state {
455 MPI3MR_DEV_CREATED = 1,
456 MPI3MR_DEV_REMOVE_HS_COMPLETED = 2,
457 };
458
459 struct mpi3mr_cmd {
460 TAILQ_ENTRY(mpi3mr_cmd) next;
461 struct mpi3mr_softc *sc;
462 union ccb *ccb;
463 void *data;
464 u_int length;
465 struct mpi3mr_target *targ;
466 u_int data_dir;
467 u_int state;
468 bus_dmamap_t dmamap;
469 struct scsi_sense_data *sense;
470 struct callout callout;
471 bool callout_owner;
472 U16 hosttag;
473 U8 req_qidx;
474 Mpi3SCSIIORequest_t io_request;
475 };
476
477 struct mpi3mr_chain {
478 bus_dmamap_t buf_dmamap;
479 void *buf;
480 bus_addr_t buf_phys;
481 };
482
483 struct mpi3mr_event_handle {
484 TAILQ_ENTRY(mpi3mr_event_handle) eh_list;
485 mpi3mr_evt_callback_t *callback;
486 void *data;
487 uint8_t mask[16];
488 };
489
490 struct mpi3mr_fw_event_work {
491 U16 event;
492 void *event_data;
493 TAILQ_ENTRY(mpi3mr_fw_event_work) ev_link;
494 U8 send_ack;
495 U8 process_event;
496 U32 event_context;
497 U16 event_data_size;
498 };
499
500 /**
501 * struct delayed_dev_rmhs_node - Delayed device removal node
502 *
503 * @list: list head
504 * @handle: Device handle
505 * @iou_rc: IO Unit Control Reason Code
506 */
507 struct delayed_dev_rmhs_node {
508 TAILQ_ENTRY(delayed_dev_rmhs_node) list;
509 U16 handle;
510 U8 iou_rc;
511 };
512
513 /**
514 * struct delayed_evtack_node - Delayed event ack node
515 *
516 * @list: list head
517 * @event: MPI3 event ID
518 * @event_ctx: Event context
519 */
520 struct delayed_evtack_node {
521 TAILQ_ENTRY(delayed_evtack_node) list;
522 U8 event;
523 U32 event_ctx;
524 };
525
526 /* Reset types */
527 enum reset_type {
528 MPI3MR_NO_RESET,
529 MPI3MR_TRIGGER_SOFT_RESET,
530 };
531
532 struct mpi3mr_reset {
533 u_int type;
534 U32 reason;
535 int status;
536 bool ioctl_reset_snapdump;
537 };
538
539 struct mpi3mr_softc {
540 device_t mpi3mr_dev;
541 struct cdev *mpi3mr_cdev;
542 u_int mpi3mr_flags;
543 #define MPI3MR_FLAGS_SHUTDOWN (1 << 0)
544 #define MPI3MR_FLAGS_DIAGRESET (1 << 1)
545 #define MPI3MR_FLAGS_ATTACH_DONE (1 << 2)
546 #define MPI3MR_FLAGS_PORT_ENABLE_DONE (1 << 3)
547 U8 id;
548 int cpu_count;
549 char name[MPI3MR_NAME_LENGTH];
550 char driver_name[MPI3MR_NAME_LENGTH];
551 int bars;
552 bus_addr_t dma_loaddr;
553 u_int mpi3mr_debug;
554 struct mpi3mr_reset reset;
555 int max_msix_vectors;
556 int msix_count;
557 bool msix_enable;
558 int io_cmds_highwater;
559 int max_chains;
560 uint32_t chain_frame_size;
561 struct sysctl_ctx_list sysctl_ctx;
562 struct sysctl_oid *sysctl_tree;
563 char fw_version[32];
564 struct mpi3mr_chain *chains;
565 struct callout periodic;
566 struct callout device_check_callout;
567
568 struct mpi3mr_cam_softc *cam_sc;
569 struct mpi3mr_cmd **cmd_list;
570 TAILQ_HEAD(, mpi3mr_cmd) cmd_list_head;
571 struct mtx cmd_pool_lock;
572
573 struct resource *mpi3mr_regs_resource;
574 bus_space_handle_t mpi3mr_bhandle;
575 bus_space_tag_t mpi3mr_btag;
576 int mpi3mr_regs_rid;
577
578 bus_dma_tag_t mpi3mr_parent_dmat;
579 bus_dma_tag_t buffer_dmat;
580
581 int num_reqs;
582 int num_replies;
583 int num_chains;
584
585 TAILQ_HEAD(, mpi3mr_event_handle) event_list;
586 struct mpi3mr_event_handle *mpi3mr_log_eh;
587 struct intr_config_hook mpi3mr_ich;
588
589 struct mtx mpi3mr_mtx;
590 struct mtx io_lock;
591 U8 intr_enabled;
592 TAILQ_HEAD(, delayed_dev_rmhs_node) delayed_rmhs_list;
593 TAILQ_HEAD(, delayed_evtack_node) delayed_evtack_cmds_list;
594
595 U16 num_admin_reqs;
596 U32 admin_req_q_sz;
597 U16 admin_req_pi;
598 U16 admin_req_ci;
599 bus_dma_tag_t admin_req_tag;
600 bus_dmamap_t admin_req_dmamap;
601 bus_addr_t admin_req_phys;
602 U8 *admin_req;
603 struct mtx admin_req_lock;
604
605 U16 num_admin_replies;
606 U32 admin_reply_q_sz;
607 U16 admin_reply_ci;
608 U8 admin_reply_ephase;
609 bus_dma_tag_t admin_reply_tag;
610 bus_dmamap_t admin_reply_dmamap;
611 bus_addr_t admin_reply_phys;
612 U8 *admin_reply;
613 struct mtx admin_reply_lock;
614 bool admin_in_use;
615
616 U32 num_reply_bufs;
617 bus_dma_tag_t reply_buf_tag;
618 bus_dmamap_t reply_buf_dmamap;
619 bus_addr_t reply_buf_phys;
620 U8 *reply_buf;
621 bus_addr_t reply_buf_dma_max_address;
622 bus_addr_t reply_buf_dma_min_address;
623
624 U16 reply_free_q_sz;
625 bus_dma_tag_t reply_free_q_tag;
626 bus_dmamap_t reply_free_q_dmamap;
627 bus_addr_t reply_free_q_phys;
628 U64 *reply_free_q;
629 struct mtx reply_free_q_lock;
630 U32 reply_free_q_host_index;
631
632 U32 num_sense_bufs;
633 bus_dma_tag_t sense_buf_tag;
634 bus_dmamap_t sense_buf_dmamap;
635 bus_addr_t sense_buf_phys;
636 U8 *sense_buf;
637
638 U16 sense_buf_q_sz;
639 bus_dma_tag_t sense_buf_q_tag;
640 bus_dmamap_t sense_buf_q_dmamap;
641 bus_addr_t sense_buf_q_phys;
642 U64 *sense_buf_q;
643 struct mtx sense_buf_q_lock;
644 U32 sense_buf_q_host_index;
645
646 void *nvme_encap_prp_list;
647 bus_addr_t nvme_encap_prp_list_dma;
648 bus_dma_tag_t nvme_encap_prp_list_dmatag;
649 bus_dmamap_t nvme_encap_prp_list_dma_dmamap;
650 U32 nvme_encap_prp_sz;
651
652 U32 ready_timeout;
653
654 struct mpi3mr_irq_context *irq_ctx;
655
656 U16 num_queues; /* Number of request/reply queues */
657 struct mpi3mr_op_req_queue *op_req_q;
658 struct mpi3mr_op_reply_queue *op_reply_q;
659 U16 num_hosttag_op_req_q;
660
661 struct mpi3mr_drvr_cmd init_cmds;
662 struct mpi3mr_ioc_facts facts;
663 U16 reply_sz;
664 U16 op_reply_sz;
665
666 U32 event_masks[MPI3_EVENT_NOTIFY_EVENTMASK_WORDS];
667
668 char fwevt_worker_name[MPI3MR_NAME_LENGTH];
669 struct workqueue_struct *fwevt_worker_thread;
670 struct mtx fwevt_lock;
671 struct mtx target_lock;
672
673 U16 max_host_ios;
674 bus_dma_tag_t chain_sgl_list_tag;
675 struct mpi3mr_chain *chain_sgl_list;
676 U16 chain_bitmap_sz;
677 void *chain_bitmap;
678 struct mtx chain_buf_lock;
679 U16 chain_buf_count;
680
681 struct mpi3mr_drvr_cmd ioctl_cmds;
682 struct mpi3mr_drvr_cmd host_tm_cmds;
683 struct mpi3mr_drvr_cmd dev_rmhs_cmds[MPI3MR_NUM_DEVRMCMD];
684 struct mpi3mr_drvr_cmd evtack_cmds[MPI3MR_NUM_EVTACKCMD];
685
686 U16 devrem_bitmap_sz;
687 void *devrem_bitmap;
688
689 U16 dev_handle_bitmap_sz;
690 void *removepend_bitmap;
691
692 U16 evtack_cmds_bitmap_sz;
693 void *evtack_cmds_bitmap;
694
695 U32 ts_update_counter;
696 U8 reset_in_progress;
697 U8 unrecoverable;
698 U8 block_ioctls;
699 U8 in_prep_ciactv_rst;
700 U16 prep_ciactv_rst_counter;
701 struct mtx reset_mutex;
702
703 U8 prepare_for_reset;
704 U16 prepare_for_reset_timeout_counter;
705
706 U16 diagsave_timeout;
707 int logging_level;
708 U16 flush_io_count;
709
710 Mpi3DriverInfoLayout_t driver_info;
711
712 U16 change_count;
713
714 U8 *log_data_buffer;
715 U16 log_data_buffer_index;
716 U16 log_data_entry_size;
717
718 U8 pel_wait_pend;
719 U8 pel_abort_requested;
720 U8 pel_class;
721 U16 pel_locale;
722
723 struct mpi3mr_drvr_cmd pel_cmds;
724 struct mpi3mr_drvr_cmd pel_abort_cmd;
725 U32 newest_seqnum;
726 void *pel_seq_number;
727 bus_addr_t pel_seq_number_dma;
728 bus_dma_tag_t pel_seq_num_dmatag;
729 bus_dmamap_t pel_seq_num_dmamap;
730 U32 pel_seq_number_sz;
731
732 struct selinfo mpi3mr_select;
733 U32 mpi3mr_poll_waiting;
734 U32 mpi3mr_aen_triggered;
735
736 U16 wait_for_port_enable;
737 U16 track_mapping_events;
738 U16 pending_map_events;
739 mpi3mr_atomic_t fw_outstanding;
740 mpi3mr_atomic_t pend_ioctls;
741 struct proc *watchdog_thread;
742 void *watchdog_chan;
743 void *tm_chan;
744 u_int8_t remove_in_progress;
745 u_int8_t watchdog_thread_active;
746 u_int8_t do_timedout_reset;
747 bool allow_ios;
748 bool secure_ctrl;
749 mpi3mr_atomic_t pend_large_data_sz;
750
751 u_int32_t io_throttle_data_length;
752 u_int32_t io_throttle_high;
753 u_int32_t io_throttle_low;
754 u_int16_t num_io_throttle_group;
755 u_int iot_enable;
756 struct mpi3mr_throttle_group_info *throttle_groups;
757
758 struct dma_memory_desc ioctl_sge[MPI3MR_NUM_IOCTL_SGE];
759 struct dma_memory_desc ioctl_chain_sge;
760 struct dma_memory_desc ioctl_resp_sge;
761 bool ioctl_sges_allocated;
762 };
763
764 static __inline uint64_t
mpi3mr_regread64(struct mpi3mr_softc * sc,uint32_t offset)765 mpi3mr_regread64(struct mpi3mr_softc *sc, uint32_t offset)
766 {
767 return bus_space_read_8(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset);
768 }
769
770 static __inline void
mpi3mr_regwrite64(struct mpi3mr_softc * sc,uint32_t offset,uint64_t val)771 mpi3mr_regwrite64(struct mpi3mr_softc *sc, uint32_t offset, uint64_t val)
772 {
773 bus_space_write_8(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset, val);
774 }
775
776 static __inline uint32_t
mpi3mr_regread(struct mpi3mr_softc * sc,uint32_t offset)777 mpi3mr_regread(struct mpi3mr_softc *sc, uint32_t offset)
778 {
779 return bus_space_read_4(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset);
780 }
781
782 static __inline void
mpi3mr_regwrite(struct mpi3mr_softc * sc,uint32_t offset,uint32_t val)783 mpi3mr_regwrite(struct mpi3mr_softc *sc, uint32_t offset, uint32_t val)
784 {
785 bus_space_write_4(sc->mpi3mr_btag, sc->mpi3mr_bhandle, offset, val);
786 }
787
788 #define MPI3MR_INFO (1 << 0) /* Basic info */
789 #define MPI3MR_FAULT (1 << 1) /* Hardware faults */
790 #define MPI3MR_EVENT (1 << 2) /* Event data from the controller */
791 #define MPI3MR_LOG (1 << 3) /* Log data from the controller */
792 #define MPI3MR_RECOVERY (1 << 4) /* Command error recovery tracing */
793 #define MPI3MR_ERROR (1 << 5) /* Fatal driver/OS APIs failure */
794 #define MPI3MR_XINFO (1 << 6) /* Additional info logs*/
795 #define MPI3MR_TRACE (1 << 7) /* Trace functions */
796 #define MPI3MR_IOT (1 << 8) /* IO throttling related debugs */
797 #define MPI3MR_DEBUG_TM (1 << 9) /* Task management related debugs */
798 #define MPI3MR_DEBUG_IOCTL (1 << 10) /* IOCTL related debugs */
799
800 #define mpi3mr_printf(sc, args...) \
801 device_printf((sc)->mpi3mr_dev, ##args)
802
803 #define mpi3mr_print_field(sc, msg, args...) \
804 printf("\t" msg, ##args)
805
806 #define mpi3mr_vprintf(sc, args...) \
807 do { \
808 if (bootverbose) \
809 mpi3mr_printf(sc, ##args); \
810 } while (0)
811
812 #define mpi3mr_dprint(sc, level, msg, args...) \
813 do { \
814 if ((sc)->mpi3mr_debug & (level)) \
815 device_printf((sc)->mpi3mr_dev, msg, ##args); \
816 } while (0)
817
818 #define MPI3MR_PRINTFIELD_START(sc, tag...) \
819 mpi3mr_printf((sc), ##tag); \
820 mpi3mr_print_field((sc), ":\n")
821 #define MPI3MR_PRINTFIELD_END(sc, tag) \
822 mpi3mr_printf((sc), tag "\n")
823 #define MPI3MR_PRINTFIELD(sc, facts, attr, fmt) \
824 mpi3mr_print_field((sc), #attr ": " #fmt "\n", (facts)->attr)
825
826 #define mpi3mr_kproc_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg) \
827 kproc_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg)
828 #define mpi3mr_kproc_exit(arg) kproc_exit(arg)
829
830 #if defined(CAM_PRIORITY_XPT)
831 #define MPI3MR_PRIORITY_XPT CAM_PRIORITY_XPT
832 #else
833 #define MPI3MR_PRIORITY_XPT 5
834 #endif
835
836 static __inline void
mpi3mr_clear_bit(int b,volatile void * p)837 mpi3mr_clear_bit(int b, volatile void *p)
838 {
839 atomic_clear_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f));
840 }
841
842 static __inline void
mpi3mr_set_bit(int b,volatile void * p)843 mpi3mr_set_bit(int b, volatile void *p)
844 {
845 atomic_set_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f));
846 }
847
848 static __inline int
mpi3mr_test_bit(int b,volatile void * p)849 mpi3mr_test_bit(int b, volatile void *p)
850 {
851 return ((volatile int *)p)[b >> 5] & (1 << (b & 0x1f));
852 }
853
854 static __inline int
mpi3mr_test_and_set_bit(int b,volatile void * p)855 mpi3mr_test_and_set_bit(int b, volatile void *p)
856 {
857 int ret = ((volatile int *)p)[b >> 5] & (1 << (b & 0x1f));
858
859 atomic_set_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f));
860 return ret;
861 }
862
863 static __inline int
mpi3mr_find_first_zero_bit(void * p,int bit_count)864 mpi3mr_find_first_zero_bit(void *p, int bit_count)
865 {
866 int i, sz, j=0;
867 U8 *loc;
868
869 sz = bit_count % 8 ? (bit_count / 8 + 1) : (bit_count / 8);
870 loc = malloc(sz, M_MPI3MR, M_NOWAIT | M_ZERO);
871
872 memcpy(loc, p, sz);
873
874 for (i = 0; i < sz; i++) {
875 j = 0;
876 while (j < 8) {
877 if (!((loc[i] >> j) & 0x1))
878 goto out;
879 j++;
880 }
881 }
882 out:
883 free(loc, M_MPI3MR);
884 return (i + j);
885 }
886
887 #define MPI3MR_DIV_ROUND_UP(n,d) (((n) + (d) - 1) / (d))
888
889 void
890 init_completion(struct completion *completion);
891
892 void
893 complete(struct completion *completion);
894
895 void wait_for_completion_timeout(struct completion *completion,
896 U32 timeout);
897 void wait_for_completion_timeout_tm(struct completion *completion,
898 U32 timeout, struct mpi3mr_softc *sc);
899 void mpi3mr_add_sg_single(void *paddr, U8 flags, U32 length,
900 bus_addr_t dma_addr);
901 void mpi3mr_enable_interrupts(struct mpi3mr_softc *sc);
902 void mpi3mr_disable_interrupts(struct mpi3mr_softc *sc);
903 void mpi3mr_memaddr_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error);
904 int mpi3mr_submit_admin_cmd(struct mpi3mr_softc *mrioc, void *admin_req,
905 U16 admin_req_sz);
906 int mpi3mr_submit_io(struct mpi3mr_softc *mrioc,
907 struct mpi3mr_op_req_queue *op_req_q, U8 *req);
908 int
909 mpi3mr_alloc_interrupts(struct mpi3mr_softc *sc, U16 setup_one);
910
911 void mpi3mr_cleanup_ioc(struct mpi3mr_softc *sc);
912 int mpi3mr_initialize_ioc(struct mpi3mr_softc *sc, U8 reason);
913 void mpi3mr_build_zero_len_sge(void *paddr);
914 int mpi3mr_issue_event_notification(struct mpi3mr_softc *sc);
915 int
916 mpi3mr_register_events(struct mpi3mr_softc *sc);
917 void mpi3mr_process_op_reply_desc(struct mpi3mr_softc *sc,
918 Mpi3DefaultReplyDescriptor_t *reply_desc, U64 *reply_dma);
919 struct mpi3mr_cmd *
920 mpi3mr_get_command(struct mpi3mr_softc *sc);
921 void
922 mpi3mr_release_command(struct mpi3mr_cmd *cmd);
923 int
924 mpi3mr_complete_io_cmd(struct mpi3mr_softc *sc,
925 struct mpi3mr_irq_context *irq_context);
926 int
927 mpi3mr_cam_detach(struct mpi3mr_softc *sc);
928 int
929 mpi3mr_cam_attach(struct mpi3mr_softc *sc);
930 struct mpi3mr_target *
931 mpi3mr_find_target_by_per_id(struct mpi3mr_cam_softc *cam_sc,
932 uint16_t per_id);
933 struct mpi3mr_target *
934 mpi3mr_find_target_by_dev_handle(struct mpi3mr_cam_softc *cam_sc,
935 uint16_t dev_handle);
936 int mpi3mr_create_device(struct mpi3mr_softc *sc,
937 Mpi3DevicePage0_t *dev_pg0);
938 void
939 mpi3mr_unmap_request(struct mpi3mr_softc *sc, struct mpi3mr_cmd *cmd);
940 void
941 init_completion(struct completion *completion);
942 void
943 complete(struct completion *completion);
944 void wait_for_completion_timeout(struct completion *completion,
945 U32 timeout);
946 void
947 poll_for_command_completion(struct mpi3mr_softc *sc,
948 struct mpi3mr_drvr_cmd *cmd, U16 wait);
949 int
950 mpi3mr_alloc_requests(struct mpi3mr_softc *sc);
951 void
952 mpi3mr_watchdog(void *arg);
953 int mpi3mr_issue_port_enable(struct mpi3mr_softc *mrioc, U8 async);
954 void
955 mpi3mr_isr(void *privdata);
956 int
957 mpi3mr_alloc_msix_queues(struct mpi3mr_softc *sc);
958 void
959 mpi3mr_destory_mtx(struct mpi3mr_softc *sc);
960 void
961 mpi3mr_free_mem(struct mpi3mr_softc *sc);
962 void
963 mpi3mr_cleanup_interrupts(struct mpi3mr_softc *sc);
964 int mpi3mr_setup_irqs(struct mpi3mr_softc *sc);
965 void mpi3mr_cleanup_event_taskq(struct mpi3mr_softc *sc);
966 void
967 mpi3mr_hexdump(void *buf, int sz, int format);
968 int mpi3mr_soft_reset_handler(struct mpi3mr_softc *sc,
969 U16 reset_reason, bool snapdump);
970 void
971 mpi3mrsas_release_simq_reinit(struct mpi3mr_cam_softc *cam_sc);
972 void
973 mpi3mr_watchdog_thread(void *arg);
974 void mpi3mr_add_device(struct mpi3mr_softc *sc, U16 per_id);
975 int mpi3mr_remove_device(struct mpi3mr_softc *sc, U16 handle);
976 int
977 mpi3mrsas_register_events(struct mpi3mr_softc *sc);
978 int mpi3mr_process_event_ack(struct mpi3mr_softc *sc, U8 event,
979 U32 event_ctx);
980 int mpi3mr_remove_device_from_os(struct mpi3mr_softc *sc, U16 handle);
981 void mpi3mr_remove_device_from_list(struct mpi3mr_softc *sc, struct mpi3mr_target *target,
982 bool must_delete);
983 void mpi3mr_update_device(struct mpi3mr_softc *mrioc,
984 struct mpi3mr_target *tgtdev, Mpi3DevicePage0_t *dev_pg0, bool is_added);
985 void mpi3mr_app_save_logdata(struct mpi3mr_softc *sc, char *event_data, U16 event_data_size);
986 void mpi3mr_set_io_divert_for_all_vd_in_tg(struct mpi3mr_softc *sc,
987 struct mpi3mr_throttle_group_info *tg, U8 divert_value);
988 enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_softc *sc);
989 void mpi3mr_poll_pend_io_completions(struct mpi3mr_softc *sc);
990 void int_to_lun(unsigned int lun, U8 *req_lun);
991 void trigger_reset_from_watchdog(struct mpi3mr_softc *sc, U8 reset_type, U16 reset_reason);
992 void mpi3mr_alloc_ioctl_dma_memory(struct mpi3mr_softc *sc);
993 #endif /*MPI3MR_H_INCLUDED*/
994