xref: /freebsd/sys/dev/mpr/mprvar.h (revision 396c556d77189a5c474d35cec6f44a762e310b7d)
1 /*-
2  * Copyright (c) 2009 Yahoo! Inc.
3  * Copyright (c) 2011-2015 LSI Corp.
4  * Copyright (c) 2013-2016 Avago Technologies
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  * Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD
29  *
30  * $FreeBSD$
31  */
32 
33 #ifndef _MPRVAR_H
34 #define _MPRVAR_H
35 
36 #define MPR_DRIVER_VERSION	"15.03.00.00-fbsd"
37 
38 #define MPR_DB_MAX_WAIT		2500
39 
40 #define MPR_REQ_FRAMES		2048
41 #define MPR_PRI_REQ_FRAMES	128
42 #define MPR_EVT_REPLY_FRAMES	32
43 #define MPR_REPLY_FRAMES	MPR_REQ_FRAMES
44 #define MPR_CHAIN_FRAMES	2048
45 #define MPR_MAXIO_PAGES		(-1)
46 #define MPR_SENSE_LEN		SSD_FULL_SIZE
47 #define MPR_MSI_MAX		1
48 #define MPR_MSIX_MAX		96
49 #define MPR_SGE64_SIZE		12
50 #define MPR_SGE32_SIZE		8
51 #define MPR_SGC_SIZE		8
52 #define MPR_DEFAULT_CHAIN_SEG_SIZE	8
53 #define MPR_MAX_CHAIN_ELEMENT_SIZE	16
54 
55 /*
56  * PCIe NVMe Specific defines
57  */
58 //SLM-for now just use the same value as a SAS disk
59 #define NVME_QDEPTH			MPR_REQ_FRAMES
60 #define PRP_ENTRY_SIZE			8
61 #define NVME_CMD_PRP1_OFFSET		24	/* PRP1 offset in NVMe cmd */
62 #define NVME_CMD_PRP2_OFFSET		32	/* PRP2 offset in NVMe cmd */
63 #define NVME_ERROR_RESPONSE_SIZE	16	/* Max NVME Error Response */
64 #define HOST_PAGE_SIZE_4K		12
65 
66 #define MPR_FUNCTRACE(sc)			\
67 	mpr_dprint((sc), MPR_TRACE, "%s\n", __func__)
68 
69 #define	CAN_SLEEP			1
70 #define	NO_SLEEP			0
71 
72 #define MPR_PERIODIC_DELAY	1	/* 1 second heartbeat/watchdog check */
73 #define MPR_ATA_ID_TIMEOUT	5	/* 5 second timeout for SATA ID cmd */
74 #define MPR_MISSING_CHECK_DELAY	10	/* 10 seconds between missing check */
75 
76 #define	IFAULT_IOP_OVER_TEMP_THRESHOLD_EXCEEDED	0x2810
77 
78 #define MPR_SCSI_RI_INVALID_FRAME	(0x00000002)
79 
80 #define DEFAULT_SPINUP_WAIT	3	/* seconds to wait for spinup */
81 
82 #include <sys/endian.h>
83 
84 /*
85  * host mapping related macro definitions
86  */
87 #define MPR_MAPTABLE_BAD_IDX	0xFFFFFFFF
88 #define MPR_DPM_BAD_IDX		0xFFFF
89 #define MPR_ENCTABLE_BAD_IDX	0xFF
90 #define MPR_MAX_MISSING_COUNT	0x0F
91 #define MPR_DEV_RESERVED	0x20000000
92 #define MPR_MAP_IN_USE		0x10000000
93 #define MPR_MAP_BAD_ID		0xFFFFFFFF
94 
95 typedef uint8_t u8;
96 typedef uint16_t u16;
97 typedef uint32_t u32;
98 typedef uint64_t u64;
99 
100 /**
101  * struct dev_mapping_table - device mapping information
102  * @physical_id: SAS address for drives or WWID for RAID volumes
103  * @device_info: bitfield provides detailed info about the device
104  * @phy_bits: bitfields indicating controller phys
105  * @dpm_entry_num: index of this device in device persistent map table
106  * @dev_handle: device handle for the device pointed by this entry
107  * @id: target id
108  * @missing_count: number of times the device not detected by driver
109  * @hide_flag: Hide this physical disk/not (foreign configuration)
110  * @init_complete: Whether the start of the day checks completed or not
111  * @TLR_bits: Turn TLR support on or off
112  */
113 struct dev_mapping_table {
114 	u64	physical_id;
115 	u32	device_info;
116 	u32	phy_bits;
117 	u16	dpm_entry_num;
118 	u16	dev_handle;
119 	u16	reserved1;
120 	u16	id;
121 	u8	missing_count;
122 	u8	init_complete;
123 	u8	TLR_bits;
124 	u8	reserved2;
125 };
126 
127 /**
128  * struct enc_mapping_table -  mapping information about an enclosure
129  * @enclosure_id: Logical ID of this enclosure
130  * @start_index: index to the entry in dev_mapping_table
131  * @phy_bits: bitfields indicating controller phys
132  * @dpm_entry_num: index of this enclosure in device persistent map table
133  * @enc_handle: device handle for the enclosure pointed by this entry
134  * @num_slots: number of slots in the enclosure
135  * @start_slot: Starting slot id
136  * @missing_count: number of times the device not detected by driver
137  * @removal_flag: used to mark the device for removal
138  * @skip_search: used as a flag to include/exclude enclosure for search
139  * @init_complete: Whether the start of the day checks completed or not
140  */
141 struct enc_mapping_table {
142 	u64	enclosure_id;
143 	u32	start_index;
144 	u32	phy_bits;
145 	u16	dpm_entry_num;
146 	u16	enc_handle;
147 	u16	num_slots;
148 	u16	start_slot;
149 	u8	missing_count;
150 	u8	removal_flag;
151 	u8	skip_search;
152 	u8	init_complete;
153 };
154 
155 /**
156  * struct map_removal_table - entries to be removed from mapping table
157  * @dpm_entry_num: index of this device in device persistent map table
158  * @dev_handle: device handle for the device pointed by this entry
159  */
160 struct map_removal_table{
161 	u16	dpm_entry_num;
162 	u16	dev_handle;
163 };
164 
165 typedef struct mpr_fw_diagnostic_buffer {
166 	size_t		size;
167 	uint8_t		extended_type;
168 	uint8_t		buffer_type;
169 	uint8_t		force_release;
170 	uint32_t	product_specific[23];
171 	uint8_t		immediate;
172 	uint8_t		enabled;
173 	uint8_t		valid_data;
174 	uint8_t		owned_by_firmware;
175 	uint32_t	unique_id;
176 } mpr_fw_diagnostic_buffer_t;
177 
178 struct mpr_softc;
179 struct mpr_command;
180 struct mprsas_softc;
181 union ccb;
182 struct mprsas_target;
183 struct mpr_column_map;
184 
185 MALLOC_DECLARE(M_MPR);
186 
187 typedef void mpr_evt_callback_t(struct mpr_softc *, uintptr_t,
188     MPI2_EVENT_NOTIFICATION_REPLY *reply);
189 typedef void mpr_command_callback_t(struct mpr_softc *, struct mpr_command *cm);
190 
191 struct mpr_chain {
192 	TAILQ_ENTRY(mpr_chain)		chain_link;
193 	void				*chain;
194 	uint64_t			chain_busaddr;
195 };
196 
197 struct mpr_prp_page {
198 	TAILQ_ENTRY(mpr_prp_page)	prp_page_link;
199 	uint64_t			*prp_page;
200 	uint64_t			prp_page_busaddr;
201 };
202 
203 /*
204  * This needs to be at least 2 to support SMP passthrough.
205  */
206 #define       MPR_IOVEC_COUNT 2
207 
208 struct mpr_command {
209 	TAILQ_ENTRY(mpr_command)	cm_link;
210 	TAILQ_ENTRY(mpr_command)	cm_recovery;
211 	struct mpr_softc		*cm_sc;
212 	union ccb			*cm_ccb;
213 	void				*cm_data;
214 	u_int				cm_length;
215 	u_int				cm_out_len;
216 	struct uio			cm_uio;
217 	struct iovec			cm_iovec[MPR_IOVEC_COUNT];
218 	u_int				cm_max_segs;
219 	u_int				cm_sglsize;
220 	void				*cm_sge;
221 	uint8_t				*cm_req;
222 	uint8_t				*cm_reply;
223 	uint32_t			cm_reply_data;
224 	mpr_command_callback_t		*cm_complete;
225 	void				*cm_complete_data;
226 	struct mprsas_target		*cm_targ;
227 	MPI2_REQUEST_DESCRIPTOR_UNION	cm_desc;
228 	u_int	                	cm_lun;
229 	u_int				cm_flags;
230 #define MPR_CM_FLAGS_POLLED		(1 << 0)
231 #define MPR_CM_FLAGS_COMPLETE		(1 << 1)
232 #define MPR_CM_FLAGS_SGE_SIMPLE		(1 << 2)
233 #define MPR_CM_FLAGS_DATAOUT		(1 << 3)
234 #define MPR_CM_FLAGS_DATAIN		(1 << 4)
235 #define MPR_CM_FLAGS_WAKEUP		(1 << 5)
236 #define MPR_CM_FLAGS_USE_UIO		(1 << 6)
237 #define MPR_CM_FLAGS_SMP_PASS		(1 << 7)
238 #define	MPR_CM_FLAGS_CHAIN_FAILED	(1 << 8)
239 #define	MPR_CM_FLAGS_ERROR_MASK		MPR_CM_FLAGS_CHAIN_FAILED
240 #define	MPR_CM_FLAGS_USE_CCB		(1 << 9)
241 #define	MPR_CM_FLAGS_SATA_ID_TIMEOUT	(1 << 10)
242 	u_int				cm_state;
243 #define MPR_CM_STATE_FREE		0
244 #define MPR_CM_STATE_BUSY		1
245 #define MPR_CM_STATE_TIMEDOUT		2
246 	bus_dmamap_t			cm_dmamap;
247 	struct scsi_sense_data		*cm_sense;
248 	uint64_t			*nvme_error_response;
249 	TAILQ_HEAD(, mpr_chain)		cm_chain_list;
250  	TAILQ_HEAD(, mpr_prp_page)	cm_prp_page_list;
251 	uint32_t			cm_req_busaddr;
252 	bus_addr_t			cm_sense_busaddr;
253 	struct callout			cm_callout;
254 };
255 
256 struct mpr_column_map {
257 	uint16_t			dev_handle;
258 	uint8_t				phys_disk_num;
259 };
260 
261 struct mpr_event_handle {
262 	TAILQ_ENTRY(mpr_event_handle)	eh_list;
263 	mpr_evt_callback_t		*callback;
264 	void				*data;
265 	uint8_t				mask[16];
266 };
267 
268 struct mpr_queue {
269 	struct mpr_softc		*sc;
270 	int				qnum;
271 	MPI2_REPLY_DESCRIPTORS_UNION	*post_queue;
272 	int				replypostindex;
273 #ifdef notyet
274 	ck_ring_buffer_t		*ringmem;
275 	ck_ring_buffer_t		*chainmem;
276 	ck_ring_t			req_ring;
277 	ck_ring_t			chain_ring;
278 #endif
279 	bus_dma_tag_t			buffer_dmat;
280 	int				io_cmds_highwater;
281 	int				chain_free_lowwater;
282 	int				chain_alloc_fail;
283 	struct resource			*irq;
284 	void				*intrhand;
285 	int				irq_rid;
286 };
287 
288 struct mpr_softc {
289 	device_t			mpr_dev;
290 	struct cdev			*mpr_cdev;
291 	u_int				mpr_flags;
292 #define MPR_FLAGS_INTX		(1 << 0)
293 #define MPR_FLAGS_MSI		(1 << 1)
294 #define MPR_FLAGS_BUSY		(1 << 2)
295 #define MPR_FLAGS_SHUTDOWN	(1 << 3)
296 #define MPR_FLAGS_DIAGRESET	(1 << 4)
297 #define	MPR_FLAGS_ATTACH_DONE	(1 << 5)
298 #define	MPR_FLAGS_GEN35_IOC	(1 << 6)
299 #define	MPR_FLAGS_REALLOCATED	(1 << 7)
300 	u_int				mpr_debug;
301 	int				msi_msgs;
302 	u_int				atomic_desc_capable;
303 	int				tm_cmds_active;
304 	int				io_cmds_active;
305 	int				io_cmds_highwater;
306 	int				chain_free;
307 	int				max_chains;
308 	int				max_io_pages;
309 	u_int				maxio;
310 	int				chain_free_lowwater;
311 	uint32_t			chain_frame_size;
312 	uint16_t			chain_seg_size;
313 	int				prp_buffer_size;
314 	int				prp_pages_free;
315 	int				prp_pages_free_lowwater;
316 	u_int				enable_ssu;
317 	int				spinup_wait_time;
318 	int				use_phynum;
319 	uint64_t			chain_alloc_fail;
320 	uint64_t			prp_page_alloc_fail;
321 	struct sysctl_ctx_list		sysctl_ctx;
322 	struct sysctl_oid		*sysctl_tree;
323 	char                            fw_version[16];
324 	struct mpr_command		*commands;
325 	struct mpr_chain		*chains;
326 	struct mpr_prp_page		*prps;
327 	struct callout			periodic;
328 	struct callout			device_check_callout;
329 	struct mpr_queue		*queues;
330 
331 	struct mprsas_softc		*sassc;
332 	TAILQ_HEAD(, mpr_command)	req_list;
333 	TAILQ_HEAD(, mpr_command)	high_priority_req_list;
334 	TAILQ_HEAD(, mpr_chain)		chain_list;
335 	TAILQ_HEAD(, mpr_prp_page)	prp_page_list;
336 	TAILQ_HEAD(, mpr_command)	tm_list;
337 	int				replypostindex;
338 	int				replyfreeindex;
339 
340 	struct resource			*mpr_regs_resource;
341 	bus_space_handle_t		mpr_bhandle;
342 	bus_space_tag_t			mpr_btag;
343 	int				mpr_regs_rid;
344 
345 	bus_dma_tag_t			mpr_parent_dmat;
346 	bus_dma_tag_t			buffer_dmat;
347 
348 	MPI2_IOC_FACTS_REPLY		*facts;
349 	int				num_reqs;
350 	int				num_replies;
351 	int				fqdepth;	/* Free queue */
352 	int				pqdepth;	/* Post queue */
353 
354 	uint8_t				event_mask[16];
355 	TAILQ_HEAD(, mpr_event_handle)	event_list;
356 	struct mpr_event_handle		*mpr_log_eh;
357 
358 	struct mtx			mpr_mtx;
359 	struct intr_config_hook		mpr_ich;
360 
361 	uint8_t				*req_frames;
362 	bus_addr_t			req_busaddr;
363 	bus_dma_tag_t			req_dmat;
364 	bus_dmamap_t			req_map;
365 
366 	uint8_t				*reply_frames;
367 	bus_addr_t			reply_busaddr;
368 	bus_dma_tag_t			reply_dmat;
369 	bus_dmamap_t			reply_map;
370 
371 	struct scsi_sense_data		*sense_frames;
372 	bus_addr_t			sense_busaddr;
373 	bus_dma_tag_t			sense_dmat;
374 	bus_dmamap_t			sense_map;
375 
376 	uint8_t				*chain_frames;
377 	bus_addr_t			chain_busaddr;
378 	bus_dma_tag_t			chain_dmat;
379 	bus_dmamap_t			chain_map;
380 
381 	uint8_t				*prp_pages;
382 	bus_addr_t			prp_page_busaddr;
383 	bus_dma_tag_t			prp_page_dmat;
384 	bus_dmamap_t			prp_page_map;
385 
386 	MPI2_REPLY_DESCRIPTORS_UNION	*post_queue;
387 	bus_addr_t			post_busaddr;
388 	uint32_t			*free_queue;
389 	bus_addr_t			free_busaddr;
390 	bus_dma_tag_t			queues_dmat;
391 	bus_dmamap_t			queues_map;
392 
393 	uint8_t				*fw_diag_buffer;
394 	bus_addr_t			fw_diag_busaddr;
395 	bus_dma_tag_t			fw_diag_dmat;
396 	bus_dmamap_t			fw_diag_map;
397 
398 	uint8_t				ir_firmware;
399 
400 	/* static config pages */
401 	Mpi2IOCPage8_t			ioc_pg8;
402 	Mpi2IOUnitPage8_t		iounit_pg8;
403 
404 	/* host mapping support */
405 	struct dev_mapping_table	*mapping_table;
406 	struct enc_mapping_table	*enclosure_table;
407 	struct map_removal_table	*removal_table;
408 	uint8_t				*dpm_entry_used;
409 	uint8_t				*dpm_flush_entry;
410 	Mpi2DriverMappingPage0_t	*dpm_pg0;
411 	uint16_t			max_devices;
412 	uint16_t			max_enclosures;
413 	uint16_t			max_expanders;
414 	uint8_t				max_volumes;
415 	uint8_t				num_enc_table_entries;
416 	uint8_t				num_rsvd_entries;
417 	uint16_t			max_dpm_entries;
418 	uint8_t				is_dpm_enable;
419 	uint8_t				track_mapping_events;
420 	uint32_t			pending_map_events;
421 
422 	/* FW diag Buffer List */
423 	mpr_fw_diagnostic_buffer_t
424 				fw_diag_buffer_list[MPI2_DIAG_BUF_TYPE_COUNT];
425 
426 	/* Event Recording IOCTL support */
427 	uint32_t			events_to_record[4];
428 	mpr_event_entry_t		recorded_events[MPR_EVENT_QUEUE_SIZE];
429 	uint8_t				event_index;
430 	uint32_t			event_number;
431 
432 	/* EEDP and TLR support */
433 	uint8_t				eedp_enabled;
434 	uint8_t				control_TLR;
435 
436 	/* Shutdown Event Handler */
437 	eventhandler_tag		shutdown_eh;
438 
439 	/* To track topo events during reset */
440 #define	MPR_DIAG_RESET_TIMEOUT	300000
441 	uint8_t				wait_for_port_enable;
442 	uint8_t				port_enable_complete;
443 	uint8_t				msleep_fake_chan;
444 
445 	/* StartStopUnit command handling at shutdown */
446 	uint32_t			SSU_refcount;
447 	uint8_t				SSU_started;
448 
449 	/* Configuration tunables */
450 	u_int				disable_msix;
451 	u_int				disable_msi;
452 	u_int				max_msix;
453 	u_int				max_reqframes;
454 	u_int				max_prireqframes;
455 	u_int				max_replyframes;
456 	u_int				max_evtframes;
457 	char				exclude_ids[80];
458 
459 	struct timeval			lastfail;
460 };
461 
462 struct mpr_config_params {
463 	MPI2_CONFIG_EXT_PAGE_HEADER_UNION	hdr;
464 	u_int		action;
465 	u_int		page_address;	/* Attributes, not a phys address */
466 	u_int		status;
467 	void		*buffer;
468 	u_int		length;
469 	int		timeout;
470 	void		(*callback)(struct mpr_softc *, struct mpr_config_params *);
471 	void		*cbdata;
472 };
473 
474 struct scsi_read_capacity_eedp
475 {
476 	uint8_t addr[8];
477 	uint8_t length[4];
478 	uint8_t protect;
479 };
480 
481 static __inline uint32_t
482 mpr_regread(struct mpr_softc *sc, uint32_t offset)
483 {
484 	return (bus_space_read_4(sc->mpr_btag, sc->mpr_bhandle, offset));
485 }
486 
487 static __inline void
488 mpr_regwrite(struct mpr_softc *sc, uint32_t offset, uint32_t val)
489 {
490 	bus_space_write_4(sc->mpr_btag, sc->mpr_bhandle, offset, val);
491 }
492 
493 /* free_queue must have Little Endian address
494  * TODO- cm_reply_data is unwanted. We can remove it.
495  * */
496 static __inline void
497 mpr_free_reply(struct mpr_softc *sc, uint32_t busaddr)
498 {
499 	if (++sc->replyfreeindex >= sc->fqdepth)
500 		sc->replyfreeindex = 0;
501 	sc->free_queue[sc->replyfreeindex] = htole32(busaddr);
502 	mpr_regwrite(sc, MPI2_REPLY_FREE_HOST_INDEX_OFFSET, sc->replyfreeindex);
503 }
504 
505 static __inline struct mpr_chain *
506 mpr_alloc_chain(struct mpr_softc *sc)
507 {
508 	struct mpr_chain *chain;
509 
510 	if ((chain = TAILQ_FIRST(&sc->chain_list)) != NULL) {
511 		TAILQ_REMOVE(&sc->chain_list, chain, chain_link);
512 		sc->chain_free--;
513 		if (sc->chain_free < sc->chain_free_lowwater)
514 			sc->chain_free_lowwater = sc->chain_free;
515 	} else
516 		sc->chain_alloc_fail++;
517 	return (chain);
518 }
519 
520 static __inline void
521 mpr_free_chain(struct mpr_softc *sc, struct mpr_chain *chain)
522 {
523 #if 0
524 	bzero(chain->chain, 128);
525 #endif
526 	sc->chain_free++;
527 	TAILQ_INSERT_TAIL(&sc->chain_list, chain, chain_link);
528 }
529 
530 static __inline struct mpr_prp_page *
531 mpr_alloc_prp_page(struct mpr_softc *sc)
532 {
533 	struct mpr_prp_page *prp_page;
534 
535 	if ((prp_page = TAILQ_FIRST(&sc->prp_page_list)) != NULL) {
536 		TAILQ_REMOVE(&sc->prp_page_list, prp_page, prp_page_link);
537 		sc->prp_pages_free--;
538 		if (sc->prp_pages_free < sc->prp_pages_free_lowwater)
539 			sc->prp_pages_free_lowwater = sc->prp_pages_free;
540 	} else
541 		sc->prp_page_alloc_fail++;
542 	return (prp_page);
543 }
544 
545 static __inline void
546 mpr_free_prp_page(struct mpr_softc *sc, struct mpr_prp_page *prp_page)
547 {
548 	sc->prp_pages_free++;
549 	TAILQ_INSERT_TAIL(&sc->prp_page_list, prp_page, prp_page_link);
550 }
551 
552 static __inline void
553 mpr_free_command(struct mpr_softc *sc, struct mpr_command *cm)
554 {
555 	struct mpr_chain *chain, *chain_temp;
556 	struct mpr_prp_page *prp_page, *prp_page_temp;
557 
558 	if (cm->cm_reply != NULL)
559 		mpr_free_reply(sc, cm->cm_reply_data);
560 	cm->cm_reply = NULL;
561 	cm->cm_flags = 0;
562 	cm->cm_complete = NULL;
563 	cm->cm_complete_data = NULL;
564 	cm->cm_ccb = NULL;
565 	cm->cm_targ = NULL;
566 	cm->cm_max_segs = 0;
567 	cm->cm_lun = 0;
568 	cm->cm_state = MPR_CM_STATE_FREE;
569 	cm->cm_data = NULL;
570 	cm->cm_length = 0;
571 	cm->cm_out_len = 0;
572 	cm->cm_sglsize = 0;
573 	cm->cm_sge = NULL;
574 
575 	TAILQ_FOREACH_SAFE(chain, &cm->cm_chain_list, chain_link, chain_temp) {
576 		TAILQ_REMOVE(&cm->cm_chain_list, chain, chain_link);
577 		mpr_free_chain(sc, chain);
578 	}
579 	TAILQ_FOREACH_SAFE(prp_page, &cm->cm_prp_page_list, prp_page_link,
580 	    prp_page_temp) {
581 		TAILQ_REMOVE(&cm->cm_prp_page_list, prp_page, prp_page_link);
582 		mpr_free_prp_page(sc, prp_page);
583 	}
584 	TAILQ_INSERT_TAIL(&sc->req_list, cm, cm_link);
585 }
586 
587 static __inline struct mpr_command *
588 mpr_alloc_command(struct mpr_softc *sc)
589 {
590 	struct mpr_command *cm;
591 
592 	cm = TAILQ_FIRST(&sc->req_list);
593 	if (cm == NULL)
594 		return (NULL);
595 
596 	TAILQ_REMOVE(&sc->req_list, cm, cm_link);
597 	KASSERT(cm->cm_state == MPR_CM_STATE_FREE, ("mpr: Allocating busy "
598 	    "command\n"));
599 	cm->cm_state = MPR_CM_STATE_BUSY;
600 	return (cm);
601 }
602 
603 static __inline void
604 mpr_free_high_priority_command(struct mpr_softc *sc, struct mpr_command *cm)
605 {
606 	struct mpr_chain *chain, *chain_temp;
607 
608 	if (cm->cm_reply != NULL)
609 		mpr_free_reply(sc, cm->cm_reply_data);
610 	cm->cm_reply = NULL;
611 	cm->cm_flags = 0;
612 	cm->cm_complete = NULL;
613 	cm->cm_complete_data = NULL;
614 	cm->cm_ccb = NULL;
615 	cm->cm_targ = NULL;
616 	cm->cm_lun = 0;
617 	cm->cm_state = MPR_CM_STATE_FREE;
618 	TAILQ_FOREACH_SAFE(chain, &cm->cm_chain_list, chain_link, chain_temp) {
619 		TAILQ_REMOVE(&cm->cm_chain_list, chain, chain_link);
620 		mpr_free_chain(sc, chain);
621 	}
622 	TAILQ_INSERT_TAIL(&sc->high_priority_req_list, cm, cm_link);
623 }
624 
625 static __inline struct mpr_command *
626 mpr_alloc_high_priority_command(struct mpr_softc *sc)
627 {
628 	struct mpr_command *cm;
629 
630 	cm = TAILQ_FIRST(&sc->high_priority_req_list);
631 	if (cm == NULL)
632 		return (NULL);
633 
634 	TAILQ_REMOVE(&sc->high_priority_req_list, cm, cm_link);
635 	KASSERT(cm->cm_state == MPR_CM_STATE_FREE, ("mpr: Allocating busy "
636 	    "command\n"));
637 	cm->cm_state = MPR_CM_STATE_BUSY;
638 	return (cm);
639 }
640 
641 static __inline void
642 mpr_lock(struct mpr_softc *sc)
643 {
644 	mtx_lock(&sc->mpr_mtx);
645 }
646 
647 static __inline void
648 mpr_unlock(struct mpr_softc *sc)
649 {
650 	mtx_unlock(&sc->mpr_mtx);
651 }
652 
653 #define MPR_INFO	(1 << 0)	/* Basic info */
654 #define MPR_FAULT	(1 << 1)	/* Hardware faults */
655 #define MPR_EVENT	(1 << 2)	/* Event data from the controller */
656 #define MPR_LOG		(1 << 3)	/* Log data from the controller */
657 #define MPR_RECOVERY	(1 << 4)	/* Command error recovery tracing */
658 #define MPR_ERROR	(1 << 5)	/* Parameter errors, programming bugs */
659 #define MPR_INIT	(1 << 6)	/* Things related to system init */
660 #define MPR_XINFO	(1 << 7)	/* More detailed/noisy info */
661 #define MPR_USER	(1 << 8)	/* Trace user-generated commands */
662 #define MPR_MAPPING	(1 << 9)	/* Trace device mappings */
663 #define MPR_TRACE	(1 << 10)	/* Function-by-function trace */
664 
665 #define	MPR_SSU_DISABLE_SSD_DISABLE_HDD	0
666 #define	MPR_SSU_ENABLE_SSD_DISABLE_HDD	1
667 #define	MPR_SSU_DISABLE_SSD_ENABLE_HDD	2
668 #define	MPR_SSU_ENABLE_SSD_ENABLE_HDD	3
669 
670 #define mpr_printf(sc, args...)				\
671 	device_printf((sc)->mpr_dev, ##args)
672 
673 #define mpr_print_field(sc, msg, args...)		\
674 	printf("\t" msg, ##args)
675 
676 #define mpr_vprintf(sc, args...)			\
677 do {							\
678 	if (bootverbose)				\
679 		mpr_printf(sc, ##args);			\
680 } while (0)
681 
682 #define mpr_dprint(sc, level, msg, args...)		\
683 do {							\
684 	if ((sc)->mpr_debug & (level))			\
685 		device_printf((sc)->mpr_dev, msg, ##args);	\
686 } while (0)
687 
688 #define MPR_PRINTFIELD_START(sc, tag...)	\
689 	mpr_printf((sc), ##tag);		\
690 	mpr_print_field((sc), ":\n")
691 #define MPR_PRINTFIELD_END(sc, tag)		\
692 	mpr_printf((sc), tag "\n")
693 #define MPR_PRINTFIELD(sc, facts, attr, fmt)	\
694 	mpr_print_field((sc), #attr ": " #fmt "\n", (facts)->attr)
695 
696 static __inline void
697 mpr_from_u64(uint64_t data, U64 *mpr)
698 {
699 	(mpr)->High = htole32((uint32_t)((data) >> 32));
700 	(mpr)->Low = htole32((uint32_t)((data) & 0xffffffff));
701 }
702 
703 static __inline uint64_t
704 mpr_to_u64(U64 *data)
705 {
706 	return (((uint64_t)le32toh(data->High) << 32) | le32toh(data->Low));
707 }
708 
709 static __inline void
710 mpr_mask_intr(struct mpr_softc *sc)
711 {
712 	uint32_t mask;
713 
714 	mask = mpr_regread(sc, MPI2_HOST_INTERRUPT_MASK_OFFSET);
715 	mask |= MPI2_HIM_REPLY_INT_MASK;
716 	mpr_regwrite(sc, MPI2_HOST_INTERRUPT_MASK_OFFSET, mask);
717 }
718 
719 static __inline void
720 mpr_unmask_intr(struct mpr_softc *sc)
721 {
722 	uint32_t mask;
723 
724 	mask = mpr_regread(sc, MPI2_HOST_INTERRUPT_MASK_OFFSET);
725 	mask &= ~MPI2_HIM_REPLY_INT_MASK;
726 	mpr_regwrite(sc, MPI2_HOST_INTERRUPT_MASK_OFFSET, mask);
727 }
728 
729 int mpr_pci_setup_interrupts(struct mpr_softc *sc);
730 void mpr_pci_free_interrupts(struct mpr_softc *sc);
731 int mpr_pci_restore(struct mpr_softc *sc);
732 
733 void mpr_get_tunables(struct mpr_softc *sc);
734 int mpr_attach(struct mpr_softc *sc);
735 int mpr_free(struct mpr_softc *sc);
736 void mpr_intr(void *);
737 void mpr_intr_msi(void *);
738 void mpr_intr_locked(void *);
739 int mpr_register_events(struct mpr_softc *, uint8_t *, mpr_evt_callback_t *,
740     void *, struct mpr_event_handle **);
741 int mpr_restart(struct mpr_softc *);
742 int mpr_update_events(struct mpr_softc *, struct mpr_event_handle *, uint8_t *);
743 int mpr_deregister_events(struct mpr_softc *, struct mpr_event_handle *);
744 void mpr_build_nvme_prp(struct mpr_softc *sc, struct mpr_command *cm,
745     Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request, void *data,
746     uint32_t data_in_sz, uint32_t data_out_sz);
747 int mpr_push_sge(struct mpr_command *, MPI2_SGE_SIMPLE64 *, size_t, int);
748 int mpr_push_ieee_sge(struct mpr_command *, void *, int);
749 int mpr_add_dmaseg(struct mpr_command *, vm_paddr_t, size_t, u_int, int);
750 int mpr_attach_sas(struct mpr_softc *sc);
751 int mpr_detach_sas(struct mpr_softc *sc);
752 int mpr_read_config_page(struct mpr_softc *, struct mpr_config_params *);
753 int mpr_write_config_page(struct mpr_softc *, struct mpr_config_params *);
754 void mpr_memaddr_cb(void *, bus_dma_segment_t *, int , int );
755 void mpr_init_sge(struct mpr_command *cm, void *req, void *sge);
756 int mpr_attach_user(struct mpr_softc *);
757 void mpr_detach_user(struct mpr_softc *);
758 void mprsas_record_event(struct mpr_softc *sc,
759     MPI2_EVENT_NOTIFICATION_REPLY *event_reply);
760 
761 int mpr_map_command(struct mpr_softc *sc, struct mpr_command *cm);
762 int mpr_wait_command(struct mpr_softc *sc, struct mpr_command **cm, int timeout,
763     int sleep_flag);
764 int mpr_request_polled(struct mpr_softc *sc, struct mpr_command **cm);
765 
766 int mpr_config_get_bios_pg3(struct mpr_softc *sc, Mpi2ConfigReply_t
767     *mpi_reply, Mpi2BiosPage3_t *config_page);
768 int mpr_config_get_raid_volume_pg0(struct mpr_softc *sc, Mpi2ConfigReply_t
769     *mpi_reply, Mpi2RaidVolPage0_t *config_page, u32 page_address);
770 int mpr_config_get_ioc_pg8(struct mpr_softc *sc, Mpi2ConfigReply_t *,
771     Mpi2IOCPage8_t *);
772 int mpr_config_get_iounit_pg8(struct mpr_softc *sc,
773     Mpi2ConfigReply_t *mpi_reply, Mpi2IOUnitPage8_t *config_page);
774 int mpr_config_get_sas_device_pg0(struct mpr_softc *, Mpi2ConfigReply_t *,
775     Mpi2SasDevicePage0_t *, u32 , u16 );
776 int mpr_config_get_pcie_device_pg0(struct mpr_softc *sc, Mpi2ConfigReply_t
777     *mpi_reply, Mpi26PCIeDevicePage0_t *config_page, u32 form, u16 handle);
778 int mpr_config_get_pcie_device_pg2(struct mpr_softc *sc, Mpi2ConfigReply_t
779     *mpi_reply, Mpi26PCIeDevicePage2_t *config_page, u32 form, u16 handle);
780 int mpr_config_get_dpm_pg0(struct mpr_softc *, Mpi2ConfigReply_t *,
781     Mpi2DriverMappingPage0_t *, u16 );
782 int mpr_config_get_raid_volume_pg1(struct mpr_softc *sc,
783     Mpi2ConfigReply_t *mpi_reply, Mpi2RaidVolPage1_t *config_page, u32 form,
784     u16 handle);
785 int mpr_config_get_volume_wwid(struct mpr_softc *sc, u16 volume_handle,
786     u64 *wwid);
787 int mpr_config_get_raid_pd_pg0(struct mpr_softc *sc,
788     Mpi2ConfigReply_t *mpi_reply, Mpi2RaidPhysDiskPage0_t *config_page,
789     u32 page_address);
790 void mprsas_ir_shutdown(struct mpr_softc *sc);
791 
792 int mpr_reinit(struct mpr_softc *sc);
793 void mprsas_handle_reinit(struct mpr_softc *sc);
794 
795 void mpr_base_static_config_pages(struct mpr_softc *sc);
796 
797 int mpr_mapping_initialize(struct mpr_softc *);
798 void mpr_mapping_topology_change_event(struct mpr_softc *,
799     Mpi2EventDataSasTopologyChangeList_t *);
800 void mpr_mapping_pcie_topology_change_event(struct mpr_softc *sc,
801     Mpi26EventDataPCIeTopologyChangeList_t *event_data);
802 void mpr_mapping_free_memory(struct mpr_softc *sc);
803 int mpr_config_set_dpm_pg0(struct mpr_softc *, Mpi2ConfigReply_t *,
804     Mpi2DriverMappingPage0_t *, u16 );
805 void mpr_mapping_exit(struct mpr_softc *);
806 void mpr_mapping_check_devices(void *);
807 int mpr_mapping_allocate_memory(struct mpr_softc *sc);
808 unsigned int mpr_mapping_get_tid(struct mpr_softc *, uint64_t , u16);
809 unsigned int mpr_mapping_get_tid_from_handle(struct mpr_softc *sc,
810     u16 handle);
811 unsigned int mpr_mapping_get_raid_tid(struct mpr_softc *sc, u64 wwid,
812     u16 volHandle);
813 unsigned int mpr_mapping_get_raid_tid_from_handle(struct mpr_softc *sc,
814     u16 volHandle);
815 void mpr_mapping_enclosure_dev_status_change_event(struct mpr_softc *,
816     Mpi2EventDataSasEnclDevStatusChange_t *event_data);
817 void mpr_mapping_ir_config_change_event(struct mpr_softc *sc,
818     Mpi2EventDataIrConfigChangeList_t *event_data);
819 
820 void mprsas_evt_handler(struct mpr_softc *sc, uintptr_t data,
821     MPI2_EVENT_NOTIFICATION_REPLY *event);
822 void mprsas_prepare_remove(struct mprsas_softc *sassc, uint16_t handle);
823 void mprsas_prepare_volume_remove(struct mprsas_softc *sassc, uint16_t handle);
824 int mprsas_startup(struct mpr_softc *sc);
825 struct mprsas_target * mprsas_find_target_by_handle(struct mprsas_softc *, int,
826     uint16_t);
827 void mprsas_realloc_targets(struct mpr_softc *sc, int maxtargets);
828 struct mpr_command * mprsas_alloc_tm(struct mpr_softc *sc);
829 void mprsas_free_tm(struct mpr_softc *sc, struct mpr_command *tm);
830 void mprsas_release_simq_reinit(struct mprsas_softc *sassc);
831 int mprsas_send_reset(struct mpr_softc *sc, struct mpr_command *tm,
832     uint8_t type);
833 
834 SYSCTL_DECL(_hw_mpr);
835 
836 /* Compatibility shims for different OS versions */
837 #if __FreeBSD_version >= 800001
838 #define mpr_kproc_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg) \
839     kproc_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg)
840 #define mpr_kproc_exit(arg)	kproc_exit(arg)
841 #else
842 #define mpr_kproc_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg) \
843     kthread_create(func, farg, proc_ptr, flags, stackpgs, fmtstr, arg)
844 #define mpr_kproc_exit(arg)	kthread_exit(arg)
845 #endif
846 
847 #if defined(CAM_PRIORITY_XPT)
848 #define MPR_PRIORITY_XPT	CAM_PRIORITY_XPT
849 #else
850 #define MPR_PRIORITY_XPT	5
851 #endif
852 
853 #if __FreeBSD_version < 800107
854 // Prior to FreeBSD-8.0 scp3_flags was not defined.
855 #define spc3_flags reserved
856 
857 #define SPC3_SID_PROTECT    0x01
858 #define SPC3_SID_3PC        0x08
859 #define SPC3_SID_TPGS_MASK  0x30
860 #define SPC3_SID_TPGS_IMPLICIT  0x10
861 #define SPC3_SID_TPGS_EXPLICIT  0x20
862 #define SPC3_SID_ACC        0x40
863 #define SPC3_SID_SCCS       0x80
864 
865 #define CAM_PRIORITY_NORMAL CAM_PRIORITY_NONE
866 #endif
867 
868 /* Definitions for SCSI unmap translation to NVMe DSM command */
869 
870 /* UNMAP block descriptor structure */
871 struct unmap_blk_desc {
872 	uint64_t slba;
873 	uint32_t nlb;
874 	uint32_t resv;
875 };
876 
877 /* UNMAP command's data */
878 struct unmap_parm_list {
879 	uint16_t unmap_data_len;
880 	uint16_t unmap_blk_desc_data_len;
881 	uint32_t resv;
882 	struct unmap_blk_desc desc[0];
883 };
884 
885 /* SCSI ADDITIONAL SENSE Codes */
886 #define FIXED_SENSE_DATA                                0x70
887 #define SCSI_ASC_NO_SENSE                               0x00
888 #define SCSI_ASC_PERIPHERAL_DEV_WRITE_FAULT             0x03
889 #define SCSI_ASC_LUN_NOT_READY                          0x04
890 #define SCSI_ASC_WARNING                                0x0B
891 #define SCSI_ASC_LOG_BLOCK_GUARD_CHECK_FAILED           0x10
892 #define SCSI_ASC_LOG_BLOCK_APPTAG_CHECK_FAILED          0x10
893 #define SCSI_ASC_LOG_BLOCK_REFTAG_CHECK_FAILED          0x10
894 #define SCSI_ASC_UNRECOVERED_READ_ERROR                 0x11
895 #define SCSI_ASC_MISCOMPARE_DURING_VERIFY               0x1D
896 #define SCSI_ASC_ACCESS_DENIED_INVALID_LUN_ID           0x20
897 #define SCSI_ASC_ILLEGAL_COMMAND                        0x20
898 #define SCSI_ASC_ILLEGAL_BLOCK                          0x21
899 #define SCSI_ASC_INVALID_CDB                            0x24
900 #define SCSI_ASC_INVALID_LUN                            0x25
901 #define SCSI_ASC_INVALID_PARAMETER                      0x26
902 #define SCSI_ASC_FORMAT_COMMAND_FAILED                  0x31
903 #define SCSI_ASC_INTERNAL_TARGET_FAILURE                0x44
904 
905 /* SCSI ADDITIONAL SENSE Code Qualifiers */
906 #define SCSI_ASCQ_CAUSE_NOT_REPORTABLE                  0x00
907 #define SCSI_ASCQ_FORMAT_COMMAND_FAILED                 0x01
908 #define SCSI_ASCQ_LOG_BLOCK_GUARD_CHECK_FAILED          0x01
909 #define SCSI_ASCQ_LOG_BLOCK_APPTAG_CHECK_FAILED         0x02
910 #define SCSI_ASCQ_LOG_BLOCK_REFTAG_CHECK_FAILED         0x03
911 #define SCSI_ASCQ_FORMAT_IN_PROGRESS                    0x04
912 #define SCSI_ASCQ_POWER_LOSS_EXPECTED                   0x08
913 #define SCSI_ASCQ_INVALID_LUN_ID                        0x09
914 
915 #endif
916 
917