1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2016 The MathWorks, Inc. All rights reserved. 14 * Copyright 2019 Joyent, Inc. 15 * Copyright 2019 Unix Software Ltd. 16 * Copyright 2021 Oxide Computer Company. 17 * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. 18 * Copyright 2022 Tintri by DDN, Inc. All rights reserved. 19 */ 20 21 #ifndef _NVME_VAR_H 22 #define _NVME_VAR_H 23 24 #include <sys/ddi.h> 25 #include <sys/sunddi.h> 26 #include <sys/blkdev.h> 27 #include <sys/taskq_impl.h> 28 #include <sys/list.h> 29 30 /* 31 * NVMe driver state 32 */ 33 34 #ifdef __cplusplus 35 extern "C" { 36 #endif 37 38 #define NVME_FMA_INIT 0x1 39 #define NVME_REGS_MAPPED 0x2 40 #define NVME_ADMIN_QUEUE 0x4 41 #define NVME_CTRL_LIMITS 0x8 42 #define NVME_INTERRUPTS 0x10 43 #define NVME_UFM_INIT 0x20 44 #define NVME_MUTEX_INIT 0x40 45 #define NVME_MGMT_INIT 0x80 46 47 #define NVME_MIN_ADMIN_QUEUE_LEN 16 48 #define NVME_MIN_IO_QUEUE_LEN 16 49 #define NVME_DEFAULT_ADMIN_QUEUE_LEN 256 50 #define NVME_DEFAULT_IO_QUEUE_LEN 1024 51 #define NVME_DEFAULT_ASYNC_EVENT_LIMIT 10 52 #define NVME_MIN_ASYNC_EVENT_LIMIT 1 53 #define NVME_DEFAULT_MIN_BLOCK_SIZE 512 54 55 56 typedef struct nvme nvme_t; 57 typedef struct nvme_namespace nvme_namespace_t; 58 typedef struct nvme_minor_state nvme_minor_state_t; 59 typedef struct nvme_dma nvme_dma_t; 60 typedef struct nvme_cmd nvme_cmd_t; 61 typedef struct nvme_cq nvme_cq_t; 62 typedef struct nvme_qpair nvme_qpair_t; 63 typedef struct nvme_task_arg nvme_task_arg_t; 64 65 struct nvme_minor_state { 66 kthread_t *nm_oexcl; 67 boolean_t nm_open; 68 }; 69 70 struct nvme_dma { 71 ddi_dma_handle_t nd_dmah; 72 ddi_acc_handle_t nd_acch; 73 ddi_dma_cookie_t nd_cookie; 74 uint_t nd_ncookie; 75 caddr_t nd_memp; 76 size_t nd_len; 77 boolean_t nd_cached; 78 }; 79 80 struct nvme_cmd { 81 struct list_node nc_list; 82 83 nvme_sqe_t nc_sqe; 84 nvme_cqe_t nc_cqe; 85 86 void (*nc_callback)(void *); 87 bd_xfer_t *nc_xfer; 88 boolean_t nc_completed; 89 boolean_t nc_dontpanic; 90 uint16_t nc_sqid; 91 92 nvme_dma_t *nc_dma; 93 nvme_dma_t *nc_prp; /* DMA for PRP lists */ 94 95 kmutex_t nc_mutex; 96 kcondvar_t nc_cv; 97 98 taskq_ent_t nc_tqent; 99 nvme_t *nc_nvme; 100 }; 101 102 struct nvme_cq { 103 size_t ncq_nentry; 104 uint16_t ncq_id; 105 106 nvme_dma_t *ncq_dma; 107 nvme_cqe_t *ncq_cq; 108 uint_t ncq_head; 109 uint_t ncq_tail; 110 uintptr_t ncq_hdbl; 111 int ncq_phase; 112 113 taskq_t *ncq_cmd_taskq; 114 115 kmutex_t ncq_mutex; 116 }; 117 118 struct nvme_qpair { 119 size_t nq_nentry; 120 121 /* submission fields */ 122 nvme_dma_t *nq_sqdma; 123 nvme_sqe_t *nq_sq; 124 uint_t nq_sqhead; 125 uint_t nq_sqtail; 126 uintptr_t nq_sqtdbl; 127 128 /* completion */ 129 nvme_cq_t *nq_cq; 130 131 /* shared structures for completion and submission */ 132 nvme_cmd_t **nq_cmd; /* active command array */ 133 uint16_t nq_next_cmd; /* next potential empty queue slot */ 134 uint_t nq_active_cmds; /* number of active cmds */ 135 136 kmutex_t nq_mutex; /* protects shared state */ 137 ksema_t nq_sema; /* semaphore to ensure q always has >= 1 empty slot */ 138 }; 139 140 struct nvme { 141 dev_info_t *n_dip; 142 int n_progress; 143 144 caddr_t n_regs; 145 ddi_acc_handle_t n_regh; 146 147 kmem_cache_t *n_cmd_cache; 148 kmem_cache_t *n_prp_cache; 149 150 size_t n_inth_sz; 151 ddi_intr_handle_t *n_inth; 152 int n_intr_cnt; 153 uint_t n_intr_pri; 154 int n_intr_cap; 155 int n_intr_type; 156 int n_intr_types; 157 158 char *n_product; 159 char *n_vendor; 160 161 nvme_version_t n_version; 162 boolean_t n_dead; 163 boolean_t n_strict_version; 164 boolean_t n_ignore_unknown_vendor_status; 165 uint32_t n_admin_queue_len; 166 uint32_t n_io_squeue_len; 167 uint32_t n_io_cqueue_len; 168 uint16_t n_async_event_limit; 169 uint_t n_min_block_size; 170 uint16_t n_abort_command_limit; 171 uint64_t n_max_data_transfer_size; 172 boolean_t n_write_cache_present; 173 boolean_t n_write_cache_enabled; 174 int n_error_log_len; 175 boolean_t n_lba_range_supported; 176 boolean_t n_auto_pst_supported; 177 boolean_t n_async_event_supported; 178 boolean_t n_progress_supported; 179 int n_submission_queues; 180 int n_completion_queues; 181 182 int n_nssr_supported; 183 int n_doorbell_stride; 184 int n_timeout; 185 int n_arbitration_mechanisms; 186 int n_cont_queues_reqd; 187 int n_max_queue_entries; 188 int n_pageshift; 189 int n_pagesize; 190 191 int n_namespace_count; 192 uint_t n_namespaces_attachable; 193 uint_t n_ioq_count; 194 uint_t n_cq_count; 195 196 nvme_identify_ctrl_t *n_idctl; 197 198 /* Pointer to the admin queue, which is always queue 0 in n_ioq. */ 199 nvme_qpair_t *n_adminq; 200 /* 201 * All command queues, including the admin queue. 202 * Its length is: n_ioq_count + 1. 203 */ 204 nvme_qpair_t **n_ioq; 205 nvme_cq_t **n_cq; 206 207 nvme_namespace_t *n_ns; 208 209 ddi_dma_attr_t n_queue_dma_attr; 210 ddi_dma_attr_t n_prp_dma_attr; 211 ddi_dma_attr_t n_sgl_dma_attr; 212 ddi_device_acc_attr_t n_reg_acc_attr; 213 ddi_iblock_cookie_t n_fm_ibc; 214 int n_fm_cap; 215 216 ksema_t n_abort_sema; 217 218 /* protects namespace management operations */ 219 kmutex_t n_mgmt_mutex; 220 221 /* protects minor node operations */ 222 kmutex_t n_minor_mutex; 223 224 /* state for devctl minor node */ 225 nvme_minor_state_t n_minor; 226 227 /* errors detected by driver */ 228 uint32_t n_dma_bind_err; 229 uint32_t n_abort_failed; 230 uint32_t n_cmd_timeout; 231 uint32_t n_cmd_aborted; 232 uint32_t n_wrong_logpage; 233 uint32_t n_unknown_logpage; 234 uint32_t n_too_many_cookies; 235 236 /* errors detected by hardware */ 237 uint32_t n_data_xfr_err; 238 uint32_t n_internal_err; 239 uint32_t n_abort_rq_err; 240 uint32_t n_abort_sq_del; 241 uint32_t n_nvm_cap_exc; 242 uint32_t n_nvm_ns_notrdy; 243 uint32_t n_nvm_ns_formatting; 244 uint32_t n_inv_cq_err; 245 uint32_t n_inv_qid_err; 246 uint32_t n_max_qsz_exc; 247 uint32_t n_inv_int_vect; 248 uint32_t n_inv_log_page; 249 uint32_t n_inv_format; 250 uint32_t n_inv_q_del; 251 uint32_t n_cnfl_attr; 252 uint32_t n_inv_prot; 253 uint32_t n_readonly; 254 255 /* errors reported by asynchronous events */ 256 uint32_t n_diagfail_event; 257 uint32_t n_persistent_event; 258 uint32_t n_transient_event; 259 uint32_t n_fw_load_event; 260 uint32_t n_reliability_event; 261 uint32_t n_temperature_event; 262 uint32_t n_spare_event; 263 uint32_t n_vendor_event; 264 uint32_t n_notice_event; 265 uint32_t n_unknown_event; 266 267 /* hot removal NDI event handling */ 268 ddi_eventcookie_t n_rm_cookie; 269 ddi_callback_id_t n_ev_rm_cb_id; 270 271 /* DDI UFM handle */ 272 ddi_ufm_handle_t *n_ufmh; 273 /* Cached Firmware Slot Information log page */ 274 nvme_fwslot_log_t *n_fwslot; 275 /* Lock protecting the cached firmware slot info */ 276 kmutex_t n_fwslot_mutex; 277 }; 278 279 struct nvme_namespace { 280 nvme_t *ns_nvme; 281 uint8_t ns_eui64[8]; 282 uint8_t ns_nguid[16]; 283 char ns_name[11]; 284 285 bd_handle_t ns_bd_hdl; 286 287 uint32_t ns_id; 288 size_t ns_block_count; 289 size_t ns_block_size; 290 size_t ns_best_block_size; 291 292 boolean_t ns_allocated; 293 boolean_t ns_active; 294 boolean_t ns_ignore; 295 boolean_t ns_attached; 296 297 nvme_identify_nsid_t *ns_idns; 298 299 /* state for attachment point minor node */ 300 nvme_minor_state_t ns_minor; 301 302 /* 303 * If a namespace has neither NGUID nor EUI64, we create a devid in 304 * nvme_prepare_devid(). 305 */ 306 char *ns_devid; 307 }; 308 309 struct nvme_task_arg { 310 nvme_t *nt_nvme; 311 nvme_cmd_t *nt_cmd; 312 }; 313 314 #ifdef __cplusplus 315 } 316 #endif 317 318 #endif /* _NVME_VAR_H */ 319