1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2016 The MathWorks, Inc. All rights reserved. 14 * Copyright 2019 Joyent, Inc. 15 * Copyright 2019 Unix Software Ltd. 16 * Copyright 2023 Oxide Computer Company. 17 * Copyright 2022 OmniOS Community Edition (OmniOSce) Association. 18 * Copyright 2022 Tintri by DDN, Inc. All rights reserved. 19 */ 20 21 #ifndef _NVME_VAR_H 22 #define _NVME_VAR_H 23 24 #include <sys/ddi.h> 25 #include <sys/sunddi.h> 26 #include <sys/blkdev.h> 27 #include <sys/taskq_impl.h> 28 #include <sys/list.h> 29 30 /* 31 * NVMe driver state 32 */ 33 34 #ifdef __cplusplus 35 extern "C" { 36 #endif 37 38 typedef enum { 39 NVME_PCI_CONFIG = 1 << 0, 40 NVME_FMA_INIT = 1 << 1, 41 NVME_REGS_MAPPED = 1 << 2, 42 NVME_ADMIN_QUEUE = 1 << 3, 43 NVME_CTRL_LIMITS = 1 << 4, 44 NVME_INTERRUPTS = 1 << 5, 45 NVME_UFM_INIT = 1 << 6, 46 NVME_MUTEX_INIT = 1 << 7, 47 NVME_MGMT_INIT = 1 << 8, 48 } nvme_progress_t; 49 50 typedef enum { 51 /* 52 * The controller fails to properly process commands on the admin queue 53 * if the first one has CID 0. Subsequent use of CID 0 doesn't present 54 * a problem. 55 */ 56 NVME_QUIRK_START_CID = 1 << 0, 57 } nvme_quirk_t; 58 59 #define NVME_MIN_ADMIN_QUEUE_LEN 16 60 #define NVME_MIN_IO_QUEUE_LEN 16 61 #define NVME_DEFAULT_ADMIN_QUEUE_LEN 256 62 #define NVME_DEFAULT_IO_QUEUE_LEN 1024 63 #define NVME_DEFAULT_ASYNC_EVENT_LIMIT 10 64 #define NVME_MIN_ASYNC_EVENT_LIMIT 1 65 #define NVME_DEFAULT_MIN_BLOCK_SIZE 512 66 67 68 typedef struct nvme nvme_t; 69 typedef struct nvme_namespace nvme_namespace_t; 70 typedef struct nvme_minor_state nvme_minor_state_t; 71 typedef struct nvme_dma nvme_dma_t; 72 typedef struct nvme_cmd nvme_cmd_t; 73 typedef struct nvme_cq nvme_cq_t; 74 typedef struct nvme_qpair nvme_qpair_t; 75 typedef struct nvme_task_arg nvme_task_arg_t; 76 77 struct nvme_minor_state { 78 kthread_t *nm_oexcl; 79 boolean_t nm_open; 80 }; 81 82 struct nvme_dma { 83 ddi_dma_handle_t nd_dmah; 84 ddi_acc_handle_t nd_acch; 85 ddi_dma_cookie_t nd_cookie; 86 uint_t nd_ncookie; 87 caddr_t nd_memp; 88 size_t nd_len; 89 boolean_t nd_cached; 90 }; 91 92 struct nvme_cmd { 93 struct list_node nc_list; 94 95 nvme_sqe_t nc_sqe; 96 nvme_cqe_t nc_cqe; 97 98 void (*nc_callback)(void *); 99 bd_xfer_t *nc_xfer; 100 boolean_t nc_completed; 101 boolean_t nc_dontpanic; 102 uint16_t nc_sqid; 103 104 nvme_dma_t *nc_dma; 105 nvme_dma_t *nc_prp; /* DMA for PRP lists */ 106 107 kmutex_t nc_mutex; 108 kcondvar_t nc_cv; 109 110 taskq_ent_t nc_tqent; 111 nvme_t *nc_nvme; 112 }; 113 114 struct nvme_cq { 115 size_t ncq_nentry; 116 uint16_t ncq_id; 117 118 nvme_dma_t *ncq_dma; 119 nvme_cqe_t *ncq_cq; 120 uint_t ncq_head; 121 uint_t ncq_tail; 122 uintptr_t ncq_hdbl; 123 int ncq_phase; 124 125 taskq_t *ncq_cmd_taskq; 126 127 kmutex_t ncq_mutex; 128 }; 129 130 struct nvme_qpair { 131 size_t nq_nentry; 132 133 /* submission fields */ 134 nvme_dma_t *nq_sqdma; 135 nvme_sqe_t *nq_sq; 136 uint_t nq_sqhead; 137 uint_t nq_sqtail; 138 uintptr_t nq_sqtdbl; 139 140 /* completion */ 141 nvme_cq_t *nq_cq; 142 143 /* shared structures for completion and submission */ 144 nvme_cmd_t **nq_cmd; /* active command array */ 145 uint16_t nq_next_cmd; /* next potential empty queue slot */ 146 uint_t nq_active_cmds; /* number of active cmds */ 147 148 kmutex_t nq_mutex; /* protects shared state */ 149 ksema_t nq_sema; /* semaphore to ensure q always has >= 1 empty slot */ 150 }; 151 152 struct nvme { 153 dev_info_t *n_dip; 154 nvme_progress_t n_progress; 155 nvme_quirk_t n_quirks; 156 157 caddr_t n_regs; 158 ddi_acc_handle_t n_regh; 159 160 kmem_cache_t *n_cmd_cache; 161 kmem_cache_t *n_prp_cache; 162 163 size_t n_inth_sz; 164 ddi_intr_handle_t *n_inth; 165 int n_intr_cnt; 166 uint_t n_intr_pri; 167 int n_intr_cap; 168 int n_intr_type; 169 int n_intr_types; 170 171 ddi_acc_handle_t n_pcicfg_handle; 172 uint16_t n_vendor_id; 173 uint16_t n_device_id; 174 uint16_t n_subsystem_vendor_id; 175 uint16_t n_subsystem_device_id; 176 uint8_t n_revision_id; 177 178 char *n_product; 179 char *n_vendor; 180 181 nvme_version_t n_version; 182 boolean_t n_dead; 183 boolean_t n_strict_version; 184 boolean_t n_ignore_unknown_vendor_status; 185 uint32_t n_admin_queue_len; 186 uint32_t n_io_squeue_len; 187 uint32_t n_io_cqueue_len; 188 uint16_t n_async_event_limit; 189 uint_t n_min_block_size; 190 uint16_t n_abort_command_limit; 191 uint64_t n_max_data_transfer_size; 192 boolean_t n_write_cache_present; 193 boolean_t n_write_cache_enabled; 194 int n_error_log_len; 195 boolean_t n_lba_range_supported; 196 boolean_t n_auto_pst_supported; 197 boolean_t n_async_event_supported; 198 boolean_t n_progress_supported; 199 int n_submission_queues; 200 int n_completion_queues; 201 202 int n_nssr_supported; 203 int n_doorbell_stride; 204 int n_timeout; 205 int n_arbitration_mechanisms; 206 int n_cont_queues_reqd; 207 int n_max_queue_entries; 208 int n_pageshift; 209 int n_pagesize; 210 211 int n_namespace_count; 212 uint_t n_namespaces_attachable; 213 uint_t n_ioq_count; 214 uint_t n_cq_count; 215 216 nvme_identify_ctrl_t *n_idctl; 217 218 /* Pointer to the admin queue, which is always queue 0 in n_ioq. */ 219 nvme_qpair_t *n_adminq; 220 /* 221 * All command queues, including the admin queue. 222 * Its length is: n_ioq_count + 1. 223 */ 224 nvme_qpair_t **n_ioq; 225 nvme_cq_t **n_cq; 226 227 nvme_namespace_t *n_ns; 228 229 ddi_dma_attr_t n_queue_dma_attr; 230 ddi_dma_attr_t n_prp_dma_attr; 231 ddi_dma_attr_t n_sgl_dma_attr; 232 ddi_device_acc_attr_t n_reg_acc_attr; 233 ddi_iblock_cookie_t n_fm_ibc; 234 int n_fm_cap; 235 236 ksema_t n_abort_sema; 237 238 /* protects namespace management operations */ 239 kmutex_t n_mgmt_mutex; 240 241 /* protects minor node operations */ 242 kmutex_t n_minor_mutex; 243 244 /* state for devctl minor node */ 245 nvme_minor_state_t n_minor; 246 247 /* errors detected by driver */ 248 uint32_t n_dma_bind_err; 249 uint32_t n_abort_failed; 250 uint32_t n_cmd_timeout; 251 uint32_t n_cmd_aborted; 252 uint32_t n_wrong_logpage; 253 uint32_t n_unknown_logpage; 254 uint32_t n_too_many_cookies; 255 uint32_t n_unknown_cid; 256 257 /* errors detected by hardware */ 258 uint32_t n_data_xfr_err; 259 uint32_t n_internal_err; 260 uint32_t n_abort_rq_err; 261 uint32_t n_abort_sq_del; 262 uint32_t n_nvm_cap_exc; 263 uint32_t n_nvm_ns_notrdy; 264 uint32_t n_nvm_ns_formatting; 265 uint32_t n_inv_cq_err; 266 uint32_t n_inv_qid_err; 267 uint32_t n_max_qsz_exc; 268 uint32_t n_inv_int_vect; 269 uint32_t n_inv_log_page; 270 uint32_t n_inv_format; 271 uint32_t n_inv_q_del; 272 uint32_t n_cnfl_attr; 273 uint32_t n_inv_prot; 274 uint32_t n_readonly; 275 276 /* errors reported by asynchronous events */ 277 uint32_t n_diagfail_event; 278 uint32_t n_persistent_event; 279 uint32_t n_transient_event; 280 uint32_t n_fw_load_event; 281 uint32_t n_reliability_event; 282 uint32_t n_temperature_event; 283 uint32_t n_spare_event; 284 uint32_t n_vendor_event; 285 uint32_t n_notice_event; 286 uint32_t n_unknown_event; 287 288 /* hot removal NDI event handling */ 289 ddi_eventcookie_t n_rm_cookie; 290 ddi_callback_id_t n_ev_rm_cb_id; 291 292 /* DDI UFM handle */ 293 ddi_ufm_handle_t *n_ufmh; 294 /* Cached Firmware Slot Information log page */ 295 nvme_fwslot_log_t *n_fwslot; 296 /* Lock protecting the cached firmware slot info */ 297 kmutex_t n_fwslot_mutex; 298 }; 299 300 struct nvme_namespace { 301 nvme_t *ns_nvme; 302 uint8_t ns_eui64[8]; 303 uint8_t ns_nguid[16]; 304 char ns_name[11]; 305 306 bd_handle_t ns_bd_hdl; 307 308 uint32_t ns_id; 309 size_t ns_block_count; 310 size_t ns_block_size; 311 size_t ns_best_block_size; 312 313 boolean_t ns_allocated; 314 boolean_t ns_active; 315 boolean_t ns_ignore; 316 boolean_t ns_attached; 317 318 nvme_identify_nsid_t *ns_idns; 319 320 /* state for attachment point minor node */ 321 nvme_minor_state_t ns_minor; 322 323 /* 324 * If a namespace has neither NGUID nor EUI64, we create a devid in 325 * nvme_prepare_devid(). 326 */ 327 char *ns_devid; 328 }; 329 330 struct nvme_task_arg { 331 nvme_t *nt_nvme; 332 nvme_cmd_t *nt_cmd; 333 }; 334 335 #ifdef __cplusplus 336 } 337 #endif 338 339 #endif /* _NVME_VAR_H */ 340