1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2018 Nexenta Systems, Inc. 14 * Copyright 2016 The MathWorks, Inc. All rights reserved. 15 * Copyright 2017 Joyent, Inc. 16 * Copyright 2019 Western Digital Corporation. 17 */ 18 19 #ifndef _NVME_VAR_H 20 #define _NVME_VAR_H 21 22 #include <sys/ddi.h> 23 #include <sys/sunddi.h> 24 #include <sys/blkdev.h> 25 #include <sys/taskq_impl.h> 26 #include <sys/list.h> 27 28 /* 29 * NVMe driver state 30 */ 31 32 #ifdef __cplusplus 33 extern "C" { 34 #endif 35 36 #define NVME_FMA_INIT 0x1 37 #define NVME_REGS_MAPPED 0x2 38 #define NVME_ADMIN_QUEUE 0x4 39 #define NVME_CTRL_LIMITS 0x8 40 #define NVME_INTERRUPTS 0x10 41 42 #define NVME_MIN_ADMIN_QUEUE_LEN 16 43 #define NVME_MIN_IO_QUEUE_LEN 16 44 #define NVME_DEFAULT_ADMIN_QUEUE_LEN 256 45 #define NVME_DEFAULT_IO_QUEUE_LEN 1024 46 #define NVME_DEFAULT_ASYNC_EVENT_LIMIT 10 47 #define NVME_MIN_ASYNC_EVENT_LIMIT 1 48 #define NVME_DEFAULT_MIN_BLOCK_SIZE 512 49 50 51 typedef struct nvme nvme_t; 52 typedef struct nvme_namespace nvme_namespace_t; 53 typedef struct nvme_minor_state nvme_minor_state_t; 54 typedef struct nvme_dma nvme_dma_t; 55 typedef struct nvme_cmd nvme_cmd_t; 56 typedef struct nvme_cq nvme_cq_t; 57 typedef struct nvme_qpair nvme_qpair_t; 58 typedef struct nvme_task_arg nvme_task_arg_t; 59 60 struct nvme_minor_state { 61 kmutex_t nm_mutex; 62 boolean_t nm_oexcl; 63 uint_t nm_ocnt; 64 }; 65 66 struct nvme_dma { 67 ddi_dma_handle_t nd_dmah; 68 ddi_acc_handle_t nd_acch; 69 ddi_dma_cookie_t nd_cookie; 70 uint_t nd_ncookie; 71 caddr_t nd_memp; 72 size_t nd_len; 73 boolean_t nd_cached; 74 }; 75 76 struct nvme_cmd { 77 struct list_node nc_list; 78 79 nvme_sqe_t nc_sqe; 80 nvme_cqe_t nc_cqe; 81 82 void (*nc_callback)(void *); 83 bd_xfer_t *nc_xfer; 84 boolean_t nc_completed; 85 boolean_t nc_dontpanic; 86 uint16_t nc_sqid; 87 88 nvme_dma_t *nc_dma; 89 90 kmutex_t nc_mutex; 91 kcondvar_t nc_cv; 92 93 taskq_ent_t nc_tqent; 94 nvme_t *nc_nvme; 95 }; 96 97 struct nvme_cq { 98 size_t ncq_nentry; 99 uint16_t ncq_id; 100 101 nvme_dma_t *ncq_dma; 102 nvme_cqe_t *ncq_cq; 103 uint_t ncq_head; 104 uint_t ncq_tail; 105 uintptr_t ncq_hdbl; 106 int ncq_phase; 107 108 kmutex_t ncq_mutex; 109 }; 110 111 struct nvme_qpair { 112 size_t nq_nentry; 113 114 nvme_dma_t *nq_sqdma; 115 nvme_sqe_t *nq_sq; 116 uint_t nq_sqhead; 117 uint_t nq_sqtail; 118 uintptr_t nq_sqtdbl; 119 120 nvme_cq_t *nq_cq; 121 122 nvme_cmd_t **nq_cmd; 123 uint16_t nq_next_cmd; 124 uint_t nq_active_cmds; 125 126 kmutex_t nq_mutex; 127 ksema_t nq_sema; 128 }; 129 130 struct nvme { 131 dev_info_t *n_dip; 132 int n_progress; 133 134 caddr_t n_regs; 135 ddi_acc_handle_t n_regh; 136 137 kmem_cache_t *n_cmd_cache; 138 kmem_cache_t *n_prp_cache; 139 140 size_t n_inth_sz; 141 ddi_intr_handle_t *n_inth; 142 int n_intr_cnt; 143 uint_t n_intr_pri; 144 int n_intr_cap; 145 int n_intr_type; 146 int n_intr_types; 147 148 char *n_product; 149 char *n_vendor; 150 151 nvme_version_t n_version; 152 boolean_t n_dead; 153 boolean_t n_strict_version; 154 boolean_t n_ignore_unknown_vendor_status; 155 uint32_t n_admin_queue_len; 156 uint32_t n_io_squeue_len; 157 uint32_t n_io_cqueue_len; 158 uint16_t n_async_event_limit; 159 uint_t n_min_block_size; 160 uint16_t n_abort_command_limit; 161 uint64_t n_max_data_transfer_size; 162 boolean_t n_write_cache_present; 163 boolean_t n_write_cache_enabled; 164 int n_error_log_len; 165 boolean_t n_lba_range_supported; 166 boolean_t n_auto_pst_supported; 167 boolean_t n_async_event_supported; 168 boolean_t n_progress_supported; 169 int n_submission_queues; 170 int n_completion_queues; 171 172 int n_nssr_supported; 173 int n_doorbell_stride; 174 int n_timeout; 175 int n_arbitration_mechanisms; 176 int n_cont_queues_reqd; 177 int n_max_queue_entries; 178 int n_pageshift; 179 int n_pagesize; 180 181 int n_namespace_count; 182 uint_t n_ioq_count; 183 uint_t n_cq_count; 184 185 nvme_identify_ctrl_t *n_idctl; 186 187 nvme_qpair_t *n_adminq; 188 nvme_qpair_t **n_ioq; 189 nvme_cq_t **n_cq; 190 191 nvme_namespace_t *n_ns; 192 193 ddi_dma_attr_t n_queue_dma_attr; 194 ddi_dma_attr_t n_prp_dma_attr; 195 ddi_dma_attr_t n_sgl_dma_attr; 196 ddi_device_acc_attr_t n_reg_acc_attr; 197 ddi_iblock_cookie_t n_fm_ibc; 198 int n_fm_cap; 199 200 ksema_t n_abort_sema; 201 202 ddi_taskq_t *n_cmd_taskq; 203 204 /* state for devctl minor node */ 205 nvme_minor_state_t n_minor; 206 207 /* errors detected by driver */ 208 uint32_t n_dma_bind_err; 209 uint32_t n_abort_failed; 210 uint32_t n_cmd_timeout; 211 uint32_t n_cmd_aborted; 212 uint32_t n_wrong_logpage; 213 uint32_t n_unknown_logpage; 214 uint32_t n_too_many_cookies; 215 216 /* errors detected by hardware */ 217 uint32_t n_data_xfr_err; 218 uint32_t n_internal_err; 219 uint32_t n_abort_rq_err; 220 uint32_t n_abort_sq_del; 221 uint32_t n_nvm_cap_exc; 222 uint32_t n_nvm_ns_notrdy; 223 uint32_t n_inv_cq_err; 224 uint32_t n_inv_qid_err; 225 uint32_t n_max_qsz_exc; 226 uint32_t n_inv_int_vect; 227 uint32_t n_inv_log_page; 228 uint32_t n_inv_format; 229 uint32_t n_inv_q_del; 230 uint32_t n_cnfl_attr; 231 uint32_t n_inv_prot; 232 uint32_t n_readonly; 233 234 /* errors reported by asynchronous events */ 235 uint32_t n_diagfail_event; 236 uint32_t n_persistent_event; 237 uint32_t n_transient_event; 238 uint32_t n_fw_load_event; 239 uint32_t n_reliability_event; 240 uint32_t n_temperature_event; 241 uint32_t n_spare_event; 242 uint32_t n_vendor_event; 243 uint32_t n_unknown_event; 244 245 }; 246 247 struct nvme_namespace { 248 nvme_t *ns_nvme; 249 uint8_t ns_eui64[8]; 250 char ns_name[17]; 251 252 bd_handle_t ns_bd_hdl; 253 254 uint32_t ns_id; 255 size_t ns_block_count; 256 size_t ns_block_size; 257 size_t ns_best_block_size; 258 259 boolean_t ns_ignore; 260 261 nvme_identify_nsid_t *ns_idns; 262 263 /* state for attachment point minor node */ 264 nvme_minor_state_t ns_minor; 265 266 /* 267 * If a namespace has no EUI64, we create a devid in 268 * nvme_prepare_devid(). 269 */ 270 char *ns_devid; 271 }; 272 273 struct nvme_task_arg { 274 nvme_t *nt_nvme; 275 nvme_cmd_t *nt_cmd; 276 }; 277 278 #ifdef __cplusplus 279 } 280 #endif 281 282 #endif /* _NVME_VAR_H */ 283