1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2021 Oxide Computer Company 14 */ 15 16 /* 17 * This file declares all constants and structures dealing with the 18 * physical ENA device. It is based on the ena_com code of the public 19 * Linux and FreeBSD drivers. While this file is based on the common 20 * code it doesn't share the same type names. Where it is useful, a 21 * "common" reference is added to include the name of the type as 22 * defined in the common code. 23 * 24 * The Linux driver defines enq_admin_aq_entry as the top-level type 25 * for admin command descriptors. From this type you can access the 26 * common bits shared by every descriptor (ena_admin_aq_common_desc) 27 * as well as the control buffer (ena_admin_ctrl_buff_info) which is 28 * present for _some_ commands. Other than that, this top-level type 29 * treats the rest of the data as an opaque array of unsigned 32-bit 30 * integers. Then, for each individual command, the Linux driver 31 * defines a dedicated type, each of which contains the following: 32 * 33 * 1. The common descriptor: ena_admin_aq_common_desc. 34 * 35 * 2. The optional control buffer desc: ena_admin_ctrl_buff_info. 36 * 37 * 3. The command-specific data. 38 * 39 * 4. Optional padding to make sure all commands are 64 bytes in size. 40 * 41 * Furthermore, there may be further common types for commands which 42 * are made up of several sub-commands, e.g. the get/set feature 43 * commands. 44 * 45 * Finally, when a command is passed to the common function for 46 * executing commands (ena_com_execute_admin_command()), it is cast as 47 * a pointer to the top-level type: ena_admin_aq_entry. 48 * 49 * This works for the Linux driver just fine, but it causes lots of 50 * repetition in the structure definitions and also means there is no 51 * easy way to determine all valid commands. This ENA driver has 52 * turned the Linux approach inside out -- the top-level type is a 53 * union of all possible commands: enahw_cmd_desc_t. Each command may 54 * then further sub-type via unions to represent its sub-commands. 55 * This same treatment was given to the response descriptor: 56 * enahw_resp_desc_t. 57 * 58 * What is the point of knowing all this? Well, when referencing the 59 * common type in the comment above the enahw_ type, you need to keep 60 * in mind that the Linux/common type will include all the common 61 * descriptor bits, whereas these types do not. 62 * 63 * The common code DOES NOT pack any of these structures, and thus 64 * neither do we. That means these structures all rely on natural 65 * compiler alignment, just as the common code does. In ena.c you will 66 * find CTASSERTs for many of these structures, to verify they are of 67 * the expected size. 68 */ 69 70 #ifndef _ENA_HW_H 71 #define _ENA_HW_H 72 73 #include <sys/ddi.h> 74 #include <sys/sunddi.h> 75 #include <sys/types.h> 76 #include <sys/debug.h> 77 #include <sys/ethernet.h> 78 79 /* 80 * The common code sets the upper limit of I/O queues to 128. In this 81 * case a "queue" is a SQ+CQ pair that forms a logical queue or ring 82 * for sending or receiving packets. Thus, at maximum, we may expect 83 * 128 Tx rings, and 128 Rx rings; though, practically speaking, the 84 * number of rings will often be limited by number of CPUs or 85 * available interrupts. 86 * 87 * common: ENA_MAX_NUM_IO_QUEUES 88 */ 89 #define ENAHW_MAX_NUM_IO_QUEUES 128 90 91 /* 92 * Generate a 32-bit bitmask where the bits between high (inclusive) 93 * and low (inclusive) are set to 1. 94 */ 95 #define GENMASK(h, l) (((~0U) - (1U << (l)) + 1) & (~0U >> (32 - 1 - (h)))) 96 97 /* 98 * Generate a 64-bit bitmask where bit b is set to 1. 99 */ 100 #define BIT(b) (1UL << (b)) 101 102 #define ENAHW_DMA_ADMINQ_ALIGNMENT 8 103 104 #define ENAHW_ADMIN_CQ_DESC_BUF_ALIGNMENT 8 105 #define ENAHW_ADMIN_SQ_DESC_BUF_ALIGNMENT 8 106 #define ENAHW_AENQ_DESC_BUF_ALIGNMENT 8 107 #define ENAHW_HOST_INFO_ALIGNMENT 8 108 #define ENAHW_HOST_INFO_ALLOC_SZ 4096 109 #define ENAHW_IO_CQ_DESC_BUF_ALIGNMENT 4096 110 #define ENAHW_IO_SQ_DESC_BUF_ALIGNMENT 8 111 112 /* 113 * BAR0 register offsets. 114 * 115 * Any register not defined in the common code was marked as a gap, 116 * using the hex address of the register as suffix. The idea is to 117 * make it clear where the gaps are and allow the 118 * ena_hw_update_reg_cache() function to display any bits stored in 119 * these gaps in case they turn out to be interesting later. 120 */ 121 #define ENAHW_REG_VERSION 0x0 122 #define ENAHW_REG_CONTROLLER_VERSION 0x4 123 #define ENAHW_REG_CAPS 0x8 124 #define ENAHW_REG_CAPS_EXT 0xc 125 #define ENAHW_REG_ASQ_BASE_LO 0x10 126 #define ENAHW_REG_ASQ_BASE_HI 0x14 127 #define ENAHW_REG_ASQ_CAPS 0x18 128 #define ENAHW_REG_GAP_1C 0x1c 129 #define ENAHW_REG_ACQ_BASE_LO 0x20 130 #define ENAHW_REG_ACQ_BASE_HI 0x24 131 #define ENAHW_REG_ACQ_CAPS 0x28 132 #define ENAHW_REG_ASQ_DB 0x2c 133 #define ENAHW_REG_ACQ_TAIL 0x30 134 #define ENAHW_REG_AENQ_CAPS 0x34 135 #define ENAHW_REG_AENQ_BASE_LO 0x38 136 #define ENAHW_REG_AENQ_BASE_HI 0x3c 137 #define ENAHW_REG_AENQ_HEAD_DB 0x40 138 #define ENAHW_REG_AENQ_TAIL 0x44 139 #define ENAHW_REG_GAP_48 0x48 140 #define ENAHW_REG_INTERRUPT_MASK 0x4c 141 #define ENAHW_REG_GAP_50 0x50 142 #define ENAHW_REG_DEV_CTL 0x54 143 #define ENAHW_REG_DEV_STS 0x58 144 #define ENAHW_REG_MMIO_REG_READ 0x5c 145 #define ENAHW_REG_MMIO_RESP_LO 0x60 146 #define ENAHW_REG_MMIO_RESP_HI 0x64 147 #define ENAHW_REG_RSS_IND_ENTRY_UPDATE 0x68 148 #define ENAHW_NUM_REGS ((ENAHW_REG_RSS_IND_ENTRY_UPDATE / 4) + 1) 149 150 /* 151 * Device Version (Register 0x0) 152 */ 153 #define ENAHW_DEV_MINOR_VSN_MASK 0xff 154 #define ENAHW_DEV_MAJOR_VSN_SHIFT 8 155 #define ENAHW_DEV_MAJOR_VSN_MASK 0xff00 156 157 #define ENAHW_DEV_MAJOR_VSN(vsn) \ 158 (((vsn) & ENAHW_DEV_MAJOR_VSN_MASK) >> ENAHW_DEV_MAJOR_VSN_SHIFT) 159 #define ENAHW_DEV_MINOR_VSN(vsn) \ 160 ((vsn) & ENAHW_DEV_MINOR_VSN_MASK) 161 162 /* 163 * Controller Version (Register 0x4) 164 */ 165 #define ENAHW_CTRL_SUBMINOR_VSN_MASK 0xff 166 #define ENAHW_CTRL_MINOR_VSN_SHIFT 8 167 #define ENAHW_CTRL_MINOR_VSN_MASK 0xff00 168 #define ENAHW_CTRL_MAJOR_VSN_SHIFT 16 169 #define ENAHW_CTRL_MAJOR_VSN_MASK 0xff0000 170 #define ENAHW_CTRL_IMPL_ID_SHIFT 24 171 #define ENAHW_CTRL_IMPL_ID_MASK 0xff000000 172 173 #define ENAHW_CTRL_MAJOR_VSN(vsn) \ 174 (((vsn) & ENAHW_CTRL_MAJOR_VSN_MASK) >> ENAHW_CTRL_MAJOR_VSN_SHIFT) 175 #define ENAHW_CTRL_MINOR_VSN(vsn) \ 176 (((vsn) & ENAHW_CTRL_MINOR_VSN_MASK) >> ENAHW_CTRL_MINOR_VSN_SHIFT) 177 #define ENAHW_CTRL_SUBMINOR_VSN(vsn) \ 178 ((vsn) & ENAHW_CTRL_SUBMINOR_VSN_MASK) 179 #define ENAHW_CTRL_IMPL_ID(vsn) \ 180 (((vsn) & ENAHW_CTRL_IMPL_ID_MASK) >> ENAHW_CTRL_IMPL_ID_SHIFT) 181 182 /* 183 * Device Caps (Register 0x8) 184 */ 185 #define ENAHW_CAPS_CONTIGUOUS_QUEUE_REQUIRED_MASK 0x1 186 #define ENAHW_CAPS_RESET_TIMEOUT_SHIFT 1 187 #define ENAHW_CAPS_RESET_TIMEOUT_MASK 0x3e 188 #define ENAHW_CAPS_RESET_TIMEOUT(v) \ 189 (((v) & ENAHW_CAPS_RESET_TIMEOUT_MASK) >> \ 190 ENAHW_CAPS_RESET_TIMEOUT_SHIFT) 191 #define ENAHW_CAPS_DMA_ADDR_WIDTH_SHIFT 8 192 #define ENAHW_CAPS_DMA_ADDR_WIDTH_MASK 0xff00 193 #define ENAHW_CAPS_DMA_ADDR_WIDTH(v) \ 194 (((v) & ENAHW_CAPS_DMA_ADDR_WIDTH_MASK) >> \ 195 ENAHW_CAPS_DMA_ADDR_WIDTH_SHIFT) 196 #define ENAHW_CAPS_ADMIN_CMD_TIMEOUT_SHIFT 16 197 #define ENAHW_CAPS_ADMIN_CMD_TIMEOUT_MASK 0xf0000 198 #define ENAHW_CAPS_ADMIN_CMD_TIMEOUT(v) \ 199 (((v) & ENAHW_CAPS_ADMIN_CMD_TIMEOUT_MASK) >> \ 200 ENAHW_CAPS_ADMIN_CMD_TIMEOUT_SHIFT) 201 202 enum enahw_reset_reason_types { 203 ENAHW_RESET_NORMAL = 0, 204 ENAHW_RESET_KEEP_ALIVE_TO = 1, 205 ENAHW_RESET_ADMIN_TO = 2, 206 ENAHW_RESET_MISS_TX_CMPL = 3, 207 ENAHW_RESET_INV_RX_REQ_ID = 4, 208 ENAHW_RESET_INV_TX_REQ_ID = 5, 209 ENAHW_RESET_TOO_MANY_RX_DESCS = 6, 210 ENAHW_RESET_INIT_ERR = 7, 211 ENAHW_RESET_DRIVER_INVALID_STATE = 8, 212 ENAHW_RESET_OS_TRIGGER = 9, 213 ENAHW_RESET_OS_NETDEV_WD = 10, 214 ENAHW_RESET_SHUTDOWN = 11, 215 ENAHW_RESET_USER_TRIGGER = 12, 216 ENAHW_RESET_GENERIC = 13, 217 ENAHW_RESET_MISS_INTERRUPT = 14, 218 ENAHW_RESET_LAST, 219 }; 220 221 /* 222 * Admin Submission Queue Caps (Register 0x18) 223 */ 224 #define ENAHW_ASQ_CAPS_DEPTH_MASK 0xffff 225 #define ENAHW_ASQ_CAPS_ENTRY_SIZE_SHIFT 16 226 #define ENAHW_ASQ_CAPS_ENTRY_SIZE_MASK 0xffff0000 227 228 #define ENAHW_ASQ_CAPS_DEPTH(x) ((x) & ENAHW_ASQ_CAPS_DEPTH_MASK) 229 230 #define ENAHW_ASQ_CAPS_ENTRY_SIZE(x) \ 231 (((x) << ENAHW_ASQ_CAPS_ENTRY_SIZE_SHIFT) & \ 232 ENAHW_ASQ_CAPS_ENTRY_SIZE_MASK) 233 234 /* 235 * Admin Completion Queue Caps (Register 0x28) 236 */ 237 #define ENAHW_ACQ_CAPS_DEPTH_MASK 0xffff 238 #define ENAHW_ACQ_CAPS_ENTRY_SIZE_SHIFT 16 239 #define ENAHW_ACQ_CAPS_ENTRY_SIZE_MASK 0xffff0000 240 241 #define ENAHW_ACQ_CAPS_DEPTH(x) ((x) & ENAHW_ACQ_CAPS_DEPTH_MASK) 242 243 #define ENAHW_ACQ_CAPS_ENTRY_SIZE(x) \ 244 (((x) << ENAHW_ACQ_CAPS_ENTRY_SIZE_SHIFT) & \ 245 ENAHW_ACQ_CAPS_ENTRY_SIZE_MASK) 246 247 /* 248 * Asynchronous Event Notification Queue Caps (Register 0x34) 249 */ 250 #define ENAHW_AENQ_CAPS_DEPTH_MASK 0xffff 251 #define ENAHW_AENQ_CAPS_ENTRY_SIZE_SHIFT 16 252 #define ENAHW_AENQ_CAPS_ENTRY_SIZE_MASK 0xffff0000 253 254 #define ENAHW_AENQ_CAPS_DEPTH(x) ((x) & ENAHW_AENQ_CAPS_DEPTH_MASK) 255 256 #define ENAHW_AENQ_CAPS_ENTRY_SIZE(x) \ 257 (((x) << ENAHW_AENQ_CAPS_ENTRY_SIZE_SHIFT) & \ 258 ENAHW_AENQ_CAPS_ENTRY_SIZE_MASK) 259 260 /* 261 * Interrupt Mask (Register 0x4c) 262 */ 263 #define ENAHW_INTR_UNMASK 0x0 264 #define ENAHW_INTR_MASK 0x1 265 266 /* 267 * Device Control (Register 0x54) 268 */ 269 #define ENAHW_DEV_CTL_DEV_RESET_MASK 0x1 270 #define ENAHW_DEV_CTL_AQ_RESTART_SHIFT 1 271 #define ENAHW_DEV_CTL_AQ_RESTART_MASK 0x2 272 #define ENAHW_DEV_CTL_QUIESCENT_SHIFT 2 273 #define ENAHW_DEV_CTL_QUIESCENT_MASK 0x4 274 #define ENAHW_DEV_CTL_IO_RESUME_SHIFT 3 275 #define ENAHW_DEV_CTL_IO_RESUME_MASK 0x8 276 #define ENAHW_DEV_CTL_RESET_REASON_SHIFT 28 277 #define ENAHW_DEV_CTL_RESET_REASON_MASK 0xf0000000 278 279 /* 280 * Device Status (Register 0x58) 281 */ 282 #define ENAHW_DEV_STS_READY_MASK 0x1 283 #define ENAHW_DEV_STS_AQ_RESTART_IN_PROGRESS_SHIFT 1 284 #define ENAHW_DEV_STS_AQ_RESTART_IN_PROGRESS_MASK 0x2 285 #define ENAHW_DEV_STS_AQ_RESTART_FINISHED_SHIFT 2 286 #define ENAHW_DEV_STS_AQ_RESTART_FINISHED_MASK 0x4 287 #define ENAHW_DEV_STS_RESET_IN_PROGRESS_SHIFT 3 288 #define ENAHW_DEV_STS_RESET_IN_PROGRESS_MASK 0x8 289 #define ENAHW_DEV_STS_RESET_FINISHED_SHIFT 4 290 #define ENAHW_DEV_STS_RESET_FINISHED_MASK 0x10 291 #define ENAHW_DEV_STS_FATAL_ERROR_SHIFT 5 292 #define ENAHW_DEV_STS_FATAL_ERROR_MASK 0x20 293 #define ENAHW_DEV_STS_QUIESCENT_STATE_IN_PROGRESS_SHIFT 6 294 #define ENAHW_DEV_STS_QUIESCENT_STATE_IN_PROGRESS_MASK 0x40 295 #define ENAHW_DEV_STS_QUIESCENT_STATE_ACHIEVED_SHIFT 7 296 #define ENAHW_DEV_STS_QUIESCENT_STATE_ACHIEVED_MASK 0x80 297 298 /* common: ena_admin_aenq_common_desc */ 299 typedef struct enahw_aenq_desc { 300 uint16_t ead_group; 301 uint16_t ead_syndrome; 302 uint8_t ead_flags; 303 uint8_t ead_rsvd1[3]; 304 uint32_t ead_ts_low; 305 uint32_t ead_ts_high; 306 307 union { 308 uint32_t raw[12]; 309 310 struct { 311 uint32_t flags; 312 } link_change; 313 314 struct { 315 uint32_t rx_drops_low; 316 uint32_t rx_drops_high; 317 uint32_t tx_drops_low; 318 uint32_t tx_drops_high; 319 } keep_alive; 320 } ead_payload; 321 } enahw_aenq_desc_t; 322 323 #define ENAHW_AENQ_DESC_PHASE_MASK BIT(0) 324 325 #define ENAHW_AENQ_DESC_PHASE(desc) \ 326 ((desc)->ead_flags & ENAHW_AENQ_DESC_PHASE_MASK) 327 328 #define ENAHW_AENQ_LINK_CHANGE_LINK_STATUS_MASK BIT(0) 329 330 /* 331 * Asynchronous Event Notification Queue groups. 332 * 333 * Note: These values represent the bit position of each feature as 334 * returned by ENAHW_FEAT_AENQ_CONFIG. We encode them this way so that 335 * they can double as an index into the AENQ handlers array. 336 * 337 * common: ena_admin_aenq_group 338 */ 339 typedef enum enahw_aenq_groups { 340 ENAHW_AENQ_GROUP_LINK_CHANGE = 0, 341 ENAHW_AENQ_GROUP_FATAL_ERROR = 1, 342 ENAHW_AENQ_GROUP_WARNING = 2, 343 ENAHW_AENQ_GROUP_NOTIFICATION = 3, 344 ENAHW_AENQ_GROUP_KEEP_ALIVE = 4, 345 ENAHW_AENQ_GROUP_REFRESH_CAPABILITIES = 5, 346 ENAHW_AENQ_GROUPS_ARR_NUM = 6, 347 } enahw_aenq_groups_t; 348 349 /* 350 * The reason for ENAHW_AENQ_GROUP_NOFIFICATION. 351 * 352 * common: ena_admin_aenq_notification_syndrome 353 */ 354 typedef enum enahw_aenq_syndrome { 355 ENAHW_AENQ_SYNDROME_UPDATE_HINTS = 2, 356 } enahw_aenq_syndrome_t; 357 358 /* 359 * ENA devices use a 48-bit memory space. 360 * 361 * common: ena_common_mem_addr 362 */ 363 typedef struct enahw_addr { 364 uint32_t ea_low; 365 uint16_t ea_high; 366 uint16_t ea_rsvd; /* must be zero */ 367 } enahw_addr_t; 368 369 /* common: ena_admin_ctrl_buff_info */ 370 struct enahw_ctrl_buff { 371 uint32_t ecb_length; 372 enahw_addr_t ecb_addr; 373 }; 374 375 /* common: ena_admin_get_set_feature_common_desc */ 376 struct enahw_feat_common { 377 /* 378 * 1:0 Select which value you want. 379 * 380 * 0x1 = Current value. 381 * 0x3 = Default value. 382 * 383 * Note: Linux seems to set this to 0 to get the value, 384 * not sure if that's a bug or just another way to get the 385 * current value. 386 * 387 * 7:3 Reserved. 388 */ 389 uint8_t efc_flags; 390 391 /* An id from enahw_feature_id_t. */ 392 uint8_t efc_id; 393 394 /* 395 * Each feature is versioned, allowing upgrades to the feature 396 * set without breaking backwards compatibility. The driver 397 * uses this field to specify which version it supports 398 * (starting from zero). Linux doesn't document this very well 399 * and sets this value to 0 for most features. We define a set 400 * of macros, underneath the enahw_feature_id_t type, clearly 401 * documenting the version we support for each feature. 402 */ 403 uint8_t efc_version; 404 uint8_t efc_rsvd; 405 }; 406 407 /* common: ena_admin_get_feat_cmd */ 408 typedef struct enahw_cmd_get_feat { 409 struct enahw_ctrl_buff ecgf_ctrl_buf; 410 struct enahw_feat_common ecgf_comm; 411 uint32_t egcf_unused[11]; 412 } enahw_cmd_get_feat_t; 413 414 /* 415 * N.B. Linux sets efc_flags to 0 (via memset) when reading the 416 * current value, but the comments say it should be 0x1. We follow the 417 * comments. 418 */ 419 #define ENAHW_GET_FEAT_FLAGS_GET_CURR_VAL(desc) \ 420 ((desc)->ecgf_comm.efc_flags) |= 0x1 421 #define ENAHW_GET_FEAT_FLAGS_GET_DEF_VAL(desc) \ 422 ((desc)->ecgf_comm.efc_flags) |= 0x3 423 424 /* 425 * Set the MTU of the device. This value does not include the L2 426 * headers or trailers, only the payload. 427 * 428 * common: ena_admin_set_feature_mtu_desc 429 */ 430 typedef struct enahw_feat_mtu { 431 uint32_t efm_mtu; 432 } enahw_feat_mtu_t; 433 434 /* common: ena_admin_set_feature_host_attr_desc */ 435 typedef struct enahw_feat_host_attr { 436 enahw_addr_t efha_os_addr; 437 enahw_addr_t efha_debug_addr; 438 uint32_t efha_debug_sz; 439 } enahw_feat_host_attr_t; 440 441 /* 442 * ENAHW_FEAT_AENQ_CONFIG 443 * 444 * common: ena_admin_feature_aenq_desc 445 */ 446 typedef struct enahw_feat_aenq { 447 /* Bitmask of AENQ groups this device supports. */ 448 uint32_t efa_supported_groups; 449 450 /* Bitmask of AENQ groups currently enabled. */ 451 uint32_t efa_enabled_groups; 452 } enahw_feat_aenq_t; 453 454 /* common: ena_admin_set_feat_cmd */ 455 typedef struct enahw_cmd_set_feat { 456 struct enahw_ctrl_buff ecsf_ctrl_buf; 457 struct enahw_feat_common ecsf_comm; 458 459 union { 460 uint32_t ecsf_raw[11]; 461 enahw_feat_host_attr_t ecsf_host_attr; 462 enahw_feat_mtu_t ecsf_mtu; 463 enahw_feat_aenq_t ecsf_aenq; 464 } ecsf_feat; 465 } enahw_cmd_set_feat_t; 466 467 /* 468 * Used to populate the host information buffer which the Nitro 469 * hypervisor supposedly uses for display, debugging, and possibly 470 * other purposes. 471 * 472 * common: ena_admin_host_info 473 */ 474 typedef struct enahw_host_info { 475 uint32_t ehi_os_type; 476 uint8_t ehi_os_dist_str[128]; 477 uint32_t ehi_os_dist; 478 uint8_t ehi_kernel_ver_str[32]; 479 uint32_t ehi_kernel_ver; 480 uint32_t ehi_driver_ver; 481 uint32_t ehi_supported_net_features[2]; 482 uint16_t ehi_ena_spec_version; 483 uint16_t ehi_bdf; 484 uint16_t ehi_num_cpus; 485 uint16_t ehi_rsvd; 486 uint32_t ehi_driver_supported_features; 487 } enahw_host_info_t; 488 489 #define ENAHW_HOST_INFO_MAJOR_MASK GENMASK(7, 0) 490 #define ENAHW_HOST_INFO_MINOR_SHIFT 8 491 #define ENAHW_HOST_INFO_MINOR_MASK GENMASK(15, 8) 492 #define ENAHW_HOST_INFO_SUB_MINOR_SHIFT 16 493 #define ENAHW_HOST_INFO_SUB_MINOR_MASK GENMASK(23, 16) 494 #define ENAHW_HOST_INFO_SPEC_MAJOR_SHIFT 8 495 #define ENAHW_HOST_INFO_MODULE_TYPE_SHIFT 24 496 #define ENAHW_HOST_INFO_MODULE_TYPE_MASK GENMASK(31, 24) 497 #define ENAHW_HOST_INFO_FUNCTION_MASK GENMASK(2, 0) 498 #define ENAHW_HOST_INFO_DEVICE_SHIFT 3 499 #define ENAHW_HOST_INFO_DEVICE_MASK GENMASK(7, 3) 500 #define ENAHW_HOST_INFO_BUS_SHIFT 8 501 #define ENAHW_HOST_INFO_BUS_MASK GENMASK(15, 8) 502 #define ENAHW_HOST_INFO_RX_OFFSET_SHIFT 1 503 #define ENAHW_HOST_INFO_RX_OFFSET_MASK BIT(1) 504 #define ENAHW_HOST_INFO_INTERRUPT_MODERATION_SHIFT 2 505 #define ENAHW_HOST_INFO_INTERRUPT_MODERATION_MASK BIT(2) 506 #define ENAHW_HOST_INFO_RX_BUF_MIRRORING_SHIFT 3 507 #define ENAHW_HOST_INFO_RX_BUF_MIRRORING_MASK BIT(3) 508 #define ENAHW_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_SHIFT 4 509 #define ENAHW_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK BIT(4) 510 511 /* common: ena_admin_os_type */ 512 enum enahw_os_type { 513 ENAHW_OS_LINUX = 1, 514 ENAHW_OS_WIN = 2, 515 ENAHW_OS_DPDK = 3, 516 ENAHW_OS_FREEBSD = 4, 517 ENAHW_OS_IPXE = 5, 518 ENAHW_OS_ESXI = 6, 519 ENAHW_OS_MACOS = 7, 520 ENAHW_OS_GROUPS_NUM = 7, 521 }; 522 523 /* 524 * Create I/O Completion Queue 525 * 526 * A completion queue is where the device writes responses to I/O 527 * requests. The admin completion queue must be created before such a 528 * command can be issued, see ena_admin_cq_init(). 529 * 530 * common: ena_admin_aq_create_cq_cmd 531 */ 532 typedef struct enahw_cmd_create_cq { 533 /* 534 * 7-6 reserved 535 * 536 * 5 interrupt mode: when set the device sends an interrupt 537 * for each completion, otherwise the driver must poll 538 * the queue. 539 * 540 * 4-0 reserved 541 */ 542 uint8_t ecq_caps_1; 543 544 /* 545 * 7-5 reserved 546 * 547 * 4-0 CQ entry size (in words): the size of a single CQ entry 548 * in multiples of 32-bit words. 549 * 550 * NOTE: According to the common code the "valid" values 551 * are 4 or 8 -- this is incorrect. The valid values are 552 * 2 and 4. The common code does have an "extended" Rx 553 * completion descriptor, ena_eth_io_rx_cdesc_ext, that 554 * is 32 bytes and thus would use a value of 8, but it is 555 * not used by the Linux or FreeBSD drivers, so we do not 556 * bother with it. 557 * 558 * Type Bytes Value 559 * enahw_tx_cdesc_t 8 2 560 * enahw_rx_cdesc_t 16 4 561 */ 562 uint8_t ecq_caps_2; 563 564 /* The number of CQ entries, must be a power of 2. */ 565 uint16_t ecq_num_descs; 566 567 /* The MSI-X vector assigned to this CQ. */ 568 uint32_t ecq_msix_vector; 569 570 /* 571 * The CQ's physical base address. The CQ memory must be 572 * physically contiguous. 573 */ 574 enahw_addr_t ecq_addr; 575 } enahw_cmd_create_cq_t; 576 577 #define ENAHW_CMD_CREATE_CQ_INTERRUPT_MODE_ENABLED_SHIFT 5 578 #define ENAHW_CMD_CREATE_CQ_INTERRUPT_MODE_ENABLED_MASK (BIT(5)) 579 #define ENAHW_CMD_CREATE_CQ_DESC_SIZE_WORDS_MASK (GENMASK(4, 0)) 580 581 #define ENAHW_CMD_CREATE_CQ_INTERRUPT_MODE_ENABLE(cmd) \ 582 ((cmd)->ecq_caps_1 |= ENAHW_CMD_CREATE_CQ_INTERRUPT_MODE_ENABLED_MASK) 583 584 #define ENAHW_CMD_CREATE_CQ_DESC_SIZE_WORDS(cmd, val) \ 585 (((cmd)->ecq_caps_2) |= \ 586 ((val) & ENAHW_CMD_CREATE_CQ_DESC_SIZE_WORDS_MASK)) 587 588 /* 589 * Destroy Completion Queue 590 * 591 * common: ena_admin_aq_destroy_cq_cmd 592 */ 593 typedef struct enahw_cmd_destroy_cq { 594 uint16_t edcq_idx; 595 uint16_t edcq_rsvd; 596 } enahw_cmd_destroy_cq_t; 597 598 /* 599 * common: ena_admin_aq_create_sq_cmd 600 */ 601 typedef struct enahw_cmd_create_sq { 602 /* 603 * 7-5 direction: 0x1 = Tx, 0x2 = Rx 604 * 4-0 reserved 605 */ 606 uint8_t ecsq_dir; 607 uint8_t ecsq_rsvd1; 608 609 /* 610 * 7 reserved 611 * 612 * 6-4 completion policy: How are completion events generated. 613 * 614 * See enahw_completion_policy_type_t for a description of 615 * the various values. 616 * 617 * 3-0 placement policy: Where the descriptor ring and 618 * headers reside. 619 * 620 * See enahw_placement_policy_t for a description of the 621 * various values. 622 */ 623 uint8_t ecsq_caps_2; 624 625 /* 626 * 7-1 reserved 627 * 628 * 0 physically contiguous: When set indicates the descriptor 629 * ring memory is physically contiguous. 630 */ 631 uint8_t ecsq_caps_3; 632 633 /* 634 * The index of the associated Completion Queue (CQ). The CQ 635 * must be created before the SQ. 636 */ 637 uint16_t ecsq_cq_idx; 638 639 /* The number of descriptors in this SQ. */ 640 uint16_t ecsq_num_descs; 641 642 /* 643 * The base physical address of the SQ. This should not be set 644 * for LLQ. Must be page aligned. 645 */ 646 enahw_addr_t ecsq_base; 647 648 /* 649 * The physical address of the head write-back pointer. Valid 650 * only when the completion policy is set to one of the head 651 * write-back modes (0x2 or 0x3). Must be cacheline size 652 * aligned. 653 */ 654 enahw_addr_t ecsq_head_wb; 655 uint32_t ecsq_rsvdw2; 656 uint32_t ecsq_rsvdw3; 657 } enahw_cmd_create_sq_t; 658 659 typedef enum enahw_sq_direction { 660 ENAHW_SQ_DIRECTION_TX = 1, 661 ENAHW_SQ_DIRECTION_RX = 2, 662 } enahw_sq_direction_t; 663 664 typedef enum enahw_placement_policy { 665 /* Descriptors and headers are in host memory. */ 666 ENAHW_PLACEMENT_POLICY_HOST = 1, 667 668 /* 669 * Descriptors and headers are in device memory (a.k.a Low 670 * Latency Queue). 671 */ 672 ENAHW_PLACEMENT_POLICY_DEV = 3, 673 } enahw_placement_policy_t; 674 675 /* 676 * DESC: Write a CQ entry for each SQ descriptor. 677 * 678 * DESC_ON_DEMAND: Write a CQ entry when requested by the SQ descriptor. 679 * 680 * HEAD_ON_DEMAND: Update head pointer when requested by the SQ 681 * descriptor. 682 * 683 * HEAD: Update head pointer for each SQ descriptor. 684 * 685 */ 686 typedef enum enahw_completion_policy_type { 687 ENAHW_COMPLETION_POLICY_DESC = 0, 688 ENAHW_COMPLETION_POLICY_DESC_ON_DEMAND = 1, 689 ENAHW_COMPLETION_POLICY_HEAD_ON_DEMAND = 2, 690 ENAHW_COMPLETION_POLICY_HEAD = 3, 691 } enahw_completion_policy_type_t; 692 693 #define ENAHW_CMD_CREATE_SQ_DIR_SHIFT 5 694 #define ENAHW_CMD_CREATE_SQ_DIR_MASK GENMASK(7, 5) 695 #define ENAHW_CMD_CREATE_SQ_PLACEMENT_POLICY_MASK GENMASK(3, 0) 696 #define ENAHW_CMD_CREATE_SQ_COMPLETION_POLICY_SHIFT 4 697 #define ENAHW_CMD_CREATE_SQ_COMPLETION_POLICY_MASK GENMASK(6, 4) 698 #define ENAHW_CMD_CREATE_SQ_PHYSMEM_CONTIG_MASK BIT(0) 699 700 #define ENAHW_CMD_CREATE_SQ_DIR(cmd, val) \ 701 (((cmd)->ecsq_dir) |= (((val) << ENAHW_CMD_CREATE_SQ_DIR_SHIFT) & \ 702 ENAHW_CMD_CREATE_SQ_DIR_MASK)) 703 704 #define ENAHW_CMD_CREATE_SQ_PLACEMENT_POLICY(cmd, val) \ 705 (((cmd)->ecsq_caps_2) |= \ 706 ((val) & ENAHW_CMD_CREATE_SQ_PLACEMENT_POLICY_MASK)) 707 708 #define ENAHW_CMD_CREATE_SQ_COMPLETION_POLICY(cmd, val) \ 709 (((cmd)->ecsq_caps_2) |= \ 710 (((val) << ENAHW_CMD_CREATE_SQ_COMPLETION_POLICY_SHIFT) & \ 711 ENAHW_CMD_CREATE_SQ_COMPLETION_POLICY_MASK)) 712 713 #define ENAHW_CMD_CREATE_SQ_PHYSMEM_CONTIG(cmd) \ 714 ((cmd)->ecsq_caps_3 |= ENAHW_CMD_CREATE_SQ_PHYSMEM_CONTIG_MASK) 715 716 /* common: ena_admin_sq */ 717 typedef struct enahw_cmd_destroy_sq { 718 uint16_t edsq_idx; 719 uint8_t edsq_dir; /* Tx/Rx */ 720 uint8_t edsq_rsvd; 721 } enahw_cmd_destroy_sq_t; 722 723 #define ENAHW_CMD_DESTROY_SQ_DIR_SHIFT 5 724 #define ENAHW_CMD_DESTROY_SQ_DIR_MASK GENMASK(7, 5) 725 726 #define ENAHW_CMD_DESTROY_SQ_DIR(cmd, val) \ 727 (((cmd)->edsq_dir) |= (((val) << ENAHW_CMD_DESTROY_SQ_DIR_SHIFT) & \ 728 ENAHW_CMD_DESTROY_SQ_DIR_MASK)) 729 730 /* common: ena_admin_aq_get_stats_cmd */ 731 typedef struct enahw_cmd_get_stats { 732 struct enahw_ctrl_buff ecgs_ctrl_buf; 733 uint8_t ecgs_type; 734 uint8_t ecgs_scope; 735 uint16_t ecgs_rsvd; 736 uint16_t ecgs_queue_idx; 737 738 /* 739 * The device ID for which to query stats from. The sentinel 740 * value 0xFFFF indicates a query of the current device. 741 * According to the common docs, a "privileged device" may 742 * query stats for other ENA devices. However the definition 743 * of this "privilege device" is not expanded upon. 744 */ 745 uint16_t ecgs_device_id; 746 } enahw_cmd_get_stats_t; 747 748 /* Query the stats for my device. */ 749 #define ENAHW_CMD_GET_STATS_MY_DEVICE_ID 0xFFFF 750 751 /* 752 * BASIC: Returns enahw_resp_basic_stats. 753 * 754 * EXTENDED: According to the Linux documentation returns a buffer in 755 * "string format" with additional statistics per queue and per device ID. 756 * 757 * ENI: According to the Linux documentation it returns "extra HW 758 * stats for a specific network interfaces". 759 * 760 * common: ena_admin_get_stats_type 761 */ 762 typedef enum enahw_get_stats_type { 763 ENAHW_GET_STATS_TYPE_BASIC = 0, 764 ENAHW_GET_STATS_TYPE_EXTENDED = 1, 765 ENAHW_GET_STATS_TYPE_ENI = 2, 766 } enahw_get_stats_type_t; 767 768 /* common: ena_admin_get_stats_scope */ 769 typedef enum enahw_get_stats_scope { 770 ENAHW_GET_STATS_SCOPE_QUEUE = 0, 771 ENAHW_GET_STATS_SCOPE_ETH = 1, 772 } enahw_get_stats_scope_t; 773 774 /* common: ena_admin_aq_entry */ 775 typedef struct enahw_cmd_desc { 776 uint16_t ecd_cmd_id; 777 uint8_t ecd_opcode; 778 uint8_t ecd_flags; 779 780 union { 781 uint32_t ecd_raw[15]; 782 enahw_cmd_get_feat_t ecd_get_feat; 783 enahw_cmd_set_feat_t ecd_set_feat; 784 enahw_cmd_create_cq_t ecd_create_cq; 785 enahw_cmd_destroy_cq_t ecd_destroy_cq; 786 enahw_cmd_create_sq_t ecd_create_sq; 787 enahw_cmd_destroy_sq_t ecd_destroy_sq; 788 enahw_cmd_get_stats_t ecd_get_stats; 789 } ecd_cmd; 790 791 } enahw_cmd_desc_t; 792 793 /* 794 * top level commands that may be sent to the Admin Queue. 795 * 796 * common: ena_admin_aq_opcode 797 */ 798 typedef enum ena_cmd_opcode { 799 ENAHW_CMD_NONE = 0, 800 ENAHW_CMD_CREATE_SQ = 1, 801 ENAHW_CMD_DESTROY_SQ = 2, 802 ENAHW_CMD_CREATE_CQ = 3, 803 ENAHW_CMD_DESTROY_CQ = 4, 804 ENAHW_CMD_GET_FEATURE = 8, 805 ENAHW_CMD_SET_FEATURE = 9, 806 ENAHW_CMD_GET_STATS = 11, 807 } enahw_cmd_opcode_t; 808 809 /* common: ENA_ADMIN_AQ_COMMON_DESC */ 810 #define ENAHW_CMD_ID_MASK GENMASK(11, 0) 811 #define ENAHW_CMD_PHASE_MASK BIT(0) 812 813 #define ENAHW_CMD_ID(desc, id) \ 814 (((desc)->ecd_cmd_id) |= ((id) & ENAHW_CMD_ID_MASK)) 815 816 /* 817 * Subcommands for ENA_ADMIN_{GET,SET}_FEATURE. 818 * 819 * common: ena_admin_aq_feature_id 820 */ 821 typedef enum enahw_feature_id { 822 ENAHW_FEAT_DEVICE_ATTRIBUTES = 1, 823 ENAHW_FEAT_MAX_QUEUES_NUM = 2, 824 ENAHW_FEAT_HW_HINTS = 3, 825 ENAHW_FEAT_LLQ = 4, 826 ENAHW_FEAT_EXTRA_PROPERTIES_STRINGS = 5, 827 ENAHW_FEAT_EXTRA_PROPERTIES_FLAGS = 6, 828 ENAHW_FEAT_MAX_QUEUES_EXT = 7, 829 ENAHW_FEAT_RSS_HASH_FUNCTION = 10, 830 ENAHW_FEAT_STATELESS_OFFLOAD_CONFIG = 11, 831 ENAHW_FEAT_RSS_INDIRECTION_TABLE_CONFIG = 12, 832 ENAHW_FEAT_MTU = 14, 833 ENAHW_FEAT_RSS_HASH_INPUT = 18, 834 ENAHW_FEAT_INTERRUPT_MODERATION = 20, 835 ENAHW_FEAT_AENQ_CONFIG = 26, 836 ENAHW_FEAT_LINK_CONFIG = 27, 837 ENAHW_FEAT_HOST_ATTR_CONFIG = 28, 838 ENAHW_FEAT_NUM = 32, 839 } enahw_feature_id_t; 840 841 /* 842 * The following macros define the maximum version we support for each 843 * feature. These are the feature versions we use to communicate with 844 * the feature command. Linux has these values spread throughout the 845 * code at the various callsites of ena_com_get_feature(). We choose 846 * to centralize our feature versions to make it easier to audit. 847 */ 848 #define ENAHW_FEAT_DEVICE_ATTRIBUTES_VER 0 849 #define ENAHW_FEAT_MAX_QUEUES_NUM_VER 0 850 #define ENAHW_FEAT_HW_HINTS_VER 0 851 #define ENAHW_FEAT_LLQ_VER 0 852 #define ENAHW_FEAT_EXTRA_PROPERTIES_STRINGS_VER 0 853 #define ENAHW_FEAT_EXTRA_PROPERTIES_FLAGS_VER 0 854 #define ENAHW_FEAT_MAX_QUEUES_EXT_VER 1 855 #define ENAHW_FEAT_RSS_HASH_FUNCTION_VER 0 856 #define ENAHW_FEAT_STATELESS_OFFLOAD_CONFIG_VER 0 857 #define ENAHW_FEAT_RSS_INDIRECTION_TABLE_CONFIG_VER 0 858 #define ENAHW_FEAT_MTU_VER 0 859 #define ENAHW_FEAT_RSS_HASH_INPUT_VER 0 860 #define ENAHW_FEAT_INTERRUPT_MODERATION_VER 0 861 #define ENAHW_FEAT_AENQ_CONFIG_VER 0 862 #define ENAHW_FEAT_LINK_CONFIG_VER 0 863 #define ENAHW_FEAT_HOST_ATTR_CONFIG_VER 0 864 865 /* common: ena_admin_link_types */ 866 typedef enum enahw_link_speeds { 867 ENAHW_LINK_SPEED_1G = 0x1, 868 ENAHW_LINK_SPEED_2_HALF_G = 0x2, 869 ENAHW_LINK_SPEED_5G = 0x4, 870 ENAHW_LINK_SPEED_10G = 0x8, 871 ENAHW_LINK_SPEED_25G = 0x10, 872 ENAHW_LINK_SPEED_40G = 0x20, 873 ENAHW_LINK_SPEED_50G = 0x40, 874 ENAHW_LINK_SPEED_100G = 0x80, 875 ENAHW_LINK_SPEED_200G = 0x100, 876 ENAHW_LINK_SPEED_400G = 0x200, 877 } enahw_link_speeds_t; 878 879 /* 880 * Response to ENAHW_FEAT_HW_HINTS. 881 * 882 * Hints from the device to the driver about what values to use for 883 * various communications between the two. A value of 0 indicates 884 * there is no hint and the driver should provide its own default. All 885 * timeout values are in milliseconds. 886 * 887 * common: ena_admin_ena_hw_hints 888 */ 889 typedef struct enahw_device_hints { 890 /* 891 * The amount of time the driver should wait for an MMIO read 892 * reply before giving up and returning an error. 893 */ 894 uint16_t edh_mmio_read_timeout; 895 896 /* 897 * If the driver has not seen an AENQ keep alive in this 898 * timeframe, then consider the device hung and perform a 899 * reset. 900 */ 901 uint16_t edh_keep_alive_timeout; 902 903 /* 904 * The timeperiod in which we expect a Tx to report 905 * completion, otherwise it is considered "missed". Initiate a 906 * device reset when the number of missed completions is 907 * greater than the threshold. 908 */ 909 uint16_t edh_tx_comp_timeout; 910 uint16_t edh_missed_tx_reset_threshold; 911 912 /* 913 * The timeperiod in which we expect an admin command to 914 * report completion. 915 */ 916 uint16_t edh_admin_comp_timeout; 917 918 /* 919 * Used by Linux to set the netdevice 'watchdog_timeo' value. 920 * This value is used by the networking stack to determine 921 * when a pending transmission has stalled. This is similar to 922 * the keep alive timeout, except its viewing progress from 923 * the perspective of the network stack itself. This differnce 924 * is subtle but important: the device could be in a state 925 * where it has a functioning keep alive heartbeat, but has a 926 * stuck Tx queue impeding forward progress of the networking 927 * stack (which in many cases results in a scenario 928 * indistinguishable form a complete host hang). 929 * 930 * The mac layer does not currently provide such 931 * functionality, though it could and should be extended to 932 * support such a feature. 933 */ 934 uint16_t edh_net_wd_timeout; 935 936 /* 937 * The maximum number of cookies/segments allowed in a DMA 938 * scatter-gather list. 939 */ 940 uint16_t edh_max_tx_sgl; 941 uint16_t edh_max_rx_sgl; 942 943 uint16_t reserved[8]; 944 } enahw_device_hints_t; 945 946 /* 947 * Response to ENAHW_FEAT_DEVICE_ATTRIBUTES. 948 * 949 * common: ena_admin_device_attr_feature_desc 950 */ 951 typedef struct enahw_feat_dev_attr { 952 uint32_t efda_impl_id; 953 uint32_t efda_device_version; 954 955 /* 956 * Bitmap representing supported get/set feature subcommands 957 * (enahw_feature_id). 958 */ 959 uint32_t efda_supported_features; 960 uint32_t efda_rsvd1; 961 962 /* Number of bits used for physical/vritual address. */ 963 uint32_t efda_phys_addr_width; 964 uint32_t efda_virt_addr_with; 965 966 /* The unicast MAC address in network byte order. */ 967 uint8_t efda_mac_addr[6]; 968 uint8_t efda_rsvd2[2]; 969 uint32_t efda_max_mtu; 970 } enahw_feat_dev_attr_t; 971 972 /* 973 * Response to ENAHW_FEAT_MAX_QUEUES_NUM. 974 * 975 * common: ena_admin_queue_feature_desc 976 */ 977 typedef struct enahw_feat_max_queue { 978 uint32_t efmq_max_sq_num; 979 uint32_t efmq_max_sq_depth; 980 uint32_t efmq_max_cq_num; 981 uint32_t efmq_max_cq_depth; 982 uint32_t efmq_max_legacy_llq_num; 983 uint32_t efmq_max_legacy_llq_depth; 984 uint32_t efmq_max_header_size; 985 986 /* 987 * The maximum number of descriptors a single Tx packet may 988 * span. This includes the meta descriptor. 989 */ 990 uint16_t efmq_max_per_packet_tx_descs; 991 992 /* 993 * The maximum number of descriptors a single Rx packet may span. 994 */ 995 uint16_t efmq_max_per_packet_rx_descs; 996 } enahw_feat_max_queue_t; 997 998 /* 999 * Response to ENAHW_FEAT_MAX_QUEUES_EXT. 1000 * 1001 * common: ena_admin_queue_ext_feature_desc 1002 */ 1003 typedef struct enahw_feat_max_queue_ext { 1004 uint8_t efmqe_version; 1005 uint8_t efmqe_rsvd[3]; 1006 1007 uint32_t efmqe_max_tx_sq_num; 1008 uint32_t efmqe_max_tx_cq_num; 1009 uint32_t efmqe_max_rx_sq_num; 1010 uint32_t efmqe_max_rx_cq_num; 1011 uint32_t efmqe_max_tx_sq_depth; 1012 uint32_t efmqe_max_tx_cq_depth; 1013 uint32_t efmqe_max_rx_sq_depth; 1014 uint32_t efmqe_max_rx_cq_depth; 1015 uint32_t efmqe_max_tx_header_size; 1016 1017 /* 1018 * The maximum number of descriptors a single Tx packet may 1019 * span. This includes the meta descriptor. 1020 */ 1021 uint16_t efmqe_max_per_packet_tx_descs; 1022 1023 /* 1024 * The maximum number of descriptors a single Rx packet may span. 1025 */ 1026 uint16_t efmqe_max_per_packet_rx_descs; 1027 } enahw_feat_max_queue_ext_t; 1028 1029 /* 1030 * Response to ENA_ADMIN_LINK_CONFIG. 1031 * 1032 * common: ena_admin_get_feature_link_desc 1033 */ 1034 typedef struct enahw_feat_link_conf { 1035 /* Link speed in Mbit/s. */ 1036 uint32_t eflc_speed; 1037 1038 /* Bit field of enahw_link_speeds_t. */ 1039 uint32_t eflc_supported; 1040 1041 /* 1042 * 31-2: reserved 1043 * 1: duplex - Full Duplex 1044 * 0: autoneg 1045 */ 1046 uint32_t eflc_flags; 1047 } enahw_feat_link_conf_t; 1048 1049 #define ENAHW_FEAT_LINK_CONF_AUTONEG_MASK BIT(0) 1050 #define ENAHW_FEAT_LINK_CONF_DUPLEX_SHIFT 1 1051 #define ENAHW_FEAT_LINK_CONF_DUPLEX_MASK BIT(1) 1052 1053 #define ENAHW_FEAT_LINK_CONF_AUTONEG(f) \ 1054 ((f)->eflc_flags & ENAHW_FEAT_LINK_CONF_AUTONEG_MASK) 1055 1056 #define ENAHW_FEAT_LINK_CONF_FULL_DUPLEX(f) \ 1057 ((((f)->eflc_flags & ENAHW_FEAT_LINK_CONF_DUPLEX_MASK) >> \ 1058 ENAHW_FEAT_LINK_CONF_DUPLEX_SHIFT) == 1) 1059 1060 /* 1061 * Response to ENAHW_FEAT_STATELESS_OFFLOAD_CONFIG. 1062 * 1063 * common: ena_admin_feature_offload_desc 1064 */ 1065 typedef struct enahw_feat_offload { 1066 /* 1067 * 0 : Tx IPv4 Header Checksum 1068 * 1 : Tx L4/IPv4 Partial Checksum 1069 * 1070 * The L4 checksum field should be initialized with pseudo 1071 * header checksum. 1072 * 1073 * 2 : Tx L4/IPv4 Checksum Full 1074 * 3 : Tx L4/IPv6 Partial Checksum 1075 * 1076 * The L4 checksum field should be initialized with pseudo 1077 * header checksum. 1078 * 1079 * 4 : Tx L4/IPv6 Checksum Full 1080 * 5 : TCP/IPv4 LSO (aka TSO) 1081 * 6 : TCP/IPv6 LSO (aka TSO) 1082 * 7 : LSO ECN 1083 */ 1084 uint32_t efo_tx; 1085 1086 /* 1087 * Receive side supported stateless offload. 1088 * 1089 * 0 : Rx IPv4 Header Checksum 1090 * 1 : Rx TCP/UDP + IPv4 Full Checksum 1091 * 2 : Rx TCP/UDP + IPv6 Full Checksum 1092 * 3 : Rx hash calculation 1093 */ 1094 uint32_t efo_rx_supported; 1095 1096 /* Linux seems to only check rx_supported. */ 1097 uint32_t efo_rx_enabled; 1098 } enahw_feat_offload_t; 1099 1100 /* Feature Offloads */ 1101 #define ENAHW_FEAT_OFFLOAD_TX_L3_IPV4_CSUM_MASK BIT(0) 1102 #define ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_PART_SHIFT 1 1103 #define ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_PART_MASK BIT(1) 1104 #define ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_FULL_SHIFT 2 1105 #define ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_FULL_MASK BIT(2) 1106 #define ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_PART_SHIFT 3 1107 #define ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_PART_MASK BIT(3) 1108 #define ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_FULL_SHIFT 4 1109 #define ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_FULL_MASK BIT(4) 1110 #define ENAHW_FEAT_OFFLOAD_TSO_IPV4_SHIFT 5 1111 #define ENAHW_FEAT_OFFLOAD_TSO_IPV4_MASK BIT(5) 1112 #define ENAHW_FEAT_OFFLOAD_TSO_IPV6_SHIFT 6 1113 #define ENAHW_FEAT_OFFLOAD_TSO_IPV6_MASK BIT(6) 1114 #define ENAHW_FEAT_OFFLOAD_TSO_ECN_SHIFT 7 1115 #define ENAHW_FEAT_OFFLOAD_TSO_ECN_MASK BIT(7) 1116 #define ENAHW_FEAT_OFFLOAD_RX_L3_IPV4_CSUM_MASK BIT(0) 1117 #define ENAHW_FEAT_OFFLOAD_RX_L4_IPV4_CSUM_SHIFT 1 1118 #define ENAHW_FEAT_OFFLOAD_RX_L4_IPV4_CSUM_MASK BIT(1) 1119 #define ENAHW_FEAT_OFFLOAD_RX_L4_IPV6_CSUM_SHIFT 2 1120 #define ENAHW_FEAT_OFFLOAD_RX_L4_IPV6_CSUM_MASK BIT(2) 1121 #define ENAHW_FEAT_OFFLOAD_RX_HASH_SHIFT 3 1122 #define ENAHW_FEAT_OFFLOAD_RX_HASH_MASK BIT(3) 1123 1124 #define ENAHW_FEAT_OFFLOAD_TX_L3_IPV4_CSUM(f) \ 1125 (((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TX_L3_IPV4_CSUM_MASK) != 0) 1126 1127 #define ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_PART(f) \ 1128 (((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_PART_MASK) != 0) 1129 1130 #define ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_FULL(f) \ 1131 (((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_FULL_MASK) != 0) 1132 1133 #define ENAHW_FEAT_OFFLOAD_TSO_IPV4(f) \ 1134 (((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TSO_IPV4_MASK) != 0) 1135 1136 #define ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_PART(f) \ 1137 (((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_PART_MASK) != 0) 1138 1139 #define ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_FULL(f) \ 1140 (((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_FULL_MASK) != 0) 1141 1142 #define ENAHW_FEAT_OFFLOAD_TSO_IPV6(f) \ 1143 (((f)->efo_tx & ENAHW_FEAT_OFFLOAD_TSO_IPV6_MASK) != 0) 1144 1145 #define ENAHW_FEAT_OFFLOAD_RX_L3_IPV4_CSUM(f) \ 1146 (((f)->efo_rx_supported & ENAHW_FEAT_OFFLOAD_RX_L3_IPV4_CSUM_MASK) != 0) 1147 1148 #define ENAHW_FEAT_OFFLOAD_RX_L4_IPV4_CSUM(f) \ 1149 (((f)->efo_rx_supported & ENAHW_FEAT_OFFLOAD_RX_L4_IPV4_CSUM_MASK) != 0) 1150 1151 #define ENAHW_FEAT_OFFLOAD_RX_L4_IPV6_CSUM(f) \ 1152 (((f)->efo_rx_supported & ENAHW_FEAT_OFFLOAD_RX_L4_IPV6_CSUM_MASK) != 0) 1153 1154 typedef union enahw_resp_get_feat { 1155 uint32_t ergf_raw[14]; 1156 enahw_feat_dev_attr_t ergf_dev_attr; 1157 enahw_feat_max_queue_t ergf_max_queue; 1158 enahw_feat_max_queue_ext_t ergf_max_queue_ext; 1159 enahw_feat_aenq_t ergf_aenq; 1160 enahw_feat_link_conf_t ergf_link_conf; 1161 enahw_feat_offload_t ergf_offload; 1162 } enahw_resp_get_feat_u; 1163 1164 /* 1165 * common: ena_admin_acq_create_cq_resp_desc 1166 */ 1167 typedef struct enahw_resp_create_cq { 1168 /* 1169 * The hardware's index for this queue. 1170 */ 1171 uint16_t ercq_idx; 1172 1173 /* 1174 * Apparently the number of descriptors granted may be 1175 * different than that requested. 1176 */ 1177 uint16_t ercq_actual_num_descs; 1178 uint32_t ercq_numa_node_reg_offset; 1179 uint32_t ercq_head_db_reg_offset; /* doorbell */ 1180 uint32_t ercq_interrupt_mask_reg_offset; /* stop intr */ 1181 } enahw_resp_create_cq_t; 1182 1183 /* common: ena_admin_acq_create_sq_resp_desc */ 1184 typedef struct enahw_resp_create_sq { 1185 uint16_t ersq_idx; 1186 uint16_t ersq_rsvdw1; 1187 uint32_t ersq_db_reg_offset; 1188 uint32_t ersq_llq_descs_reg_offset; 1189 uint32_t ersq_llq_headers_reg_offset; 1190 } enahw_resp_create_sq_t; 1191 1192 /* common: ena_admin_basic_stats */ 1193 typedef struct enahw_resp_basic_stats { 1194 uint32_t erbs_tx_bytes_low; 1195 uint32_t erbs_tx_bytes_high; 1196 uint32_t erbs_tx_pkts_low; 1197 uint32_t erbs_tx_pkts_high; 1198 uint32_t erbs_rx_bytes_low; 1199 uint32_t erbs_rx_bytes_high; 1200 uint32_t erbs_rx_pkts_low; 1201 uint32_t erbs_rx_pkts_high; 1202 uint32_t erbs_rx_drops_low; 1203 uint32_t erbs_rx_drops_high; 1204 uint32_t erbs_tx_drops_low; 1205 uint32_t erbs_tx_drops_high; 1206 } enahw_resp_basic_stats_t; 1207 1208 /* common: ena_admin_eni_stats */ 1209 typedef struct enahw_resp_eni_stats { 1210 /* 1211 * The number of inbound packets dropped due to aggregate 1212 * inbound bandwidth allowance being exceeded. 1213 */ 1214 uint64_t eres_bw_in_exceeded; 1215 1216 /* 1217 * The number of outbound packets dropped due to aggregated outbound 1218 * bandwidth allowance being exceeded. 1219 */ 1220 uint64_t eres_bw_out_exceeded; 1221 1222 /* 1223 * The number of packets dropped due to the Packets Per Second 1224 * allowance being exceeded. 1225 */ 1226 uint64_t eres_pps_exceeded; 1227 1228 /* 1229 * The number of packets dropped due to connection tracking 1230 * allowance being exceeded and leading to failure in 1231 * establishment of new connections. 1232 */ 1233 uint64_t eres_conns_exceeded; 1234 1235 /* 1236 * The number of packets dropped due to linklocal packet rate 1237 * allowance being exceeded. 1238 */ 1239 uint64_t eres_linklocal_exceeded; 1240 } enahw_resp_eni_stats_t; 1241 1242 /* 1243 * common: ena_admin_acq_entry 1244 */ 1245 typedef struct enahw_resp_desc { 1246 /* The index of the completed command. */ 1247 uint16_t erd_cmd_id; 1248 1249 /* The status of the command (enahw_resp_status_t). */ 1250 uint8_t erd_status; 1251 1252 /* 1253 * 7-1 Reserved 1254 * 0 Phase 1255 */ 1256 uint8_t erd_flags; 1257 1258 /* Extended status. */ 1259 uint16_t erd_ext_status; 1260 1261 /* 1262 * The AQ entry (enahw_cmd_desc) index which has been consumed 1263 * by the device and can be reused. However, this field is not 1264 * used in the other drivers, and it seems to be redundant 1265 * with the erd_idx field. 1266 */ 1267 uint16_t erd_sq_head_idx; 1268 1269 union { 1270 uint32_t raw[14]; 1271 enahw_resp_get_feat_u erd_get_feat; 1272 enahw_resp_create_cq_t erd_create_cq; 1273 /* destroy_cq: No command-specific response. */ 1274 enahw_resp_create_sq_t erd_create_sq; 1275 /* destroy_sq: No command-specific response. */ 1276 enahw_resp_basic_stats_t erd_basic_stats; 1277 enahw_resp_eni_stats_t erd_eni_stats; 1278 } erd_resp; 1279 } enahw_resp_desc_t; 1280 1281 /* common: ENA_ADMIN_ACQ_COMMON_DESC */ 1282 #define ENAHW_RESP_CMD_ID_MASK GENMASK(11, 0) 1283 #define ENAHW_RESP_PHASE_MASK 0x1 1284 1285 #define ENAHW_RESP_CMD_ID(desc) \ 1286 (((desc)->erd_cmd_id) & ENAHW_RESP_CMD_ID_MASK) 1287 1288 /* 1289 * The response status of an Admin Queue command. 1290 * 1291 * common: ena_admin_aq_completion_status 1292 */ 1293 typedef enum enahw_resp_status { 1294 ENAHW_RESP_SUCCESS = 0, 1295 ENAHW_RESP_RESOURCE_ALLOCATION_FAILURE = 1, 1296 ENAHW_RESP_BAD_OPCODE = 2, 1297 ENAHW_RESP_UNSUPPORTED_OPCODE = 3, 1298 ENAHW_RESP_MALFORMED_REQUEST = 4, 1299 /* 1300 * At this place in the common code it mentions that there is 1301 * "additional status" in the reponse descriptor's 1302 * erd_ext_status field. As the common code never actually 1303 * uses this field it's hard to know the exact meaning of the 1304 * comment. My best guess is the illegal parameter error 1305 * stores additional context in the erd_ext_status field. But 1306 * how to interpret that additional context is anyone's guess. 1307 */ 1308 ENAHW_RESP_ILLEGAL_PARAMETER = 5, 1309 ENAHW_RESP_UNKNOWN_ERROR = 6, 1310 ENAHW_RESP_RESOURCE_BUSY = 7, 1311 } enahw_resp_status_t; 1312 1313 /* 1314 * Not really a device structure, more of a helper to debug register values. 1315 */ 1316 typedef struct enahw_reg_nv { 1317 char *ern_name; 1318 uint32_t ern_offset; 1319 uint32_t ern_value; 1320 } enahw_reg_nv_t; 1321 1322 /* 1323 * I/O macros and strcutures. 1324 * ------------------------- 1325 */ 1326 1327 /* 1328 * The device's L3 and L4 protocol numbers. These are specific to the 1329 * ENA device and not to be confused with IANA protocol numbers. 1330 * 1331 * common: ena_eth_io_l3_proto_index 1332 */ 1333 typedef enum enahw_io_l3_proto { 1334 ENAHW_IO_L3_PROTO_UNKNOWN = 0, 1335 ENAHW_IO_L3_PROTO_IPV4 = 8, 1336 ENAHW_IO_L3_PROTO_IPV6 = 11, 1337 ENAHW_IO_L3_PROTO_FCOE = 21, 1338 ENAHW_IO_L3_PROTO_ROCE = 22, 1339 } enahw_io_l3_proto_t; 1340 1341 /* common: ena_eth_io_l4_proto_index */ 1342 typedef enum enahw_io_l4_proto { 1343 ENAHW_IO_L4_PROTO_UNKNOWN = 0, 1344 ENAHW_IO_L4_PROTO_TCP = 12, 1345 ENAHW_IO_L4_PROTO_UDP = 13, 1346 ENAHW_IO_L4_PROTO_ROUTEABLE_ROCE = 23, 1347 } enahw_io_l4_proto_t; 1348 1349 /* common: ena_eth_io_tx_desc */ 1350 typedef struct enahw_tx_data_desc { 1351 /* 1352 * 15-0 Buffer Length (LENGTH) 1353 * 1354 * The buffer length in bytes. This should NOT include the 1355 * Ethernet FCS bytes. 1356 * 1357 * 21-16 Request ID High Bits [15-10] (REQ_ID_HI) 1358 * 22 Reserved Zero 1359 * 23 Metadata Flag always zero (META_DESC) 1360 * 1361 * This flag indicates if the descriptor is a metadata 1362 * descriptor or not. In this case we are defining the Tx 1363 * descriptor, so it's always zero. 1364 * 1365 * 24 Phase bit (PHASE) 1366 * 25 Reserved Zero 1367 * 26 First Descriptor Bit (FIRST) 1368 * 1369 * Indicates this is the first descriptor for the frame. 1370 * 1371 * 27 Last Descriptor Bit (LAST) 1372 * 1373 * Indicates this is the last descriptor for the frame. 1374 * 1375 * 28 Completion Request Bit (COMP_REQ) 1376 * 1377 * Indicates if completion should be posted after the 1378 * frame is transmitted. This bit is only valid on the 1379 * first descriptor. 1380 * 1381 * 31-29 Reserved Zero 1382 */ 1383 uint32_t etd_len_ctrl; 1384 1385 /* 1386 * 3-0 L3 Protocol Number (L3_PROTO_IDX) 1387 * 1388 * The L3 protocol type, one of enahw_io_l3_proto_t. This 1389 * field is required when L3_CSUM_EN or TSO_EN is set. 1390 * 1391 * 4 Don't Fragment Bit (DF) 1392 * 1393 * The value of IPv4 DF. This value must copy the value 1394 * found in the packet's IPv4 header. 1395 * 1396 * 6-5 Reserved Zero 1397 * 7 TSO Bit (TSO_EN) 1398 * 1399 * Enable TCP Segment Offload. 1400 * 1401 * 12-8 L4 Protocol Number (L4_PROTO_IDX) 1402 * 1403 * The L4 protocol type, one of enahw_io_l4_proto_t. This 1404 * field is required when L4_CSUM_EN or TSO_EN are 1405 * set. 1406 * 1407 * 13 L3 Checksum Offload (L3_CSUM_EN) 1408 * 1409 * Enable IPv4 header checksum offload. 1410 * 1411 * 14 L4 Checksum Offload (L4_CSUM_EN) 1412 * 1413 * Enable TCP/UDP checksum offload. 1414 * 1415 * 15 Ethernet FCS Disable (ETHERNET_FCS_DIS) 1416 * 1417 * Disable the device's Ethernet Frame Check sequence. 1418 * 1419 * 16 Reserved Zero 1420 * 17 L4 Partial Checksum Present (L4_CSUM_PARTIAL) 1421 * 1422 * When set it indicates the host has already provided 1423 * the pseudo-header checksum. Otherwise, it is up to the 1424 * device to calculate it. 1425 * 1426 * When set and using TSO the host stack must remember 1427 * not to include the TCP segment length in the supplied 1428 * pseudo-header. 1429 * 1430 * The host stack should provide the pseudo-header 1431 * checksum when using IPv6 with Routing Headers. 1432 * 1433 * 21-18 Reserved Zero 1434 * 31-22 Request ID Low [9-0] (REQ_ID_LO) 1435 */ 1436 uint32_t etd_meta_ctrl; 1437 1438 /* The low 32 bits of the buffer address. */ 1439 uint32_t etd_buff_addr_lo; 1440 1441 /* 1442 * address high and header size 1443 * 1444 * 15-0 Buffer Address High [47-32] (ADDR_HI) 1445 * 1446 * The upper 15 bits of the buffer address. 1447 * 1448 * 23-16 Reserved Zero 1449 * 31-24 Header Length (HEADER_LENGTH) 1450 * 1451 * This field has dubious documentation in the 1452 * common/Linux driver code, even contradicting itself in 1453 * the same sentence. Here's what it says, verbatim: 1454 * 1455 * > Header length. For Low Latency Queues, this fields 1456 * > indicates the number of bytes written to the 1457 * > headers' memory. For normal queues, if packet is TCP 1458 * > or UDP, and longer than max_header_size, then this 1459 * > field should be set to the sum of L4 header offset 1460 * > and L4 header size(without options), otherwise, this 1461 * > field should be set to 0. For both modes, this field 1462 * > must not exceed the max_header_size. max_header_size 1463 * > value is reported by the Max Queues Feature 1464 * > descriptor 1465 * 1466 * Here's what one _might_ ascertain from the above. 1467 * 1468 * 1. This field should always be set in the case of 1469 * LLQs/device placement. 1470 * 1471 * 2. This field must _never_ exceed the max header size 1472 * as reported by feature detection. In our code this 1473 * would be efmq_max_header_size for older ENA devices 1474 * and efmqe_max_tx_header_size for newer ones. One 1475 * empirical data point from a t3.small (with newer 1476 * device) is a max Tx header size of 128 bytes. 1477 * 1478 * 3. If the packet is TCP or UDP, and the packet (or the 1479 * headers?) is longer than the max header size, then 1480 * this field should be set to the total header size 1481 * with the exception of TCP header options. 1482 * Otherwise, if the packet is not TCP or UDP, or if 1483 * the packet (or header length?) _does not_ exceed 1484 * the max header size, then set this value to 0. 1485 * 1486 * One might think, based on (3), that when the header 1487 * size exceeds the max this field needs to be set, but 1488 * that contradicts (2), which dictates that the total 1489 * header size can never exceed the max. Sure enough, the 1490 * Linux code drops all packets with headers that exceed 1491 * the max. So in that case it would mean that "and 1492 * longer than max_header_size" is referring to the total 1493 * packet length. So for most workloads, the TCP/UDP 1494 * packets should have this field set, to indicate their 1495 * header length. This matches with Linux, which seems to 1496 * set header length regardless of IP protocol. 1497 * 1498 * However, the FreeBSD code tells a different story. In 1499 * it's non-LLQ Tx path it has the following comment, 1500 * verbatim: 1501 * 1502 * > header_len is just a hint for the device. Because 1503 * > FreeBSD is not giving us information about packet 1504 * > header length and it is not guaranteed that all 1505 * > packet headers will be in the 1st mbuf, setting 1506 * > header_len to 0 is making the device ignore this 1507 * > value and resolve header on it's own. 1508 * 1509 * According to this we can just set the value to zero 1510 * and let the device figure it out. This maps better to 1511 * illumos, where we also allow the header to potentially 1512 * span multiple mblks (though we do have access to the 1513 * header sizes via mac_ether_offload_info_t). 1514 * 1515 * The upshot: for now we take advantage of the device's 1516 * ability to determine the header length on its own, at 1517 * the potential cost of some performance (not measured). 1518 */ 1519 uint32_t etd_buff_addr_hi_hdr_sz; 1520 } enahw_tx_data_desc_t; 1521 1522 #define ENAHW_TX_DESC_LENGTH_MASK GENMASK(15, 0) 1523 #define ENAHW_TX_DESC_REQ_ID_HI_SHIFT 16 1524 #define ENAHW_TX_DESC_REQ_ID_HI_MASK GENMASK(21, 16) 1525 #define ENAHW_TX_DESC_META_DESC_SHIFT 23 1526 #define ENAHW_TX_DESC_META_DESC_MASK BIT(23) 1527 #define ENAHW_TX_DESC_PHASE_SHIFT 24 1528 #define ENAHW_TX_DESC_PHASE_MASK BIT(24) 1529 #define ENAHW_TX_DESC_FIRST_SHIFT 26 1530 #define ENAHW_TX_DESC_FIRST_MASK BIT(26) 1531 #define ENAHW_TX_DESC_LAST_SHIFT 27 1532 #define ENAHW_TX_DESC_LAST_MASK BIT(27) 1533 #define ENAHW_TX_DESC_COMP_REQ_SHIFT 28 1534 #define ENAHW_TX_DESC_COMP_REQ_MASK BIT(28) 1535 #define ENAHW_TX_DESC_L3_PROTO_IDX_MASK GENMASK(3, 0) 1536 #define ENAHW_TX_DESC_DF_SHIFT 4 1537 #define ENAHW_TX_DESC_DF_MASK BIT(4) 1538 #define ENAHW_TX_DESC_TSO_EN_SHIFT 7 1539 #define ENAHW_TX_DESC_TSO_EN_MASK BIT(7) 1540 #define ENAHW_TX_DESC_L4_PROTO_IDX_SHIFT 8 1541 #define ENAHW_TX_DESC_L4_PROTO_IDX_MASK GENMASK(12, 8) 1542 #define ENAHW_TX_DESC_L3_CSUM_EN_SHIFT 13 1543 #define ENAHW_TX_DESC_L3_CSUM_EN_MASK BIT(13) 1544 #define ENAHW_TX_DESC_L4_CSUM_EN_SHIFT 14 1545 #define ENAHW_TX_DESC_L4_CSUM_EN_MASK BIT(14) 1546 #define ENAHW_TX_DESC_ETHERNET_FCS_DIS_SHIFT 15 1547 #define ENAHW_TX_DESC_ETHERNET_FCS_DIS_MASK BIT(15) 1548 #define ENAHW_TX_DESC_L4_CSUM_PARTIAL_SHIFT 17 1549 #define ENAHW_TX_DESC_L4_CSUM_PARTIAL_MASK BIT(17) 1550 #define ENAHW_TX_DESC_REQ_ID_LO_SHIFT 22 1551 #define ENAHW_TX_DESC_REQ_ID_LO_MASK GENMASK(31, 22) 1552 #define ENAHW_TX_DESC_ADDR_HI_MASK GENMASK(15, 0) 1553 #define ENAHW_TX_DESC_HEADER_LENGTH_SHIFT 24 1554 #define ENAHW_TX_DESC_HEADER_LENGTH_MASK GENMASK(31, 24) 1555 1556 #define ENAHW_TX_DESC_LENGTH(desc, len) \ 1557 (((desc)->etd_len_ctrl) |= ((len) & ENAHW_TX_DESC_LENGTH_MASK)) 1558 1559 #define ENAHW_TX_DESC_FIRST_ON(desc) \ 1560 (((desc)->etd_len_ctrl) |= ENAHW_TX_DESC_FIRST_MASK) 1561 1562 #define ENAHW_TX_DESC_FIRST_OFF(desc) \ 1563 (((desc)->etd_len_ctrl) &= ~ENAHW_TX_DESC_FIRST_MASK) 1564 1565 #define ENAHW_TX_DESC_REQID_HI(desc, reqid) \ 1566 (((desc)->etd_len_ctrl) |= \ 1567 ((((reqid) >> 10) << ENAHW_TX_DESC_REQ_ID_HI_SHIFT) & \ 1568 ENAHW_TX_DESC_REQ_ID_HI_MASK)) 1569 1570 #define ENAHW_TX_DESC_REQID_LO(desc, reqid) \ 1571 (((desc)->etd_meta_ctrl) |= \ 1572 (((reqid) << ENAHW_TX_DESC_REQ_ID_LO_SHIFT) & \ 1573 ENAHW_TX_DESC_REQ_ID_LO_MASK)) 1574 1575 #define ENAHW_TX_DESC_PHASE(desc, phase) \ 1576 (((desc)->etd_len_ctrl) |= (((phase) << ENAHW_TX_DESC_PHASE_SHIFT) & \ 1577 ENAHW_TX_DESC_PHASE_MASK)) 1578 1579 #define ENAHW_TX_DESC_LAST_ON(desc) \ 1580 (((desc)->etd_len_ctrl) |= ENAHW_TX_DESC_LAST_MASK) 1581 1582 #define ENAHW_TX_DESC_LAST_OFF(desc) \ 1583 (((desc)->etd_len_ctrl) &= ~ENAHW_TX_DESC_LAST_MASK) 1584 1585 #define ENAHW_TX_DESC_COMP_REQ_ON(desc) \ 1586 (((desc)->etd_len_ctrl) |= ENAHW_TX_DESC_COMP_REQ_MASK) 1587 1588 #define ENAHW_TX_DESC_COMP_REQ_OFF(desc) \ 1589 (((desc)->etd_len_ctrl) &= ~ENAHW_TX_DESC_COMP_REQ_MASK) 1590 1591 #define ENAHW_TX_DESC_META_DESC_ON(desc) \ 1592 (((desc)->etd_len_ctrl) |= ENAHW_TX_DESC_META_DESC_MASK) 1593 1594 #define ENAHW_TX_DESC_META_DESC_OFF(desc) \ 1595 (((desc)->etd_len_ctrl) &= ~ENAHW_TX_DESC_META_DESC_MASK) 1596 1597 #define ENAHW_TX_DESC_ADDR_LO(desc, addr) \ 1598 (((desc)->etd_buff_addr_lo) = (addr)) 1599 1600 #define ENAHW_TX_DESC_ADDR_HI(desc, addr) \ 1601 (((desc)->etd_buff_addr_hi_hdr_sz) |= \ 1602 (((addr) >> 32) & ENAHW_TX_DESC_ADDR_HI_MASK)) 1603 1604 #define ENAHW_TX_DESC_HEADER_LENGTH(desc, len) \ 1605 (((desc)->etd_buff_addr_hi_hdr_sz) |= \ 1606 (((len) << ENAHW_TX_DESC_HEADER_LENGTH_SHIFT) & \ 1607 ENAHW_TX_DESC_HEADER_LENGTH_MASK)) 1608 1609 #define ENAHW_TX_DESC_DF_ON(desc) \ 1610 ((desc)->etd_meta_ctrl |= ENAHW_TX_DESC_DF_MASK) 1611 1612 #define ENAHW_TX_DESC_TSO_OFF(desc) \ 1613 (((desc)->etd_meta_ctrl) &= ~ENAHW_TX_DESC_TSO_EN_MASK) 1614 1615 #define ENAHW_TX_DESC_L3_CSUM_OFF(desc) \ 1616 (((desc)->etd_meta_ctrl) &= ~ENAHW_TX_DESC_L3_CSUM_EN_MASK) 1617 1618 #define ENAHW_TX_DESC_L4_CSUM_OFF(desc) \ 1619 (((desc)->etd_meta_ctrl) &= ~ENAHW_TX_DESC_L4_CSUM_EN_MASK) 1620 1621 #define ENAHW_TX_DESC_L4_CSUM_PARTIAL_ON(desc) \ 1622 (((desc)->etd_meta_ctrl) &= ~ENAHW_TX_DESC_L4_CSUM_PARTIAL_MASK) 1623 1624 /* common: ena_eth_io_tx_meta_desc */ 1625 typedef struct enahw_tx_meta_desc { 1626 /* 1627 * 9-0 Request ID Low [9-0] (REQ_ID_LO) 1628 * 13-10 Reserved Zero 1629 * 14 Extended Metadata Valid (EXT_VALID) 1630 * 1631 * When set this descriptor contains valid extended 1632 * metadata. The extended metadata includes the L3/L4 1633 * length and offset fields as well as the MSS bits. This 1634 * is needed for TSO. 1635 * 1636 * 15 Reserved Zero 1637 * 19-16 MSS High Bits (MSS_HI) 1638 * 20 Meta Type (ETH_META_TYPE) 1639 * 1640 * If enabled this is an extended metadata descriptor. 1641 * This seems redundant with EXT_VALID. 1642 * 1643 * 21 Meta Store (META_STORE) 1644 * 1645 * Store the extended metadata in the queue cache. 1646 * 1647 * 22 Reserved Zero 1648 * 23 Metadata Flag (META_DESC) -- always one 1649 * 24 Phase (PHASE) 1650 * 25 Reserved Zero 1651 * 26 First Descriptor Bit (FIRST) 1652 * 27 Last Descriptor Bit (LAST) 1653 * 28 Completion Request Bit (COMP_REQ) 1654 * 31-29 Reserved Zero 1655 */ 1656 uint32_t etmd_len_ctrl; 1657 1658 /* 1659 * 5-0 Request ID High Bits [15-10] (REQ_ID_HI) 1660 * 31-6 Reserved Zero 1661 */ 1662 uint32_t etmd_word1; 1663 1664 /* 1665 * 7-0 L3 Header Length (L3_HDR_LEN) 1666 * 15:8 L3 Header Offset (L3_HDR_OFF) 1667 * 21:16 L4 Header Length in Words (L4_HDR_LEN_IN_WORDS) 1668 * 1669 * Specifies the L4 header length in words. The device 1670 * assumes the L4 header follows directly after the L3 1671 * header and that the L4 offset is equal to L3_HDR_OFF + 1672 * L3_HDR_LEN. 1673 * 1674 * 31-22 MSS Low Bits (MSS_LO) 1675 */ 1676 uint32_t etmd_word2; 1677 uint32_t etmd_reserved; 1678 } enahw_tx_meta_desc_t; 1679 1680 /* common: N/A */ 1681 typedef union enahw_tx_desc { 1682 enahw_tx_data_desc_t etd_data; 1683 enahw_tx_meta_desc_t etd_meta; 1684 } enahw_tx_desc_t; 1685 1686 /* common: ena_eth_io_tx_cdesc */ 1687 typedef struct enahw_tx_cdesc { 1688 /* 1689 * 15-0 Request ID Bits 1690 * 16 Reserved Zero 1691 */ 1692 uint16_t etc_req_id; 1693 1694 /* 1695 * Presumably the status of the Tx, though the Linux driver 1696 * never checks this field. 1697 */ 1698 uint8_t etc_status; 1699 1700 /* 1701 * 0 Phase 1702 * 7-1 Reserved Zero 1703 */ 1704 uint8_t etc_flags; 1705 1706 /* 1707 * This isn't documented or used in the Linux driver, but 1708 * these probably store the submission queue ID and the 1709 * submission queue head index. 1710 */ 1711 uint16_t etc_sub_qid; 1712 uint16_t etc_sq_head_idx; 1713 } enahw_tx_cdesc_t; 1714 1715 #define ENAHW_TX_CDESC_PHASE_SHIFT 0 1716 #define ENAHW_TX_CDESC_PHASE_MASK BIT(0) 1717 1718 #define ENAHW_TX_CDESC_GET_PHASE(cdesc) \ 1719 ((cdesc)->etc_flags & ENAHW_TX_CDESC_PHASE_MASK) 1720 1721 /* common: ena_eth_io_rx_desc */ 1722 typedef struct enahw_rx_desc { 1723 /* 1724 * The length of the buffer provided by the host, in bytes. 1725 * Use the value of 0 to indicate 64K. 1726 */ 1727 uint16_t erd_length; 1728 uint8_t erd_reserved1; 1729 1730 /* 1731 * 0 Phase (PHASE) 1732 * 1 Reserved Zero 1733 * 2 First (FIRST) 1734 * 1735 * Indicates this is the first descriptor for the frame. 1736 * 1737 * 3 Last (LAST) 1738 * 1739 * Indicates this is the last descriptor for the frame. 1740 * 1741 * 4 Completion Request (COMP_REQ) 1742 * 1743 * Indicates that a completion request should be generated 1744 * for this descriptor. 1745 * 1746 * 7-5 Reserved Zero 1747 */ 1748 uint8_t erd_ctrl; 1749 1750 /* 1751 * 15-0 Request ID 1752 * 16 Reserved 0 1753 */ 1754 uint16_t erd_req_id; 1755 uint16_t erd_reserved2; 1756 1757 /* The physical address of the buffer provided by the host. */ 1758 uint32_t erd_buff_addr_lo; 1759 uint16_t erd_buff_addr_hi; 1760 uint16_t erd_reserved3; 1761 } enahw_rx_desc_t; 1762 1763 #define ENAHW_RX_DESC_PHASE_MASK BIT(0) 1764 #define ENAHW_RX_DESC_FIRST_SHIFT 2 1765 #define ENAHW_RX_DESC_FIRST_MASK BIT(2) 1766 #define ENAHW_RX_DESC_LAST_SHIFT 3 1767 #define ENAHW_RX_DESC_LAST_MASK BIT(3) 1768 #define ENAHW_RX_DESC_COMP_REQ_SHIFT 4 1769 #define ENAHW_RX_DESC_COMP_REQ_MASK BIT(4) 1770 1771 #define ENAHW_RX_DESC_SET_PHASE(desc, val) \ 1772 ((desc)->erd_ctrl |= ((val) & ENAHW_RX_DESC_PHASE_MASK)) 1773 1774 #define ENAHW_RX_DESC_SET_FIRST(desc) \ 1775 ((desc)->erd_ctrl |= ENAHW_RX_DESC_FIRST_MASK) 1776 1777 #define ENAHW_RX_DESC_SET_LAST(desc) \ 1778 ((desc)->erd_ctrl |= ENAHW_RX_DESC_LAST_MASK) 1779 1780 #define ENAHW_RX_DESC_SET_COMP_REQ(desc) \ 1781 ((desc)->erd_ctrl |= ENAHW_RX_DESC_COMP_REQ_MASK) 1782 1783 /* 1784 * Ethernet parsing information is only valid when last == 1. 1785 * 1786 * common: ena_eth_io_rx_cdesc_base 1787 */ 1788 typedef struct enahw_rx_cdesc { 1789 /* 1790 * 4-0 L3 Protocol Number (L3_PROTO) 1791 * 1792 * The L3 protocol type, one of enahw_io_l3_proto_t. 1793 * 1794 * 6-5 (SRC_VLAN_CNT) 1795 * 7 Reserved Zero 1796 * 12-8 L4 Protocol Number (L4_PROTO) 1797 * 13 L3 Checksum Error (L3_CSUM_ERR) 1798 * 1799 * When set either the L3 checksum failed to match or the 1800 * controller didn't attempt to validate the checksum. 1801 * This bit is valid only when L3_PROTO indicates an IPv4 1802 * packet. 1803 * 1804 * 14 L4 Checksum Error (L4_CSUM_ERR) 1805 * 1806 * When set either the L4 checksum failed to match or the 1807 * controller didn't attempt to validate the checksum. 1808 * This bit is valid only when L4_PROTO indicates a 1809 * TCP/UDP packet, IPV4_FRAG is not set, and 1810 * L4_CSUM_CHECKED is set. 1811 * 1812 * 15 IPv4 Fragmented (IPV4_FRAG) 1813 * 16 L4 Checksum Validated (L4_CSUM_CHECKED) 1814 * 1815 * When set it indicates the device attempted to validate 1816 * the L4 checksum. 1817 * 1818 * 23-17 Reserved Zero 1819 * 24 Phase (PHASE) 1820 * 25 (L3_CSUM2) 1821 * 1822 * According to the Linux source this is the "second 1823 * checksum engine result". It's never checked. 1824 * 1825 * 26 First Descriptor Bit (FIRST) 1826 * 1827 * Indicates the first descriptor for the frame. 1828 * 1829 * 27 Last Descriptor Bit (LAST) 1830 * 1831 * Indicates the last descriptor for the frame. 1832 * 1833 * 29-28 Reserved Zero 1834 * 30 Buffer Type (BUFFER) 1835 * 1836 * When enabled indicates this is a data descriptor. 1837 * Otherwse, it is a metadata descriptor. 1838 * 1839 * 31 : reserved31 1840 */ 1841 uint32_t erc_status; 1842 uint16_t erc_length; 1843 uint16_t erc_req_id; 1844 1845 /* 32-bit hash result */ 1846 uint32_t erc_hash; 1847 uint16_t erc_sub_qid; 1848 1849 /* 1850 * The device may choose to offset the start of the header 1851 * data (which implies this value only applies to the first 1852 * descriptor). When and why the device does this is not 1853 * documented in the common code. The most likely case would 1854 * be for IP header alignment. 1855 */ 1856 uint8_t erc_offset; 1857 uint8_t erc_reserved; 1858 } enahw_rx_cdesc_t; 1859 1860 #define ENAHW_RX_CDESC_L3_PROTO_MASK GENMASK(4, 0) 1861 #define ENAHW_RX_CDESC_SRC_VLAN_CNT_SHIFT 5 1862 #define ENAHW_RX_CDESC_SRC_VLAN_CNT_MASK GENMASK(6, 5) 1863 #define ENAHW_RX_CDESC_L4_PROTO_SHIFT 8 1864 #define ENAHW_RX_CDESC_L4_PROTO_MASK GENMASK(12, 8) 1865 #define ENAHW_RX_CDESC_L3_CSUM_ERR_SHIFT 13 1866 #define ENAHW_RX_CDESC_L3_CSUM_ERR_MASK BIT(13) 1867 #define ENAHW_RX_CDESC_L4_CSUM_ERR_SHIFT 14 1868 #define ENAHW_RX_CDESC_L4_CSUM_ERR_MASK BIT(14) 1869 #define ENAHW_RX_CDESC_IPV4_FRAG_SHIFT 15 1870 #define ENAHW_RX_CDESC_IPV4_FRAG_MASK BIT(15) 1871 #define ENAHW_RX_CDESC_L4_CSUM_CHECKED_SHIFT 16 1872 #define ENAHW_RX_CDESC_L4_CSUM_CHECKED_MASK BIT(16) 1873 #define ENAHW_RX_CDESC_PHASE_SHIFT 24 1874 #define ENAHW_RX_CDESC_PHASE_MASK BIT(24) 1875 #define ENAHW_RX_CDESC_L3_CSUM2_SHIFT 25 1876 #define ENAHW_RX_CDESC_L3_CSUM2_MASK BIT(25) 1877 #define ENAHW_RX_CDESC_FIRST_SHIFT 26 1878 #define ENAHW_RX_CDESC_FIRST_MASK BIT(26) 1879 #define ENAHW_RX_CDESC_LAST_SHIFT 27 1880 #define ENAHW_RX_CDESC_LAST_MASK BIT(27) 1881 #define ENAHW_RX_CDESC_BUFFER_SHIFT 30 1882 #define ENAHW_RX_CDESC_BUFFER_MASK BIT(30) 1883 1884 #define ENAHW_RX_CDESC_L3_PROTO(desc) \ 1885 ((desc)->erc_status & ENAHW_RX_CDESC_L3_PROTO_MASK) 1886 1887 #define ENAHW_RX_CDESC_L3_CSUM_ERR(desc) \ 1888 ((((desc)->erc_status & ENAHW_RX_CDESC_L3_CSUM_ERR_MASK) >> \ 1889 ENAHW_RX_CDESC_L3_CSUM_ERR_SHIFT) != 0) 1890 1891 #define ENAHW_RX_CDESC_L4_PROTO(desc) \ 1892 (((desc)->erc_status & ENAHW_RX_CDESC_L4_PROTO_MASK) >> \ 1893 ENAHW_RX_CDESC_L4_PROTO_SHIFT) 1894 1895 #define ENAHW_RX_CDESC_L4_CSUM_CHECKED(desc) \ 1896 ((((desc)->erc_status & ENAHW_RX_CDESC_L4_CSUM_CHECKED_MASK) >> \ 1897 ENAHW_RX_CDESC_L4_CSUM_CHECKED_SHIFT) != 0) 1898 1899 #define ENAHW_RX_CDESC_L4_CSUM_ERR(desc) \ 1900 ((((desc)->erc_status & ENAHW_RX_CDESC_L4_CSUM_ERR_MASK) >> \ 1901 ENAHW_RX_CDESC_L4_CSUM_ERR_SHIFT) != 0) 1902 1903 #define ENAHW_RX_CDESC_PHASE(desc) \ 1904 (((desc)->erc_status & ENAHW_RX_CDESC_PHASE_MASK) >> \ 1905 ENAHW_RX_CDESC_PHASE_SHIFT) 1906 1907 #define ENAHW_RX_CDESC_FIRST(desc) \ 1908 ((((desc)->erc_status & ENAHW_RX_CDESC_FIRST_MASK) >> \ 1909 ENAHW_RX_CDESC_FIRST_SHIFT) == 1) 1910 1911 #define ENAHW_RX_CDESC_LAST(desc) \ 1912 ((((desc)->erc_status & ENAHW_RX_CDESC_LAST_MASK) >> \ 1913 ENAHW_RX_CDESC_LAST_SHIFT) == 1) 1914 1915 /* 1916 * Controls for the interrupt register mapped to each Rx/Tx CQ. 1917 */ 1918 #define ENAHW_REG_INTR_RX_DELAY_MASK GENMASK(14, 0) 1919 #define ENAHW_REG_INTR_TX_DELAY_SHIFT 15 1920 #define ENAHW_REG_INTR_TX_DELAY_MASK GENMASK(29, 15) 1921 #define ENAHW_REG_INTR_UNMASK_SHIFT 30 1922 #define ENAHW_REG_INTR_UNMASK_MASK BIT(30) 1923 1924 #define ENAHW_REG_INTR_UNMASK(val) \ 1925 ((val) |= ENAHW_REG_INTR_UNMASK_MASK) 1926 1927 #define ENAHW_REG_INTR_MASK(val) \ 1928 ((val) &= ~ENAHW_REG_INTR_UNMASK_MASK) 1929 1930 #endif /* _ENA_HW_H */ 1931