1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #ifndef ENA_COM 7 #define ENA_COM 8 9 #include <linux/compiler.h> 10 #include <linux/delay.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/gfp.h> 13 #include <linux/io.h> 14 #include <linux/prefetch.h> 15 #include <linux/sched.h> 16 #include <linux/sizes.h> 17 #include <linux/spinlock.h> 18 #include <linux/types.h> 19 #include <linux/wait.h> 20 #include <linux/netdevice.h> 21 22 #include "ena_common_defs.h" 23 #include "ena_admin_defs.h" 24 #include "ena_eth_io_defs.h" 25 #include "ena_regs_defs.h" 26 27 #undef pr_fmt 28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 29 30 #define ENA_MAX_NUM_IO_QUEUES 128U 31 /* We need to queues for each IO (on for Tx and one for Rx) */ 32 #define ENA_TOTAL_NUM_QUEUES (2 * (ENA_MAX_NUM_IO_QUEUES)) 33 34 #define ENA_MAX_HANDLERS 256 35 36 #define ENA_MAX_PHYS_ADDR_SIZE_BITS 48 37 38 /* Unit in usec */ 39 #define ENA_REG_READ_TIMEOUT 200000 40 41 #define ADMIN_SQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_aq_entry)) 42 #define ADMIN_CQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_acq_entry)) 43 #define ADMIN_AENQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_aenq_entry)) 44 45 #define ENA_CUSTOMER_METRICS_BUFFER_SIZE 512 46 47 /*****************************************************************************/ 48 /*****************************************************************************/ 49 /* ENA adaptive interrupt moderation settings */ 50 51 #define ENA_INTR_INITIAL_TX_INTERVAL_USECS 64 52 #define ENA_INTR_INITIAL_RX_INTERVAL_USECS 20 53 #define ENA_DEFAULT_INTR_DELAY_RESOLUTION 1 54 55 #define ENA_HASH_KEY_SIZE 40 56 57 #define ENA_HW_HINTS_NO_TIMEOUT 0xFFFF 58 59 #define ENA_FEATURE_MAX_QUEUE_EXT_VER 1 60 61 struct ena_llq_configurations { 62 enum ena_admin_llq_header_location llq_header_location; 63 enum ena_admin_llq_ring_entry_size llq_ring_entry_size; 64 enum ena_admin_llq_stride_ctrl llq_stride_ctrl; 65 enum ena_admin_llq_num_descs_before_header llq_num_decs_before_header; 66 u16 llq_ring_entry_size_value; 67 }; 68 69 enum queue_direction { 70 ENA_COM_IO_QUEUE_DIRECTION_TX, 71 ENA_COM_IO_QUEUE_DIRECTION_RX 72 }; 73 74 struct ena_com_buf { 75 dma_addr_t paddr; /**< Buffer physical address */ 76 u16 len; /**< Buffer length in bytes */ 77 }; 78 79 struct ena_com_rx_buf_info { 80 u16 len; 81 u16 req_id; 82 }; 83 84 struct ena_com_io_desc_addr { 85 u8 __iomem *pbuf_dev_addr; /* LLQ address */ 86 u8 *virt_addr; 87 dma_addr_t phys_addr; 88 }; 89 90 struct ena_com_tx_meta { 91 u16 mss; 92 u16 l3_hdr_len; 93 u16 l3_hdr_offset; 94 u16 l4_hdr_len; /* In words */ 95 }; 96 97 struct ena_com_llq_info { 98 u16 header_location_ctrl; 99 u16 desc_stride_ctrl; 100 u16 desc_list_entry_size_ctrl; 101 u16 desc_list_entry_size; 102 u16 descs_num_before_header; 103 u16 descs_per_entry; 104 u16 max_entries_in_tx_burst; 105 bool disable_meta_caching; 106 }; 107 108 struct ena_com_io_cq { 109 struct ena_com_io_desc_addr cdesc_addr; 110 111 /* Interrupt unmask register */ 112 u32 __iomem *unmask_reg; 113 114 /* numa configuration register (for TPH) */ 115 u32 __iomem *numa_node_cfg_reg; 116 117 /* The value to write to the above register to unmask 118 * the interrupt of this queue 119 */ 120 u32 msix_vector ____cacheline_aligned; 121 122 enum queue_direction direction; 123 124 /* holds the number of cdesc of the current packet */ 125 u16 cur_rx_pkt_cdesc_count; 126 /* save the first cdesc idx of the current packet */ 127 u16 cur_rx_pkt_cdesc_start_idx; 128 129 u16 q_depth; 130 /* Caller qid */ 131 u16 qid; 132 133 /* Device queue index */ 134 u16 idx; 135 u16 head; 136 u8 phase; 137 u8 cdesc_entry_size_in_bytes; 138 139 } ____cacheline_aligned; 140 141 struct ena_com_io_bounce_buffer_control { 142 u8 *base_buffer; 143 u16 next_to_use; 144 u16 buffer_size; 145 u16 buffers_num; /* Must be a power of 2 */ 146 }; 147 148 /* This struct is to keep tracking the current location of the next llq entry */ 149 struct ena_com_llq_pkt_ctrl { 150 u8 *curr_bounce_buf; 151 u16 idx; 152 u16 descs_left_in_line; 153 }; 154 155 struct ena_com_io_sq { 156 struct ena_com_io_desc_addr desc_addr; 157 158 u32 __iomem *db_addr; 159 160 enum queue_direction direction; 161 enum ena_admin_placement_policy_type mem_queue_type; 162 163 bool disable_meta_caching; 164 165 u32 msix_vector; 166 struct ena_com_tx_meta cached_tx_meta; 167 struct ena_com_llq_info llq_info; 168 struct ena_com_llq_pkt_ctrl llq_buf_ctrl; 169 struct ena_com_io_bounce_buffer_control bounce_buf_ctrl; 170 171 u16 q_depth; 172 u16 qid; 173 174 u16 idx; 175 u16 tail; 176 u16 next_to_comp; 177 u16 llq_last_copy_tail; 178 u32 tx_max_header_size; 179 u8 phase; 180 u8 desc_entry_size; 181 u8 dma_addr_bits; 182 u16 entries_in_tx_burst_left; 183 } ____cacheline_aligned; 184 185 struct ena_com_admin_cq { 186 struct ena_admin_acq_entry *entries; 187 dma_addr_t dma_addr; 188 189 u16 head; 190 u8 phase; 191 }; 192 193 struct ena_com_admin_sq { 194 struct ena_admin_aq_entry *entries; 195 dma_addr_t dma_addr; 196 197 u32 __iomem *db_addr; 198 199 u16 head; 200 u16 tail; 201 u8 phase; 202 203 }; 204 205 struct ena_com_stats_admin { 206 u64 aborted_cmd; 207 u64 submitted_cmd; 208 u64 completed_cmd; 209 u64 out_of_space; 210 u64 no_completion; 211 }; 212 213 struct ena_com_admin_queue { 214 void *q_dmadev; 215 struct ena_com_dev *ena_dev; 216 spinlock_t q_lock; /* spinlock for the admin queue */ 217 218 struct ena_comp_ctx *comp_ctx; 219 u32 completion_timeout; 220 u16 q_depth; 221 struct ena_com_admin_cq cq; 222 struct ena_com_admin_sq sq; 223 224 /* Indicate if the admin queue should poll for completion */ 225 bool polling; 226 227 u16 curr_cmd_id; 228 229 /* Indicate that the ena was initialized and can 230 * process new admin commands 231 */ 232 bool running_state; 233 234 /* Count the number of outstanding admin commands */ 235 atomic_t outstanding_cmds; 236 237 struct ena_com_stats_admin stats; 238 }; 239 240 struct ena_aenq_handlers; 241 242 struct ena_com_aenq { 243 u16 head; 244 u8 phase; 245 struct ena_admin_aenq_entry *entries; 246 dma_addr_t dma_addr; 247 u16 q_depth; 248 struct ena_aenq_handlers *aenq_handlers; 249 }; 250 251 struct ena_com_mmio_read { 252 struct ena_admin_ena_mmio_req_read_less_resp *read_resp; 253 dma_addr_t read_resp_dma_addr; 254 u32 reg_read_to; /* in us */ 255 u16 seq_num; 256 bool readless_supported; 257 /* spin lock to ensure a single outstanding read */ 258 spinlock_t lock; 259 }; 260 261 struct ena_rss { 262 /* Indirect table */ 263 u16 *host_rss_ind_tbl; 264 struct ena_admin_rss_ind_table_entry *rss_ind_tbl; 265 dma_addr_t rss_ind_tbl_dma_addr; 266 u16 tbl_log_size; 267 268 /* Hash key */ 269 enum ena_admin_hash_functions hash_func; 270 struct ena_admin_feature_rss_flow_hash_control *hash_key; 271 dma_addr_t hash_key_dma_addr; 272 u32 hash_init_val; 273 274 /* Flow Control */ 275 struct ena_admin_feature_rss_hash_control *hash_ctrl; 276 dma_addr_t hash_ctrl_dma_addr; 277 278 }; 279 280 struct ena_customer_metrics { 281 /* in correlation with ENA_ADMIN_CUSTOMER_METRICS_SUPPORT_MASK 282 * and ena_admin_customer_metrics_id 283 */ 284 u64 supported_metrics; 285 dma_addr_t buffer_dma_addr; 286 void *buffer_virt_addr; 287 u32 buffer_len; 288 }; 289 290 struct ena_host_attribute { 291 /* Debug area */ 292 u8 *debug_area_virt_addr; 293 dma_addr_t debug_area_dma_addr; 294 u32 debug_area_size; 295 296 /* Host information */ 297 struct ena_admin_host_info *host_info; 298 dma_addr_t host_info_dma_addr; 299 }; 300 301 /* Each ena_dev is a PCI function. */ 302 struct ena_com_dev { 303 struct ena_com_admin_queue admin_queue; 304 struct ena_com_aenq aenq; 305 struct ena_com_io_cq io_cq_queues[ENA_TOTAL_NUM_QUEUES]; 306 struct ena_com_io_sq io_sq_queues[ENA_TOTAL_NUM_QUEUES]; 307 u8 __iomem *reg_bar; 308 void __iomem *mem_bar; 309 void *dmadev; 310 struct net_device *net_device; 311 312 enum ena_admin_placement_policy_type tx_mem_queue_type; 313 u32 tx_max_header_size; 314 u16 stats_func; /* Selected function for extended statistic dump */ 315 u16 stats_queue; /* Selected queue for extended statistic dump */ 316 317 u32 ena_min_poll_delay_us; 318 319 struct ena_com_mmio_read mmio_read; 320 321 struct ena_rss rss; 322 u32 supported_features; 323 u32 capabilities; 324 u32 dma_addr_bits; 325 326 struct ena_host_attribute host_attr; 327 bool adaptive_coalescing; 328 u16 intr_delay_resolution; 329 330 /* interrupt moderation intervals are in usec divided by 331 * intr_delay_resolution, which is supplied by the device. 332 */ 333 u32 intr_moder_tx_interval; 334 u32 intr_moder_rx_interval; 335 336 struct ena_intr_moder_entry *intr_moder_tbl; 337 338 struct ena_com_llq_info llq_info; 339 340 struct ena_customer_metrics customer_metrics; 341 }; 342 343 struct ena_com_dev_get_features_ctx { 344 struct ena_admin_queue_feature_desc max_queues; 345 struct ena_admin_queue_ext_feature_desc max_queue_ext; 346 struct ena_admin_device_attr_feature_desc dev_attr; 347 struct ena_admin_feature_aenq_desc aenq; 348 struct ena_admin_feature_offload_desc offload; 349 struct ena_admin_ena_hw_hints hw_hints; 350 struct ena_admin_feature_llq_desc llq; 351 }; 352 353 struct ena_com_create_io_ctx { 354 enum ena_admin_placement_policy_type mem_queue_type; 355 enum queue_direction direction; 356 int numa_node; 357 u32 msix_vector; 358 u16 queue_size; 359 u16 qid; 360 }; 361 362 typedef void (*ena_aenq_handler)(void *data, 363 struct ena_admin_aenq_entry *aenq_e); 364 365 /* Holds aenq handlers. Indexed by AENQ event group */ 366 struct ena_aenq_handlers { 367 ena_aenq_handler handlers[ENA_MAX_HANDLERS]; 368 ena_aenq_handler unimplemented_handler; 369 }; 370 371 /*****************************************************************************/ 372 /*****************************************************************************/ 373 374 /* ena_com_mmio_reg_read_request_init - Init the mmio reg read mechanism 375 * @ena_dev: ENA communication layer struct 376 * 377 * Initialize the register read mechanism. 378 * 379 * @note: This method must be the first stage in the initialization sequence. 380 * 381 * @return - 0 on success, negative value on failure. 382 */ 383 int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev); 384 385 /* ena_com_set_mmio_read_mode - Enable/disable the indirect mmio reg read mechanism 386 * @ena_dev: ENA communication layer struct 387 * @readless_supported: readless mode (enable/disable) 388 */ 389 void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, 390 bool readless_supported); 391 392 /* ena_com_mmio_reg_read_request_write_dev_addr - Write the mmio reg read return 393 * value physical address. 394 * @ena_dev: ENA communication layer struct 395 */ 396 void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev); 397 398 /* ena_com_mmio_reg_read_request_destroy - Destroy the mmio reg read mechanism 399 * @ena_dev: ENA communication layer struct 400 */ 401 void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev); 402 403 /* ena_com_admin_init - Init the admin and the async queues 404 * @ena_dev: ENA communication layer struct 405 * @aenq_handlers: Those handlers to be called upon event. 406 * 407 * Initialize the admin submission and completion queues. 408 * Initialize the asynchronous events notification queues. 409 * 410 * @return - 0 on success, negative value on failure. 411 */ 412 int ena_com_admin_init(struct ena_com_dev *ena_dev, 413 struct ena_aenq_handlers *aenq_handlers); 414 415 /* ena_com_admin_destroy - Destroy the admin and the async events queues. 416 * @ena_dev: ENA communication layer struct 417 * 418 * @note: Before calling this method, the caller must validate that the device 419 * won't send any additional admin completions/aenq. 420 * To achieve that, a FLR is recommended. 421 */ 422 void ena_com_admin_destroy(struct ena_com_dev *ena_dev); 423 424 /* ena_com_dev_reset - Perform device FLR to the device. 425 * @ena_dev: ENA communication layer struct 426 * @reset_reason: Specify what is the trigger for the reset in case of an error. 427 * 428 * @return - 0 on success, negative value on failure. 429 */ 430 int ena_com_dev_reset(struct ena_com_dev *ena_dev, 431 enum ena_regs_reset_reason_types reset_reason); 432 433 /* ena_com_create_io_queue - Create io queue. 434 * @ena_dev: ENA communication layer struct 435 * @ctx - create context structure 436 * 437 * Create the submission and the completion queues. 438 * 439 * @return - 0 on success, negative value on failure. 440 */ 441 int ena_com_create_io_queue(struct ena_com_dev *ena_dev, 442 struct ena_com_create_io_ctx *ctx); 443 444 /* ena_com_destroy_io_queue - Destroy IO queue with the queue id - qid. 445 * @ena_dev: ENA communication layer struct 446 * @qid - the caller virtual queue id. 447 */ 448 void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid); 449 450 /* ena_com_get_io_handlers - Return the io queue handlers 451 * @ena_dev: ENA communication layer struct 452 * @qid - the caller virtual queue id. 453 * @io_sq - IO submission queue handler 454 * @io_cq - IO completion queue handler. 455 * 456 * @return - 0 on success, negative value on failure. 457 */ 458 int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid, 459 struct ena_com_io_sq **io_sq, 460 struct ena_com_io_cq **io_cq); 461 462 /* ena_com_admin_aenq_enable - ENAble asynchronous event notifications 463 * @ena_dev: ENA communication layer struct 464 * 465 * After this method, aenq event can be received via AENQ. 466 */ 467 void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev); 468 469 /* ena_com_set_admin_running_state - Set the state of the admin queue 470 * @ena_dev: ENA communication layer struct 471 * 472 * Change the state of the admin queue (enable/disable) 473 */ 474 void ena_com_set_admin_running_state(struct ena_com_dev *ena_dev, bool state); 475 476 /* ena_com_get_admin_running_state - Get the admin queue state 477 * @ena_dev: ENA communication layer struct 478 * 479 * Retrieve the state of the admin queue (enable/disable) 480 * 481 * @return - current polling mode (enable/disable) 482 */ 483 bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev); 484 485 /* ena_com_set_admin_polling_mode - Set the admin completion queue polling mode 486 * @ena_dev: ENA communication layer struct 487 * @polling: ENAble/Disable polling mode 488 * 489 * Set the admin completion mode. 490 */ 491 void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling); 492 493 /* ena_com_admin_q_comp_intr_handler - admin queue interrupt handler 494 * @ena_dev: ENA communication layer struct 495 * 496 * This method goes over the admin completion queue and wakes up all the pending 497 * threads that wait on the commands wait event. 498 * 499 * @note: Should be called after MSI-X interrupt. 500 */ 501 void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev); 502 503 /* ena_com_aenq_intr_handler - AENQ interrupt handler 504 * @ena_dev: ENA communication layer struct 505 * 506 * This method goes over the async event notification queue and calls the proper 507 * aenq handler. 508 */ 509 void ena_com_aenq_intr_handler(struct ena_com_dev *ena_dev, void *data); 510 511 /* ena_com_abort_admin_commands - Abort all the outstanding admin commands. 512 * @ena_dev: ENA communication layer struct 513 * 514 * This method aborts all the outstanding admin commands. 515 * The caller should then call ena_com_wait_for_abort_completion to make sure 516 * all the commands were completed. 517 */ 518 void ena_com_abort_admin_commands(struct ena_com_dev *ena_dev); 519 520 /* ena_com_wait_for_abort_completion - Wait for admin commands abort. 521 * @ena_dev: ENA communication layer struct 522 * 523 * This method waits until all the outstanding admin commands are completed. 524 */ 525 void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev); 526 527 /* ena_com_validate_version - Validate the device parameters 528 * @ena_dev: ENA communication layer struct 529 * 530 * This method verifies the device parameters are the same as the saved 531 * parameters in ena_dev. 532 * This method is useful after device reset, to validate the device mac address 533 * and the device offloads are the same as before the reset. 534 * 535 * @return - 0 on success negative value otherwise. 536 */ 537 int ena_com_validate_version(struct ena_com_dev *ena_dev); 538 539 /* ena_com_get_link_params - Retrieve physical link parameters. 540 * @ena_dev: ENA communication layer struct 541 * @resp: Link parameters 542 * 543 * Retrieve the physical link parameters, 544 * like speed, auto-negotiation and full duplex support. 545 * 546 * @return - 0 on Success negative value otherwise. 547 */ 548 int ena_com_get_link_params(struct ena_com_dev *ena_dev, 549 struct ena_admin_get_feat_resp *resp); 550 551 /* ena_com_get_dma_width - Retrieve physical dma address width the device 552 * supports. 553 * @ena_dev: ENA communication layer struct 554 * 555 * Retrieve the maximum physical address bits the device can handle. 556 * 557 * @return: > 0 on Success and negative value otherwise. 558 */ 559 int ena_com_get_dma_width(struct ena_com_dev *ena_dev); 560 561 /* ena_com_set_aenq_config - Set aenq groups configurations 562 * @ena_dev: ENA communication layer struct 563 * @groups flag: bit fields flags of enum ena_admin_aenq_group. 564 * 565 * Configure which aenq event group the driver would like to receive. 566 * 567 * @return: 0 on Success and negative value otherwise. 568 */ 569 int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag); 570 571 /* ena_com_get_dev_attr_feat - Get device features 572 * @ena_dev: ENA communication layer struct 573 * @get_feat_ctx: returned context that contain the get features. 574 * 575 * @return: 0 on Success and negative value otherwise. 576 */ 577 int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, 578 struct ena_com_dev_get_features_ctx *get_feat_ctx); 579 580 /* ena_com_get_eni_stats - Get extended network interface statistics 581 * @ena_dev: ENA communication layer struct 582 * @stats: stats return value 583 * 584 * @return: 0 on Success and negative value otherwise. 585 */ 586 int ena_com_get_eni_stats(struct ena_com_dev *ena_dev, 587 struct ena_admin_eni_stats *stats); 588 589 /* ena_com_get_ena_srd_info - Get ENA SRD network interface statistics 590 * @ena_dev: ENA communication layer struct 591 * @info: ena srd stats and flags 592 * 593 * @return: 0 on Success and negative value otherwise. 594 */ 595 int ena_com_get_ena_srd_info(struct ena_com_dev *ena_dev, 596 struct ena_admin_ena_srd_info *info); 597 598 /* ena_com_get_customer_metrics - Get customer metrics for network interface 599 * @ena_dev: ENA communication layer struct 600 * @buffer: buffer for returned customer metrics 601 * @len: size of the buffer 602 * 603 * @return: 0 on Success and negative value otherwise. 604 */ 605 int ena_com_get_customer_metrics(struct ena_com_dev *ena_dev, char *buffer, u32 len); 606 607 /* ena_com_set_dev_mtu - Configure the device mtu. 608 * @ena_dev: ENA communication layer struct 609 * @mtu: mtu value 610 * 611 * @return: 0 on Success and negative value otherwise. 612 */ 613 int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, u32 mtu); 614 615 /* ena_com_rss_init - Init RSS 616 * @ena_dev: ENA communication layer struct 617 * @log_size: indirection log size 618 * 619 * Allocate RSS/RFS resources. 620 * The caller then can configure rss using ena_com_set_hash_function, 621 * ena_com_set_hash_ctrl and ena_com_indirect_table_set. 622 * 623 * @return: 0 on Success and negative value otherwise. 624 */ 625 int ena_com_rss_init(struct ena_com_dev *ena_dev, u16 log_size); 626 627 /* ena_com_rss_destroy - Destroy rss 628 * @ena_dev: ENA communication layer struct 629 * 630 * Free all the RSS/RFS resources. 631 */ 632 void ena_com_rss_destroy(struct ena_com_dev *ena_dev); 633 634 /* ena_com_get_current_hash_function - Get RSS hash function 635 * @ena_dev: ENA communication layer struct 636 * 637 * Return the current hash function. 638 * @return: 0 or one of the ena_admin_hash_functions values. 639 */ 640 int ena_com_get_current_hash_function(struct ena_com_dev *ena_dev); 641 642 /* ena_com_fill_hash_function - Fill RSS hash function 643 * @ena_dev: ENA communication layer struct 644 * @func: The hash function (Toeplitz or crc) 645 * @key: Hash key (for toeplitz hash) 646 * @key_len: key length (max length 10 DW) 647 * @init_val: initial value for the hash function 648 * 649 * Fill the ena_dev resources with the desire hash function, hash key, key_len 650 * and key initial value (if needed by the hash function). 651 * To flush the key into the device the caller should call 652 * ena_com_set_hash_function. 653 * 654 * @return: 0 on Success and negative value otherwise. 655 */ 656 int ena_com_fill_hash_function(struct ena_com_dev *ena_dev, 657 enum ena_admin_hash_functions func, 658 const u8 *key, u16 key_len, u32 init_val); 659 660 /* ena_com_set_hash_function - Flush the hash function and it dependencies to 661 * the device. 662 * @ena_dev: ENA communication layer struct 663 * 664 * Flush the hash function and it dependencies (key, key length and 665 * initial value) if needed. 666 * 667 * @note: Prior to this method the caller should call ena_com_fill_hash_function 668 * 669 * @return: 0 on Success and negative value otherwise. 670 */ 671 int ena_com_set_hash_function(struct ena_com_dev *ena_dev); 672 673 /* ena_com_get_hash_function - Retrieve the hash function from the device. 674 * @ena_dev: ENA communication layer struct 675 * @func: hash function 676 * 677 * Retrieve the hash function from the device. 678 * 679 * @note: If the caller called ena_com_fill_hash_function but didn't flush 680 * it to the device, the new configuration will be lost. 681 * 682 * @return: 0 on Success and negative value otherwise. 683 */ 684 int ena_com_get_hash_function(struct ena_com_dev *ena_dev, 685 enum ena_admin_hash_functions *func); 686 687 /* ena_com_get_hash_key - Retrieve the hash key 688 * @ena_dev: ENA communication layer struct 689 * @key: hash key 690 * 691 * Retrieve the hash key. 692 * 693 * @note: If the caller called ena_com_fill_hash_key but didn't flush 694 * it to the device, the new configuration will be lost. 695 * 696 * @return: 0 on Success and negative value otherwise. 697 */ 698 int ena_com_get_hash_key(struct ena_com_dev *ena_dev, u8 *key); 699 /* ena_com_fill_hash_ctrl - Fill RSS hash control 700 * @ena_dev: ENA communication layer struct. 701 * @proto: The protocol to configure. 702 * @hash_fields: bit mask of ena_admin_flow_hash_fields 703 * 704 * Fill the ena_dev resources with the desire hash control (the ethernet 705 * fields that take part of the hash) for a specific protocol. 706 * To flush the hash control to the device, the caller should call 707 * ena_com_set_hash_ctrl. 708 * 709 * @return: 0 on Success and negative value otherwise. 710 */ 711 int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev, 712 enum ena_admin_flow_hash_proto proto, 713 u16 hash_fields); 714 715 /* ena_com_set_hash_ctrl - Flush the hash control resources to the device. 716 * @ena_dev: ENA communication layer struct 717 * 718 * Flush the hash control (the ethernet fields that take part of the hash) 719 * 720 * @note: Prior to this method the caller should call ena_com_fill_hash_ctrl. 721 * 722 * @return: 0 on Success and negative value otherwise. 723 */ 724 int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev); 725 726 /* ena_com_get_hash_ctrl - Retrieve the hash control from the device. 727 * @ena_dev: ENA communication layer struct 728 * @proto: The protocol to retrieve. 729 * @fields: bit mask of ena_admin_flow_hash_fields. 730 * 731 * Retrieve the hash control from the device. 732 * 733 * @note: If the caller called ena_com_fill_hash_ctrl but didn't flush 734 * it to the device, the new configuration will be lost. 735 * 736 * @return: 0 on Success and negative value otherwise. 737 */ 738 int ena_com_get_hash_ctrl(struct ena_com_dev *ena_dev, 739 enum ena_admin_flow_hash_proto proto, 740 u16 *fields); 741 742 /* ena_com_set_default_hash_ctrl - Set the hash control to a default 743 * configuration. 744 * @ena_dev: ENA communication layer struct 745 * 746 * Fill the ena_dev resources with the default hash control configuration. 747 * To flush the hash control to the device, the caller should call 748 * ena_com_set_hash_ctrl. 749 * 750 * @return: 0 on Success and negative value otherwise. 751 */ 752 int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev); 753 754 /* ena_com_indirect_table_fill_entry - Fill a single entry in the RSS 755 * indirection table 756 * @ena_dev: ENA communication layer struct. 757 * @entry_idx - indirection table entry. 758 * @entry_value - redirection value 759 * 760 * Fill a single entry of the RSS indirection table in the ena_dev resources. 761 * To flush the indirection table to the device, the called should call 762 * ena_com_indirect_table_set. 763 * 764 * @return: 0 on Success and negative value otherwise. 765 */ 766 int ena_com_indirect_table_fill_entry(struct ena_com_dev *ena_dev, 767 u16 entry_idx, u16 entry_value); 768 769 /* ena_com_indirect_table_set - Flush the indirection table to the device. 770 * @ena_dev: ENA communication layer struct 771 * 772 * Flush the indirection hash control to the device. 773 * Prior to this method the caller should call ena_com_indirect_table_fill_entry 774 * 775 * @return: 0 on Success and negative value otherwise. 776 */ 777 int ena_com_indirect_table_set(struct ena_com_dev *ena_dev); 778 779 /* ena_com_indirect_table_get - Retrieve the indirection table from the device. 780 * @ena_dev: ENA communication layer struct 781 * @ind_tbl: indirection table 782 * 783 * Retrieve the RSS indirection table from the device. 784 * 785 * @note: If the caller called ena_com_indirect_table_fill_entry but didn't flush 786 * it to the device, the new configuration will be lost. 787 * 788 * @return: 0 on Success and negative value otherwise. 789 */ 790 int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl); 791 792 /* ena_com_allocate_host_info - Allocate host info resources. 793 * @ena_dev: ENA communication layer struct 794 * 795 * @return: 0 on Success and negative value otherwise. 796 */ 797 int ena_com_allocate_host_info(struct ena_com_dev *ena_dev); 798 799 /* ena_com_allocate_debug_area - Allocate debug area. 800 * @ena_dev: ENA communication layer struct 801 * @debug_area_size - debug area size. 802 * 803 * @return: 0 on Success and negative value otherwise. 804 */ 805 int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev, 806 u32 debug_area_size); 807 808 /* ena_com_allocate_customer_metrics_buffer - Allocate customer metrics resources. 809 * @ena_dev: ENA communication layer struct 810 * 811 * @return: 0 on Success and negative value otherwise. 812 */ 813 int ena_com_allocate_customer_metrics_buffer(struct ena_com_dev *ena_dev); 814 815 /* ena_com_delete_debug_area - Free the debug area resources. 816 * @ena_dev: ENA communication layer struct 817 * 818 * Free the allocated debug area. 819 */ 820 void ena_com_delete_debug_area(struct ena_com_dev *ena_dev); 821 822 /* ena_com_delete_host_info - Free the host info resources. 823 * @ena_dev: ENA communication layer struct 824 * 825 * Free the allocated host info. 826 */ 827 void ena_com_delete_host_info(struct ena_com_dev *ena_dev); 828 829 /* ena_com_delete_customer_metrics_buffer - Free the customer metrics resources. 830 * @ena_dev: ENA communication layer struct 831 * 832 * Free the allocated customer metrics area. 833 */ 834 void ena_com_delete_customer_metrics_buffer(struct ena_com_dev *ena_dev); 835 836 /* ena_com_set_host_attributes - Update the device with the host 837 * attributes (debug area and host info) base address. 838 * @ena_dev: ENA communication layer struct 839 * 840 * @return: 0 on Success and negative value otherwise. 841 */ 842 int ena_com_set_host_attributes(struct ena_com_dev *ena_dev); 843 844 /* ena_com_create_io_cq - Create io completion queue. 845 * @ena_dev: ENA communication layer struct 846 * @io_cq - io completion queue handler 847 848 * Create IO completion queue. 849 * 850 * @return - 0 on success, negative value on failure. 851 */ 852 int ena_com_create_io_cq(struct ena_com_dev *ena_dev, 853 struct ena_com_io_cq *io_cq); 854 855 /* ena_com_destroy_io_cq - Destroy io completion queue. 856 * @ena_dev: ENA communication layer struct 857 * @io_cq - io completion queue handler 858 859 * Destroy IO completion queue. 860 * 861 * @return - 0 on success, negative value on failure. 862 */ 863 int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev, 864 struct ena_com_io_cq *io_cq); 865 866 /* ena_com_execute_admin_command - Execute admin command 867 * @admin_queue: admin queue. 868 * @cmd: the admin command to execute. 869 * @cmd_size: the command size. 870 * @cmd_completion: command completion return value. 871 * @cmd_comp_size: command completion size. 872 873 * Submit an admin command and then wait until the device returns a 874 * completion. 875 * The completion will be copied into cmd_comp. 876 * 877 * @return - 0 on success, negative value on failure. 878 */ 879 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue, 880 struct ena_admin_aq_entry *cmd, 881 size_t cmd_size, 882 struct ena_admin_acq_entry *cmd_comp, 883 size_t cmd_comp_size); 884 885 /* ena_com_init_interrupt_moderation - Init interrupt moderation 886 * @ena_dev: ENA communication layer struct 887 * 888 * @return - 0 on success, negative value on failure. 889 */ 890 int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev); 891 892 /* ena_com_interrupt_moderation_supported - Return if interrupt moderation 893 * capability is supported by the device. 894 * 895 * @return - supported or not. 896 */ 897 bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev); 898 899 /* ena_com_update_nonadaptive_moderation_interval_tx - Update the 900 * non-adaptive interval in Tx direction. 901 * @ena_dev: ENA communication layer struct 902 * @tx_coalesce_usecs: Interval in usec. 903 * 904 * @return - 0 on success, negative value on failure. 905 */ 906 int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev, 907 u32 tx_coalesce_usecs); 908 909 /* ena_com_update_nonadaptive_moderation_interval_rx - Update the 910 * non-adaptive interval in Rx direction. 911 * @ena_dev: ENA communication layer struct 912 * @rx_coalesce_usecs: Interval in usec. 913 * 914 * @return - 0 on success, negative value on failure. 915 */ 916 int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev, 917 u32 rx_coalesce_usecs); 918 919 /* ena_com_get_nonadaptive_moderation_interval_tx - Retrieve the 920 * non-adaptive interval in Tx direction. 921 * @ena_dev: ENA communication layer struct 922 * 923 * @return - interval in usec 924 */ 925 unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev); 926 927 /* ena_com_get_nonadaptive_moderation_interval_rx - Retrieve the 928 * non-adaptive interval in Rx direction. 929 * @ena_dev: ENA communication layer struct 930 * 931 * @return - interval in usec 932 */ 933 unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev); 934 935 /* ena_com_config_dev_mode - Configure the placement policy of the device. 936 * @ena_dev: ENA communication layer struct 937 * @llq_features: LLQ feature descriptor, retrieve via 938 * ena_com_get_dev_attr_feat. 939 * @ena_llq_config: The default driver LLQ parameters configurations 940 */ 941 int ena_com_config_dev_mode(struct ena_com_dev *ena_dev, 942 struct ena_admin_feature_llq_desc *llq_features, 943 struct ena_llq_configurations *llq_default_config); 944 945 /* ena_com_io_sq_to_ena_dev - Extract ena_com_dev using contained field io_sq. 946 * @io_sq: IO submit queue struct 947 * 948 * @return - ena_com_dev struct extracted from io_sq 949 */ 950 static inline struct ena_com_dev *ena_com_io_sq_to_ena_dev(struct ena_com_io_sq *io_sq) 951 { 952 return container_of(io_sq, struct ena_com_dev, io_sq_queues[io_sq->qid]); 953 } 954 955 /* ena_com_io_cq_to_ena_dev - Extract ena_com_dev using contained field io_cq. 956 * @io_sq: IO submit queue struct 957 * 958 * @return - ena_com_dev struct extracted from io_sq 959 */ 960 static inline struct ena_com_dev *ena_com_io_cq_to_ena_dev(struct ena_com_io_cq *io_cq) 961 { 962 return container_of(io_cq, struct ena_com_dev, io_cq_queues[io_cq->qid]); 963 } 964 965 static inline bool ena_com_get_adaptive_moderation_enabled(struct ena_com_dev *ena_dev) 966 { 967 return ena_dev->adaptive_coalescing; 968 } 969 970 static inline void ena_com_enable_adaptive_moderation(struct ena_com_dev *ena_dev) 971 { 972 ena_dev->adaptive_coalescing = true; 973 } 974 975 static inline void ena_com_disable_adaptive_moderation(struct ena_com_dev *ena_dev) 976 { 977 ena_dev->adaptive_coalescing = false; 978 } 979 980 /* ena_com_get_cap - query whether device supports a capability. 981 * @ena_dev: ENA communication layer struct 982 * @cap_id: enum value representing the capability 983 * 984 * @return - true if capability is supported or false otherwise 985 */ 986 static inline bool ena_com_get_cap(struct ena_com_dev *ena_dev, 987 enum ena_admin_aq_caps_id cap_id) 988 { 989 return !!(ena_dev->capabilities & BIT(cap_id)); 990 } 991 992 /* ena_com_get_customer_metric_support - query whether device supports a given customer metric. 993 * @ena_dev: ENA communication layer struct 994 * @metric_id: enum value representing the customer metric 995 * 996 * @return - true if customer metric is supported or false otherwise 997 */ 998 static inline bool ena_com_get_customer_metric_support(struct ena_com_dev *ena_dev, 999 enum ena_admin_customer_metrics_id metric_id) 1000 { 1001 return !!(ena_dev->customer_metrics.supported_metrics & BIT(metric_id)); 1002 } 1003 1004 /* ena_com_get_customer_metric_count - return the number of supported customer metrics. 1005 * @ena_dev: ENA communication layer struct 1006 * 1007 * @return - the number of supported customer metrics 1008 */ 1009 static inline int ena_com_get_customer_metric_count(struct ena_com_dev *ena_dev) 1010 { 1011 return hweight64(ena_dev->customer_metrics.supported_metrics); 1012 } 1013 1014 /* ena_com_update_intr_reg - Prepare interrupt register 1015 * @intr_reg: interrupt register to update. 1016 * @rx_delay_interval: Rx interval in usecs 1017 * @tx_delay_interval: Tx interval in usecs 1018 * @unmask: unmask enable/disable 1019 * 1020 * Prepare interrupt update register with the supplied parameters. 1021 */ 1022 static inline void ena_com_update_intr_reg(struct ena_eth_io_intr_reg *intr_reg, 1023 u32 rx_delay_interval, 1024 u32 tx_delay_interval, 1025 bool unmask) 1026 { 1027 intr_reg->intr_control = 0; 1028 intr_reg->intr_control |= rx_delay_interval & 1029 ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK; 1030 1031 intr_reg->intr_control |= 1032 (tx_delay_interval << ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT) 1033 & ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK; 1034 1035 if (unmask) 1036 intr_reg->intr_control |= ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK; 1037 } 1038 1039 static inline u8 *ena_com_get_next_bounce_buffer(struct ena_com_io_bounce_buffer_control *bounce_buf_ctrl) 1040 { 1041 u16 size, buffers_num; 1042 u8 *buf; 1043 1044 size = bounce_buf_ctrl->buffer_size; 1045 buffers_num = bounce_buf_ctrl->buffers_num; 1046 1047 buf = bounce_buf_ctrl->base_buffer + 1048 (bounce_buf_ctrl->next_to_use++ & (buffers_num - 1)) * size; 1049 1050 prefetchw(bounce_buf_ctrl->base_buffer + 1051 (bounce_buf_ctrl->next_to_use & (buffers_num - 1)) * size); 1052 1053 return buf; 1054 } 1055 1056 #endif /* !(ENA_COM) */ 1057