1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #ifndef ENA_COM 7 #define ENA_COM 8 9 #include <linux/compiler.h> 10 #include <linux/delay.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/gfp.h> 13 #include <linux/io.h> 14 #include <linux/prefetch.h> 15 #include <linux/sched.h> 16 #include <linux/sizes.h> 17 #include <linux/spinlock.h> 18 #include <linux/types.h> 19 #include <linux/wait.h> 20 #include <linux/netdevice.h> 21 22 #include "ena_common_defs.h" 23 #include "ena_admin_defs.h" 24 #include "ena_eth_io_defs.h" 25 #include "ena_regs_defs.h" 26 27 #undef pr_fmt 28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 29 30 #define ENA_MAX_NUM_IO_QUEUES 128U 31 /* We need to queues for each IO (on for Tx and one for Rx) */ 32 #define ENA_TOTAL_NUM_QUEUES (2 * (ENA_MAX_NUM_IO_QUEUES)) 33 34 #define ENA_MAX_HANDLERS 256 35 36 #define ENA_MAX_PHYS_ADDR_SIZE_BITS 48 37 38 /* Unit in usec */ 39 #define ENA_REG_READ_TIMEOUT 200000 40 41 #define ADMIN_SQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_aq_entry)) 42 #define ADMIN_CQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_acq_entry)) 43 #define ADMIN_AENQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_aenq_entry)) 44 45 #define ENA_CUSTOMER_METRICS_BUFFER_SIZE 512 46 47 /*****************************************************************************/ 48 /*****************************************************************************/ 49 /* ENA adaptive interrupt moderation settings */ 50 51 #define ENA_INTR_INITIAL_TX_INTERVAL_USECS 64 52 #define ENA_INTR_INITIAL_RX_INTERVAL_USECS 20 53 #define ENA_DEFAULT_INTR_DELAY_RESOLUTION 1 54 55 #define ENA_HASH_KEY_SIZE 40 56 57 #define ENA_HW_HINTS_NO_TIMEOUT 0xFFFF 58 59 #define ENA_FEATURE_MAX_QUEUE_EXT_VER 1 60 61 struct ena_llq_configurations { 62 enum ena_admin_llq_header_location llq_header_location; 63 enum ena_admin_llq_ring_entry_size llq_ring_entry_size; 64 enum ena_admin_llq_stride_ctrl llq_stride_ctrl; 65 enum ena_admin_llq_num_descs_before_header llq_num_decs_before_header; 66 u16 llq_ring_entry_size_value; 67 }; 68 69 enum queue_direction { 70 ENA_COM_IO_QUEUE_DIRECTION_TX, 71 ENA_COM_IO_QUEUE_DIRECTION_RX 72 }; 73 74 struct ena_com_buf { 75 dma_addr_t paddr; /**< Buffer physical address */ 76 u16 len; /**< Buffer length in bytes */ 77 }; 78 79 struct ena_com_rx_buf_info { 80 u16 len; 81 u16 req_id; 82 }; 83 84 struct ena_com_io_desc_addr { 85 u8 __iomem *pbuf_dev_addr; /* LLQ address */ 86 u8 *virt_addr; 87 dma_addr_t phys_addr; 88 }; 89 90 struct ena_com_tx_meta { 91 u16 mss; 92 u16 l3_hdr_len; 93 u16 l3_hdr_offset; 94 u16 l4_hdr_len; /* In words */ 95 }; 96 97 struct ena_com_llq_info { 98 u16 header_location_ctrl; 99 u16 desc_stride_ctrl; 100 u16 desc_list_entry_size_ctrl; 101 u16 desc_list_entry_size; 102 u16 descs_num_before_header; 103 u16 descs_per_entry; 104 u16 max_entries_in_tx_burst; 105 bool disable_meta_caching; 106 }; 107 108 struct ena_com_io_cq { 109 struct ena_com_io_desc_addr cdesc_addr; 110 111 /* Interrupt unmask register */ 112 u32 __iomem *unmask_reg; 113 114 /* numa configuration register (for TPH) */ 115 u32 __iomem *numa_node_cfg_reg; 116 117 /* The value to write to the above register to unmask 118 * the interrupt of this queue 119 */ 120 u32 msix_vector ____cacheline_aligned; 121 122 enum queue_direction direction; 123 124 /* holds the number of cdesc of the current packet */ 125 u16 cur_rx_pkt_cdesc_count; 126 /* save the first cdesc idx of the current packet */ 127 u16 cur_rx_pkt_cdesc_start_idx; 128 129 u16 q_depth; 130 /* Caller qid */ 131 u16 qid; 132 133 /* Device queue index */ 134 u16 idx; 135 u16 head; 136 u8 phase; 137 u8 cdesc_entry_size_in_bytes; 138 139 } ____cacheline_aligned; 140 141 struct ena_com_io_bounce_buffer_control { 142 u8 *base_buffer; 143 u16 next_to_use; 144 u16 buffer_size; 145 u16 buffers_num; /* Must be a power of 2 */ 146 }; 147 148 /* This struct is to keep tracking the current location of the next llq entry */ 149 struct ena_com_llq_pkt_ctrl { 150 u8 *curr_bounce_buf; 151 u16 idx; 152 u16 descs_left_in_line; 153 }; 154 155 struct ena_com_io_sq { 156 struct ena_com_io_desc_addr desc_addr; 157 158 u32 __iomem *db_addr; 159 160 enum queue_direction direction; 161 enum ena_admin_placement_policy_type mem_queue_type; 162 163 bool disable_meta_caching; 164 165 u32 msix_vector; 166 struct ena_com_tx_meta cached_tx_meta; 167 struct ena_com_llq_info llq_info; 168 struct ena_com_llq_pkt_ctrl llq_buf_ctrl; 169 struct ena_com_io_bounce_buffer_control bounce_buf_ctrl; 170 171 u16 q_depth; 172 u16 qid; 173 174 u16 idx; 175 u16 tail; 176 u16 next_to_comp; 177 u16 llq_last_copy_tail; 178 u32 tx_max_header_size; 179 u8 phase; 180 u8 desc_entry_size; 181 u8 dma_addr_bits; 182 u16 entries_in_tx_burst_left; 183 } ____cacheline_aligned; 184 185 struct ena_com_admin_cq { 186 struct ena_admin_acq_entry *entries; 187 dma_addr_t dma_addr; 188 189 u16 head; 190 u8 phase; 191 }; 192 193 struct ena_com_admin_sq { 194 struct ena_admin_aq_entry *entries; 195 dma_addr_t dma_addr; 196 197 u32 __iomem *db_addr; 198 199 u16 head; 200 u16 tail; 201 u8 phase; 202 203 }; 204 205 struct ena_com_stats_admin { 206 u64 aborted_cmd; 207 u64 submitted_cmd; 208 u64 completed_cmd; 209 u64 out_of_space; 210 u64 no_completion; 211 }; 212 213 struct ena_com_admin_queue { 214 void *q_dmadev; 215 struct ena_com_dev *ena_dev; 216 spinlock_t q_lock; /* spinlock for the admin queue */ 217 218 struct ena_comp_ctx *comp_ctx; 219 u32 completion_timeout; 220 u16 q_depth; 221 struct ena_com_admin_cq cq; 222 struct ena_com_admin_sq sq; 223 224 /* Indicate if the admin queue should poll for completion */ 225 bool polling; 226 227 /* Define if fallback to polling mode should occur */ 228 bool auto_polling; 229 230 u16 curr_cmd_id; 231 232 /* Indicate that the ena was initialized and can 233 * process new admin commands 234 */ 235 bool running_state; 236 237 /* Count the number of outstanding admin commands */ 238 atomic_t outstanding_cmds; 239 240 struct ena_com_stats_admin stats; 241 }; 242 243 struct ena_aenq_handlers; 244 245 struct ena_com_aenq { 246 u16 head; 247 u8 phase; 248 struct ena_admin_aenq_entry *entries; 249 dma_addr_t dma_addr; 250 u16 q_depth; 251 struct ena_aenq_handlers *aenq_handlers; 252 }; 253 254 struct ena_com_mmio_read { 255 struct ena_admin_ena_mmio_req_read_less_resp *read_resp; 256 dma_addr_t read_resp_dma_addr; 257 u32 reg_read_to; /* in us */ 258 u16 seq_num; 259 bool readless_supported; 260 /* spin lock to ensure a single outstanding read */ 261 spinlock_t lock; 262 }; 263 264 struct ena_rss { 265 /* Indirect table */ 266 u16 *host_rss_ind_tbl; 267 struct ena_admin_rss_ind_table_entry *rss_ind_tbl; 268 dma_addr_t rss_ind_tbl_dma_addr; 269 u16 tbl_log_size; 270 271 /* Hash key */ 272 enum ena_admin_hash_functions hash_func; 273 struct ena_admin_feature_rss_flow_hash_control *hash_key; 274 dma_addr_t hash_key_dma_addr; 275 u32 hash_init_val; 276 277 /* Flow Control */ 278 struct ena_admin_feature_rss_hash_control *hash_ctrl; 279 dma_addr_t hash_ctrl_dma_addr; 280 281 }; 282 283 struct ena_customer_metrics { 284 /* in correlation with ENA_ADMIN_CUSTOMER_METRICS_SUPPORT_MASK 285 * and ena_admin_customer_metrics_id 286 */ 287 u64 supported_metrics; 288 dma_addr_t buffer_dma_addr; 289 void *buffer_virt_addr; 290 u32 buffer_len; 291 }; 292 293 struct ena_host_attribute { 294 /* Debug area */ 295 u8 *debug_area_virt_addr; 296 dma_addr_t debug_area_dma_addr; 297 u32 debug_area_size; 298 299 /* Host information */ 300 struct ena_admin_host_info *host_info; 301 dma_addr_t host_info_dma_addr; 302 }; 303 304 /* Each ena_dev is a PCI function. */ 305 struct ena_com_dev { 306 struct ena_com_admin_queue admin_queue; 307 struct ena_com_aenq aenq; 308 struct ena_com_io_cq io_cq_queues[ENA_TOTAL_NUM_QUEUES]; 309 struct ena_com_io_sq io_sq_queues[ENA_TOTAL_NUM_QUEUES]; 310 u8 __iomem *reg_bar; 311 void __iomem *mem_bar; 312 void *dmadev; 313 struct net_device *net_device; 314 315 enum ena_admin_placement_policy_type tx_mem_queue_type; 316 u32 tx_max_header_size; 317 u16 stats_func; /* Selected function for extended statistic dump */ 318 u16 stats_queue; /* Selected queue for extended statistic dump */ 319 320 u32 ena_min_poll_delay_us; 321 322 struct ena_com_mmio_read mmio_read; 323 324 struct ena_rss rss; 325 u32 supported_features; 326 u32 capabilities; 327 u32 dma_addr_bits; 328 329 struct ena_host_attribute host_attr; 330 bool adaptive_coalescing; 331 u16 intr_delay_resolution; 332 333 /* interrupt moderation intervals are in usec divided by 334 * intr_delay_resolution, which is supplied by the device. 335 */ 336 u32 intr_moder_tx_interval; 337 u32 intr_moder_rx_interval; 338 339 struct ena_intr_moder_entry *intr_moder_tbl; 340 341 struct ena_com_llq_info llq_info; 342 343 struct ena_customer_metrics customer_metrics; 344 }; 345 346 struct ena_com_dev_get_features_ctx { 347 struct ena_admin_queue_feature_desc max_queues; 348 struct ena_admin_queue_ext_feature_desc max_queue_ext; 349 struct ena_admin_device_attr_feature_desc dev_attr; 350 struct ena_admin_feature_aenq_desc aenq; 351 struct ena_admin_feature_offload_desc offload; 352 struct ena_admin_ena_hw_hints hw_hints; 353 struct ena_admin_feature_llq_desc llq; 354 }; 355 356 struct ena_com_create_io_ctx { 357 enum ena_admin_placement_policy_type mem_queue_type; 358 enum queue_direction direction; 359 int numa_node; 360 u32 msix_vector; 361 u16 queue_size; 362 u16 qid; 363 }; 364 365 typedef void (*ena_aenq_handler)(void *data, 366 struct ena_admin_aenq_entry *aenq_e); 367 368 /* Holds aenq handlers. Indexed by AENQ event group */ 369 struct ena_aenq_handlers { 370 ena_aenq_handler handlers[ENA_MAX_HANDLERS]; 371 ena_aenq_handler unimplemented_handler; 372 }; 373 374 /*****************************************************************************/ 375 /*****************************************************************************/ 376 377 /* ena_com_mmio_reg_read_request_init - Init the mmio reg read mechanism 378 * @ena_dev: ENA communication layer struct 379 * 380 * Initialize the register read mechanism. 381 * 382 * @note: This method must be the first stage in the initialization sequence. 383 * 384 * @return - 0 on success, negative value on failure. 385 */ 386 int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev); 387 388 /* ena_com_set_mmio_read_mode - Enable/disable the indirect mmio reg read mechanism 389 * @ena_dev: ENA communication layer struct 390 * @readless_supported: readless mode (enable/disable) 391 */ 392 void ena_com_set_mmio_read_mode(struct ena_com_dev *ena_dev, 393 bool readless_supported); 394 395 /* ena_com_mmio_reg_read_request_write_dev_addr - Write the mmio reg read return 396 * value physical address. 397 * @ena_dev: ENA communication layer struct 398 */ 399 void ena_com_mmio_reg_read_request_write_dev_addr(struct ena_com_dev *ena_dev); 400 401 /* ena_com_mmio_reg_read_request_destroy - Destroy the mmio reg read mechanism 402 * @ena_dev: ENA communication layer struct 403 */ 404 void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev); 405 406 /* ena_com_admin_init - Init the admin and the async queues 407 * @ena_dev: ENA communication layer struct 408 * @aenq_handlers: Those handlers to be called upon event. 409 * 410 * Initialize the admin submission and completion queues. 411 * Initialize the asynchronous events notification queues. 412 * 413 * @return - 0 on success, negative value on failure. 414 */ 415 int ena_com_admin_init(struct ena_com_dev *ena_dev, 416 struct ena_aenq_handlers *aenq_handlers); 417 418 /* ena_com_admin_destroy - Destroy the admin and the async events queues. 419 * @ena_dev: ENA communication layer struct 420 * 421 * @note: Before calling this method, the caller must validate that the device 422 * won't send any additional admin completions/aenq. 423 * To achieve that, a FLR is recommended. 424 */ 425 void ena_com_admin_destroy(struct ena_com_dev *ena_dev); 426 427 /* ena_com_dev_reset - Perform device FLR to the device. 428 * @ena_dev: ENA communication layer struct 429 * @reset_reason: Specify what is the trigger for the reset in case of an error. 430 * 431 * @return - 0 on success, negative value on failure. 432 */ 433 int ena_com_dev_reset(struct ena_com_dev *ena_dev, 434 enum ena_regs_reset_reason_types reset_reason); 435 436 /* ena_com_create_io_queue - Create io queue. 437 * @ena_dev: ENA communication layer struct 438 * @ctx - create context structure 439 * 440 * Create the submission and the completion queues. 441 * 442 * @return - 0 on success, negative value on failure. 443 */ 444 int ena_com_create_io_queue(struct ena_com_dev *ena_dev, 445 struct ena_com_create_io_ctx *ctx); 446 447 /* ena_com_destroy_io_queue - Destroy IO queue with the queue id - qid. 448 * @ena_dev: ENA communication layer struct 449 * @qid - the caller virtual queue id. 450 */ 451 void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid); 452 453 /* ena_com_get_io_handlers - Return the io queue handlers 454 * @ena_dev: ENA communication layer struct 455 * @qid - the caller virtual queue id. 456 * @io_sq - IO submission queue handler 457 * @io_cq - IO completion queue handler. 458 * 459 * @return - 0 on success, negative value on failure. 460 */ 461 int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid, 462 struct ena_com_io_sq **io_sq, 463 struct ena_com_io_cq **io_cq); 464 465 /* ena_com_admin_aenq_enable - ENAble asynchronous event notifications 466 * @ena_dev: ENA communication layer struct 467 * 468 * After this method, aenq event can be received via AENQ. 469 */ 470 void ena_com_admin_aenq_enable(struct ena_com_dev *ena_dev); 471 472 /* ena_com_set_admin_running_state - Set the state of the admin queue 473 * @ena_dev: ENA communication layer struct 474 * 475 * Change the state of the admin queue (enable/disable) 476 */ 477 void ena_com_set_admin_running_state(struct ena_com_dev *ena_dev, bool state); 478 479 /* ena_com_get_admin_running_state - Get the admin queue state 480 * @ena_dev: ENA communication layer struct 481 * 482 * Retrieve the state of the admin queue (enable/disable) 483 * 484 * @return - current polling mode (enable/disable) 485 */ 486 bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev); 487 488 /* ena_com_set_admin_polling_mode - Set the admin completion queue polling mode 489 * @ena_dev: ENA communication layer struct 490 * @polling: ENAble/Disable polling mode 491 * 492 * Set the admin completion mode. 493 */ 494 void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling); 495 496 /* ena_com_set_admin_auto_polling_mode - Enable autoswitch to polling mode 497 * @ena_dev: ENA communication layer struct 498 * @polling: Enable/Disable polling mode 499 * 500 * Set the autopolling mode. 501 * If autopolling is on: 502 * In case of missing interrupt when data is available switch to polling. 503 */ 504 void ena_com_set_admin_auto_polling_mode(struct ena_com_dev *ena_dev, 505 bool polling); 506 507 /* ena_com_admin_q_comp_intr_handler - admin queue interrupt handler 508 * @ena_dev: ENA communication layer struct 509 * 510 * This method goes over the admin completion queue and wakes up all the pending 511 * threads that wait on the commands wait event. 512 * 513 * @note: Should be called after MSI-X interrupt. 514 */ 515 void ena_com_admin_q_comp_intr_handler(struct ena_com_dev *ena_dev); 516 517 /* ena_com_aenq_intr_handler - AENQ interrupt handler 518 * @ena_dev: ENA communication layer struct 519 * 520 * This method goes over the async event notification queue and calls the proper 521 * aenq handler. 522 */ 523 void ena_com_aenq_intr_handler(struct ena_com_dev *ena_dev, void *data); 524 525 /* ena_com_abort_admin_commands - Abort all the outstanding admin commands. 526 * @ena_dev: ENA communication layer struct 527 * 528 * This method aborts all the outstanding admin commands. 529 * The caller should then call ena_com_wait_for_abort_completion to make sure 530 * all the commands were completed. 531 */ 532 void ena_com_abort_admin_commands(struct ena_com_dev *ena_dev); 533 534 /* ena_com_wait_for_abort_completion - Wait for admin commands abort. 535 * @ena_dev: ENA communication layer struct 536 * 537 * This method waits until all the outstanding admin commands are completed. 538 */ 539 void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev); 540 541 /* ena_com_validate_version - Validate the device parameters 542 * @ena_dev: ENA communication layer struct 543 * 544 * This method verifies the device parameters are the same as the saved 545 * parameters in ena_dev. 546 * This method is useful after device reset, to validate the device mac address 547 * and the device offloads are the same as before the reset. 548 * 549 * @return - 0 on success negative value otherwise. 550 */ 551 int ena_com_validate_version(struct ena_com_dev *ena_dev); 552 553 /* ena_com_get_link_params - Retrieve physical link parameters. 554 * @ena_dev: ENA communication layer struct 555 * @resp: Link parameters 556 * 557 * Retrieve the physical link parameters, 558 * like speed, auto-negotiation and full duplex support. 559 * 560 * @return - 0 on Success negative value otherwise. 561 */ 562 int ena_com_get_link_params(struct ena_com_dev *ena_dev, 563 struct ena_admin_get_feat_resp *resp); 564 565 /* ena_com_get_dma_width - Retrieve physical dma address width the device 566 * supports. 567 * @ena_dev: ENA communication layer struct 568 * 569 * Retrieve the maximum physical address bits the device can handle. 570 * 571 * @return: > 0 on Success and negative value otherwise. 572 */ 573 int ena_com_get_dma_width(struct ena_com_dev *ena_dev); 574 575 /* ena_com_set_aenq_config - Set aenq groups configurations 576 * @ena_dev: ENA communication layer struct 577 * @groups flag: bit fields flags of enum ena_admin_aenq_group. 578 * 579 * Configure which aenq event group the driver would like to receive. 580 * 581 * @return: 0 on Success and negative value otherwise. 582 */ 583 int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag); 584 585 /* ena_com_get_dev_attr_feat - Get device features 586 * @ena_dev: ENA communication layer struct 587 * @get_feat_ctx: returned context that contain the get features. 588 * 589 * @return: 0 on Success and negative value otherwise. 590 */ 591 int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, 592 struct ena_com_dev_get_features_ctx *get_feat_ctx); 593 594 /* ena_com_get_dev_basic_stats - Get device basic statistics 595 * @ena_dev: ENA communication layer struct 596 * @stats: stats return value 597 * 598 * @return: 0 on Success and negative value otherwise. 599 */ 600 int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev, 601 struct ena_admin_basic_stats *stats); 602 603 /* ena_com_get_eni_stats - Get extended network interface statistics 604 * @ena_dev: ENA communication layer struct 605 * @stats: stats return value 606 * 607 * @return: 0 on Success and negative value otherwise. 608 */ 609 int ena_com_get_eni_stats(struct ena_com_dev *ena_dev, 610 struct ena_admin_eni_stats *stats); 611 612 /* ena_com_get_ena_srd_info - Get ENA SRD network interface statistics 613 * @ena_dev: ENA communication layer struct 614 * @info: ena srd stats and flags 615 * 616 * @return: 0 on Success and negative value otherwise. 617 */ 618 int ena_com_get_ena_srd_info(struct ena_com_dev *ena_dev, 619 struct ena_admin_ena_srd_info *info); 620 621 /* ena_com_get_customer_metrics - Get customer metrics for network interface 622 * @ena_dev: ENA communication layer struct 623 * @buffer: buffer for returned customer metrics 624 * @len: size of the buffer 625 * 626 * @return: 0 on Success and negative value otherwise. 627 */ 628 int ena_com_get_customer_metrics(struct ena_com_dev *ena_dev, char *buffer, u32 len); 629 630 /* ena_com_set_dev_mtu - Configure the device mtu. 631 * @ena_dev: ENA communication layer struct 632 * @mtu: mtu value 633 * 634 * @return: 0 on Success and negative value otherwise. 635 */ 636 int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, u32 mtu); 637 638 /* ena_com_get_offload_settings - Retrieve the device offloads capabilities 639 * @ena_dev: ENA communication layer struct 640 * @offlad: offload return value 641 * 642 * @return: 0 on Success and negative value otherwise. 643 */ 644 int ena_com_get_offload_settings(struct ena_com_dev *ena_dev, 645 struct ena_admin_feature_offload_desc *offload); 646 647 /* ena_com_rss_init - Init RSS 648 * @ena_dev: ENA communication layer struct 649 * @log_size: indirection log size 650 * 651 * Allocate RSS/RFS resources. 652 * The caller then can configure rss using ena_com_set_hash_function, 653 * ena_com_set_hash_ctrl and ena_com_indirect_table_set. 654 * 655 * @return: 0 on Success and negative value otherwise. 656 */ 657 int ena_com_rss_init(struct ena_com_dev *ena_dev, u16 log_size); 658 659 /* ena_com_rss_destroy - Destroy rss 660 * @ena_dev: ENA communication layer struct 661 * 662 * Free all the RSS/RFS resources. 663 */ 664 void ena_com_rss_destroy(struct ena_com_dev *ena_dev); 665 666 /* ena_com_get_current_hash_function - Get RSS hash function 667 * @ena_dev: ENA communication layer struct 668 * 669 * Return the current hash function. 670 * @return: 0 or one of the ena_admin_hash_functions values. 671 */ 672 int ena_com_get_current_hash_function(struct ena_com_dev *ena_dev); 673 674 /* ena_com_fill_hash_function - Fill RSS hash function 675 * @ena_dev: ENA communication layer struct 676 * @func: The hash function (Toeplitz or crc) 677 * @key: Hash key (for toeplitz hash) 678 * @key_len: key length (max length 10 DW) 679 * @init_val: initial value for the hash function 680 * 681 * Fill the ena_dev resources with the desire hash function, hash key, key_len 682 * and key initial value (if needed by the hash function). 683 * To flush the key into the device the caller should call 684 * ena_com_set_hash_function. 685 * 686 * @return: 0 on Success and negative value otherwise. 687 */ 688 int ena_com_fill_hash_function(struct ena_com_dev *ena_dev, 689 enum ena_admin_hash_functions func, 690 const u8 *key, u16 key_len, u32 init_val); 691 692 /* ena_com_set_hash_function - Flush the hash function and it dependencies to 693 * the device. 694 * @ena_dev: ENA communication layer struct 695 * 696 * Flush the hash function and it dependencies (key, key length and 697 * initial value) if needed. 698 * 699 * @note: Prior to this method the caller should call ena_com_fill_hash_function 700 * 701 * @return: 0 on Success and negative value otherwise. 702 */ 703 int ena_com_set_hash_function(struct ena_com_dev *ena_dev); 704 705 /* ena_com_get_hash_function - Retrieve the hash function from the device. 706 * @ena_dev: ENA communication layer struct 707 * @func: hash function 708 * 709 * Retrieve the hash function from the device. 710 * 711 * @note: If the caller called ena_com_fill_hash_function but didn't flush 712 * it to the device, the new configuration will be lost. 713 * 714 * @return: 0 on Success and negative value otherwise. 715 */ 716 int ena_com_get_hash_function(struct ena_com_dev *ena_dev, 717 enum ena_admin_hash_functions *func); 718 719 /* ena_com_get_hash_key - Retrieve the hash key 720 * @ena_dev: ENA communication layer struct 721 * @key: hash key 722 * 723 * Retrieve the hash key. 724 * 725 * @note: If the caller called ena_com_fill_hash_key but didn't flush 726 * it to the device, the new configuration will be lost. 727 * 728 * @return: 0 on Success and negative value otherwise. 729 */ 730 int ena_com_get_hash_key(struct ena_com_dev *ena_dev, u8 *key); 731 /* ena_com_fill_hash_ctrl - Fill RSS hash control 732 * @ena_dev: ENA communication layer struct. 733 * @proto: The protocol to configure. 734 * @hash_fields: bit mask of ena_admin_flow_hash_fields 735 * 736 * Fill the ena_dev resources with the desire hash control (the ethernet 737 * fields that take part of the hash) for a specific protocol. 738 * To flush the hash control to the device, the caller should call 739 * ena_com_set_hash_ctrl. 740 * 741 * @return: 0 on Success and negative value otherwise. 742 */ 743 int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev, 744 enum ena_admin_flow_hash_proto proto, 745 u16 hash_fields); 746 747 /* ena_com_set_hash_ctrl - Flush the hash control resources to the device. 748 * @ena_dev: ENA communication layer struct 749 * 750 * Flush the hash control (the ethernet fields that take part of the hash) 751 * 752 * @note: Prior to this method the caller should call ena_com_fill_hash_ctrl. 753 * 754 * @return: 0 on Success and negative value otherwise. 755 */ 756 int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev); 757 758 /* ena_com_get_hash_ctrl - Retrieve the hash control from the device. 759 * @ena_dev: ENA communication layer struct 760 * @proto: The protocol to retrieve. 761 * @fields: bit mask of ena_admin_flow_hash_fields. 762 * 763 * Retrieve the hash control from the device. 764 * 765 * @note: If the caller called ena_com_fill_hash_ctrl but didn't flush 766 * it to the device, the new configuration will be lost. 767 * 768 * @return: 0 on Success and negative value otherwise. 769 */ 770 int ena_com_get_hash_ctrl(struct ena_com_dev *ena_dev, 771 enum ena_admin_flow_hash_proto proto, 772 u16 *fields); 773 774 /* ena_com_set_default_hash_ctrl - Set the hash control to a default 775 * configuration. 776 * @ena_dev: ENA communication layer struct 777 * 778 * Fill the ena_dev resources with the default hash control configuration. 779 * To flush the hash control to the device, the caller should call 780 * ena_com_set_hash_ctrl. 781 * 782 * @return: 0 on Success and negative value otherwise. 783 */ 784 int ena_com_set_default_hash_ctrl(struct ena_com_dev *ena_dev); 785 786 /* ena_com_indirect_table_fill_entry - Fill a single entry in the RSS 787 * indirection table 788 * @ena_dev: ENA communication layer struct. 789 * @entry_idx - indirection table entry. 790 * @entry_value - redirection value 791 * 792 * Fill a single entry of the RSS indirection table in the ena_dev resources. 793 * To flush the indirection table to the device, the called should call 794 * ena_com_indirect_table_set. 795 * 796 * @return: 0 on Success and negative value otherwise. 797 */ 798 int ena_com_indirect_table_fill_entry(struct ena_com_dev *ena_dev, 799 u16 entry_idx, u16 entry_value); 800 801 /* ena_com_indirect_table_set - Flush the indirection table to the device. 802 * @ena_dev: ENA communication layer struct 803 * 804 * Flush the indirection hash control to the device. 805 * Prior to this method the caller should call ena_com_indirect_table_fill_entry 806 * 807 * @return: 0 on Success and negative value otherwise. 808 */ 809 int ena_com_indirect_table_set(struct ena_com_dev *ena_dev); 810 811 /* ena_com_indirect_table_get - Retrieve the indirection table from the device. 812 * @ena_dev: ENA communication layer struct 813 * @ind_tbl: indirection table 814 * 815 * Retrieve the RSS indirection table from the device. 816 * 817 * @note: If the caller called ena_com_indirect_table_fill_entry but didn't flush 818 * it to the device, the new configuration will be lost. 819 * 820 * @return: 0 on Success and negative value otherwise. 821 */ 822 int ena_com_indirect_table_get(struct ena_com_dev *ena_dev, u32 *ind_tbl); 823 824 /* ena_com_allocate_host_info - Allocate host info resources. 825 * @ena_dev: ENA communication layer struct 826 * 827 * @return: 0 on Success and negative value otherwise. 828 */ 829 int ena_com_allocate_host_info(struct ena_com_dev *ena_dev); 830 831 /* ena_com_allocate_debug_area - Allocate debug area. 832 * @ena_dev: ENA communication layer struct 833 * @debug_area_size - debug area size. 834 * 835 * @return: 0 on Success and negative value otherwise. 836 */ 837 int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev, 838 u32 debug_area_size); 839 840 /* ena_com_allocate_customer_metrics_buffer - Allocate customer metrics resources. 841 * @ena_dev: ENA communication layer struct 842 * 843 * @return: 0 on Success and negative value otherwise. 844 */ 845 int ena_com_allocate_customer_metrics_buffer(struct ena_com_dev *ena_dev); 846 847 /* ena_com_delete_debug_area - Free the debug area resources. 848 * @ena_dev: ENA communication layer struct 849 * 850 * Free the allocated debug area. 851 */ 852 void ena_com_delete_debug_area(struct ena_com_dev *ena_dev); 853 854 /* ena_com_delete_host_info - Free the host info resources. 855 * @ena_dev: ENA communication layer struct 856 * 857 * Free the allocated host info. 858 */ 859 void ena_com_delete_host_info(struct ena_com_dev *ena_dev); 860 861 /* ena_com_delete_customer_metrics_buffer - Free the customer metrics resources. 862 * @ena_dev: ENA communication layer struct 863 * 864 * Free the allocated customer metrics area. 865 */ 866 void ena_com_delete_customer_metrics_buffer(struct ena_com_dev *ena_dev); 867 868 /* ena_com_set_host_attributes - Update the device with the host 869 * attributes (debug area and host info) base address. 870 * @ena_dev: ENA communication layer struct 871 * 872 * @return: 0 on Success and negative value otherwise. 873 */ 874 int ena_com_set_host_attributes(struct ena_com_dev *ena_dev); 875 876 /* ena_com_create_io_cq - Create io completion queue. 877 * @ena_dev: ENA communication layer struct 878 * @io_cq - io completion queue handler 879 880 * Create IO completion queue. 881 * 882 * @return - 0 on success, negative value on failure. 883 */ 884 int ena_com_create_io_cq(struct ena_com_dev *ena_dev, 885 struct ena_com_io_cq *io_cq); 886 887 /* ena_com_destroy_io_cq - Destroy io completion queue. 888 * @ena_dev: ENA communication layer struct 889 * @io_cq - io completion queue handler 890 891 * Destroy IO completion queue. 892 * 893 * @return - 0 on success, negative value on failure. 894 */ 895 int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev, 896 struct ena_com_io_cq *io_cq); 897 898 /* ena_com_execute_admin_command - Execute admin command 899 * @admin_queue: admin queue. 900 * @cmd: the admin command to execute. 901 * @cmd_size: the command size. 902 * @cmd_completion: command completion return value. 903 * @cmd_comp_size: command completion size. 904 905 * Submit an admin command and then wait until the device returns a 906 * completion. 907 * The completion will be copied into cmd_comp. 908 * 909 * @return - 0 on success, negative value on failure. 910 */ 911 int ena_com_execute_admin_command(struct ena_com_admin_queue *admin_queue, 912 struct ena_admin_aq_entry *cmd, 913 size_t cmd_size, 914 struct ena_admin_acq_entry *cmd_comp, 915 size_t cmd_comp_size); 916 917 /* ena_com_init_interrupt_moderation - Init interrupt moderation 918 * @ena_dev: ENA communication layer struct 919 * 920 * @return - 0 on success, negative value on failure. 921 */ 922 int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev); 923 924 /* ena_com_interrupt_moderation_supported - Return if interrupt moderation 925 * capability is supported by the device. 926 * 927 * @return - supported or not. 928 */ 929 bool ena_com_interrupt_moderation_supported(struct ena_com_dev *ena_dev); 930 931 /* ena_com_update_nonadaptive_moderation_interval_tx - Update the 932 * non-adaptive interval in Tx direction. 933 * @ena_dev: ENA communication layer struct 934 * @tx_coalesce_usecs: Interval in usec. 935 * 936 * @return - 0 on success, negative value on failure. 937 */ 938 int ena_com_update_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev, 939 u32 tx_coalesce_usecs); 940 941 /* ena_com_update_nonadaptive_moderation_interval_rx - Update the 942 * non-adaptive interval in Rx direction. 943 * @ena_dev: ENA communication layer struct 944 * @rx_coalesce_usecs: Interval in usec. 945 * 946 * @return - 0 on success, negative value on failure. 947 */ 948 int ena_com_update_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev, 949 u32 rx_coalesce_usecs); 950 951 /* ena_com_get_nonadaptive_moderation_interval_tx - Retrieve the 952 * non-adaptive interval in Tx direction. 953 * @ena_dev: ENA communication layer struct 954 * 955 * @return - interval in usec 956 */ 957 unsigned int ena_com_get_nonadaptive_moderation_interval_tx(struct ena_com_dev *ena_dev); 958 959 /* ena_com_get_nonadaptive_moderation_interval_rx - Retrieve the 960 * non-adaptive interval in Rx direction. 961 * @ena_dev: ENA communication layer struct 962 * 963 * @return - interval in usec 964 */ 965 unsigned int ena_com_get_nonadaptive_moderation_interval_rx(struct ena_com_dev *ena_dev); 966 967 /* ena_com_config_dev_mode - Configure the placement policy of the device. 968 * @ena_dev: ENA communication layer struct 969 * @llq_features: LLQ feature descriptor, retrieve via 970 * ena_com_get_dev_attr_feat. 971 * @ena_llq_config: The default driver LLQ parameters configurations 972 */ 973 int ena_com_config_dev_mode(struct ena_com_dev *ena_dev, 974 struct ena_admin_feature_llq_desc *llq_features, 975 struct ena_llq_configurations *llq_default_config); 976 977 /* ena_com_io_sq_to_ena_dev - Extract ena_com_dev using contained field io_sq. 978 * @io_sq: IO submit queue struct 979 * 980 * @return - ena_com_dev struct extracted from io_sq 981 */ 982 static inline struct ena_com_dev *ena_com_io_sq_to_ena_dev(struct ena_com_io_sq *io_sq) 983 { 984 return container_of(io_sq, struct ena_com_dev, io_sq_queues[io_sq->qid]); 985 } 986 987 /* ena_com_io_cq_to_ena_dev - Extract ena_com_dev using contained field io_cq. 988 * @io_sq: IO submit queue struct 989 * 990 * @return - ena_com_dev struct extracted from io_sq 991 */ 992 static inline struct ena_com_dev *ena_com_io_cq_to_ena_dev(struct ena_com_io_cq *io_cq) 993 { 994 return container_of(io_cq, struct ena_com_dev, io_cq_queues[io_cq->qid]); 995 } 996 997 static inline bool ena_com_get_adaptive_moderation_enabled(struct ena_com_dev *ena_dev) 998 { 999 return ena_dev->adaptive_coalescing; 1000 } 1001 1002 static inline void ena_com_enable_adaptive_moderation(struct ena_com_dev *ena_dev) 1003 { 1004 ena_dev->adaptive_coalescing = true; 1005 } 1006 1007 static inline void ena_com_disable_adaptive_moderation(struct ena_com_dev *ena_dev) 1008 { 1009 ena_dev->adaptive_coalescing = false; 1010 } 1011 1012 /* ena_com_get_cap - query whether device supports a capability. 1013 * @ena_dev: ENA communication layer struct 1014 * @cap_id: enum value representing the capability 1015 * 1016 * @return - true if capability is supported or false otherwise 1017 */ 1018 static inline bool ena_com_get_cap(struct ena_com_dev *ena_dev, 1019 enum ena_admin_aq_caps_id cap_id) 1020 { 1021 return !!(ena_dev->capabilities & BIT(cap_id)); 1022 } 1023 1024 /* ena_com_get_customer_metric_support - query whether device supports a given customer metric. 1025 * @ena_dev: ENA communication layer struct 1026 * @metric_id: enum value representing the customer metric 1027 * 1028 * @return - true if customer metric is supported or false otherwise 1029 */ 1030 static inline bool ena_com_get_customer_metric_support(struct ena_com_dev *ena_dev, 1031 enum ena_admin_customer_metrics_id metric_id) 1032 { 1033 return !!(ena_dev->customer_metrics.supported_metrics & BIT(metric_id)); 1034 } 1035 1036 /* ena_com_get_customer_metric_count - return the number of supported customer metrics. 1037 * @ena_dev: ENA communication layer struct 1038 * 1039 * @return - the number of supported customer metrics 1040 */ 1041 static inline int ena_com_get_customer_metric_count(struct ena_com_dev *ena_dev) 1042 { 1043 return hweight64(ena_dev->customer_metrics.supported_metrics); 1044 } 1045 1046 /* ena_com_update_intr_reg - Prepare interrupt register 1047 * @intr_reg: interrupt register to update. 1048 * @rx_delay_interval: Rx interval in usecs 1049 * @tx_delay_interval: Tx interval in usecs 1050 * @unmask: unmask enable/disable 1051 * 1052 * Prepare interrupt update register with the supplied parameters. 1053 */ 1054 static inline void ena_com_update_intr_reg(struct ena_eth_io_intr_reg *intr_reg, 1055 u32 rx_delay_interval, 1056 u32 tx_delay_interval, 1057 bool unmask) 1058 { 1059 intr_reg->intr_control = 0; 1060 intr_reg->intr_control |= rx_delay_interval & 1061 ENA_ETH_IO_INTR_REG_RX_INTR_DELAY_MASK; 1062 1063 intr_reg->intr_control |= 1064 (tx_delay_interval << ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_SHIFT) 1065 & ENA_ETH_IO_INTR_REG_TX_INTR_DELAY_MASK; 1066 1067 if (unmask) 1068 intr_reg->intr_control |= ENA_ETH_IO_INTR_REG_INTR_UNMASK_MASK; 1069 } 1070 1071 static inline u8 *ena_com_get_next_bounce_buffer(struct ena_com_io_bounce_buffer_control *bounce_buf_ctrl) 1072 { 1073 u16 size, buffers_num; 1074 u8 *buf; 1075 1076 size = bounce_buf_ctrl->buffer_size; 1077 buffers_num = bounce_buf_ctrl->buffers_num; 1078 1079 buf = bounce_buf_ctrl->base_buffer + 1080 (bounce_buf_ctrl->next_to_use++ & (buffers_num - 1)) * size; 1081 1082 prefetchw(bounce_buf_ctrl->base_buffer + 1083 (bounce_buf_ctrl->next_to_use & (buffers_num - 1)) * size); 1084 1085 return buf; 1086 } 1087 1088 #endif /* !(ENA_COM) */ 1089