1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #include <linux/ethtool.h> 7 #include <linux/pci.h> 8 #include <linux/net_tstamp.h> 9 10 #include "ena_netdev.h" 11 #include "ena_xdp.h" 12 #include "ena_phc.h" 13 14 struct ena_stats { 15 char name[ETH_GSTRING_LEN]; 16 int stat_offset; 17 }; 18 19 struct ena_hw_metrics { 20 char name[ETH_GSTRING_LEN]; 21 }; 22 23 #define ENA_STAT_ENA_COM_ENTRY(stat) { \ 24 .name = #stat, \ 25 .stat_offset = offsetof(struct ena_com_stats_admin, stat) / sizeof(u64) \ 26 } 27 28 #define ENA_STAT_ENTRY(stat, stat_type) { \ 29 .name = #stat, \ 30 .stat_offset = offsetof(struct ena_stats_##stat_type, stat) / sizeof(u64) \ 31 } 32 33 #define ENA_STAT_HW_ENTRY(stat, stat_type) { \ 34 .name = #stat, \ 35 .stat_offset = offsetof(struct ena_admin_##stat_type, stat) / sizeof(u64) \ 36 } 37 38 #define ENA_STAT_RX_ENTRY(stat) \ 39 ENA_STAT_ENTRY(stat, rx) 40 41 #define ENA_STAT_TX_ENTRY(stat) \ 42 ENA_STAT_ENTRY(stat, tx) 43 44 #define ENA_STAT_GLOBAL_ENTRY(stat) \ 45 ENA_STAT_ENTRY(stat, dev) 46 47 #define ENA_STAT_ENI_ENTRY(stat) \ 48 ENA_STAT_HW_ENTRY(stat, eni_stats) 49 50 #define ENA_STAT_ENA_SRD_ENTRY(stat) \ 51 ENA_STAT_HW_ENTRY(stat, ena_srd_stats) 52 53 #define ENA_STAT_ENA_SRD_MODE_ENTRY(stat) { \ 54 .name = #stat, \ 55 .stat_offset = offsetof(struct ena_admin_ena_srd_info, flags) / sizeof(u64) \ 56 } 57 58 #define ENA_METRIC_ENI_ENTRY(stat) { \ 59 .name = #stat \ 60 } 61 62 static const struct ena_stats ena_stats_global_strings[] = { 63 ENA_STAT_GLOBAL_ENTRY(tx_timeout), 64 ENA_STAT_GLOBAL_ENTRY(suspend), 65 ENA_STAT_GLOBAL_ENTRY(resume), 66 ENA_STAT_GLOBAL_ENTRY(wd_expired), 67 ENA_STAT_GLOBAL_ENTRY(interface_up), 68 ENA_STAT_GLOBAL_ENTRY(interface_down), 69 ENA_STAT_GLOBAL_ENTRY(admin_q_pause), 70 ENA_STAT_GLOBAL_ENTRY(reset_fail), 71 }; 72 73 /* A partial list of hw stats. Used when admin command 74 * with type ENA_ADMIN_GET_STATS_TYPE_CUSTOMER_METRICS is not supported 75 */ 76 static const struct ena_stats ena_stats_eni_strings[] = { 77 ENA_STAT_ENI_ENTRY(bw_in_allowance_exceeded), 78 ENA_STAT_ENI_ENTRY(bw_out_allowance_exceeded), 79 ENA_STAT_ENI_ENTRY(pps_allowance_exceeded), 80 ENA_STAT_ENI_ENTRY(conntrack_allowance_exceeded), 81 ENA_STAT_ENI_ENTRY(linklocal_allowance_exceeded), 82 }; 83 84 static const struct ena_hw_metrics ena_hw_stats_strings[] = { 85 ENA_METRIC_ENI_ENTRY(bw_in_allowance_exceeded), 86 ENA_METRIC_ENI_ENTRY(bw_out_allowance_exceeded), 87 ENA_METRIC_ENI_ENTRY(pps_allowance_exceeded), 88 ENA_METRIC_ENI_ENTRY(conntrack_allowance_exceeded), 89 ENA_METRIC_ENI_ENTRY(linklocal_allowance_exceeded), 90 ENA_METRIC_ENI_ENTRY(conntrack_allowance_available), 91 }; 92 93 static const struct ena_stats ena_srd_info_strings[] = { 94 ENA_STAT_ENA_SRD_MODE_ENTRY(ena_srd_mode), 95 ENA_STAT_ENA_SRD_ENTRY(ena_srd_tx_pkts), 96 ENA_STAT_ENA_SRD_ENTRY(ena_srd_eligible_tx_pkts), 97 ENA_STAT_ENA_SRD_ENTRY(ena_srd_rx_pkts), 98 ENA_STAT_ENA_SRD_ENTRY(ena_srd_resource_utilization) 99 }; 100 101 static const struct ena_stats ena_stats_tx_strings[] = { 102 ENA_STAT_TX_ENTRY(cnt), 103 ENA_STAT_TX_ENTRY(bytes), 104 ENA_STAT_TX_ENTRY(queue_stop), 105 ENA_STAT_TX_ENTRY(queue_wakeup), 106 ENA_STAT_TX_ENTRY(dma_mapping_err), 107 ENA_STAT_TX_ENTRY(linearize), 108 ENA_STAT_TX_ENTRY(linearize_failed), 109 ENA_STAT_TX_ENTRY(napi_comp), 110 ENA_STAT_TX_ENTRY(tx_poll), 111 ENA_STAT_TX_ENTRY(doorbells), 112 ENA_STAT_TX_ENTRY(prepare_ctx_err), 113 ENA_STAT_TX_ENTRY(bad_req_id), 114 ENA_STAT_TX_ENTRY(llq_buffer_copy), 115 ENA_STAT_TX_ENTRY(missed_tx), 116 ENA_STAT_TX_ENTRY(unmask_interrupt), 117 }; 118 119 static const struct ena_stats ena_stats_rx_strings[] = { 120 ENA_STAT_RX_ENTRY(cnt), 121 ENA_STAT_RX_ENTRY(bytes), 122 ENA_STAT_RX_ENTRY(rx_copybreak_pkt), 123 ENA_STAT_RX_ENTRY(csum_good), 124 ENA_STAT_RX_ENTRY(refil_partial), 125 ENA_STAT_RX_ENTRY(csum_bad), 126 ENA_STAT_RX_ENTRY(page_alloc_fail), 127 ENA_STAT_RX_ENTRY(skb_alloc_fail), 128 ENA_STAT_RX_ENTRY(dma_mapping_err), 129 ENA_STAT_RX_ENTRY(bad_desc_num), 130 ENA_STAT_RX_ENTRY(bad_req_id), 131 ENA_STAT_RX_ENTRY(empty_rx_ring), 132 ENA_STAT_RX_ENTRY(csum_unchecked), 133 ENA_STAT_RX_ENTRY(xdp_aborted), 134 ENA_STAT_RX_ENTRY(xdp_drop), 135 ENA_STAT_RX_ENTRY(xdp_pass), 136 ENA_STAT_RX_ENTRY(xdp_tx), 137 ENA_STAT_RX_ENTRY(xdp_invalid), 138 ENA_STAT_RX_ENTRY(xdp_redirect), 139 }; 140 141 static const struct ena_stats ena_stats_ena_com_strings[] = { 142 ENA_STAT_ENA_COM_ENTRY(aborted_cmd), 143 ENA_STAT_ENA_COM_ENTRY(submitted_cmd), 144 ENA_STAT_ENA_COM_ENTRY(completed_cmd), 145 ENA_STAT_ENA_COM_ENTRY(out_of_space), 146 ENA_STAT_ENA_COM_ENTRY(no_completion), 147 }; 148 149 #define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings) 150 #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) 151 #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) 152 #define ENA_STATS_ARRAY_ENA_COM ARRAY_SIZE(ena_stats_ena_com_strings) 153 #define ENA_STATS_ARRAY_ENI ARRAY_SIZE(ena_stats_eni_strings) 154 #define ENA_STATS_ARRAY_ENA_SRD ARRAY_SIZE(ena_srd_info_strings) 155 #define ENA_METRICS_ARRAY_ENI ARRAY_SIZE(ena_hw_stats_strings) 156 157 static void ena_safe_update_stat(u64 *src, u64 *dst, 158 struct u64_stats_sync *syncp) 159 { 160 unsigned int start; 161 162 do { 163 start = u64_stats_fetch_begin(syncp); 164 *(dst) = *src; 165 } while (u64_stats_fetch_retry(syncp, start)); 166 } 167 168 static void ena_metrics_stats(struct ena_adapter *adapter, u64 **data) 169 { 170 struct ena_com_dev *dev = adapter->ena_dev; 171 const struct ena_stats *ena_stats; 172 u64 *ptr; 173 int i; 174 175 if (ena_com_get_cap(dev, ENA_ADMIN_CUSTOMER_METRICS)) { 176 u32 supported_metrics_count; 177 int len; 178 179 supported_metrics_count = ena_com_get_customer_metric_count(dev); 180 len = supported_metrics_count * sizeof(u64); 181 182 /* Fill the data buffer, and advance its pointer */ 183 ena_com_get_customer_metrics(dev, (char *)(*data), len); 184 (*data) += supported_metrics_count; 185 186 } else if (ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS)) { 187 ena_com_get_eni_stats(dev, &adapter->eni_stats); 188 /* Updating regardless of rc - once we told ethtool how many stats we have 189 * it will print that much stats. We can't leave holes in the stats 190 */ 191 for (i = 0; i < ENA_STATS_ARRAY_ENI; i++) { 192 ena_stats = &ena_stats_eni_strings[i]; 193 194 ptr = (u64 *)&adapter->eni_stats + 195 ena_stats->stat_offset; 196 197 ena_safe_update_stat(ptr, (*data)++, &adapter->syncp); 198 } 199 } 200 201 if (ena_com_get_cap(dev, ENA_ADMIN_ENA_SRD_INFO)) { 202 ena_com_get_ena_srd_info(dev, &adapter->ena_srd_info); 203 /* Get ENA SRD mode */ 204 ptr = (u64 *)&adapter->ena_srd_info; 205 ena_safe_update_stat(ptr, (*data)++, &adapter->syncp); 206 for (i = 1; i < ENA_STATS_ARRAY_ENA_SRD; i++) { 207 ena_stats = &ena_srd_info_strings[i]; 208 /* Wrapped within an outer struct - need to accommodate an 209 * additional offset of the ENA SRD mode that was already processed 210 */ 211 ptr = (u64 *)&adapter->ena_srd_info + 212 ena_stats->stat_offset + 1; 213 214 ena_safe_update_stat(ptr, (*data)++, &adapter->syncp); 215 } 216 } 217 } 218 219 static void ena_queue_stats(struct ena_adapter *adapter, u64 **data) 220 { 221 const struct ena_stats *ena_stats; 222 struct ena_ring *ring; 223 224 u64 *ptr; 225 int i, j; 226 227 for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) { 228 /* Tx stats */ 229 ring = &adapter->tx_ring[i]; 230 231 for (j = 0; j < ENA_STATS_ARRAY_TX; j++) { 232 ena_stats = &ena_stats_tx_strings[j]; 233 234 ptr = (u64 *)&ring->tx_stats + ena_stats->stat_offset; 235 236 ena_safe_update_stat(ptr, (*data)++, &ring->syncp); 237 } 238 /* XDP TX queues don't have a RX queue counterpart */ 239 if (!ENA_IS_XDP_INDEX(adapter, i)) { 240 /* Rx stats */ 241 ring = &adapter->rx_ring[i]; 242 243 for (j = 0; j < ENA_STATS_ARRAY_RX; j++) { 244 ena_stats = &ena_stats_rx_strings[j]; 245 246 ptr = (u64 *)&ring->rx_stats + 247 ena_stats->stat_offset; 248 249 ena_safe_update_stat(ptr, (*data)++, &ring->syncp); 250 } 251 } 252 } 253 } 254 255 static void ena_dev_admin_queue_stats(struct ena_adapter *adapter, u64 **data) 256 { 257 const struct ena_stats *ena_stats; 258 u64 *ptr; 259 int i; 260 261 for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) { 262 ena_stats = &ena_stats_ena_com_strings[i]; 263 264 ptr = (u64 *)&adapter->ena_dev->admin_queue.stats + 265 ena_stats->stat_offset; 266 267 *(*data)++ = *ptr; 268 } 269 } 270 271 static void ena_get_stats(struct ena_adapter *adapter, 272 u64 *data, 273 bool hw_stats_needed) 274 { 275 const struct ena_stats *ena_stats; 276 u64 *ptr; 277 int i; 278 279 for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) { 280 ena_stats = &ena_stats_global_strings[i]; 281 282 ptr = (u64 *)&adapter->dev_stats + ena_stats->stat_offset; 283 284 ena_safe_update_stat(ptr, data++, &adapter->syncp); 285 } 286 287 if (hw_stats_needed) 288 ena_metrics_stats(adapter, &data); 289 290 ena_queue_stats(adapter, &data); 291 ena_dev_admin_queue_stats(adapter, &data); 292 } 293 294 static void ena_get_ethtool_stats(struct net_device *netdev, 295 struct ethtool_stats *stats, 296 u64 *data) 297 { 298 struct ena_adapter *adapter = netdev_priv(netdev); 299 300 ena_get_stats(adapter, data, true); 301 } 302 303 static int ena_get_ts_info(struct net_device *netdev, 304 struct kernel_ethtool_ts_info *info) 305 { 306 struct ena_adapter *adapter = netdev_priv(netdev); 307 308 info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE; 309 310 info->phc_index = ena_phc_get_index(adapter); 311 312 return 0; 313 } 314 315 static int ena_get_sw_stats_count(struct ena_adapter *adapter) 316 { 317 return adapter->num_io_queues * (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX) 318 + adapter->xdp_num_queues * ENA_STATS_ARRAY_TX 319 + ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM; 320 } 321 322 static int ena_get_hw_stats_count(struct ena_adapter *adapter) 323 { 324 struct ena_com_dev *dev = adapter->ena_dev; 325 int count; 326 327 count = ENA_STATS_ARRAY_ENA_SRD * ena_com_get_cap(dev, ENA_ADMIN_ENA_SRD_INFO); 328 329 if (ena_com_get_cap(dev, ENA_ADMIN_CUSTOMER_METRICS)) 330 count += ena_com_get_customer_metric_count(dev); 331 else if (ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS)) 332 count += ENA_STATS_ARRAY_ENI; 333 334 return count; 335 } 336 337 int ena_get_sset_count(struct net_device *netdev, int sset) 338 { 339 struct ena_adapter *adapter = netdev_priv(netdev); 340 341 switch (sset) { 342 case ETH_SS_STATS: 343 return ena_get_sw_stats_count(adapter) + 344 ena_get_hw_stats_count(adapter); 345 } 346 347 return -EOPNOTSUPP; 348 } 349 350 static void ena_metrics_stats_strings(struct ena_adapter *adapter, u8 **data) 351 { 352 struct ena_com_dev *dev = adapter->ena_dev; 353 const struct ena_hw_metrics *ena_metrics; 354 const struct ena_stats *ena_stats; 355 int i; 356 357 if (ena_com_get_cap(dev, ENA_ADMIN_CUSTOMER_METRICS)) { 358 for (i = 0; i < ENA_METRICS_ARRAY_ENI; i++) { 359 if (ena_com_get_customer_metric_support(dev, i)) { 360 ena_metrics = &ena_hw_stats_strings[i]; 361 ethtool_puts(data, ena_metrics->name); 362 } 363 } 364 } else if (ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS)) { 365 for (i = 0; i < ENA_STATS_ARRAY_ENI; i++) { 366 ena_stats = &ena_stats_eni_strings[i]; 367 ethtool_puts(data, ena_stats->name); 368 } 369 } 370 371 if (ena_com_get_cap(dev, ENA_ADMIN_ENA_SRD_INFO)) { 372 for (i = 0; i < ENA_STATS_ARRAY_ENA_SRD; i++) { 373 ena_stats = &ena_srd_info_strings[i]; 374 ethtool_puts(data, ena_stats->name); 375 } 376 } 377 } 378 379 static void ena_queue_strings(struct ena_adapter *adapter, u8 **data) 380 { 381 const struct ena_stats *ena_stats; 382 bool is_xdp; 383 int i, j; 384 385 for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) { 386 is_xdp = ENA_IS_XDP_INDEX(adapter, i); 387 /* Tx stats */ 388 for (j = 0; j < ENA_STATS_ARRAY_TX; j++) { 389 ena_stats = &ena_stats_tx_strings[j]; 390 391 ethtool_sprintf(data, 392 "queue_%u_%s_%s", i, 393 is_xdp ? "xdp_tx" : "tx", 394 ena_stats->name); 395 } 396 397 /* In XDP there isn't an RX queue counterpart */ 398 if (is_xdp) 399 continue; 400 401 for (j = 0; j < ENA_STATS_ARRAY_RX; j++) { 402 ena_stats = &ena_stats_rx_strings[j]; 403 404 ethtool_sprintf(data, "queue_%u_rx_%s", i, ena_stats->name); 405 } 406 } 407 } 408 409 static void ena_com_dev_strings(u8 **data) 410 { 411 const struct ena_stats *ena_stats; 412 int i; 413 414 for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) { 415 ena_stats = &ena_stats_ena_com_strings[i]; 416 417 ethtool_sprintf(data, 418 "ena_admin_q_%s", ena_stats->name); 419 } 420 } 421 422 static void ena_get_strings(struct ena_adapter *adapter, 423 u8 *data, 424 bool hw_stats_needed) 425 { 426 const struct ena_stats *ena_stats; 427 int i; 428 429 for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) { 430 ena_stats = &ena_stats_global_strings[i]; 431 ethtool_puts(&data, ena_stats->name); 432 } 433 434 if (hw_stats_needed) 435 ena_metrics_stats_strings(adapter, &data); 436 437 ena_queue_strings(adapter, &data); 438 ena_com_dev_strings(&data); 439 } 440 441 static void ena_get_ethtool_strings(struct net_device *netdev, 442 u32 sset, 443 u8 *data) 444 { 445 struct ena_adapter *adapter = netdev_priv(netdev); 446 447 switch (sset) { 448 case ETH_SS_STATS: 449 ena_get_strings(adapter, data, true); 450 break; 451 } 452 } 453 454 static int ena_get_link_ksettings(struct net_device *netdev, 455 struct ethtool_link_ksettings *link_ksettings) 456 { 457 struct ena_adapter *adapter = netdev_priv(netdev); 458 struct ena_com_dev *ena_dev = adapter->ena_dev; 459 struct ena_admin_get_feature_link_desc *link; 460 struct ena_admin_get_feat_resp feat_resp; 461 int rc; 462 463 rc = ena_com_get_link_params(ena_dev, &feat_resp); 464 if (rc) 465 return rc; 466 467 link = &feat_resp.u.link; 468 link_ksettings->base.speed = link->speed; 469 470 if (link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) { 471 ethtool_link_ksettings_add_link_mode(link_ksettings, 472 supported, Autoneg); 473 ethtool_link_ksettings_add_link_mode(link_ksettings, 474 supported, Autoneg); 475 } 476 477 link_ksettings->base.autoneg = 478 (link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) ? 479 AUTONEG_ENABLE : AUTONEG_DISABLE; 480 481 link_ksettings->base.duplex = DUPLEX_FULL; 482 483 return 0; 484 } 485 486 static int ena_get_coalesce(struct net_device *net_dev, 487 struct ethtool_coalesce *coalesce, 488 struct kernel_ethtool_coalesce *kernel_coal, 489 struct netlink_ext_ack *extack) 490 { 491 struct ena_adapter *adapter = netdev_priv(net_dev); 492 struct ena_com_dev *ena_dev = adapter->ena_dev; 493 494 if (!ena_com_interrupt_moderation_supported(ena_dev)) 495 return -EOPNOTSUPP; 496 497 coalesce->tx_coalesce_usecs = 498 ena_com_get_nonadaptive_moderation_interval_tx(ena_dev) * 499 ena_dev->intr_delay_resolution; 500 501 coalesce->rx_coalesce_usecs = 502 ena_com_get_nonadaptive_moderation_interval_rx(ena_dev) 503 * ena_dev->intr_delay_resolution; 504 505 coalesce->use_adaptive_rx_coalesce = 506 ena_com_get_adaptive_moderation_enabled(ena_dev); 507 508 return 0; 509 } 510 511 static void ena_update_tx_rings_nonadaptive_intr_moderation(struct ena_adapter *adapter) 512 { 513 unsigned int val; 514 int i; 515 516 val = ena_com_get_nonadaptive_moderation_interval_tx(adapter->ena_dev); 517 518 for (i = 0; i < adapter->num_io_queues; i++) 519 adapter->tx_ring[i].smoothed_interval = val; 520 } 521 522 static void ena_update_rx_rings_nonadaptive_intr_moderation(struct ena_adapter *adapter) 523 { 524 unsigned int val; 525 int i; 526 527 val = ena_com_get_nonadaptive_moderation_interval_rx(adapter->ena_dev); 528 529 for (i = 0; i < adapter->num_io_queues; i++) 530 adapter->rx_ring[i].smoothed_interval = val; 531 } 532 533 static int ena_set_coalesce(struct net_device *net_dev, 534 struct ethtool_coalesce *coalesce, 535 struct kernel_ethtool_coalesce *kernel_coal, 536 struct netlink_ext_ack *extack) 537 { 538 struct ena_adapter *adapter = netdev_priv(net_dev); 539 struct ena_com_dev *ena_dev = adapter->ena_dev; 540 int rc; 541 542 if (!ena_com_interrupt_moderation_supported(ena_dev)) 543 return -EOPNOTSUPP; 544 545 rc = ena_com_update_nonadaptive_moderation_interval_tx(ena_dev, 546 coalesce->tx_coalesce_usecs); 547 if (rc) 548 return rc; 549 550 ena_update_tx_rings_nonadaptive_intr_moderation(adapter); 551 552 rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev, 553 coalesce->rx_coalesce_usecs); 554 if (rc) 555 return rc; 556 557 ena_update_rx_rings_nonadaptive_intr_moderation(adapter); 558 559 if (coalesce->use_adaptive_rx_coalesce && 560 !ena_com_get_adaptive_moderation_enabled(ena_dev)) 561 ena_com_enable_adaptive_moderation(ena_dev); 562 563 if (!coalesce->use_adaptive_rx_coalesce && 564 ena_com_get_adaptive_moderation_enabled(ena_dev)) 565 ena_com_disable_adaptive_moderation(ena_dev); 566 567 return 0; 568 } 569 570 static u32 ena_get_msglevel(struct net_device *netdev) 571 { 572 struct ena_adapter *adapter = netdev_priv(netdev); 573 574 return adapter->msg_enable; 575 } 576 577 static void ena_set_msglevel(struct net_device *netdev, u32 value) 578 { 579 struct ena_adapter *adapter = netdev_priv(netdev); 580 581 adapter->msg_enable = value; 582 } 583 584 static void ena_get_drvinfo(struct net_device *dev, 585 struct ethtool_drvinfo *info) 586 { 587 struct ena_adapter *adapter = netdev_priv(dev); 588 ssize_t ret = 0; 589 590 ret = strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); 591 if (ret < 0) 592 netif_dbg(adapter, drv, dev, 593 "module name will be truncated, status = %zd\n", ret); 594 595 ret = strscpy(info->bus_info, pci_name(adapter->pdev), 596 sizeof(info->bus_info)); 597 if (ret < 0) 598 netif_dbg(adapter, drv, dev, 599 "bus info will be truncated, status = %zd\n", ret); 600 } 601 602 static void ena_get_ringparam(struct net_device *netdev, 603 struct ethtool_ringparam *ring, 604 struct kernel_ethtool_ringparam *kernel_ring, 605 struct netlink_ext_ack *extack) 606 { 607 struct ena_adapter *adapter = netdev_priv(netdev); 608 609 ring->tx_max_pending = adapter->max_tx_ring_size; 610 ring->rx_max_pending = adapter->max_rx_ring_size; 611 if (adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 612 bool large_llq_supported = adapter->large_llq_header_supported; 613 614 kernel_ring->tx_push = true; 615 kernel_ring->tx_push_buf_len = adapter->ena_dev->tx_max_header_size; 616 if (large_llq_supported) 617 kernel_ring->tx_push_buf_max_len = ENA_LLQ_LARGE_HEADER; 618 else 619 kernel_ring->tx_push_buf_max_len = ENA_LLQ_HEADER; 620 } else { 621 kernel_ring->tx_push = false; 622 kernel_ring->tx_push_buf_max_len = 0; 623 kernel_ring->tx_push_buf_len = 0; 624 } 625 626 ring->tx_pending = adapter->tx_ring[0].ring_size; 627 ring->rx_pending = adapter->rx_ring[0].ring_size; 628 } 629 630 static int ena_set_ringparam(struct net_device *netdev, 631 struct ethtool_ringparam *ring, 632 struct kernel_ethtool_ringparam *kernel_ring, 633 struct netlink_ext_ack *extack) 634 { 635 struct ena_adapter *adapter = netdev_priv(netdev); 636 u32 new_tx_size, new_rx_size, new_tx_push_buf_len; 637 bool changed = false; 638 639 new_tx_size = ring->tx_pending < ENA_MIN_RING_SIZE ? 640 ENA_MIN_RING_SIZE : ring->tx_pending; 641 new_tx_size = rounddown_pow_of_two(new_tx_size); 642 643 new_rx_size = ring->rx_pending < ENA_MIN_RING_SIZE ? 644 ENA_MIN_RING_SIZE : ring->rx_pending; 645 new_rx_size = rounddown_pow_of_two(new_rx_size); 646 647 changed |= new_tx_size != adapter->requested_tx_ring_size || 648 new_rx_size != adapter->requested_rx_ring_size; 649 650 /* This value is ignored if LLQ is not supported */ 651 new_tx_push_buf_len = adapter->ena_dev->tx_max_header_size; 652 653 if ((adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) != 654 kernel_ring->tx_push) { 655 NL_SET_ERR_MSG_MOD(extack, "Push mode state cannot be modified"); 656 return -EINVAL; 657 } 658 659 /* Validate that the push buffer is supported on the underlying device */ 660 if (kernel_ring->tx_push_buf_len) { 661 enum ena_admin_placement_policy_type placement; 662 663 new_tx_push_buf_len = kernel_ring->tx_push_buf_len; 664 665 placement = adapter->ena_dev->tx_mem_queue_type; 666 if (placement == ENA_ADMIN_PLACEMENT_POLICY_HOST) 667 return -EOPNOTSUPP; 668 669 if (new_tx_push_buf_len != ENA_LLQ_HEADER && 670 new_tx_push_buf_len != ENA_LLQ_LARGE_HEADER) { 671 bool large_llq_sup = adapter->large_llq_header_supported; 672 char large_llq_size_str[40]; 673 674 snprintf(large_llq_size_str, 40, ", %lu", ENA_LLQ_LARGE_HEADER); 675 676 NL_SET_ERR_MSG_FMT_MOD(extack, 677 "Supported tx push buff values: [%lu%s]", 678 ENA_LLQ_HEADER, 679 large_llq_sup ? large_llq_size_str : ""); 680 681 return -EINVAL; 682 } 683 684 changed |= new_tx_push_buf_len != adapter->ena_dev->tx_max_header_size; 685 } 686 687 if (!changed) 688 return 0; 689 690 return ena_update_queue_params(adapter, new_tx_size, new_rx_size, 691 new_tx_push_buf_len); 692 } 693 694 static u32 ena_flow_hash_to_flow_type(u16 hash_fields) 695 { 696 u32 data = 0; 697 698 if (hash_fields & ENA_ADMIN_RSS_L2_DA) 699 data |= RXH_L2DA; 700 701 if (hash_fields & ENA_ADMIN_RSS_L3_DA) 702 data |= RXH_IP_DST; 703 704 if (hash_fields & ENA_ADMIN_RSS_L3_SA) 705 data |= RXH_IP_SRC; 706 707 if (hash_fields & ENA_ADMIN_RSS_L4_DP) 708 data |= RXH_L4_B_2_3; 709 710 if (hash_fields & ENA_ADMIN_RSS_L4_SP) 711 data |= RXH_L4_B_0_1; 712 713 return data; 714 } 715 716 static u16 ena_flow_data_to_flow_hash(u32 hash_fields) 717 { 718 u16 data = 0; 719 720 if (hash_fields & RXH_L2DA) 721 data |= ENA_ADMIN_RSS_L2_DA; 722 723 if (hash_fields & RXH_IP_DST) 724 data |= ENA_ADMIN_RSS_L3_DA; 725 726 if (hash_fields & RXH_IP_SRC) 727 data |= ENA_ADMIN_RSS_L3_SA; 728 729 if (hash_fields & RXH_L4_B_2_3) 730 data |= ENA_ADMIN_RSS_L4_DP; 731 732 if (hash_fields & RXH_L4_B_0_1) 733 data |= ENA_ADMIN_RSS_L4_SP; 734 735 return data; 736 } 737 738 static int ena_get_rxfh_fields(struct net_device *netdev, 739 struct ethtool_rxfh_fields *cmd) 740 { 741 struct ena_adapter *adapter = netdev_priv(netdev); 742 struct ena_com_dev *ena_dev = adapter->ena_dev; 743 enum ena_admin_flow_hash_proto proto; 744 u16 hash_fields; 745 int rc; 746 747 cmd->data = 0; 748 749 switch (cmd->flow_type) { 750 case TCP_V4_FLOW: 751 proto = ENA_ADMIN_RSS_TCP4; 752 break; 753 case UDP_V4_FLOW: 754 proto = ENA_ADMIN_RSS_UDP4; 755 break; 756 case TCP_V6_FLOW: 757 proto = ENA_ADMIN_RSS_TCP6; 758 break; 759 case UDP_V6_FLOW: 760 proto = ENA_ADMIN_RSS_UDP6; 761 break; 762 case IPV4_FLOW: 763 proto = ENA_ADMIN_RSS_IP4; 764 break; 765 case IPV6_FLOW: 766 proto = ENA_ADMIN_RSS_IP6; 767 break; 768 case ETHER_FLOW: 769 proto = ENA_ADMIN_RSS_NOT_IP; 770 break; 771 case AH_V4_FLOW: 772 case ESP_V4_FLOW: 773 case AH_V6_FLOW: 774 case ESP_V6_FLOW: 775 case SCTP_V4_FLOW: 776 case AH_ESP_V4_FLOW: 777 return -EOPNOTSUPP; 778 default: 779 return -EINVAL; 780 } 781 782 rc = ena_com_get_hash_ctrl(ena_dev, proto, &hash_fields); 783 if (rc) 784 return rc; 785 786 cmd->data = ena_flow_hash_to_flow_type(hash_fields); 787 788 return 0; 789 } 790 791 static int ena_set_rxfh_fields(struct net_device *netdev, 792 const struct ethtool_rxfh_fields *cmd, 793 struct netlink_ext_ack *extack) 794 { 795 struct ena_adapter *adapter = netdev_priv(netdev); 796 struct ena_com_dev *ena_dev = adapter->ena_dev; 797 enum ena_admin_flow_hash_proto proto; 798 u16 hash_fields; 799 800 switch (cmd->flow_type) { 801 case TCP_V4_FLOW: 802 proto = ENA_ADMIN_RSS_TCP4; 803 break; 804 case UDP_V4_FLOW: 805 proto = ENA_ADMIN_RSS_UDP4; 806 break; 807 case TCP_V6_FLOW: 808 proto = ENA_ADMIN_RSS_TCP6; 809 break; 810 case UDP_V6_FLOW: 811 proto = ENA_ADMIN_RSS_UDP6; 812 break; 813 case IPV4_FLOW: 814 proto = ENA_ADMIN_RSS_IP4; 815 break; 816 case IPV6_FLOW: 817 proto = ENA_ADMIN_RSS_IP6; 818 break; 819 case ETHER_FLOW: 820 proto = ENA_ADMIN_RSS_NOT_IP; 821 break; 822 case AH_V4_FLOW: 823 case ESP_V4_FLOW: 824 case AH_V6_FLOW: 825 case ESP_V6_FLOW: 826 case SCTP_V4_FLOW: 827 case AH_ESP_V4_FLOW: 828 return -EOPNOTSUPP; 829 default: 830 return -EINVAL; 831 } 832 833 hash_fields = ena_flow_data_to_flow_hash(cmd->data); 834 835 return ena_com_fill_hash_ctrl(ena_dev, proto, hash_fields); 836 } 837 838 static int ena_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info, 839 u32 *rules) 840 { 841 struct ena_adapter *adapter = netdev_priv(netdev); 842 int rc = 0; 843 844 switch (info->cmd) { 845 case ETHTOOL_GRXRINGS: 846 info->data = adapter->num_io_queues; 847 rc = 0; 848 break; 849 case ETHTOOL_GRXCLSRLCNT: 850 case ETHTOOL_GRXCLSRULE: 851 case ETHTOOL_GRXCLSRLALL: 852 default: 853 netif_err(adapter, drv, netdev, 854 "Command parameter %d is not supported\n", info->cmd); 855 rc = -EOPNOTSUPP; 856 } 857 858 return rc; 859 } 860 861 static u32 ena_get_rxfh_indir_size(struct net_device *netdev) 862 { 863 return ENA_RX_RSS_TABLE_SIZE; 864 } 865 866 static u32 ena_get_rxfh_key_size(struct net_device *netdev) 867 { 868 struct ena_adapter *adapter = netdev_priv(netdev); 869 struct ena_rss *rss = &adapter->ena_dev->rss; 870 871 return rss->hash_key ? ENA_HASH_KEY_SIZE : 0; 872 } 873 874 static int ena_indirection_table_set(struct ena_adapter *adapter, 875 const u32 *indir) 876 { 877 struct ena_com_dev *ena_dev = adapter->ena_dev; 878 int i, rc; 879 880 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 881 rc = ena_com_indirect_table_fill_entry(ena_dev, 882 i, 883 ENA_IO_RXQ_IDX(indir[i])); 884 if (unlikely(rc)) { 885 netif_err(adapter, drv, adapter->netdev, 886 "Cannot fill indirect table (index is too large)\n"); 887 return rc; 888 } 889 } 890 891 rc = ena_com_indirect_table_set(ena_dev); 892 if (rc) { 893 netif_err(adapter, drv, adapter->netdev, 894 "Cannot set indirect table\n"); 895 return rc == -EPERM ? -EOPNOTSUPP : rc; 896 } 897 return rc; 898 } 899 900 static int ena_indirection_table_get(struct ena_adapter *adapter, u32 *indir) 901 { 902 struct ena_com_dev *ena_dev = adapter->ena_dev; 903 int i, rc; 904 905 if (!indir) 906 return 0; 907 908 rc = ena_com_indirect_table_get(ena_dev, indir); 909 if (rc) 910 return rc; 911 912 /* Our internal representation of the indices is: even indices 913 * for Tx and uneven indices for Rx. We need to convert the Rx 914 * indices to be consecutive 915 */ 916 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) 917 indir[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(indir[i]); 918 919 return rc; 920 } 921 922 static int ena_get_rxfh(struct net_device *netdev, 923 struct ethtool_rxfh_param *rxfh) 924 { 925 struct ena_adapter *adapter = netdev_priv(netdev); 926 enum ena_admin_hash_functions ena_func; 927 u8 func; 928 int rc; 929 930 rc = ena_indirection_table_get(adapter, rxfh->indir); 931 if (rc) 932 return rc; 933 934 /* We call this function in order to check if the device 935 * supports getting/setting the hash function. 936 */ 937 rc = ena_com_get_hash_function(adapter->ena_dev, &ena_func); 938 if (rc) { 939 if (rc == -EOPNOTSUPP) 940 rc = 0; 941 942 return rc; 943 } 944 945 rc = ena_com_get_hash_key(adapter->ena_dev, rxfh->key); 946 if (rc) 947 return rc; 948 949 switch (ena_func) { 950 case ENA_ADMIN_TOEPLITZ: 951 func = ETH_RSS_HASH_TOP; 952 break; 953 case ENA_ADMIN_CRC32: 954 func = ETH_RSS_HASH_CRC32; 955 break; 956 default: 957 netif_err(adapter, drv, netdev, 958 "Command parameter is not supported\n"); 959 return -EOPNOTSUPP; 960 } 961 962 rxfh->hfunc = func; 963 964 return 0; 965 } 966 967 static int ena_set_rxfh(struct net_device *netdev, 968 struct ethtool_rxfh_param *rxfh, 969 struct netlink_ext_ack *extack) 970 { 971 struct ena_adapter *adapter = netdev_priv(netdev); 972 struct ena_com_dev *ena_dev = adapter->ena_dev; 973 enum ena_admin_hash_functions func = 0; 974 int rc; 975 976 if (rxfh->indir) { 977 rc = ena_indirection_table_set(adapter, rxfh->indir); 978 if (rc) 979 return rc; 980 } 981 982 switch (rxfh->hfunc) { 983 case ETH_RSS_HASH_NO_CHANGE: 984 func = ena_com_get_current_hash_function(ena_dev); 985 break; 986 case ETH_RSS_HASH_TOP: 987 func = ENA_ADMIN_TOEPLITZ; 988 break; 989 case ETH_RSS_HASH_CRC32: 990 func = ENA_ADMIN_CRC32; 991 break; 992 default: 993 netif_err(adapter, drv, netdev, "Unsupported hfunc %d\n", 994 rxfh->hfunc); 995 return -EOPNOTSUPP; 996 } 997 998 if (rxfh->key || func) { 999 rc = ena_com_fill_hash_function(ena_dev, func, rxfh->key, 1000 ENA_HASH_KEY_SIZE, 1001 0xFFFFFFFF); 1002 if (unlikely(rc)) { 1003 netif_err(adapter, drv, netdev, "Cannot fill key\n"); 1004 return rc == -EPERM ? -EOPNOTSUPP : rc; 1005 } 1006 } 1007 1008 return 0; 1009 } 1010 1011 static void ena_get_channels(struct net_device *netdev, 1012 struct ethtool_channels *channels) 1013 { 1014 struct ena_adapter *adapter = netdev_priv(netdev); 1015 1016 channels->max_combined = adapter->max_num_io_queues; 1017 channels->combined_count = adapter->num_io_queues; 1018 } 1019 1020 static int ena_set_channels(struct net_device *netdev, 1021 struct ethtool_channels *channels) 1022 { 1023 struct ena_adapter *adapter = netdev_priv(netdev); 1024 u32 count = channels->combined_count; 1025 /* The check for max value is already done in ethtool */ 1026 if (count < ENA_MIN_NUM_IO_QUEUES) 1027 return -EINVAL; 1028 1029 if (!ena_xdp_legal_queue_count(adapter, count)) { 1030 if (ena_xdp_present(adapter)) 1031 return -EINVAL; 1032 1033 xdp_clear_features_flag(netdev); 1034 } else { 1035 xdp_set_features_flag(netdev, 1036 NETDEV_XDP_ACT_BASIC | 1037 NETDEV_XDP_ACT_REDIRECT); 1038 } 1039 1040 return ena_update_queue_count(adapter, count); 1041 } 1042 1043 static int ena_get_tunable(struct net_device *netdev, 1044 const struct ethtool_tunable *tuna, void *data) 1045 { 1046 struct ena_adapter *adapter = netdev_priv(netdev); 1047 int ret = 0; 1048 1049 switch (tuna->id) { 1050 case ETHTOOL_RX_COPYBREAK: 1051 *(u32 *)data = adapter->rx_copybreak; 1052 break; 1053 default: 1054 ret = -EINVAL; 1055 break; 1056 } 1057 1058 return ret; 1059 } 1060 1061 static int ena_set_tunable(struct net_device *netdev, 1062 const struct ethtool_tunable *tuna, 1063 const void *data) 1064 { 1065 struct ena_adapter *adapter = netdev_priv(netdev); 1066 int ret = 0; 1067 u32 len; 1068 1069 switch (tuna->id) { 1070 case ETHTOOL_RX_COPYBREAK: 1071 len = *(u32 *)data; 1072 ret = ena_set_rx_copybreak(adapter, len); 1073 break; 1074 default: 1075 ret = -EINVAL; 1076 break; 1077 } 1078 1079 return ret; 1080 } 1081 1082 static const struct ethtool_ops ena_ethtool_ops = { 1083 .supported_coalesce_params = ETHTOOL_COALESCE_USECS | 1084 ETHTOOL_COALESCE_USE_ADAPTIVE_RX, 1085 .supported_ring_params = ETHTOOL_RING_USE_TX_PUSH_BUF_LEN | 1086 ETHTOOL_RING_USE_TX_PUSH, 1087 .get_link_ksettings = ena_get_link_ksettings, 1088 .get_drvinfo = ena_get_drvinfo, 1089 .get_msglevel = ena_get_msglevel, 1090 .set_msglevel = ena_set_msglevel, 1091 .get_link = ethtool_op_get_link, 1092 .get_coalesce = ena_get_coalesce, 1093 .set_coalesce = ena_set_coalesce, 1094 .get_ringparam = ena_get_ringparam, 1095 .set_ringparam = ena_set_ringparam, 1096 .get_sset_count = ena_get_sset_count, 1097 .get_strings = ena_get_ethtool_strings, 1098 .get_ethtool_stats = ena_get_ethtool_stats, 1099 .get_rxnfc = ena_get_rxnfc, 1100 .get_rxfh_indir_size = ena_get_rxfh_indir_size, 1101 .get_rxfh_key_size = ena_get_rxfh_key_size, 1102 .get_rxfh = ena_get_rxfh, 1103 .set_rxfh = ena_set_rxfh, 1104 .get_rxfh_fields = ena_get_rxfh_fields, 1105 .set_rxfh_fields = ena_set_rxfh_fields, 1106 .get_channels = ena_get_channels, 1107 .set_channels = ena_set_channels, 1108 .get_tunable = ena_get_tunable, 1109 .set_tunable = ena_set_tunable, 1110 .get_ts_info = ena_get_ts_info, 1111 }; 1112 1113 void ena_set_ethtool_ops(struct net_device *netdev) 1114 { 1115 netdev->ethtool_ops = &ena_ethtool_ops; 1116 } 1117 1118 static void ena_dump_stats_ex(struct ena_adapter *adapter, u8 *buf) 1119 { 1120 struct net_device *netdev = adapter->netdev; 1121 u8 *strings_buf; 1122 u64 *data_buf; 1123 int strings_num; 1124 int i, rc; 1125 1126 strings_num = ena_get_sw_stats_count(adapter); 1127 if (strings_num <= 0) { 1128 netif_err(adapter, drv, netdev, "Can't get stats num\n"); 1129 return; 1130 } 1131 1132 strings_buf = kcalloc(strings_num, ETH_GSTRING_LEN, GFP_ATOMIC); 1133 if (!strings_buf) { 1134 netif_err(adapter, drv, netdev, 1135 "Failed to allocate strings_buf\n"); 1136 return; 1137 } 1138 1139 data_buf = kcalloc(strings_num, sizeof(u64), GFP_ATOMIC); 1140 if (!data_buf) { 1141 netif_err(adapter, drv, netdev, 1142 "Failed to allocate data buf\n"); 1143 kfree(strings_buf); 1144 return; 1145 } 1146 1147 ena_get_strings(adapter, strings_buf, false); 1148 ena_get_stats(adapter, data_buf, false); 1149 1150 /* If there is a buffer, dump stats, otherwise print them to dmesg */ 1151 if (buf) 1152 for (i = 0; i < strings_num; i++) { 1153 rc = snprintf(buf, ETH_GSTRING_LEN + sizeof(u64), 1154 "%s %llu\n", 1155 strings_buf + i * ETH_GSTRING_LEN, 1156 data_buf[i]); 1157 buf += rc; 1158 } 1159 else 1160 for (i = 0; i < strings_num; i++) 1161 netif_err(adapter, drv, netdev, "%s: %llu\n", 1162 strings_buf + i * ETH_GSTRING_LEN, 1163 data_buf[i]); 1164 1165 kfree(strings_buf); 1166 kfree(data_buf); 1167 } 1168 1169 void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf) 1170 { 1171 if (!buf) 1172 return; 1173 1174 ena_dump_stats_ex(adapter, buf); 1175 } 1176 1177 void ena_dump_stats_to_dmesg(struct ena_adapter *adapter) 1178 { 1179 ena_dump_stats_ex(adapter, NULL); 1180 } 1181