1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved. 4 */ 5 6 #include <linux/ethtool.h> 7 #include <linux/pci.h> 8 #include <linux/net_tstamp.h> 9 10 #include "ena_netdev.h" 11 #include "ena_xdp.h" 12 #include "ena_phc.h" 13 14 struct ena_stats { 15 char name[ETH_GSTRING_LEN]; 16 int stat_offset; 17 }; 18 19 struct ena_hw_metrics { 20 char name[ETH_GSTRING_LEN]; 21 }; 22 23 #define ENA_STAT_ENA_COM_ENTRY(stat) { \ 24 .name = #stat, \ 25 .stat_offset = offsetof(struct ena_com_stats_admin, stat) / sizeof(u64) \ 26 } 27 28 #define ENA_STAT_ENTRY(stat, stat_type) { \ 29 .name = #stat, \ 30 .stat_offset = offsetof(struct ena_stats_##stat_type, stat) / sizeof(u64) \ 31 } 32 33 #define ENA_STAT_HW_ENTRY(stat, stat_type) { \ 34 .name = #stat, \ 35 .stat_offset = offsetof(struct ena_admin_##stat_type, stat) / sizeof(u64) \ 36 } 37 38 #define ENA_STAT_RX_ENTRY(stat) \ 39 ENA_STAT_ENTRY(stat, rx) 40 41 #define ENA_STAT_TX_ENTRY(stat) \ 42 ENA_STAT_ENTRY(stat, tx) 43 44 #define ENA_STAT_GLOBAL_ENTRY(stat) \ 45 ENA_STAT_ENTRY(stat, dev) 46 47 #define ENA_STAT_ENI_ENTRY(stat) \ 48 ENA_STAT_HW_ENTRY(stat, eni_stats) 49 50 #define ENA_STAT_ENA_SRD_ENTRY(stat) \ 51 ENA_STAT_HW_ENTRY(stat, ena_srd_stats) 52 53 #define ENA_STAT_ENA_SRD_MODE_ENTRY(stat) { \ 54 .name = #stat, \ 55 .stat_offset = offsetof(struct ena_admin_ena_srd_info, flags) / sizeof(u64) \ 56 } 57 58 #define ENA_METRIC_ENI_ENTRY(stat) { \ 59 .name = #stat \ 60 } 61 62 static const struct ena_stats ena_stats_global_strings[] = { 63 ENA_STAT_GLOBAL_ENTRY(tx_timeout), 64 ENA_STAT_GLOBAL_ENTRY(suspend), 65 ENA_STAT_GLOBAL_ENTRY(resume), 66 ENA_STAT_GLOBAL_ENTRY(wd_expired), 67 ENA_STAT_GLOBAL_ENTRY(interface_up), 68 ENA_STAT_GLOBAL_ENTRY(interface_down), 69 ENA_STAT_GLOBAL_ENTRY(admin_q_pause), 70 ENA_STAT_GLOBAL_ENTRY(reset_fail), 71 }; 72 73 /* A partial list of hw stats. Used when admin command 74 * with type ENA_ADMIN_GET_STATS_TYPE_CUSTOMER_METRICS is not supported 75 */ 76 static const struct ena_stats ena_stats_eni_strings[] = { 77 ENA_STAT_ENI_ENTRY(bw_in_allowance_exceeded), 78 ENA_STAT_ENI_ENTRY(bw_out_allowance_exceeded), 79 ENA_STAT_ENI_ENTRY(pps_allowance_exceeded), 80 ENA_STAT_ENI_ENTRY(conntrack_allowance_exceeded), 81 ENA_STAT_ENI_ENTRY(linklocal_allowance_exceeded), 82 }; 83 84 static const struct ena_hw_metrics ena_hw_stats_strings[] = { 85 ENA_METRIC_ENI_ENTRY(bw_in_allowance_exceeded), 86 ENA_METRIC_ENI_ENTRY(bw_out_allowance_exceeded), 87 ENA_METRIC_ENI_ENTRY(pps_allowance_exceeded), 88 ENA_METRIC_ENI_ENTRY(conntrack_allowance_exceeded), 89 ENA_METRIC_ENI_ENTRY(linklocal_allowance_exceeded), 90 ENA_METRIC_ENI_ENTRY(conntrack_allowance_available), 91 }; 92 93 static const struct ena_stats ena_srd_info_strings[] = { 94 ENA_STAT_ENA_SRD_MODE_ENTRY(ena_srd_mode), 95 ENA_STAT_ENA_SRD_ENTRY(ena_srd_tx_pkts), 96 ENA_STAT_ENA_SRD_ENTRY(ena_srd_eligible_tx_pkts), 97 ENA_STAT_ENA_SRD_ENTRY(ena_srd_rx_pkts), 98 ENA_STAT_ENA_SRD_ENTRY(ena_srd_resource_utilization) 99 }; 100 101 static const struct ena_stats ena_stats_tx_strings[] = { 102 ENA_STAT_TX_ENTRY(cnt), 103 ENA_STAT_TX_ENTRY(bytes), 104 ENA_STAT_TX_ENTRY(queue_stop), 105 ENA_STAT_TX_ENTRY(queue_wakeup), 106 ENA_STAT_TX_ENTRY(dma_mapping_err), 107 ENA_STAT_TX_ENTRY(linearize), 108 ENA_STAT_TX_ENTRY(linearize_failed), 109 ENA_STAT_TX_ENTRY(napi_comp), 110 ENA_STAT_TX_ENTRY(tx_poll), 111 ENA_STAT_TX_ENTRY(doorbells), 112 ENA_STAT_TX_ENTRY(prepare_ctx_err), 113 ENA_STAT_TX_ENTRY(bad_req_id), 114 ENA_STAT_TX_ENTRY(llq_buffer_copy), 115 ENA_STAT_TX_ENTRY(missed_tx), 116 ENA_STAT_TX_ENTRY(unmask_interrupt), 117 }; 118 119 static const struct ena_stats ena_stats_rx_strings[] = { 120 ENA_STAT_RX_ENTRY(cnt), 121 ENA_STAT_RX_ENTRY(bytes), 122 ENA_STAT_RX_ENTRY(rx_copybreak_pkt), 123 ENA_STAT_RX_ENTRY(csum_good), 124 ENA_STAT_RX_ENTRY(refil_partial), 125 ENA_STAT_RX_ENTRY(csum_bad), 126 ENA_STAT_RX_ENTRY(page_alloc_fail), 127 ENA_STAT_RX_ENTRY(skb_alloc_fail), 128 ENA_STAT_RX_ENTRY(dma_mapping_err), 129 ENA_STAT_RX_ENTRY(bad_desc_num), 130 ENA_STAT_RX_ENTRY(bad_req_id), 131 ENA_STAT_RX_ENTRY(empty_rx_ring), 132 ENA_STAT_RX_ENTRY(csum_unchecked), 133 ENA_STAT_RX_ENTRY(xdp_aborted), 134 ENA_STAT_RX_ENTRY(xdp_drop), 135 ENA_STAT_RX_ENTRY(xdp_pass), 136 ENA_STAT_RX_ENTRY(xdp_tx), 137 ENA_STAT_RX_ENTRY(xdp_invalid), 138 ENA_STAT_RX_ENTRY(xdp_redirect), 139 }; 140 141 static const struct ena_stats ena_stats_ena_com_strings[] = { 142 ENA_STAT_ENA_COM_ENTRY(aborted_cmd), 143 ENA_STAT_ENA_COM_ENTRY(submitted_cmd), 144 ENA_STAT_ENA_COM_ENTRY(completed_cmd), 145 ENA_STAT_ENA_COM_ENTRY(out_of_space), 146 ENA_STAT_ENA_COM_ENTRY(no_completion), 147 }; 148 149 #define ENA_STATS_ARRAY_GLOBAL ARRAY_SIZE(ena_stats_global_strings) 150 #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) 151 #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) 152 #define ENA_STATS_ARRAY_ENA_COM ARRAY_SIZE(ena_stats_ena_com_strings) 153 #define ENA_STATS_ARRAY_ENI ARRAY_SIZE(ena_stats_eni_strings) 154 #define ENA_STATS_ARRAY_ENA_SRD ARRAY_SIZE(ena_srd_info_strings) 155 #define ENA_METRICS_ARRAY_ENI ARRAY_SIZE(ena_hw_stats_strings) 156 157 static void ena_safe_update_stat(u64 *src, u64 *dst, 158 struct u64_stats_sync *syncp) 159 { 160 unsigned int start; 161 162 do { 163 start = u64_stats_fetch_begin(syncp); 164 *(dst) = *src; 165 } while (u64_stats_fetch_retry(syncp, start)); 166 } 167 168 static void ena_metrics_stats(struct ena_adapter *adapter, u64 **data) 169 { 170 struct ena_com_dev *dev = adapter->ena_dev; 171 const struct ena_stats *ena_stats; 172 u64 *ptr; 173 int i; 174 175 if (ena_com_get_cap(dev, ENA_ADMIN_CUSTOMER_METRICS)) { 176 u32 supported_metrics_count; 177 int len; 178 179 supported_metrics_count = ena_com_get_customer_metric_count(dev); 180 len = supported_metrics_count * sizeof(u64); 181 182 /* Fill the data buffer, and advance its pointer */ 183 ena_com_get_customer_metrics(dev, (char *)(*data), len); 184 (*data) += supported_metrics_count; 185 186 } else if (ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS)) { 187 ena_com_get_eni_stats(dev, &adapter->eni_stats); 188 /* Updating regardless of rc - once we told ethtool how many stats we have 189 * it will print that much stats. We can't leave holes in the stats 190 */ 191 for (i = 0; i < ENA_STATS_ARRAY_ENI; i++) { 192 ena_stats = &ena_stats_eni_strings[i]; 193 194 ptr = (u64 *)&adapter->eni_stats + 195 ena_stats->stat_offset; 196 197 ena_safe_update_stat(ptr, (*data)++, &adapter->syncp); 198 } 199 } 200 201 if (ena_com_get_cap(dev, ENA_ADMIN_ENA_SRD_INFO)) { 202 ena_com_get_ena_srd_info(dev, &adapter->ena_srd_info); 203 /* Get ENA SRD mode */ 204 ptr = (u64 *)&adapter->ena_srd_info; 205 ena_safe_update_stat(ptr, (*data)++, &adapter->syncp); 206 for (i = 1; i < ENA_STATS_ARRAY_ENA_SRD; i++) { 207 ena_stats = &ena_srd_info_strings[i]; 208 /* Wrapped within an outer struct - need to accommodate an 209 * additional offset of the ENA SRD mode that was already processed 210 */ 211 ptr = (u64 *)&adapter->ena_srd_info + 212 ena_stats->stat_offset + 1; 213 214 ena_safe_update_stat(ptr, (*data)++, &adapter->syncp); 215 } 216 } 217 } 218 219 static void ena_queue_stats(struct ena_adapter *adapter, u64 **data) 220 { 221 const struct ena_stats *ena_stats; 222 struct ena_ring *ring; 223 224 u64 *ptr; 225 int i, j; 226 227 for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) { 228 /* Tx stats */ 229 ring = &adapter->tx_ring[i]; 230 231 for (j = 0; j < ENA_STATS_ARRAY_TX; j++) { 232 ena_stats = &ena_stats_tx_strings[j]; 233 234 ptr = (u64 *)&ring->tx_stats + ena_stats->stat_offset; 235 236 ena_safe_update_stat(ptr, (*data)++, &ring->syncp); 237 } 238 /* XDP TX queues don't have a RX queue counterpart */ 239 if (!ENA_IS_XDP_INDEX(adapter, i)) { 240 /* Rx stats */ 241 ring = &adapter->rx_ring[i]; 242 243 for (j = 0; j < ENA_STATS_ARRAY_RX; j++) { 244 ena_stats = &ena_stats_rx_strings[j]; 245 246 ptr = (u64 *)&ring->rx_stats + 247 ena_stats->stat_offset; 248 249 ena_safe_update_stat(ptr, (*data)++, &ring->syncp); 250 } 251 } 252 } 253 } 254 255 static void ena_dev_admin_queue_stats(struct ena_adapter *adapter, u64 **data) 256 { 257 const struct ena_stats *ena_stats; 258 u64 *ptr; 259 int i; 260 261 for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) { 262 ena_stats = &ena_stats_ena_com_strings[i]; 263 264 ptr = (u64 *)&adapter->ena_dev->admin_queue.stats + 265 ena_stats->stat_offset; 266 267 *(*data)++ = *ptr; 268 } 269 } 270 271 static void ena_get_stats(struct ena_adapter *adapter, 272 u64 *data, 273 bool hw_stats_needed) 274 { 275 const struct ena_stats *ena_stats; 276 u64 *ptr; 277 int i; 278 279 for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) { 280 ena_stats = &ena_stats_global_strings[i]; 281 282 ptr = (u64 *)&adapter->dev_stats + ena_stats->stat_offset; 283 284 ena_safe_update_stat(ptr, data++, &adapter->syncp); 285 } 286 287 if (hw_stats_needed) 288 ena_metrics_stats(adapter, &data); 289 290 ena_queue_stats(adapter, &data); 291 ena_dev_admin_queue_stats(adapter, &data); 292 } 293 294 static void ena_get_ethtool_stats(struct net_device *netdev, 295 struct ethtool_stats *stats, 296 u64 *data) 297 { 298 struct ena_adapter *adapter = netdev_priv(netdev); 299 300 ena_get_stats(adapter, data, true); 301 } 302 303 static int ena_get_ts_info(struct net_device *netdev, 304 struct kernel_ethtool_ts_info *info) 305 { 306 struct ena_adapter *adapter = netdev_priv(netdev); 307 308 info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE; 309 310 info->phc_index = ena_phc_get_index(adapter); 311 312 return 0; 313 } 314 315 static int ena_get_sw_stats_count(struct ena_adapter *adapter) 316 { 317 return adapter->num_io_queues * (ENA_STATS_ARRAY_TX + ENA_STATS_ARRAY_RX) 318 + adapter->xdp_num_queues * ENA_STATS_ARRAY_TX 319 + ENA_STATS_ARRAY_GLOBAL + ENA_STATS_ARRAY_ENA_COM; 320 } 321 322 static int ena_get_hw_stats_count(struct ena_adapter *adapter) 323 { 324 struct ena_com_dev *dev = adapter->ena_dev; 325 int count; 326 327 count = ENA_STATS_ARRAY_ENA_SRD * ena_com_get_cap(dev, ENA_ADMIN_ENA_SRD_INFO); 328 329 if (ena_com_get_cap(dev, ENA_ADMIN_CUSTOMER_METRICS)) 330 count += ena_com_get_customer_metric_count(dev); 331 else if (ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS)) 332 count += ENA_STATS_ARRAY_ENI; 333 334 return count; 335 } 336 337 int ena_get_sset_count(struct net_device *netdev, int sset) 338 { 339 struct ena_adapter *adapter = netdev_priv(netdev); 340 341 switch (sset) { 342 case ETH_SS_STATS: 343 return ena_get_sw_stats_count(adapter) + 344 ena_get_hw_stats_count(adapter); 345 } 346 347 return -EOPNOTSUPP; 348 } 349 350 static void ena_metrics_stats_strings(struct ena_adapter *adapter, u8 **data) 351 { 352 struct ena_com_dev *dev = adapter->ena_dev; 353 const struct ena_hw_metrics *ena_metrics; 354 const struct ena_stats *ena_stats; 355 int i; 356 357 if (ena_com_get_cap(dev, ENA_ADMIN_CUSTOMER_METRICS)) { 358 for (i = 0; i < ENA_METRICS_ARRAY_ENI; i++) { 359 if (ena_com_get_customer_metric_support(dev, i)) { 360 ena_metrics = &ena_hw_stats_strings[i]; 361 ethtool_puts(data, ena_metrics->name); 362 } 363 } 364 } else if (ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS)) { 365 for (i = 0; i < ENA_STATS_ARRAY_ENI; i++) { 366 ena_stats = &ena_stats_eni_strings[i]; 367 ethtool_puts(data, ena_stats->name); 368 } 369 } 370 371 if (ena_com_get_cap(dev, ENA_ADMIN_ENA_SRD_INFO)) { 372 for (i = 0; i < ENA_STATS_ARRAY_ENA_SRD; i++) { 373 ena_stats = &ena_srd_info_strings[i]; 374 ethtool_puts(data, ena_stats->name); 375 } 376 } 377 } 378 379 static void ena_queue_strings(struct ena_adapter *adapter, u8 **data) 380 { 381 const struct ena_stats *ena_stats; 382 bool is_xdp; 383 int i, j; 384 385 for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) { 386 is_xdp = ENA_IS_XDP_INDEX(adapter, i); 387 /* Tx stats */ 388 for (j = 0; j < ENA_STATS_ARRAY_TX; j++) { 389 ena_stats = &ena_stats_tx_strings[j]; 390 391 ethtool_sprintf(data, 392 "queue_%u_%s_%s", i, 393 is_xdp ? "xdp_tx" : "tx", 394 ena_stats->name); 395 } 396 397 /* In XDP there isn't an RX queue counterpart */ 398 if (is_xdp) 399 continue; 400 401 for (j = 0; j < ENA_STATS_ARRAY_RX; j++) { 402 ena_stats = &ena_stats_rx_strings[j]; 403 404 ethtool_sprintf(data, "queue_%u_rx_%s", i, ena_stats->name); 405 } 406 } 407 } 408 409 static void ena_com_dev_strings(u8 **data) 410 { 411 const struct ena_stats *ena_stats; 412 int i; 413 414 for (i = 0; i < ENA_STATS_ARRAY_ENA_COM; i++) { 415 ena_stats = &ena_stats_ena_com_strings[i]; 416 417 ethtool_sprintf(data, 418 "ena_admin_q_%s", ena_stats->name); 419 } 420 } 421 422 static void ena_get_strings(struct ena_adapter *adapter, 423 u8 *data, 424 bool hw_stats_needed) 425 { 426 const struct ena_stats *ena_stats; 427 int i; 428 429 for (i = 0; i < ENA_STATS_ARRAY_GLOBAL; i++) { 430 ena_stats = &ena_stats_global_strings[i]; 431 ethtool_puts(&data, ena_stats->name); 432 } 433 434 if (hw_stats_needed) 435 ena_metrics_stats_strings(adapter, &data); 436 437 ena_queue_strings(adapter, &data); 438 ena_com_dev_strings(&data); 439 } 440 441 static void ena_get_ethtool_strings(struct net_device *netdev, 442 u32 sset, 443 u8 *data) 444 { 445 struct ena_adapter *adapter = netdev_priv(netdev); 446 447 switch (sset) { 448 case ETH_SS_STATS: 449 ena_get_strings(adapter, data, true); 450 break; 451 } 452 } 453 454 static int ena_get_link_ksettings(struct net_device *netdev, 455 struct ethtool_link_ksettings *link_ksettings) 456 { 457 struct ena_adapter *adapter = netdev_priv(netdev); 458 struct ena_com_dev *ena_dev = adapter->ena_dev; 459 struct ena_admin_get_feature_link_desc *link; 460 struct ena_admin_get_feat_resp feat_resp; 461 int rc; 462 463 rc = ena_com_get_link_params(ena_dev, &feat_resp); 464 if (rc) 465 return rc; 466 467 link = &feat_resp.u.link; 468 link_ksettings->base.speed = link->speed; 469 470 if (link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) { 471 ethtool_link_ksettings_add_link_mode(link_ksettings, 472 supported, Autoneg); 473 ethtool_link_ksettings_add_link_mode(link_ksettings, 474 supported, Autoneg); 475 } 476 477 link_ksettings->base.autoneg = 478 (link->flags & ENA_ADMIN_GET_FEATURE_LINK_DESC_AUTONEG_MASK) ? 479 AUTONEG_ENABLE : AUTONEG_DISABLE; 480 481 link_ksettings->base.duplex = DUPLEX_FULL; 482 483 return 0; 484 } 485 486 static int ena_get_coalesce(struct net_device *net_dev, 487 struct ethtool_coalesce *coalesce, 488 struct kernel_ethtool_coalesce *kernel_coal, 489 struct netlink_ext_ack *extack) 490 { 491 struct ena_adapter *adapter = netdev_priv(net_dev); 492 struct ena_com_dev *ena_dev = adapter->ena_dev; 493 494 if (!ena_com_interrupt_moderation_supported(ena_dev)) 495 return -EOPNOTSUPP; 496 497 coalesce->tx_coalesce_usecs = 498 ena_com_get_nonadaptive_moderation_interval_tx(ena_dev) * 499 ena_dev->intr_delay_resolution; 500 501 coalesce->rx_coalesce_usecs = 502 ena_com_get_nonadaptive_moderation_interval_rx(ena_dev) 503 * ena_dev->intr_delay_resolution; 504 505 coalesce->use_adaptive_rx_coalesce = 506 ena_com_get_adaptive_moderation_enabled(ena_dev); 507 508 return 0; 509 } 510 511 static void ena_update_tx_rings_nonadaptive_intr_moderation(struct ena_adapter *adapter) 512 { 513 unsigned int val; 514 int i; 515 516 val = ena_com_get_nonadaptive_moderation_interval_tx(adapter->ena_dev); 517 518 for (i = 0; i < adapter->num_io_queues; i++) 519 adapter->tx_ring[i].smoothed_interval = val; 520 } 521 522 static void ena_update_rx_rings_nonadaptive_intr_moderation(struct ena_adapter *adapter) 523 { 524 unsigned int val; 525 int i; 526 527 val = ena_com_get_nonadaptive_moderation_interval_rx(adapter->ena_dev); 528 529 for (i = 0; i < adapter->num_io_queues; i++) 530 adapter->rx_ring[i].smoothed_interval = val; 531 } 532 533 static int ena_set_coalesce(struct net_device *net_dev, 534 struct ethtool_coalesce *coalesce, 535 struct kernel_ethtool_coalesce *kernel_coal, 536 struct netlink_ext_ack *extack) 537 { 538 struct ena_adapter *adapter = netdev_priv(net_dev); 539 struct ena_com_dev *ena_dev = adapter->ena_dev; 540 int rc; 541 542 if (!ena_com_interrupt_moderation_supported(ena_dev)) 543 return -EOPNOTSUPP; 544 545 rc = ena_com_update_nonadaptive_moderation_interval_tx(ena_dev, 546 coalesce->tx_coalesce_usecs); 547 if (rc) 548 return rc; 549 550 ena_update_tx_rings_nonadaptive_intr_moderation(adapter); 551 552 rc = ena_com_update_nonadaptive_moderation_interval_rx(ena_dev, 553 coalesce->rx_coalesce_usecs); 554 if (rc) 555 return rc; 556 557 ena_update_rx_rings_nonadaptive_intr_moderation(adapter); 558 559 if (coalesce->use_adaptive_rx_coalesce && 560 !ena_com_get_adaptive_moderation_enabled(ena_dev)) 561 ena_com_enable_adaptive_moderation(ena_dev); 562 563 if (!coalesce->use_adaptive_rx_coalesce && 564 ena_com_get_adaptive_moderation_enabled(ena_dev)) 565 ena_com_disable_adaptive_moderation(ena_dev); 566 567 return 0; 568 } 569 570 static u32 ena_get_msglevel(struct net_device *netdev) 571 { 572 struct ena_adapter *adapter = netdev_priv(netdev); 573 574 return adapter->msg_enable; 575 } 576 577 static void ena_set_msglevel(struct net_device *netdev, u32 value) 578 { 579 struct ena_adapter *adapter = netdev_priv(netdev); 580 581 adapter->msg_enable = value; 582 } 583 584 static void ena_get_drvinfo(struct net_device *dev, 585 struct ethtool_drvinfo *info) 586 { 587 struct ena_adapter *adapter = netdev_priv(dev); 588 ssize_t ret = 0; 589 590 ret = strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); 591 if (ret < 0) 592 netif_dbg(adapter, drv, dev, 593 "module name will be truncated, status = %zd\n", ret); 594 595 ret = strscpy(info->bus_info, pci_name(adapter->pdev), 596 sizeof(info->bus_info)); 597 if (ret < 0) 598 netif_dbg(adapter, drv, dev, 599 "bus info will be truncated, status = %zd\n", ret); 600 } 601 602 static void ena_get_ringparam(struct net_device *netdev, 603 struct ethtool_ringparam *ring, 604 struct kernel_ethtool_ringparam *kernel_ring, 605 struct netlink_ext_ack *extack) 606 { 607 struct ena_adapter *adapter = netdev_priv(netdev); 608 609 ring->tx_max_pending = adapter->max_tx_ring_size; 610 ring->rx_max_pending = adapter->max_rx_ring_size; 611 if (adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { 612 bool large_llq_supported = adapter->large_llq_header_supported; 613 614 kernel_ring->tx_push = true; 615 kernel_ring->tx_push_buf_len = adapter->ena_dev->tx_max_header_size; 616 if (large_llq_supported) 617 kernel_ring->tx_push_buf_max_len = ENA_LLQ_LARGE_HEADER; 618 else 619 kernel_ring->tx_push_buf_max_len = ENA_LLQ_HEADER; 620 } else { 621 kernel_ring->tx_push = false; 622 kernel_ring->tx_push_buf_max_len = 0; 623 kernel_ring->tx_push_buf_len = 0; 624 } 625 626 ring->tx_pending = adapter->tx_ring[0].ring_size; 627 ring->rx_pending = adapter->rx_ring[0].ring_size; 628 } 629 630 static int ena_set_ringparam(struct net_device *netdev, 631 struct ethtool_ringparam *ring, 632 struct kernel_ethtool_ringparam *kernel_ring, 633 struct netlink_ext_ack *extack) 634 { 635 struct ena_adapter *adapter = netdev_priv(netdev); 636 u32 new_tx_size, new_rx_size, new_tx_push_buf_len; 637 bool changed = false; 638 639 new_tx_size = ring->tx_pending < ENA_MIN_RING_SIZE ? 640 ENA_MIN_RING_SIZE : ring->tx_pending; 641 new_tx_size = rounddown_pow_of_two(new_tx_size); 642 643 new_rx_size = ring->rx_pending < ENA_MIN_RING_SIZE ? 644 ENA_MIN_RING_SIZE : ring->rx_pending; 645 new_rx_size = rounddown_pow_of_two(new_rx_size); 646 647 changed |= new_tx_size != adapter->requested_tx_ring_size || 648 new_rx_size != adapter->requested_rx_ring_size; 649 650 /* This value is ignored if LLQ is not supported */ 651 new_tx_push_buf_len = adapter->ena_dev->tx_max_header_size; 652 653 if ((adapter->ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) != 654 kernel_ring->tx_push) { 655 NL_SET_ERR_MSG_MOD(extack, "Push mode state cannot be modified"); 656 return -EINVAL; 657 } 658 659 /* Validate that the push buffer is supported on the underlying device */ 660 if (kernel_ring->tx_push_buf_len) { 661 enum ena_admin_placement_policy_type placement; 662 663 new_tx_push_buf_len = kernel_ring->tx_push_buf_len; 664 665 placement = adapter->ena_dev->tx_mem_queue_type; 666 if (placement == ENA_ADMIN_PLACEMENT_POLICY_HOST) 667 return -EOPNOTSUPP; 668 669 if (new_tx_push_buf_len != ENA_LLQ_HEADER && 670 new_tx_push_buf_len != ENA_LLQ_LARGE_HEADER) { 671 bool large_llq_sup = adapter->large_llq_header_supported; 672 char large_llq_size_str[40]; 673 674 snprintf(large_llq_size_str, 40, ", %lu", ENA_LLQ_LARGE_HEADER); 675 676 NL_SET_ERR_MSG_FMT_MOD(extack, 677 "Supported tx push buff values: [%lu%s]", 678 ENA_LLQ_HEADER, 679 large_llq_sup ? large_llq_size_str : ""); 680 681 return -EINVAL; 682 } 683 684 changed |= new_tx_push_buf_len != adapter->ena_dev->tx_max_header_size; 685 } 686 687 if (!changed) 688 return 0; 689 690 return ena_update_queue_params(adapter, new_tx_size, new_rx_size, 691 new_tx_push_buf_len); 692 } 693 694 static u32 ena_flow_hash_to_flow_type(u16 hash_fields) 695 { 696 u32 data = 0; 697 698 if (hash_fields & ENA_ADMIN_RSS_L2_DA) 699 data |= RXH_L2DA; 700 701 if (hash_fields & ENA_ADMIN_RSS_L3_DA) 702 data |= RXH_IP_DST; 703 704 if (hash_fields & ENA_ADMIN_RSS_L3_SA) 705 data |= RXH_IP_SRC; 706 707 if (hash_fields & ENA_ADMIN_RSS_L4_DP) 708 data |= RXH_L4_B_2_3; 709 710 if (hash_fields & ENA_ADMIN_RSS_L4_SP) 711 data |= RXH_L4_B_0_1; 712 713 return data; 714 } 715 716 static u16 ena_flow_data_to_flow_hash(u32 hash_fields) 717 { 718 u16 data = 0; 719 720 if (hash_fields & RXH_L2DA) 721 data |= ENA_ADMIN_RSS_L2_DA; 722 723 if (hash_fields & RXH_IP_DST) 724 data |= ENA_ADMIN_RSS_L3_DA; 725 726 if (hash_fields & RXH_IP_SRC) 727 data |= ENA_ADMIN_RSS_L3_SA; 728 729 if (hash_fields & RXH_L4_B_2_3) 730 data |= ENA_ADMIN_RSS_L4_DP; 731 732 if (hash_fields & RXH_L4_B_0_1) 733 data |= ENA_ADMIN_RSS_L4_SP; 734 735 return data; 736 } 737 738 static int ena_get_rxfh_fields(struct net_device *netdev, 739 struct ethtool_rxfh_fields *cmd) 740 { 741 struct ena_adapter *adapter = netdev_priv(netdev); 742 struct ena_com_dev *ena_dev = adapter->ena_dev; 743 enum ena_admin_flow_hash_proto proto; 744 u16 hash_fields; 745 int rc; 746 747 cmd->data = 0; 748 749 switch (cmd->flow_type) { 750 case TCP_V4_FLOW: 751 proto = ENA_ADMIN_RSS_TCP4; 752 break; 753 case UDP_V4_FLOW: 754 proto = ENA_ADMIN_RSS_UDP4; 755 break; 756 case TCP_V6_FLOW: 757 proto = ENA_ADMIN_RSS_TCP6; 758 break; 759 case UDP_V6_FLOW: 760 proto = ENA_ADMIN_RSS_UDP6; 761 break; 762 case IPV4_FLOW: 763 proto = ENA_ADMIN_RSS_IP4; 764 break; 765 case IPV6_FLOW: 766 proto = ENA_ADMIN_RSS_IP6; 767 break; 768 case ETHER_FLOW: 769 proto = ENA_ADMIN_RSS_NOT_IP; 770 break; 771 case AH_V4_FLOW: 772 case ESP_V4_FLOW: 773 case AH_V6_FLOW: 774 case ESP_V6_FLOW: 775 case SCTP_V4_FLOW: 776 case AH_ESP_V4_FLOW: 777 return -EOPNOTSUPP; 778 default: 779 return -EINVAL; 780 } 781 782 rc = ena_com_get_hash_ctrl(ena_dev, proto, &hash_fields); 783 if (rc) 784 return rc; 785 786 cmd->data = ena_flow_hash_to_flow_type(hash_fields); 787 788 return 0; 789 } 790 791 static int ena_set_rxfh_fields(struct net_device *netdev, 792 const struct ethtool_rxfh_fields *cmd, 793 struct netlink_ext_ack *extack) 794 { 795 struct ena_adapter *adapter = netdev_priv(netdev); 796 struct ena_com_dev *ena_dev = adapter->ena_dev; 797 enum ena_admin_flow_hash_proto proto; 798 u16 hash_fields; 799 800 switch (cmd->flow_type) { 801 case TCP_V4_FLOW: 802 proto = ENA_ADMIN_RSS_TCP4; 803 break; 804 case UDP_V4_FLOW: 805 proto = ENA_ADMIN_RSS_UDP4; 806 break; 807 case TCP_V6_FLOW: 808 proto = ENA_ADMIN_RSS_TCP6; 809 break; 810 case UDP_V6_FLOW: 811 proto = ENA_ADMIN_RSS_UDP6; 812 break; 813 case IPV4_FLOW: 814 proto = ENA_ADMIN_RSS_IP4; 815 break; 816 case IPV6_FLOW: 817 proto = ENA_ADMIN_RSS_IP6; 818 break; 819 case ETHER_FLOW: 820 proto = ENA_ADMIN_RSS_NOT_IP; 821 break; 822 case AH_V4_FLOW: 823 case ESP_V4_FLOW: 824 case AH_V6_FLOW: 825 case ESP_V6_FLOW: 826 case SCTP_V4_FLOW: 827 case AH_ESP_V4_FLOW: 828 return -EOPNOTSUPP; 829 default: 830 return -EINVAL; 831 } 832 833 hash_fields = ena_flow_data_to_flow_hash(cmd->data); 834 835 return ena_com_fill_hash_ctrl(ena_dev, proto, hash_fields); 836 } 837 838 static int ena_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *info, 839 u32 *rules) 840 { 841 struct ena_adapter *adapter = netdev_priv(netdev); 842 int rc = 0; 843 844 switch (info->cmd) { 845 case ETHTOOL_GRXRINGS: 846 info->data = adapter->num_io_queues; 847 rc = 0; 848 break; 849 case ETHTOOL_GRXCLSRLCNT: 850 case ETHTOOL_GRXCLSRULE: 851 case ETHTOOL_GRXCLSRLALL: 852 default: 853 netif_err(adapter, drv, netdev, 854 "Command parameter %d is not supported\n", info->cmd); 855 rc = -EOPNOTSUPP; 856 } 857 858 return rc; 859 } 860 861 static u32 ena_get_rxfh_indir_size(struct net_device *netdev) 862 { 863 return ENA_RX_RSS_TABLE_SIZE; 864 } 865 866 static u32 ena_get_rxfh_key_size(struct net_device *netdev) 867 { 868 return ENA_HASH_KEY_SIZE; 869 } 870 871 static int ena_indirection_table_set(struct ena_adapter *adapter, 872 const u32 *indir) 873 { 874 struct ena_com_dev *ena_dev = adapter->ena_dev; 875 int i, rc; 876 877 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) { 878 rc = ena_com_indirect_table_fill_entry(ena_dev, 879 i, 880 ENA_IO_RXQ_IDX(indir[i])); 881 if (unlikely(rc)) { 882 netif_err(adapter, drv, adapter->netdev, 883 "Cannot fill indirect table (index is too large)\n"); 884 return rc; 885 } 886 } 887 888 rc = ena_com_indirect_table_set(ena_dev); 889 if (rc) { 890 netif_err(adapter, drv, adapter->netdev, 891 "Cannot set indirect table\n"); 892 return rc == -EPERM ? -EOPNOTSUPP : rc; 893 } 894 return rc; 895 } 896 897 static int ena_indirection_table_get(struct ena_adapter *adapter, u32 *indir) 898 { 899 struct ena_com_dev *ena_dev = adapter->ena_dev; 900 int i, rc; 901 902 if (!indir) 903 return 0; 904 905 rc = ena_com_indirect_table_get(ena_dev, indir); 906 if (rc) 907 return rc; 908 909 /* Our internal representation of the indices is: even indices 910 * for Tx and uneven indices for Rx. We need to convert the Rx 911 * indices to be consecutive 912 */ 913 for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) 914 indir[i] = ENA_IO_RXQ_IDX_TO_COMBINED_IDX(indir[i]); 915 916 return rc; 917 } 918 919 static int ena_get_rxfh(struct net_device *netdev, 920 struct ethtool_rxfh_param *rxfh) 921 { 922 struct ena_adapter *adapter = netdev_priv(netdev); 923 enum ena_admin_hash_functions ena_func; 924 u8 func; 925 int rc; 926 927 rc = ena_indirection_table_get(adapter, rxfh->indir); 928 if (rc) 929 return rc; 930 931 /* We call this function in order to check if the device 932 * supports getting/setting the hash function. 933 */ 934 rc = ena_com_get_hash_function(adapter->ena_dev, &ena_func); 935 if (rc) { 936 if (rc == -EOPNOTSUPP) 937 rc = 0; 938 939 return rc; 940 } 941 942 rc = ena_com_get_hash_key(adapter->ena_dev, rxfh->key); 943 if (rc) 944 return rc; 945 946 switch (ena_func) { 947 case ENA_ADMIN_TOEPLITZ: 948 func = ETH_RSS_HASH_TOP; 949 break; 950 case ENA_ADMIN_CRC32: 951 func = ETH_RSS_HASH_CRC32; 952 break; 953 default: 954 netif_err(adapter, drv, netdev, 955 "Command parameter is not supported\n"); 956 return -EOPNOTSUPP; 957 } 958 959 rxfh->hfunc = func; 960 961 return 0; 962 } 963 964 static int ena_set_rxfh(struct net_device *netdev, 965 struct ethtool_rxfh_param *rxfh, 966 struct netlink_ext_ack *extack) 967 { 968 struct ena_adapter *adapter = netdev_priv(netdev); 969 struct ena_com_dev *ena_dev = adapter->ena_dev; 970 enum ena_admin_hash_functions func = 0; 971 int rc; 972 973 if (rxfh->indir) { 974 rc = ena_indirection_table_set(adapter, rxfh->indir); 975 if (rc) 976 return rc; 977 } 978 979 switch (rxfh->hfunc) { 980 case ETH_RSS_HASH_NO_CHANGE: 981 func = ena_com_get_current_hash_function(ena_dev); 982 break; 983 case ETH_RSS_HASH_TOP: 984 func = ENA_ADMIN_TOEPLITZ; 985 break; 986 case ETH_RSS_HASH_CRC32: 987 func = ENA_ADMIN_CRC32; 988 break; 989 default: 990 netif_err(adapter, drv, netdev, "Unsupported hfunc %d\n", 991 rxfh->hfunc); 992 return -EOPNOTSUPP; 993 } 994 995 if (rxfh->key || func) { 996 rc = ena_com_fill_hash_function(ena_dev, func, rxfh->key, 997 ENA_HASH_KEY_SIZE, 998 0xFFFFFFFF); 999 if (unlikely(rc)) { 1000 netif_err(adapter, drv, netdev, "Cannot fill key\n"); 1001 return rc == -EPERM ? -EOPNOTSUPP : rc; 1002 } 1003 } 1004 1005 return 0; 1006 } 1007 1008 static void ena_get_channels(struct net_device *netdev, 1009 struct ethtool_channels *channels) 1010 { 1011 struct ena_adapter *adapter = netdev_priv(netdev); 1012 1013 channels->max_combined = adapter->max_num_io_queues; 1014 channels->combined_count = adapter->num_io_queues; 1015 } 1016 1017 static int ena_set_channels(struct net_device *netdev, 1018 struct ethtool_channels *channels) 1019 { 1020 struct ena_adapter *adapter = netdev_priv(netdev); 1021 u32 count = channels->combined_count; 1022 /* The check for max value is already done in ethtool */ 1023 if (count < ENA_MIN_NUM_IO_QUEUES) 1024 return -EINVAL; 1025 1026 if (!ena_xdp_legal_queue_count(adapter, count)) { 1027 if (ena_xdp_present(adapter)) 1028 return -EINVAL; 1029 1030 xdp_clear_features_flag(netdev); 1031 } else { 1032 xdp_set_features_flag(netdev, 1033 NETDEV_XDP_ACT_BASIC | 1034 NETDEV_XDP_ACT_REDIRECT); 1035 } 1036 1037 return ena_update_queue_count(adapter, count); 1038 } 1039 1040 static int ena_get_tunable(struct net_device *netdev, 1041 const struct ethtool_tunable *tuna, void *data) 1042 { 1043 struct ena_adapter *adapter = netdev_priv(netdev); 1044 int ret = 0; 1045 1046 switch (tuna->id) { 1047 case ETHTOOL_RX_COPYBREAK: 1048 *(u32 *)data = adapter->rx_copybreak; 1049 break; 1050 default: 1051 ret = -EINVAL; 1052 break; 1053 } 1054 1055 return ret; 1056 } 1057 1058 static int ena_set_tunable(struct net_device *netdev, 1059 const struct ethtool_tunable *tuna, 1060 const void *data) 1061 { 1062 struct ena_adapter *adapter = netdev_priv(netdev); 1063 int ret = 0; 1064 u32 len; 1065 1066 switch (tuna->id) { 1067 case ETHTOOL_RX_COPYBREAK: 1068 len = *(u32 *)data; 1069 ret = ena_set_rx_copybreak(adapter, len); 1070 break; 1071 default: 1072 ret = -EINVAL; 1073 break; 1074 } 1075 1076 return ret; 1077 } 1078 1079 static const struct ethtool_ops ena_ethtool_ops = { 1080 .supported_coalesce_params = ETHTOOL_COALESCE_USECS | 1081 ETHTOOL_COALESCE_USE_ADAPTIVE_RX, 1082 .supported_ring_params = ETHTOOL_RING_USE_TX_PUSH_BUF_LEN | 1083 ETHTOOL_RING_USE_TX_PUSH, 1084 .get_link_ksettings = ena_get_link_ksettings, 1085 .get_drvinfo = ena_get_drvinfo, 1086 .get_msglevel = ena_get_msglevel, 1087 .set_msglevel = ena_set_msglevel, 1088 .get_link = ethtool_op_get_link, 1089 .get_coalesce = ena_get_coalesce, 1090 .set_coalesce = ena_set_coalesce, 1091 .get_ringparam = ena_get_ringparam, 1092 .set_ringparam = ena_set_ringparam, 1093 .get_sset_count = ena_get_sset_count, 1094 .get_strings = ena_get_ethtool_strings, 1095 .get_ethtool_stats = ena_get_ethtool_stats, 1096 .get_rxnfc = ena_get_rxnfc, 1097 .get_rxfh_indir_size = ena_get_rxfh_indir_size, 1098 .get_rxfh_key_size = ena_get_rxfh_key_size, 1099 .get_rxfh = ena_get_rxfh, 1100 .set_rxfh = ena_set_rxfh, 1101 .get_rxfh_fields = ena_get_rxfh_fields, 1102 .set_rxfh_fields = ena_set_rxfh_fields, 1103 .get_channels = ena_get_channels, 1104 .set_channels = ena_set_channels, 1105 .get_tunable = ena_get_tunable, 1106 .set_tunable = ena_set_tunable, 1107 .get_ts_info = ena_get_ts_info, 1108 }; 1109 1110 void ena_set_ethtool_ops(struct net_device *netdev) 1111 { 1112 netdev->ethtool_ops = &ena_ethtool_ops; 1113 } 1114 1115 static void ena_dump_stats_ex(struct ena_adapter *adapter, u8 *buf) 1116 { 1117 struct net_device *netdev = adapter->netdev; 1118 u8 *strings_buf; 1119 u64 *data_buf; 1120 int strings_num; 1121 int i, rc; 1122 1123 strings_num = ena_get_sw_stats_count(adapter); 1124 if (strings_num <= 0) { 1125 netif_err(adapter, drv, netdev, "Can't get stats num\n"); 1126 return; 1127 } 1128 1129 strings_buf = kcalloc(strings_num, ETH_GSTRING_LEN, GFP_ATOMIC); 1130 if (!strings_buf) { 1131 netif_err(adapter, drv, netdev, 1132 "Failed to allocate strings_buf\n"); 1133 return; 1134 } 1135 1136 data_buf = kcalloc(strings_num, sizeof(u64), GFP_ATOMIC); 1137 if (!data_buf) { 1138 netif_err(adapter, drv, netdev, 1139 "Failed to allocate data buf\n"); 1140 kfree(strings_buf); 1141 return; 1142 } 1143 1144 ena_get_strings(adapter, strings_buf, false); 1145 ena_get_stats(adapter, data_buf, false); 1146 1147 /* If there is a buffer, dump stats, otherwise print them to dmesg */ 1148 if (buf) 1149 for (i = 0; i < strings_num; i++) { 1150 rc = snprintf(buf, ETH_GSTRING_LEN + sizeof(u64), 1151 "%s %llu\n", 1152 strings_buf + i * ETH_GSTRING_LEN, 1153 data_buf[i]); 1154 buf += rc; 1155 } 1156 else 1157 for (i = 0; i < strings_num; i++) 1158 netif_err(adapter, drv, netdev, "%s: %llu\n", 1159 strings_buf + i * ETH_GSTRING_LEN, 1160 data_buf[i]); 1161 1162 kfree(strings_buf); 1163 kfree(data_buf); 1164 } 1165 1166 void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf) 1167 { 1168 if (!buf) 1169 return; 1170 1171 ena_dump_stats_ex(adapter, buf); 1172 } 1173 1174 void ena_dump_stats_to_dmesg(struct ena_adapter *adapter) 1175 { 1176 ena_dump_stats_ex(adapter, NULL); 1177 } 1178