1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2024 Google LLC 5 */ 6 7 #include <linux/rtnetlink.h> 8 #include "gve.h" 9 #include "gve_adminq.h" 10 #include "gve_dqo.h" 11 #include "gve_utils.h" 12 13 static void gve_get_drvinfo(struct net_device *netdev, 14 struct ethtool_drvinfo *info) 15 { 16 struct gve_priv *priv = netdev_priv(netdev); 17 18 strscpy(info->driver, gve_driver_name, sizeof(info->driver)); 19 strscpy(info->version, gve_version_str, sizeof(info->version)); 20 strscpy(info->bus_info, pci_name(priv->pdev), sizeof(info->bus_info)); 21 } 22 23 static void gve_set_msglevel(struct net_device *netdev, u32 value) 24 { 25 struct gve_priv *priv = netdev_priv(netdev); 26 27 priv->msg_enable = value; 28 } 29 30 static u32 gve_get_msglevel(struct net_device *netdev) 31 { 32 struct gve_priv *priv = netdev_priv(netdev); 33 34 return priv->msg_enable; 35 } 36 37 /* For the following stats column string names, make sure the order 38 * matches how it is filled in the code. For xdp_aborted, xdp_drop, 39 * xdp_pass, xdp_tx, xdp_redirect, make sure it also matches the order 40 * as declared in enum xdp_action inside file uapi/linux/bpf.h . 41 */ 42 static const char gve_gstrings_main_stats[][ETH_GSTRING_LEN] = { 43 "rx_packets", "rx_hsplit_pkt", "tx_packets", "rx_bytes", 44 "tx_bytes", "rx_dropped", "tx_dropped", "tx_timeouts", 45 "rx_skb_alloc_fail", "rx_buf_alloc_fail", "rx_desc_err_dropped_pkt", 46 "rx_hsplit_unsplit_pkt", 47 "interface_up_cnt", "interface_down_cnt", "reset_cnt", 48 "page_alloc_fail", "dma_mapping_error", "stats_report_trigger_cnt", 49 }; 50 51 static const char gve_gstrings_rx_stats[][ETH_GSTRING_LEN] = { 52 "rx_posted_desc[%u]", "rx_completed_desc[%u]", "rx_consumed_desc[%u]", 53 "rx_bytes[%u]", "rx_hsplit_bytes[%u]", "rx_cont_packet_cnt[%u]", 54 "rx_frag_flip_cnt[%u]", "rx_frag_copy_cnt[%u]", "rx_frag_alloc_cnt[%u]", 55 "rx_dropped_pkt[%u]", "rx_copybreak_pkt[%u]", "rx_copied_pkt[%u]", 56 "rx_queue_drop_cnt[%u]", "rx_no_buffers_posted[%u]", 57 "rx_drops_packet_over_mru[%u]", "rx_drops_invalid_checksum[%u]", 58 "rx_xdp_aborted[%u]", "rx_xdp_drop[%u]", "rx_xdp_pass[%u]", 59 "rx_xdp_tx[%u]", "rx_xdp_redirect[%u]", 60 "rx_xdp_tx_errors[%u]", "rx_xdp_redirect_errors[%u]", "rx_xdp_alloc_fails[%u]", 61 }; 62 63 static const char gve_gstrings_tx_stats[][ETH_GSTRING_LEN] = { 64 "tx_posted_desc[%u]", "tx_completed_desc[%u]", "tx_consumed_desc[%u]", "tx_bytes[%u]", 65 "tx_wake[%u]", "tx_stop[%u]", "tx_event_counter[%u]", 66 "tx_dma_mapping_error[%u]", 67 "tx_xsk_sent[%u]", "tx_xdp_xmit[%u]", "tx_xdp_xmit_errors[%u]" 68 }; 69 70 static const char gve_gstrings_adminq_stats[][ETH_GSTRING_LEN] __nonstring_array = { 71 "adminq_prod_cnt", "adminq_cmd_fail", "adminq_timeouts", 72 "adminq_describe_device_cnt", "adminq_cfg_device_resources_cnt", 73 "adminq_register_page_list_cnt", "adminq_unregister_page_list_cnt", 74 "adminq_create_tx_queue_cnt", "adminq_create_rx_queue_cnt", 75 "adminq_destroy_tx_queue_cnt", "adminq_destroy_rx_queue_cnt", 76 "adminq_dcfg_device_resources_cnt", "adminq_set_driver_parameter_cnt", 77 "adminq_report_stats_cnt", "adminq_report_link_speed_cnt", "adminq_get_ptype_map_cnt", 78 "adminq_query_flow_rules", "adminq_cfg_flow_rule", "adminq_cfg_rss_cnt", 79 "adminq_query_rss_cnt", "adminq_report_nic_timestamp_cnt", 80 }; 81 82 static const char gve_gstrings_priv_flags[][ETH_GSTRING_LEN] = { 83 "report-stats", 84 }; 85 86 #define GVE_MAIN_STATS_LEN ARRAY_SIZE(gve_gstrings_main_stats) 87 #define GVE_ADMINQ_STATS_LEN ARRAY_SIZE(gve_gstrings_adminq_stats) 88 #define NUM_GVE_TX_CNTS ARRAY_SIZE(gve_gstrings_tx_stats) 89 #define NUM_GVE_RX_CNTS ARRAY_SIZE(gve_gstrings_rx_stats) 90 #define GVE_PRIV_FLAGS_STR_LEN ARRAY_SIZE(gve_gstrings_priv_flags) 91 92 static void gve_get_strings(struct net_device *netdev, u32 stringset, u8 *data) 93 { 94 struct gve_priv *priv = netdev_priv(netdev); 95 u8 *s = (char *)data; 96 int num_tx_queues; 97 int i, j; 98 99 num_tx_queues = gve_num_tx_queues(priv); 100 switch (stringset) { 101 case ETH_SS_STATS: 102 for (i = 0; i < ARRAY_SIZE(gve_gstrings_main_stats); i++) 103 ethtool_puts(&s, gve_gstrings_main_stats[i]); 104 105 for (i = 0; i < priv->rx_cfg.num_queues; i++) 106 for (j = 0; j < NUM_GVE_RX_CNTS; j++) 107 ethtool_sprintf(&s, gve_gstrings_rx_stats[j], 108 i); 109 110 for (i = 0; i < num_tx_queues; i++) 111 for (j = 0; j < NUM_GVE_TX_CNTS; j++) 112 ethtool_sprintf(&s, gve_gstrings_tx_stats[j], 113 i); 114 115 for (i = 0; i < ARRAY_SIZE(gve_gstrings_adminq_stats); i++) 116 ethtool_cpy(&s, gve_gstrings_adminq_stats[i]); 117 118 break; 119 120 case ETH_SS_PRIV_FLAGS: 121 for (i = 0; i < ARRAY_SIZE(gve_gstrings_priv_flags); i++) 122 ethtool_puts(&s, gve_gstrings_priv_flags[i]); 123 break; 124 125 default: 126 break; 127 } 128 } 129 130 static int gve_get_sset_count(struct net_device *netdev, int sset) 131 { 132 struct gve_priv *priv = netdev_priv(netdev); 133 int num_tx_queues; 134 135 num_tx_queues = gve_num_tx_queues(priv); 136 switch (sset) { 137 case ETH_SS_STATS: 138 return GVE_MAIN_STATS_LEN + GVE_ADMINQ_STATS_LEN + 139 (priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS) + 140 (num_tx_queues * NUM_GVE_TX_CNTS); 141 case ETH_SS_PRIV_FLAGS: 142 return GVE_PRIV_FLAGS_STR_LEN; 143 default: 144 return -EOPNOTSUPP; 145 } 146 } 147 148 static void 149 gve_get_ethtool_stats(struct net_device *netdev, 150 struct ethtool_stats *stats, u64 *data) 151 { 152 u64 tmp_rx_pkts, tmp_rx_hsplit_pkt, tmp_rx_bytes, tmp_rx_hsplit_bytes, 153 tmp_rx_skb_alloc_fail, tmp_rx_buf_alloc_fail, 154 tmp_rx_desc_err_dropped_pkt, tmp_rx_hsplit_unsplit_pkt, 155 tmp_tx_pkts, tmp_tx_bytes; 156 u64 rx_buf_alloc_fail, rx_desc_err_dropped_pkt, rx_hsplit_unsplit_pkt, 157 rx_pkts, rx_hsplit_pkt, rx_skb_alloc_fail, rx_bytes, tx_pkts, tx_bytes, 158 tx_dropped; 159 int stats_idx, base_stats_idx, max_stats_idx; 160 struct stats *report_stats; 161 int *rx_qid_to_stats_idx; 162 int *tx_qid_to_stats_idx; 163 int num_stopped_rxqs = 0; 164 int num_stopped_txqs = 0; 165 struct gve_priv *priv; 166 bool skip_nic_stats; 167 unsigned int start; 168 int num_tx_queues; 169 int ring; 170 int i, j; 171 172 ASSERT_RTNL(); 173 174 priv = netdev_priv(netdev); 175 num_tx_queues = gve_num_tx_queues(priv); 176 report_stats = priv->stats_report->stats; 177 rx_qid_to_stats_idx = kmalloc_array(priv->rx_cfg.num_queues, 178 sizeof(int), GFP_KERNEL); 179 if (!rx_qid_to_stats_idx) 180 return; 181 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { 182 rx_qid_to_stats_idx[ring] = -1; 183 if (!gve_rx_was_added_to_block(priv, ring)) 184 num_stopped_rxqs++; 185 } 186 tx_qid_to_stats_idx = kmalloc_array(num_tx_queues, 187 sizeof(int), GFP_KERNEL); 188 if (!tx_qid_to_stats_idx) { 189 kfree(rx_qid_to_stats_idx); 190 return; 191 } 192 for (ring = 0; ring < num_tx_queues; ring++) { 193 tx_qid_to_stats_idx[ring] = -1; 194 if (!gve_tx_was_added_to_block(priv, ring)) 195 num_stopped_txqs++; 196 } 197 198 for (rx_pkts = 0, rx_bytes = 0, rx_hsplit_pkt = 0, 199 rx_skb_alloc_fail = 0, rx_buf_alloc_fail = 0, 200 rx_desc_err_dropped_pkt = 0, rx_hsplit_unsplit_pkt = 0, 201 ring = 0; 202 ring < priv->rx_cfg.num_queues; ring++) { 203 if (priv->rx) { 204 do { 205 struct gve_rx_ring *rx = &priv->rx[ring]; 206 207 start = 208 u64_stats_fetch_begin(&priv->rx[ring].statss); 209 tmp_rx_pkts = rx->rpackets; 210 tmp_rx_hsplit_pkt = rx->rx_hsplit_pkt; 211 tmp_rx_bytes = rx->rbytes; 212 tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail; 213 tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail; 214 tmp_rx_desc_err_dropped_pkt = 215 rx->rx_desc_err_dropped_pkt; 216 tmp_rx_hsplit_unsplit_pkt = 217 rx->rx_hsplit_unsplit_pkt; 218 } while (u64_stats_fetch_retry(&priv->rx[ring].statss, 219 start)); 220 rx_pkts += tmp_rx_pkts; 221 rx_hsplit_pkt += tmp_rx_hsplit_pkt; 222 rx_bytes += tmp_rx_bytes; 223 rx_skb_alloc_fail += tmp_rx_skb_alloc_fail; 224 rx_buf_alloc_fail += tmp_rx_buf_alloc_fail; 225 rx_desc_err_dropped_pkt += tmp_rx_desc_err_dropped_pkt; 226 rx_hsplit_unsplit_pkt += tmp_rx_hsplit_unsplit_pkt; 227 } 228 } 229 for (tx_pkts = 0, tx_bytes = 0, tx_dropped = 0, ring = 0; 230 ring < num_tx_queues; ring++) { 231 if (priv->tx) { 232 do { 233 start = 234 u64_stats_fetch_begin(&priv->tx[ring].statss); 235 tmp_tx_pkts = priv->tx[ring].pkt_done; 236 tmp_tx_bytes = priv->tx[ring].bytes_done; 237 } while (u64_stats_fetch_retry(&priv->tx[ring].statss, 238 start)); 239 tx_pkts += tmp_tx_pkts; 240 tx_bytes += tmp_tx_bytes; 241 tx_dropped += priv->tx[ring].dropped_pkt; 242 } 243 } 244 245 i = 0; 246 data[i++] = rx_pkts; 247 data[i++] = rx_hsplit_pkt; 248 data[i++] = tx_pkts; 249 data[i++] = rx_bytes; 250 data[i++] = tx_bytes; 251 /* total rx dropped packets */ 252 data[i++] = rx_skb_alloc_fail + rx_buf_alloc_fail + 253 rx_desc_err_dropped_pkt; 254 data[i++] = tx_dropped; 255 data[i++] = priv->tx_timeo_cnt; 256 data[i++] = rx_skb_alloc_fail; 257 data[i++] = rx_buf_alloc_fail; 258 data[i++] = rx_desc_err_dropped_pkt; 259 data[i++] = rx_hsplit_unsplit_pkt; 260 data[i++] = priv->interface_up_cnt; 261 data[i++] = priv->interface_down_cnt; 262 data[i++] = priv->reset_cnt; 263 data[i++] = priv->page_alloc_fail; 264 data[i++] = priv->dma_mapping_error; 265 data[i++] = priv->stats_report_trigger_cnt; 266 i = GVE_MAIN_STATS_LEN; 267 268 /* For rx cross-reporting stats, start from nic rx stats in report */ 269 base_stats_idx = GVE_TX_STATS_REPORT_NUM * num_tx_queues + 270 GVE_RX_STATS_REPORT_NUM * priv->rx_cfg.num_queues; 271 /* The boundary between driver stats and NIC stats shifts if there are 272 * stopped queues. 273 */ 274 base_stats_idx += NIC_RX_STATS_REPORT_NUM * num_stopped_rxqs + 275 NIC_TX_STATS_REPORT_NUM * num_stopped_txqs; 276 max_stats_idx = NIC_RX_STATS_REPORT_NUM * 277 (priv->rx_cfg.num_queues - num_stopped_rxqs) + 278 base_stats_idx; 279 /* Preprocess the stats report for rx, map queue id to start index */ 280 skip_nic_stats = false; 281 for (stats_idx = base_stats_idx; stats_idx < max_stats_idx; 282 stats_idx += NIC_RX_STATS_REPORT_NUM) { 283 u32 stat_name = be32_to_cpu(report_stats[stats_idx].stat_name); 284 u32 queue_id = be32_to_cpu(report_stats[stats_idx].queue_id); 285 286 if (stat_name == 0) { 287 /* no stats written by NIC yet */ 288 skip_nic_stats = true; 289 break; 290 } 291 if (queue_id < 0 || queue_id >= priv->rx_cfg.num_queues) { 292 net_err_ratelimited("Invalid rxq id in NIC stats\n"); 293 continue; 294 } 295 rx_qid_to_stats_idx[queue_id] = stats_idx; 296 } 297 /* walk RX rings */ 298 if (priv->rx) { 299 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { 300 struct gve_rx_ring *rx = &priv->rx[ring]; 301 302 data[i++] = rx->fill_cnt; 303 data[i++] = rx->cnt; 304 data[i++] = rx->fill_cnt - rx->cnt; 305 do { 306 start = 307 u64_stats_fetch_begin(&priv->rx[ring].statss); 308 tmp_rx_bytes = rx->rbytes; 309 tmp_rx_hsplit_bytes = rx->rx_hsplit_bytes; 310 tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail; 311 tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail; 312 tmp_rx_desc_err_dropped_pkt = 313 rx->rx_desc_err_dropped_pkt; 314 } while (u64_stats_fetch_retry(&priv->rx[ring].statss, 315 start)); 316 data[i++] = tmp_rx_bytes; 317 data[i++] = tmp_rx_hsplit_bytes; 318 data[i++] = rx->rx_cont_packet_cnt; 319 data[i++] = rx->rx_frag_flip_cnt; 320 data[i++] = rx->rx_frag_copy_cnt; 321 data[i++] = rx->rx_frag_alloc_cnt; 322 /* rx dropped packets */ 323 data[i++] = tmp_rx_skb_alloc_fail + 324 tmp_rx_buf_alloc_fail + 325 tmp_rx_desc_err_dropped_pkt; 326 data[i++] = rx->rx_copybreak_pkt; 327 data[i++] = rx->rx_copied_pkt; 328 /* stats from NIC */ 329 stats_idx = rx_qid_to_stats_idx[ring]; 330 if (skip_nic_stats || stats_idx < 0) { 331 /* skip NIC rx stats */ 332 i += NIC_RX_STATS_REPORT_NUM; 333 } else { 334 for (j = 0; j < NIC_RX_STATS_REPORT_NUM; j++) { 335 u64 value = 336 be64_to_cpu(report_stats[stats_idx + j].value); 337 338 data[i++] = value; 339 } 340 } 341 /* XDP rx counters */ 342 do { 343 start = u64_stats_fetch_begin(&priv->rx[ring].statss); 344 for (j = 0; j < GVE_XDP_ACTIONS; j++) 345 data[i + j] = rx->xdp_actions[j]; 346 data[i + j++] = rx->xdp_tx_errors; 347 data[i + j++] = rx->xdp_redirect_errors; 348 data[i + j++] = rx->xdp_alloc_fails; 349 } while (u64_stats_fetch_retry(&priv->rx[ring].statss, 350 start)); 351 i += GVE_XDP_ACTIONS + 3; /* XDP rx counters */ 352 } 353 } else { 354 i += priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS; 355 } 356 357 /* For tx cross-reporting stats, start from nic tx stats in report */ 358 base_stats_idx = max_stats_idx; 359 max_stats_idx = NIC_TX_STATS_REPORT_NUM * 360 (num_tx_queues - num_stopped_txqs) + 361 max_stats_idx; 362 /* Preprocess the stats report for tx, map queue id to start index */ 363 skip_nic_stats = false; 364 for (stats_idx = base_stats_idx; stats_idx < max_stats_idx; 365 stats_idx += NIC_TX_STATS_REPORT_NUM) { 366 u32 stat_name = be32_to_cpu(report_stats[stats_idx].stat_name); 367 u32 queue_id = be32_to_cpu(report_stats[stats_idx].queue_id); 368 369 if (stat_name == 0) { 370 /* no stats written by NIC yet */ 371 skip_nic_stats = true; 372 break; 373 } 374 if (queue_id < 0 || queue_id >= num_tx_queues) { 375 net_err_ratelimited("Invalid txq id in NIC stats\n"); 376 continue; 377 } 378 tx_qid_to_stats_idx[queue_id] = stats_idx; 379 } 380 /* walk TX rings */ 381 if (priv->tx) { 382 for (ring = 0; ring < num_tx_queues; ring++) { 383 struct gve_tx_ring *tx = &priv->tx[ring]; 384 385 if (gve_is_gqi(priv)) { 386 data[i++] = tx->req; 387 data[i++] = tx->done; 388 data[i++] = tx->req - tx->done; 389 } else { 390 /* DQO doesn't currently support 391 * posted/completed descriptor counts; 392 */ 393 data[i++] = 0; 394 data[i++] = 0; 395 data[i++] = 396 (tx->dqo_tx.tail - tx->dqo_tx.head) & 397 tx->mask; 398 } 399 do { 400 start = 401 u64_stats_fetch_begin(&priv->tx[ring].statss); 402 tmp_tx_bytes = tx->bytes_done; 403 } while (u64_stats_fetch_retry(&priv->tx[ring].statss, 404 start)); 405 data[i++] = tmp_tx_bytes; 406 data[i++] = tx->wake_queue; 407 data[i++] = tx->stop_queue; 408 data[i++] = gve_tx_load_event_counter(priv, tx); 409 data[i++] = tx->dma_mapping_error; 410 /* stats from NIC */ 411 stats_idx = tx_qid_to_stats_idx[ring]; 412 if (skip_nic_stats || stats_idx < 0) { 413 /* skip NIC tx stats */ 414 i += NIC_TX_STATS_REPORT_NUM; 415 } else { 416 for (j = 0; j < NIC_TX_STATS_REPORT_NUM; j++) { 417 u64 value = 418 be64_to_cpu(report_stats[stats_idx + j].value); 419 data[i++] = value; 420 } 421 } 422 /* XDP counters */ 423 do { 424 start = u64_stats_fetch_begin(&priv->tx[ring].statss); 425 data[i] = tx->xdp_xsk_sent; 426 data[i + 1] = tx->xdp_xmit; 427 data[i + 2] = tx->xdp_xmit_errors; 428 } while (u64_stats_fetch_retry(&priv->tx[ring].statss, 429 start)); 430 i += 3; /* XDP tx counters */ 431 } 432 } else { 433 i += num_tx_queues * NUM_GVE_TX_CNTS; 434 } 435 436 kfree(rx_qid_to_stats_idx); 437 kfree(tx_qid_to_stats_idx); 438 /* AQ Stats */ 439 data[i++] = priv->adminq_prod_cnt; 440 data[i++] = priv->adminq_cmd_fail; 441 data[i++] = priv->adminq_timeouts; 442 data[i++] = priv->adminq_describe_device_cnt; 443 data[i++] = priv->adminq_cfg_device_resources_cnt; 444 data[i++] = priv->adminq_register_page_list_cnt; 445 data[i++] = priv->adminq_unregister_page_list_cnt; 446 data[i++] = priv->adminq_create_tx_queue_cnt; 447 data[i++] = priv->adminq_create_rx_queue_cnt; 448 data[i++] = priv->adminq_destroy_tx_queue_cnt; 449 data[i++] = priv->adminq_destroy_rx_queue_cnt; 450 data[i++] = priv->adminq_dcfg_device_resources_cnt; 451 data[i++] = priv->adminq_set_driver_parameter_cnt; 452 data[i++] = priv->adminq_report_stats_cnt; 453 data[i++] = priv->adminq_report_link_speed_cnt; 454 data[i++] = priv->adminq_get_ptype_map_cnt; 455 data[i++] = priv->adminq_query_flow_rules_cnt; 456 data[i++] = priv->adminq_cfg_flow_rule_cnt; 457 data[i++] = priv->adminq_cfg_rss_cnt; 458 data[i++] = priv->adminq_query_rss_cnt; 459 data[i++] = priv->adminq_report_nic_timestamp_cnt; 460 } 461 462 static void gve_get_channels(struct net_device *netdev, 463 struct ethtool_channels *cmd) 464 { 465 struct gve_priv *priv = netdev_priv(netdev); 466 467 cmd->max_rx = priv->rx_cfg.max_queues; 468 cmd->max_tx = priv->tx_cfg.max_queues; 469 cmd->max_other = 0; 470 cmd->max_combined = 0; 471 cmd->rx_count = priv->rx_cfg.num_queues; 472 cmd->tx_count = priv->tx_cfg.num_queues; 473 cmd->other_count = 0; 474 cmd->combined_count = 0; 475 } 476 477 static int gve_set_channels(struct net_device *netdev, 478 struct ethtool_channels *cmd) 479 { 480 struct gve_priv *priv = netdev_priv(netdev); 481 struct gve_tx_queue_config new_tx_cfg = priv->tx_cfg; 482 struct gve_rx_queue_config new_rx_cfg = priv->rx_cfg; 483 struct ethtool_channels old_settings; 484 int new_tx = cmd->tx_count; 485 int new_rx = cmd->rx_count; 486 bool reset_rss = false; 487 488 gve_get_channels(netdev, &old_settings); 489 490 /* Changing combined is not allowed */ 491 if (cmd->combined_count != old_settings.combined_count) 492 return -EINVAL; 493 494 if (!new_rx || !new_tx) 495 return -EINVAL; 496 497 if (priv->xdp_prog) { 498 if (new_tx != new_rx || 499 (2 * new_tx > priv->tx_cfg.max_queues)) { 500 dev_err(&priv->pdev->dev, "The number of configured RX queues should be equal to the number of configured TX queues and the number of configured RX/TX queues should be less than or equal to half the maximum number of RX/TX queues when XDP program is installed"); 501 return -EINVAL; 502 } 503 504 /* One XDP TX queue per RX queue. */ 505 new_tx_cfg.num_xdp_queues = new_rx; 506 } else { 507 new_tx_cfg.num_xdp_queues = 0; 508 } 509 510 if (new_rx != priv->rx_cfg.num_queues && 511 priv->cache_rss_config && !netif_is_rxfh_configured(netdev)) 512 reset_rss = true; 513 514 new_tx_cfg.num_queues = new_tx; 515 new_rx_cfg.num_queues = new_rx; 516 517 return gve_adjust_queues(priv, new_rx_cfg, new_tx_cfg, reset_rss); 518 } 519 520 static void gve_get_ringparam(struct net_device *netdev, 521 struct ethtool_ringparam *cmd, 522 struct kernel_ethtool_ringparam *kernel_cmd, 523 struct netlink_ext_ack *extack) 524 { 525 struct gve_priv *priv = netdev_priv(netdev); 526 527 cmd->rx_max_pending = priv->max_rx_desc_cnt; 528 cmd->tx_max_pending = priv->max_tx_desc_cnt; 529 cmd->rx_pending = priv->rx_desc_cnt; 530 cmd->tx_pending = priv->tx_desc_cnt; 531 532 kernel_cmd->rx_buf_len = priv->rx_cfg.packet_buffer_size; 533 534 if (!gve_header_split_supported(priv)) 535 kernel_cmd->tcp_data_split = ETHTOOL_TCP_DATA_SPLIT_UNKNOWN; 536 else if (priv->header_split_enabled) 537 kernel_cmd->tcp_data_split = ETHTOOL_TCP_DATA_SPLIT_ENABLED; 538 else 539 kernel_cmd->tcp_data_split = ETHTOOL_TCP_DATA_SPLIT_DISABLED; 540 } 541 542 static int gve_validate_req_ring_size(struct gve_priv *priv, u16 new_tx_desc_cnt, 543 u16 new_rx_desc_cnt) 544 { 545 /* check for valid range */ 546 if (new_tx_desc_cnt < priv->min_tx_desc_cnt || 547 new_tx_desc_cnt > priv->max_tx_desc_cnt || 548 new_rx_desc_cnt < priv->min_rx_desc_cnt || 549 new_rx_desc_cnt > priv->max_rx_desc_cnt) { 550 dev_err(&priv->pdev->dev, "Requested descriptor count out of range\n"); 551 return -EINVAL; 552 } 553 554 if (!is_power_of_2(new_tx_desc_cnt) || !is_power_of_2(new_rx_desc_cnt)) { 555 dev_err(&priv->pdev->dev, "Requested descriptor count has to be a power of 2\n"); 556 return -EINVAL; 557 } 558 return 0; 559 } 560 561 static int gve_set_ring_sizes_config(struct gve_priv *priv, u16 new_tx_desc_cnt, 562 u16 new_rx_desc_cnt, 563 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, 564 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) 565 { 566 if (new_tx_desc_cnt == priv->tx_desc_cnt && 567 new_rx_desc_cnt == priv->rx_desc_cnt) 568 return 0; 569 570 if (!priv->modify_ring_size_enabled) { 571 dev_err(&priv->pdev->dev, "Modify ring size is not supported.\n"); 572 return -EOPNOTSUPP; 573 } 574 575 if (gve_validate_req_ring_size(priv, new_tx_desc_cnt, new_rx_desc_cnt)) 576 return -EINVAL; 577 578 tx_alloc_cfg->ring_size = new_tx_desc_cnt; 579 rx_alloc_cfg->ring_size = new_rx_desc_cnt; 580 return 0; 581 } 582 583 static int gve_set_ringparam(struct net_device *netdev, 584 struct ethtool_ringparam *cmd, 585 struct kernel_ethtool_ringparam *kernel_cmd, 586 struct netlink_ext_ack *extack) 587 { 588 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 589 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 590 struct gve_priv *priv = netdev_priv(netdev); 591 int err; 592 593 gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 594 595 err = gve_set_rx_buf_len_config(priv, kernel_cmd->rx_buf_len, extack, 596 &rx_alloc_cfg); 597 if (err) 598 return err; 599 600 err = gve_set_hsplit_config(priv, kernel_cmd->tcp_data_split, 601 &rx_alloc_cfg); 602 if (err) 603 return err; 604 605 err = gve_set_ring_sizes_config(priv, cmd->tx_pending, cmd->rx_pending, 606 &tx_alloc_cfg, &rx_alloc_cfg); 607 if (err) 608 return err; 609 610 if (netif_running(priv->dev)) { 611 err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); 612 if (err) 613 return err; 614 } else { 615 /* Set ring params for the next up */ 616 priv->rx_cfg.packet_buffer_size = 617 rx_alloc_cfg.packet_buffer_size; 618 priv->header_split_enabled = rx_alloc_cfg.enable_header_split; 619 priv->tx_desc_cnt = tx_alloc_cfg.ring_size; 620 priv->rx_desc_cnt = rx_alloc_cfg.ring_size; 621 } 622 return 0; 623 } 624 625 static int gve_user_reset(struct net_device *netdev, u32 *flags) 626 { 627 struct gve_priv *priv = netdev_priv(netdev); 628 629 if (*flags == ETH_RESET_ALL) { 630 *flags = 0; 631 return gve_reset(priv, true); 632 } 633 634 return -EOPNOTSUPP; 635 } 636 637 static int gve_get_tunable(struct net_device *netdev, 638 const struct ethtool_tunable *etuna, void *value) 639 { 640 struct gve_priv *priv = netdev_priv(netdev); 641 642 switch (etuna->id) { 643 case ETHTOOL_RX_COPYBREAK: 644 *(u32 *)value = priv->rx_copybreak; 645 return 0; 646 default: 647 return -EOPNOTSUPP; 648 } 649 } 650 651 static int gve_set_tunable(struct net_device *netdev, 652 const struct ethtool_tunable *etuna, 653 const void *value) 654 { 655 struct gve_priv *priv = netdev_priv(netdev); 656 u32 len; 657 658 switch (etuna->id) { 659 case ETHTOOL_RX_COPYBREAK: 660 { 661 u32 max_copybreak = priv->rx_cfg.packet_buffer_size; 662 663 len = *(u32 *)value; 664 if (len > max_copybreak) 665 return -EINVAL; 666 priv->rx_copybreak = len; 667 return 0; 668 } 669 default: 670 return -EOPNOTSUPP; 671 } 672 } 673 674 static u32 gve_get_priv_flags(struct net_device *netdev) 675 { 676 struct gve_priv *priv = netdev_priv(netdev); 677 u32 ret_flags = 0; 678 679 /* Only 1 flag exists currently: report-stats (BIT(0)), so set that flag. */ 680 if (priv->ethtool_flags & BIT(0)) 681 ret_flags |= BIT(0); 682 return ret_flags; 683 } 684 685 static int gve_set_priv_flags(struct net_device *netdev, u32 flags) 686 { 687 struct gve_priv *priv = netdev_priv(netdev); 688 u64 ori_flags, new_flags; 689 int num_tx_queues; 690 691 num_tx_queues = gve_num_tx_queues(priv); 692 ori_flags = READ_ONCE(priv->ethtool_flags); 693 new_flags = ori_flags; 694 695 /* Only one priv flag exists: report-stats (BIT(0))*/ 696 if (flags & BIT(0)) 697 new_flags |= BIT(0); 698 else 699 new_flags &= ~(BIT(0)); 700 priv->ethtool_flags = new_flags; 701 /* start report-stats timer when user turns report stats on. */ 702 if (flags & BIT(0)) { 703 mod_timer(&priv->stats_report_timer, 704 round_jiffies(jiffies + 705 msecs_to_jiffies(priv->stats_report_timer_period))); 706 } 707 /* Zero off gve stats when report-stats turned off and */ 708 /* delete report stats timer. */ 709 if (!(flags & BIT(0)) && (ori_flags & BIT(0))) { 710 int tx_stats_num = GVE_TX_STATS_REPORT_NUM * 711 num_tx_queues; 712 int rx_stats_num = GVE_RX_STATS_REPORT_NUM * 713 priv->rx_cfg.num_queues; 714 715 memset(priv->stats_report->stats, 0, (tx_stats_num + rx_stats_num) * 716 sizeof(struct stats)); 717 timer_delete_sync(&priv->stats_report_timer); 718 } 719 return 0; 720 } 721 722 static int gve_get_link_ksettings(struct net_device *netdev, 723 struct ethtool_link_ksettings *cmd) 724 { 725 struct gve_priv *priv = netdev_priv(netdev); 726 int err = 0; 727 728 if (priv->link_speed == 0) 729 err = gve_adminq_report_link_speed(priv); 730 731 cmd->base.speed = priv->link_speed; 732 733 cmd->base.duplex = DUPLEX_FULL; 734 735 return err; 736 } 737 738 static int gve_get_coalesce(struct net_device *netdev, 739 struct ethtool_coalesce *ec, 740 struct kernel_ethtool_coalesce *kernel_ec, 741 struct netlink_ext_ack *extack) 742 { 743 struct gve_priv *priv = netdev_priv(netdev); 744 745 if (gve_is_gqi(priv)) 746 return -EOPNOTSUPP; 747 ec->tx_coalesce_usecs = priv->tx_coalesce_usecs; 748 ec->rx_coalesce_usecs = priv->rx_coalesce_usecs; 749 750 return 0; 751 } 752 753 static int gve_set_coalesce(struct net_device *netdev, 754 struct ethtool_coalesce *ec, 755 struct kernel_ethtool_coalesce *kernel_ec, 756 struct netlink_ext_ack *extack) 757 { 758 struct gve_priv *priv = netdev_priv(netdev); 759 u32 tx_usecs_orig = priv->tx_coalesce_usecs; 760 u32 rx_usecs_orig = priv->rx_coalesce_usecs; 761 int idx; 762 763 if (gve_is_gqi(priv)) 764 return -EOPNOTSUPP; 765 766 if (ec->tx_coalesce_usecs > GVE_MAX_ITR_INTERVAL_DQO || 767 ec->rx_coalesce_usecs > GVE_MAX_ITR_INTERVAL_DQO) 768 return -EINVAL; 769 priv->tx_coalesce_usecs = ec->tx_coalesce_usecs; 770 priv->rx_coalesce_usecs = ec->rx_coalesce_usecs; 771 772 if (tx_usecs_orig != priv->tx_coalesce_usecs) { 773 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) { 774 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); 775 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 776 777 gve_set_itr_coalesce_usecs_dqo(priv, block, 778 priv->tx_coalesce_usecs); 779 } 780 } 781 782 if (rx_usecs_orig != priv->rx_coalesce_usecs) { 783 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 784 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); 785 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 786 787 gve_set_itr_coalesce_usecs_dqo(priv, block, 788 priv->rx_coalesce_usecs); 789 } 790 } 791 792 return 0; 793 } 794 795 static int gve_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 796 { 797 struct gve_priv *priv = netdev_priv(netdev); 798 int err = 0; 799 800 if (!(netdev->features & NETIF_F_NTUPLE)) 801 return -EOPNOTSUPP; 802 803 switch (cmd->cmd) { 804 case ETHTOOL_SRXCLSRLINS: 805 err = gve_add_flow_rule(priv, cmd); 806 break; 807 case ETHTOOL_SRXCLSRLDEL: 808 err = gve_del_flow_rule(priv, cmd); 809 break; 810 default: 811 err = -EOPNOTSUPP; 812 break; 813 } 814 815 return err; 816 } 817 818 static int gve_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, u32 *rule_locs) 819 { 820 struct gve_priv *priv = netdev_priv(netdev); 821 int err = 0; 822 823 switch (cmd->cmd) { 824 case ETHTOOL_GRXRINGS: 825 cmd->data = priv->rx_cfg.num_queues; 826 break; 827 case ETHTOOL_GRXCLSRLCNT: 828 if (!priv->max_flow_rules) 829 return -EOPNOTSUPP; 830 831 err = gve_adminq_query_flow_rules(priv, GVE_FLOW_RULE_QUERY_STATS, 0); 832 if (err) 833 return err; 834 835 cmd->rule_cnt = priv->num_flow_rules; 836 cmd->data = priv->max_flow_rules; 837 break; 838 case ETHTOOL_GRXCLSRULE: 839 err = gve_get_flow_rule_entry(priv, cmd); 840 break; 841 case ETHTOOL_GRXCLSRLALL: 842 err = gve_get_flow_rule_ids(priv, cmd, (u32 *)rule_locs); 843 break; 844 default: 845 err = -EOPNOTSUPP; 846 break; 847 } 848 849 return err; 850 } 851 852 static u32 gve_get_rxfh_key_size(struct net_device *netdev) 853 { 854 struct gve_priv *priv = netdev_priv(netdev); 855 856 return priv->rss_key_size; 857 } 858 859 static u32 gve_get_rxfh_indir_size(struct net_device *netdev) 860 { 861 struct gve_priv *priv = netdev_priv(netdev); 862 863 return priv->rss_lut_size; 864 } 865 866 static void gve_get_rss_config_cache(struct gve_priv *priv, 867 struct ethtool_rxfh_param *rxfh) 868 { 869 struct gve_rss_config *rss_config = &priv->rss_config; 870 871 rxfh->hfunc = ETH_RSS_HASH_TOP; 872 873 if (rxfh->key) { 874 rxfh->key_size = priv->rss_key_size; 875 memcpy(rxfh->key, rss_config->hash_key, priv->rss_key_size); 876 } 877 878 if (rxfh->indir) { 879 rxfh->indir_size = priv->rss_lut_size; 880 memcpy(rxfh->indir, rss_config->hash_lut, 881 priv->rss_lut_size * sizeof(*rxfh->indir)); 882 } 883 } 884 885 static int gve_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) 886 { 887 struct gve_priv *priv = netdev_priv(netdev); 888 889 if (!priv->rss_key_size || !priv->rss_lut_size) 890 return -EOPNOTSUPP; 891 892 if (priv->cache_rss_config) { 893 gve_get_rss_config_cache(priv, rxfh); 894 return 0; 895 } 896 897 return gve_adminq_query_rss_config(priv, rxfh); 898 } 899 900 static void gve_set_rss_config_cache(struct gve_priv *priv, 901 struct ethtool_rxfh_param *rxfh) 902 { 903 struct gve_rss_config *rss_config = &priv->rss_config; 904 905 if (rxfh->key) 906 memcpy(rss_config->hash_key, rxfh->key, priv->rss_key_size); 907 908 if (rxfh->indir) 909 memcpy(rss_config->hash_lut, rxfh->indir, 910 priv->rss_lut_size * sizeof(*rxfh->indir)); 911 } 912 913 static int gve_set_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh, 914 struct netlink_ext_ack *extack) 915 { 916 struct gve_priv *priv = netdev_priv(netdev); 917 int err; 918 919 if (!priv->rss_key_size || !priv->rss_lut_size) 920 return -EOPNOTSUPP; 921 922 err = gve_adminq_configure_rss(priv, rxfh); 923 if (err) { 924 NL_SET_ERR_MSG_MOD(extack, "Fail to configure RSS config"); 925 return err; 926 } 927 928 if (priv->cache_rss_config) 929 gve_set_rss_config_cache(priv, rxfh); 930 931 return 0; 932 } 933 934 static int gve_get_ts_info(struct net_device *netdev, 935 struct kernel_ethtool_ts_info *info) 936 { 937 struct gve_priv *priv = netdev_priv(netdev); 938 939 ethtool_op_get_ts_info(netdev, info); 940 941 if (priv->nic_timestamp_supported) { 942 info->so_timestamping |= SOF_TIMESTAMPING_RX_HARDWARE | 943 SOF_TIMESTAMPING_RAW_HARDWARE; 944 945 info->rx_filters |= BIT(HWTSTAMP_FILTER_NONE) | 946 BIT(HWTSTAMP_FILTER_ALL); 947 948 if (priv->ptp) 949 info->phc_index = ptp_clock_index(priv->ptp->clock); 950 } 951 952 return 0; 953 } 954 955 const struct ethtool_ops gve_ethtool_ops = { 956 .supported_coalesce_params = ETHTOOL_COALESCE_USECS, 957 .supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT | 958 ETHTOOL_RING_USE_RX_BUF_LEN, 959 .get_drvinfo = gve_get_drvinfo, 960 .get_strings = gve_get_strings, 961 .get_sset_count = gve_get_sset_count, 962 .get_ethtool_stats = gve_get_ethtool_stats, 963 .set_msglevel = gve_set_msglevel, 964 .get_msglevel = gve_get_msglevel, 965 .set_channels = gve_set_channels, 966 .get_channels = gve_get_channels, 967 .set_rxnfc = gve_set_rxnfc, 968 .get_rxnfc = gve_get_rxnfc, 969 .get_rxfh_indir_size = gve_get_rxfh_indir_size, 970 .get_rxfh_key_size = gve_get_rxfh_key_size, 971 .get_rxfh = gve_get_rxfh, 972 .set_rxfh = gve_set_rxfh, 973 .get_link = ethtool_op_get_link, 974 .get_coalesce = gve_get_coalesce, 975 .set_coalesce = gve_set_coalesce, 976 .get_ringparam = gve_get_ringparam, 977 .set_ringparam = gve_set_ringparam, 978 .reset = gve_user_reset, 979 .get_tunable = gve_get_tunable, 980 .set_tunable = gve_set_tunable, 981 .get_priv_flags = gve_get_priv_flags, 982 .set_priv_flags = gve_set_priv_flags, 983 .get_link_ksettings = gve_get_link_ksettings, 984 .get_ts_info = gve_get_ts_info, 985 }; 986