1 // SPDX-License-Identifier: (GPL-2.0 OR MIT) 2 /* Google virtual Ethernet (gve) driver 3 * 4 * Copyright (C) 2015-2024 Google LLC 5 */ 6 7 #include <linux/rtnetlink.h> 8 #include "gve.h" 9 #include "gve_adminq.h" 10 #include "gve_dqo.h" 11 #include "gve_utils.h" 12 13 static void gve_get_drvinfo(struct net_device *netdev, 14 struct ethtool_drvinfo *info) 15 { 16 struct gve_priv *priv = netdev_priv(netdev); 17 18 strscpy(info->driver, gve_driver_name, sizeof(info->driver)); 19 strscpy(info->version, gve_version_str, sizeof(info->version)); 20 strscpy(info->bus_info, pci_name(priv->pdev), sizeof(info->bus_info)); 21 } 22 23 static void gve_set_msglevel(struct net_device *netdev, u32 value) 24 { 25 struct gve_priv *priv = netdev_priv(netdev); 26 27 priv->msg_enable = value; 28 } 29 30 static u32 gve_get_msglevel(struct net_device *netdev) 31 { 32 struct gve_priv *priv = netdev_priv(netdev); 33 34 return priv->msg_enable; 35 } 36 37 /* For the following stats column string names, make sure the order 38 * matches how it is filled in the code. For xdp_aborted, xdp_drop, 39 * xdp_pass, xdp_tx, xdp_redirect, make sure it also matches the order 40 * as declared in enum xdp_action inside file uapi/linux/bpf.h . 41 */ 42 static const char gve_gstrings_main_stats[][ETH_GSTRING_LEN] = { 43 "rx_packets", "rx_hsplit_pkt", "tx_packets", "rx_bytes", 44 "tx_bytes", "rx_dropped", "tx_dropped", "tx_timeouts", 45 "rx_skb_alloc_fail", "rx_buf_alloc_fail", "rx_desc_err_dropped_pkt", 46 "rx_hsplit_unsplit_pkt", 47 "interface_up_cnt", "interface_down_cnt", "reset_cnt", 48 "page_alloc_fail", "dma_mapping_error", "stats_report_trigger_cnt", 49 }; 50 51 static const char gve_gstrings_rx_stats[][ETH_GSTRING_LEN] = { 52 "rx_posted_desc[%u]", "rx_completed_desc[%u]", "rx_consumed_desc[%u]", 53 "rx_bytes[%u]", "rx_hsplit_bytes[%u]", "rx_cont_packet_cnt[%u]", 54 "rx_frag_flip_cnt[%u]", "rx_frag_copy_cnt[%u]", "rx_frag_alloc_cnt[%u]", 55 "rx_dropped_pkt[%u]", "rx_copybreak_pkt[%u]", "rx_copied_pkt[%u]", 56 "rx_queue_drop_cnt[%u]", "rx_no_buffers_posted[%u]", 57 "rx_drops_packet_over_mru[%u]", "rx_drops_invalid_checksum[%u]", 58 "rx_xdp_aborted[%u]", "rx_xdp_drop[%u]", "rx_xdp_pass[%u]", 59 "rx_xdp_tx[%u]", "rx_xdp_redirect[%u]", 60 "rx_xdp_tx_errors[%u]", "rx_xdp_redirect_errors[%u]", "rx_xdp_alloc_fails[%u]", 61 }; 62 63 static const char gve_gstrings_tx_stats[][ETH_GSTRING_LEN] = { 64 "tx_posted_desc[%u]", "tx_completed_desc[%u]", "tx_consumed_desc[%u]", "tx_bytes[%u]", 65 "tx_wake[%u]", "tx_stop[%u]", "tx_event_counter[%u]", 66 "tx_dma_mapping_error[%u]", 67 "tx_xsk_sent[%u]", "tx_xdp_xmit[%u]", "tx_xdp_xmit_errors[%u]" 68 }; 69 70 static const char gve_gstrings_adminq_stats[][ETH_GSTRING_LEN] __nonstring_array = { 71 "adminq_prod_cnt", "adminq_cmd_fail", "adminq_timeouts", 72 "adminq_describe_device_cnt", "adminq_cfg_device_resources_cnt", 73 "adminq_register_page_list_cnt", "adminq_unregister_page_list_cnt", 74 "adminq_create_tx_queue_cnt", "adminq_create_rx_queue_cnt", 75 "adminq_destroy_tx_queue_cnt", "adminq_destroy_rx_queue_cnt", 76 "adminq_dcfg_device_resources_cnt", "adminq_set_driver_parameter_cnt", 77 "adminq_report_stats_cnt", "adminq_report_link_speed_cnt", "adminq_get_ptype_map_cnt", 78 "adminq_query_flow_rules", "adminq_cfg_flow_rule", "adminq_cfg_rss_cnt", 79 "adminq_query_rss_cnt", "adminq_report_nic_timestamp_cnt", 80 }; 81 82 static const char gve_gstrings_priv_flags[][ETH_GSTRING_LEN] = { 83 "report-stats", 84 }; 85 86 #define GVE_MAIN_STATS_LEN ARRAY_SIZE(gve_gstrings_main_stats) 87 #define GVE_ADMINQ_STATS_LEN ARRAY_SIZE(gve_gstrings_adminq_stats) 88 #define NUM_GVE_TX_CNTS ARRAY_SIZE(gve_gstrings_tx_stats) 89 #define NUM_GVE_RX_CNTS ARRAY_SIZE(gve_gstrings_rx_stats) 90 #define GVE_PRIV_FLAGS_STR_LEN ARRAY_SIZE(gve_gstrings_priv_flags) 91 92 static void gve_get_strings(struct net_device *netdev, u32 stringset, u8 *data) 93 { 94 struct gve_priv *priv = netdev_priv(netdev); 95 u8 *s = (char *)data; 96 int num_tx_queues; 97 int i, j; 98 99 num_tx_queues = gve_num_tx_queues(priv); 100 switch (stringset) { 101 case ETH_SS_STATS: 102 for (i = 0; i < ARRAY_SIZE(gve_gstrings_main_stats); i++) 103 ethtool_puts(&s, gve_gstrings_main_stats[i]); 104 105 for (i = 0; i < priv->rx_cfg.num_queues; i++) 106 for (j = 0; j < NUM_GVE_RX_CNTS; j++) 107 ethtool_sprintf(&s, gve_gstrings_rx_stats[j], 108 i); 109 110 for (i = 0; i < num_tx_queues; i++) 111 for (j = 0; j < NUM_GVE_TX_CNTS; j++) 112 ethtool_sprintf(&s, gve_gstrings_tx_stats[j], 113 i); 114 115 for (i = 0; i < ARRAY_SIZE(gve_gstrings_adminq_stats); i++) 116 ethtool_cpy(&s, gve_gstrings_adminq_stats[i]); 117 118 break; 119 120 case ETH_SS_PRIV_FLAGS: 121 for (i = 0; i < ARRAY_SIZE(gve_gstrings_priv_flags); i++) 122 ethtool_puts(&s, gve_gstrings_priv_flags[i]); 123 break; 124 125 default: 126 break; 127 } 128 } 129 130 static int gve_get_sset_count(struct net_device *netdev, int sset) 131 { 132 struct gve_priv *priv = netdev_priv(netdev); 133 int num_tx_queues; 134 135 num_tx_queues = gve_num_tx_queues(priv); 136 switch (sset) { 137 case ETH_SS_STATS: 138 return GVE_MAIN_STATS_LEN + GVE_ADMINQ_STATS_LEN + 139 (priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS) + 140 (num_tx_queues * NUM_GVE_TX_CNTS); 141 case ETH_SS_PRIV_FLAGS: 142 return GVE_PRIV_FLAGS_STR_LEN; 143 default: 144 return -EOPNOTSUPP; 145 } 146 } 147 148 static void 149 gve_get_ethtool_stats(struct net_device *netdev, 150 struct ethtool_stats *stats, u64 *data) 151 { 152 u64 tmp_rx_pkts, tmp_rx_hsplit_pkt, tmp_rx_bytes, tmp_rx_hsplit_bytes, 153 tmp_rx_skb_alloc_fail, tmp_rx_buf_alloc_fail, 154 tmp_rx_desc_err_dropped_pkt, tmp_rx_hsplit_unsplit_pkt, 155 tmp_tx_pkts, tmp_tx_bytes; 156 u64 rx_buf_alloc_fail, rx_desc_err_dropped_pkt, rx_hsplit_unsplit_pkt, 157 rx_pkts, rx_hsplit_pkt, rx_skb_alloc_fail, rx_bytes, tx_pkts, tx_bytes, 158 tx_dropped; 159 int stats_idx, base_stats_idx, max_stats_idx; 160 struct stats *report_stats; 161 int *rx_qid_to_stats_idx; 162 int *tx_qid_to_stats_idx; 163 int num_stopped_rxqs = 0; 164 int num_stopped_txqs = 0; 165 struct gve_priv *priv; 166 bool skip_nic_stats; 167 unsigned int start; 168 int num_tx_queues; 169 int ring; 170 int i, j; 171 172 ASSERT_RTNL(); 173 174 priv = netdev_priv(netdev); 175 num_tx_queues = gve_num_tx_queues(priv); 176 report_stats = priv->stats_report->stats; 177 rx_qid_to_stats_idx = kmalloc_array(priv->rx_cfg.num_queues, 178 sizeof(int), GFP_KERNEL); 179 if (!rx_qid_to_stats_idx) 180 return; 181 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { 182 rx_qid_to_stats_idx[ring] = -1; 183 if (!gve_rx_was_added_to_block(priv, ring)) 184 num_stopped_rxqs++; 185 } 186 tx_qid_to_stats_idx = kmalloc_array(num_tx_queues, 187 sizeof(int), GFP_KERNEL); 188 if (!tx_qid_to_stats_idx) { 189 kfree(rx_qid_to_stats_idx); 190 return; 191 } 192 for (ring = 0; ring < num_tx_queues; ring++) { 193 tx_qid_to_stats_idx[ring] = -1; 194 if (!gve_tx_was_added_to_block(priv, ring)) 195 num_stopped_txqs++; 196 } 197 198 for (rx_pkts = 0, rx_bytes = 0, rx_hsplit_pkt = 0, 199 rx_skb_alloc_fail = 0, rx_buf_alloc_fail = 0, 200 rx_desc_err_dropped_pkt = 0, rx_hsplit_unsplit_pkt = 0, 201 ring = 0; 202 ring < priv->rx_cfg.num_queues; ring++) { 203 if (priv->rx) { 204 do { 205 struct gve_rx_ring *rx = &priv->rx[ring]; 206 207 start = 208 u64_stats_fetch_begin(&priv->rx[ring].statss); 209 tmp_rx_pkts = rx->rpackets; 210 tmp_rx_hsplit_pkt = rx->rx_hsplit_pkt; 211 tmp_rx_bytes = rx->rbytes; 212 tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail; 213 tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail; 214 tmp_rx_desc_err_dropped_pkt = 215 rx->rx_desc_err_dropped_pkt; 216 tmp_rx_hsplit_unsplit_pkt = 217 rx->rx_hsplit_unsplit_pkt; 218 } while (u64_stats_fetch_retry(&priv->rx[ring].statss, 219 start)); 220 rx_pkts += tmp_rx_pkts; 221 rx_hsplit_pkt += tmp_rx_hsplit_pkt; 222 rx_bytes += tmp_rx_bytes; 223 rx_skb_alloc_fail += tmp_rx_skb_alloc_fail; 224 rx_buf_alloc_fail += tmp_rx_buf_alloc_fail; 225 rx_desc_err_dropped_pkt += tmp_rx_desc_err_dropped_pkt; 226 rx_hsplit_unsplit_pkt += tmp_rx_hsplit_unsplit_pkt; 227 } 228 } 229 for (tx_pkts = 0, tx_bytes = 0, tx_dropped = 0, ring = 0; 230 ring < num_tx_queues; ring++) { 231 if (priv->tx) { 232 do { 233 start = 234 u64_stats_fetch_begin(&priv->tx[ring].statss); 235 tmp_tx_pkts = priv->tx[ring].pkt_done; 236 tmp_tx_bytes = priv->tx[ring].bytes_done; 237 } while (u64_stats_fetch_retry(&priv->tx[ring].statss, 238 start)); 239 tx_pkts += tmp_tx_pkts; 240 tx_bytes += tmp_tx_bytes; 241 tx_dropped += priv->tx[ring].dropped_pkt; 242 } 243 } 244 245 i = 0; 246 data[i++] = rx_pkts; 247 data[i++] = rx_hsplit_pkt; 248 data[i++] = tx_pkts; 249 data[i++] = rx_bytes; 250 data[i++] = tx_bytes; 251 /* total rx dropped packets */ 252 data[i++] = rx_skb_alloc_fail + rx_buf_alloc_fail + 253 rx_desc_err_dropped_pkt; 254 data[i++] = tx_dropped; 255 data[i++] = priv->tx_timeo_cnt; 256 data[i++] = rx_skb_alloc_fail; 257 data[i++] = rx_buf_alloc_fail; 258 data[i++] = rx_desc_err_dropped_pkt; 259 data[i++] = rx_hsplit_unsplit_pkt; 260 data[i++] = priv->interface_up_cnt; 261 data[i++] = priv->interface_down_cnt; 262 data[i++] = priv->reset_cnt; 263 data[i++] = priv->page_alloc_fail; 264 data[i++] = priv->dma_mapping_error; 265 data[i++] = priv->stats_report_trigger_cnt; 266 i = GVE_MAIN_STATS_LEN; 267 268 /* For rx cross-reporting stats, start from nic rx stats in report */ 269 base_stats_idx = GVE_TX_STATS_REPORT_NUM * num_tx_queues + 270 GVE_RX_STATS_REPORT_NUM * priv->rx_cfg.num_queues; 271 /* The boundary between driver stats and NIC stats shifts if there are 272 * stopped queues. 273 */ 274 base_stats_idx += NIC_RX_STATS_REPORT_NUM * num_stopped_rxqs + 275 NIC_TX_STATS_REPORT_NUM * num_stopped_txqs; 276 max_stats_idx = NIC_RX_STATS_REPORT_NUM * 277 (priv->rx_cfg.num_queues - num_stopped_rxqs) + 278 base_stats_idx; 279 /* Preprocess the stats report for rx, map queue id to start index */ 280 skip_nic_stats = false; 281 for (stats_idx = base_stats_idx; stats_idx < max_stats_idx; 282 stats_idx += NIC_RX_STATS_REPORT_NUM) { 283 u32 stat_name = be32_to_cpu(report_stats[stats_idx].stat_name); 284 u32 queue_id = be32_to_cpu(report_stats[stats_idx].queue_id); 285 286 if (stat_name == 0) { 287 /* no stats written by NIC yet */ 288 skip_nic_stats = true; 289 break; 290 } 291 if (queue_id < 0 || queue_id >= priv->rx_cfg.num_queues) { 292 net_err_ratelimited("Invalid rxq id in NIC stats\n"); 293 continue; 294 } 295 rx_qid_to_stats_idx[queue_id] = stats_idx; 296 } 297 /* walk RX rings */ 298 if (priv->rx) { 299 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) { 300 struct gve_rx_ring *rx = &priv->rx[ring]; 301 302 data[i++] = rx->fill_cnt; 303 data[i++] = rx->cnt; 304 data[i++] = rx->fill_cnt - rx->cnt; 305 do { 306 start = 307 u64_stats_fetch_begin(&priv->rx[ring].statss); 308 tmp_rx_bytes = rx->rbytes; 309 tmp_rx_hsplit_bytes = rx->rx_hsplit_bytes; 310 tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail; 311 tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail; 312 tmp_rx_desc_err_dropped_pkt = 313 rx->rx_desc_err_dropped_pkt; 314 } while (u64_stats_fetch_retry(&priv->rx[ring].statss, 315 start)); 316 data[i++] = tmp_rx_bytes; 317 data[i++] = tmp_rx_hsplit_bytes; 318 data[i++] = rx->rx_cont_packet_cnt; 319 data[i++] = rx->rx_frag_flip_cnt; 320 data[i++] = rx->rx_frag_copy_cnt; 321 data[i++] = rx->rx_frag_alloc_cnt; 322 /* rx dropped packets */ 323 data[i++] = tmp_rx_skb_alloc_fail + 324 tmp_rx_buf_alloc_fail + 325 tmp_rx_desc_err_dropped_pkt; 326 data[i++] = rx->rx_copybreak_pkt; 327 data[i++] = rx->rx_copied_pkt; 328 /* stats from NIC */ 329 stats_idx = rx_qid_to_stats_idx[ring]; 330 if (skip_nic_stats || stats_idx < 0) { 331 /* skip NIC rx stats */ 332 i += NIC_RX_STATS_REPORT_NUM; 333 } else { 334 for (j = 0; j < NIC_RX_STATS_REPORT_NUM; j++) { 335 u64 value = 336 be64_to_cpu(report_stats[stats_idx + j].value); 337 338 data[i++] = value; 339 } 340 } 341 /* XDP rx counters */ 342 do { 343 start = u64_stats_fetch_begin(&priv->rx[ring].statss); 344 for (j = 0; j < GVE_XDP_ACTIONS; j++) 345 data[i + j] = rx->xdp_actions[j]; 346 data[i + j++] = rx->xdp_tx_errors; 347 data[i + j++] = rx->xdp_redirect_errors; 348 data[i + j++] = rx->xdp_alloc_fails; 349 } while (u64_stats_fetch_retry(&priv->rx[ring].statss, 350 start)); 351 i += GVE_XDP_ACTIONS + 3; /* XDP rx counters */ 352 } 353 } else { 354 i += priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS; 355 } 356 357 /* For tx cross-reporting stats, start from nic tx stats in report */ 358 base_stats_idx = max_stats_idx; 359 max_stats_idx = NIC_TX_STATS_REPORT_NUM * 360 (num_tx_queues - num_stopped_txqs) + 361 max_stats_idx; 362 /* Preprocess the stats report for tx, map queue id to start index */ 363 skip_nic_stats = false; 364 for (stats_idx = base_stats_idx; stats_idx < max_stats_idx; 365 stats_idx += NIC_TX_STATS_REPORT_NUM) { 366 u32 stat_name = be32_to_cpu(report_stats[stats_idx].stat_name); 367 u32 queue_id = be32_to_cpu(report_stats[stats_idx].queue_id); 368 369 if (stat_name == 0) { 370 /* no stats written by NIC yet */ 371 skip_nic_stats = true; 372 break; 373 } 374 if (queue_id < 0 || queue_id >= num_tx_queues) { 375 net_err_ratelimited("Invalid txq id in NIC stats\n"); 376 continue; 377 } 378 tx_qid_to_stats_idx[queue_id] = stats_idx; 379 } 380 /* walk TX rings */ 381 if (priv->tx) { 382 for (ring = 0; ring < num_tx_queues; ring++) { 383 struct gve_tx_ring *tx = &priv->tx[ring]; 384 385 if (gve_is_gqi(priv)) { 386 data[i++] = tx->req; 387 data[i++] = tx->done; 388 data[i++] = tx->req - tx->done; 389 } else { 390 /* DQO doesn't currently support 391 * posted/completed descriptor counts; 392 */ 393 data[i++] = 0; 394 data[i++] = 0; 395 data[i++] = 396 (tx->dqo_tx.tail - tx->dqo_tx.head) & 397 tx->mask; 398 } 399 do { 400 start = 401 u64_stats_fetch_begin(&priv->tx[ring].statss); 402 tmp_tx_bytes = tx->bytes_done; 403 } while (u64_stats_fetch_retry(&priv->tx[ring].statss, 404 start)); 405 data[i++] = tmp_tx_bytes; 406 data[i++] = tx->wake_queue; 407 data[i++] = tx->stop_queue; 408 data[i++] = gve_tx_load_event_counter(priv, tx); 409 data[i++] = tx->dma_mapping_error; 410 /* stats from NIC */ 411 stats_idx = tx_qid_to_stats_idx[ring]; 412 if (skip_nic_stats || stats_idx < 0) { 413 /* skip NIC tx stats */ 414 i += NIC_TX_STATS_REPORT_NUM; 415 } else { 416 for (j = 0; j < NIC_TX_STATS_REPORT_NUM; j++) { 417 u64 value = 418 be64_to_cpu(report_stats[stats_idx + j].value); 419 data[i++] = value; 420 } 421 } 422 /* XDP counters */ 423 do { 424 start = u64_stats_fetch_begin(&priv->tx[ring].statss); 425 data[i] = tx->xdp_xsk_sent; 426 data[i + 1] = tx->xdp_xmit; 427 data[i + 2] = tx->xdp_xmit_errors; 428 } while (u64_stats_fetch_retry(&priv->tx[ring].statss, 429 start)); 430 i += 3; /* XDP tx counters */ 431 } 432 } else { 433 i += num_tx_queues * NUM_GVE_TX_CNTS; 434 } 435 436 kfree(rx_qid_to_stats_idx); 437 kfree(tx_qid_to_stats_idx); 438 /* AQ Stats */ 439 data[i++] = priv->adminq_prod_cnt; 440 data[i++] = priv->adminq_cmd_fail; 441 data[i++] = priv->adminq_timeouts; 442 data[i++] = priv->adminq_describe_device_cnt; 443 data[i++] = priv->adminq_cfg_device_resources_cnt; 444 data[i++] = priv->adminq_register_page_list_cnt; 445 data[i++] = priv->adminq_unregister_page_list_cnt; 446 data[i++] = priv->adminq_create_tx_queue_cnt; 447 data[i++] = priv->adminq_create_rx_queue_cnt; 448 data[i++] = priv->adminq_destroy_tx_queue_cnt; 449 data[i++] = priv->adminq_destroy_rx_queue_cnt; 450 data[i++] = priv->adminq_dcfg_device_resources_cnt; 451 data[i++] = priv->adminq_set_driver_parameter_cnt; 452 data[i++] = priv->adminq_report_stats_cnt; 453 data[i++] = priv->adminq_report_link_speed_cnt; 454 data[i++] = priv->adminq_get_ptype_map_cnt; 455 data[i++] = priv->adminq_query_flow_rules_cnt; 456 data[i++] = priv->adminq_cfg_flow_rule_cnt; 457 data[i++] = priv->adminq_cfg_rss_cnt; 458 data[i++] = priv->adminq_query_rss_cnt; 459 data[i++] = priv->adminq_report_nic_timestamp_cnt; 460 } 461 462 static void gve_get_channels(struct net_device *netdev, 463 struct ethtool_channels *cmd) 464 { 465 struct gve_priv *priv = netdev_priv(netdev); 466 467 cmd->max_rx = priv->rx_cfg.max_queues; 468 cmd->max_tx = priv->tx_cfg.max_queues; 469 cmd->max_other = 0; 470 cmd->max_combined = 0; 471 cmd->rx_count = priv->rx_cfg.num_queues; 472 cmd->tx_count = priv->tx_cfg.num_queues; 473 cmd->other_count = 0; 474 cmd->combined_count = 0; 475 } 476 477 static int gve_set_channels(struct net_device *netdev, 478 struct ethtool_channels *cmd) 479 { 480 struct gve_priv *priv = netdev_priv(netdev); 481 struct gve_tx_queue_config new_tx_cfg = priv->tx_cfg; 482 struct gve_rx_queue_config new_rx_cfg = priv->rx_cfg; 483 struct ethtool_channels old_settings; 484 int new_tx = cmd->tx_count; 485 int new_rx = cmd->rx_count; 486 bool reset_rss = false; 487 488 gve_get_channels(netdev, &old_settings); 489 490 /* Changing combined is not allowed */ 491 if (cmd->combined_count != old_settings.combined_count) 492 return -EINVAL; 493 494 if (!new_rx || !new_tx) 495 return -EINVAL; 496 497 if (priv->xdp_prog) { 498 if (new_tx != new_rx || 499 (2 * new_tx > priv->tx_cfg.max_queues)) { 500 dev_err(&priv->pdev->dev, "The number of configured RX queues should be equal to the number of configured TX queues and the number of configured RX/TX queues should be less than or equal to half the maximum number of RX/TX queues when XDP program is installed"); 501 return -EINVAL; 502 } 503 504 /* One XDP TX queue per RX queue. */ 505 new_tx_cfg.num_xdp_queues = new_rx; 506 } else { 507 new_tx_cfg.num_xdp_queues = 0; 508 } 509 510 if (new_rx != priv->rx_cfg.num_queues && 511 priv->cache_rss_config && !netif_is_rxfh_configured(netdev)) 512 reset_rss = true; 513 514 new_tx_cfg.num_queues = new_tx; 515 new_rx_cfg.num_queues = new_rx; 516 517 return gve_adjust_queues(priv, new_rx_cfg, new_tx_cfg, reset_rss); 518 } 519 520 static void gve_get_ringparam(struct net_device *netdev, 521 struct ethtool_ringparam *cmd, 522 struct kernel_ethtool_ringparam *kernel_cmd, 523 struct netlink_ext_ack *extack) 524 { 525 struct gve_priv *priv = netdev_priv(netdev); 526 527 cmd->rx_max_pending = priv->max_rx_desc_cnt; 528 cmd->tx_max_pending = priv->max_tx_desc_cnt; 529 cmd->rx_pending = priv->rx_desc_cnt; 530 cmd->tx_pending = priv->tx_desc_cnt; 531 532 if (!gve_header_split_supported(priv)) 533 kernel_cmd->tcp_data_split = ETHTOOL_TCP_DATA_SPLIT_UNKNOWN; 534 else if (priv->header_split_enabled) 535 kernel_cmd->tcp_data_split = ETHTOOL_TCP_DATA_SPLIT_ENABLED; 536 else 537 kernel_cmd->tcp_data_split = ETHTOOL_TCP_DATA_SPLIT_DISABLED; 538 } 539 540 static int gve_adjust_ring_sizes(struct gve_priv *priv, 541 u16 new_tx_desc_cnt, 542 u16 new_rx_desc_cnt) 543 { 544 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0}; 545 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0}; 546 int err; 547 548 /* get current queue configuration */ 549 gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); 550 551 /* copy over the new ring_size from ethtool */ 552 tx_alloc_cfg.ring_size = new_tx_desc_cnt; 553 rx_alloc_cfg.ring_size = new_rx_desc_cnt; 554 555 if (netif_running(priv->dev)) { 556 err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg); 557 if (err) 558 return err; 559 } 560 561 /* Set new ring_size for the next up */ 562 priv->tx_desc_cnt = new_tx_desc_cnt; 563 priv->rx_desc_cnt = new_rx_desc_cnt; 564 565 return 0; 566 } 567 568 static int gve_validate_req_ring_size(struct gve_priv *priv, u16 new_tx_desc_cnt, 569 u16 new_rx_desc_cnt) 570 { 571 /* check for valid range */ 572 if (new_tx_desc_cnt < priv->min_tx_desc_cnt || 573 new_tx_desc_cnt > priv->max_tx_desc_cnt || 574 new_rx_desc_cnt < priv->min_rx_desc_cnt || 575 new_rx_desc_cnt > priv->max_rx_desc_cnt) { 576 dev_err(&priv->pdev->dev, "Requested descriptor count out of range\n"); 577 return -EINVAL; 578 } 579 580 if (!is_power_of_2(new_tx_desc_cnt) || !is_power_of_2(new_rx_desc_cnt)) { 581 dev_err(&priv->pdev->dev, "Requested descriptor count has to be a power of 2\n"); 582 return -EINVAL; 583 } 584 return 0; 585 } 586 587 static int gve_set_ringparam(struct net_device *netdev, 588 struct ethtool_ringparam *cmd, 589 struct kernel_ethtool_ringparam *kernel_cmd, 590 struct netlink_ext_ack *extack) 591 { 592 struct gve_priv *priv = netdev_priv(netdev); 593 u16 new_tx_cnt, new_rx_cnt; 594 int err; 595 596 err = gve_set_hsplit_config(priv, kernel_cmd->tcp_data_split); 597 if (err) 598 return err; 599 600 if (cmd->tx_pending == priv->tx_desc_cnt && cmd->rx_pending == priv->rx_desc_cnt) 601 return 0; 602 603 if (!priv->modify_ring_size_enabled) { 604 dev_err(&priv->pdev->dev, "Modify ring size is not supported.\n"); 605 return -EOPNOTSUPP; 606 } 607 608 new_tx_cnt = cmd->tx_pending; 609 new_rx_cnt = cmd->rx_pending; 610 611 if (gve_validate_req_ring_size(priv, new_tx_cnt, new_rx_cnt)) 612 return -EINVAL; 613 614 return gve_adjust_ring_sizes(priv, new_tx_cnt, new_rx_cnt); 615 } 616 617 static int gve_user_reset(struct net_device *netdev, u32 *flags) 618 { 619 struct gve_priv *priv = netdev_priv(netdev); 620 621 if (*flags == ETH_RESET_ALL) { 622 *flags = 0; 623 return gve_reset(priv, true); 624 } 625 626 return -EOPNOTSUPP; 627 } 628 629 static int gve_get_tunable(struct net_device *netdev, 630 const struct ethtool_tunable *etuna, void *value) 631 { 632 struct gve_priv *priv = netdev_priv(netdev); 633 634 switch (etuna->id) { 635 case ETHTOOL_RX_COPYBREAK: 636 *(u32 *)value = priv->rx_copybreak; 637 return 0; 638 default: 639 return -EOPNOTSUPP; 640 } 641 } 642 643 static int gve_set_tunable(struct net_device *netdev, 644 const struct ethtool_tunable *etuna, 645 const void *value) 646 { 647 struct gve_priv *priv = netdev_priv(netdev); 648 u32 len; 649 650 switch (etuna->id) { 651 case ETHTOOL_RX_COPYBREAK: 652 { 653 u32 max_copybreak = priv->rx_cfg.packet_buffer_size; 654 655 len = *(u32 *)value; 656 if (len > max_copybreak) 657 return -EINVAL; 658 priv->rx_copybreak = len; 659 return 0; 660 } 661 default: 662 return -EOPNOTSUPP; 663 } 664 } 665 666 static u32 gve_get_priv_flags(struct net_device *netdev) 667 { 668 struct gve_priv *priv = netdev_priv(netdev); 669 u32 ret_flags = 0; 670 671 /* Only 1 flag exists currently: report-stats (BIT(0)), so set that flag. */ 672 if (priv->ethtool_flags & BIT(0)) 673 ret_flags |= BIT(0); 674 return ret_flags; 675 } 676 677 static int gve_set_priv_flags(struct net_device *netdev, u32 flags) 678 { 679 struct gve_priv *priv = netdev_priv(netdev); 680 u64 ori_flags, new_flags; 681 int num_tx_queues; 682 683 num_tx_queues = gve_num_tx_queues(priv); 684 ori_flags = READ_ONCE(priv->ethtool_flags); 685 new_flags = ori_flags; 686 687 /* Only one priv flag exists: report-stats (BIT(0))*/ 688 if (flags & BIT(0)) 689 new_flags |= BIT(0); 690 else 691 new_flags &= ~(BIT(0)); 692 priv->ethtool_flags = new_flags; 693 /* start report-stats timer when user turns report stats on. */ 694 if (flags & BIT(0)) { 695 mod_timer(&priv->stats_report_timer, 696 round_jiffies(jiffies + 697 msecs_to_jiffies(priv->stats_report_timer_period))); 698 } 699 /* Zero off gve stats when report-stats turned off and */ 700 /* delete report stats timer. */ 701 if (!(flags & BIT(0)) && (ori_flags & BIT(0))) { 702 int tx_stats_num = GVE_TX_STATS_REPORT_NUM * 703 num_tx_queues; 704 int rx_stats_num = GVE_RX_STATS_REPORT_NUM * 705 priv->rx_cfg.num_queues; 706 707 memset(priv->stats_report->stats, 0, (tx_stats_num + rx_stats_num) * 708 sizeof(struct stats)); 709 timer_delete_sync(&priv->stats_report_timer); 710 } 711 return 0; 712 } 713 714 static int gve_get_link_ksettings(struct net_device *netdev, 715 struct ethtool_link_ksettings *cmd) 716 { 717 struct gve_priv *priv = netdev_priv(netdev); 718 int err = 0; 719 720 if (priv->link_speed == 0) 721 err = gve_adminq_report_link_speed(priv); 722 723 cmd->base.speed = priv->link_speed; 724 725 cmd->base.duplex = DUPLEX_FULL; 726 727 return err; 728 } 729 730 static int gve_get_coalesce(struct net_device *netdev, 731 struct ethtool_coalesce *ec, 732 struct kernel_ethtool_coalesce *kernel_ec, 733 struct netlink_ext_ack *extack) 734 { 735 struct gve_priv *priv = netdev_priv(netdev); 736 737 if (gve_is_gqi(priv)) 738 return -EOPNOTSUPP; 739 ec->tx_coalesce_usecs = priv->tx_coalesce_usecs; 740 ec->rx_coalesce_usecs = priv->rx_coalesce_usecs; 741 742 return 0; 743 } 744 745 static int gve_set_coalesce(struct net_device *netdev, 746 struct ethtool_coalesce *ec, 747 struct kernel_ethtool_coalesce *kernel_ec, 748 struct netlink_ext_ack *extack) 749 { 750 struct gve_priv *priv = netdev_priv(netdev); 751 u32 tx_usecs_orig = priv->tx_coalesce_usecs; 752 u32 rx_usecs_orig = priv->rx_coalesce_usecs; 753 int idx; 754 755 if (gve_is_gqi(priv)) 756 return -EOPNOTSUPP; 757 758 if (ec->tx_coalesce_usecs > GVE_MAX_ITR_INTERVAL_DQO || 759 ec->rx_coalesce_usecs > GVE_MAX_ITR_INTERVAL_DQO) 760 return -EINVAL; 761 priv->tx_coalesce_usecs = ec->tx_coalesce_usecs; 762 priv->rx_coalesce_usecs = ec->rx_coalesce_usecs; 763 764 if (tx_usecs_orig != priv->tx_coalesce_usecs) { 765 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) { 766 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx); 767 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 768 769 gve_set_itr_coalesce_usecs_dqo(priv, block, 770 priv->tx_coalesce_usecs); 771 } 772 } 773 774 if (rx_usecs_orig != priv->rx_coalesce_usecs) { 775 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) { 776 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx); 777 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx]; 778 779 gve_set_itr_coalesce_usecs_dqo(priv, block, 780 priv->rx_coalesce_usecs); 781 } 782 } 783 784 return 0; 785 } 786 787 static int gve_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 788 { 789 struct gve_priv *priv = netdev_priv(netdev); 790 int err = 0; 791 792 if (!(netdev->features & NETIF_F_NTUPLE)) 793 return -EOPNOTSUPP; 794 795 switch (cmd->cmd) { 796 case ETHTOOL_SRXCLSRLINS: 797 err = gve_add_flow_rule(priv, cmd); 798 break; 799 case ETHTOOL_SRXCLSRLDEL: 800 err = gve_del_flow_rule(priv, cmd); 801 break; 802 default: 803 err = -EOPNOTSUPP; 804 break; 805 } 806 807 return err; 808 } 809 810 static int gve_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, u32 *rule_locs) 811 { 812 struct gve_priv *priv = netdev_priv(netdev); 813 int err = 0; 814 815 switch (cmd->cmd) { 816 case ETHTOOL_GRXRINGS: 817 cmd->data = priv->rx_cfg.num_queues; 818 break; 819 case ETHTOOL_GRXCLSRLCNT: 820 if (!priv->max_flow_rules) 821 return -EOPNOTSUPP; 822 823 err = gve_adminq_query_flow_rules(priv, GVE_FLOW_RULE_QUERY_STATS, 0); 824 if (err) 825 return err; 826 827 cmd->rule_cnt = priv->num_flow_rules; 828 cmd->data = priv->max_flow_rules; 829 break; 830 case ETHTOOL_GRXCLSRULE: 831 err = gve_get_flow_rule_entry(priv, cmd); 832 break; 833 case ETHTOOL_GRXCLSRLALL: 834 err = gve_get_flow_rule_ids(priv, cmd, (u32 *)rule_locs); 835 break; 836 default: 837 err = -EOPNOTSUPP; 838 break; 839 } 840 841 return err; 842 } 843 844 static u32 gve_get_rxfh_key_size(struct net_device *netdev) 845 { 846 struct gve_priv *priv = netdev_priv(netdev); 847 848 return priv->rss_key_size; 849 } 850 851 static u32 gve_get_rxfh_indir_size(struct net_device *netdev) 852 { 853 struct gve_priv *priv = netdev_priv(netdev); 854 855 return priv->rss_lut_size; 856 } 857 858 static void gve_get_rss_config_cache(struct gve_priv *priv, 859 struct ethtool_rxfh_param *rxfh) 860 { 861 struct gve_rss_config *rss_config = &priv->rss_config; 862 863 rxfh->hfunc = ETH_RSS_HASH_TOP; 864 865 if (rxfh->key) { 866 rxfh->key_size = priv->rss_key_size; 867 memcpy(rxfh->key, rss_config->hash_key, priv->rss_key_size); 868 } 869 870 if (rxfh->indir) { 871 rxfh->indir_size = priv->rss_lut_size; 872 memcpy(rxfh->indir, rss_config->hash_lut, 873 priv->rss_lut_size * sizeof(*rxfh->indir)); 874 } 875 } 876 877 static int gve_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) 878 { 879 struct gve_priv *priv = netdev_priv(netdev); 880 881 if (!priv->rss_key_size || !priv->rss_lut_size) 882 return -EOPNOTSUPP; 883 884 if (priv->cache_rss_config) { 885 gve_get_rss_config_cache(priv, rxfh); 886 return 0; 887 } 888 889 return gve_adminq_query_rss_config(priv, rxfh); 890 } 891 892 static void gve_set_rss_config_cache(struct gve_priv *priv, 893 struct ethtool_rxfh_param *rxfh) 894 { 895 struct gve_rss_config *rss_config = &priv->rss_config; 896 897 if (rxfh->key) 898 memcpy(rss_config->hash_key, rxfh->key, priv->rss_key_size); 899 900 if (rxfh->indir) 901 memcpy(rss_config->hash_lut, rxfh->indir, 902 priv->rss_lut_size * sizeof(*rxfh->indir)); 903 } 904 905 static int gve_set_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh, 906 struct netlink_ext_ack *extack) 907 { 908 struct gve_priv *priv = netdev_priv(netdev); 909 int err; 910 911 if (!priv->rss_key_size || !priv->rss_lut_size) 912 return -EOPNOTSUPP; 913 914 err = gve_adminq_configure_rss(priv, rxfh); 915 if (err) { 916 NL_SET_ERR_MSG_MOD(extack, "Fail to configure RSS config"); 917 return err; 918 } 919 920 if (priv->cache_rss_config) 921 gve_set_rss_config_cache(priv, rxfh); 922 923 return 0; 924 } 925 926 static int gve_get_ts_info(struct net_device *netdev, 927 struct kernel_ethtool_ts_info *info) 928 { 929 struct gve_priv *priv = netdev_priv(netdev); 930 931 ethtool_op_get_ts_info(netdev, info); 932 933 if (priv->nic_timestamp_supported) { 934 info->so_timestamping |= SOF_TIMESTAMPING_RX_HARDWARE | 935 SOF_TIMESTAMPING_RAW_HARDWARE; 936 937 info->rx_filters |= BIT(HWTSTAMP_FILTER_NONE) | 938 BIT(HWTSTAMP_FILTER_ALL); 939 940 if (priv->ptp) 941 info->phc_index = ptp_clock_index(priv->ptp->clock); 942 } 943 944 return 0; 945 } 946 947 const struct ethtool_ops gve_ethtool_ops = { 948 .supported_coalesce_params = ETHTOOL_COALESCE_USECS, 949 .supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT, 950 .get_drvinfo = gve_get_drvinfo, 951 .get_strings = gve_get_strings, 952 .get_sset_count = gve_get_sset_count, 953 .get_ethtool_stats = gve_get_ethtool_stats, 954 .set_msglevel = gve_set_msglevel, 955 .get_msglevel = gve_get_msglevel, 956 .set_channels = gve_set_channels, 957 .get_channels = gve_get_channels, 958 .set_rxnfc = gve_set_rxnfc, 959 .get_rxnfc = gve_get_rxnfc, 960 .get_rxfh_indir_size = gve_get_rxfh_indir_size, 961 .get_rxfh_key_size = gve_get_rxfh_key_size, 962 .get_rxfh = gve_get_rxfh, 963 .set_rxfh = gve_set_rxfh, 964 .get_link = ethtool_op_get_link, 965 .get_coalesce = gve_get_coalesce, 966 .set_coalesce = gve_set_coalesce, 967 .get_ringparam = gve_get_ringparam, 968 .set_ringparam = gve_set_ringparam, 969 .reset = gve_user_reset, 970 .get_tunable = gve_get_tunable, 971 .set_tunable = gve_set_tunable, 972 .get_priv_flags = gve_get_priv_flags, 973 .set_priv_flags = gve_set_priv_flags, 974 .get_link_ksettings = gve_get_link_ksettings, 975 .get_ts_info = gve_get_ts_info, 976 }; 977