1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
2 /* Google virtual Ethernet (gve) driver
3 *
4 * Copyright (C) 2015-2024 Google LLC
5 */
6
7 #include <linux/rtnetlink.h>
8 #include "gve.h"
9 #include "gve_adminq.h"
10 #include "gve_dqo.h"
11 #include "gve_utils.h"
12
gve_get_drvinfo(struct net_device * netdev,struct ethtool_drvinfo * info)13 static void gve_get_drvinfo(struct net_device *netdev,
14 struct ethtool_drvinfo *info)
15 {
16 struct gve_priv *priv = netdev_priv(netdev);
17
18 strscpy(info->driver, gve_driver_name, sizeof(info->driver));
19 strscpy(info->version, gve_version_str, sizeof(info->version));
20 strscpy(info->bus_info, pci_name(priv->pdev), sizeof(info->bus_info));
21 }
22
gve_set_msglevel(struct net_device * netdev,u32 value)23 static void gve_set_msglevel(struct net_device *netdev, u32 value)
24 {
25 struct gve_priv *priv = netdev_priv(netdev);
26
27 priv->msg_enable = value;
28 }
29
gve_get_msglevel(struct net_device * netdev)30 static u32 gve_get_msglevel(struct net_device *netdev)
31 {
32 struct gve_priv *priv = netdev_priv(netdev);
33
34 return priv->msg_enable;
35 }
36
37 /* For the following stats column string names, make sure the order
38 * matches how it is filled in the code. For xdp_aborted, xdp_drop,
39 * xdp_pass, xdp_tx, xdp_redirect, make sure it also matches the order
40 * as declared in enum xdp_action inside file uapi/linux/bpf.h .
41 */
42 static const char gve_gstrings_main_stats[][ETH_GSTRING_LEN] = {
43 "rx_packets", "rx_hsplit_pkt", "tx_packets", "rx_bytes",
44 "tx_bytes", "rx_dropped", "tx_dropped", "tx_timeouts",
45 "rx_skb_alloc_fail", "rx_buf_alloc_fail", "rx_desc_err_dropped_pkt",
46 "rx_hsplit_unsplit_pkt",
47 "interface_up_cnt", "interface_down_cnt", "reset_cnt",
48 "page_alloc_fail", "dma_mapping_error", "stats_report_trigger_cnt",
49 };
50
51 static const char gve_gstrings_rx_stats[][ETH_GSTRING_LEN] = {
52 "rx_posted_desc[%u]", "rx_completed_desc[%u]", "rx_consumed_desc[%u]",
53 "rx_bytes[%u]", "rx_hsplit_bytes[%u]", "rx_cont_packet_cnt[%u]",
54 "rx_frag_flip_cnt[%u]", "rx_frag_copy_cnt[%u]", "rx_frag_alloc_cnt[%u]",
55 "rx_dropped_pkt[%u]", "rx_copybreak_pkt[%u]", "rx_copied_pkt[%u]",
56 "rx_queue_drop_cnt[%u]", "rx_no_buffers_posted[%u]",
57 "rx_drops_packet_over_mru[%u]", "rx_drops_invalid_checksum[%u]",
58 "rx_xdp_aborted[%u]", "rx_xdp_drop[%u]", "rx_xdp_pass[%u]",
59 "rx_xdp_tx[%u]", "rx_xdp_redirect[%u]",
60 "rx_xdp_tx_errors[%u]", "rx_xdp_redirect_errors[%u]", "rx_xdp_alloc_fails[%u]",
61 };
62
63 static const char gve_gstrings_tx_stats[][ETH_GSTRING_LEN] = {
64 "tx_posted_desc[%u]", "tx_completed_desc[%u]", "tx_consumed_desc[%u]", "tx_bytes[%u]",
65 "tx_wake[%u]", "tx_stop[%u]", "tx_event_counter[%u]",
66 "tx_dma_mapping_error[%u]",
67 "tx_xsk_sent[%u]", "tx_xdp_xmit[%u]", "tx_xdp_xmit_errors[%u]"
68 };
69
70 static const char gve_gstrings_adminq_stats[][ETH_GSTRING_LEN] __nonstring_array = {
71 "adminq_prod_cnt", "adminq_cmd_fail", "adminq_timeouts",
72 "adminq_describe_device_cnt", "adminq_cfg_device_resources_cnt",
73 "adminq_register_page_list_cnt", "adminq_unregister_page_list_cnt",
74 "adminq_create_tx_queue_cnt", "adminq_create_rx_queue_cnt",
75 "adminq_destroy_tx_queue_cnt", "adminq_destroy_rx_queue_cnt",
76 "adminq_dcfg_device_resources_cnt", "adminq_set_driver_parameter_cnt",
77 "adminq_report_stats_cnt", "adminq_report_link_speed_cnt", "adminq_get_ptype_map_cnt",
78 "adminq_query_flow_rules", "adminq_cfg_flow_rule", "adminq_cfg_rss_cnt",
79 "adminq_query_rss_cnt", "adminq_report_nic_timestamp_cnt",
80 };
81
82 static const char gve_gstrings_priv_flags[][ETH_GSTRING_LEN] = {
83 "report-stats",
84 };
85
86 #define GVE_MAIN_STATS_LEN ARRAY_SIZE(gve_gstrings_main_stats)
87 #define GVE_ADMINQ_STATS_LEN ARRAY_SIZE(gve_gstrings_adminq_stats)
88 #define NUM_GVE_TX_CNTS ARRAY_SIZE(gve_gstrings_tx_stats)
89 #define NUM_GVE_RX_CNTS ARRAY_SIZE(gve_gstrings_rx_stats)
90 #define GVE_PRIV_FLAGS_STR_LEN ARRAY_SIZE(gve_gstrings_priv_flags)
91
gve_get_strings(struct net_device * netdev,u32 stringset,u8 * data)92 static void gve_get_strings(struct net_device *netdev, u32 stringset, u8 *data)
93 {
94 struct gve_priv *priv = netdev_priv(netdev);
95 u8 *s = (char *)data;
96 int num_tx_queues;
97 int i, j;
98
99 num_tx_queues = gve_num_tx_queues(priv);
100 switch (stringset) {
101 case ETH_SS_STATS:
102 for (i = 0; i < ARRAY_SIZE(gve_gstrings_main_stats); i++)
103 ethtool_puts(&s, gve_gstrings_main_stats[i]);
104
105 for (i = 0; i < priv->rx_cfg.num_queues; i++)
106 for (j = 0; j < NUM_GVE_RX_CNTS; j++)
107 ethtool_sprintf(&s, gve_gstrings_rx_stats[j],
108 i);
109
110 for (i = 0; i < num_tx_queues; i++)
111 for (j = 0; j < NUM_GVE_TX_CNTS; j++)
112 ethtool_sprintf(&s, gve_gstrings_tx_stats[j],
113 i);
114
115 for (i = 0; i < ARRAY_SIZE(gve_gstrings_adminq_stats); i++)
116 ethtool_cpy(&s, gve_gstrings_adminq_stats[i]);
117
118 break;
119
120 case ETH_SS_PRIV_FLAGS:
121 for (i = 0; i < ARRAY_SIZE(gve_gstrings_priv_flags); i++)
122 ethtool_puts(&s, gve_gstrings_priv_flags[i]);
123 break;
124
125 default:
126 break;
127 }
128 }
129
gve_get_sset_count(struct net_device * netdev,int sset)130 static int gve_get_sset_count(struct net_device *netdev, int sset)
131 {
132 struct gve_priv *priv = netdev_priv(netdev);
133 int num_tx_queues;
134
135 num_tx_queues = gve_num_tx_queues(priv);
136 switch (sset) {
137 case ETH_SS_STATS:
138 return GVE_MAIN_STATS_LEN + GVE_ADMINQ_STATS_LEN +
139 (priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS) +
140 (num_tx_queues * NUM_GVE_TX_CNTS);
141 case ETH_SS_PRIV_FLAGS:
142 return GVE_PRIV_FLAGS_STR_LEN;
143 default:
144 return -EOPNOTSUPP;
145 }
146 }
147
148 static void
gve_get_ethtool_stats(struct net_device * netdev,struct ethtool_stats * stats,u64 * data)149 gve_get_ethtool_stats(struct net_device *netdev,
150 struct ethtool_stats *stats, u64 *data)
151 {
152 u64 tmp_rx_pkts, tmp_rx_hsplit_pkt, tmp_rx_bytes, tmp_rx_hsplit_bytes,
153 tmp_rx_skb_alloc_fail, tmp_rx_buf_alloc_fail,
154 tmp_rx_desc_err_dropped_pkt, tmp_rx_hsplit_unsplit_pkt,
155 tmp_tx_pkts, tmp_tx_bytes;
156 u64 rx_buf_alloc_fail, rx_desc_err_dropped_pkt, rx_hsplit_unsplit_pkt,
157 rx_pkts, rx_hsplit_pkt, rx_skb_alloc_fail, rx_bytes, tx_pkts, tx_bytes,
158 tx_dropped;
159 int stats_idx, base_stats_idx, max_stats_idx;
160 struct stats *report_stats;
161 int *rx_qid_to_stats_idx;
162 int *tx_qid_to_stats_idx;
163 int num_stopped_rxqs = 0;
164 int num_stopped_txqs = 0;
165 struct gve_priv *priv;
166 bool skip_nic_stats;
167 unsigned int start;
168 int num_tx_queues;
169 int ring;
170 int i, j;
171
172 ASSERT_RTNL();
173
174 priv = netdev_priv(netdev);
175 num_tx_queues = gve_num_tx_queues(priv);
176 report_stats = priv->stats_report->stats;
177 rx_qid_to_stats_idx = kmalloc_array(priv->rx_cfg.num_queues,
178 sizeof(int), GFP_KERNEL);
179 if (!rx_qid_to_stats_idx)
180 return;
181 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) {
182 rx_qid_to_stats_idx[ring] = -1;
183 if (!gve_rx_was_added_to_block(priv, ring))
184 num_stopped_rxqs++;
185 }
186 tx_qid_to_stats_idx = kmalloc_array(num_tx_queues,
187 sizeof(int), GFP_KERNEL);
188 if (!tx_qid_to_stats_idx) {
189 kfree(rx_qid_to_stats_idx);
190 return;
191 }
192 for (ring = 0; ring < num_tx_queues; ring++) {
193 tx_qid_to_stats_idx[ring] = -1;
194 if (!gve_tx_was_added_to_block(priv, ring))
195 num_stopped_txqs++;
196 }
197
198 for (rx_pkts = 0, rx_bytes = 0, rx_hsplit_pkt = 0,
199 rx_skb_alloc_fail = 0, rx_buf_alloc_fail = 0,
200 rx_desc_err_dropped_pkt = 0, rx_hsplit_unsplit_pkt = 0,
201 ring = 0;
202 ring < priv->rx_cfg.num_queues; ring++) {
203 if (priv->rx) {
204 do {
205 struct gve_rx_ring *rx = &priv->rx[ring];
206
207 start =
208 u64_stats_fetch_begin(&priv->rx[ring].statss);
209 tmp_rx_pkts = rx->rpackets;
210 tmp_rx_hsplit_pkt = rx->rx_hsplit_pkt;
211 tmp_rx_bytes = rx->rbytes;
212 tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail;
213 tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail;
214 tmp_rx_desc_err_dropped_pkt =
215 rx->rx_desc_err_dropped_pkt;
216 tmp_rx_hsplit_unsplit_pkt =
217 rx->rx_hsplit_unsplit_pkt;
218 } while (u64_stats_fetch_retry(&priv->rx[ring].statss,
219 start));
220 rx_pkts += tmp_rx_pkts;
221 rx_hsplit_pkt += tmp_rx_hsplit_pkt;
222 rx_bytes += tmp_rx_bytes;
223 rx_skb_alloc_fail += tmp_rx_skb_alloc_fail;
224 rx_buf_alloc_fail += tmp_rx_buf_alloc_fail;
225 rx_desc_err_dropped_pkt += tmp_rx_desc_err_dropped_pkt;
226 rx_hsplit_unsplit_pkt += tmp_rx_hsplit_unsplit_pkt;
227 }
228 }
229 for (tx_pkts = 0, tx_bytes = 0, tx_dropped = 0, ring = 0;
230 ring < num_tx_queues; ring++) {
231 if (priv->tx) {
232 do {
233 start =
234 u64_stats_fetch_begin(&priv->tx[ring].statss);
235 tmp_tx_pkts = priv->tx[ring].pkt_done;
236 tmp_tx_bytes = priv->tx[ring].bytes_done;
237 } while (u64_stats_fetch_retry(&priv->tx[ring].statss,
238 start));
239 tx_pkts += tmp_tx_pkts;
240 tx_bytes += tmp_tx_bytes;
241 tx_dropped += priv->tx[ring].dropped_pkt;
242 }
243 }
244
245 i = 0;
246 data[i++] = rx_pkts;
247 data[i++] = rx_hsplit_pkt;
248 data[i++] = tx_pkts;
249 data[i++] = rx_bytes;
250 data[i++] = tx_bytes;
251 /* total rx dropped packets */
252 data[i++] = rx_skb_alloc_fail + rx_buf_alloc_fail +
253 rx_desc_err_dropped_pkt;
254 data[i++] = tx_dropped;
255 data[i++] = priv->tx_timeo_cnt;
256 data[i++] = rx_skb_alloc_fail;
257 data[i++] = rx_buf_alloc_fail;
258 data[i++] = rx_desc_err_dropped_pkt;
259 data[i++] = rx_hsplit_unsplit_pkt;
260 data[i++] = priv->interface_up_cnt;
261 data[i++] = priv->interface_down_cnt;
262 data[i++] = priv->reset_cnt;
263 data[i++] = priv->page_alloc_fail;
264 data[i++] = priv->dma_mapping_error;
265 data[i++] = priv->stats_report_trigger_cnt;
266 i = GVE_MAIN_STATS_LEN;
267
268 /* For rx cross-reporting stats, start from nic rx stats in report */
269 base_stats_idx = GVE_TX_STATS_REPORT_NUM * num_tx_queues +
270 GVE_RX_STATS_REPORT_NUM * priv->rx_cfg.num_queues;
271 /* The boundary between driver stats and NIC stats shifts if there are
272 * stopped queues.
273 */
274 base_stats_idx += NIC_RX_STATS_REPORT_NUM * num_stopped_rxqs +
275 NIC_TX_STATS_REPORT_NUM * num_stopped_txqs;
276 max_stats_idx = NIC_RX_STATS_REPORT_NUM *
277 (priv->rx_cfg.num_queues - num_stopped_rxqs) +
278 base_stats_idx;
279 /* Preprocess the stats report for rx, map queue id to start index */
280 skip_nic_stats = false;
281 for (stats_idx = base_stats_idx; stats_idx < max_stats_idx;
282 stats_idx += NIC_RX_STATS_REPORT_NUM) {
283 u32 stat_name = be32_to_cpu(report_stats[stats_idx].stat_name);
284 u32 queue_id = be32_to_cpu(report_stats[stats_idx].queue_id);
285
286 if (stat_name == 0) {
287 /* no stats written by NIC yet */
288 skip_nic_stats = true;
289 break;
290 }
291 if (queue_id < 0 || queue_id >= priv->rx_cfg.num_queues) {
292 net_err_ratelimited("Invalid rxq id in NIC stats\n");
293 continue;
294 }
295 rx_qid_to_stats_idx[queue_id] = stats_idx;
296 }
297 /* walk RX rings */
298 if (priv->rx) {
299 for (ring = 0; ring < priv->rx_cfg.num_queues; ring++) {
300 struct gve_rx_ring *rx = &priv->rx[ring];
301
302 data[i++] = rx->fill_cnt;
303 data[i++] = rx->cnt;
304 data[i++] = rx->fill_cnt - rx->cnt;
305 do {
306 start =
307 u64_stats_fetch_begin(&priv->rx[ring].statss);
308 tmp_rx_bytes = rx->rbytes;
309 tmp_rx_hsplit_bytes = rx->rx_hsplit_bytes;
310 tmp_rx_skb_alloc_fail = rx->rx_skb_alloc_fail;
311 tmp_rx_buf_alloc_fail = rx->rx_buf_alloc_fail;
312 tmp_rx_desc_err_dropped_pkt =
313 rx->rx_desc_err_dropped_pkt;
314 } while (u64_stats_fetch_retry(&priv->rx[ring].statss,
315 start));
316 data[i++] = tmp_rx_bytes;
317 data[i++] = tmp_rx_hsplit_bytes;
318 data[i++] = rx->rx_cont_packet_cnt;
319 data[i++] = rx->rx_frag_flip_cnt;
320 data[i++] = rx->rx_frag_copy_cnt;
321 data[i++] = rx->rx_frag_alloc_cnt;
322 /* rx dropped packets */
323 data[i++] = tmp_rx_skb_alloc_fail +
324 tmp_rx_buf_alloc_fail +
325 tmp_rx_desc_err_dropped_pkt;
326 data[i++] = rx->rx_copybreak_pkt;
327 data[i++] = rx->rx_copied_pkt;
328 /* stats from NIC */
329 stats_idx = rx_qid_to_stats_idx[ring];
330 if (skip_nic_stats || stats_idx < 0) {
331 /* skip NIC rx stats */
332 i += NIC_RX_STATS_REPORT_NUM;
333 } else {
334 for (j = 0; j < NIC_RX_STATS_REPORT_NUM; j++) {
335 u64 value =
336 be64_to_cpu(report_stats[stats_idx + j].value);
337
338 data[i++] = value;
339 }
340 }
341 /* XDP rx counters */
342 do {
343 start = u64_stats_fetch_begin(&priv->rx[ring].statss);
344 for (j = 0; j < GVE_XDP_ACTIONS; j++)
345 data[i + j] = rx->xdp_actions[j];
346 data[i + j++] = rx->xdp_tx_errors;
347 data[i + j++] = rx->xdp_redirect_errors;
348 data[i + j++] = rx->xdp_alloc_fails;
349 } while (u64_stats_fetch_retry(&priv->rx[ring].statss,
350 start));
351 i += GVE_XDP_ACTIONS + 3; /* XDP rx counters */
352 }
353 } else {
354 i += priv->rx_cfg.num_queues * NUM_GVE_RX_CNTS;
355 }
356
357 /* For tx cross-reporting stats, start from nic tx stats in report */
358 base_stats_idx = max_stats_idx;
359 max_stats_idx = NIC_TX_STATS_REPORT_NUM *
360 (num_tx_queues - num_stopped_txqs) +
361 max_stats_idx;
362 /* Preprocess the stats report for tx, map queue id to start index */
363 skip_nic_stats = false;
364 for (stats_idx = base_stats_idx; stats_idx < max_stats_idx;
365 stats_idx += NIC_TX_STATS_REPORT_NUM) {
366 u32 stat_name = be32_to_cpu(report_stats[stats_idx].stat_name);
367 u32 queue_id = be32_to_cpu(report_stats[stats_idx].queue_id);
368
369 if (stat_name == 0) {
370 /* no stats written by NIC yet */
371 skip_nic_stats = true;
372 break;
373 }
374 if (queue_id < 0 || queue_id >= num_tx_queues) {
375 net_err_ratelimited("Invalid txq id in NIC stats\n");
376 continue;
377 }
378 tx_qid_to_stats_idx[queue_id] = stats_idx;
379 }
380 /* walk TX rings */
381 if (priv->tx) {
382 for (ring = 0; ring < num_tx_queues; ring++) {
383 struct gve_tx_ring *tx = &priv->tx[ring];
384
385 if (gve_is_gqi(priv)) {
386 data[i++] = tx->req;
387 data[i++] = tx->done;
388 data[i++] = tx->req - tx->done;
389 } else {
390 /* DQO doesn't currently support
391 * posted/completed descriptor counts;
392 */
393 data[i++] = 0;
394 data[i++] = 0;
395 data[i++] =
396 (tx->dqo_tx.tail - tx->dqo_tx.head) &
397 tx->mask;
398 }
399 do {
400 start =
401 u64_stats_fetch_begin(&priv->tx[ring].statss);
402 tmp_tx_bytes = tx->bytes_done;
403 } while (u64_stats_fetch_retry(&priv->tx[ring].statss,
404 start));
405 data[i++] = tmp_tx_bytes;
406 data[i++] = tx->wake_queue;
407 data[i++] = tx->stop_queue;
408 data[i++] = gve_tx_load_event_counter(priv, tx);
409 data[i++] = tx->dma_mapping_error;
410 /* stats from NIC */
411 stats_idx = tx_qid_to_stats_idx[ring];
412 if (skip_nic_stats || stats_idx < 0) {
413 /* skip NIC tx stats */
414 i += NIC_TX_STATS_REPORT_NUM;
415 } else {
416 for (j = 0; j < NIC_TX_STATS_REPORT_NUM; j++) {
417 u64 value =
418 be64_to_cpu(report_stats[stats_idx + j].value);
419 data[i++] = value;
420 }
421 }
422 /* XDP counters */
423 do {
424 start = u64_stats_fetch_begin(&priv->tx[ring].statss);
425 data[i] = tx->xdp_xsk_sent;
426 data[i + 1] = tx->xdp_xmit;
427 data[i + 2] = tx->xdp_xmit_errors;
428 } while (u64_stats_fetch_retry(&priv->tx[ring].statss,
429 start));
430 i += 3; /* XDP tx counters */
431 }
432 } else {
433 i += num_tx_queues * NUM_GVE_TX_CNTS;
434 }
435
436 kfree(rx_qid_to_stats_idx);
437 kfree(tx_qid_to_stats_idx);
438 /* AQ Stats */
439 data[i++] = priv->adminq_prod_cnt;
440 data[i++] = priv->adminq_cmd_fail;
441 data[i++] = priv->adminq_timeouts;
442 data[i++] = priv->adminq_describe_device_cnt;
443 data[i++] = priv->adminq_cfg_device_resources_cnt;
444 data[i++] = priv->adminq_register_page_list_cnt;
445 data[i++] = priv->adminq_unregister_page_list_cnt;
446 data[i++] = priv->adminq_create_tx_queue_cnt;
447 data[i++] = priv->adminq_create_rx_queue_cnt;
448 data[i++] = priv->adminq_destroy_tx_queue_cnt;
449 data[i++] = priv->adminq_destroy_rx_queue_cnt;
450 data[i++] = priv->adminq_dcfg_device_resources_cnt;
451 data[i++] = priv->adminq_set_driver_parameter_cnt;
452 data[i++] = priv->adminq_report_stats_cnt;
453 data[i++] = priv->adminq_report_link_speed_cnt;
454 data[i++] = priv->adminq_get_ptype_map_cnt;
455 data[i++] = priv->adminq_query_flow_rules_cnt;
456 data[i++] = priv->adminq_cfg_flow_rule_cnt;
457 data[i++] = priv->adminq_cfg_rss_cnt;
458 data[i++] = priv->adminq_query_rss_cnt;
459 data[i++] = priv->adminq_report_nic_timestamp_cnt;
460 }
461
gve_get_channels(struct net_device * netdev,struct ethtool_channels * cmd)462 static void gve_get_channels(struct net_device *netdev,
463 struct ethtool_channels *cmd)
464 {
465 struct gve_priv *priv = netdev_priv(netdev);
466
467 cmd->max_rx = priv->rx_cfg.max_queues;
468 cmd->max_tx = priv->tx_cfg.max_queues;
469 cmd->max_other = 0;
470 cmd->max_combined = 0;
471 cmd->rx_count = priv->rx_cfg.num_queues;
472 cmd->tx_count = priv->tx_cfg.num_queues;
473 cmd->other_count = 0;
474 cmd->combined_count = 0;
475 }
476
gve_set_channels(struct net_device * netdev,struct ethtool_channels * cmd)477 static int gve_set_channels(struct net_device *netdev,
478 struct ethtool_channels *cmd)
479 {
480 struct gve_priv *priv = netdev_priv(netdev);
481 struct gve_tx_queue_config new_tx_cfg = priv->tx_cfg;
482 struct gve_rx_queue_config new_rx_cfg = priv->rx_cfg;
483 struct ethtool_channels old_settings;
484 int new_tx = cmd->tx_count;
485 int new_rx = cmd->rx_count;
486 bool reset_rss = false;
487
488 gve_get_channels(netdev, &old_settings);
489
490 /* Changing combined is not allowed */
491 if (cmd->combined_count != old_settings.combined_count)
492 return -EINVAL;
493
494 if (!new_rx || !new_tx)
495 return -EINVAL;
496
497 if (priv->xdp_prog) {
498 if (new_tx != new_rx ||
499 (2 * new_tx > priv->tx_cfg.max_queues)) {
500 dev_err(&priv->pdev->dev, "The number of configured RX queues should be equal to the number of configured TX queues and the number of configured RX/TX queues should be less than or equal to half the maximum number of RX/TX queues when XDP program is installed");
501 return -EINVAL;
502 }
503
504 /* One XDP TX queue per RX queue. */
505 new_tx_cfg.num_xdp_queues = new_rx;
506 } else {
507 new_tx_cfg.num_xdp_queues = 0;
508 }
509
510 if (new_rx != priv->rx_cfg.num_queues &&
511 priv->cache_rss_config && !netif_is_rxfh_configured(netdev))
512 reset_rss = true;
513
514 new_tx_cfg.num_queues = new_tx;
515 new_rx_cfg.num_queues = new_rx;
516
517 return gve_adjust_queues(priv, new_rx_cfg, new_tx_cfg, reset_rss);
518 }
519
gve_get_ringparam(struct net_device * netdev,struct ethtool_ringparam * cmd,struct kernel_ethtool_ringparam * kernel_cmd,struct netlink_ext_ack * extack)520 static void gve_get_ringparam(struct net_device *netdev,
521 struct ethtool_ringparam *cmd,
522 struct kernel_ethtool_ringparam *kernel_cmd,
523 struct netlink_ext_ack *extack)
524 {
525 struct gve_priv *priv = netdev_priv(netdev);
526
527 cmd->rx_max_pending = priv->max_rx_desc_cnt;
528 cmd->tx_max_pending = priv->max_tx_desc_cnt;
529 cmd->rx_pending = priv->rx_desc_cnt;
530 cmd->tx_pending = priv->tx_desc_cnt;
531
532 kernel_cmd->rx_buf_len = priv->rx_cfg.packet_buffer_size;
533
534 if (!gve_header_split_supported(priv))
535 kernel_cmd->tcp_data_split = ETHTOOL_TCP_DATA_SPLIT_UNKNOWN;
536 else if (priv->header_split_enabled)
537 kernel_cmd->tcp_data_split = ETHTOOL_TCP_DATA_SPLIT_ENABLED;
538 else
539 kernel_cmd->tcp_data_split = ETHTOOL_TCP_DATA_SPLIT_DISABLED;
540 }
541
gve_validate_req_ring_size(struct gve_priv * priv,u16 new_tx_desc_cnt,u16 new_rx_desc_cnt)542 static int gve_validate_req_ring_size(struct gve_priv *priv, u16 new_tx_desc_cnt,
543 u16 new_rx_desc_cnt)
544 {
545 /* check for valid range */
546 if (new_tx_desc_cnt < priv->min_tx_desc_cnt ||
547 new_tx_desc_cnt > priv->max_tx_desc_cnt ||
548 new_rx_desc_cnt < priv->min_rx_desc_cnt ||
549 new_rx_desc_cnt > priv->max_rx_desc_cnt) {
550 dev_err(&priv->pdev->dev, "Requested descriptor count out of range\n");
551 return -EINVAL;
552 }
553
554 if (!is_power_of_2(new_tx_desc_cnt) || !is_power_of_2(new_rx_desc_cnt)) {
555 dev_err(&priv->pdev->dev, "Requested descriptor count has to be a power of 2\n");
556 return -EINVAL;
557 }
558 return 0;
559 }
560
gve_set_ring_sizes_config(struct gve_priv * priv,u16 new_tx_desc_cnt,u16 new_rx_desc_cnt,struct gve_tx_alloc_rings_cfg * tx_alloc_cfg,struct gve_rx_alloc_rings_cfg * rx_alloc_cfg)561 static int gve_set_ring_sizes_config(struct gve_priv *priv, u16 new_tx_desc_cnt,
562 u16 new_rx_desc_cnt,
563 struct gve_tx_alloc_rings_cfg *tx_alloc_cfg,
564 struct gve_rx_alloc_rings_cfg *rx_alloc_cfg)
565 {
566 if (new_tx_desc_cnt == priv->tx_desc_cnt &&
567 new_rx_desc_cnt == priv->rx_desc_cnt)
568 return 0;
569
570 if (!priv->modify_ring_size_enabled) {
571 dev_err(&priv->pdev->dev, "Modify ring size is not supported.\n");
572 return -EOPNOTSUPP;
573 }
574
575 if (gve_validate_req_ring_size(priv, new_tx_desc_cnt, new_rx_desc_cnt))
576 return -EINVAL;
577
578 tx_alloc_cfg->ring_size = new_tx_desc_cnt;
579 rx_alloc_cfg->ring_size = new_rx_desc_cnt;
580 return 0;
581 }
582
gve_set_ringparam(struct net_device * netdev,struct ethtool_ringparam * cmd,struct kernel_ethtool_ringparam * kernel_cmd,struct netlink_ext_ack * extack)583 static int gve_set_ringparam(struct net_device *netdev,
584 struct ethtool_ringparam *cmd,
585 struct kernel_ethtool_ringparam *kernel_cmd,
586 struct netlink_ext_ack *extack)
587 {
588 struct gve_tx_alloc_rings_cfg tx_alloc_cfg = {0};
589 struct gve_rx_alloc_rings_cfg rx_alloc_cfg = {0};
590 struct gve_priv *priv = netdev_priv(netdev);
591 int err;
592
593 gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg);
594
595 err = gve_set_rx_buf_len_config(priv, kernel_cmd->rx_buf_len, extack,
596 &rx_alloc_cfg);
597 if (err)
598 return err;
599
600 err = gve_set_hsplit_config(priv, kernel_cmd->tcp_data_split,
601 &rx_alloc_cfg);
602 if (err)
603 return err;
604
605 err = gve_set_ring_sizes_config(priv, cmd->tx_pending, cmd->rx_pending,
606 &tx_alloc_cfg, &rx_alloc_cfg);
607 if (err)
608 return err;
609
610 if (netif_running(priv->dev)) {
611 err = gve_adjust_config(priv, &tx_alloc_cfg, &rx_alloc_cfg);
612 if (err)
613 return err;
614 } else {
615 /* Set ring params for the next up */
616 priv->rx_cfg.packet_buffer_size =
617 rx_alloc_cfg.packet_buffer_size;
618 priv->header_split_enabled = rx_alloc_cfg.enable_header_split;
619 priv->tx_desc_cnt = tx_alloc_cfg.ring_size;
620 priv->rx_desc_cnt = rx_alloc_cfg.ring_size;
621 }
622 return 0;
623 }
624
gve_user_reset(struct net_device * netdev,u32 * flags)625 static int gve_user_reset(struct net_device *netdev, u32 *flags)
626 {
627 struct gve_priv *priv = netdev_priv(netdev);
628
629 if (*flags == ETH_RESET_ALL) {
630 *flags = 0;
631 return gve_reset(priv, true);
632 }
633
634 return -EOPNOTSUPP;
635 }
636
gve_get_tunable(struct net_device * netdev,const struct ethtool_tunable * etuna,void * value)637 static int gve_get_tunable(struct net_device *netdev,
638 const struct ethtool_tunable *etuna, void *value)
639 {
640 struct gve_priv *priv = netdev_priv(netdev);
641
642 switch (etuna->id) {
643 case ETHTOOL_RX_COPYBREAK:
644 *(u32 *)value = priv->rx_copybreak;
645 return 0;
646 default:
647 return -EOPNOTSUPP;
648 }
649 }
650
gve_set_tunable(struct net_device * netdev,const struct ethtool_tunable * etuna,const void * value)651 static int gve_set_tunable(struct net_device *netdev,
652 const struct ethtool_tunable *etuna,
653 const void *value)
654 {
655 struct gve_priv *priv = netdev_priv(netdev);
656 u32 len;
657
658 switch (etuna->id) {
659 case ETHTOOL_RX_COPYBREAK:
660 {
661 u32 max_copybreak = priv->rx_cfg.packet_buffer_size;
662
663 len = *(u32 *)value;
664 if (len > max_copybreak)
665 return -EINVAL;
666 priv->rx_copybreak = len;
667 return 0;
668 }
669 default:
670 return -EOPNOTSUPP;
671 }
672 }
673
gve_get_priv_flags(struct net_device * netdev)674 static u32 gve_get_priv_flags(struct net_device *netdev)
675 {
676 struct gve_priv *priv = netdev_priv(netdev);
677 u32 ret_flags = 0;
678
679 /* Only 1 flag exists currently: report-stats (BIT(0)), so set that flag. */
680 if (priv->ethtool_flags & BIT(0))
681 ret_flags |= BIT(0);
682 return ret_flags;
683 }
684
gve_set_priv_flags(struct net_device * netdev,u32 flags)685 static int gve_set_priv_flags(struct net_device *netdev, u32 flags)
686 {
687 struct gve_priv *priv = netdev_priv(netdev);
688 u64 ori_flags, new_flags;
689 int num_tx_queues;
690
691 num_tx_queues = gve_num_tx_queues(priv);
692 ori_flags = READ_ONCE(priv->ethtool_flags);
693 new_flags = ori_flags;
694
695 /* Only one priv flag exists: report-stats (BIT(0))*/
696 if (flags & BIT(0))
697 new_flags |= BIT(0);
698 else
699 new_flags &= ~(BIT(0));
700 priv->ethtool_flags = new_flags;
701 /* start report-stats timer when user turns report stats on. */
702 if (flags & BIT(0)) {
703 mod_timer(&priv->stats_report_timer,
704 round_jiffies(jiffies +
705 msecs_to_jiffies(priv->stats_report_timer_period)));
706 }
707 /* Zero off gve stats when report-stats turned off and */
708 /* delete report stats timer. */
709 if (!(flags & BIT(0)) && (ori_flags & BIT(0))) {
710 int tx_stats_num = GVE_TX_STATS_REPORT_NUM *
711 num_tx_queues;
712 int rx_stats_num = GVE_RX_STATS_REPORT_NUM *
713 priv->rx_cfg.num_queues;
714
715 memset(priv->stats_report->stats, 0, (tx_stats_num + rx_stats_num) *
716 sizeof(struct stats));
717 timer_delete_sync(&priv->stats_report_timer);
718 }
719 return 0;
720 }
721
gve_get_link_ksettings(struct net_device * netdev,struct ethtool_link_ksettings * cmd)722 static int gve_get_link_ksettings(struct net_device *netdev,
723 struct ethtool_link_ksettings *cmd)
724 {
725 struct gve_priv *priv = netdev_priv(netdev);
726 int err = 0;
727
728 if (priv->link_speed == 0)
729 err = gve_adminq_report_link_speed(priv);
730
731 cmd->base.speed = priv->link_speed;
732
733 cmd->base.duplex = DUPLEX_FULL;
734
735 return err;
736 }
737
gve_get_coalesce(struct net_device * netdev,struct ethtool_coalesce * ec,struct kernel_ethtool_coalesce * kernel_ec,struct netlink_ext_ack * extack)738 static int gve_get_coalesce(struct net_device *netdev,
739 struct ethtool_coalesce *ec,
740 struct kernel_ethtool_coalesce *kernel_ec,
741 struct netlink_ext_ack *extack)
742 {
743 struct gve_priv *priv = netdev_priv(netdev);
744
745 if (gve_is_gqi(priv))
746 return -EOPNOTSUPP;
747 ec->tx_coalesce_usecs = priv->tx_coalesce_usecs;
748 ec->rx_coalesce_usecs = priv->rx_coalesce_usecs;
749
750 return 0;
751 }
752
gve_set_coalesce(struct net_device * netdev,struct ethtool_coalesce * ec,struct kernel_ethtool_coalesce * kernel_ec,struct netlink_ext_ack * extack)753 static int gve_set_coalesce(struct net_device *netdev,
754 struct ethtool_coalesce *ec,
755 struct kernel_ethtool_coalesce *kernel_ec,
756 struct netlink_ext_ack *extack)
757 {
758 struct gve_priv *priv = netdev_priv(netdev);
759 u32 tx_usecs_orig = priv->tx_coalesce_usecs;
760 u32 rx_usecs_orig = priv->rx_coalesce_usecs;
761 int idx;
762
763 if (gve_is_gqi(priv))
764 return -EOPNOTSUPP;
765
766 if (ec->tx_coalesce_usecs > GVE_MAX_ITR_INTERVAL_DQO ||
767 ec->rx_coalesce_usecs > GVE_MAX_ITR_INTERVAL_DQO)
768 return -EINVAL;
769 priv->tx_coalesce_usecs = ec->tx_coalesce_usecs;
770 priv->rx_coalesce_usecs = ec->rx_coalesce_usecs;
771
772 if (tx_usecs_orig != priv->tx_coalesce_usecs) {
773 for (idx = 0; idx < priv->tx_cfg.num_queues; idx++) {
774 int ntfy_idx = gve_tx_idx_to_ntfy(priv, idx);
775 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
776
777 gve_set_itr_coalesce_usecs_dqo(priv, block,
778 priv->tx_coalesce_usecs);
779 }
780 }
781
782 if (rx_usecs_orig != priv->rx_coalesce_usecs) {
783 for (idx = 0; idx < priv->rx_cfg.num_queues; idx++) {
784 int ntfy_idx = gve_rx_idx_to_ntfy(priv, idx);
785 struct gve_notify_block *block = &priv->ntfy_blocks[ntfy_idx];
786
787 gve_set_itr_coalesce_usecs_dqo(priv, block,
788 priv->rx_coalesce_usecs);
789 }
790 }
791
792 return 0;
793 }
794
gve_set_rxnfc(struct net_device * netdev,struct ethtool_rxnfc * cmd)795 static int gve_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd)
796 {
797 struct gve_priv *priv = netdev_priv(netdev);
798 int err = 0;
799
800 if (!(netdev->features & NETIF_F_NTUPLE))
801 return -EOPNOTSUPP;
802
803 switch (cmd->cmd) {
804 case ETHTOOL_SRXCLSRLINS:
805 err = gve_add_flow_rule(priv, cmd);
806 break;
807 case ETHTOOL_SRXCLSRLDEL:
808 err = gve_del_flow_rule(priv, cmd);
809 break;
810 default:
811 err = -EOPNOTSUPP;
812 break;
813 }
814
815 return err;
816 }
817
gve_get_rxnfc(struct net_device * netdev,struct ethtool_rxnfc * cmd,u32 * rule_locs)818 static int gve_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, u32 *rule_locs)
819 {
820 struct gve_priv *priv = netdev_priv(netdev);
821 int err = 0;
822
823 switch (cmd->cmd) {
824 case ETHTOOL_GRXRINGS:
825 cmd->data = priv->rx_cfg.num_queues;
826 break;
827 case ETHTOOL_GRXCLSRLCNT:
828 if (!priv->max_flow_rules)
829 return -EOPNOTSUPP;
830
831 err = gve_adminq_query_flow_rules(priv, GVE_FLOW_RULE_QUERY_STATS, 0);
832 if (err)
833 return err;
834
835 cmd->rule_cnt = priv->num_flow_rules;
836 cmd->data = priv->max_flow_rules;
837 break;
838 case ETHTOOL_GRXCLSRULE:
839 err = gve_get_flow_rule_entry(priv, cmd);
840 break;
841 case ETHTOOL_GRXCLSRLALL:
842 err = gve_get_flow_rule_ids(priv, cmd, (u32 *)rule_locs);
843 break;
844 default:
845 err = -EOPNOTSUPP;
846 break;
847 }
848
849 return err;
850 }
851
gve_get_rxfh_key_size(struct net_device * netdev)852 static u32 gve_get_rxfh_key_size(struct net_device *netdev)
853 {
854 struct gve_priv *priv = netdev_priv(netdev);
855
856 return priv->rss_key_size;
857 }
858
gve_get_rxfh_indir_size(struct net_device * netdev)859 static u32 gve_get_rxfh_indir_size(struct net_device *netdev)
860 {
861 struct gve_priv *priv = netdev_priv(netdev);
862
863 return priv->rss_lut_size;
864 }
865
gve_get_rss_config_cache(struct gve_priv * priv,struct ethtool_rxfh_param * rxfh)866 static void gve_get_rss_config_cache(struct gve_priv *priv,
867 struct ethtool_rxfh_param *rxfh)
868 {
869 struct gve_rss_config *rss_config = &priv->rss_config;
870
871 rxfh->hfunc = ETH_RSS_HASH_TOP;
872
873 if (rxfh->key) {
874 rxfh->key_size = priv->rss_key_size;
875 memcpy(rxfh->key, rss_config->hash_key, priv->rss_key_size);
876 }
877
878 if (rxfh->indir) {
879 rxfh->indir_size = priv->rss_lut_size;
880 memcpy(rxfh->indir, rss_config->hash_lut,
881 priv->rss_lut_size * sizeof(*rxfh->indir));
882 }
883 }
884
gve_get_rxfh(struct net_device * netdev,struct ethtool_rxfh_param * rxfh)885 static int gve_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh)
886 {
887 struct gve_priv *priv = netdev_priv(netdev);
888
889 if (!priv->rss_key_size || !priv->rss_lut_size)
890 return -EOPNOTSUPP;
891
892 if (priv->cache_rss_config) {
893 gve_get_rss_config_cache(priv, rxfh);
894 return 0;
895 }
896
897 return gve_adminq_query_rss_config(priv, rxfh);
898 }
899
gve_set_rss_config_cache(struct gve_priv * priv,struct ethtool_rxfh_param * rxfh)900 static void gve_set_rss_config_cache(struct gve_priv *priv,
901 struct ethtool_rxfh_param *rxfh)
902 {
903 struct gve_rss_config *rss_config = &priv->rss_config;
904
905 if (rxfh->key)
906 memcpy(rss_config->hash_key, rxfh->key, priv->rss_key_size);
907
908 if (rxfh->indir)
909 memcpy(rss_config->hash_lut, rxfh->indir,
910 priv->rss_lut_size * sizeof(*rxfh->indir));
911 }
912
gve_set_rxfh(struct net_device * netdev,struct ethtool_rxfh_param * rxfh,struct netlink_ext_ack * extack)913 static int gve_set_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh,
914 struct netlink_ext_ack *extack)
915 {
916 struct gve_priv *priv = netdev_priv(netdev);
917 int err;
918
919 if (!priv->rss_key_size || !priv->rss_lut_size)
920 return -EOPNOTSUPP;
921
922 err = gve_adminq_configure_rss(priv, rxfh);
923 if (err) {
924 NL_SET_ERR_MSG_MOD(extack, "Fail to configure RSS config");
925 return err;
926 }
927
928 if (priv->cache_rss_config)
929 gve_set_rss_config_cache(priv, rxfh);
930
931 return 0;
932 }
933
gve_get_ts_info(struct net_device * netdev,struct kernel_ethtool_ts_info * info)934 static int gve_get_ts_info(struct net_device *netdev,
935 struct kernel_ethtool_ts_info *info)
936 {
937 struct gve_priv *priv = netdev_priv(netdev);
938
939 ethtool_op_get_ts_info(netdev, info);
940
941 if (gve_is_clock_enabled(priv)) {
942 info->so_timestamping |= SOF_TIMESTAMPING_RX_HARDWARE |
943 SOF_TIMESTAMPING_RAW_HARDWARE;
944
945 info->rx_filters |= BIT(HWTSTAMP_FILTER_NONE) |
946 BIT(HWTSTAMP_FILTER_ALL);
947
948 if (priv->ptp)
949 info->phc_index = ptp_clock_index(priv->ptp->clock);
950 }
951
952 return 0;
953 }
954
955 const struct ethtool_ops gve_ethtool_ops = {
956 .supported_coalesce_params = ETHTOOL_COALESCE_USECS,
957 .supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT |
958 ETHTOOL_RING_USE_RX_BUF_LEN,
959 .get_drvinfo = gve_get_drvinfo,
960 .get_strings = gve_get_strings,
961 .get_sset_count = gve_get_sset_count,
962 .get_ethtool_stats = gve_get_ethtool_stats,
963 .set_msglevel = gve_set_msglevel,
964 .get_msglevel = gve_get_msglevel,
965 .set_channels = gve_set_channels,
966 .get_channels = gve_get_channels,
967 .set_rxnfc = gve_set_rxnfc,
968 .get_rxnfc = gve_get_rxnfc,
969 .get_rxfh_indir_size = gve_get_rxfh_indir_size,
970 .get_rxfh_key_size = gve_get_rxfh_key_size,
971 .get_rxfh = gve_get_rxfh,
972 .set_rxfh = gve_set_rxfh,
973 .get_link = ethtool_op_get_link,
974 .get_coalesce = gve_get_coalesce,
975 .set_coalesce = gve_set_coalesce,
976 .get_ringparam = gve_get_ringparam,
977 .set_ringparam = gve_set_ringparam,
978 .reset = gve_user_reset,
979 .get_tunable = gve_get_tunable,
980 .set_tunable = gve_set_tunable,
981 .get_priv_flags = gve_get_priv_flags,
982 .set_priv_flags = gve_set_priv_flags,
983 .get_link_ksettings = gve_get_link_ksettings,
984 .get_ts_info = gve_get_ts_info,
985 };
986