1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright (c) 2021, Microsoft Corporation. */
3
4 #include <linux/inetdevice.h>
5 #include <linux/etherdevice.h>
6 #include <linux/ethtool.h>
7
8 #include <net/mana/mana.h>
9
10 struct mana_stats_desc {
11 char name[ETH_GSTRING_LEN];
12 u16 offset;
13 };
14
15 static const struct mana_stats_desc mana_eth_stats[] = {
16 {"stop_queue", offsetof(struct mana_ethtool_stats, stop_queue)},
17 {"wake_queue", offsetof(struct mana_ethtool_stats, wake_queue)},
18 {"tx_cq_err", offsetof(struct mana_ethtool_stats, tx_cqe_err)},
19 {"tx_cqe_unknown_type", offsetof(struct mana_ethtool_stats,
20 tx_cqe_unknown_type)},
21 {"tx_linear_pkt_cnt", offsetof(struct mana_ethtool_stats,
22 tx_linear_pkt_cnt)},
23 {"rx_cqe_unknown_type", offsetof(struct mana_ethtool_stats,
24 rx_cqe_unknown_type)},
25 };
26
27 static const struct mana_stats_desc mana_hc_stats[] = {
28 {"hc_rx_discards_no_wqe", offsetof(struct mana_ethtool_hc_stats,
29 hc_rx_discards_no_wqe)},
30 {"hc_rx_err_vport_disabled", offsetof(struct mana_ethtool_hc_stats,
31 hc_rx_err_vport_disabled)},
32 {"hc_rx_bytes", offsetof(struct mana_ethtool_hc_stats, hc_rx_bytes)},
33 {"hc_rx_ucast_pkts", offsetof(struct mana_ethtool_hc_stats,
34 hc_rx_ucast_pkts)},
35 {"hc_rx_ucast_bytes", offsetof(struct mana_ethtool_hc_stats,
36 hc_rx_ucast_bytes)},
37 {"hc_rx_bcast_pkts", offsetof(struct mana_ethtool_hc_stats,
38 hc_rx_bcast_pkts)},
39 {"hc_rx_bcast_bytes", offsetof(struct mana_ethtool_hc_stats,
40 hc_rx_bcast_bytes)},
41 {"hc_rx_mcast_pkts", offsetof(struct mana_ethtool_hc_stats,
42 hc_rx_mcast_pkts)},
43 {"hc_rx_mcast_bytes", offsetof(struct mana_ethtool_hc_stats,
44 hc_rx_mcast_bytes)},
45 {"hc_tx_err_gf_disabled", offsetof(struct mana_ethtool_hc_stats,
46 hc_tx_err_gf_disabled)},
47 {"hc_tx_err_vport_disabled", offsetof(struct mana_ethtool_hc_stats,
48 hc_tx_err_vport_disabled)},
49 {"hc_tx_err_inval_vportoffset_pkt",
50 offsetof(struct mana_ethtool_hc_stats,
51 hc_tx_err_inval_vportoffset_pkt)},
52 {"hc_tx_err_vlan_enforcement", offsetof(struct mana_ethtool_hc_stats,
53 hc_tx_err_vlan_enforcement)},
54 {"hc_tx_err_eth_type_enforcement",
55 offsetof(struct mana_ethtool_hc_stats, hc_tx_err_eth_type_enforcement)},
56 {"hc_tx_err_sa_enforcement", offsetof(struct mana_ethtool_hc_stats,
57 hc_tx_err_sa_enforcement)},
58 {"hc_tx_err_sqpdid_enforcement",
59 offsetof(struct mana_ethtool_hc_stats, hc_tx_err_sqpdid_enforcement)},
60 {"hc_tx_err_cqpdid_enforcement",
61 offsetof(struct mana_ethtool_hc_stats, hc_tx_err_cqpdid_enforcement)},
62 {"hc_tx_err_mtu_violation", offsetof(struct mana_ethtool_hc_stats,
63 hc_tx_err_mtu_violation)},
64 {"hc_tx_err_inval_oob", offsetof(struct mana_ethtool_hc_stats,
65 hc_tx_err_inval_oob)},
66 {"hc_tx_err_gdma", offsetof(struct mana_ethtool_hc_stats,
67 hc_tx_err_gdma)},
68 {"hc_tx_bytes", offsetof(struct mana_ethtool_hc_stats, hc_tx_bytes)},
69 {"hc_tx_ucast_pkts", offsetof(struct mana_ethtool_hc_stats,
70 hc_tx_ucast_pkts)},
71 {"hc_tx_ucast_bytes", offsetof(struct mana_ethtool_hc_stats,
72 hc_tx_ucast_bytes)},
73 {"hc_tx_bcast_pkts", offsetof(struct mana_ethtool_hc_stats,
74 hc_tx_bcast_pkts)},
75 {"hc_tx_bcast_bytes", offsetof(struct mana_ethtool_hc_stats,
76 hc_tx_bcast_bytes)},
77 {"hc_tx_mcast_pkts", offsetof(struct mana_ethtool_hc_stats,
78 hc_tx_mcast_pkts)},
79 {"hc_tx_mcast_bytes", offsetof(struct mana_ethtool_hc_stats,
80 hc_tx_mcast_bytes)},
81 };
82
83 static const struct mana_stats_desc mana_phy_stats[] = {
84 { "hc_rx_pkt_drop_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_drop_phy) },
85 { "hc_tx_pkt_drop_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_drop_phy) },
86 { "hc_tc0_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc0_phy) },
87 { "hc_tc0_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc0_phy) },
88 { "hc_tc0_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc0_phy) },
89 { "hc_tc0_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc0_phy) },
90 { "hc_tc1_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc1_phy) },
91 { "hc_tc1_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc1_phy) },
92 { "hc_tc1_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc1_phy) },
93 { "hc_tc1_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc1_phy) },
94 { "hc_tc2_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc2_phy) },
95 { "hc_tc2_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc2_phy) },
96 { "hc_tc2_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc2_phy) },
97 { "hc_tc2_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc2_phy) },
98 { "hc_tc3_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc3_phy) },
99 { "hc_tc3_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc3_phy) },
100 { "hc_tc3_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc3_phy) },
101 { "hc_tc3_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc3_phy) },
102 { "hc_tc4_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc4_phy) },
103 { "hc_tc4_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc4_phy) },
104 { "hc_tc4_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc4_phy) },
105 { "hc_tc4_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc4_phy) },
106 { "hc_tc5_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc5_phy) },
107 { "hc_tc5_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc5_phy) },
108 { "hc_tc5_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc5_phy) },
109 { "hc_tc5_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc5_phy) },
110 { "hc_tc6_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc6_phy) },
111 { "hc_tc6_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc6_phy) },
112 { "hc_tc6_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc6_phy) },
113 { "hc_tc6_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc6_phy) },
114 { "hc_tc7_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc7_phy) },
115 { "hc_tc7_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc7_phy) },
116 { "hc_tc7_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc7_phy) },
117 { "hc_tc7_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc7_phy) },
118 { "hc_tc0_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc0_phy) },
119 { "hc_tc0_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc0_phy) },
120 { "hc_tc1_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc1_phy) },
121 { "hc_tc1_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc1_phy) },
122 { "hc_tc2_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc2_phy) },
123 { "hc_tc2_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc2_phy) },
124 { "hc_tc3_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc3_phy) },
125 { "hc_tc3_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc3_phy) },
126 { "hc_tc4_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc4_phy) },
127 { "hc_tc4_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc4_phy) },
128 { "hc_tc5_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc5_phy) },
129 { "hc_tc5_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc5_phy) },
130 { "hc_tc6_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc6_phy) },
131 { "hc_tc6_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc6_phy) },
132 { "hc_tc7_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc7_phy) },
133 { "hc_tc7_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc7_phy) },
134 };
135
136 static const char mana_priv_flags[MANA_PRIV_FLAG_MAX][ETH_GSTRING_LEN] = {
137 [MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF] = "full-page-rx"
138 };
139
mana_get_sset_count(struct net_device * ndev,int stringset)140 static int mana_get_sset_count(struct net_device *ndev, int stringset)
141 {
142 struct mana_port_context *apc = netdev_priv(ndev);
143 unsigned int num_queues = apc->num_queues;
144
145 switch (stringset) {
146 case ETH_SS_STATS:
147 return ARRAY_SIZE(mana_eth_stats) +
148 ARRAY_SIZE(mana_phy_stats) +
149 ARRAY_SIZE(mana_hc_stats) +
150 num_queues * (MANA_STATS_RX_COUNT + MANA_STATS_TX_COUNT);
151
152 case ETH_SS_PRIV_FLAGS:
153 return MANA_PRIV_FLAG_MAX;
154
155 default:
156 return -EINVAL;
157 }
158 }
159
mana_get_strings_stats(struct mana_port_context * apc,u8 ** data)160 static void mana_get_strings_stats(struct mana_port_context *apc, u8 **data)
161 {
162 unsigned int num_queues = apc->num_queues;
163 int i, j;
164
165 for (i = 0; i < ARRAY_SIZE(mana_eth_stats); i++)
166 ethtool_puts(data, mana_eth_stats[i].name);
167
168 for (i = 0; i < ARRAY_SIZE(mana_hc_stats); i++)
169 ethtool_puts(data, mana_hc_stats[i].name);
170
171 for (i = 0; i < ARRAY_SIZE(mana_phy_stats); i++)
172 ethtool_puts(data, mana_phy_stats[i].name);
173
174 for (i = 0; i < num_queues; i++) {
175 ethtool_sprintf(data, "rx_%d_packets", i);
176 ethtool_sprintf(data, "rx_%d_bytes", i);
177 ethtool_sprintf(data, "rx_%d_xdp_drop", i);
178 ethtool_sprintf(data, "rx_%d_xdp_tx", i);
179 ethtool_sprintf(data, "rx_%d_xdp_redirect", i);
180 ethtool_sprintf(data, "rx_%d_pkt_len0_err", i);
181 for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++)
182 ethtool_sprintf(data,
183 "rx_%d_coalesced_cqe_%d",
184 i,
185 j + 2);
186 }
187
188 for (i = 0; i < num_queues; i++) {
189 ethtool_sprintf(data, "tx_%d_packets", i);
190 ethtool_sprintf(data, "tx_%d_bytes", i);
191 ethtool_sprintf(data, "tx_%d_xdp_xmit", i);
192 ethtool_sprintf(data, "tx_%d_tso_packets", i);
193 ethtool_sprintf(data, "tx_%d_tso_bytes", i);
194 ethtool_sprintf(data, "tx_%d_tso_inner_packets", i);
195 ethtool_sprintf(data, "tx_%d_tso_inner_bytes", i);
196 ethtool_sprintf(data, "tx_%d_long_pkt_fmt", i);
197 ethtool_sprintf(data, "tx_%d_short_pkt_fmt", i);
198 ethtool_sprintf(data, "tx_%d_csum_partial", i);
199 ethtool_sprintf(data, "tx_%d_mana_map_err", i);
200 }
201 }
202
mana_get_strings_priv_flags(u8 ** data)203 static void mana_get_strings_priv_flags(u8 **data)
204 {
205 int i;
206
207 for (i = 0; i < MANA_PRIV_FLAG_MAX; i++)
208 ethtool_puts(data, mana_priv_flags[i]);
209 }
210
mana_get_strings(struct net_device * ndev,u32 stringset,u8 * data)211 static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data)
212 {
213 struct mana_port_context *apc = netdev_priv(ndev);
214
215 switch (stringset) {
216 case ETH_SS_STATS:
217 mana_get_strings_stats(apc, &data);
218 break;
219 case ETH_SS_PRIV_FLAGS:
220 mana_get_strings_priv_flags(&data);
221 break;
222 default:
223 break;
224 }
225 }
226
mana_get_ethtool_stats(struct net_device * ndev,struct ethtool_stats * e_stats,u64 * data)227 static void mana_get_ethtool_stats(struct net_device *ndev,
228 struct ethtool_stats *e_stats, u64 *data)
229 {
230 struct mana_port_context *apc = netdev_priv(ndev);
231 unsigned int num_queues = apc->num_queues;
232 void *eth_stats = &apc->eth_stats;
233 void *hc_stats = &apc->ac->hc_stats;
234 void *phy_stats = &apc->phy_stats;
235 struct mana_stats_rx *rx_stats;
236 struct mana_stats_tx *tx_stats;
237 unsigned int start;
238 u64 packets, bytes;
239 u64 xdp_redirect;
240 u64 xdp_xmit;
241 u64 xdp_drop;
242 u64 xdp_tx;
243 u64 pkt_len0_err;
244 u64 coalesced_cqe[MANA_CQE_COAL_PKTS_8 - 1];
245 u64 tso_packets;
246 u64 tso_bytes;
247 u64 tso_inner_packets;
248 u64 tso_inner_bytes;
249 u64 long_pkt_fmt;
250 u64 short_pkt_fmt;
251 u64 csum_partial;
252 u64 mana_map_err;
253 int q, i = 0, j;
254
255 if (!apc->port_is_up)
256 return;
257
258 /* We call this mana function to get the phy stats from GDMA and includes
259 * aggregate tx/rx drop counters, Per-TC(Traffic Channel) tx/rx and pause
260 * counters.
261 */
262 mana_query_phy_stats(apc);
263
264 for (q = 0; q < ARRAY_SIZE(mana_eth_stats); q++)
265 data[i++] = *(u64 *)(eth_stats + mana_eth_stats[q].offset);
266
267 for (q = 0; q < ARRAY_SIZE(mana_hc_stats); q++)
268 data[i++] = *(u64 *)(hc_stats + mana_hc_stats[q].offset);
269
270 for (q = 0; q < ARRAY_SIZE(mana_phy_stats); q++)
271 data[i++] = *(u64 *)(phy_stats + mana_phy_stats[q].offset);
272
273 for (q = 0; q < num_queues; q++) {
274 rx_stats = &apc->rxqs[q]->stats;
275
276 do {
277 start = u64_stats_fetch_begin(&rx_stats->syncp);
278 packets = rx_stats->packets;
279 bytes = rx_stats->bytes;
280 xdp_drop = rx_stats->xdp_drop;
281 xdp_tx = rx_stats->xdp_tx;
282 xdp_redirect = rx_stats->xdp_redirect;
283 pkt_len0_err = rx_stats->pkt_len0_err;
284 for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++)
285 coalesced_cqe[j] = rx_stats->coalesced_cqe[j];
286 } while (u64_stats_fetch_retry(&rx_stats->syncp, start));
287
288 data[i++] = packets;
289 data[i++] = bytes;
290 data[i++] = xdp_drop;
291 data[i++] = xdp_tx;
292 data[i++] = xdp_redirect;
293 data[i++] = pkt_len0_err;
294 for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++)
295 data[i++] = coalesced_cqe[j];
296 }
297
298 for (q = 0; q < num_queues; q++) {
299 tx_stats = &apc->tx_qp[q]->txq.stats;
300
301 do {
302 start = u64_stats_fetch_begin(&tx_stats->syncp);
303 packets = tx_stats->packets;
304 bytes = tx_stats->bytes;
305 xdp_xmit = tx_stats->xdp_xmit;
306 tso_packets = tx_stats->tso_packets;
307 tso_bytes = tx_stats->tso_bytes;
308 tso_inner_packets = tx_stats->tso_inner_packets;
309 tso_inner_bytes = tx_stats->tso_inner_bytes;
310 long_pkt_fmt = tx_stats->long_pkt_fmt;
311 short_pkt_fmt = tx_stats->short_pkt_fmt;
312 csum_partial = tx_stats->csum_partial;
313 mana_map_err = tx_stats->mana_map_err;
314 } while (u64_stats_fetch_retry(&tx_stats->syncp, start));
315
316 data[i++] = packets;
317 data[i++] = bytes;
318 data[i++] = xdp_xmit;
319 data[i++] = tso_packets;
320 data[i++] = tso_bytes;
321 data[i++] = tso_inner_packets;
322 data[i++] = tso_inner_bytes;
323 data[i++] = long_pkt_fmt;
324 data[i++] = short_pkt_fmt;
325 data[i++] = csum_partial;
326 data[i++] = mana_map_err;
327 }
328 }
329
mana_get_rx_ring_count(struct net_device * ndev)330 static u32 mana_get_rx_ring_count(struct net_device *ndev)
331 {
332 struct mana_port_context *apc = netdev_priv(ndev);
333
334 return apc->num_queues;
335 }
336
mana_get_rxfh_key_size(struct net_device * ndev)337 static u32 mana_get_rxfh_key_size(struct net_device *ndev)
338 {
339 return MANA_HASH_KEY_SIZE;
340 }
341
mana_rss_indir_size(struct net_device * ndev)342 static u32 mana_rss_indir_size(struct net_device *ndev)
343 {
344 struct mana_port_context *apc = netdev_priv(ndev);
345
346 return apc->indir_table_sz;
347 }
348
mana_get_rxfh(struct net_device * ndev,struct ethtool_rxfh_param * rxfh)349 static int mana_get_rxfh(struct net_device *ndev,
350 struct ethtool_rxfh_param *rxfh)
351 {
352 struct mana_port_context *apc = netdev_priv(ndev);
353 int i;
354
355 rxfh->hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */
356
357 if (rxfh->indir) {
358 for (i = 0; i < apc->indir_table_sz; i++)
359 rxfh->indir[i] = apc->indir_table[i];
360 }
361
362 if (rxfh->key)
363 memcpy(rxfh->key, apc->hashkey, MANA_HASH_KEY_SIZE);
364
365 return 0;
366 }
367
mana_set_rxfh(struct net_device * ndev,struct ethtool_rxfh_param * rxfh,struct netlink_ext_ack * extack)368 static int mana_set_rxfh(struct net_device *ndev,
369 struct ethtool_rxfh_param *rxfh,
370 struct netlink_ext_ack *extack)
371 {
372 struct mana_port_context *apc = netdev_priv(ndev);
373 bool update_hash = false, update_table = false;
374 u8 save_key[MANA_HASH_KEY_SIZE];
375 u32 *save_table;
376 int i, err;
377
378 if (!apc->port_is_up)
379 return -EOPNOTSUPP;
380
381 if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE &&
382 rxfh->hfunc != ETH_RSS_HASH_TOP)
383 return -EOPNOTSUPP;
384
385 save_table = kcalloc(apc->indir_table_sz, sizeof(u32), GFP_KERNEL);
386 if (!save_table)
387 return -ENOMEM;
388
389 if (rxfh->indir) {
390 for (i = 0; i < apc->indir_table_sz; i++)
391 if (rxfh->indir[i] >= apc->num_queues) {
392 err = -EINVAL;
393 goto cleanup;
394 }
395
396 update_table = true;
397 for (i = 0; i < apc->indir_table_sz; i++) {
398 save_table[i] = apc->indir_table[i];
399 apc->indir_table[i] = rxfh->indir[i];
400 }
401 }
402
403 if (rxfh->key) {
404 update_hash = true;
405 memcpy(save_key, apc->hashkey, MANA_HASH_KEY_SIZE);
406 memcpy(apc->hashkey, rxfh->key, MANA_HASH_KEY_SIZE);
407 }
408
409 err = mana_config_rss(apc, TRI_STATE_TRUE, update_hash, update_table);
410
411 if (err) { /* recover to original values */
412 if (update_table) {
413 for (i = 0; i < apc->indir_table_sz; i++)
414 apc->indir_table[i] = save_table[i];
415 }
416
417 if (update_hash)
418 memcpy(apc->hashkey, save_key, MANA_HASH_KEY_SIZE);
419
420 mana_config_rss(apc, TRI_STATE_TRUE, update_hash, update_table);
421 }
422
423 cleanup:
424 kfree(save_table);
425
426 return err;
427 }
428
mana_get_channels(struct net_device * ndev,struct ethtool_channels * channel)429 static void mana_get_channels(struct net_device *ndev,
430 struct ethtool_channels *channel)
431 {
432 struct mana_port_context *apc = netdev_priv(ndev);
433
434 channel->max_combined = apc->max_queues;
435 channel->combined_count = apc->num_queues;
436 }
437
438 #define MANA_RX_CQE_NSEC_DEF 2048
mana_get_coalesce(struct net_device * ndev,struct ethtool_coalesce * ec,struct kernel_ethtool_coalesce * kernel_coal,struct netlink_ext_ack * extack)439 static int mana_get_coalesce(struct net_device *ndev,
440 struct ethtool_coalesce *ec,
441 struct kernel_ethtool_coalesce *kernel_coal,
442 struct netlink_ext_ack *extack)
443 {
444 struct mana_port_context *apc = netdev_priv(ndev);
445
446 kernel_coal->rx_cqe_frames =
447 apc->cqe8_coalescing_enable ? MANA_CQE_COAL_PKTS_8 :
448 apc->cqe_coalescing_enable ? MANA_RXCOMP_OOB_NUM_PPI : 1;
449
450 kernel_coal->rx_cqe_nsecs = apc->cqe_coalescing_timeout_ns;
451
452 /* Return the default timeout value for old FW not providing
453 * this value.
454 */
455 if (apc->port_is_up && apc->cqe_coalescing_enable &&
456 !kernel_coal->rx_cqe_nsecs)
457 kernel_coal->rx_cqe_nsecs = MANA_RX_CQE_NSEC_DEF;
458
459 ec->rx_coalesce_usecs = apc->intr_modr_rx_usec;
460 ec->rx_max_coalesced_frames = apc->intr_modr_rx_comp;
461
462 ec->tx_coalesce_usecs = apc->intr_modr_tx_usec;
463 ec->tx_max_coalesced_frames = apc->intr_modr_tx_comp;
464
465 ec->use_adaptive_rx_coalesce = apc->rx_dim_enabled;
466 ec->use_adaptive_tx_coalesce = apc->tx_dim_enabled;
467
468 return 0;
469 }
470
mana_set_coalesce(struct net_device * ndev,struct ethtool_coalesce * ec,struct kernel_ethtool_coalesce * kernel_coal,struct netlink_ext_ack * extack)471 static int mana_set_coalesce(struct net_device *ndev,
472 struct ethtool_coalesce *ec,
473 struct kernel_ethtool_coalesce *kernel_coal,
474 struct netlink_ext_ack *extack)
475 {
476 struct mana_port_context *apc = netdev_priv(ndev);
477 struct {
478 u16 intr_modr_rx_usec;
479 u16 intr_modr_rx_comp;
480 u16 intr_modr_tx_usec;
481 u16 intr_modr_tx_comp;
482 u8 cqe_coalescing_enable;
483 u8 cqe8_coalescing_enable;
484 bool rx_dim_enabled;
485 bool tx_dim_enabled;
486 } saved;
487 bool modr_changed = false;
488 bool dim_changed = false;
489 struct gdma_context *gc;
490 u32 max_cqe_frames;
491 int err;
492
493 gc = apc->ac->gdma_dev->gdma_context;
494 max_cqe_frames = gc->cqe8_coalescing_sup ? MANA_CQE_COAL_PKTS_8 :
495 MANA_RXCOMP_OOB_NUM_PPI;
496
497 /* Both static and dynamic interrupt moderation (DIM) rely on the
498 * same HW capability advertised by the PF.
499 */
500 if ((ec->use_adaptive_rx_coalesce || ec->use_adaptive_tx_coalesce ||
501 ec->rx_coalesce_usecs || ec->tx_coalesce_usecs ||
502 ec->rx_max_coalesced_frames || ec->tx_max_coalesced_frames) &&
503 !(gc->pf_cap_flags1 & GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION)) {
504 NL_SET_ERR_MSG(extack,
505 "Interrupt Moderation is not supported by HW");
506 return -EOPNOTSUPP;
507 }
508
509 if (kernel_coal->rx_cqe_frames != 1 &&
510 kernel_coal->rx_cqe_frames != MANA_RXCOMP_OOB_NUM_PPI &&
511 kernel_coal->rx_cqe_frames != max_cqe_frames) {
512 NL_SET_ERR_MSG_FMT(extack,
513 "rx-frames must be 1 or %u%s, got %u",
514 MANA_RXCOMP_OOB_NUM_PPI,
515 gc->cqe8_coalescing_sup ? " or 8" : "",
516 kernel_coal->rx_cqe_frames);
517 return -EINVAL;
518 }
519
520 if (ec->rx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX ||
521 ec->tx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX) {
522 NL_SET_ERR_MSG_FMT(extack,
523 "coalesce usecs must be <= %lu",
524 MANA_INTR_MODR_USEC_MAX);
525 return -EINVAL;
526 }
527
528 if (ec->rx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX ||
529 ec->tx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX) {
530 NL_SET_ERR_MSG_FMT(extack,
531 "coalesce frames must be <= %lu",
532 MANA_INTR_MODR_COMP_MAX);
533 return -EINVAL;
534 }
535
536 if (ec->rx_coalesce_usecs != apc->intr_modr_rx_usec ||
537 ec->rx_max_coalesced_frames != apc->intr_modr_rx_comp ||
538 ec->tx_coalesce_usecs != apc->intr_modr_tx_usec ||
539 ec->tx_max_coalesced_frames != apc->intr_modr_tx_comp)
540 modr_changed = true;
541
542 saved.intr_modr_rx_usec = apc->intr_modr_rx_usec;
543 saved.intr_modr_rx_comp = apc->intr_modr_rx_comp;
544 saved.intr_modr_tx_usec = apc->intr_modr_tx_usec;
545 saved.intr_modr_tx_comp = apc->intr_modr_tx_comp;
546
547 apc->intr_modr_rx_usec = ec->rx_coalesce_usecs;
548 apc->intr_modr_rx_comp = ec->rx_max_coalesced_frames;
549 apc->intr_modr_tx_usec = ec->tx_coalesce_usecs;
550 apc->intr_modr_tx_comp = ec->tx_max_coalesced_frames;
551
552 if (!!ec->use_adaptive_rx_coalesce != apc->rx_dim_enabled ||
553 !!ec->use_adaptive_tx_coalesce != apc->tx_dim_enabled)
554 dim_changed = true;
555
556 saved.rx_dim_enabled = apc->rx_dim_enabled;
557 saved.tx_dim_enabled = apc->tx_dim_enabled;
558
559 saved.cqe_coalescing_enable = apc->cqe_coalescing_enable;
560 saved.cqe8_coalescing_enable = apc->cqe8_coalescing_enable;
561 apc->cqe_coalescing_enable =
562 kernel_coal->rx_cqe_frames >= MANA_RXCOMP_OOB_NUM_PPI;
563 apc->cqe8_coalescing_enable =
564 kernel_coal->rx_cqe_frames == MANA_CQE_COAL_PKTS_8;
565
566 if (!apc->port_is_up) {
567 WRITE_ONCE(apc->rx_dim_enabled, !!ec->use_adaptive_rx_coalesce);
568 WRITE_ONCE(apc->tx_dim_enabled, !!ec->use_adaptive_tx_coalesce);
569 return 0;
570 }
571
572 if (apc->cqe_coalescing_enable != saved.cqe_coalescing_enable ||
573 apc->cqe8_coalescing_enable != saved.cqe8_coalescing_enable) {
574 /* CQE coalescing setting is applied via RSS configuration. */
575 err = mana_config_rss(apc, TRI_STATE_TRUE, false, false);
576 if (err) {
577 netdev_err(ndev, "Change CQE coalescing failed: %d\n",
578 err);
579 apc->cqe_coalescing_enable =
580 saved.cqe_coalescing_enable;
581 apc->cqe8_coalescing_enable =
582 saved.cqe8_coalescing_enable;
583 apc->intr_modr_rx_usec = saved.intr_modr_rx_usec;
584 apc->intr_modr_rx_comp = saved.intr_modr_rx_comp;
585 apc->intr_modr_tx_usec = saved.intr_modr_tx_usec;
586 apc->intr_modr_tx_comp = saved.intr_modr_tx_comp;
587 return err;
588 }
589 }
590
591 if (modr_changed || dim_changed) {
592 bool new_rx_dim = !!ec->use_adaptive_rx_coalesce;
593 bool new_tx_dim = !!ec->use_adaptive_tx_coalesce;
594 bool disable_rx_dim = saved.rx_dim_enabled && !new_rx_dim;
595 bool disable_tx_dim = saved.tx_dim_enabled && !new_tx_dim;
596 bool enable_rx_dim = !saved.rx_dim_enabled && new_rx_dim;
597 bool enable_tx_dim = !saved.tx_dim_enabled && new_tx_dim;
598 int q;
599
600 /* On disable: clear the per-port flag first and
601 * synchronize_net() so any in-flight NAPI poll observes
602 * the new value and will not schedule further DIM work;
603 * then drain pending work and restore the static
604 * moderation values.
605 */
606 if (disable_rx_dim)
607 WRITE_ONCE(apc->rx_dim_enabled, false);
608 if (disable_tx_dim)
609 WRITE_ONCE(apc->tx_dim_enabled, false);
610 if (disable_rx_dim || disable_tx_dim)
611 synchronize_net();
612
613 for (q = 0; q < apc->num_queues; q++) {
614 struct mana_cq *rx_cq = &apc->rxqs[q]->rx_cq;
615 struct mana_cq *tx_cq = &apc->tx_qp[q]->tx_cq;
616
617 if (disable_rx_dim)
618 mana_dim_change(rx_cq, false);
619 else if (enable_rx_dim)
620 mana_dim_change(rx_cq, true);
621 else if (!new_rx_dim && modr_changed)
622 mana_gd_ring_dim(rx_cq->gdma_cq,
623 apc->intr_modr_rx_usec, true,
624 apc->intr_modr_rx_comp, true);
625
626 if (disable_tx_dim)
627 mana_dim_change(tx_cq, false);
628 else if (enable_tx_dim)
629 mana_dim_change(tx_cq, true);
630 else if (!new_tx_dim && modr_changed)
631 mana_gd_ring_dim(tx_cq->gdma_cq,
632 apc->intr_modr_tx_usec, true,
633 apc->intr_modr_tx_comp, true);
634 }
635
636 /* Publish the enable flag with release semantics so a
637 * concurrent NAPI poll that observes it set also sees the DIM
638 * (re)init done by mana_dim_change() above.
639 */
640 if (enable_rx_dim)
641 /* pairs with smp_load_acquire() in mana_update_rx_dim() */
642 smp_store_release(&apc->rx_dim_enabled, true);
643 if (enable_tx_dim)
644 /* pairs with smp_load_acquire() in mana_update_tx_dim() */
645 smp_store_release(&apc->tx_dim_enabled, true);
646 }
647
648 return 0;
649 }
650
651 /* mana_set_channels - change the number of queues on a port
652 *
653 * Returns -EBUSY if RDMA holds the vport with EQs sized to the
654 * current num_queues.
655 */
mana_set_channels(struct net_device * ndev,struct ethtool_channels * channels)656 static int mana_set_channels(struct net_device *ndev,
657 struct ethtool_channels *channels)
658 {
659 struct mana_port_context *apc = netdev_priv(ndev);
660 unsigned int new_count = channels->combined_count;
661 unsigned int old_count = apc->num_queues;
662 int err;
663
664 /* Set channel_changing to block RDMA from grabbing the vport
665 * during the detach/attach window. mana_cfg_vport() checks
666 * this flag under vport_mutex and returns -EBUSY if set.
667 */
668 mutex_lock(&apc->vport_mutex);
669 if (!apc->port_is_up && apc->vport_use_count) {
670 mutex_unlock(&apc->vport_mutex);
671 return -EBUSY;
672 }
673 apc->channel_changing = true;
674 mutex_unlock(&apc->vport_mutex);
675
676 err = mana_pre_alloc_rxbufs(apc, ndev->mtu, new_count);
677 if (err) {
678 netdev_err(ndev, "Insufficient memory for new allocations");
679 goto clear_flag;
680 }
681
682 err = mana_detach(ndev, false);
683 if (err) {
684 netdev_err(ndev, "mana_detach failed: %d\n", err);
685 goto out;
686 }
687
688 apc->num_queues = new_count;
689 err = mana_attach(ndev);
690 if (err) {
691 apc->num_queues = old_count;
692 netdev_err(ndev, "mana_attach failed: %d\n", err);
693 }
694
695 out:
696 mana_pre_dealloc_rxbufs(apc);
697 clear_flag:
698 mutex_lock(&apc->vport_mutex);
699 apc->channel_changing = false;
700 mutex_unlock(&apc->vport_mutex);
701 return err;
702 }
703
mana_get_ringparam(struct net_device * ndev,struct ethtool_ringparam * ring,struct kernel_ethtool_ringparam * kernel_ring,struct netlink_ext_ack * extack)704 static void mana_get_ringparam(struct net_device *ndev,
705 struct ethtool_ringparam *ring,
706 struct kernel_ethtool_ringparam *kernel_ring,
707 struct netlink_ext_ack *extack)
708 {
709 struct mana_port_context *apc = netdev_priv(ndev);
710
711 ring->rx_pending = apc->rx_queue_size;
712 ring->tx_pending = apc->tx_queue_size;
713 ring->rx_max_pending = MAX_RX_BUFFERS_PER_QUEUE;
714 ring->tx_max_pending = MAX_TX_BUFFERS_PER_QUEUE;
715 }
716
mana_set_ringparam(struct net_device * ndev,struct ethtool_ringparam * ring,struct kernel_ethtool_ringparam * kernel_ring,struct netlink_ext_ack * extack)717 static int mana_set_ringparam(struct net_device *ndev,
718 struct ethtool_ringparam *ring,
719 struct kernel_ethtool_ringparam *kernel_ring,
720 struct netlink_ext_ack *extack)
721 {
722 struct mana_port_context *apc = netdev_priv(ndev);
723 u32 new_tx, new_rx;
724 u32 old_tx, old_rx;
725 int err;
726
727 old_tx = apc->tx_queue_size;
728 old_rx = apc->rx_queue_size;
729
730 if (ring->tx_pending < MIN_TX_BUFFERS_PER_QUEUE) {
731 NL_SET_ERR_MSG_FMT(extack, "tx:%d less than the min:%d", ring->tx_pending,
732 MIN_TX_BUFFERS_PER_QUEUE);
733 return -EINVAL;
734 }
735
736 if (ring->rx_pending < MIN_RX_BUFFERS_PER_QUEUE) {
737 NL_SET_ERR_MSG_FMT(extack, "rx:%d less than the min:%d", ring->rx_pending,
738 MIN_RX_BUFFERS_PER_QUEUE);
739 return -EINVAL;
740 }
741
742 new_rx = roundup_pow_of_two(ring->rx_pending);
743 new_tx = roundup_pow_of_two(ring->tx_pending);
744 netdev_info(ndev, "Using nearest power of 2 values for Txq:%d Rxq:%d\n",
745 new_tx, new_rx);
746
747 /* pre-allocating new buffers to prevent failures in mana_attach() later */
748 apc->rx_queue_size = new_rx;
749 err = mana_pre_alloc_rxbufs(apc, ndev->mtu, apc->num_queues);
750 apc->rx_queue_size = old_rx;
751 if (err) {
752 netdev_err(ndev, "Insufficient memory for new allocations\n");
753 return err;
754 }
755
756 err = mana_detach(ndev, false);
757 if (err) {
758 netdev_err(ndev, "mana_detach failed: %d\n", err);
759 goto out;
760 }
761
762 apc->tx_queue_size = new_tx;
763 apc->rx_queue_size = new_rx;
764
765 err = mana_attach(ndev);
766 if (err) {
767 netdev_err(ndev, "mana_attach failed: %d\n", err);
768 apc->tx_queue_size = old_tx;
769 apc->rx_queue_size = old_rx;
770 }
771 out:
772 mana_pre_dealloc_rxbufs(apc);
773 return err;
774 }
775
mana_get_link_ksettings(struct net_device * ndev,struct ethtool_link_ksettings * cmd)776 static int mana_get_link_ksettings(struct net_device *ndev,
777 struct ethtool_link_ksettings *cmd)
778 {
779 struct mana_port_context *apc = netdev_priv(ndev);
780 int err;
781
782 err = mana_query_link_cfg(apc);
783 cmd->base.speed = (err) ? SPEED_UNKNOWN : apc->max_speed;
784
785 cmd->base.duplex = DUPLEX_FULL;
786 cmd->base.port = PORT_OTHER;
787
788 return 0;
789 }
790
mana_get_priv_flags(struct net_device * ndev)791 static u32 mana_get_priv_flags(struct net_device *ndev)
792 {
793 struct mana_port_context *apc = netdev_priv(ndev);
794
795 return apc->priv_flags;
796 }
797
mana_set_priv_flags(struct net_device * ndev,u32 priv_flags)798 static int mana_set_priv_flags(struct net_device *ndev, u32 priv_flags)
799 {
800 struct mana_port_context *apc = netdev_priv(ndev);
801 u32 changed = apc->priv_flags ^ priv_flags;
802 u32 old_priv_flags = apc->priv_flags;
803 int err = 0;
804
805 if (!changed)
806 return 0;
807
808 /* Reject unknown bits */
809 if (priv_flags & ~GENMASK(MANA_PRIV_FLAG_MAX - 1, 0))
810 return -EINVAL;
811
812 apc->priv_flags = priv_flags;
813
814 if (changed & BIT(MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF)) {
815 if (!apc->port_is_up)
816 return 0;
817
818 /* If XDP is attached or MTU is jumbo, single-buffer-per-page
819 * is already forced regardless of this flag. Skip the
820 * expensive detach/attach cycle since nothing changes.
821 */
822 if (ndev->mtu + MANA_RXBUF_PAD > PAGE_SIZE / 2 ||
823 mana_xdp_get(apc))
824 return 0;
825
826 /* Block RDMA from grabbing the vport during detach/attach */
827 mutex_lock(&apc->vport_mutex);
828 apc->channel_changing = true;
829 mutex_unlock(&apc->vport_mutex);
830
831 err = mana_pre_alloc_rxbufs(apc, ndev->mtu, apc->num_queues);
832 if (err) {
833 netdev_err(ndev,
834 "Insufficient memory for new allocations\n");
835 apc->priv_flags = old_priv_flags;
836 goto clear_flag;
837 }
838
839 err = mana_detach(ndev, false);
840 if (err) {
841 netdev_err(ndev, "mana_detach failed: %d\n", err);
842 apc->priv_flags = old_priv_flags;
843 goto out;
844 }
845
846 err = mana_attach(ndev);
847 if (err) {
848 netdev_err(ndev, "mana_attach failed: %d\n", err);
849 apc->priv_flags = old_priv_flags;
850 }
851 }
852
853 out:
854 mana_pre_dealloc_rxbufs(apc);
855 clear_flag:
856 mutex_lock(&apc->vport_mutex);
857 apc->channel_changing = false;
858 mutex_unlock(&apc->vport_mutex);
859
860 return err;
861 }
862
863 const struct ethtool_ops mana_ethtool_ops = {
864 .supported_coalesce_params = ETHTOOL_COALESCE_RX_CQE_FRAMES |
865 ETHTOOL_COALESCE_RX_USECS |
866 ETHTOOL_COALESCE_RX_MAX_FRAMES |
867 ETHTOOL_COALESCE_TX_USECS |
868 ETHTOOL_COALESCE_TX_MAX_FRAMES |
869 ETHTOOL_COALESCE_USE_ADAPTIVE_RX |
870 ETHTOOL_COALESCE_USE_ADAPTIVE_TX,
871 .op_needs_rtnl = ETHTOOL_OP_NEEDS_RTNL_SCHANNELS |
872 ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM |
873 ETHTOOL_OP_NEEDS_RTNL_SPFLAGS |
874 ETHTOOL_OP_NEEDS_RTNL_GLINK,
875 .get_ethtool_stats = mana_get_ethtool_stats,
876 .get_sset_count = mana_get_sset_count,
877 .get_strings = mana_get_strings,
878 .get_rx_ring_count = mana_get_rx_ring_count,
879 .get_rxfh_key_size = mana_get_rxfh_key_size,
880 .get_rxfh_indir_size = mana_rss_indir_size,
881 .get_rxfh = mana_get_rxfh,
882 .set_rxfh = mana_set_rxfh,
883 .get_channels = mana_get_channels,
884 .set_channels = mana_set_channels,
885 .get_coalesce = mana_get_coalesce,
886 .set_coalesce = mana_set_coalesce,
887 .get_ringparam = mana_get_ringparam,
888 .set_ringparam = mana_set_ringparam,
889 .get_link_ksettings = mana_get_link_ksettings,
890 .get_link = ethtool_op_get_link,
891 .get_priv_flags = mana_get_priv_flags,
892 .set_priv_flags = mana_set_priv_flags,
893 };
894