1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #include <linux/inetdevice.h> 5 #include <linux/etherdevice.h> 6 #include <linux/ethtool.h> 7 8 #include <net/mana/mana.h> 9 10 struct mana_stats_desc { 11 char name[ETH_GSTRING_LEN]; 12 u16 offset; 13 }; 14 15 static const struct mana_stats_desc mana_eth_stats[] = { 16 {"stop_queue", offsetof(struct mana_ethtool_stats, stop_queue)}, 17 {"wake_queue", offsetof(struct mana_ethtool_stats, wake_queue)}, 18 {"tx_cq_err", offsetof(struct mana_ethtool_stats, tx_cqe_err)}, 19 {"tx_cqe_unknown_type", offsetof(struct mana_ethtool_stats, 20 tx_cqe_unknown_type)}, 21 {"tx_linear_pkt_cnt", offsetof(struct mana_ethtool_stats, 22 tx_linear_pkt_cnt)}, 23 {"rx_cqe_unknown_type", offsetof(struct mana_ethtool_stats, 24 rx_cqe_unknown_type)}, 25 }; 26 27 static const struct mana_stats_desc mana_hc_stats[] = { 28 {"hc_rx_discards_no_wqe", offsetof(struct mana_ethtool_hc_stats, 29 hc_rx_discards_no_wqe)}, 30 {"hc_rx_err_vport_disabled", offsetof(struct mana_ethtool_hc_stats, 31 hc_rx_err_vport_disabled)}, 32 {"hc_rx_bytes", offsetof(struct mana_ethtool_hc_stats, hc_rx_bytes)}, 33 {"hc_rx_ucast_pkts", offsetof(struct mana_ethtool_hc_stats, 34 hc_rx_ucast_pkts)}, 35 {"hc_rx_ucast_bytes", offsetof(struct mana_ethtool_hc_stats, 36 hc_rx_ucast_bytes)}, 37 {"hc_rx_bcast_pkts", offsetof(struct mana_ethtool_hc_stats, 38 hc_rx_bcast_pkts)}, 39 {"hc_rx_bcast_bytes", offsetof(struct mana_ethtool_hc_stats, 40 hc_rx_bcast_bytes)}, 41 {"hc_rx_mcast_pkts", offsetof(struct mana_ethtool_hc_stats, 42 hc_rx_mcast_pkts)}, 43 {"hc_rx_mcast_bytes", offsetof(struct mana_ethtool_hc_stats, 44 hc_rx_mcast_bytes)}, 45 {"hc_tx_err_gf_disabled", offsetof(struct mana_ethtool_hc_stats, 46 hc_tx_err_gf_disabled)}, 47 {"hc_tx_err_vport_disabled", offsetof(struct mana_ethtool_hc_stats, 48 hc_tx_err_vport_disabled)}, 49 {"hc_tx_err_inval_vportoffset_pkt", 50 offsetof(struct mana_ethtool_hc_stats, 51 hc_tx_err_inval_vportoffset_pkt)}, 52 {"hc_tx_err_vlan_enforcement", offsetof(struct mana_ethtool_hc_stats, 53 hc_tx_err_vlan_enforcement)}, 54 {"hc_tx_err_eth_type_enforcement", 55 offsetof(struct mana_ethtool_hc_stats, hc_tx_err_eth_type_enforcement)}, 56 {"hc_tx_err_sa_enforcement", offsetof(struct mana_ethtool_hc_stats, 57 hc_tx_err_sa_enforcement)}, 58 {"hc_tx_err_sqpdid_enforcement", 59 offsetof(struct mana_ethtool_hc_stats, hc_tx_err_sqpdid_enforcement)}, 60 {"hc_tx_err_cqpdid_enforcement", 61 offsetof(struct mana_ethtool_hc_stats, hc_tx_err_cqpdid_enforcement)}, 62 {"hc_tx_err_mtu_violation", offsetof(struct mana_ethtool_hc_stats, 63 hc_tx_err_mtu_violation)}, 64 {"hc_tx_err_inval_oob", offsetof(struct mana_ethtool_hc_stats, 65 hc_tx_err_inval_oob)}, 66 {"hc_tx_err_gdma", offsetof(struct mana_ethtool_hc_stats, 67 hc_tx_err_gdma)}, 68 {"hc_tx_bytes", offsetof(struct mana_ethtool_hc_stats, hc_tx_bytes)}, 69 {"hc_tx_ucast_pkts", offsetof(struct mana_ethtool_hc_stats, 70 hc_tx_ucast_pkts)}, 71 {"hc_tx_ucast_bytes", offsetof(struct mana_ethtool_hc_stats, 72 hc_tx_ucast_bytes)}, 73 {"hc_tx_bcast_pkts", offsetof(struct mana_ethtool_hc_stats, 74 hc_tx_bcast_pkts)}, 75 {"hc_tx_bcast_bytes", offsetof(struct mana_ethtool_hc_stats, 76 hc_tx_bcast_bytes)}, 77 {"hc_tx_mcast_pkts", offsetof(struct mana_ethtool_hc_stats, 78 hc_tx_mcast_pkts)}, 79 {"hc_tx_mcast_bytes", offsetof(struct mana_ethtool_hc_stats, 80 hc_tx_mcast_bytes)}, 81 }; 82 83 static const struct mana_stats_desc mana_phy_stats[] = { 84 { "hc_rx_pkt_drop_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_drop_phy) }, 85 { "hc_tx_pkt_drop_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_drop_phy) }, 86 { "hc_tc0_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc0_phy) }, 87 { "hc_tc0_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc0_phy) }, 88 { "hc_tc0_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc0_phy) }, 89 { "hc_tc0_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc0_phy) }, 90 { "hc_tc1_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc1_phy) }, 91 { "hc_tc1_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc1_phy) }, 92 { "hc_tc1_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc1_phy) }, 93 { "hc_tc1_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc1_phy) }, 94 { "hc_tc2_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc2_phy) }, 95 { "hc_tc2_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc2_phy) }, 96 { "hc_tc2_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc2_phy) }, 97 { "hc_tc2_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc2_phy) }, 98 { "hc_tc3_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc3_phy) }, 99 { "hc_tc3_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc3_phy) }, 100 { "hc_tc3_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc3_phy) }, 101 { "hc_tc3_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc3_phy) }, 102 { "hc_tc4_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc4_phy) }, 103 { "hc_tc4_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc4_phy) }, 104 { "hc_tc4_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc4_phy) }, 105 { "hc_tc4_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc4_phy) }, 106 { "hc_tc5_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc5_phy) }, 107 { "hc_tc5_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc5_phy) }, 108 { "hc_tc5_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc5_phy) }, 109 { "hc_tc5_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc5_phy) }, 110 { "hc_tc6_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc6_phy) }, 111 { "hc_tc6_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc6_phy) }, 112 { "hc_tc6_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc6_phy) }, 113 { "hc_tc6_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc6_phy) }, 114 { "hc_tc7_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc7_phy) }, 115 { "hc_tc7_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc7_phy) }, 116 { "hc_tc7_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc7_phy) }, 117 { "hc_tc7_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc7_phy) }, 118 { "hc_tc0_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc0_phy) }, 119 { "hc_tc0_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc0_phy) }, 120 { "hc_tc1_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc1_phy) }, 121 { "hc_tc1_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc1_phy) }, 122 { "hc_tc2_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc2_phy) }, 123 { "hc_tc2_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc2_phy) }, 124 { "hc_tc3_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc3_phy) }, 125 { "hc_tc3_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc3_phy) }, 126 { "hc_tc4_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc4_phy) }, 127 { "hc_tc4_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc4_phy) }, 128 { "hc_tc5_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc5_phy) }, 129 { "hc_tc5_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc5_phy) }, 130 { "hc_tc6_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc6_phy) }, 131 { "hc_tc6_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc6_phy) }, 132 { "hc_tc7_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc7_phy) }, 133 { "hc_tc7_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc7_phy) }, 134 }; 135 136 static const char mana_priv_flags[MANA_PRIV_FLAG_MAX][ETH_GSTRING_LEN] = { 137 [MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF] = "full-page-rx" 138 }; 139 140 static int mana_get_sset_count(struct net_device *ndev, int stringset) 141 { 142 struct mana_port_context *apc = netdev_priv(ndev); 143 unsigned int num_queues = apc->num_queues; 144 145 switch (stringset) { 146 case ETH_SS_STATS: 147 return ARRAY_SIZE(mana_eth_stats) + 148 ARRAY_SIZE(mana_phy_stats) + 149 ARRAY_SIZE(mana_hc_stats) + 150 num_queues * (MANA_STATS_RX_COUNT + MANA_STATS_TX_COUNT); 151 152 case ETH_SS_PRIV_FLAGS: 153 return MANA_PRIV_FLAG_MAX; 154 155 default: 156 return -EINVAL; 157 } 158 } 159 160 static void mana_get_strings_stats(struct mana_port_context *apc, u8 **data) 161 { 162 unsigned int num_queues = apc->num_queues; 163 int i, j; 164 165 for (i = 0; i < ARRAY_SIZE(mana_eth_stats); i++) 166 ethtool_puts(data, mana_eth_stats[i].name); 167 168 for (i = 0; i < ARRAY_SIZE(mana_hc_stats); i++) 169 ethtool_puts(data, mana_hc_stats[i].name); 170 171 for (i = 0; i < ARRAY_SIZE(mana_phy_stats); i++) 172 ethtool_puts(data, mana_phy_stats[i].name); 173 174 for (i = 0; i < num_queues; i++) { 175 ethtool_sprintf(data, "rx_%d_packets", i); 176 ethtool_sprintf(data, "rx_%d_bytes", i); 177 ethtool_sprintf(data, "rx_%d_xdp_drop", i); 178 ethtool_sprintf(data, "rx_%d_xdp_tx", i); 179 ethtool_sprintf(data, "rx_%d_xdp_redirect", i); 180 ethtool_sprintf(data, "rx_%d_pkt_len0_err", i); 181 for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++) 182 ethtool_sprintf(data, 183 "rx_%d_coalesced_cqe_%d", 184 i, 185 j + 2); 186 } 187 188 for (i = 0; i < num_queues; i++) { 189 ethtool_sprintf(data, "tx_%d_packets", i); 190 ethtool_sprintf(data, "tx_%d_bytes", i); 191 ethtool_sprintf(data, "tx_%d_xdp_xmit", i); 192 ethtool_sprintf(data, "tx_%d_tso_packets", i); 193 ethtool_sprintf(data, "tx_%d_tso_bytes", i); 194 ethtool_sprintf(data, "tx_%d_tso_inner_packets", i); 195 ethtool_sprintf(data, "tx_%d_tso_inner_bytes", i); 196 ethtool_sprintf(data, "tx_%d_long_pkt_fmt", i); 197 ethtool_sprintf(data, "tx_%d_short_pkt_fmt", i); 198 ethtool_sprintf(data, "tx_%d_csum_partial", i); 199 ethtool_sprintf(data, "tx_%d_mana_map_err", i); 200 } 201 } 202 203 static void mana_get_strings_priv_flags(u8 **data) 204 { 205 int i; 206 207 for (i = 0; i < MANA_PRIV_FLAG_MAX; i++) 208 ethtool_puts(data, mana_priv_flags[i]); 209 } 210 211 static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data) 212 { 213 struct mana_port_context *apc = netdev_priv(ndev); 214 215 switch (stringset) { 216 case ETH_SS_STATS: 217 mana_get_strings_stats(apc, &data); 218 break; 219 case ETH_SS_PRIV_FLAGS: 220 mana_get_strings_priv_flags(&data); 221 break; 222 default: 223 break; 224 } 225 } 226 227 static void mana_get_ethtool_stats(struct net_device *ndev, 228 struct ethtool_stats *e_stats, u64 *data) 229 { 230 struct mana_port_context *apc = netdev_priv(ndev); 231 unsigned int num_queues = apc->num_queues; 232 void *eth_stats = &apc->eth_stats; 233 void *hc_stats = &apc->ac->hc_stats; 234 void *phy_stats = &apc->phy_stats; 235 struct mana_stats_rx *rx_stats; 236 struct mana_stats_tx *tx_stats; 237 unsigned int start; 238 u64 packets, bytes; 239 u64 xdp_redirect; 240 u64 xdp_xmit; 241 u64 xdp_drop; 242 u64 xdp_tx; 243 u64 pkt_len0_err; 244 u64 coalesced_cqe[MANA_CQE_COAL_PKTS_8 - 1]; 245 u64 tso_packets; 246 u64 tso_bytes; 247 u64 tso_inner_packets; 248 u64 tso_inner_bytes; 249 u64 long_pkt_fmt; 250 u64 short_pkt_fmt; 251 u64 csum_partial; 252 u64 mana_map_err; 253 int q, i = 0, j; 254 255 if (!apc->port_is_up) 256 return; 257 258 /* We call this mana function to get the phy stats from GDMA and includes 259 * aggregate tx/rx drop counters, Per-TC(Traffic Channel) tx/rx and pause 260 * counters. 261 */ 262 mana_query_phy_stats(apc); 263 264 for (q = 0; q < ARRAY_SIZE(mana_eth_stats); q++) 265 data[i++] = *(u64 *)(eth_stats + mana_eth_stats[q].offset); 266 267 for (q = 0; q < ARRAY_SIZE(mana_hc_stats); q++) 268 data[i++] = *(u64 *)(hc_stats + mana_hc_stats[q].offset); 269 270 for (q = 0; q < ARRAY_SIZE(mana_phy_stats); q++) 271 data[i++] = *(u64 *)(phy_stats + mana_phy_stats[q].offset); 272 273 for (q = 0; q < num_queues; q++) { 274 rx_stats = &apc->rxqs[q]->stats; 275 276 do { 277 start = u64_stats_fetch_begin(&rx_stats->syncp); 278 packets = rx_stats->packets; 279 bytes = rx_stats->bytes; 280 xdp_drop = rx_stats->xdp_drop; 281 xdp_tx = rx_stats->xdp_tx; 282 xdp_redirect = rx_stats->xdp_redirect; 283 pkt_len0_err = rx_stats->pkt_len0_err; 284 for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++) 285 coalesced_cqe[j] = rx_stats->coalesced_cqe[j]; 286 } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); 287 288 data[i++] = packets; 289 data[i++] = bytes; 290 data[i++] = xdp_drop; 291 data[i++] = xdp_tx; 292 data[i++] = xdp_redirect; 293 data[i++] = pkt_len0_err; 294 for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++) 295 data[i++] = coalesced_cqe[j]; 296 } 297 298 for (q = 0; q < num_queues; q++) { 299 tx_stats = &apc->tx_qp[q]->txq.stats; 300 301 do { 302 start = u64_stats_fetch_begin(&tx_stats->syncp); 303 packets = tx_stats->packets; 304 bytes = tx_stats->bytes; 305 xdp_xmit = tx_stats->xdp_xmit; 306 tso_packets = tx_stats->tso_packets; 307 tso_bytes = tx_stats->tso_bytes; 308 tso_inner_packets = tx_stats->tso_inner_packets; 309 tso_inner_bytes = tx_stats->tso_inner_bytes; 310 long_pkt_fmt = tx_stats->long_pkt_fmt; 311 short_pkt_fmt = tx_stats->short_pkt_fmt; 312 csum_partial = tx_stats->csum_partial; 313 mana_map_err = tx_stats->mana_map_err; 314 } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); 315 316 data[i++] = packets; 317 data[i++] = bytes; 318 data[i++] = xdp_xmit; 319 data[i++] = tso_packets; 320 data[i++] = tso_bytes; 321 data[i++] = tso_inner_packets; 322 data[i++] = tso_inner_bytes; 323 data[i++] = long_pkt_fmt; 324 data[i++] = short_pkt_fmt; 325 data[i++] = csum_partial; 326 data[i++] = mana_map_err; 327 } 328 } 329 330 static u32 mana_get_rx_ring_count(struct net_device *ndev) 331 { 332 struct mana_port_context *apc = netdev_priv(ndev); 333 334 return apc->num_queues; 335 } 336 337 static u32 mana_get_rxfh_key_size(struct net_device *ndev) 338 { 339 return MANA_HASH_KEY_SIZE; 340 } 341 342 static u32 mana_rss_indir_size(struct net_device *ndev) 343 { 344 struct mana_port_context *apc = netdev_priv(ndev); 345 346 return apc->indir_table_sz; 347 } 348 349 static int mana_get_rxfh(struct net_device *ndev, 350 struct ethtool_rxfh_param *rxfh) 351 { 352 struct mana_port_context *apc = netdev_priv(ndev); 353 int i; 354 355 rxfh->hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */ 356 357 if (rxfh->indir) { 358 for (i = 0; i < apc->indir_table_sz; i++) 359 rxfh->indir[i] = apc->indir_table[i]; 360 } 361 362 if (rxfh->key) 363 memcpy(rxfh->key, apc->hashkey, MANA_HASH_KEY_SIZE); 364 365 return 0; 366 } 367 368 static int mana_set_rxfh(struct net_device *ndev, 369 struct ethtool_rxfh_param *rxfh, 370 struct netlink_ext_ack *extack) 371 { 372 struct mana_port_context *apc = netdev_priv(ndev); 373 bool update_hash = false, update_table = false; 374 u8 save_key[MANA_HASH_KEY_SIZE]; 375 u32 *save_table; 376 int i, err; 377 378 if (!apc->port_is_up) 379 return -EOPNOTSUPP; 380 381 if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && 382 rxfh->hfunc != ETH_RSS_HASH_TOP) 383 return -EOPNOTSUPP; 384 385 save_table = kcalloc(apc->indir_table_sz, sizeof(u32), GFP_KERNEL); 386 if (!save_table) 387 return -ENOMEM; 388 389 if (rxfh->indir) { 390 for (i = 0; i < apc->indir_table_sz; i++) 391 if (rxfh->indir[i] >= apc->num_queues) { 392 err = -EINVAL; 393 goto cleanup; 394 } 395 396 update_table = true; 397 for (i = 0; i < apc->indir_table_sz; i++) { 398 save_table[i] = apc->indir_table[i]; 399 apc->indir_table[i] = rxfh->indir[i]; 400 } 401 } 402 403 if (rxfh->key) { 404 update_hash = true; 405 memcpy(save_key, apc->hashkey, MANA_HASH_KEY_SIZE); 406 memcpy(apc->hashkey, rxfh->key, MANA_HASH_KEY_SIZE); 407 } 408 409 err = mana_config_rss(apc, TRI_STATE_TRUE, update_hash, update_table); 410 411 if (err) { /* recover to original values */ 412 if (update_table) { 413 for (i = 0; i < apc->indir_table_sz; i++) 414 apc->indir_table[i] = save_table[i]; 415 } 416 417 if (update_hash) 418 memcpy(apc->hashkey, save_key, MANA_HASH_KEY_SIZE); 419 420 mana_config_rss(apc, TRI_STATE_TRUE, update_hash, update_table); 421 } 422 423 cleanup: 424 kfree(save_table); 425 426 return err; 427 } 428 429 static void mana_get_channels(struct net_device *ndev, 430 struct ethtool_channels *channel) 431 { 432 struct mana_port_context *apc = netdev_priv(ndev); 433 434 channel->max_combined = apc->max_queues; 435 channel->combined_count = apc->num_queues; 436 } 437 438 #define MANA_RX_CQE_NSEC_DEF 2048 439 static int mana_get_coalesce(struct net_device *ndev, 440 struct ethtool_coalesce *ec, 441 struct kernel_ethtool_coalesce *kernel_coal, 442 struct netlink_ext_ack *extack) 443 { 444 struct mana_port_context *apc = netdev_priv(ndev); 445 446 kernel_coal->rx_cqe_frames = 447 apc->cqe8_coalescing_enable ? MANA_CQE_COAL_PKTS_8 : 448 apc->cqe_coalescing_enable ? MANA_RXCOMP_OOB_NUM_PPI : 1; 449 450 kernel_coal->rx_cqe_nsecs = apc->cqe_coalescing_timeout_ns; 451 452 /* Return the default timeout value for old FW not providing 453 * this value. 454 */ 455 if (apc->port_is_up && apc->cqe_coalescing_enable && 456 !kernel_coal->rx_cqe_nsecs) 457 kernel_coal->rx_cqe_nsecs = MANA_RX_CQE_NSEC_DEF; 458 459 ec->rx_coalesce_usecs = apc->intr_modr_rx_usec; 460 ec->rx_max_coalesced_frames = apc->intr_modr_rx_comp; 461 462 ec->tx_coalesce_usecs = apc->intr_modr_tx_usec; 463 ec->tx_max_coalesced_frames = apc->intr_modr_tx_comp; 464 465 ec->use_adaptive_rx_coalesce = apc->rx_dim_enabled; 466 ec->use_adaptive_tx_coalesce = apc->tx_dim_enabled; 467 468 return 0; 469 } 470 471 static int mana_set_coalesce(struct net_device *ndev, 472 struct ethtool_coalesce *ec, 473 struct kernel_ethtool_coalesce *kernel_coal, 474 struct netlink_ext_ack *extack) 475 { 476 struct mana_port_context *apc = netdev_priv(ndev); 477 struct { 478 u16 intr_modr_rx_usec; 479 u16 intr_modr_rx_comp; 480 u16 intr_modr_tx_usec; 481 u16 intr_modr_tx_comp; 482 u8 cqe_coalescing_enable; 483 u8 cqe8_coalescing_enable; 484 bool rx_dim_enabled; 485 bool tx_dim_enabled; 486 } saved; 487 bool modr_changed = false; 488 bool dim_changed = false; 489 struct gdma_context *gc; 490 u32 max_cqe_frames; 491 int err; 492 493 gc = apc->ac->gdma_dev->gdma_context; 494 max_cqe_frames = gc->cqe8_coalescing_sup ? MANA_CQE_COAL_PKTS_8 : 495 MANA_RXCOMP_OOB_NUM_PPI; 496 497 /* Both static and dynamic interrupt moderation (DIM) rely on the 498 * same HW capability advertised by the PF. 499 */ 500 if ((ec->use_adaptive_rx_coalesce || ec->use_adaptive_tx_coalesce || 501 ec->rx_coalesce_usecs || ec->tx_coalesce_usecs || 502 ec->rx_max_coalesced_frames || ec->tx_max_coalesced_frames) && 503 !(gc->pf_cap_flags1 & GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION)) { 504 NL_SET_ERR_MSG(extack, 505 "Interrupt Moderation is not supported by HW"); 506 return -EOPNOTSUPP; 507 } 508 509 if (kernel_coal->rx_cqe_frames != 1 && 510 kernel_coal->rx_cqe_frames != MANA_RXCOMP_OOB_NUM_PPI && 511 kernel_coal->rx_cqe_frames != max_cqe_frames) { 512 NL_SET_ERR_MSG_FMT(extack, 513 "rx-frames must be 1 or %u%s, got %u", 514 MANA_RXCOMP_OOB_NUM_PPI, 515 gc->cqe8_coalescing_sup ? " or 8" : "", 516 kernel_coal->rx_cqe_frames); 517 return -EINVAL; 518 } 519 520 if (ec->rx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX || 521 ec->tx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX) { 522 NL_SET_ERR_MSG_FMT(extack, 523 "coalesce usecs must be <= %lu", 524 MANA_INTR_MODR_USEC_MAX); 525 return -EINVAL; 526 } 527 528 if (ec->rx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX || 529 ec->tx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX) { 530 NL_SET_ERR_MSG_FMT(extack, 531 "coalesce frames must be <= %lu", 532 MANA_INTR_MODR_COMP_MAX); 533 return -EINVAL; 534 } 535 536 if (ec->rx_coalesce_usecs != apc->intr_modr_rx_usec || 537 ec->rx_max_coalesced_frames != apc->intr_modr_rx_comp || 538 ec->tx_coalesce_usecs != apc->intr_modr_tx_usec || 539 ec->tx_max_coalesced_frames != apc->intr_modr_tx_comp) 540 modr_changed = true; 541 542 saved.intr_modr_rx_usec = apc->intr_modr_rx_usec; 543 saved.intr_modr_rx_comp = apc->intr_modr_rx_comp; 544 saved.intr_modr_tx_usec = apc->intr_modr_tx_usec; 545 saved.intr_modr_tx_comp = apc->intr_modr_tx_comp; 546 547 apc->intr_modr_rx_usec = ec->rx_coalesce_usecs; 548 apc->intr_modr_rx_comp = ec->rx_max_coalesced_frames; 549 apc->intr_modr_tx_usec = ec->tx_coalesce_usecs; 550 apc->intr_modr_tx_comp = ec->tx_max_coalesced_frames; 551 552 if (!!ec->use_adaptive_rx_coalesce != apc->rx_dim_enabled || 553 !!ec->use_adaptive_tx_coalesce != apc->tx_dim_enabled) 554 dim_changed = true; 555 556 saved.rx_dim_enabled = apc->rx_dim_enabled; 557 saved.tx_dim_enabled = apc->tx_dim_enabled; 558 559 saved.cqe_coalescing_enable = apc->cqe_coalescing_enable; 560 saved.cqe8_coalescing_enable = apc->cqe8_coalescing_enable; 561 apc->cqe_coalescing_enable = 562 kernel_coal->rx_cqe_frames >= MANA_RXCOMP_OOB_NUM_PPI; 563 apc->cqe8_coalescing_enable = 564 kernel_coal->rx_cqe_frames == MANA_CQE_COAL_PKTS_8; 565 566 if (!apc->port_is_up) { 567 WRITE_ONCE(apc->rx_dim_enabled, !!ec->use_adaptive_rx_coalesce); 568 WRITE_ONCE(apc->tx_dim_enabled, !!ec->use_adaptive_tx_coalesce); 569 return 0; 570 } 571 572 if (apc->cqe_coalescing_enable != saved.cqe_coalescing_enable || 573 apc->cqe8_coalescing_enable != saved.cqe8_coalescing_enable) { 574 /* CQE coalescing setting is applied via RSS configuration. */ 575 err = mana_config_rss(apc, TRI_STATE_TRUE, false, false); 576 if (err) { 577 netdev_err(ndev, "Change CQE coalescing failed: %d\n", 578 err); 579 apc->cqe_coalescing_enable = 580 saved.cqe_coalescing_enable; 581 apc->cqe8_coalescing_enable = 582 saved.cqe8_coalescing_enable; 583 apc->intr_modr_rx_usec = saved.intr_modr_rx_usec; 584 apc->intr_modr_rx_comp = saved.intr_modr_rx_comp; 585 apc->intr_modr_tx_usec = saved.intr_modr_tx_usec; 586 apc->intr_modr_tx_comp = saved.intr_modr_tx_comp; 587 return err; 588 } 589 } 590 591 if (modr_changed || dim_changed) { 592 bool new_rx_dim = !!ec->use_adaptive_rx_coalesce; 593 bool new_tx_dim = !!ec->use_adaptive_tx_coalesce; 594 bool disable_rx_dim = saved.rx_dim_enabled && !new_rx_dim; 595 bool disable_tx_dim = saved.tx_dim_enabled && !new_tx_dim; 596 bool enable_rx_dim = !saved.rx_dim_enabled && new_rx_dim; 597 bool enable_tx_dim = !saved.tx_dim_enabled && new_tx_dim; 598 int q; 599 600 /* On disable: clear the per-port flag first and 601 * synchronize_net() so any in-flight NAPI poll observes 602 * the new value and will not schedule further DIM work; 603 * then drain pending work and restore the static 604 * moderation values. 605 */ 606 if (disable_rx_dim) 607 WRITE_ONCE(apc->rx_dim_enabled, false); 608 if (disable_tx_dim) 609 WRITE_ONCE(apc->tx_dim_enabled, false); 610 if (disable_rx_dim || disable_tx_dim) 611 synchronize_net(); 612 613 for (q = 0; q < apc->num_queues; q++) { 614 struct mana_cq *rx_cq = &apc->rxqs[q]->rx_cq; 615 struct mana_cq *tx_cq = &apc->tx_qp[q]->tx_cq; 616 617 if (disable_rx_dim) 618 mana_dim_change(rx_cq, false); 619 else if (enable_rx_dim) 620 mana_dim_change(rx_cq, true); 621 else if (!new_rx_dim && modr_changed) 622 mana_gd_ring_dim(rx_cq->gdma_cq, 623 apc->intr_modr_rx_usec, true, 624 apc->intr_modr_rx_comp, true); 625 626 if (disable_tx_dim) 627 mana_dim_change(tx_cq, false); 628 else if (enable_tx_dim) 629 mana_dim_change(tx_cq, true); 630 else if (!new_tx_dim && modr_changed) 631 mana_gd_ring_dim(tx_cq->gdma_cq, 632 apc->intr_modr_tx_usec, true, 633 apc->intr_modr_tx_comp, true); 634 } 635 636 /* Publish the enable flag with release semantics so a 637 * concurrent NAPI poll that observes it set also sees the DIM 638 * (re)init done by mana_dim_change() above. 639 */ 640 if (enable_rx_dim) 641 /* pairs with smp_load_acquire() in mana_update_rx_dim() */ 642 smp_store_release(&apc->rx_dim_enabled, true); 643 if (enable_tx_dim) 644 /* pairs with smp_load_acquire() in mana_update_tx_dim() */ 645 smp_store_release(&apc->tx_dim_enabled, true); 646 } 647 648 return 0; 649 } 650 651 /* mana_set_channels - change the number of queues on a port 652 * 653 * Returns -EBUSY if RDMA holds the vport with EQs sized to the 654 * current num_queues. 655 */ 656 static int mana_set_channels(struct net_device *ndev, 657 struct ethtool_channels *channels) 658 { 659 struct mana_port_context *apc = netdev_priv(ndev); 660 unsigned int new_count = channels->combined_count; 661 unsigned int old_count = apc->num_queues; 662 int err; 663 664 /* Set channel_changing to block RDMA from grabbing the vport 665 * during the detach/attach window. mana_cfg_vport() checks 666 * this flag under vport_mutex and returns -EBUSY if set. 667 */ 668 mutex_lock(&apc->vport_mutex); 669 if (!apc->port_is_up && apc->vport_use_count) { 670 mutex_unlock(&apc->vport_mutex); 671 return -EBUSY; 672 } 673 apc->channel_changing = true; 674 mutex_unlock(&apc->vport_mutex); 675 676 err = mana_pre_alloc_rxbufs(apc, ndev->mtu, new_count); 677 if (err) { 678 netdev_err(ndev, "Insufficient memory for new allocations"); 679 goto clear_flag; 680 } 681 682 err = mana_detach(ndev, false); 683 if (err) { 684 netdev_err(ndev, "mana_detach failed: %d\n", err); 685 goto out; 686 } 687 688 apc->num_queues = new_count; 689 err = mana_attach(ndev); 690 if (err) { 691 apc->num_queues = old_count; 692 netdev_err(ndev, "mana_attach failed: %d\n", err); 693 } 694 695 out: 696 mana_pre_dealloc_rxbufs(apc); 697 clear_flag: 698 mutex_lock(&apc->vport_mutex); 699 apc->channel_changing = false; 700 mutex_unlock(&apc->vport_mutex); 701 return err; 702 } 703 704 static void mana_get_ringparam(struct net_device *ndev, 705 struct ethtool_ringparam *ring, 706 struct kernel_ethtool_ringparam *kernel_ring, 707 struct netlink_ext_ack *extack) 708 { 709 struct mana_port_context *apc = netdev_priv(ndev); 710 711 ring->rx_pending = apc->rx_queue_size; 712 ring->tx_pending = apc->tx_queue_size; 713 ring->rx_max_pending = MAX_RX_BUFFERS_PER_QUEUE; 714 ring->tx_max_pending = MAX_TX_BUFFERS_PER_QUEUE; 715 } 716 717 static int mana_set_ringparam(struct net_device *ndev, 718 struct ethtool_ringparam *ring, 719 struct kernel_ethtool_ringparam *kernel_ring, 720 struct netlink_ext_ack *extack) 721 { 722 struct mana_port_context *apc = netdev_priv(ndev); 723 u32 new_tx, new_rx; 724 u32 old_tx, old_rx; 725 int err; 726 727 old_tx = apc->tx_queue_size; 728 old_rx = apc->rx_queue_size; 729 730 if (ring->tx_pending < MIN_TX_BUFFERS_PER_QUEUE) { 731 NL_SET_ERR_MSG_FMT(extack, "tx:%d less than the min:%d", ring->tx_pending, 732 MIN_TX_BUFFERS_PER_QUEUE); 733 return -EINVAL; 734 } 735 736 if (ring->rx_pending < MIN_RX_BUFFERS_PER_QUEUE) { 737 NL_SET_ERR_MSG_FMT(extack, "rx:%d less than the min:%d", ring->rx_pending, 738 MIN_RX_BUFFERS_PER_QUEUE); 739 return -EINVAL; 740 } 741 742 new_rx = roundup_pow_of_two(ring->rx_pending); 743 new_tx = roundup_pow_of_two(ring->tx_pending); 744 netdev_info(ndev, "Using nearest power of 2 values for Txq:%d Rxq:%d\n", 745 new_tx, new_rx); 746 747 /* pre-allocating new buffers to prevent failures in mana_attach() later */ 748 apc->rx_queue_size = new_rx; 749 err = mana_pre_alloc_rxbufs(apc, ndev->mtu, apc->num_queues); 750 apc->rx_queue_size = old_rx; 751 if (err) { 752 netdev_err(ndev, "Insufficient memory for new allocations\n"); 753 return err; 754 } 755 756 err = mana_detach(ndev, false); 757 if (err) { 758 netdev_err(ndev, "mana_detach failed: %d\n", err); 759 goto out; 760 } 761 762 apc->tx_queue_size = new_tx; 763 apc->rx_queue_size = new_rx; 764 765 err = mana_attach(ndev); 766 if (err) { 767 netdev_err(ndev, "mana_attach failed: %d\n", err); 768 apc->tx_queue_size = old_tx; 769 apc->rx_queue_size = old_rx; 770 } 771 out: 772 mana_pre_dealloc_rxbufs(apc); 773 return err; 774 } 775 776 static int mana_get_link_ksettings(struct net_device *ndev, 777 struct ethtool_link_ksettings *cmd) 778 { 779 struct mana_port_context *apc = netdev_priv(ndev); 780 int err; 781 782 err = mana_query_link_cfg(apc); 783 cmd->base.speed = (err) ? SPEED_UNKNOWN : apc->max_speed; 784 785 cmd->base.duplex = DUPLEX_FULL; 786 cmd->base.port = PORT_OTHER; 787 788 return 0; 789 } 790 791 static u32 mana_get_priv_flags(struct net_device *ndev) 792 { 793 struct mana_port_context *apc = netdev_priv(ndev); 794 795 return apc->priv_flags; 796 } 797 798 static int mana_set_priv_flags(struct net_device *ndev, u32 priv_flags) 799 { 800 struct mana_port_context *apc = netdev_priv(ndev); 801 u32 changed = apc->priv_flags ^ priv_flags; 802 u32 old_priv_flags = apc->priv_flags; 803 int err = 0; 804 805 if (!changed) 806 return 0; 807 808 /* Reject unknown bits */ 809 if (priv_flags & ~GENMASK(MANA_PRIV_FLAG_MAX - 1, 0)) 810 return -EINVAL; 811 812 apc->priv_flags = priv_flags; 813 814 if (changed & BIT(MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF)) { 815 if (!apc->port_is_up) 816 return 0; 817 818 /* If XDP is attached or MTU is jumbo, single-buffer-per-page 819 * is already forced regardless of this flag. Skip the 820 * expensive detach/attach cycle since nothing changes. 821 */ 822 if (ndev->mtu + MANA_RXBUF_PAD > PAGE_SIZE / 2 || 823 mana_xdp_get(apc)) 824 return 0; 825 826 /* Block RDMA from grabbing the vport during detach/attach */ 827 mutex_lock(&apc->vport_mutex); 828 apc->channel_changing = true; 829 mutex_unlock(&apc->vport_mutex); 830 831 err = mana_pre_alloc_rxbufs(apc, ndev->mtu, apc->num_queues); 832 if (err) { 833 netdev_err(ndev, 834 "Insufficient memory for new allocations\n"); 835 apc->priv_flags = old_priv_flags; 836 goto clear_flag; 837 } 838 839 err = mana_detach(ndev, false); 840 if (err) { 841 netdev_err(ndev, "mana_detach failed: %d\n", err); 842 apc->priv_flags = old_priv_flags; 843 goto out; 844 } 845 846 err = mana_attach(ndev); 847 if (err) { 848 netdev_err(ndev, "mana_attach failed: %d\n", err); 849 apc->priv_flags = old_priv_flags; 850 } 851 } 852 853 out: 854 mana_pre_dealloc_rxbufs(apc); 855 clear_flag: 856 mutex_lock(&apc->vport_mutex); 857 apc->channel_changing = false; 858 mutex_unlock(&apc->vport_mutex); 859 860 return err; 861 } 862 863 const struct ethtool_ops mana_ethtool_ops = { 864 .supported_coalesce_params = ETHTOOL_COALESCE_RX_CQE_FRAMES | 865 ETHTOOL_COALESCE_RX_USECS | 866 ETHTOOL_COALESCE_RX_MAX_FRAMES | 867 ETHTOOL_COALESCE_TX_USECS | 868 ETHTOOL_COALESCE_TX_MAX_FRAMES | 869 ETHTOOL_COALESCE_USE_ADAPTIVE_RX | 870 ETHTOOL_COALESCE_USE_ADAPTIVE_TX, 871 .op_needs_rtnl = ETHTOOL_OP_NEEDS_RTNL_SCHANNELS | 872 ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM | 873 ETHTOOL_OP_NEEDS_RTNL_SPFLAGS | 874 ETHTOOL_OP_NEEDS_RTNL_GLINK, 875 .get_ethtool_stats = mana_get_ethtool_stats, 876 .get_sset_count = mana_get_sset_count, 877 .get_strings = mana_get_strings, 878 .get_rx_ring_count = mana_get_rx_ring_count, 879 .get_rxfh_key_size = mana_get_rxfh_key_size, 880 .get_rxfh_indir_size = mana_rss_indir_size, 881 .get_rxfh = mana_get_rxfh, 882 .set_rxfh = mana_set_rxfh, 883 .get_channels = mana_get_channels, 884 .set_channels = mana_set_channels, 885 .get_coalesce = mana_get_coalesce, 886 .set_coalesce = mana_set_coalesce, 887 .get_ringparam = mana_get_ringparam, 888 .set_ringparam = mana_set_ringparam, 889 .get_link_ksettings = mana_get_link_ksettings, 890 .get_link = ethtool_op_get_link, 891 .get_priv_flags = mana_get_priv_flags, 892 .set_priv_flags = mana_set_priv_flags, 893 }; 894