1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */ 3 4 #include <linux/ethtool.h> 5 #include <linux/netdevice.h> 6 #include <linux/pci.h> 7 #include <net/ipv6.h> 8 9 #include "fbnic.h" 10 #include "fbnic_netdev.h" 11 #include "fbnic_tlv.h" 12 13 struct fbnic_stat { 14 u8 string[ETH_GSTRING_LEN]; 15 unsigned int size; 16 unsigned int offset; 17 }; 18 19 #define FBNIC_STAT_FIELDS(type, name, stat) { \ 20 .string = name, \ 21 .size = sizeof_field(struct type, stat), \ 22 .offset = offsetof(struct type, stat), \ 23 } 24 25 /* Hardware statistics not captured in rtnl_link_stats */ 26 #define FBNIC_HW_STAT(name, stat) \ 27 FBNIC_STAT_FIELDS(fbnic_hw_stats, name, stat) 28 29 static const struct fbnic_stat fbnic_gstrings_hw_stats[] = { 30 /* RPC */ 31 FBNIC_HW_STAT("rpc_unkn_etype", rpc.unkn_etype), 32 FBNIC_HW_STAT("rpc_unkn_ext_hdr", rpc.unkn_ext_hdr), 33 FBNIC_HW_STAT("rpc_ipv4_frag", rpc.ipv4_frag), 34 FBNIC_HW_STAT("rpc_ipv6_frag", rpc.ipv6_frag), 35 FBNIC_HW_STAT("rpc_ipv4_esp", rpc.ipv4_esp), 36 FBNIC_HW_STAT("rpc_ipv6_esp", rpc.ipv6_esp), 37 FBNIC_HW_STAT("rpc_tcp_opt_err", rpc.tcp_opt_err), 38 FBNIC_HW_STAT("rpc_out_of_hdr_err", rpc.out_of_hdr_err), 39 }; 40 41 #define FBNIC_HW_FIXED_STATS_LEN ARRAY_SIZE(fbnic_gstrings_hw_stats) 42 #define FBNIC_HW_STATS_LEN FBNIC_HW_FIXED_STATS_LEN 43 44 static void 45 fbnic_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) 46 { 47 struct fbnic_net *fbn = netdev_priv(netdev); 48 struct fbnic_dev *fbd = fbn->fbd; 49 50 fbnic_get_fw_ver_commit_str(fbd, drvinfo->fw_version, 51 sizeof(drvinfo->fw_version)); 52 } 53 54 static int fbnic_get_regs_len(struct net_device *netdev) 55 { 56 struct fbnic_net *fbn = netdev_priv(netdev); 57 58 return fbnic_csr_regs_len(fbn->fbd) * sizeof(u32); 59 } 60 61 static void fbnic_get_regs(struct net_device *netdev, 62 struct ethtool_regs *regs, void *data) 63 { 64 struct fbnic_net *fbn = netdev_priv(netdev); 65 66 fbnic_csr_get_regs(fbn->fbd, data, ®s->version); 67 } 68 69 static struct fbnic_net *fbnic_clone_create(struct fbnic_net *orig) 70 { 71 struct fbnic_net *clone; 72 73 clone = kmemdup(orig, sizeof(*orig), GFP_KERNEL); 74 if (!clone) 75 return NULL; 76 77 memset(clone->tx, 0, sizeof(clone->tx)); 78 memset(clone->rx, 0, sizeof(clone->rx)); 79 memset(clone->napi, 0, sizeof(clone->napi)); 80 return clone; 81 } 82 83 static void fbnic_clone_swap_cfg(struct fbnic_net *orig, 84 struct fbnic_net *clone) 85 { 86 swap(clone->rcq_size, orig->rcq_size); 87 swap(clone->hpq_size, orig->hpq_size); 88 swap(clone->ppq_size, orig->ppq_size); 89 swap(clone->txq_size, orig->txq_size); 90 swap(clone->num_rx_queues, orig->num_rx_queues); 91 swap(clone->num_tx_queues, orig->num_tx_queues); 92 swap(clone->num_napi, orig->num_napi); 93 } 94 95 static void fbnic_aggregate_vector_counters(struct fbnic_net *fbn, 96 struct fbnic_napi_vector *nv) 97 { 98 int i, j; 99 100 for (i = 0; i < nv->txt_count; i++) { 101 fbnic_aggregate_ring_tx_counters(fbn, &nv->qt[i].sub0); 102 fbnic_aggregate_ring_tx_counters(fbn, &nv->qt[i].sub1); 103 fbnic_aggregate_ring_tx_counters(fbn, &nv->qt[i].cmpl); 104 } 105 106 for (j = 0; j < nv->rxt_count; j++, i++) { 107 fbnic_aggregate_ring_rx_counters(fbn, &nv->qt[i].sub0); 108 fbnic_aggregate_ring_rx_counters(fbn, &nv->qt[i].sub1); 109 fbnic_aggregate_ring_rx_counters(fbn, &nv->qt[i].cmpl); 110 } 111 } 112 113 static void fbnic_clone_swap(struct fbnic_net *orig, 114 struct fbnic_net *clone) 115 { 116 struct fbnic_dev *fbd = orig->fbd; 117 unsigned int i; 118 119 for (i = 0; i < max(clone->num_napi, orig->num_napi); i++) 120 fbnic_synchronize_irq(fbd, FBNIC_NON_NAPI_VECTORS + i); 121 for (i = 0; i < orig->num_napi; i++) 122 fbnic_aggregate_vector_counters(orig, orig->napi[i]); 123 124 fbnic_clone_swap_cfg(orig, clone); 125 126 for (i = 0; i < ARRAY_SIZE(orig->napi); i++) 127 swap(clone->napi[i], orig->napi[i]); 128 for (i = 0; i < ARRAY_SIZE(orig->tx); i++) 129 swap(clone->tx[i], orig->tx[i]); 130 for (i = 0; i < ARRAY_SIZE(orig->rx); i++) 131 swap(clone->rx[i], orig->rx[i]); 132 } 133 134 static void fbnic_clone_free(struct fbnic_net *clone) 135 { 136 kfree(clone); 137 } 138 139 static int fbnic_get_coalesce(struct net_device *netdev, 140 struct ethtool_coalesce *ec, 141 struct kernel_ethtool_coalesce *kernel_coal, 142 struct netlink_ext_ack *extack) 143 { 144 struct fbnic_net *fbn = netdev_priv(netdev); 145 146 ec->tx_coalesce_usecs = fbn->tx_usecs; 147 ec->rx_coalesce_usecs = fbn->rx_usecs; 148 ec->rx_max_coalesced_frames = fbn->rx_max_frames; 149 150 return 0; 151 } 152 153 static int fbnic_set_coalesce(struct net_device *netdev, 154 struct ethtool_coalesce *ec, 155 struct kernel_ethtool_coalesce *kernel_coal, 156 struct netlink_ext_ack *extack) 157 { 158 struct fbnic_net *fbn = netdev_priv(netdev); 159 160 /* Verify against hardware limits */ 161 if (ec->rx_coalesce_usecs > FIELD_MAX(FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT)) { 162 NL_SET_ERR_MSG_MOD(extack, "rx_usecs is above device max"); 163 return -EINVAL; 164 } 165 if (ec->tx_coalesce_usecs > FIELD_MAX(FBNIC_INTR_CQ_REARM_TCQ_TIMEOUT)) { 166 NL_SET_ERR_MSG_MOD(extack, "tx_usecs is above device max"); 167 return -EINVAL; 168 } 169 if (ec->rx_max_coalesced_frames > 170 FIELD_MAX(FBNIC_QUEUE_RIM_THRESHOLD_RCD_MASK) / 171 FBNIC_MIN_RXD_PER_FRAME) { 172 NL_SET_ERR_MSG_MOD(extack, "rx_frames is above device max"); 173 return -EINVAL; 174 } 175 176 fbn->tx_usecs = ec->tx_coalesce_usecs; 177 fbn->rx_usecs = ec->rx_coalesce_usecs; 178 fbn->rx_max_frames = ec->rx_max_coalesced_frames; 179 180 if (netif_running(netdev)) { 181 int i; 182 183 for (i = 0; i < fbn->num_napi; i++) { 184 struct fbnic_napi_vector *nv = fbn->napi[i]; 185 186 fbnic_config_txrx_usecs(nv, 0); 187 fbnic_config_rx_frames(nv); 188 } 189 } 190 191 return 0; 192 } 193 194 static void 195 fbnic_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, 196 struct kernel_ethtool_ringparam *kernel_ring, 197 struct netlink_ext_ack *extack) 198 { 199 struct fbnic_net *fbn = netdev_priv(netdev); 200 201 ring->rx_max_pending = FBNIC_QUEUE_SIZE_MAX; 202 ring->rx_mini_max_pending = FBNIC_QUEUE_SIZE_MAX; 203 ring->rx_jumbo_max_pending = FBNIC_QUEUE_SIZE_MAX; 204 ring->tx_max_pending = FBNIC_QUEUE_SIZE_MAX; 205 206 ring->rx_pending = fbn->rcq_size; 207 ring->rx_mini_pending = fbn->hpq_size; 208 ring->rx_jumbo_pending = fbn->ppq_size; 209 ring->tx_pending = fbn->txq_size; 210 } 211 212 static void fbnic_set_rings(struct fbnic_net *fbn, 213 struct ethtool_ringparam *ring) 214 { 215 fbn->rcq_size = ring->rx_pending; 216 fbn->hpq_size = ring->rx_mini_pending; 217 fbn->ppq_size = ring->rx_jumbo_pending; 218 fbn->txq_size = ring->tx_pending; 219 } 220 221 static int 222 fbnic_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, 223 struct kernel_ethtool_ringparam *kernel_ring, 224 struct netlink_ext_ack *extack) 225 226 { 227 struct fbnic_net *fbn = netdev_priv(netdev); 228 struct fbnic_net *clone; 229 int err; 230 231 ring->rx_pending = roundup_pow_of_two(ring->rx_pending); 232 ring->rx_mini_pending = roundup_pow_of_two(ring->rx_mini_pending); 233 ring->rx_jumbo_pending = roundup_pow_of_two(ring->rx_jumbo_pending); 234 ring->tx_pending = roundup_pow_of_two(ring->tx_pending); 235 236 /* These are absolute minimums allowing the device and driver to operate 237 * but not necessarily guarantee reasonable performance. Settings below 238 * Rx queue size of 128 and BDQs smaller than 64 are likely suboptimal 239 * at best. 240 */ 241 if (ring->rx_pending < max(FBNIC_QUEUE_SIZE_MIN, FBNIC_RX_DESC_MIN) || 242 ring->rx_mini_pending < FBNIC_QUEUE_SIZE_MIN || 243 ring->rx_jumbo_pending < FBNIC_QUEUE_SIZE_MIN || 244 ring->tx_pending < max(FBNIC_QUEUE_SIZE_MIN, FBNIC_TX_DESC_MIN)) { 245 NL_SET_ERR_MSG_MOD(extack, "requested ring size too small"); 246 return -EINVAL; 247 } 248 249 if (!netif_running(netdev)) { 250 fbnic_set_rings(fbn, ring); 251 return 0; 252 } 253 254 clone = fbnic_clone_create(fbn); 255 if (!clone) 256 return -ENOMEM; 257 258 fbnic_set_rings(clone, ring); 259 260 err = fbnic_alloc_napi_vectors(clone); 261 if (err) 262 goto err_free_clone; 263 264 err = fbnic_alloc_resources(clone); 265 if (err) 266 goto err_free_napis; 267 268 fbnic_down_noidle(fbn); 269 err = fbnic_wait_all_queues_idle(fbn->fbd, true); 270 if (err) 271 goto err_start_stack; 272 273 err = fbnic_set_netif_queues(clone); 274 if (err) 275 goto err_start_stack; 276 277 /* Nothing can fail past this point */ 278 fbnic_flush(fbn); 279 280 fbnic_clone_swap(fbn, clone); 281 282 fbnic_up(fbn); 283 284 fbnic_free_resources(clone); 285 fbnic_free_napi_vectors(clone); 286 fbnic_clone_free(clone); 287 288 return 0; 289 290 err_start_stack: 291 fbnic_flush(fbn); 292 fbnic_up(fbn); 293 fbnic_free_resources(clone); 294 err_free_napis: 295 fbnic_free_napi_vectors(clone); 296 err_free_clone: 297 fbnic_clone_free(clone); 298 return err; 299 } 300 301 static void fbnic_get_strings(struct net_device *dev, u32 sset, u8 *data) 302 { 303 int i; 304 305 switch (sset) { 306 case ETH_SS_STATS: 307 for (i = 0; i < FBNIC_HW_STATS_LEN; i++) 308 ethtool_puts(&data, fbnic_gstrings_hw_stats[i].string); 309 break; 310 } 311 } 312 313 static void fbnic_get_ethtool_stats(struct net_device *dev, 314 struct ethtool_stats *stats, u64 *data) 315 { 316 struct fbnic_net *fbn = netdev_priv(dev); 317 const struct fbnic_stat *stat; 318 int i; 319 320 fbnic_get_hw_stats(fbn->fbd); 321 322 for (i = 0; i < FBNIC_HW_STATS_LEN; i++) { 323 stat = &fbnic_gstrings_hw_stats[i]; 324 data[i] = *(u64 *)((u8 *)&fbn->fbd->hw_stats + stat->offset); 325 } 326 } 327 328 static int fbnic_get_sset_count(struct net_device *dev, int sset) 329 { 330 switch (sset) { 331 case ETH_SS_STATS: 332 return FBNIC_HW_STATS_LEN; 333 default: 334 return -EOPNOTSUPP; 335 } 336 } 337 338 static int fbnic_get_rss_hash_idx(u32 flow_type) 339 { 340 switch (flow_type & ~(FLOW_EXT | FLOW_MAC_EXT | FLOW_RSS)) { 341 case TCP_V4_FLOW: 342 return FBNIC_TCP4_HASH_OPT; 343 case TCP_V6_FLOW: 344 return FBNIC_TCP6_HASH_OPT; 345 case UDP_V4_FLOW: 346 return FBNIC_UDP4_HASH_OPT; 347 case UDP_V6_FLOW: 348 return FBNIC_UDP6_HASH_OPT; 349 case AH_V4_FLOW: 350 case ESP_V4_FLOW: 351 case AH_ESP_V4_FLOW: 352 case SCTP_V4_FLOW: 353 case IPV4_FLOW: 354 case IPV4_USER_FLOW: 355 return FBNIC_IPV4_HASH_OPT; 356 case AH_V6_FLOW: 357 case ESP_V6_FLOW: 358 case AH_ESP_V6_FLOW: 359 case SCTP_V6_FLOW: 360 case IPV6_FLOW: 361 case IPV6_USER_FLOW: 362 return FBNIC_IPV6_HASH_OPT; 363 case ETHER_FLOW: 364 return FBNIC_ETHER_HASH_OPT; 365 } 366 367 return -1; 368 } 369 370 static int 371 fbnic_get_rss_hash_opts(struct fbnic_net *fbn, struct ethtool_rxnfc *cmd) 372 { 373 int hash_opt_idx = fbnic_get_rss_hash_idx(cmd->flow_type); 374 375 if (hash_opt_idx < 0) 376 return -EINVAL; 377 378 /* Report options from rss_en table in fbn */ 379 cmd->data = fbn->rss_flow_hash[hash_opt_idx]; 380 381 return 0; 382 } 383 384 static int fbnic_get_cls_rule_all(struct fbnic_net *fbn, 385 struct ethtool_rxnfc *cmd, 386 u32 *rule_locs) 387 { 388 struct fbnic_dev *fbd = fbn->fbd; 389 int i, cnt = 0; 390 391 /* Report maximum rule count */ 392 cmd->data = FBNIC_RPC_ACT_TBL_NFC_ENTRIES; 393 394 for (i = 0; i < FBNIC_RPC_ACT_TBL_NFC_ENTRIES; i++) { 395 int idx = i + FBNIC_RPC_ACT_TBL_NFC_OFFSET; 396 struct fbnic_act_tcam *act_tcam; 397 398 act_tcam = &fbd->act_tcam[idx]; 399 if (act_tcam->state != FBNIC_TCAM_S_VALID) 400 continue; 401 402 if (rule_locs) { 403 if (cnt == cmd->rule_cnt) 404 return -EMSGSIZE; 405 406 rule_locs[cnt] = i; 407 } 408 409 cnt++; 410 } 411 412 return cnt; 413 } 414 415 static int fbnic_get_cls_rule(struct fbnic_net *fbn, struct ethtool_rxnfc *cmd) 416 { 417 struct ethtool_rx_flow_spec *fsp; 418 struct fbnic_dev *fbd = fbn->fbd; 419 struct fbnic_act_tcam *act_tcam; 420 int idx; 421 422 fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; 423 424 if (fsp->location >= FBNIC_RPC_ACT_TBL_NFC_ENTRIES) 425 return -EINVAL; 426 427 idx = fsp->location + FBNIC_RPC_ACT_TBL_NFC_OFFSET; 428 act_tcam = &fbd->act_tcam[idx]; 429 430 if (act_tcam->state != FBNIC_TCAM_S_VALID) 431 return -EINVAL; 432 433 /* Report maximum rule count */ 434 cmd->data = FBNIC_RPC_ACT_TBL_NFC_ENTRIES; 435 436 /* Set flow type field */ 437 if (!(act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_IP_VALID)) { 438 fsp->flow_type = ETHER_FLOW; 439 if (!FIELD_GET(FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX, 440 act_tcam->mask.tcam[1])) { 441 struct fbnic_mac_addr *mac_addr; 442 443 idx = FIELD_GET(FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX, 444 act_tcam->value.tcam[1]); 445 mac_addr = &fbd->mac_addr[idx]; 446 447 ether_addr_copy(fsp->h_u.ether_spec.h_dest, 448 mac_addr->value.addr8); 449 eth_broadcast_addr(fsp->m_u.ether_spec.h_dest); 450 } 451 } else if (act_tcam->value.tcam[1] & 452 FBNIC_RPC_TCAM_ACT1_OUTER_IP_VALID) { 453 fsp->flow_type = IPV6_USER_FLOW; 454 fsp->h_u.usr_ip6_spec.l4_proto = IPPROTO_IPV6; 455 fsp->m_u.usr_ip6_spec.l4_proto = 0xff; 456 457 if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX, 458 act_tcam->mask.tcam[0])) { 459 struct fbnic_ip_addr *ip_addr; 460 int i; 461 462 idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX, 463 act_tcam->value.tcam[0]); 464 ip_addr = &fbd->ipo_src[idx]; 465 466 for (i = 0; i < 4; i++) { 467 fsp->h_u.usr_ip6_spec.ip6src[i] = 468 ip_addr->value.s6_addr32[i]; 469 fsp->m_u.usr_ip6_spec.ip6src[i] = 470 ~ip_addr->mask.s6_addr32[i]; 471 } 472 } 473 474 if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX, 475 act_tcam->mask.tcam[0])) { 476 struct fbnic_ip_addr *ip_addr; 477 int i; 478 479 idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX, 480 act_tcam->value.tcam[0]); 481 ip_addr = &fbd->ipo_dst[idx]; 482 483 for (i = 0; i < 4; i++) { 484 fsp->h_u.usr_ip6_spec.ip6dst[i] = 485 ip_addr->value.s6_addr32[i]; 486 fsp->m_u.usr_ip6_spec.ip6dst[i] = 487 ~ip_addr->mask.s6_addr32[i]; 488 } 489 } 490 } else if ((act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_IP_IS_V6)) { 491 if (act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_L4_VALID) { 492 if (act_tcam->value.tcam[1] & 493 FBNIC_RPC_TCAM_ACT1_L4_IS_UDP) 494 fsp->flow_type = UDP_V6_FLOW; 495 else 496 fsp->flow_type = TCP_V6_FLOW; 497 fsp->h_u.tcp_ip6_spec.psrc = 498 cpu_to_be16(act_tcam->value.tcam[3]); 499 fsp->m_u.tcp_ip6_spec.psrc = 500 cpu_to_be16(~act_tcam->mask.tcam[3]); 501 fsp->h_u.tcp_ip6_spec.pdst = 502 cpu_to_be16(act_tcam->value.tcam[4]); 503 fsp->m_u.tcp_ip6_spec.pdst = 504 cpu_to_be16(~act_tcam->mask.tcam[4]); 505 } else { 506 fsp->flow_type = IPV6_USER_FLOW; 507 } 508 509 if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX, 510 act_tcam->mask.tcam[0])) { 511 struct fbnic_ip_addr *ip_addr; 512 int i; 513 514 idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX, 515 act_tcam->value.tcam[0]); 516 ip_addr = &fbd->ip_src[idx]; 517 518 for (i = 0; i < 4; i++) { 519 fsp->h_u.usr_ip6_spec.ip6src[i] = 520 ip_addr->value.s6_addr32[i]; 521 fsp->m_u.usr_ip6_spec.ip6src[i] = 522 ~ip_addr->mask.s6_addr32[i]; 523 } 524 } 525 526 if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPDST_IDX, 527 act_tcam->mask.tcam[0])) { 528 struct fbnic_ip_addr *ip_addr; 529 int i; 530 531 idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPDST_IDX, 532 act_tcam->value.tcam[0]); 533 ip_addr = &fbd->ip_dst[idx]; 534 535 for (i = 0; i < 4; i++) { 536 fsp->h_u.usr_ip6_spec.ip6dst[i] = 537 ip_addr->value.s6_addr32[i]; 538 fsp->m_u.usr_ip6_spec.ip6dst[i] = 539 ~ip_addr->mask.s6_addr32[i]; 540 } 541 } 542 } else { 543 if (act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_L4_VALID) { 544 if (act_tcam->value.tcam[1] & 545 FBNIC_RPC_TCAM_ACT1_L4_IS_UDP) 546 fsp->flow_type = UDP_V4_FLOW; 547 else 548 fsp->flow_type = TCP_V4_FLOW; 549 fsp->h_u.tcp_ip4_spec.psrc = 550 cpu_to_be16(act_tcam->value.tcam[3]); 551 fsp->m_u.tcp_ip4_spec.psrc = 552 cpu_to_be16(~act_tcam->mask.tcam[3]); 553 fsp->h_u.tcp_ip4_spec.pdst = 554 cpu_to_be16(act_tcam->value.tcam[4]); 555 fsp->m_u.tcp_ip4_spec.pdst = 556 cpu_to_be16(~act_tcam->mask.tcam[4]); 557 } else { 558 fsp->flow_type = IPV4_USER_FLOW; 559 fsp->h_u.usr_ip4_spec.ip_ver = ETH_RX_NFC_IP4; 560 } 561 562 if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX, 563 act_tcam->mask.tcam[0])) { 564 struct fbnic_ip_addr *ip_addr; 565 566 idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX, 567 act_tcam->value.tcam[0]); 568 ip_addr = &fbd->ip_src[idx]; 569 570 fsp->h_u.usr_ip4_spec.ip4src = 571 ip_addr->value.s6_addr32[3]; 572 fsp->m_u.usr_ip4_spec.ip4src = 573 ~ip_addr->mask.s6_addr32[3]; 574 } 575 576 if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPDST_IDX, 577 act_tcam->mask.tcam[0])) { 578 struct fbnic_ip_addr *ip_addr; 579 580 idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPDST_IDX, 581 act_tcam->value.tcam[0]); 582 ip_addr = &fbd->ip_dst[idx]; 583 584 fsp->h_u.usr_ip4_spec.ip4dst = 585 ip_addr->value.s6_addr32[3]; 586 fsp->m_u.usr_ip4_spec.ip4dst = 587 ~ip_addr->mask.s6_addr32[3]; 588 } 589 } 590 591 /* Record action */ 592 if (act_tcam->dest & FBNIC_RPC_ACT_TBL0_DROP) 593 fsp->ring_cookie = RX_CLS_FLOW_DISC; 594 else if (act_tcam->dest & FBNIC_RPC_ACT_TBL0_Q_SEL) 595 fsp->ring_cookie = FIELD_GET(FBNIC_RPC_ACT_TBL0_Q_ID, 596 act_tcam->dest); 597 else 598 fsp->flow_type |= FLOW_RSS; 599 600 cmd->rss_context = FIELD_GET(FBNIC_RPC_ACT_TBL0_RSS_CTXT_ID, 601 act_tcam->dest); 602 603 return 0; 604 } 605 606 static int fbnic_get_rxnfc(struct net_device *netdev, 607 struct ethtool_rxnfc *cmd, u32 *rule_locs) 608 { 609 struct fbnic_net *fbn = netdev_priv(netdev); 610 int ret = -EOPNOTSUPP; 611 u32 special = 0; 612 613 switch (cmd->cmd) { 614 case ETHTOOL_GRXRINGS: 615 cmd->data = fbn->num_rx_queues; 616 ret = 0; 617 break; 618 case ETHTOOL_GRXFH: 619 ret = fbnic_get_rss_hash_opts(fbn, cmd); 620 break; 621 case ETHTOOL_GRXCLSRULE: 622 ret = fbnic_get_cls_rule(fbn, cmd); 623 break; 624 case ETHTOOL_GRXCLSRLCNT: 625 rule_locs = NULL; 626 special = RX_CLS_LOC_SPECIAL; 627 fallthrough; 628 case ETHTOOL_GRXCLSRLALL: 629 ret = fbnic_get_cls_rule_all(fbn, cmd, rule_locs); 630 if (ret < 0) 631 break; 632 633 cmd->data |= special; 634 cmd->rule_cnt = ret; 635 ret = 0; 636 break; 637 } 638 639 return ret; 640 } 641 642 #define FBNIC_L2_HASH_OPTIONS \ 643 (RXH_L2DA | RXH_DISCARD) 644 #define FBNIC_L3_HASH_OPTIONS \ 645 (FBNIC_L2_HASH_OPTIONS | RXH_IP_SRC | RXH_IP_DST) 646 #define FBNIC_L4_HASH_OPTIONS \ 647 (FBNIC_L3_HASH_OPTIONS | RXH_L4_B_0_1 | RXH_L4_B_2_3) 648 649 static int 650 fbnic_set_rss_hash_opts(struct fbnic_net *fbn, const struct ethtool_rxnfc *cmd) 651 { 652 int hash_opt_idx; 653 654 /* Verify the type requested is correct */ 655 hash_opt_idx = fbnic_get_rss_hash_idx(cmd->flow_type); 656 if (hash_opt_idx < 0) 657 return -EINVAL; 658 659 /* Verify the fields asked for can actually be assigned based on type */ 660 if (cmd->data & ~FBNIC_L4_HASH_OPTIONS || 661 (hash_opt_idx > FBNIC_L4_HASH_OPT && 662 cmd->data & ~FBNIC_L3_HASH_OPTIONS) || 663 (hash_opt_idx > FBNIC_IP_HASH_OPT && 664 cmd->data & ~FBNIC_L2_HASH_OPTIONS)) 665 return -EINVAL; 666 667 fbn->rss_flow_hash[hash_opt_idx] = cmd->data; 668 669 if (netif_running(fbn->netdev)) { 670 fbnic_rss_reinit(fbn->fbd, fbn); 671 fbnic_write_rules(fbn->fbd); 672 } 673 674 return 0; 675 } 676 677 static int fbnic_cls_rule_any_loc(struct fbnic_dev *fbd) 678 { 679 int i; 680 681 for (i = FBNIC_RPC_ACT_TBL_NFC_ENTRIES; i--;) { 682 int idx = i + FBNIC_RPC_ACT_TBL_NFC_OFFSET; 683 684 if (fbd->act_tcam[idx].state != FBNIC_TCAM_S_VALID) 685 return i; 686 } 687 688 return -ENOSPC; 689 } 690 691 static int fbnic_set_cls_rule_ins(struct fbnic_net *fbn, 692 const struct ethtool_rxnfc *cmd) 693 { 694 u16 flow_value = 0, flow_mask = 0xffff, ip_value = 0, ip_mask = 0xffff; 695 u16 sport = 0, sport_mask = ~0, dport = 0, dport_mask = ~0; 696 u16 misc = 0, misc_mask = ~0; 697 u32 dest = FIELD_PREP(FBNIC_RPC_ACT_TBL0_DEST_MASK, 698 FBNIC_RPC_ACT_TBL0_DEST_HOST); 699 struct fbnic_ip_addr *ip_src = NULL, *ip_dst = NULL; 700 struct fbnic_mac_addr *mac_addr = NULL; 701 struct ethtool_rx_flow_spec *fsp; 702 struct fbnic_dev *fbd = fbn->fbd; 703 struct fbnic_act_tcam *act_tcam; 704 struct in6_addr *addr6, *mask6; 705 struct in_addr *addr4, *mask4; 706 int hash_idx, location; 707 u32 flow_type; 708 int idx, j; 709 710 fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; 711 712 if (fsp->location != RX_CLS_LOC_ANY) 713 return -EINVAL; 714 location = fbnic_cls_rule_any_loc(fbd); 715 if (location < 0) 716 return location; 717 718 if (fsp->ring_cookie == RX_CLS_FLOW_DISC) { 719 dest = FBNIC_RPC_ACT_TBL0_DROP; 720 } else if (fsp->flow_type & FLOW_RSS) { 721 if (cmd->rss_context == 1) 722 dest |= FBNIC_RPC_ACT_TBL0_RSS_CTXT_ID; 723 } else { 724 u32 ring_idx = ethtool_get_flow_spec_ring(fsp->ring_cookie); 725 726 if (ring_idx >= fbn->num_rx_queues) 727 return -EINVAL; 728 729 dest |= FBNIC_RPC_ACT_TBL0_Q_SEL | 730 FIELD_PREP(FBNIC_RPC_ACT_TBL0_Q_ID, ring_idx); 731 } 732 733 idx = location + FBNIC_RPC_ACT_TBL_NFC_OFFSET; 734 act_tcam = &fbd->act_tcam[idx]; 735 736 /* Do not allow overwriting for now. 737 * To support overwriting rules we will need to add logic to free 738 * any IP or MACDA TCAMs that may be associated with the old rule. 739 */ 740 if (act_tcam->state != FBNIC_TCAM_S_DISABLED) 741 return -EBUSY; 742 743 flow_type = fsp->flow_type & ~(FLOW_EXT | FLOW_RSS); 744 hash_idx = fbnic_get_rss_hash_idx(flow_type); 745 746 switch (flow_type) { 747 case UDP_V4_FLOW: 748 udp4_flow: 749 flow_value |= FBNIC_RPC_TCAM_ACT1_L4_IS_UDP; 750 fallthrough; 751 case TCP_V4_FLOW: 752 tcp4_flow: 753 flow_value |= FBNIC_RPC_TCAM_ACT1_L4_VALID; 754 flow_mask &= ~(FBNIC_RPC_TCAM_ACT1_L4_IS_UDP | 755 FBNIC_RPC_TCAM_ACT1_L4_VALID); 756 757 sport = be16_to_cpu(fsp->h_u.tcp_ip4_spec.psrc); 758 sport_mask = ~be16_to_cpu(fsp->m_u.tcp_ip4_spec.psrc); 759 dport = be16_to_cpu(fsp->h_u.tcp_ip4_spec.pdst); 760 dport_mask = ~be16_to_cpu(fsp->m_u.tcp_ip4_spec.pdst); 761 goto ip4_flow; 762 case IP_USER_FLOW: 763 if (!fsp->m_u.usr_ip4_spec.proto) 764 goto ip4_flow; 765 if (fsp->m_u.usr_ip4_spec.proto != 0xff) 766 return -EINVAL; 767 if (fsp->h_u.usr_ip4_spec.proto == IPPROTO_UDP) 768 goto udp4_flow; 769 if (fsp->h_u.usr_ip4_spec.proto == IPPROTO_TCP) 770 goto tcp4_flow; 771 return -EINVAL; 772 ip4_flow: 773 addr4 = (struct in_addr *)&fsp->h_u.usr_ip4_spec.ip4src; 774 mask4 = (struct in_addr *)&fsp->m_u.usr_ip4_spec.ip4src; 775 if (mask4->s_addr) { 776 ip_src = __fbnic_ip4_sync(fbd, fbd->ip_src, 777 addr4, mask4); 778 if (!ip_src) 779 return -ENOSPC; 780 781 set_bit(idx, ip_src->act_tcam); 782 ip_value |= FBNIC_RPC_TCAM_ACT0_IPSRC_VALID | 783 FIELD_PREP(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX, 784 ip_src - fbd->ip_src); 785 ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_IPSRC_VALID | 786 FBNIC_RPC_TCAM_ACT0_IPSRC_IDX); 787 } 788 789 addr4 = (struct in_addr *)&fsp->h_u.usr_ip4_spec.ip4dst; 790 mask4 = (struct in_addr *)&fsp->m_u.usr_ip4_spec.ip4dst; 791 if (mask4->s_addr) { 792 ip_dst = __fbnic_ip4_sync(fbd, fbd->ip_dst, 793 addr4, mask4); 794 if (!ip_dst) { 795 if (ip_src && ip_src->state == FBNIC_TCAM_S_ADD) 796 memset(ip_src, 0, sizeof(*ip_src)); 797 return -ENOSPC; 798 } 799 800 set_bit(idx, ip_dst->act_tcam); 801 ip_value |= FBNIC_RPC_TCAM_ACT0_IPDST_VALID | 802 FIELD_PREP(FBNIC_RPC_TCAM_ACT0_IPDST_IDX, 803 ip_dst - fbd->ip_dst); 804 ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_IPDST_VALID | 805 FBNIC_RPC_TCAM_ACT0_IPDST_IDX); 806 } 807 flow_value |= FBNIC_RPC_TCAM_ACT1_IP_VALID | 808 FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID; 809 flow_mask &= ~(FBNIC_RPC_TCAM_ACT1_IP_IS_V6 | 810 FBNIC_RPC_TCAM_ACT1_IP_VALID | 811 FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID); 812 break; 813 case UDP_V6_FLOW: 814 udp6_flow: 815 flow_value |= FBNIC_RPC_TCAM_ACT1_L4_IS_UDP; 816 fallthrough; 817 case TCP_V6_FLOW: 818 tcp6_flow: 819 flow_value |= FBNIC_RPC_TCAM_ACT1_L4_VALID; 820 flow_mask &= ~(FBNIC_RPC_TCAM_ACT1_L4_IS_UDP | 821 FBNIC_RPC_TCAM_ACT1_L4_VALID); 822 823 sport = be16_to_cpu(fsp->h_u.tcp_ip6_spec.psrc); 824 sport_mask = ~be16_to_cpu(fsp->m_u.tcp_ip6_spec.psrc); 825 dport = be16_to_cpu(fsp->h_u.tcp_ip6_spec.pdst); 826 dport_mask = ~be16_to_cpu(fsp->m_u.tcp_ip6_spec.pdst); 827 goto ipv6_flow; 828 case IPV6_USER_FLOW: 829 if (!fsp->m_u.usr_ip6_spec.l4_proto) 830 goto ipv6_flow; 831 832 if (fsp->m_u.usr_ip6_spec.l4_proto != 0xff) 833 return -EINVAL; 834 if (fsp->h_u.usr_ip6_spec.l4_proto == IPPROTO_UDP) 835 goto udp6_flow; 836 if (fsp->h_u.usr_ip6_spec.l4_proto == IPPROTO_TCP) 837 goto tcp6_flow; 838 if (fsp->h_u.usr_ip6_spec.l4_proto != IPPROTO_IPV6) 839 return -EINVAL; 840 841 addr6 = (struct in6_addr *)fsp->h_u.usr_ip6_spec.ip6src; 842 mask6 = (struct in6_addr *)fsp->m_u.usr_ip6_spec.ip6src; 843 if (!ipv6_addr_any(mask6)) { 844 ip_src = __fbnic_ip6_sync(fbd, fbd->ipo_src, 845 addr6, mask6); 846 if (!ip_src) 847 return -ENOSPC; 848 849 set_bit(idx, ip_src->act_tcam); 850 ip_value |= 851 FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_VALID | 852 FIELD_PREP(FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX, 853 ip_src - fbd->ipo_src); 854 ip_mask &= 855 ~(FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_VALID | 856 FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX); 857 } 858 859 addr6 = (struct in6_addr *)fsp->h_u.usr_ip6_spec.ip6dst; 860 mask6 = (struct in6_addr *)fsp->m_u.usr_ip6_spec.ip6dst; 861 if (!ipv6_addr_any(mask6)) { 862 ip_dst = __fbnic_ip6_sync(fbd, fbd->ipo_dst, 863 addr6, mask6); 864 if (!ip_dst) { 865 if (ip_src && ip_src->state == FBNIC_TCAM_S_ADD) 866 memset(ip_src, 0, sizeof(*ip_src)); 867 return -ENOSPC; 868 } 869 870 set_bit(idx, ip_dst->act_tcam); 871 ip_value |= 872 FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_VALID | 873 FIELD_PREP(FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX, 874 ip_dst - fbd->ipo_dst); 875 ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_VALID | 876 FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX); 877 } 878 879 flow_value |= FBNIC_RPC_TCAM_ACT1_OUTER_IP_VALID; 880 flow_mask &= FBNIC_RPC_TCAM_ACT1_OUTER_IP_VALID; 881 ipv6_flow: 882 addr6 = (struct in6_addr *)fsp->h_u.usr_ip6_spec.ip6src; 883 mask6 = (struct in6_addr *)fsp->m_u.usr_ip6_spec.ip6src; 884 if (!ip_src && !ipv6_addr_any(mask6)) { 885 ip_src = __fbnic_ip6_sync(fbd, fbd->ip_src, 886 addr6, mask6); 887 if (!ip_src) 888 return -ENOSPC; 889 890 set_bit(idx, ip_src->act_tcam); 891 ip_value |= FBNIC_RPC_TCAM_ACT0_IPSRC_VALID | 892 FIELD_PREP(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX, 893 ip_src - fbd->ip_src); 894 ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_IPSRC_VALID | 895 FBNIC_RPC_TCAM_ACT0_IPSRC_IDX); 896 } 897 898 addr6 = (struct in6_addr *)fsp->h_u.usr_ip6_spec.ip6dst; 899 mask6 = (struct in6_addr *)fsp->m_u.usr_ip6_spec.ip6dst; 900 if (!ip_dst && !ipv6_addr_any(mask6)) { 901 ip_dst = __fbnic_ip6_sync(fbd, fbd->ip_dst, 902 addr6, mask6); 903 if (!ip_dst) { 904 if (ip_src && ip_src->state == FBNIC_TCAM_S_ADD) 905 memset(ip_src, 0, sizeof(*ip_src)); 906 return -ENOSPC; 907 } 908 909 set_bit(idx, ip_dst->act_tcam); 910 ip_value |= FBNIC_RPC_TCAM_ACT0_IPDST_VALID | 911 FIELD_PREP(FBNIC_RPC_TCAM_ACT0_IPDST_IDX, 912 ip_dst - fbd->ip_dst); 913 ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_IPDST_VALID | 914 FBNIC_RPC_TCAM_ACT0_IPDST_IDX); 915 } 916 917 flow_value |= FBNIC_RPC_TCAM_ACT1_IP_IS_V6 | 918 FBNIC_RPC_TCAM_ACT1_IP_VALID | 919 FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID; 920 flow_mask &= ~(FBNIC_RPC_TCAM_ACT1_IP_IS_V6 | 921 FBNIC_RPC_TCAM_ACT1_IP_VALID | 922 FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID); 923 break; 924 case ETHER_FLOW: 925 if (!is_zero_ether_addr(fsp->m_u.ether_spec.h_dest)) { 926 u8 *addr = fsp->h_u.ether_spec.h_dest; 927 u8 *mask = fsp->m_u.ether_spec.h_dest; 928 929 /* Do not allow MAC addr of 0 */ 930 if (is_zero_ether_addr(addr)) 931 return -EINVAL; 932 933 /* Only support full MAC address to avoid 934 * conflicts with other MAC addresses. 935 */ 936 if (!is_broadcast_ether_addr(mask)) 937 return -EINVAL; 938 939 if (is_multicast_ether_addr(addr)) 940 mac_addr = __fbnic_mc_sync(fbd, addr); 941 else 942 mac_addr = __fbnic_uc_sync(fbd, addr); 943 944 if (!mac_addr) 945 return -ENOSPC; 946 947 set_bit(idx, mac_addr->act_tcam); 948 flow_value |= 949 FIELD_PREP(FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX, 950 mac_addr - fbd->mac_addr); 951 flow_mask &= ~FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX; 952 } 953 954 flow_value |= FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID; 955 flow_mask &= ~FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID; 956 break; 957 default: 958 return -EINVAL; 959 } 960 961 /* Write action table values */ 962 act_tcam->dest = dest; 963 act_tcam->rss_en_mask = fbnic_flow_hash_2_rss_en_mask(fbn, hash_idx); 964 965 /* Write IP Match value/mask to action_tcam[0] */ 966 act_tcam->value.tcam[0] = ip_value; 967 act_tcam->mask.tcam[0] = ip_mask; 968 969 /* Write flow type value/mask to action_tcam[1] */ 970 act_tcam->value.tcam[1] = flow_value; 971 act_tcam->mask.tcam[1] = flow_mask; 972 973 /* Write error, DSCP, extra L4 matches to action_tcam[2] */ 974 act_tcam->value.tcam[2] = misc; 975 act_tcam->mask.tcam[2] = misc_mask; 976 977 /* Write source/destination port values */ 978 act_tcam->value.tcam[3] = sport; 979 act_tcam->mask.tcam[3] = sport_mask; 980 act_tcam->value.tcam[4] = dport; 981 act_tcam->mask.tcam[4] = dport_mask; 982 983 for (j = 5; j < FBNIC_RPC_TCAM_ACT_WORD_LEN; j++) 984 act_tcam->mask.tcam[j] = 0xffff; 985 986 act_tcam->state = FBNIC_TCAM_S_UPDATE; 987 fsp->location = location; 988 989 if (netif_running(fbn->netdev)) { 990 fbnic_write_rules(fbd); 991 if (ip_src || ip_dst) 992 fbnic_write_ip_addr(fbd); 993 if (mac_addr) 994 fbnic_write_macda(fbd); 995 } 996 997 return 0; 998 } 999 1000 static void fbnic_clear_nfc_macda(struct fbnic_net *fbn, 1001 unsigned int tcam_idx) 1002 { 1003 struct fbnic_dev *fbd = fbn->fbd; 1004 int idx; 1005 1006 for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) 1007 __fbnic_xc_unsync(&fbd->mac_addr[idx], tcam_idx); 1008 1009 /* Write updates to hardware */ 1010 if (netif_running(fbn->netdev)) 1011 fbnic_write_macda(fbd); 1012 } 1013 1014 static void fbnic_clear_nfc_ip_addr(struct fbnic_net *fbn, 1015 unsigned int tcam_idx) 1016 { 1017 struct fbnic_dev *fbd = fbn->fbd; 1018 int idx; 1019 1020 for (idx = ARRAY_SIZE(fbd->ip_src); idx--;) 1021 __fbnic_ip_unsync(&fbd->ip_src[idx], tcam_idx); 1022 for (idx = ARRAY_SIZE(fbd->ip_dst); idx--;) 1023 __fbnic_ip_unsync(&fbd->ip_dst[idx], tcam_idx); 1024 for (idx = ARRAY_SIZE(fbd->ipo_src); idx--;) 1025 __fbnic_ip_unsync(&fbd->ipo_src[idx], tcam_idx); 1026 for (idx = ARRAY_SIZE(fbd->ipo_dst); idx--;) 1027 __fbnic_ip_unsync(&fbd->ipo_dst[idx], tcam_idx); 1028 1029 /* Write updates to hardware */ 1030 if (netif_running(fbn->netdev)) 1031 fbnic_write_ip_addr(fbd); 1032 } 1033 1034 static int fbnic_set_cls_rule_del(struct fbnic_net *fbn, 1035 const struct ethtool_rxnfc *cmd) 1036 { 1037 struct ethtool_rx_flow_spec *fsp; 1038 struct fbnic_dev *fbd = fbn->fbd; 1039 struct fbnic_act_tcam *act_tcam; 1040 int idx; 1041 1042 fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; 1043 1044 if (fsp->location >= FBNIC_RPC_ACT_TBL_NFC_ENTRIES) 1045 return -EINVAL; 1046 1047 idx = fsp->location + FBNIC_RPC_ACT_TBL_NFC_OFFSET; 1048 act_tcam = &fbd->act_tcam[idx]; 1049 1050 if (act_tcam->state != FBNIC_TCAM_S_VALID) 1051 return -EINVAL; 1052 1053 act_tcam->state = FBNIC_TCAM_S_DELETE; 1054 1055 if ((act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID) && 1056 (~act_tcam->mask.tcam[1] & FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX)) 1057 fbnic_clear_nfc_macda(fbn, idx); 1058 1059 if ((act_tcam->value.tcam[0] & 1060 (FBNIC_RPC_TCAM_ACT0_IPSRC_VALID | 1061 FBNIC_RPC_TCAM_ACT0_IPDST_VALID | 1062 FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_VALID | 1063 FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_VALID)) && 1064 (~act_tcam->mask.tcam[0] & 1065 (FBNIC_RPC_TCAM_ACT0_IPSRC_IDX | 1066 FBNIC_RPC_TCAM_ACT0_IPDST_IDX | 1067 FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX | 1068 FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX))) 1069 fbnic_clear_nfc_ip_addr(fbn, idx); 1070 1071 if (netif_running(fbn->netdev)) 1072 fbnic_write_rules(fbd); 1073 1074 return 0; 1075 } 1076 1077 static int fbnic_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) 1078 { 1079 struct fbnic_net *fbn = netdev_priv(netdev); 1080 int ret = -EOPNOTSUPP; 1081 1082 switch (cmd->cmd) { 1083 case ETHTOOL_SRXFH: 1084 ret = fbnic_set_rss_hash_opts(fbn, cmd); 1085 break; 1086 case ETHTOOL_SRXCLSRLINS: 1087 ret = fbnic_set_cls_rule_ins(fbn, cmd); 1088 break; 1089 case ETHTOOL_SRXCLSRLDEL: 1090 ret = fbnic_set_cls_rule_del(fbn, cmd); 1091 break; 1092 } 1093 1094 return ret; 1095 } 1096 1097 static u32 fbnic_get_rxfh_key_size(struct net_device *netdev) 1098 { 1099 return FBNIC_RPC_RSS_KEY_BYTE_LEN; 1100 } 1101 1102 static u32 fbnic_get_rxfh_indir_size(struct net_device *netdev) 1103 { 1104 return FBNIC_RPC_RSS_TBL_SIZE; 1105 } 1106 1107 static int 1108 fbnic_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) 1109 { 1110 struct fbnic_net *fbn = netdev_priv(netdev); 1111 unsigned int i; 1112 1113 rxfh->hfunc = ETH_RSS_HASH_TOP; 1114 1115 if (rxfh->key) { 1116 for (i = 0; i < FBNIC_RPC_RSS_KEY_BYTE_LEN; i++) { 1117 u32 rss_key = fbn->rss_key[i / 4] << ((i % 4) * 8); 1118 1119 rxfh->key[i] = rss_key >> 24; 1120 } 1121 } 1122 1123 if (rxfh->indir) { 1124 for (i = 0; i < FBNIC_RPC_RSS_TBL_SIZE; i++) 1125 rxfh->indir[i] = fbn->indir_tbl[0][i]; 1126 } 1127 1128 return 0; 1129 } 1130 1131 static unsigned int 1132 fbnic_set_indir(struct fbnic_net *fbn, unsigned int idx, const u32 *indir) 1133 { 1134 unsigned int i, changes = 0; 1135 1136 for (i = 0; i < FBNIC_RPC_RSS_TBL_SIZE; i++) { 1137 if (fbn->indir_tbl[idx][i] == indir[i]) 1138 continue; 1139 1140 fbn->indir_tbl[idx][i] = indir[i]; 1141 changes++; 1142 } 1143 1144 return changes; 1145 } 1146 1147 static int 1148 fbnic_set_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh, 1149 struct netlink_ext_ack *extack) 1150 { 1151 struct fbnic_net *fbn = netdev_priv(netdev); 1152 unsigned int i, changes = 0; 1153 1154 if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && 1155 rxfh->hfunc != ETH_RSS_HASH_TOP) 1156 return -EINVAL; 1157 1158 if (rxfh->key) { 1159 u32 rss_key = 0; 1160 1161 for (i = FBNIC_RPC_RSS_KEY_BYTE_LEN; i--;) { 1162 rss_key >>= 8; 1163 rss_key |= (u32)(rxfh->key[i]) << 24; 1164 1165 if (i % 4) 1166 continue; 1167 1168 if (fbn->rss_key[i / 4] == rss_key) 1169 continue; 1170 1171 fbn->rss_key[i / 4] = rss_key; 1172 changes++; 1173 } 1174 } 1175 1176 if (rxfh->indir) 1177 changes += fbnic_set_indir(fbn, 0, rxfh->indir); 1178 1179 if (changes && netif_running(netdev)) 1180 fbnic_rss_reinit_hw(fbn->fbd, fbn); 1181 1182 return 0; 1183 } 1184 1185 static int 1186 fbnic_modify_rxfh_context(struct net_device *netdev, 1187 struct ethtool_rxfh_context *ctx, 1188 const struct ethtool_rxfh_param *rxfh, 1189 struct netlink_ext_ack *extack) 1190 { 1191 struct fbnic_net *fbn = netdev_priv(netdev); 1192 const u32 *indir = rxfh->indir; 1193 unsigned int changes; 1194 1195 if (!indir) 1196 indir = ethtool_rxfh_context_indir(ctx); 1197 1198 changes = fbnic_set_indir(fbn, rxfh->rss_context, indir); 1199 if (changes && netif_running(netdev)) 1200 fbnic_rss_reinit_hw(fbn->fbd, fbn); 1201 1202 return 0; 1203 } 1204 1205 static int 1206 fbnic_create_rxfh_context(struct net_device *netdev, 1207 struct ethtool_rxfh_context *ctx, 1208 const struct ethtool_rxfh_param *rxfh, 1209 struct netlink_ext_ack *extack) 1210 { 1211 struct fbnic_net *fbn = netdev_priv(netdev); 1212 1213 if (rxfh->hfunc && rxfh->hfunc != ETH_RSS_HASH_TOP) { 1214 NL_SET_ERR_MSG_MOD(extack, "RSS hash function not supported"); 1215 return -EOPNOTSUPP; 1216 } 1217 ctx->hfunc = ETH_RSS_HASH_TOP; 1218 1219 if (!rxfh->indir) { 1220 u32 *indir = ethtool_rxfh_context_indir(ctx); 1221 unsigned int num_rx = fbn->num_rx_queues; 1222 unsigned int i; 1223 1224 for (i = 0; i < FBNIC_RPC_RSS_TBL_SIZE; i++) 1225 indir[i] = ethtool_rxfh_indir_default(i, num_rx); 1226 } 1227 1228 return fbnic_modify_rxfh_context(netdev, ctx, rxfh, extack); 1229 } 1230 1231 static int 1232 fbnic_remove_rxfh_context(struct net_device *netdev, 1233 struct ethtool_rxfh_context *ctx, u32 rss_context, 1234 struct netlink_ext_ack *extack) 1235 { 1236 /* Nothing to do, contexts are allocated statically */ 1237 return 0; 1238 } 1239 1240 static void fbnic_get_channels(struct net_device *netdev, 1241 struct ethtool_channels *ch) 1242 { 1243 struct fbnic_net *fbn = netdev_priv(netdev); 1244 struct fbnic_dev *fbd = fbn->fbd; 1245 1246 ch->max_rx = fbd->max_num_queues; 1247 ch->max_tx = fbd->max_num_queues; 1248 ch->max_combined = min(ch->max_rx, ch->max_tx); 1249 ch->max_other = FBNIC_NON_NAPI_VECTORS; 1250 1251 if (fbn->num_rx_queues > fbn->num_napi || 1252 fbn->num_tx_queues > fbn->num_napi) 1253 ch->combined_count = min(fbn->num_rx_queues, 1254 fbn->num_tx_queues); 1255 else 1256 ch->combined_count = 1257 fbn->num_rx_queues + fbn->num_tx_queues - fbn->num_napi; 1258 ch->rx_count = fbn->num_rx_queues - ch->combined_count; 1259 ch->tx_count = fbn->num_tx_queues - ch->combined_count; 1260 ch->other_count = FBNIC_NON_NAPI_VECTORS; 1261 } 1262 1263 static void fbnic_set_queues(struct fbnic_net *fbn, struct ethtool_channels *ch, 1264 unsigned int max_napis) 1265 { 1266 fbn->num_rx_queues = ch->rx_count + ch->combined_count; 1267 fbn->num_tx_queues = ch->tx_count + ch->combined_count; 1268 fbn->num_napi = min(ch->rx_count + ch->tx_count + ch->combined_count, 1269 max_napis); 1270 } 1271 1272 static int fbnic_set_channels(struct net_device *netdev, 1273 struct ethtool_channels *ch) 1274 { 1275 struct fbnic_net *fbn = netdev_priv(netdev); 1276 unsigned int max_napis, standalone; 1277 struct fbnic_dev *fbd = fbn->fbd; 1278 struct fbnic_net *clone; 1279 int err; 1280 1281 max_napis = fbd->num_irqs - FBNIC_NON_NAPI_VECTORS; 1282 standalone = ch->rx_count + ch->tx_count; 1283 1284 /* Limits for standalone queues: 1285 * - each queue has it's own NAPI (num_napi >= rx + tx + combined) 1286 * - combining queues (combined not 0, rx or tx must be 0) 1287 */ 1288 if ((ch->rx_count && ch->tx_count && ch->combined_count) || 1289 (standalone && standalone + ch->combined_count > max_napis) || 1290 ch->rx_count + ch->combined_count > fbd->max_num_queues || 1291 ch->tx_count + ch->combined_count > fbd->max_num_queues || 1292 ch->other_count != FBNIC_NON_NAPI_VECTORS) 1293 return -EINVAL; 1294 1295 if (!netif_running(netdev)) { 1296 fbnic_set_queues(fbn, ch, max_napis); 1297 fbnic_reset_indir_tbl(fbn); 1298 return 0; 1299 } 1300 1301 clone = fbnic_clone_create(fbn); 1302 if (!clone) 1303 return -ENOMEM; 1304 1305 fbnic_set_queues(clone, ch, max_napis); 1306 1307 err = fbnic_alloc_napi_vectors(clone); 1308 if (err) 1309 goto err_free_clone; 1310 1311 err = fbnic_alloc_resources(clone); 1312 if (err) 1313 goto err_free_napis; 1314 1315 fbnic_down_noidle(fbn); 1316 err = fbnic_wait_all_queues_idle(fbn->fbd, true); 1317 if (err) 1318 goto err_start_stack; 1319 1320 err = fbnic_set_netif_queues(clone); 1321 if (err) 1322 goto err_start_stack; 1323 1324 /* Nothing can fail past this point */ 1325 fbnic_flush(fbn); 1326 1327 fbnic_clone_swap(fbn, clone); 1328 1329 /* Reset RSS indirection table */ 1330 fbnic_reset_indir_tbl(fbn); 1331 1332 fbnic_up(fbn); 1333 1334 fbnic_free_resources(clone); 1335 fbnic_free_napi_vectors(clone); 1336 fbnic_clone_free(clone); 1337 1338 return 0; 1339 1340 err_start_stack: 1341 fbnic_flush(fbn); 1342 fbnic_up(fbn); 1343 fbnic_free_resources(clone); 1344 err_free_napis: 1345 fbnic_free_napi_vectors(clone); 1346 err_free_clone: 1347 fbnic_clone_free(clone); 1348 return err; 1349 } 1350 1351 static int 1352 fbnic_get_ts_info(struct net_device *netdev, 1353 struct kernel_ethtool_ts_info *tsinfo) 1354 { 1355 struct fbnic_net *fbn = netdev_priv(netdev); 1356 1357 tsinfo->phc_index = ptp_clock_index(fbn->fbd->ptp); 1358 1359 tsinfo->so_timestamping = 1360 SOF_TIMESTAMPING_TX_SOFTWARE | 1361 SOF_TIMESTAMPING_TX_HARDWARE | 1362 SOF_TIMESTAMPING_RX_HARDWARE | 1363 SOF_TIMESTAMPING_RAW_HARDWARE; 1364 1365 tsinfo->tx_types = 1366 BIT(HWTSTAMP_TX_OFF) | 1367 BIT(HWTSTAMP_TX_ON); 1368 1369 tsinfo->rx_filters = 1370 BIT(HWTSTAMP_FILTER_NONE) | 1371 BIT(HWTSTAMP_FILTER_PTP_V1_L4_EVENT) | 1372 BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT) | 1373 BIT(HWTSTAMP_FILTER_PTP_V2_L2_EVENT) | 1374 BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) | 1375 BIT(HWTSTAMP_FILTER_ALL); 1376 1377 return 0; 1378 } 1379 1380 static void fbnic_get_ts_stats(struct net_device *netdev, 1381 struct ethtool_ts_stats *ts_stats) 1382 { 1383 struct fbnic_net *fbn = netdev_priv(netdev); 1384 u64 ts_packets, ts_lost; 1385 struct fbnic_ring *ring; 1386 unsigned int start; 1387 int i; 1388 1389 ts_stats->pkts = fbn->tx_stats.twq.ts_packets; 1390 ts_stats->lost = fbn->tx_stats.twq.ts_lost; 1391 for (i = 0; i < fbn->num_tx_queues; i++) { 1392 ring = fbn->tx[i]; 1393 do { 1394 start = u64_stats_fetch_begin(&ring->stats.syncp); 1395 ts_packets = ring->stats.twq.ts_packets; 1396 ts_lost = ring->stats.twq.ts_lost; 1397 } while (u64_stats_fetch_retry(&ring->stats.syncp, start)); 1398 ts_stats->pkts += ts_packets; 1399 ts_stats->lost += ts_lost; 1400 } 1401 } 1402 1403 static void fbnic_set_counter(u64 *stat, struct fbnic_stat_counter *counter) 1404 { 1405 if (counter->reported) 1406 *stat = counter->value; 1407 } 1408 1409 static void 1410 fbnic_get_eth_mac_stats(struct net_device *netdev, 1411 struct ethtool_eth_mac_stats *eth_mac_stats) 1412 { 1413 struct fbnic_net *fbn = netdev_priv(netdev); 1414 struct fbnic_mac_stats *mac_stats; 1415 struct fbnic_dev *fbd = fbn->fbd; 1416 const struct fbnic_mac *mac; 1417 1418 mac_stats = &fbd->hw_stats.mac; 1419 mac = fbd->mac; 1420 1421 mac->get_eth_mac_stats(fbd, false, &mac_stats->eth_mac); 1422 1423 fbnic_set_counter(ð_mac_stats->FramesTransmittedOK, 1424 &mac_stats->eth_mac.FramesTransmittedOK); 1425 fbnic_set_counter(ð_mac_stats->FramesReceivedOK, 1426 &mac_stats->eth_mac.FramesReceivedOK); 1427 fbnic_set_counter(ð_mac_stats->FrameCheckSequenceErrors, 1428 &mac_stats->eth_mac.FrameCheckSequenceErrors); 1429 fbnic_set_counter(ð_mac_stats->AlignmentErrors, 1430 &mac_stats->eth_mac.AlignmentErrors); 1431 fbnic_set_counter(ð_mac_stats->OctetsTransmittedOK, 1432 &mac_stats->eth_mac.OctetsTransmittedOK); 1433 fbnic_set_counter(ð_mac_stats->FramesLostDueToIntMACXmitError, 1434 &mac_stats->eth_mac.FramesLostDueToIntMACXmitError); 1435 fbnic_set_counter(ð_mac_stats->OctetsReceivedOK, 1436 &mac_stats->eth_mac.OctetsReceivedOK); 1437 fbnic_set_counter(ð_mac_stats->FramesLostDueToIntMACRcvError, 1438 &mac_stats->eth_mac.FramesLostDueToIntMACRcvError); 1439 fbnic_set_counter(ð_mac_stats->MulticastFramesXmittedOK, 1440 &mac_stats->eth_mac.MulticastFramesXmittedOK); 1441 fbnic_set_counter(ð_mac_stats->BroadcastFramesXmittedOK, 1442 &mac_stats->eth_mac.BroadcastFramesXmittedOK); 1443 fbnic_set_counter(ð_mac_stats->MulticastFramesReceivedOK, 1444 &mac_stats->eth_mac.MulticastFramesReceivedOK); 1445 fbnic_set_counter(ð_mac_stats->BroadcastFramesReceivedOK, 1446 &mac_stats->eth_mac.BroadcastFramesReceivedOK); 1447 fbnic_set_counter(ð_mac_stats->FrameTooLongErrors, 1448 &mac_stats->eth_mac.FrameTooLongErrors); 1449 } 1450 1451 static const struct ethtool_ops fbnic_ethtool_ops = { 1452 .supported_coalesce_params = 1453 ETHTOOL_COALESCE_USECS | 1454 ETHTOOL_COALESCE_RX_MAX_FRAMES, 1455 .rxfh_max_num_contexts = FBNIC_RPC_RSS_TBL_COUNT, 1456 .get_drvinfo = fbnic_get_drvinfo, 1457 .get_regs_len = fbnic_get_regs_len, 1458 .get_regs = fbnic_get_regs, 1459 .get_coalesce = fbnic_get_coalesce, 1460 .set_coalesce = fbnic_set_coalesce, 1461 .get_ringparam = fbnic_get_ringparam, 1462 .set_ringparam = fbnic_set_ringparam, 1463 .get_strings = fbnic_get_strings, 1464 .get_ethtool_stats = fbnic_get_ethtool_stats, 1465 .get_sset_count = fbnic_get_sset_count, 1466 .get_rxnfc = fbnic_get_rxnfc, 1467 .set_rxnfc = fbnic_set_rxnfc, 1468 .get_rxfh_key_size = fbnic_get_rxfh_key_size, 1469 .get_rxfh_indir_size = fbnic_get_rxfh_indir_size, 1470 .get_rxfh = fbnic_get_rxfh, 1471 .set_rxfh = fbnic_set_rxfh, 1472 .create_rxfh_context = fbnic_create_rxfh_context, 1473 .modify_rxfh_context = fbnic_modify_rxfh_context, 1474 .remove_rxfh_context = fbnic_remove_rxfh_context, 1475 .get_channels = fbnic_get_channels, 1476 .set_channels = fbnic_set_channels, 1477 .get_ts_info = fbnic_get_ts_info, 1478 .get_ts_stats = fbnic_get_ts_stats, 1479 .get_eth_mac_stats = fbnic_get_eth_mac_stats, 1480 }; 1481 1482 void fbnic_set_ethtool_ops(struct net_device *dev) 1483 { 1484 dev->ethtool_ops = &fbnic_ethtool_ops; 1485 } 1486