1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/bitmap.h> 6 #include <linux/etherdevice.h> 7 #include <linux/lockdep.h> 8 #include <linux/netdevice.h> 9 #include <linux/rcupdate.h> 10 #include <linux/rtnetlink.h> 11 #include <linux/slab.h> 12 13 #include "../nfpcore/nfp.h" 14 #include "../nfpcore/nfp_cpp.h" 15 #include "../nfpcore/nfp_nsp.h" 16 #include "../nfp_app.h" 17 #include "../nfp_main.h" 18 #include "../nfp_net.h" 19 #include "../nfp_net_repr.h" 20 #include "../nfp_port.h" 21 #include "main.h" 22 23 static u32 nfp_abm_portid(enum nfp_repr_type rtype, unsigned int id) 24 { 25 return FIELD_PREP(NFP_ABM_PORTID_TYPE, rtype) | 26 FIELD_PREP(NFP_ABM_PORTID_ID, id); 27 } 28 29 static int 30 nfp_abm_setup_tc(struct nfp_app *app, struct net_device *netdev, 31 enum tc_setup_type type, void *type_data) 32 { 33 struct nfp_repr *repr = netdev_priv(netdev); 34 struct nfp_port *port; 35 36 port = nfp_port_from_netdev(netdev); 37 if (!port || port->type != NFP_PORT_PF_PORT) 38 return -EOPNOTSUPP; 39 40 switch (type) { 41 case TC_SETUP_ROOT_QDISC: 42 return nfp_abm_setup_root(netdev, repr->app_priv, type_data); 43 case TC_SETUP_QDISC_MQ: 44 return nfp_abm_setup_tc_mq(netdev, repr->app_priv, type_data); 45 case TC_SETUP_QDISC_RED: 46 return nfp_abm_setup_tc_red(netdev, repr->app_priv, type_data); 47 case TC_SETUP_QDISC_GRED: 48 return nfp_abm_setup_tc_gred(netdev, repr->app_priv, type_data); 49 case TC_SETUP_BLOCK: 50 return nfp_abm_setup_cls_block(netdev, repr, type_data); 51 default: 52 return -EOPNOTSUPP; 53 } 54 } 55 56 static struct net_device * 57 nfp_abm_repr_get(struct nfp_app *app, u32 port_id, bool *redir_egress) 58 { 59 enum nfp_repr_type rtype; 60 struct nfp_reprs *reprs; 61 u8 port; 62 63 rtype = FIELD_GET(NFP_ABM_PORTID_TYPE, port_id); 64 port = FIELD_GET(NFP_ABM_PORTID_ID, port_id); 65 66 reprs = rcu_dereference(app->reprs[rtype]); 67 if (!reprs) 68 return NULL; 69 70 if (port >= reprs->num_reprs) 71 return NULL; 72 73 return rcu_dereference(reprs->reprs[port]); 74 } 75 76 static int 77 nfp_abm_spawn_repr(struct nfp_app *app, struct nfp_abm_link *alink, 78 enum nfp_port_type ptype) 79 { 80 struct net_device *netdev; 81 enum nfp_repr_type rtype; 82 struct nfp_reprs *reprs; 83 struct nfp_repr *repr; 84 struct nfp_port *port; 85 unsigned int txqs; 86 int err; 87 88 if (ptype == NFP_PORT_PHYS_PORT) { 89 rtype = NFP_REPR_TYPE_PHYS_PORT; 90 txqs = 1; 91 } else { 92 rtype = NFP_REPR_TYPE_PF; 93 txqs = alink->vnic->max_rx_rings; 94 } 95 96 netdev = nfp_repr_alloc_mqs(app, txqs, 1); 97 if (!netdev) 98 return -ENOMEM; 99 repr = netdev_priv(netdev); 100 repr->app_priv = alink; 101 102 port = nfp_port_alloc(app, ptype, netdev); 103 if (IS_ERR(port)) { 104 err = PTR_ERR(port); 105 goto err_free_repr; 106 } 107 108 if (ptype == NFP_PORT_PHYS_PORT) { 109 port->eth_forced = true; 110 err = nfp_port_init_phy_port(app->pf, app, port, alink->id); 111 if (err) 112 goto err_free_port; 113 } else { 114 port->pf_id = alink->abm->pf_id; 115 port->pf_split = app->pf->max_data_vnics > 1; 116 port->pf_split_id = alink->id; 117 port->vnic = alink->vnic->dp.ctrl_bar; 118 } 119 120 SET_NETDEV_DEV(netdev, &alink->vnic->pdev->dev); 121 eth_hw_addr_random(netdev); 122 123 err = nfp_repr_init(app, netdev, nfp_abm_portid(rtype, alink->id), 124 port, alink->vnic->dp.netdev); 125 if (err) 126 goto err_free_port; 127 128 reprs = nfp_reprs_get_locked(app, rtype); 129 WARN(nfp_repr_get_locked(app, reprs, alink->id), "duplicate repr"); 130 rtnl_lock(); 131 rcu_assign_pointer(reprs->reprs[alink->id], netdev); 132 rtnl_unlock(); 133 134 nfp_info(app->cpp, "%s Port %d Representor(%s) created\n", 135 ptype == NFP_PORT_PF_PORT ? "PCIe" : "Phys", 136 alink->id, netdev->name); 137 138 return 0; 139 140 err_free_port: 141 nfp_port_free(port); 142 err_free_repr: 143 nfp_repr_free(netdev); 144 return err; 145 } 146 147 static void 148 nfp_abm_kill_repr(struct nfp_app *app, struct nfp_abm_link *alink, 149 enum nfp_repr_type rtype) 150 { 151 struct net_device *netdev; 152 struct nfp_reprs *reprs; 153 154 reprs = nfp_reprs_get_locked(app, rtype); 155 netdev = nfp_repr_get_locked(app, reprs, alink->id); 156 if (!netdev) 157 return; 158 rtnl_lock(); 159 rcu_assign_pointer(reprs->reprs[alink->id], NULL); 160 rtnl_unlock(); 161 synchronize_rcu(); 162 /* Cast to make sure nfp_repr_clean_and_free() takes a nfp_repr */ 163 nfp_repr_clean_and_free((struct nfp_repr *)netdev_priv(netdev)); 164 } 165 166 static void 167 nfp_abm_kill_reprs(struct nfp_abm *abm, struct nfp_abm_link *alink) 168 { 169 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PF); 170 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PHYS_PORT); 171 } 172 173 static void nfp_abm_kill_reprs_all(struct nfp_abm *abm) 174 { 175 struct nfp_pf *pf = abm->app->pf; 176 struct nfp_net *nn; 177 178 list_for_each_entry(nn, &pf->vnics, vnic_list) 179 nfp_abm_kill_reprs(abm, (struct nfp_abm_link *)nn->app_priv); 180 } 181 182 static enum devlink_eswitch_mode nfp_abm_eswitch_mode_get(struct nfp_app *app) 183 { 184 struct nfp_abm *abm = app->priv; 185 186 return abm->eswitch_mode; 187 } 188 189 static int nfp_abm_eswitch_set_legacy(struct nfp_abm *abm) 190 { 191 nfp_abm_kill_reprs_all(abm); 192 nfp_abm_ctrl_qm_disable(abm); 193 194 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY; 195 return 0; 196 } 197 198 static void nfp_abm_eswitch_clean_up(struct nfp_abm *abm) 199 { 200 if (abm->eswitch_mode != DEVLINK_ESWITCH_MODE_LEGACY) 201 WARN_ON(nfp_abm_eswitch_set_legacy(abm)); 202 } 203 204 static int nfp_abm_eswitch_set_switchdev(struct nfp_abm *abm) 205 { 206 struct nfp_app *app = abm->app; 207 struct nfp_pf *pf = app->pf; 208 struct nfp_net *nn; 209 int err; 210 211 if (!abm->red_support) 212 return -EOPNOTSUPP; 213 214 err = nfp_abm_ctrl_qm_enable(abm); 215 if (err) 216 return err; 217 218 list_for_each_entry(nn, &pf->vnics, vnic_list) { 219 struct nfp_abm_link *alink = nn->app_priv; 220 221 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PHYS_PORT); 222 if (err) 223 goto err_kill_all_reprs; 224 225 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PF_PORT); 226 if (err) 227 goto err_kill_all_reprs; 228 } 229 230 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 231 return 0; 232 233 err_kill_all_reprs: 234 nfp_abm_kill_reprs_all(abm); 235 nfp_abm_ctrl_qm_disable(abm); 236 return err; 237 } 238 239 static int nfp_abm_eswitch_mode_set(struct nfp_app *app, u16 mode) 240 { 241 struct nfp_abm *abm = app->priv; 242 243 if (abm->eswitch_mode == mode) 244 return 0; 245 246 switch (mode) { 247 case DEVLINK_ESWITCH_MODE_LEGACY: 248 return nfp_abm_eswitch_set_legacy(abm); 249 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 250 return nfp_abm_eswitch_set_switchdev(abm); 251 default: 252 return -EINVAL; 253 } 254 } 255 256 static void 257 nfp_abm_vnic_set_mac(struct nfp_pf *pf, struct nfp_abm *abm, struct nfp_net *nn, 258 unsigned int id) 259 { 260 struct nfp_eth_table_port *eth_port = &pf->eth_tbl->ports[id]; 261 u8 mac_addr[ETH_ALEN]; 262 struct nfp_nsp *nsp; 263 char hwinfo[32]; 264 int err; 265 266 if (id > pf->eth_tbl->count) { 267 nfp_warn(pf->cpp, "No entry for persistent MAC address\n"); 268 eth_hw_addr_random(nn->dp.netdev); 269 return; 270 } 271 272 snprintf(hwinfo, sizeof(hwinfo), "eth%u.mac.pf%u", 273 eth_port->eth_index, abm->pf_id); 274 275 nsp = nfp_nsp_open(pf->cpp); 276 if (IS_ERR(nsp)) { 277 nfp_warn(pf->cpp, "Failed to access the NSP for persistent MAC address: %ld\n", 278 PTR_ERR(nsp)); 279 eth_hw_addr_random(nn->dp.netdev); 280 return; 281 } 282 283 if (!nfp_nsp_has_hwinfo_lookup(nsp)) { 284 nfp_warn(pf->cpp, "NSP doesn't support PF MAC generation\n"); 285 eth_hw_addr_random(nn->dp.netdev); 286 nfp_nsp_close(nsp); 287 return; 288 } 289 290 err = nfp_nsp_hwinfo_lookup(nsp, hwinfo, sizeof(hwinfo)); 291 nfp_nsp_close(nsp); 292 if (err) { 293 nfp_warn(pf->cpp, "Reading persistent MAC address failed: %d\n", 294 err); 295 eth_hw_addr_random(nn->dp.netdev); 296 return; 297 } 298 299 if (sscanf(hwinfo, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx", 300 &mac_addr[0], &mac_addr[1], &mac_addr[2], 301 &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) { 302 nfp_warn(pf->cpp, "Can't parse persistent MAC address (%s)\n", 303 hwinfo); 304 eth_hw_addr_random(nn->dp.netdev); 305 return; 306 } 307 308 ether_addr_copy(nn->dp.netdev->dev_addr, mac_addr); 309 ether_addr_copy(nn->dp.netdev->perm_addr, mac_addr); 310 } 311 312 static int 313 nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) 314 { 315 struct nfp_eth_table_port *eth_port = &app->pf->eth_tbl->ports[id]; 316 struct nfp_abm *abm = app->priv; 317 struct nfp_abm_link *alink; 318 int err; 319 320 alink = kzalloc(sizeof(*alink), GFP_KERNEL); 321 if (!alink) 322 return -ENOMEM; 323 nn->app_priv = alink; 324 alink->abm = abm; 325 alink->vnic = nn; 326 alink->id = id; 327 alink->total_queues = alink->vnic->max_rx_rings; 328 329 INIT_LIST_HEAD(&alink->dscp_map); 330 331 err = nfp_abm_ctrl_read_params(alink); 332 if (err) 333 goto err_free_alink; 334 335 alink->prio_map = kzalloc(abm->prio_map_len, GFP_KERNEL); 336 if (!alink->prio_map) { 337 err = -ENOMEM; 338 goto err_free_alink; 339 } 340 341 /* This is a multi-host app, make sure MAC/PHY is up, but don't 342 * make the MAC/PHY state follow the state of any of the ports. 343 */ 344 err = nfp_eth_set_configured(app->cpp, eth_port->index, true); 345 if (err < 0) 346 goto err_free_priomap; 347 348 netif_keep_dst(nn->dp.netdev); 349 350 nfp_abm_vnic_set_mac(app->pf, abm, nn, id); 351 INIT_RADIX_TREE(&alink->qdiscs, GFP_KERNEL); 352 353 return 0; 354 355 err_free_priomap: 356 kfree(alink->prio_map); 357 err_free_alink: 358 kfree(alink); 359 return err; 360 } 361 362 static void nfp_abm_vnic_free(struct nfp_app *app, struct nfp_net *nn) 363 { 364 struct nfp_abm_link *alink = nn->app_priv; 365 366 nfp_abm_kill_reprs(alink->abm, alink); 367 WARN(!radix_tree_empty(&alink->qdiscs), "left over qdiscs\n"); 368 kfree(alink->prio_map); 369 kfree(alink); 370 } 371 372 static int nfp_abm_vnic_init(struct nfp_app *app, struct nfp_net *nn) 373 { 374 struct nfp_abm_link *alink = nn->app_priv; 375 376 if (nfp_abm_has_prio(alink->abm)) 377 return nfp_abm_ctrl_prio_map_update(alink, alink->prio_map); 378 return 0; 379 } 380 381 static u64 * 382 nfp_abm_port_get_stats(struct nfp_app *app, struct nfp_port *port, u64 *data) 383 { 384 struct nfp_repr *repr = netdev_priv(port->netdev); 385 struct nfp_abm_link *alink; 386 unsigned int i; 387 388 if (port->type != NFP_PORT_PF_PORT) 389 return data; 390 alink = repr->app_priv; 391 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) { 392 *data++ = nfp_abm_ctrl_stat_non_sto(alink, i); 393 *data++ = nfp_abm_ctrl_stat_sto(alink, i); 394 } 395 return data; 396 } 397 398 static int 399 nfp_abm_port_get_stats_count(struct nfp_app *app, struct nfp_port *port) 400 { 401 struct nfp_repr *repr = netdev_priv(port->netdev); 402 struct nfp_abm_link *alink; 403 404 if (port->type != NFP_PORT_PF_PORT) 405 return 0; 406 alink = repr->app_priv; 407 return alink->vnic->dp.num_r_vecs * 2; 408 } 409 410 static u8 * 411 nfp_abm_port_get_stats_strings(struct nfp_app *app, struct nfp_port *port, 412 u8 *data) 413 { 414 struct nfp_repr *repr = netdev_priv(port->netdev); 415 struct nfp_abm_link *alink; 416 unsigned int i; 417 418 if (port->type != NFP_PORT_PF_PORT) 419 return data; 420 alink = repr->app_priv; 421 for (i = 0; i < alink->vnic->dp.num_r_vecs; i++) { 422 ethtool_sprintf(&data, "q%u_no_wait", i); 423 ethtool_sprintf(&data, "q%u_delayed", i); 424 } 425 return data; 426 } 427 428 static int nfp_abm_fw_init_reset(struct nfp_abm *abm) 429 { 430 unsigned int i; 431 432 if (!abm->red_support) 433 return 0; 434 435 for (i = 0; i < abm->num_bands * NFP_NET_MAX_RX_RINGS; i++) { 436 __nfp_abm_ctrl_set_q_lvl(abm, i, NFP_ABM_LVL_INFINITY); 437 __nfp_abm_ctrl_set_q_act(abm, i, NFP_ABM_ACT_DROP); 438 } 439 440 return nfp_abm_ctrl_qm_disable(abm); 441 } 442 443 static int nfp_abm_init(struct nfp_app *app) 444 { 445 struct nfp_pf *pf = app->pf; 446 struct nfp_reprs *reprs; 447 struct nfp_abm *abm; 448 int err; 449 450 if (!pf->eth_tbl) { 451 nfp_err(pf->cpp, "ABM NIC requires ETH table\n"); 452 return -EINVAL; 453 } 454 if (pf->max_data_vnics != pf->eth_tbl->count) { 455 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n", 456 pf->max_data_vnics, pf->eth_tbl->count); 457 return -EINVAL; 458 } 459 if (!pf->mac_stats_bar) { 460 nfp_warn(app->cpp, "ABM NIC requires mac_stats symbol\n"); 461 return -EINVAL; 462 } 463 464 abm = kzalloc(sizeof(*abm), GFP_KERNEL); 465 if (!abm) 466 return -ENOMEM; 467 app->priv = abm; 468 abm->app = app; 469 470 err = nfp_abm_ctrl_find_addrs(abm); 471 if (err) 472 goto err_free_abm; 473 474 err = -ENOMEM; 475 abm->num_thresholds = array_size(abm->num_bands, NFP_NET_MAX_RX_RINGS); 476 abm->threshold_undef = bitmap_zalloc(abm->num_thresholds, GFP_KERNEL); 477 if (!abm->threshold_undef) 478 goto err_free_abm; 479 480 abm->thresholds = kvcalloc(abm->num_thresholds, 481 sizeof(*abm->thresholds), GFP_KERNEL); 482 if (!abm->thresholds) 483 goto err_free_thresh_umap; 484 485 abm->actions = kvcalloc(abm->num_thresholds, sizeof(*abm->actions), 486 GFP_KERNEL); 487 if (!abm->actions) 488 goto err_free_thresh; 489 490 /* We start in legacy mode, make sure advanced queuing is disabled */ 491 err = nfp_abm_fw_init_reset(abm); 492 if (err) 493 goto err_free_act; 494 495 err = -ENOMEM; 496 reprs = nfp_reprs_alloc(pf->max_data_vnics); 497 if (!reprs) 498 goto err_free_act; 499 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PHYS_PORT], reprs); 500 501 reprs = nfp_reprs_alloc(pf->max_data_vnics); 502 if (!reprs) 503 goto err_free_phys; 504 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PF], reprs); 505 506 return 0; 507 508 err_free_phys: 509 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 510 err_free_act: 511 kvfree(abm->actions); 512 err_free_thresh: 513 kvfree(abm->thresholds); 514 err_free_thresh_umap: 515 bitmap_free(abm->threshold_undef); 516 err_free_abm: 517 kfree(abm); 518 app->priv = NULL; 519 return err; 520 } 521 522 static void nfp_abm_clean(struct nfp_app *app) 523 { 524 struct nfp_abm *abm = app->priv; 525 526 nfp_abm_eswitch_clean_up(abm); 527 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF); 528 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 529 bitmap_free(abm->threshold_undef); 530 kvfree(abm->actions); 531 kvfree(abm->thresholds); 532 kfree(abm); 533 app->priv = NULL; 534 } 535 536 const struct nfp_app_type app_abm = { 537 .id = NFP_APP_ACTIVE_BUFFER_MGMT_NIC, 538 .name = "abm", 539 540 .init = nfp_abm_init, 541 .clean = nfp_abm_clean, 542 543 .vnic_alloc = nfp_abm_vnic_alloc, 544 .vnic_free = nfp_abm_vnic_free, 545 .vnic_init = nfp_abm_vnic_init, 546 547 .port_get_stats = nfp_abm_port_get_stats, 548 .port_get_stats_count = nfp_abm_port_get_stats_count, 549 .port_get_stats_strings = nfp_abm_port_get_stats_strings, 550 551 .setup_tc = nfp_abm_setup_tc, 552 553 .eswitch_mode_get = nfp_abm_eswitch_mode_get, 554 .eswitch_mode_set = nfp_abm_eswitch_mode_set, 555 556 .dev_get = nfp_abm_repr_get, 557 }; 558