1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) 2 /* 3 * Copyright (C) 2018 Netronome Systems, Inc. 4 * 5 * This software is dual licensed under the GNU General License Version 2, 6 * June 1991 as shown in the file COPYING in the top-level directory of this 7 * source tree or the BSD 2-Clause License provided below. You have the 8 * option to license this software under the complete terms of either license. 9 * 10 * The BSD 2-Clause License: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * 1. Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * 2. Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include <linux/bitfield.h> 36 #include <linux/etherdevice.h> 37 #include <linux/lockdep.h> 38 #include <linux/netdevice.h> 39 #include <linux/rcupdate.h> 40 #include <linux/slab.h> 41 #include <net/pkt_cls.h> 42 #include <net/pkt_sched.h> 43 #include <net/red.h> 44 45 #include "../nfpcore/nfp.h" 46 #include "../nfpcore/nfp_cpp.h" 47 #include "../nfpcore/nfp_nsp.h" 48 #include "../nfp_app.h" 49 #include "../nfp_main.h" 50 #include "../nfp_net.h" 51 #include "../nfp_net_repr.h" 52 #include "../nfp_port.h" 53 #include "main.h" 54 55 static u32 nfp_abm_portid(enum nfp_repr_type rtype, unsigned int id) 56 { 57 return FIELD_PREP(NFP_ABM_PORTID_TYPE, rtype) | 58 FIELD_PREP(NFP_ABM_PORTID_ID, id); 59 } 60 61 static int nfp_abm_reset_stats(struct nfp_abm_link *alink) 62 { 63 int err; 64 65 err = nfp_abm_ctrl_read_stats(alink, &alink->qdiscs[0].stats); 66 if (err) 67 return err; 68 alink->qdiscs[0].stats.backlog_pkts = 0; 69 alink->qdiscs[0].stats.backlog_bytes = 0; 70 71 err = nfp_abm_ctrl_read_xstats(alink, &alink->qdiscs[0].xstats); 72 if (err) 73 return err; 74 75 return 0; 76 } 77 78 static void 79 nfp_abm_red_destroy(struct net_device *netdev, struct nfp_abm_link *alink, 80 u32 handle) 81 { 82 struct nfp_port *port = nfp_port_from_netdev(netdev); 83 84 if (handle != alink->qdiscs[0].handle) 85 return; 86 87 alink->qdiscs[0].handle = TC_H_UNSPEC; 88 port->tc_offload_cnt = 0; 89 nfp_abm_ctrl_set_all_q_lvls(alink, ~0); 90 } 91 92 static int 93 nfp_abm_red_replace(struct net_device *netdev, struct nfp_abm_link *alink, 94 struct tc_red_qopt_offload *opt) 95 { 96 struct nfp_port *port = nfp_port_from_netdev(netdev); 97 int err; 98 99 if (opt->set.min != opt->set.max || !opt->set.is_ecn) { 100 nfp_warn(alink->abm->app->cpp, 101 "RED offload failed - unsupported parameters\n"); 102 err = -EINVAL; 103 goto err_destroy; 104 } 105 err = nfp_abm_ctrl_set_all_q_lvls(alink, opt->set.min); 106 if (err) 107 goto err_destroy; 108 109 /* Reset stats only on new qdisc */ 110 if (alink->qdiscs[0].handle != opt->handle) { 111 err = nfp_abm_reset_stats(alink); 112 if (err) 113 goto err_destroy; 114 } 115 116 alink->qdiscs[0].handle = opt->handle; 117 port->tc_offload_cnt = 1; 118 119 return 0; 120 err_destroy: 121 /* If the qdisc keeps on living, but we can't offload undo changes */ 122 if (alink->qdiscs[0].handle == opt->handle) { 123 opt->set.qstats->qlen -= alink->qdiscs[0].stats.backlog_pkts; 124 opt->set.qstats->backlog -= 125 alink->qdiscs[0].stats.backlog_bytes; 126 } 127 if (alink->qdiscs[0].handle != TC_H_UNSPEC) 128 nfp_abm_red_destroy(netdev, alink, alink->qdiscs[0].handle); 129 return err; 130 } 131 132 static void 133 nfp_abm_update_stats(struct nfp_alink_stats *new, struct nfp_alink_stats *old, 134 struct tc_qopt_offload_stats *stats) 135 { 136 _bstats_update(stats->bstats, new->tx_bytes - old->tx_bytes, 137 new->tx_pkts - old->tx_pkts); 138 stats->qstats->qlen += new->backlog_pkts - old->backlog_pkts; 139 stats->qstats->backlog += new->backlog_bytes - old->backlog_bytes; 140 stats->qstats->overlimits += new->overlimits - old->overlimits; 141 stats->qstats->drops += new->drops - old->drops; 142 } 143 144 static int 145 nfp_abm_red_stats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt) 146 { 147 struct nfp_alink_stats *prev_stats; 148 struct nfp_alink_stats stats; 149 int err; 150 151 if (alink->qdiscs[0].handle != opt->handle) 152 return -EOPNOTSUPP; 153 prev_stats = &alink->qdiscs[0].stats; 154 155 err = nfp_abm_ctrl_read_stats(alink, &stats); 156 if (err) 157 return err; 158 159 nfp_abm_update_stats(&stats, prev_stats, &opt->stats); 160 161 *prev_stats = stats; 162 163 return 0; 164 } 165 166 static int 167 nfp_abm_red_xstats(struct nfp_abm_link *alink, struct tc_red_qopt_offload *opt) 168 { 169 struct nfp_alink_xstats *prev_xstats; 170 struct nfp_alink_xstats xstats; 171 int err; 172 173 if (alink->qdiscs[0].handle != opt->handle) 174 return -EOPNOTSUPP; 175 prev_xstats = &alink->qdiscs[0].xstats; 176 177 err = nfp_abm_ctrl_read_xstats(alink, &xstats); 178 if (err) 179 return err; 180 181 opt->xstats->forced_mark += xstats.ecn_marked - prev_xstats->ecn_marked; 182 opt->xstats->pdrop += xstats.pdrop - prev_xstats->pdrop; 183 184 *prev_xstats = xstats; 185 186 return 0; 187 } 188 189 static int 190 nfp_abm_setup_tc_red(struct net_device *netdev, struct nfp_abm_link *alink, 191 struct tc_red_qopt_offload *opt) 192 { 193 if (opt->parent != TC_H_ROOT) 194 return -EOPNOTSUPP; 195 196 switch (opt->command) { 197 case TC_RED_REPLACE: 198 return nfp_abm_red_replace(netdev, alink, opt); 199 case TC_RED_DESTROY: 200 nfp_abm_red_destroy(netdev, alink, opt->handle); 201 return 0; 202 case TC_RED_STATS: 203 return nfp_abm_red_stats(alink, opt); 204 case TC_RED_XSTATS: 205 return nfp_abm_red_xstats(alink, opt); 206 default: 207 return -EOPNOTSUPP; 208 } 209 } 210 211 static int 212 nfp_abm_setup_tc(struct nfp_app *app, struct net_device *netdev, 213 enum tc_setup_type type, void *type_data) 214 { 215 struct nfp_repr *repr = netdev_priv(netdev); 216 struct nfp_port *port; 217 218 port = nfp_port_from_netdev(netdev); 219 if (!port || port->type != NFP_PORT_PF_PORT) 220 return -EOPNOTSUPP; 221 222 switch (type) { 223 case TC_SETUP_QDISC_RED: 224 return nfp_abm_setup_tc_red(netdev, repr->app_priv, type_data); 225 default: 226 return -EOPNOTSUPP; 227 } 228 } 229 230 static struct net_device *nfp_abm_repr_get(struct nfp_app *app, u32 port_id) 231 { 232 enum nfp_repr_type rtype; 233 struct nfp_reprs *reprs; 234 u8 port; 235 236 rtype = FIELD_GET(NFP_ABM_PORTID_TYPE, port_id); 237 port = FIELD_GET(NFP_ABM_PORTID_ID, port_id); 238 239 reprs = rcu_dereference(app->reprs[rtype]); 240 if (!reprs) 241 return NULL; 242 243 if (port >= reprs->num_reprs) 244 return NULL; 245 246 return rcu_dereference(reprs->reprs[port]); 247 } 248 249 static int 250 nfp_abm_spawn_repr(struct nfp_app *app, struct nfp_abm_link *alink, 251 enum nfp_port_type ptype) 252 { 253 struct net_device *netdev; 254 enum nfp_repr_type rtype; 255 struct nfp_reprs *reprs; 256 struct nfp_repr *repr; 257 struct nfp_port *port; 258 int err; 259 260 if (ptype == NFP_PORT_PHYS_PORT) 261 rtype = NFP_REPR_TYPE_PHYS_PORT; 262 else 263 rtype = NFP_REPR_TYPE_PF; 264 265 netdev = nfp_repr_alloc(app); 266 if (!netdev) 267 return -ENOMEM; 268 repr = netdev_priv(netdev); 269 repr->app_priv = alink; 270 271 port = nfp_port_alloc(app, ptype, netdev); 272 if (IS_ERR(port)) { 273 err = PTR_ERR(port); 274 goto err_free_repr; 275 } 276 277 if (ptype == NFP_PORT_PHYS_PORT) { 278 port->eth_forced = true; 279 err = nfp_port_init_phy_port(app->pf, app, port, alink->id); 280 if (err) 281 goto err_free_port; 282 } else { 283 port->pf_id = alink->abm->pf_id; 284 port->pf_split = app->pf->max_data_vnics > 1; 285 port->pf_split_id = alink->id; 286 port->vnic = alink->vnic->dp.ctrl_bar; 287 } 288 289 SET_NETDEV_DEV(netdev, &alink->vnic->pdev->dev); 290 eth_hw_addr_random(netdev); 291 292 err = nfp_repr_init(app, netdev, nfp_abm_portid(rtype, alink->id), 293 port, alink->vnic->dp.netdev); 294 if (err) 295 goto err_free_port; 296 297 reprs = nfp_reprs_get_locked(app, rtype); 298 WARN(nfp_repr_get_locked(app, reprs, alink->id), "duplicate repr"); 299 rcu_assign_pointer(reprs->reprs[alink->id], netdev); 300 301 nfp_info(app->cpp, "%s Port %d Representor(%s) created\n", 302 ptype == NFP_PORT_PF_PORT ? "PCIe" : "Phys", 303 alink->id, netdev->name); 304 305 return 0; 306 307 err_free_port: 308 nfp_port_free(port); 309 err_free_repr: 310 nfp_repr_free(netdev); 311 return err; 312 } 313 314 static void 315 nfp_abm_kill_repr(struct nfp_app *app, struct nfp_abm_link *alink, 316 enum nfp_repr_type rtype) 317 { 318 struct net_device *netdev; 319 struct nfp_reprs *reprs; 320 321 reprs = nfp_reprs_get_locked(app, rtype); 322 netdev = nfp_repr_get_locked(app, reprs, alink->id); 323 if (!netdev) 324 return; 325 rcu_assign_pointer(reprs->reprs[alink->id], NULL); 326 synchronize_rcu(); 327 /* Cast to make sure nfp_repr_clean_and_free() takes a nfp_repr */ 328 nfp_repr_clean_and_free((struct nfp_repr *)netdev_priv(netdev)); 329 } 330 331 static void 332 nfp_abm_kill_reprs(struct nfp_abm *abm, struct nfp_abm_link *alink) 333 { 334 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PF); 335 nfp_abm_kill_repr(abm->app, alink, NFP_REPR_TYPE_PHYS_PORT); 336 } 337 338 static void nfp_abm_kill_reprs_all(struct nfp_abm *abm) 339 { 340 struct nfp_pf *pf = abm->app->pf; 341 struct nfp_net *nn; 342 343 list_for_each_entry(nn, &pf->vnics, vnic_list) 344 nfp_abm_kill_reprs(abm, (struct nfp_abm_link *)nn->app_priv); 345 } 346 347 static enum devlink_eswitch_mode nfp_abm_eswitch_mode_get(struct nfp_app *app) 348 { 349 struct nfp_abm *abm = app->priv; 350 351 return abm->eswitch_mode; 352 } 353 354 static int nfp_abm_eswitch_set_legacy(struct nfp_abm *abm) 355 { 356 nfp_abm_kill_reprs_all(abm); 357 nfp_abm_ctrl_qm_disable(abm); 358 359 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY; 360 return 0; 361 } 362 363 static void nfp_abm_eswitch_clean_up(struct nfp_abm *abm) 364 { 365 if (abm->eswitch_mode != DEVLINK_ESWITCH_MODE_LEGACY) 366 WARN_ON(nfp_abm_eswitch_set_legacy(abm)); 367 } 368 369 static int nfp_abm_eswitch_set_switchdev(struct nfp_abm *abm) 370 { 371 struct nfp_app *app = abm->app; 372 struct nfp_pf *pf = app->pf; 373 struct nfp_net *nn; 374 int err; 375 376 err = nfp_abm_ctrl_qm_enable(abm); 377 if (err) 378 return err; 379 380 list_for_each_entry(nn, &pf->vnics, vnic_list) { 381 struct nfp_abm_link *alink = nn->app_priv; 382 383 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PHYS_PORT); 384 if (err) 385 goto err_kill_all_reprs; 386 387 err = nfp_abm_spawn_repr(app, alink, NFP_PORT_PF_PORT); 388 if (err) 389 goto err_kill_all_reprs; 390 } 391 392 abm->eswitch_mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 393 return 0; 394 395 err_kill_all_reprs: 396 nfp_abm_kill_reprs_all(abm); 397 nfp_abm_ctrl_qm_disable(abm); 398 return err; 399 } 400 401 static int nfp_abm_eswitch_mode_set(struct nfp_app *app, u16 mode) 402 { 403 struct nfp_abm *abm = app->priv; 404 405 if (abm->eswitch_mode == mode) 406 return 0; 407 408 switch (mode) { 409 case DEVLINK_ESWITCH_MODE_LEGACY: 410 return nfp_abm_eswitch_set_legacy(abm); 411 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 412 return nfp_abm_eswitch_set_switchdev(abm); 413 default: 414 return -EINVAL; 415 } 416 } 417 418 static void 419 nfp_abm_vnic_set_mac(struct nfp_pf *pf, struct nfp_abm *abm, struct nfp_net *nn, 420 unsigned int id) 421 { 422 struct nfp_eth_table_port *eth_port = &pf->eth_tbl->ports[id]; 423 u8 mac_addr[ETH_ALEN]; 424 const char *mac_str; 425 char name[32]; 426 427 if (id > pf->eth_tbl->count) { 428 nfp_warn(pf->cpp, "No entry for persistent MAC address\n"); 429 eth_hw_addr_random(nn->dp.netdev); 430 return; 431 } 432 433 snprintf(name, sizeof(name), "eth%u.mac.pf%u", 434 eth_port->eth_index, abm->pf_id); 435 436 mac_str = nfp_hwinfo_lookup(pf->hwinfo, name); 437 if (!mac_str) { 438 nfp_warn(pf->cpp, "Can't lookup persistent MAC address (%s)\n", 439 name); 440 eth_hw_addr_random(nn->dp.netdev); 441 return; 442 } 443 444 if (sscanf(mac_str, "%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx", 445 &mac_addr[0], &mac_addr[1], &mac_addr[2], 446 &mac_addr[3], &mac_addr[4], &mac_addr[5]) != 6) { 447 nfp_warn(pf->cpp, "Can't parse persistent MAC address (%s)\n", 448 mac_str); 449 eth_hw_addr_random(nn->dp.netdev); 450 return; 451 } 452 453 ether_addr_copy(nn->dp.netdev->dev_addr, mac_addr); 454 ether_addr_copy(nn->dp.netdev->perm_addr, mac_addr); 455 } 456 457 static int 458 nfp_abm_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) 459 { 460 struct nfp_eth_table_port *eth_port = &app->pf->eth_tbl->ports[id]; 461 struct nfp_abm *abm = app->priv; 462 struct nfp_abm_link *alink; 463 int err; 464 465 alink = kzalloc(sizeof(*alink), GFP_KERNEL); 466 if (!alink) 467 return -ENOMEM; 468 nn->app_priv = alink; 469 alink->abm = abm; 470 alink->vnic = nn; 471 alink->id = id; 472 473 /* This is a multi-host app, make sure MAC/PHY is up, but don't 474 * make the MAC/PHY state follow the state of any of the ports. 475 */ 476 err = nfp_eth_set_configured(app->cpp, eth_port->index, true); 477 if (err < 0) 478 goto err_free_alink; 479 480 netif_keep_dst(nn->dp.netdev); 481 482 nfp_abm_vnic_set_mac(app->pf, abm, nn, id); 483 nfp_abm_ctrl_read_params(alink); 484 485 return 0; 486 487 err_free_alink: 488 kfree(alink); 489 return err; 490 } 491 492 static void nfp_abm_vnic_free(struct nfp_app *app, struct nfp_net *nn) 493 { 494 struct nfp_abm_link *alink = nn->app_priv; 495 496 nfp_abm_kill_reprs(alink->abm, alink); 497 kfree(alink); 498 } 499 500 static int nfp_abm_init(struct nfp_app *app) 501 { 502 struct nfp_pf *pf = app->pf; 503 struct nfp_reprs *reprs; 504 struct nfp_abm *abm; 505 int err; 506 507 if (!pf->eth_tbl) { 508 nfp_err(pf->cpp, "ABM NIC requires ETH table\n"); 509 return -EINVAL; 510 } 511 if (pf->max_data_vnics != pf->eth_tbl->count) { 512 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n", 513 pf->max_data_vnics, pf->eth_tbl->count); 514 return -EINVAL; 515 } 516 if (!pf->mac_stats_bar) { 517 nfp_warn(app->cpp, "ABM NIC requires mac_stats symbol\n"); 518 return -EINVAL; 519 } 520 521 abm = kzalloc(sizeof(*abm), GFP_KERNEL); 522 if (!abm) 523 return -ENOMEM; 524 app->priv = abm; 525 abm->app = app; 526 527 err = nfp_abm_ctrl_find_addrs(abm); 528 if (err) 529 goto err_free_abm; 530 531 /* We start in legacy mode, make sure advanced queuing is disabled */ 532 err = nfp_abm_ctrl_qm_disable(abm); 533 if (err) 534 goto err_free_abm; 535 536 err = -ENOMEM; 537 reprs = nfp_reprs_alloc(pf->max_data_vnics); 538 if (!reprs) 539 goto err_free_abm; 540 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PHYS_PORT], reprs); 541 542 reprs = nfp_reprs_alloc(pf->max_data_vnics); 543 if (!reprs) 544 goto err_free_phys; 545 RCU_INIT_POINTER(app->reprs[NFP_REPR_TYPE_PF], reprs); 546 547 return 0; 548 549 err_free_phys: 550 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 551 err_free_abm: 552 kfree(abm); 553 app->priv = NULL; 554 return err; 555 } 556 557 static void nfp_abm_clean(struct nfp_app *app) 558 { 559 struct nfp_abm *abm = app->priv; 560 561 nfp_abm_eswitch_clean_up(abm); 562 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF); 563 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 564 kfree(abm); 565 app->priv = NULL; 566 } 567 568 const struct nfp_app_type app_abm = { 569 .id = NFP_APP_ACTIVE_BUFFER_MGMT_NIC, 570 .name = "abm", 571 572 .init = nfp_abm_init, 573 .clean = nfp_abm_clean, 574 575 .vnic_alloc = nfp_abm_vnic_alloc, 576 .vnic_free = nfp_abm_vnic_free, 577 578 .setup_tc = nfp_abm_setup_tc, 579 580 .eswitch_mode_get = nfp_abm_eswitch_mode_get, 581 .eswitch_mode_set = nfp_abm_eswitch_mode_set, 582 583 .repr_get = nfp_abm_repr_get, 584 }; 585