1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */ 3 4 #include <linux/etherdevice.h> 5 #include <linux/lockdep.h> 6 #include <linux/pci.h> 7 #include <linux/skbuff.h> 8 #include <linux/vmalloc.h> 9 #include <net/devlink.h> 10 #include <net/dst_metadata.h> 11 12 #include "main.h" 13 #include "../nfpcore/nfp_cpp.h" 14 #include "../nfpcore/nfp_nffw.h" 15 #include "../nfpcore/nfp_nsp.h" 16 #include "../nfp_app.h" 17 #include "../nfp_main.h" 18 #include "../nfp_net.h" 19 #include "../nfp_net_repr.h" 20 #include "../nfp_port.h" 21 #include "./cmsg.h" 22 23 #define NFP_FLOWER_ALLOWED_VER 0x0001000000010000UL 24 25 #define NFP_MIN_INT_PORT_ID 1 26 #define NFP_MAX_INT_PORT_ID 256 27 28 static const char *nfp_flower_extra_cap(struct nfp_app *app, struct nfp_net *nn) 29 { 30 return "FLOWER"; 31 } 32 33 static enum devlink_eswitch_mode eswitch_mode_get(struct nfp_app *app) 34 { 35 return DEVLINK_ESWITCH_MODE_SWITCHDEV; 36 } 37 38 static int 39 nfp_flower_lookup_internal_port_id(struct nfp_flower_priv *priv, 40 struct net_device *netdev) 41 { 42 struct net_device *entry; 43 int i, id = 0; 44 45 rcu_read_lock(); 46 idr_for_each_entry(&priv->internal_ports.port_ids, entry, i) 47 if (entry == netdev) { 48 id = i; 49 break; 50 } 51 rcu_read_unlock(); 52 53 return id; 54 } 55 56 static int 57 nfp_flower_get_internal_port_id(struct nfp_app *app, struct net_device *netdev) 58 { 59 struct nfp_flower_priv *priv = app->priv; 60 int id; 61 62 id = nfp_flower_lookup_internal_port_id(priv, netdev); 63 if (id > 0) 64 return id; 65 66 idr_preload(GFP_ATOMIC); 67 spin_lock_bh(&priv->internal_ports.lock); 68 id = idr_alloc(&priv->internal_ports.port_ids, netdev, 69 NFP_MIN_INT_PORT_ID, NFP_MAX_INT_PORT_ID, GFP_ATOMIC); 70 spin_unlock_bh(&priv->internal_ports.lock); 71 idr_preload_end(); 72 73 return id; 74 } 75 76 u32 nfp_flower_get_port_id_from_netdev(struct nfp_app *app, 77 struct net_device *netdev) 78 { 79 struct nfp_flower_priv *priv = app->priv; 80 int ext_port; 81 int gid; 82 83 if (nfp_netdev_is_nfp_repr(netdev)) { 84 return nfp_repr_get_port_id(netdev); 85 } else if (nfp_flower_internal_port_can_offload(app, netdev)) { 86 ext_port = nfp_flower_get_internal_port_id(app, netdev); 87 if (ext_port < 0) 88 return 0; 89 90 return nfp_flower_internal_port_get_port_id(ext_port); 91 } else if (netif_is_lag_master(netdev) && 92 priv->flower_ext_feats & NFP_FL_FEATS_TUNNEL_NEIGH_LAG) { 93 gid = nfp_flower_lag_get_output_id(app, netdev); 94 if (gid < 0) 95 return 0; 96 97 return (NFP_FL_LAG_OUT | gid); 98 } 99 100 return 0; 101 } 102 103 static struct net_device * 104 nfp_flower_get_netdev_from_internal_port_id(struct nfp_app *app, int port_id) 105 { 106 struct nfp_flower_priv *priv = app->priv; 107 struct net_device *netdev; 108 109 rcu_read_lock(); 110 netdev = idr_find(&priv->internal_ports.port_ids, port_id); 111 rcu_read_unlock(); 112 113 return netdev; 114 } 115 116 static void 117 nfp_flower_free_internal_port_id(struct nfp_app *app, struct net_device *netdev) 118 { 119 struct nfp_flower_priv *priv = app->priv; 120 int id; 121 122 id = nfp_flower_lookup_internal_port_id(priv, netdev); 123 if (!id) 124 return; 125 126 spin_lock_bh(&priv->internal_ports.lock); 127 idr_remove(&priv->internal_ports.port_ids, id); 128 spin_unlock_bh(&priv->internal_ports.lock); 129 } 130 131 static int 132 nfp_flower_internal_port_event_handler(struct nfp_app *app, 133 struct net_device *netdev, 134 unsigned long event) 135 { 136 if (event == NETDEV_UNREGISTER && 137 nfp_flower_internal_port_can_offload(app, netdev)) 138 nfp_flower_free_internal_port_id(app, netdev); 139 140 return NOTIFY_OK; 141 } 142 143 static void nfp_flower_internal_port_init(struct nfp_flower_priv *priv) 144 { 145 spin_lock_init(&priv->internal_ports.lock); 146 idr_init(&priv->internal_ports.port_ids); 147 } 148 149 static void nfp_flower_internal_port_cleanup(struct nfp_flower_priv *priv) 150 { 151 idr_destroy(&priv->internal_ports.port_ids); 152 } 153 154 static struct nfp_flower_non_repr_priv * 155 nfp_flower_non_repr_priv_lookup(struct nfp_app *app, struct net_device *netdev) 156 { 157 struct nfp_flower_priv *priv = app->priv; 158 struct nfp_flower_non_repr_priv *entry; 159 160 ASSERT_RTNL(); 161 162 list_for_each_entry(entry, &priv->non_repr_priv, list) 163 if (entry->netdev == netdev) 164 return entry; 165 166 return NULL; 167 } 168 169 void 170 __nfp_flower_non_repr_priv_get(struct nfp_flower_non_repr_priv *non_repr_priv) 171 { 172 non_repr_priv->ref_count++; 173 } 174 175 struct nfp_flower_non_repr_priv * 176 nfp_flower_non_repr_priv_get(struct nfp_app *app, struct net_device *netdev) 177 { 178 struct nfp_flower_priv *priv = app->priv; 179 struct nfp_flower_non_repr_priv *entry; 180 181 entry = nfp_flower_non_repr_priv_lookup(app, netdev); 182 if (entry) 183 goto inc_ref; 184 185 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 186 if (!entry) 187 return NULL; 188 189 entry->netdev = netdev; 190 list_add(&entry->list, &priv->non_repr_priv); 191 192 inc_ref: 193 __nfp_flower_non_repr_priv_get(entry); 194 return entry; 195 } 196 197 void 198 __nfp_flower_non_repr_priv_put(struct nfp_flower_non_repr_priv *non_repr_priv) 199 { 200 if (--non_repr_priv->ref_count) 201 return; 202 203 list_del(&non_repr_priv->list); 204 kfree(non_repr_priv); 205 } 206 207 void 208 nfp_flower_non_repr_priv_put(struct nfp_app *app, struct net_device *netdev) 209 { 210 struct nfp_flower_non_repr_priv *entry; 211 212 entry = nfp_flower_non_repr_priv_lookup(app, netdev); 213 if (!entry) 214 return; 215 216 __nfp_flower_non_repr_priv_put(entry); 217 } 218 219 static enum nfp_repr_type 220 nfp_flower_repr_get_type_and_port(struct nfp_app *app, u32 port_id, u8 *port) 221 { 222 switch (FIELD_GET(NFP_FLOWER_CMSG_PORT_TYPE, port_id)) { 223 case NFP_FLOWER_CMSG_PORT_TYPE_PHYS_PORT: 224 *port = FIELD_GET(NFP_FLOWER_CMSG_PORT_PHYS_PORT_NUM, 225 port_id); 226 return NFP_REPR_TYPE_PHYS_PORT; 227 228 case NFP_FLOWER_CMSG_PORT_TYPE_PCIE_PORT: 229 *port = FIELD_GET(NFP_FLOWER_CMSG_PORT_VNIC, port_id); 230 if (FIELD_GET(NFP_FLOWER_CMSG_PORT_VNIC_TYPE, port_id) == 231 NFP_FLOWER_CMSG_PORT_VNIC_TYPE_PF) 232 return NFP_REPR_TYPE_PF; 233 else 234 return NFP_REPR_TYPE_VF; 235 } 236 237 return __NFP_REPR_TYPE_MAX; 238 } 239 240 static struct net_device * 241 nfp_flower_dev_get(struct nfp_app *app, u32 port_id, bool *redir_egress) 242 { 243 enum nfp_repr_type repr_type; 244 struct nfp_reprs *reprs; 245 u8 port = 0; 246 247 /* Check if the port is internal. */ 248 if (FIELD_GET(NFP_FLOWER_CMSG_PORT_TYPE, port_id) == 249 NFP_FLOWER_CMSG_PORT_TYPE_OTHER_PORT) { 250 if (redir_egress) 251 *redir_egress = true; 252 port = FIELD_GET(NFP_FLOWER_CMSG_PORT_PHYS_PORT_NUM, port_id); 253 return nfp_flower_get_netdev_from_internal_port_id(app, port); 254 } 255 256 repr_type = nfp_flower_repr_get_type_and_port(app, port_id, &port); 257 if (repr_type > NFP_REPR_TYPE_MAX) 258 return NULL; 259 260 reprs = rcu_dereference(app->reprs[repr_type]); 261 if (!reprs) 262 return NULL; 263 264 if (port >= reprs->num_reprs) 265 return NULL; 266 267 return rcu_dereference(reprs->reprs[port]); 268 } 269 270 static int 271 nfp_flower_reprs_reify(struct nfp_app *app, enum nfp_repr_type type, 272 bool exists) 273 { 274 struct nfp_reprs *reprs; 275 int i, err, count = 0; 276 277 reprs = rcu_dereference_protected(app->reprs[type], 278 nfp_app_is_locked(app)); 279 if (!reprs) 280 return 0; 281 282 for (i = 0; i < reprs->num_reprs; i++) { 283 struct net_device *netdev; 284 285 netdev = nfp_repr_get_locked(app, reprs, i); 286 if (netdev) { 287 struct nfp_repr *repr = netdev_priv(netdev); 288 289 err = nfp_flower_cmsg_portreify(repr, exists); 290 if (err) 291 return err; 292 count++; 293 } 294 } 295 296 return count; 297 } 298 299 static int 300 nfp_flower_wait_repr_reify(struct nfp_app *app, atomic_t *replies, int tot_repl) 301 { 302 struct nfp_flower_priv *priv = app->priv; 303 304 if (!tot_repl) 305 return 0; 306 307 assert_nfp_app_locked(app); 308 if (!wait_event_timeout(priv->reify_wait_queue, 309 atomic_read(replies) >= tot_repl, 310 NFP_FL_REPLY_TIMEOUT)) { 311 nfp_warn(app->cpp, "Not all reprs responded to reify\n"); 312 return -EIO; 313 } 314 315 return 0; 316 } 317 318 static int 319 nfp_flower_repr_netdev_open(struct nfp_app *app, struct nfp_repr *repr) 320 { 321 int err; 322 323 err = nfp_flower_cmsg_portmod(repr, true, repr->netdev->mtu, false); 324 if (err) 325 return err; 326 327 netif_tx_wake_all_queues(repr->netdev); 328 329 return 0; 330 } 331 332 static int 333 nfp_flower_repr_netdev_stop(struct nfp_app *app, struct nfp_repr *repr) 334 { 335 netif_tx_disable(repr->netdev); 336 337 return nfp_flower_cmsg_portmod(repr, false, repr->netdev->mtu, false); 338 } 339 340 static void 341 nfp_flower_repr_netdev_clean(struct nfp_app *app, struct net_device *netdev) 342 { 343 struct nfp_repr *repr = netdev_priv(netdev); 344 345 kfree(repr->app_priv); 346 } 347 348 static void 349 nfp_flower_repr_netdev_preclean(struct nfp_app *app, struct net_device *netdev) 350 { 351 struct nfp_repr *repr = netdev_priv(netdev); 352 struct nfp_flower_priv *priv = app->priv; 353 atomic_t *replies = &priv->reify_replies; 354 int err; 355 356 atomic_set(replies, 0); 357 err = nfp_flower_cmsg_portreify(repr, false); 358 if (err) { 359 nfp_warn(app->cpp, "Failed to notify firmware about repr destruction\n"); 360 return; 361 } 362 363 nfp_flower_wait_repr_reify(app, replies, 1); 364 } 365 366 static void nfp_flower_sriov_disable(struct nfp_app *app) 367 { 368 struct nfp_flower_priv *priv = app->priv; 369 370 if (!priv->nn) 371 return; 372 373 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_VF); 374 } 375 376 static int 377 nfp_flower_spawn_vnic_reprs(struct nfp_app *app, 378 enum nfp_flower_cmsg_port_vnic_type vnic_type, 379 enum nfp_repr_type repr_type, unsigned int cnt) 380 { 381 u8 nfp_pcie = nfp_cppcore_pcie_unit(app->pf->cpp); 382 struct nfp_flower_priv *priv = app->priv; 383 atomic_t *replies = &priv->reify_replies; 384 struct nfp_flower_repr_priv *repr_priv; 385 enum nfp_port_type port_type; 386 struct nfp_repr *nfp_repr; 387 struct nfp_reprs *reprs; 388 int i, err, reify_cnt; 389 const u8 queue = 0; 390 391 port_type = repr_type == NFP_REPR_TYPE_PF ? NFP_PORT_PF_PORT : 392 NFP_PORT_VF_PORT; 393 394 reprs = nfp_reprs_alloc(cnt); 395 if (!reprs) 396 return -ENOMEM; 397 398 for (i = 0; i < cnt; i++) { 399 struct net_device *repr; 400 struct nfp_port *port; 401 u32 port_id; 402 403 repr = nfp_repr_alloc(app); 404 if (!repr) { 405 err = -ENOMEM; 406 goto err_reprs_clean; 407 } 408 409 repr_priv = kzalloc(sizeof(*repr_priv), GFP_KERNEL); 410 if (!repr_priv) { 411 err = -ENOMEM; 412 nfp_repr_free(repr); 413 goto err_reprs_clean; 414 } 415 416 nfp_repr = netdev_priv(repr); 417 nfp_repr->app_priv = repr_priv; 418 repr_priv->nfp_repr = nfp_repr; 419 420 /* For now we only support 1 PF */ 421 WARN_ON(repr_type == NFP_REPR_TYPE_PF && i); 422 423 port = nfp_port_alloc(app, port_type, repr); 424 if (IS_ERR(port)) { 425 err = PTR_ERR(port); 426 kfree(repr_priv); 427 nfp_repr_free(repr); 428 goto err_reprs_clean; 429 } 430 if (repr_type == NFP_REPR_TYPE_PF) { 431 port->pf_id = i; 432 port->vnic = priv->nn->dp.ctrl_bar; 433 } else { 434 port->pf_id = 0; 435 port->vf_id = i; 436 port->vnic = 437 app->pf->vf_cfg_mem + i * NFP_NET_CFG_BAR_SZ; 438 } 439 440 eth_hw_addr_random(repr); 441 442 port_id = nfp_flower_cmsg_pcie_port(nfp_pcie, vnic_type, 443 i, queue); 444 err = nfp_repr_init(app, repr, 445 port_id, port, priv->nn->dp.netdev); 446 if (err) { 447 kfree(repr_priv); 448 nfp_port_free(port); 449 nfp_repr_free(repr); 450 goto err_reprs_clean; 451 } 452 453 RCU_INIT_POINTER(reprs->reprs[i], repr); 454 nfp_info(app->cpp, "%s%d Representor(%s) created\n", 455 repr_type == NFP_REPR_TYPE_PF ? "PF" : "VF", i, 456 repr->name); 457 } 458 459 nfp_app_reprs_set(app, repr_type, reprs); 460 461 atomic_set(replies, 0); 462 reify_cnt = nfp_flower_reprs_reify(app, repr_type, true); 463 if (reify_cnt < 0) { 464 err = reify_cnt; 465 nfp_warn(app->cpp, "Failed to notify firmware about repr creation\n"); 466 goto err_reprs_remove; 467 } 468 469 err = nfp_flower_wait_repr_reify(app, replies, reify_cnt); 470 if (err) 471 goto err_reprs_remove; 472 473 return 0; 474 err_reprs_remove: 475 reprs = nfp_app_reprs_set(app, repr_type, NULL); 476 err_reprs_clean: 477 nfp_reprs_clean_and_free(app, reprs); 478 return err; 479 } 480 481 static int nfp_flower_sriov_enable(struct nfp_app *app, int num_vfs) 482 { 483 struct nfp_flower_priv *priv = app->priv; 484 485 if (!priv->nn) 486 return 0; 487 488 return nfp_flower_spawn_vnic_reprs(app, 489 NFP_FLOWER_CMSG_PORT_VNIC_TYPE_VF, 490 NFP_REPR_TYPE_VF, num_vfs); 491 } 492 493 static int 494 nfp_flower_spawn_phy_reprs(struct nfp_app *app, struct nfp_flower_priv *priv) 495 { 496 struct nfp_eth_table *eth_tbl = app->pf->eth_tbl; 497 atomic_t *replies = &priv->reify_replies; 498 struct nfp_flower_repr_priv *repr_priv; 499 struct nfp_repr *nfp_repr; 500 struct sk_buff *ctrl_skb; 501 struct nfp_reprs *reprs; 502 int err, reify_cnt; 503 unsigned int i; 504 505 ctrl_skb = nfp_flower_cmsg_mac_repr_start(app, eth_tbl->count); 506 if (!ctrl_skb) 507 return -ENOMEM; 508 509 reprs = nfp_reprs_alloc(eth_tbl->max_index + 1); 510 if (!reprs) { 511 err = -ENOMEM; 512 goto err_free_ctrl_skb; 513 } 514 515 for (i = 0; i < eth_tbl->count; i++) { 516 unsigned int phys_port = eth_tbl->ports[i].index; 517 struct net_device *repr; 518 struct nfp_port *port; 519 u32 cmsg_port_id; 520 521 repr = nfp_repr_alloc(app); 522 if (!repr) { 523 err = -ENOMEM; 524 goto err_reprs_clean; 525 } 526 527 repr_priv = kzalloc(sizeof(*repr_priv), GFP_KERNEL); 528 if (!repr_priv) { 529 err = -ENOMEM; 530 nfp_repr_free(repr); 531 goto err_reprs_clean; 532 } 533 534 nfp_repr = netdev_priv(repr); 535 nfp_repr->app_priv = repr_priv; 536 repr_priv->nfp_repr = nfp_repr; 537 538 port = nfp_port_alloc(app, NFP_PORT_PHYS_PORT, repr); 539 if (IS_ERR(port)) { 540 err = PTR_ERR(port); 541 kfree(repr_priv); 542 nfp_repr_free(repr); 543 goto err_reprs_clean; 544 } 545 err = nfp_port_init_phy_port(app->pf, app, port, i); 546 if (err) { 547 kfree(repr_priv); 548 nfp_port_free(port); 549 nfp_repr_free(repr); 550 goto err_reprs_clean; 551 } 552 553 SET_NETDEV_DEV(repr, &priv->nn->pdev->dev); 554 nfp_net_get_mac_addr(app->pf, repr, port); 555 556 cmsg_port_id = nfp_flower_cmsg_phys_port(phys_port); 557 err = nfp_repr_init(app, repr, 558 cmsg_port_id, port, priv->nn->dp.netdev); 559 if (err) { 560 kfree(repr_priv); 561 nfp_port_free(port); 562 nfp_repr_free(repr); 563 goto err_reprs_clean; 564 } 565 566 nfp_flower_cmsg_mac_repr_add(ctrl_skb, i, 567 eth_tbl->ports[i].nbi, 568 eth_tbl->ports[i].base, 569 phys_port); 570 571 RCU_INIT_POINTER(reprs->reprs[phys_port], repr); 572 nfp_info(app->cpp, "Phys Port %d Representor(%s) created\n", 573 phys_port, repr->name); 574 } 575 576 nfp_app_reprs_set(app, NFP_REPR_TYPE_PHYS_PORT, reprs); 577 578 /* The REIFY/MAC_REPR control messages should be sent after the MAC 579 * representors are registered using nfp_app_reprs_set(). This is 580 * because the firmware may respond with control messages for the 581 * MAC representors, f.e. to provide the driver with information 582 * about their state, and without registration the driver will drop 583 * any such messages. 584 */ 585 atomic_set(replies, 0); 586 reify_cnt = nfp_flower_reprs_reify(app, NFP_REPR_TYPE_PHYS_PORT, true); 587 if (reify_cnt < 0) { 588 err = reify_cnt; 589 nfp_warn(app->cpp, "Failed to notify firmware about repr creation\n"); 590 goto err_reprs_remove; 591 } 592 593 err = nfp_flower_wait_repr_reify(app, replies, reify_cnt); 594 if (err) 595 goto err_reprs_remove; 596 597 nfp_ctrl_tx(app->ctrl, ctrl_skb); 598 599 return 0; 600 err_reprs_remove: 601 reprs = nfp_app_reprs_set(app, NFP_REPR_TYPE_PHYS_PORT, NULL); 602 err_reprs_clean: 603 nfp_reprs_clean_and_free(app, reprs); 604 err_free_ctrl_skb: 605 kfree_skb(ctrl_skb); 606 return err; 607 } 608 609 static int nfp_flower_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, 610 unsigned int id) 611 { 612 if (id > 0) { 613 nfp_warn(app->cpp, "FlowerNIC doesn't support more than one data vNIC\n"); 614 goto err_invalid_port; 615 } 616 617 eth_hw_addr_random(nn->dp.netdev); 618 netif_keep_dst(nn->dp.netdev); 619 nn->vnic_no_name = true; 620 621 return 0; 622 623 err_invalid_port: 624 nn->port = nfp_port_alloc(app, NFP_PORT_INVALID, nn->dp.netdev); 625 return PTR_ERR_OR_ZERO(nn->port); 626 } 627 628 static void nfp_flower_vnic_clean(struct nfp_app *app, struct nfp_net *nn) 629 { 630 struct nfp_flower_priv *priv = app->priv; 631 632 if (app->pf->num_vfs) 633 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_VF); 634 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF); 635 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 636 637 priv->nn = NULL; 638 } 639 640 static int nfp_flower_vnic_init(struct nfp_app *app, struct nfp_net *nn) 641 { 642 struct nfp_flower_priv *priv = app->priv; 643 int err; 644 645 priv->nn = nn; 646 647 err = nfp_flower_spawn_phy_reprs(app, app->priv); 648 if (err) 649 goto err_clear_nn; 650 651 err = nfp_flower_spawn_vnic_reprs(app, 652 NFP_FLOWER_CMSG_PORT_VNIC_TYPE_PF, 653 NFP_REPR_TYPE_PF, 1); 654 if (err) 655 goto err_destroy_reprs_phy; 656 657 if (app->pf->num_vfs) { 658 err = nfp_flower_spawn_vnic_reprs(app, 659 NFP_FLOWER_CMSG_PORT_VNIC_TYPE_VF, 660 NFP_REPR_TYPE_VF, 661 app->pf->num_vfs); 662 if (err) 663 goto err_destroy_reprs_pf; 664 } 665 666 return 0; 667 668 err_destroy_reprs_pf: 669 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PF); 670 err_destroy_reprs_phy: 671 nfp_reprs_clean_and_free_by_type(app, NFP_REPR_TYPE_PHYS_PORT); 672 err_clear_nn: 673 priv->nn = NULL; 674 return err; 675 } 676 677 static void nfp_flower_wait_host_bit(struct nfp_app *app) 678 { 679 unsigned long err_at; 680 u64 feat; 681 int err; 682 683 /* Wait for HOST_ACK flag bit to propagate */ 684 err_at = jiffies + msecs_to_jiffies(100); 685 do { 686 feat = nfp_rtsym_read_le(app->pf->rtbl, 687 "_abi_flower_combined_features_global", 688 &err); 689 if (time_is_before_eq_jiffies(err_at)) { 690 nfp_warn(app->cpp, 691 "HOST_ACK bit not propagated in FW.\n"); 692 break; 693 } 694 usleep_range(1000, 2000); 695 } while (!err && !(feat & NFP_FL_FEATS_HOST_ACK)); 696 697 if (err) 698 nfp_warn(app->cpp, 699 "Could not read global features entry from FW\n"); 700 } 701 702 static int nfp_flower_sync_feature_bits(struct nfp_app *app) 703 { 704 struct nfp_flower_priv *app_priv = app->priv; 705 int err; 706 707 /* Tell the firmware of the host supported features. */ 708 err = nfp_rtsym_write_le(app->pf->rtbl, "_abi_flower_host_mask", 709 app_priv->flower_ext_feats | 710 NFP_FL_FEATS_HOST_ACK); 711 if (!err) 712 nfp_flower_wait_host_bit(app); 713 else if (err != -ENOENT) 714 return err; 715 716 /* Tell the firmware that the driver supports lag. */ 717 err = nfp_rtsym_write_le(app->pf->rtbl, 718 "_abi_flower_balance_sync_enable", 1); 719 if (!err) { 720 app_priv->flower_en_feats |= NFP_FL_ENABLE_LAG; 721 nfp_flower_lag_init(&app_priv->nfp_lag); 722 } else if (err == -ENOENT) { 723 nfp_warn(app->cpp, "LAG not supported by FW.\n"); 724 } else { 725 return err; 726 } 727 728 if (app_priv->flower_ext_feats & NFP_FL_FEATS_FLOW_MOD) { 729 /* Tell the firmware that the driver supports flow merging. */ 730 err = nfp_rtsym_write_le(app->pf->rtbl, 731 "_abi_flower_merge_hint_enable", 1); 732 if (!err) { 733 app_priv->flower_en_feats |= NFP_FL_ENABLE_FLOW_MERGE; 734 nfp_flower_internal_port_init(app_priv); 735 } else if (err == -ENOENT) { 736 nfp_warn(app->cpp, 737 "Flow merge not supported by FW.\n"); 738 } else { 739 return err; 740 } 741 } else { 742 nfp_warn(app->cpp, "Flow mod/merge not supported by FW.\n"); 743 } 744 745 return 0; 746 } 747 748 static int nfp_flower_init(struct nfp_app *app) 749 { 750 u64 version, features, ctx_count, num_mems; 751 const struct nfp_pf *pf = app->pf; 752 struct nfp_flower_priv *app_priv; 753 int err; 754 755 if (!pf->eth_tbl) { 756 nfp_warn(app->cpp, "FlowerNIC requires eth table\n"); 757 return -EINVAL; 758 } 759 760 if (!pf->mac_stats_bar) { 761 nfp_warn(app->cpp, "FlowerNIC requires mac_stats BAR\n"); 762 return -EINVAL; 763 } 764 765 if (!pf->vf_cfg_bar) { 766 nfp_warn(app->cpp, "FlowerNIC requires vf_cfg BAR\n"); 767 return -EINVAL; 768 } 769 770 version = nfp_rtsym_read_le(app->pf->rtbl, "hw_flower_version", &err); 771 if (err) { 772 nfp_warn(app->cpp, "FlowerNIC requires hw_flower_version memory symbol\n"); 773 return err; 774 } 775 776 num_mems = nfp_rtsym_read_le(app->pf->rtbl, "CONFIG_FC_HOST_CTX_SPLIT", 777 &err); 778 if (err) { 779 nfp_warn(app->cpp, 780 "FlowerNIC: unsupported host context memory: %d\n", 781 err); 782 err = 0; 783 num_mems = 1; 784 } 785 786 if (!FIELD_FIT(NFP_FL_STAT_ID_MU_NUM, num_mems) || !num_mems) { 787 nfp_warn(app->cpp, 788 "FlowerNIC: invalid host context memory: %llu\n", 789 num_mems); 790 return -EINVAL; 791 } 792 793 ctx_count = nfp_rtsym_read_le(app->pf->rtbl, "CONFIG_FC_HOST_CTX_COUNT", 794 &err); 795 if (err) { 796 nfp_warn(app->cpp, 797 "FlowerNIC: unsupported host context count: %d\n", 798 err); 799 err = 0; 800 ctx_count = BIT(17); 801 } 802 803 /* We need to ensure hardware has enough flower capabilities. */ 804 if (version != NFP_FLOWER_ALLOWED_VER) { 805 nfp_warn(app->cpp, "FlowerNIC: unsupported firmware version\n"); 806 return -EINVAL; 807 } 808 809 app_priv = vzalloc(sizeof(struct nfp_flower_priv)); 810 if (!app_priv) 811 return -ENOMEM; 812 813 app_priv->total_mem_units = num_mems; 814 app_priv->active_mem_unit = 0; 815 app_priv->stats_ring_size = roundup_pow_of_two(ctx_count); 816 app->priv = app_priv; 817 app_priv->app = app; 818 skb_queue_head_init(&app_priv->cmsg_skbs_high); 819 skb_queue_head_init(&app_priv->cmsg_skbs_low); 820 INIT_WORK(&app_priv->cmsg_work, nfp_flower_cmsg_process_rx); 821 init_waitqueue_head(&app_priv->reify_wait_queue); 822 823 init_waitqueue_head(&app_priv->mtu_conf.wait_q); 824 spin_lock_init(&app_priv->mtu_conf.lock); 825 826 err = nfp_flower_metadata_init(app, ctx_count, num_mems); 827 if (err) 828 goto err_free_app_priv; 829 830 /* Extract the extra features supported by the firmware. */ 831 features = nfp_rtsym_read_le(app->pf->rtbl, 832 "_abi_flower_extra_features", &err); 833 if (err) 834 app_priv->flower_ext_feats = 0; 835 else 836 app_priv->flower_ext_feats = features & NFP_FL_FEATS_HOST; 837 838 err = nfp_flower_sync_feature_bits(app); 839 if (err) 840 goto err_cleanup; 841 842 if (app_priv->flower_ext_feats & NFP_FL_FEATS_VF_RLIM) 843 nfp_flower_qos_init(app); 844 845 INIT_LIST_HEAD(&app_priv->indr_block_cb_priv); 846 INIT_LIST_HEAD(&app_priv->non_repr_priv); 847 app_priv->pre_tun_rule_cnt = 0; 848 849 return 0; 850 851 err_cleanup: 852 if (app_priv->flower_en_feats & NFP_FL_ENABLE_LAG) 853 nfp_flower_lag_cleanup(&app_priv->nfp_lag); 854 nfp_flower_metadata_cleanup(app); 855 err_free_app_priv: 856 vfree(app->priv); 857 return err; 858 } 859 860 static void nfp_flower_clean(struct nfp_app *app) 861 { 862 struct nfp_flower_priv *app_priv = app->priv; 863 864 skb_queue_purge(&app_priv->cmsg_skbs_high); 865 skb_queue_purge(&app_priv->cmsg_skbs_low); 866 flush_work(&app_priv->cmsg_work); 867 868 if (app_priv->flower_ext_feats & NFP_FL_FEATS_VF_RLIM) 869 nfp_flower_qos_cleanup(app); 870 871 if (app_priv->flower_en_feats & NFP_FL_ENABLE_LAG) 872 nfp_flower_lag_cleanup(&app_priv->nfp_lag); 873 874 if (app_priv->flower_en_feats & NFP_FL_ENABLE_FLOW_MERGE) 875 nfp_flower_internal_port_cleanup(app_priv); 876 877 nfp_flower_metadata_cleanup(app); 878 vfree(app->priv); 879 app->priv = NULL; 880 } 881 882 static bool nfp_flower_check_ack(struct nfp_flower_priv *app_priv) 883 { 884 bool ret; 885 886 spin_lock_bh(&app_priv->mtu_conf.lock); 887 ret = app_priv->mtu_conf.ack; 888 spin_unlock_bh(&app_priv->mtu_conf.lock); 889 890 return ret; 891 } 892 893 static int 894 nfp_flower_repr_change_mtu(struct nfp_app *app, struct net_device *netdev, 895 int new_mtu) 896 { 897 struct nfp_flower_priv *app_priv = app->priv; 898 struct nfp_repr *repr = netdev_priv(netdev); 899 int err; 900 901 /* Only need to config FW for physical port MTU change. */ 902 if (repr->port->type != NFP_PORT_PHYS_PORT) 903 return 0; 904 905 if (!(app_priv->flower_ext_feats & NFP_FL_NBI_MTU_SETTING)) { 906 nfp_err(app->cpp, "Physical port MTU setting not supported\n"); 907 return -EINVAL; 908 } 909 910 spin_lock_bh(&app_priv->mtu_conf.lock); 911 app_priv->mtu_conf.ack = false; 912 app_priv->mtu_conf.requested_val = new_mtu; 913 app_priv->mtu_conf.portnum = repr->dst->u.port_info.port_id; 914 spin_unlock_bh(&app_priv->mtu_conf.lock); 915 916 err = nfp_flower_cmsg_portmod(repr, netif_carrier_ok(netdev), new_mtu, 917 true); 918 if (err) { 919 spin_lock_bh(&app_priv->mtu_conf.lock); 920 app_priv->mtu_conf.requested_val = 0; 921 spin_unlock_bh(&app_priv->mtu_conf.lock); 922 return err; 923 } 924 925 /* Wait for fw to ack the change. */ 926 if (!wait_event_timeout(app_priv->mtu_conf.wait_q, 927 nfp_flower_check_ack(app_priv), 928 NFP_FL_REPLY_TIMEOUT)) { 929 spin_lock_bh(&app_priv->mtu_conf.lock); 930 app_priv->mtu_conf.requested_val = 0; 931 spin_unlock_bh(&app_priv->mtu_conf.lock); 932 nfp_warn(app->cpp, "MTU change not verified with fw\n"); 933 return -EIO; 934 } 935 936 return 0; 937 } 938 939 static int nfp_flower_start(struct nfp_app *app) 940 { 941 struct nfp_flower_priv *app_priv = app->priv; 942 int err; 943 944 if (app_priv->flower_en_feats & NFP_FL_ENABLE_LAG) { 945 err = nfp_flower_lag_reset(&app_priv->nfp_lag); 946 if (err) 947 return err; 948 } 949 950 err = flow_indr_dev_register(nfp_flower_indr_setup_tc_cb, app); 951 if (err) 952 return err; 953 954 err = nfp_tunnel_config_start(app); 955 if (err) 956 goto err_tunnel_config; 957 958 return 0; 959 960 err_tunnel_config: 961 flow_indr_dev_unregister(nfp_flower_indr_setup_tc_cb, app, 962 nfp_flower_setup_indr_tc_release); 963 return err; 964 } 965 966 static void nfp_flower_stop(struct nfp_app *app) 967 { 968 nfp_tunnel_config_stop(app); 969 970 flow_indr_dev_unregister(nfp_flower_indr_setup_tc_cb, app, 971 nfp_flower_setup_indr_tc_release); 972 } 973 974 static int 975 nfp_flower_netdev_event(struct nfp_app *app, struct net_device *netdev, 976 unsigned long event, void *ptr) 977 { 978 struct nfp_flower_priv *app_priv = app->priv; 979 int ret; 980 981 if (app_priv->flower_en_feats & NFP_FL_ENABLE_LAG) { 982 ret = nfp_flower_lag_netdev_event(app_priv, netdev, event, ptr); 983 if (ret & NOTIFY_STOP_MASK) 984 return ret; 985 } 986 987 ret = nfp_flower_internal_port_event_handler(app, netdev, event); 988 if (ret & NOTIFY_STOP_MASK) 989 return ret; 990 991 return nfp_tunnel_mac_event_handler(app, netdev, event, ptr); 992 } 993 994 const struct nfp_app_type app_flower = { 995 .id = NFP_APP_FLOWER_NIC, 996 .name = "flower", 997 998 .ctrl_cap_mask = ~0U, 999 .ctrl_has_meta = true, 1000 1001 .extra_cap = nfp_flower_extra_cap, 1002 1003 .init = nfp_flower_init, 1004 .clean = nfp_flower_clean, 1005 1006 .repr_change_mtu = nfp_flower_repr_change_mtu, 1007 1008 .vnic_alloc = nfp_flower_vnic_alloc, 1009 .vnic_init = nfp_flower_vnic_init, 1010 .vnic_clean = nfp_flower_vnic_clean, 1011 1012 .repr_preclean = nfp_flower_repr_netdev_preclean, 1013 .repr_clean = nfp_flower_repr_netdev_clean, 1014 1015 .repr_open = nfp_flower_repr_netdev_open, 1016 .repr_stop = nfp_flower_repr_netdev_stop, 1017 1018 .start = nfp_flower_start, 1019 .stop = nfp_flower_stop, 1020 1021 .netdev_event = nfp_flower_netdev_event, 1022 1023 .ctrl_msg_rx = nfp_flower_cmsg_rx, 1024 1025 .sriov_enable = nfp_flower_sriov_enable, 1026 .sriov_disable = nfp_flower_sriov_disable, 1027 1028 .eswitch_mode_get = eswitch_mode_get, 1029 .dev_get = nfp_flower_dev_get, 1030 1031 .setup_tc = nfp_flower_setup_tc, 1032 }; 1033