1 /* Broadcom NetXtreme-C/E network driver. 2 * 3 * Copyright (c) 2016-2018 Broadcom Limited 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation. 8 */ 9 10 #include <linux/module.h> 11 12 #include <linux/kernel.h> 13 #include <linux/errno.h> 14 #include <linux/interrupt.h> 15 #include <linux/pci.h> 16 #include <linux/netdevice.h> 17 #include <linux/rtnetlink.h> 18 #include <linux/bitops.h> 19 #include <linux/irq.h> 20 #include <asm/byteorder.h> 21 #include <linux/bitmap.h> 22 #include <linux/auxiliary_bus.h> 23 #include <net/netdev_lock.h> 24 #include <linux/bnxt/hsi.h> 25 #include <linux/bnxt/ulp.h> 26 27 #include "bnxt.h" 28 #include "bnxt_hwrm.h" 29 30 static DEFINE_IDA(bnxt_aux_dev_ids); 31 32 struct bnxt_aux_device { 33 const char *name; 34 }; 35 36 static void bnxt_auxdev_set_state(struct bnxt *bp, int idx, int state) 37 { 38 bp->auxdev_state[idx] = state; 39 } 40 41 static bool bnxt_auxdev_is_init(struct bnxt *bp, int idx) 42 { 43 return (bp->auxdev_state[idx] == BNXT_ADEV_STATE_INIT); 44 } 45 46 static bool bnxt_auxdev_is_active(struct bnxt *bp, int idx) 47 { 48 return (bp->auxdev_state[idx] == BNXT_ADEV_STATE_ADD); 49 } 50 51 static struct bnxt_aux_device bnxt_aux_devices[__BNXT_AUXDEV_MAX] = {{ 52 .name = "rdma", 53 }, { 54 .name = "fwctl", 55 }}; 56 57 static void bnxt_fill_msix_vecs(struct bnxt *bp, struct bnxt_msix_entry *ent) 58 { 59 struct bnxt_en_dev *edev = bp->edev[BNXT_AUXDEV_RDMA]; 60 int num_msix, i; 61 62 if (!edev->ulp_tbl->msix_requested) { 63 netdev_warn(bp->dev, "Requested MSI-X vectors insufficient\n"); 64 return; 65 } 66 num_msix = edev->ulp_tbl->msix_requested; 67 for (i = 0; i < num_msix; i++) { 68 ent[i].vector = bp->irq_tbl[i].vector; 69 ent[i].ring_idx = i; 70 if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) 71 ent[i].db_offset = bp->db_offset; 72 else 73 ent[i].db_offset = i * 0x80; 74 } 75 } 76 77 int bnxt_get_ulp_msix_num(struct bnxt *bp) 78 { 79 struct bnxt_en_dev *edev = bp->edev[BNXT_AUXDEV_RDMA]; 80 81 if (edev) 82 return edev->ulp_num_msix_vec; 83 return 0; 84 } 85 86 void bnxt_set_ulp_msix_num(struct bnxt *bp, int num) 87 { 88 struct bnxt_en_dev *edev = bp->edev[BNXT_AUXDEV_RDMA]; 89 90 if (edev) 91 edev->ulp_num_msix_vec = num; 92 } 93 94 int bnxt_get_ulp_msix_num_in_use(struct bnxt *bp) 95 { 96 struct bnxt_en_dev *edev = bp->edev[BNXT_AUXDEV_RDMA]; 97 98 if (bnxt_ulp_registered(edev)) 99 return edev->ulp_num_msix_vec; 100 return 0; 101 } 102 103 int bnxt_get_ulp_stat_ctxs(struct bnxt *bp) 104 { 105 struct bnxt_en_dev *edev = bp->edev[BNXT_AUXDEV_RDMA]; 106 107 if (edev) 108 return edev->ulp_num_ctxs; 109 return 0; 110 } 111 112 void bnxt_set_ulp_stat_ctxs(struct bnxt *bp, int num_ulp_ctx) 113 { 114 struct bnxt_en_dev *edev = bp->edev[BNXT_AUXDEV_RDMA]; 115 116 if (edev) 117 edev->ulp_num_ctxs = num_ulp_ctx; 118 } 119 120 int bnxt_get_ulp_stat_ctxs_in_use(struct bnxt *bp) 121 { 122 struct bnxt_en_dev *edev = bp->edev[BNXT_AUXDEV_RDMA]; 123 124 if (bnxt_ulp_registered(edev)) 125 return edev->ulp_num_ctxs; 126 return 0; 127 } 128 129 void bnxt_set_dflt_ulp_stat_ctxs(struct bnxt *bp) 130 { 131 struct bnxt_en_dev *edev = bp->edev[BNXT_AUXDEV_RDMA]; 132 133 if (edev) { 134 edev->ulp_num_ctxs = BNXT_MIN_ROCE_STAT_CTXS; 135 /* Reserve one additional stat_ctx for PF0 (except 136 * on 1-port NICs) as it also creates one stat_ctx 137 * for PF1 in case of RoCE bonding. 138 */ 139 if (BNXT_PF(bp) && !bp->pf.port_id && 140 bp->port_count > 1) 141 edev->ulp_num_ctxs++; 142 143 /* Reserve one additional stat_ctx when the device is capable 144 * of supporting port mirroring on RDMA device. 145 */ 146 if (BNXT_MIRROR_ON_ROCE_CAP(bp)) 147 edev->ulp_num_ctxs++; 148 } 149 } 150 151 int bnxt_register_dev(struct bnxt_en_dev *edev, 152 struct bnxt_ulp_ops *ulp_ops, 153 void *handle) 154 { 155 struct net_device *dev = edev->net; 156 struct bnxt *bp = netdev_priv(dev); 157 unsigned int max_stat_ctxs; 158 struct bnxt_ulp *ulp; 159 int rc = 0; 160 161 netdev_lock(dev); 162 mutex_lock(&edev->en_dev_lock); 163 if (!bp->irq_tbl) { 164 rc = -ENODEV; 165 goto exit; 166 } 167 max_stat_ctxs = bnxt_get_max_func_stat_ctxs(bp); 168 if (max_stat_ctxs <= BNXT_MIN_ROCE_STAT_CTXS || 169 bp->cp_nr_rings == max_stat_ctxs) { 170 rc = -ENOMEM; 171 goto exit; 172 } 173 174 ulp = edev->ulp_tbl; 175 ulp->handle = handle; 176 rcu_assign_pointer(ulp->ulp_ops, ulp_ops); 177 178 if (test_bit(BNXT_STATE_OPEN, &bp->state)) { 179 rc = bnxt_hwrm_vnic_cfg(bp, &bp->vnic_info[BNXT_VNIC_DEFAULT]); 180 if (rc) { 181 netdev_err(dev, "Failed to configure dual VNIC mode\n"); 182 RCU_INIT_POINTER(ulp->ulp_ops, NULL); 183 goto exit; 184 } 185 } 186 187 edev->ulp_tbl->msix_requested = bnxt_get_ulp_msix_num(bp); 188 189 bnxt_fill_msix_vecs(bp, edev->msix_entries); 190 exit: 191 mutex_unlock(&edev->en_dev_lock); 192 netdev_unlock(dev); 193 return rc; 194 } 195 EXPORT_SYMBOL(bnxt_register_dev); 196 197 void bnxt_unregister_dev(struct bnxt_en_dev *edev) 198 { 199 struct net_device *dev = edev->net; 200 struct bnxt *bp = netdev_priv(dev); 201 struct bnxt_ulp *ulp; 202 203 ulp = edev->ulp_tbl; 204 netdev_lock(dev); 205 mutex_lock(&edev->en_dev_lock); 206 edev->ulp_tbl->msix_requested = 0; 207 208 if (ulp->max_async_event_id) 209 bnxt_hwrm_func_drv_rgtr(bp, NULL, 0, true); 210 211 RCU_INIT_POINTER(ulp->ulp_ops, NULL); 212 synchronize_rcu(); 213 ulp->max_async_event_id = 0; 214 ulp->async_events_bmap = NULL; 215 mutex_unlock(&edev->en_dev_lock); 216 netdev_unlock(dev); 217 return; 218 } 219 EXPORT_SYMBOL(bnxt_unregister_dev); 220 221 static int bnxt_set_dflt_ulp_msix(struct bnxt *bp) 222 { 223 int roce_msix = BNXT_MAX_ROCE_MSIX; 224 225 if (BNXT_VF(bp)) 226 roce_msix = BNXT_MAX_ROCE_MSIX_VF; 227 else if (bp->port_partition_type) 228 roce_msix = BNXT_MAX_ROCE_MSIX_NPAR_PF; 229 230 /* NQ MSIX vectors should match the number of CPUs plus 1 more for 231 * the CREQ MSIX, up to the default. 232 */ 233 return min_t(int, roce_msix, num_online_cpus() + 1); 234 } 235 236 int bnxt_send_msg(struct bnxt_en_dev *edev, 237 struct bnxt_fw_msg *fw_msg) 238 { 239 struct net_device *dev = edev->net; 240 struct bnxt *bp = netdev_priv(dev); 241 struct output *resp; 242 struct input *req; 243 u32 resp_len; 244 int rc; 245 246 if (bp->fw_reset_state) 247 return -EBUSY; 248 249 rc = hwrm_req_init(bp, req, 0 /* don't care */); 250 if (rc) 251 return rc; 252 253 rc = hwrm_req_replace(bp, req, fw_msg->msg, fw_msg->msg_len); 254 if (rc) 255 goto drop_req; 256 257 hwrm_req_timeout(bp, req, fw_msg->timeout); 258 resp = hwrm_req_hold(bp, req); 259 rc = hwrm_req_send(bp, req); 260 resp_len = le16_to_cpu(resp->resp_len); 261 if (resp_len) { 262 if (fw_msg->resp_max_len < resp_len) 263 resp_len = fw_msg->resp_max_len; 264 265 memcpy(fw_msg->resp, resp, resp_len); 266 } 267 drop_req: 268 hwrm_req_drop(bp, req); 269 return rc; 270 } 271 EXPORT_SYMBOL(bnxt_send_msg); 272 273 void bnxt_ulp_stop(struct bnxt *bp) 274 { 275 int i; 276 277 mutex_lock(&bp->auxdev_lock); 278 for (i = 0; i < __BNXT_AUXDEV_MAX; i++) { 279 struct bnxt_aux_priv *aux_priv; 280 struct auxiliary_device *adev; 281 struct bnxt_en_dev *edev; 282 283 if (!bnxt_auxdev_is_active(bp, i)) 284 continue; 285 286 aux_priv = bp->aux_priv[i]; 287 edev = bp->edev[i]; 288 mutex_lock(&edev->en_dev_lock); 289 if (i == BNXT_AUXDEV_FWCTL) { 290 edev->flags |= BNXT_EN_FLAG_ULP_STOPPED; 291 mutex_unlock(&edev->en_dev_lock); 292 continue; 293 } 294 if (!bnxt_ulp_registered(edev) || 295 (edev->flags & BNXT_EN_FLAG_ULP_STOPPED)) { 296 mutex_unlock(&edev->en_dev_lock); 297 continue; 298 } 299 300 edev->flags |= BNXT_EN_FLAG_ULP_STOPPED; 301 302 adev = &aux_priv->aux_dev; 303 if (adev->dev.driver) { 304 const struct auxiliary_driver *adrv; 305 pm_message_t pm = {}; 306 307 adrv = to_auxiliary_drv(adev->dev.driver); 308 edev->en_state = bp->state; 309 adrv->suspend(adev, pm); 310 } 311 mutex_unlock(&edev->en_dev_lock); 312 } 313 mutex_unlock(&bp->auxdev_lock); 314 } 315 316 void bnxt_ulp_start(struct bnxt *bp) 317 { 318 int i; 319 320 mutex_lock(&bp->auxdev_lock); 321 for (i = 0; i < __BNXT_AUXDEV_MAX; i++) { 322 struct bnxt_aux_priv *aux_priv; 323 struct auxiliary_device *adev; 324 struct bnxt_en_dev *edev; 325 326 if (!bnxt_auxdev_is_active(bp, i)) 327 continue; 328 329 aux_priv = bp->aux_priv[i]; 330 edev = bp->edev[i]; 331 mutex_lock(&edev->en_dev_lock); 332 if (i == BNXT_AUXDEV_FWCTL || !bnxt_ulp_registered(edev) || 333 !(edev->flags & BNXT_EN_FLAG_ULP_STOPPED)) { 334 goto clear_flag_continue; 335 } 336 337 if (edev->ulp_tbl->msix_requested) 338 bnxt_fill_msix_vecs(bp, edev->msix_entries); 339 340 341 adev = &aux_priv->aux_dev; 342 if (adev->dev.driver) { 343 const struct auxiliary_driver *adrv; 344 345 adrv = to_auxiliary_drv(adev->dev.driver); 346 edev->en_state = bp->state; 347 adrv->resume(adev); 348 } 349 clear_flag_continue: 350 edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED; 351 mutex_unlock(&edev->en_dev_lock); 352 } 353 mutex_unlock(&bp->auxdev_lock); 354 } 355 356 void bnxt_ulp_irq_stop(struct bnxt *bp) 357 { 358 struct bnxt_en_dev *edev = bp->edev[BNXT_AUXDEV_RDMA]; 359 struct bnxt_ulp_ops *ops; 360 bool reset = false; 361 362 if (!edev) 363 return; 364 365 if (bnxt_ulp_registered(edev)) { 366 struct bnxt_ulp *ulp = edev->ulp_tbl; 367 368 if (!ulp->msix_requested) 369 return; 370 371 ops = netdev_lock_dereference(ulp->ulp_ops, bp->dev); 372 if (!ops || !ops->ulp_irq_stop) 373 return; 374 if (test_bit(BNXT_STATE_FW_RESET_DET, &bp->state)) 375 reset = true; 376 ops->ulp_irq_stop(ulp->handle, reset); 377 } 378 } 379 380 void bnxt_ulp_irq_restart(struct bnxt *bp, int err) 381 { 382 struct bnxt_en_dev *edev = bp->edev[BNXT_AUXDEV_RDMA]; 383 struct bnxt_ulp_ops *ops; 384 385 if (!edev) 386 return; 387 388 if (bnxt_ulp_registered(edev)) { 389 struct bnxt_ulp *ulp = edev->ulp_tbl; 390 struct bnxt_msix_entry *ent = NULL; 391 392 if (!ulp->msix_requested) 393 return; 394 395 ops = netdev_lock_dereference(ulp->ulp_ops, bp->dev); 396 if (!ops || !ops->ulp_irq_restart) 397 return; 398 399 if (!err) { 400 ent = kzalloc_objs(*ent, ulp->msix_requested); 401 if (!ent) 402 return; 403 bnxt_fill_msix_vecs(bp, ent); 404 } 405 ops->ulp_irq_restart(ulp->handle, ent); 406 kfree(ent); 407 } 408 } 409 410 void bnxt_ulp_async_events(struct bnxt *bp, struct hwrm_async_event_cmpl *cmpl) 411 { 412 u16 event_id = le16_to_cpu(cmpl->event_id); 413 struct bnxt_en_dev *edev = bp->edev[BNXT_AUXDEV_RDMA]; 414 struct bnxt_ulp_ops *ops; 415 struct bnxt_ulp *ulp; 416 417 if (!bnxt_ulp_registered(edev)) 418 return; 419 ulp = edev->ulp_tbl; 420 421 rcu_read_lock(); 422 423 ops = rcu_dereference(ulp->ulp_ops); 424 if (!ops || !ops->ulp_async_notifier) 425 goto exit_unlock_rcu; 426 if (!ulp->async_events_bmap || event_id > ulp->max_async_event_id) 427 goto exit_unlock_rcu; 428 429 /* Read max_async_event_id first before testing the bitmap. */ 430 smp_rmb(); 431 432 if (test_bit(event_id, ulp->async_events_bmap)) 433 ops->ulp_async_notifier(ulp->handle, cmpl); 434 exit_unlock_rcu: 435 rcu_read_unlock(); 436 } 437 438 void bnxt_register_async_events(struct bnxt_en_dev *edev, 439 unsigned long *events_bmap, u16 max_id) 440 { 441 struct net_device *dev = edev->net; 442 struct bnxt *bp = netdev_priv(dev); 443 struct bnxt_ulp *ulp; 444 445 ulp = edev->ulp_tbl; 446 ulp->async_events_bmap = events_bmap; 447 /* Make sure bnxt_ulp_async_events() sees this order */ 448 smp_wmb(); 449 ulp->max_async_event_id = max_id; 450 bnxt_hwrm_func_drv_rgtr(bp, events_bmap, max_id + 1, true); 451 } 452 EXPORT_SYMBOL(bnxt_register_async_events); 453 454 void bnxt_aux_devices_uninit(struct bnxt *bp) 455 { 456 struct bnxt_aux_priv *aux_priv; 457 struct auxiliary_device *adev; 458 int idx; 459 460 mutex_lock(&bp->auxdev_lock); 461 for (idx = 0; idx < __BNXT_AUXDEV_MAX; idx++) { 462 if (bnxt_auxdev_is_init(bp, idx)) { 463 aux_priv = bp->aux_priv[idx]; 464 adev = &aux_priv->aux_dev; 465 auxiliary_device_uninit(adev); 466 } 467 } 468 mutex_unlock(&bp->auxdev_lock); 469 } 470 471 static void bnxt_aux_dev_release(struct device *dev) 472 { 473 struct bnxt_aux_priv *aux_priv = 474 container_of(dev, struct bnxt_aux_priv, aux_dev.dev); 475 struct bnxt *bp = netdev_priv(aux_priv->edev->net); 476 477 kfree(aux_priv->edev->ulp_tbl); 478 bp->edev[aux_priv->id] = NULL; 479 kfree(aux_priv->edev); 480 bp->aux_priv[aux_priv->id] = NULL; 481 kfree(aux_priv); 482 } 483 484 void bnxt_aux_devices_del(struct bnxt *bp) 485 { 486 int idx; 487 488 mutex_lock(&bp->auxdev_lock); 489 for (idx = 0; idx < __BNXT_AUXDEV_MAX; idx++) { 490 if (bnxt_auxdev_is_active(bp, idx)) { 491 auxiliary_device_delete(&bp->aux_priv[idx]->aux_dev); 492 bnxt_auxdev_set_state(bp, idx, BNXT_ADEV_STATE_INIT); 493 } 494 } 495 mutex_unlock(&bp->auxdev_lock); 496 } 497 498 static void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt *bp) 499 { 500 edev->net = bp->dev; 501 edev->pdev = bp->pdev; 502 edev->l2_db_size = bp->db_size; 503 edev->l2_db_size_nc = bp->db_size; 504 edev->l2_db_offset = bp->db_offset; 505 mutex_init(&edev->en_dev_lock); 506 507 if (bp->flags & BNXT_FLAG_ROCEV1_CAP) 508 edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP; 509 if (bp->flags & BNXT_FLAG_ROCEV2_CAP) 510 edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP; 511 if (bp->flags & BNXT_FLAG_VF) 512 edev->flags |= BNXT_EN_FLAG_VF; 513 if (BNXT_ROCE_VF_RESC_CAP(bp)) 514 edev->flags |= BNXT_EN_FLAG_ROCE_VF_RES_MGMT; 515 if (BNXT_SW_RES_LMT(bp)) 516 edev->flags |= BNXT_EN_FLAG_SW_RES_LMT; 517 518 edev->chip_num = bp->chip_num; 519 edev->hw_ring_stats_size = bp->hw_ring_stats_size; 520 edev->pf_port_id = bp->pf.port_id; 521 edev->en_state = bp->state; 522 edev->bar0 = bp->bar0; 523 } 524 525 void bnxt_aux_devices_add(struct bnxt *bp) 526 { 527 struct auxiliary_device *aux_dev; 528 int rc, idx; 529 530 mutex_lock(&bp->auxdev_lock); 531 for (idx = 0; idx < __BNXT_AUXDEV_MAX; idx++) { 532 if (bnxt_auxdev_is_init(bp, idx)) { 533 aux_dev = &bp->aux_priv[idx]->aux_dev; 534 rc = auxiliary_device_add(aux_dev); 535 if (rc) { 536 netdev_warn(bp->dev, "Failed to add auxiliary device for auxdev type %d\n", 537 idx); 538 auxiliary_device_uninit(aux_dev); 539 if (idx == BNXT_AUXDEV_RDMA) 540 bp->flags &= ~BNXT_FLAG_ROCE_CAP; 541 continue; 542 } 543 bnxt_auxdev_set_state(bp, idx, BNXT_ADEV_STATE_ADD); 544 } 545 } 546 mutex_unlock(&bp->auxdev_lock); 547 } 548 549 void bnxt_aux_devices_init(struct bnxt *bp) 550 { 551 struct auxiliary_device *aux_dev; 552 struct bnxt_aux_priv *aux_priv; 553 struct bnxt_en_dev *edev; 554 struct bnxt_ulp *ulp; 555 int rc, idx; 556 557 mutex_lock(&bp->auxdev_lock); 558 for (idx = 0; idx < __BNXT_AUXDEV_MAX; idx++) { 559 bnxt_auxdev_set_state(bp, idx, BNXT_ADEV_STATE_NONE); 560 561 if (idx == BNXT_AUXDEV_RDMA && 562 !(bp->flags & BNXT_FLAG_ROCE_CAP)) 563 continue; 564 565 aux_priv = kzalloc_obj(*aux_priv); 566 if (!aux_priv) 567 goto next_auxdev; 568 569 aux_dev = &aux_priv->aux_dev; 570 aux_dev->id = bp->auxdev_id; 571 aux_dev->name = bnxt_aux_devices[idx].name; 572 aux_dev->dev.parent = &bp->pdev->dev; 573 aux_dev->dev.release = bnxt_aux_dev_release; 574 575 rc = auxiliary_device_init(aux_dev); 576 if (rc) { 577 kfree(aux_priv); 578 goto next_auxdev; 579 } 580 bp->aux_priv[idx] = aux_priv; 581 582 /* From this point, all cleanup will happen via the .release 583 * callback & any error unwinding will need to include a call 584 * to auxiliary_device_uninit. 585 */ 586 edev = kzalloc_obj(*edev); 587 if (!edev) 588 goto aux_dev_uninit; 589 590 aux_priv->edev = edev; 591 bnxt_set_edev_info(edev, bp); 592 593 ulp = kzalloc_obj(*ulp); 594 if (!ulp) 595 goto aux_dev_uninit; 596 597 edev->ulp_tbl = ulp; 598 bp->edev[idx] = edev; 599 if (idx == BNXT_AUXDEV_RDMA) 600 bp->ulp_num_msix_want = bnxt_set_dflt_ulp_msix(bp); 601 aux_priv->id = idx; 602 bnxt_auxdev_set_state(bp, idx, BNXT_ADEV_STATE_INIT); 603 604 continue; 605 aux_dev_uninit: 606 auxiliary_device_uninit(aux_dev); 607 next_auxdev: 608 if (idx == BNXT_AUXDEV_RDMA) 609 bp->flags &= ~BNXT_FLAG_ROCE_CAP; 610 } 611 mutex_unlock(&bp->auxdev_lock); 612 } 613 614 int bnxt_auxdev_id_alloc(struct bnxt *bp) 615 { 616 bp->auxdev_id = ida_alloc(&bnxt_aux_dev_ids, GFP_KERNEL); 617 if (bp->auxdev_id < 0) 618 return bp->auxdev_id; 619 620 return 0; 621 } 622 623 void bnxt_auxdev_id_free(struct bnxt *bp, int id) 624 { 625 if (bp->auxdev_id >= 0) 626 ida_free(&bnxt_aux_dev_ids, id); 627 } 628