1 /*- 2 * Broadcom NetXtreme-C/E network driver. 3 * 4 * Copyright (c) 2024 Broadcom, All Rights Reserved. 5 * The term Broadcom refers to Broadcom Limited and/or its subsidiaries 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 26 * THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <linux/module.h> 30 #include <linux/kernel.h> 31 #include <linux/errno.h> 32 #include <linux/interrupt.h> 33 #include <linux/pci.h> 34 #include <linux/netdevice.h> 35 #include <linux/bitops.h> 36 #include <asm/byteorder.h> 37 #include <linux/bitmap.h> 38 #include <linux/rcupdate.h> 39 #include <net/if.h> 40 #include <net/if_dl.h> 41 #include <net/if_media.h> 42 #include <net/if_var.h> 43 #include <net/ethernet.h> 44 #include <net/iflib.h> 45 46 #include "hsi_struct_def.h" 47 #include "bnxt.h" 48 #include "bnxt_hwrm.h" 49 #include "bnxt_ulp.h" 50 #include "bnxt_log.h" 51 52 void bnxt_destroy_irq(struct bnxt_softc *softc); 53 54 static int bnxt_register_dev(struct bnxt_en_dev *edev, int ulp_id, 55 struct bnxt_ulp_ops *ulp_ops, void *handle) 56 { 57 struct bnxt_softc *bp = edev->softc; 58 struct bnxt_ulp *ulp; 59 int rc = 0; 60 61 if (ulp_id >= BNXT_MAX_ULP) 62 return -EINVAL; 63 64 mtx_lock(&bp->en_ops_lock); 65 ulp = &edev->ulp_tbl[ulp_id]; 66 if (rcu_access_pointer(ulp->ulp_ops)) { 67 device_printf(bp->dev, "ulp id %d already registered\n", ulp_id); 68 rc = -EBUSY; 69 goto exit; 70 } 71 72 edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED; 73 atomic_set(&ulp->ref_count, 0); 74 ulp->handle = handle; 75 rcu_assign_pointer(ulp->ulp_ops, ulp_ops); 76 77 if (ulp_id == BNXT_ROCE_ULP) { 78 if (bnxt_drv_state_test(bp, BNXT_STATE_OPEN) && bp->is_dev_init) 79 bnxt_hwrm_vnic_cfg(bp, &bp->vnic_info); 80 } 81 82 exit: 83 mtx_unlock(&bp->en_ops_lock); 84 return rc; 85 } 86 87 static int bnxt_unregister_dev(struct bnxt_en_dev *edev, int ulp_id) 88 { 89 struct bnxt_softc *bp = edev->softc; 90 struct bnxt_ulp *ulp; 91 int i = 0; 92 93 if (ulp_id >= BNXT_MAX_ULP) 94 return -EINVAL; 95 96 ulp = &edev->ulp_tbl[ulp_id]; 97 if (!rcu_access_pointer(ulp->ulp_ops)) { 98 device_printf(bp->dev, "ulp id %d not registered\n", ulp_id); 99 return -EINVAL; 100 } 101 if (ulp_id == BNXT_ROCE_ULP && ulp->msix_requested) 102 edev->en_ops->bnxt_free_msix(edev, ulp_id); 103 104 mtx_lock(&bp->en_ops_lock); 105 RCU_INIT_POINTER(ulp->ulp_ops, NULL); 106 mtx_unlock(&bp->en_ops_lock); 107 synchronize_rcu(); 108 mtx_lock(&bp->en_ops_lock); 109 ulp->max_async_event_id = 0; 110 ulp->async_events_bmap = NULL; 111 mtx_unlock(&bp->en_ops_lock); 112 while (atomic_read(&ulp->ref_count) != 0 && i < 10) { 113 pause_sbt("bxtms", SBT_1MS * 100, 0, C_HARDCLOCK); 114 i++; 115 } 116 /* Tell the caller a ULP consumer may still hold a reference into torn-down state. */ 117 if (atomic_read(&ulp->ref_count) != 0) 118 return ETIMEDOUT; 119 return 0; 120 } 121 122 static void bnxt_fill_msix_vecs(struct bnxt_softc *bp, struct bnxt_msix_entry *ent) 123 { 124 struct bnxt_en_dev *edev = bp->edev; 125 int num_msix, idx, i; 126 127 num_msix = edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested; 128 idx = edev->ulp_tbl[BNXT_ROCE_ULP].msix_base; 129 for (i = 0; i < num_msix; i++) { 130 ent[i].vector = bp->irq_tbl[idx + i].vector; 131 ent[i].ring_idx = idx + i; 132 if (BNXT_CHIP_P5_PLUS(bp)) 133 ent[i].db_offset = bp->db_offset; 134 else 135 ent[i].db_offset = (idx + i) * 0x80; 136 137 } 138 } 139 140 static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, int ulp_id, 141 struct bnxt_msix_entry *ent, int num_msix) 142 { 143 struct bnxt_softc *bp = edev->softc; 144 int avail_msix, idx; 145 146 if (ulp_id != BNXT_ROCE_ULP) 147 return -EINVAL; 148 149 if (edev->ulp_tbl[ulp_id].msix_requested) 150 return -EAGAIN; 151 152 idx = bp->total_irqs - BNXT_ROCE_IRQ_COUNT; 153 avail_msix = BNXT_ROCE_IRQ_COUNT; 154 155 mtx_lock(&bp->en_ops_lock); 156 edev->ulp_tbl[ulp_id].msix_base = idx; 157 edev->ulp_tbl[ulp_id].msix_requested = avail_msix; 158 159 bnxt_fill_msix_vecs(bp, ent); 160 edev->flags |= BNXT_EN_FLAG_MSIX_REQUESTED; 161 mtx_unlock(&bp->en_ops_lock); 162 return avail_msix; 163 } 164 165 static int bnxt_free_msix_vecs(struct bnxt_en_dev *edev, int ulp_id) 166 { 167 struct bnxt_softc *bp = edev->softc; 168 169 if (ulp_id != BNXT_ROCE_ULP) 170 return -EINVAL; 171 172 if (!(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED)) 173 return 0; 174 175 mtx_lock(&bp->en_ops_lock); 176 edev->ulp_tbl[ulp_id].msix_requested = 0; 177 edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED; 178 if (edev->flags & BNXT_EN_FLAG_ULP_STOPPED) 179 goto stopped; 180 181 stopped: 182 mtx_unlock(&bp->en_ops_lock); 183 184 return 0; 185 } 186 187 int bnxt_get_ulp_msix_num(struct bnxt_softc *bp) 188 { 189 if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { 190 struct bnxt_en_dev *edev = bp->edev; 191 192 return edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested; 193 } 194 return 0; 195 } 196 197 int bnxt_get_ulp_msix_base(struct bnxt_softc *bp) 198 { 199 if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { 200 struct bnxt_en_dev *edev = bp->edev; 201 202 if (edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested) 203 return edev->ulp_tbl[BNXT_ROCE_ULP].msix_base; 204 } 205 return 0; 206 } 207 208 static int bnxt_send_msg(struct bnxt_en_dev *edev, int ulp_id, 209 struct bnxt_fw_msg *fw_msg) 210 { 211 struct bnxt_softc *softc = edev->softc; 212 int rc; 213 214 if ((ulp_id != BNXT_ROCE_ULP) && softc->fw_reset_state) 215 return -EBUSY; 216 217 rc = bnxt_hwrm_passthrough(softc, fw_msg->msg, fw_msg->msg_len, fw_msg->resp, 218 fw_msg->resp_max_len, fw_msg->timeout); 219 return rc; 220 } 221 222 static void bnxt_ulp_get(struct bnxt_ulp *ulp) 223 { 224 atomic_inc(&ulp->ref_count); 225 } 226 227 static void bnxt_ulp_put(struct bnxt_ulp *ulp) 228 { 229 atomic_dec(&ulp->ref_count); 230 } 231 232 void bnxt_ulp_stop(struct bnxt_softc *bp) 233 { 234 struct bnxt_en_dev *edev = bp->edev; 235 struct bnxt_ulp_ops *ops; 236 int i; 237 238 if (!edev) 239 return; 240 241 edev->flags |= BNXT_EN_FLAG_ULP_STOPPED; 242 edev->en_state = bp->state; 243 for (i = 0; i < BNXT_MAX_ULP; i++) { 244 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 245 246 ops = ulp->ulp_ops; 247 if (!ops || !ops->ulp_stop) 248 continue; 249 ops->ulp_stop(ulp->handle); 250 } 251 } 252 253 void bnxt_ulp_start(struct bnxt_softc *bp, int err) 254 { 255 struct bnxt_en_dev *edev = bp->edev; 256 struct bnxt_ulp_ops *ops; 257 int i; 258 259 if (!edev) 260 return; 261 262 edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED; 263 edev->en_state = bp->state; 264 265 if (err) 266 return; 267 268 for (i = 0; i < BNXT_MAX_ULP; i++) { 269 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 270 271 ops = ulp->ulp_ops; 272 if (!ops || !ops->ulp_start) 273 continue; 274 ops->ulp_start(ulp->handle); 275 } 276 } 277 278 void bnxt_ulp_sriov_cfg(struct bnxt_softc *bp, int num_vfs) 279 { 280 struct bnxt_en_dev *edev = bp->edev; 281 struct bnxt_ulp_ops *ops; 282 int i; 283 284 if (!edev) 285 return; 286 287 for (i = 0; i < BNXT_MAX_ULP; i++) { 288 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 289 290 rcu_read_lock(); 291 ops = rcu_dereference(ulp->ulp_ops); 292 if (!ops || !ops->ulp_sriov_config) { 293 rcu_read_unlock(); 294 continue; 295 } 296 bnxt_ulp_get(ulp); 297 rcu_read_unlock(); 298 ops->ulp_sriov_config(ulp->handle, num_vfs); 299 bnxt_ulp_put(ulp); 300 } 301 } 302 303 void bnxt_ulp_shutdown(struct bnxt_softc *bp) 304 { 305 struct bnxt_en_dev *edev = bp->edev; 306 struct bnxt_ulp_ops *ops; 307 int i; 308 309 if (!edev) 310 return; 311 312 for (i = 0; i < BNXT_MAX_ULP; i++) { 313 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 314 315 ops = ulp->ulp_ops; 316 if (!ops || !ops->ulp_shutdown) 317 continue; 318 ops->ulp_shutdown(ulp->handle); 319 } 320 } 321 322 void bnxt_ulp_irq_stop(struct bnxt_softc *bp) 323 { 324 struct bnxt_en_dev *edev = bp->edev; 325 struct bnxt_ulp_ops *ops; 326 327 if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED)) 328 return; 329 330 if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { 331 struct bnxt_ulp *ulp = &edev->ulp_tbl[BNXT_ROCE_ULP]; 332 333 if (!ulp->msix_requested) 334 return; 335 336 ops = ulp->ulp_ops; 337 if (!ops || !ops->ulp_irq_stop) 338 return; 339 ops->ulp_irq_stop(ulp->handle); 340 } 341 } 342 343 void 344 bnxt_logger_ulp_live_data(void *d) 345 { 346 struct bnxt_en_dev *edev; 347 struct bnxt_softc *bp; 348 349 bp = d; 350 edev = bp->edev; 351 352 if (!edev) 353 return; 354 355 if (bnxt_ulp_registered(edev, BNXT_ROCE_ULP)) { 356 struct bnxt_ulp_ops *ops; 357 struct bnxt_ulp *ulp; 358 359 ulp = edev->ulp_tbl; 360 //ops = rtnl_dereference(ulp->ulp_ops); 361 ops = ulp->ulp_ops; 362 if (!ops || !ops->ulp_log_live) 363 return; 364 365 ops->ulp_log_live(ulp->handle); 366 } 367 } 368 369 void 370 bnxt_ulp_log_live(struct bnxt_en_dev *edev, u16 logger_id, 371 const char *format, ...) 372 { 373 bnxt_log_live(if_getsoftc(edev->net), logger_id, format); 374 } 375 EXPORT_SYMBOL(bnxt_ulp_log_live); 376 377 void bnxt_ulp_async_events(struct bnxt_softc *bp, struct hwrm_async_event_cmpl *cmpl) 378 { 379 u16 event_id = le16toh(cmpl->event_id); 380 struct bnxt_en_dev *edev = bp->edev; 381 struct bnxt_ulp_ops *ops; 382 int i; 383 384 if (!edev) 385 return; 386 387 rcu_read_lock(); 388 for (i = 0; i < BNXT_MAX_ULP; i++) { 389 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 390 391 ops = rcu_dereference(ulp->ulp_ops); 392 if (!ops || !ops->ulp_async_notifier) 393 continue; 394 if (!ulp->async_events_bmap || 395 event_id > ulp->max_async_event_id) 396 continue; 397 398 /* Read max_async_event_id first before testing the bitmap. */ 399 atomic_thread_fence_acq(); 400 if (edev->flags & BNXT_EN_FLAG_ULP_STOPPED) 401 continue; 402 403 if (test_bit(event_id, ulp->async_events_bmap)) 404 ops->ulp_async_notifier(ulp->handle, cmpl); 405 } 406 rcu_read_unlock(); 407 } 408 409 static int bnxt_register_async_events(struct bnxt_en_dev *edev, int ulp_id, 410 unsigned long *events_bmap, u16 max_id) 411 { 412 struct bnxt_softc *bp = edev->softc; 413 struct bnxt_ulp *ulp; 414 415 if (ulp_id >= BNXT_MAX_ULP) 416 return -EINVAL; 417 418 mtx_lock(&bp->en_ops_lock); 419 ulp = &edev->ulp_tbl[ulp_id]; 420 ulp->async_events_bmap = events_bmap; 421 atomic_thread_fence_rel(); 422 ulp->max_async_event_id = max_id; 423 bnxt_hwrm_func_drv_rgtr(bp, events_bmap, max_id + 1, true); 424 mtx_unlock(&bp->en_ops_lock); 425 return 0; 426 } 427 428 void bnxt_destroy_irq(struct bnxt_softc *softc) 429 { 430 kfree(softc->irq_tbl); 431 } 432 433 /* 434 * Build the IRQ table from scratch for isc_ntxqsets + irq_count vectors; 435 * usable by any feature needing private IRQs without prior setup. 436 */ 437 int bnxt_populate_irq(struct bnxt_softc *softc, int irq_count) 438 { 439 struct resource_list *rl = NULL; 440 struct resource_list_entry *rle = NULL; 441 struct bnxt_msix_tbl *irq_tbl = NULL; 442 struct pci_devinfo *dinfo = NULL; 443 int i; 444 445 softc->total_irqs = softc->scctx->isc_ntxqsets + irq_count; 446 irq_tbl = kzalloc(softc->total_irqs * sizeof(*softc->irq_tbl), GFP_KERNEL); 447 448 if (!irq_tbl) { 449 device_printf(softc->dev, "Failed to allocate IRQ table\n"); 450 return -1; 451 } 452 453 dinfo = device_get_ivars(softc->pdev->dev.bsddev); 454 rl = &dinfo->resources; 455 rle = resource_list_find(rl, SYS_RES_IRQ, 1); 456 if (rle == NULL) { 457 device_printf(softc->dev, 458 "No default resources for rid = %d, type = %d\n", 459 1, SYS_RES_IRQ); 460 kfree(irq_tbl); 461 return -1; 462 } 463 464 softc->pdev->dev.irq_start = rle->start; 465 softc->pdev->dev.irq_end = rle->start + softc->total_irqs; 466 467 for (i = 0; i < softc->total_irqs; i++) { 468 irq_tbl[i].entry = i; 469 irq_tbl[i].vector = softc->pdev->dev.irq_start + i; 470 } 471 472 softc->irq_tbl = irq_tbl; 473 474 return 0; 475 } 476 477 /* 478 * Grow the IRQ table by BNXT_ROCE_IRQ_COUNT on top of existing entries (e.g. 479 * MPC's) instead of reassigning total_irqs, to avoid reclaiming MPC's rids. 480 */ 481 static int bnxt_populate_irq_roce(struct bnxt_softc *softc) 482 { 483 struct resource_list *rl = NULL; 484 struct resource_list_entry *rle = NULL; 485 struct pci_devinfo *dinfo = NULL; 486 struct bnxt_msix_tbl *irq_tbl; 487 int prev_total_irqs = softc->total_irqs; 488 int i; 489 490 softc->total_irqs += BNXT_ROCE_IRQ_COUNT; 491 irq_tbl = krealloc(softc->irq_tbl, 492 softc->total_irqs * sizeof(*softc->irq_tbl), GFP_KERNEL); 493 494 if (!irq_tbl) { 495 device_printf(softc->dev, "Failed to allocate IRQ table\n"); 496 /* Roll back total_irqs since the table itself wasn't grown. */ 497 softc->total_irqs = prev_total_irqs; 498 return -1; 499 } 500 501 dinfo = device_get_ivars(softc->pdev->dev.bsddev); 502 rl = &dinfo->resources; 503 rle = resource_list_find(rl, SYS_RES_IRQ, 1); 504 if (rle == NULL) { 505 device_printf(softc->dev, 506 "No default resources for rid = %d, type = %d\n", 507 1, SYS_RES_IRQ); 508 /* irq_tbl isn't installed yet; free it and roll back total_irqs. */ 509 kfree(irq_tbl); 510 softc->total_irqs = prev_total_irqs; 511 return -1; 512 } 513 514 softc->pdev->dev.irq_start = rle->start; 515 softc->pdev->dev.irq_end = rle->start + softc->total_irqs; 516 517 for (i = prev_total_irqs; i < softc->total_irqs; i++) { 518 irq_tbl[i].entry = i; 519 irq_tbl[i].vector = softc->pdev->dev.irq_start + i; 520 } 521 522 softc->irq_tbl = irq_tbl; 523 524 return 0; 525 } 526 527 static const struct bnxt_en_ops bnxt_en_ops_tbl = { 528 .bnxt_register_device = bnxt_register_dev, 529 .bnxt_unregister_device = bnxt_unregister_dev, 530 .bnxt_request_msix = bnxt_req_msix_vecs, 531 .bnxt_free_msix = bnxt_free_msix_vecs, 532 .bnxt_send_fw_msg = bnxt_send_msg, 533 .bnxt_register_fw_async_events = bnxt_register_async_events, 534 }; 535 536 void bnxt_aux_dev_release(struct device *dev) 537 { 538 struct bnxt_aux_dev *bnxt_adev = 539 container_of(dev, struct bnxt_aux_dev, aux_dev.dev); 540 struct bnxt_softc *bp = bnxt_adev->edev->softc; 541 542 kfree(bnxt_adev->edev); 543 bnxt_adev->edev = NULL; 544 bp->edev = NULL; 545 } 546 547 static inline void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt_softc *bp) 548 { 549 edev->en_ops = &bnxt_en_ops_tbl; 550 edev->net = bp->ifp; 551 edev->pdev = bp->pdev; 552 edev->softc = bp; 553 edev->l2_db_size = bp->db_size; 554 edev->l2_db_offset = bp->db_offset; 555 mtx_init(&bp->en_ops_lock, "Ethernet ops lock", NULL, MTX_DEF); 556 557 if (bp->flags & BNXT_FLAG_ROCEV1_CAP) 558 edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP; 559 if (bp->flags & BNXT_FLAG_ROCEV2_CAP) 560 edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP; 561 if (bp->is_asym_q) 562 edev->flags |= BNXT_EN_FLAG_ASYM_Q; 563 if (BNXT_SW_RES_LMT(bp)) 564 edev->flags |= BNXT_EN_FLAG_SW_RES_LMT; 565 edev->hwrm_bar = bp->hwrm_bar; 566 edev->port_partition_type = bp->port_partition_type; 567 edev->ulp_version = BNXT_ULP_VERSION; 568 memcpy(edev->board_part_number, bp->board_partno, BNXT_VPD_PN_FLD_LEN - 1); 569 } 570 571 int bnxt_rdma_aux_device_del(struct bnxt_softc *softc) 572 { 573 struct bnxt_aux_dev *bnxt_adev = softc->aux_dev; 574 struct auxiliary_device *adev; 575 576 adev = &bnxt_adev->aux_dev; 577 auxiliary_device_delete(adev); 578 auxiliary_device_uninit(adev); 579 bnxt_destroy_irq(softc); 580 581 return 0; 582 } 583 584 int bnxt_rdma_aux_device_add(struct bnxt_softc *bp) 585 { 586 struct bnxt_aux_dev *bnxt_adev = bp->aux_dev; 587 struct bnxt_en_dev *edev = bnxt_adev->edev; 588 struct auxiliary_device *aux_dev; 589 int ret = -1; 590 591 if (bnxt_populate_irq_roce(bp)) 592 return ret; 593 594 device_printf(bp->dev, "V:D:SV:SD %x:%x:%x:%x, irq 0x%x, " 595 "devfn 0x%x, cla 0x%x, rev 0x%x, msi_en 0x%x\n", 596 bp->pdev->vendor, bp->pdev->device, bp->pdev->subsystem_vendor, 597 bp->pdev->subsystem_device, bp->pdev->irq, bp->pdev->devfn, 598 bp->pdev->class, bp->pdev->revision, bp->pdev->msi_enabled); 599 600 aux_dev = &bnxt_adev->aux_dev; 601 aux_dev->id = bnxt_adev->id; 602 aux_dev->name = "rdma"; 603 aux_dev->dev.parent = &bp->pdev->dev; 604 aux_dev->dev.release = bnxt_aux_dev_release; 605 606 if (!edev) { 607 edev = kzalloc(sizeof(*edev), GFP_KERNEL); 608 if (!edev) 609 return -ENOMEM; 610 } 611 612 bnxt_set_edev_info(edev, bp); 613 bnxt_adev->edev = edev; 614 bp->edev = edev; 615 616 ret = auxiliary_device_init(aux_dev); 617 if (ret) 618 goto err_free_edev; 619 620 ret = auxiliary_device_add(aux_dev); 621 if (ret) 622 goto err_dev_uninit; 623 624 return 0; 625 err_dev_uninit: 626 auxiliary_device_uninit(aux_dev); 627 err_free_edev: 628 kfree(edev); 629 bnxt_adev->edev = NULL; 630 bp->edev = NULL; 631 return ret; 632 } 633