1 /*- 2 * Broadcom NetXtreme-C/E network driver. 3 * 4 * Copyright (c) 2024 Broadcom, All Rights Reserved. 5 * The term Broadcom refers to Broadcom Limited and/or its subsidiaries 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS' 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 26 * THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <linux/module.h> 30 #include <linux/kernel.h> 31 #include <linux/errno.h> 32 #include <linux/interrupt.h> 33 #include <linux/pci.h> 34 #include <linux/netdevice.h> 35 #include <linux/bitops.h> 36 #include <linux/delay.h> 37 #include <asm/byteorder.h> 38 #include <linux/bitmap.h> 39 #include <linux/rcupdate.h> 40 #include <net/if.h> 41 #include <net/if_dl.h> 42 #include <net/if_media.h> 43 #include <net/if_var.h> 44 #include <net/ethernet.h> 45 #include <net/iflib.h> 46 47 #include "hsi_struct_def.h" 48 #include "bnxt.h" 49 #include "bnxt_hwrm.h" 50 #include "bnxt_ulp.h" 51 52 void bnxt_destroy_irq(struct bnxt_softc *softc); 53 54 static int bnxt_register_dev(struct bnxt_en_dev *edev, int ulp_id, 55 struct bnxt_ulp_ops *ulp_ops, void *handle) 56 { 57 struct bnxt_softc *bp = edev->softc; 58 struct bnxt_ulp *ulp; 59 int rc = 0; 60 61 if (ulp_id >= BNXT_MAX_ULP) 62 return -EINVAL; 63 64 mtx_lock(&bp->en_ops_lock); 65 ulp = &edev->ulp_tbl[ulp_id]; 66 if (rcu_access_pointer(ulp->ulp_ops)) { 67 device_printf(bp->dev, "ulp id %d already registered\n", ulp_id); 68 rc = -EBUSY; 69 goto exit; 70 } 71 72 edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED; 73 atomic_set(&ulp->ref_count, 0); 74 ulp->handle = handle; 75 rcu_assign_pointer(ulp->ulp_ops, ulp_ops); 76 77 if (ulp_id == BNXT_ROCE_ULP) { 78 if (test_bit(BNXT_STATE_OPEN, &bp->state) && bp->is_dev_init) 79 bnxt_hwrm_vnic_cfg(bp, &bp->vnic_info); 80 } 81 82 exit: 83 mtx_unlock(&bp->en_ops_lock); 84 return rc; 85 } 86 87 static int bnxt_unregister_dev(struct bnxt_en_dev *edev, int ulp_id) 88 { 89 struct bnxt_softc *bp = edev->softc; 90 struct bnxt_ulp *ulp; 91 int i = 0; 92 93 if (ulp_id >= BNXT_MAX_ULP) 94 return -EINVAL; 95 96 ulp = &edev->ulp_tbl[ulp_id]; 97 if (!rcu_access_pointer(ulp->ulp_ops)) { 98 device_printf(bp->dev, "ulp id %d not registered\n", ulp_id); 99 return -EINVAL; 100 } 101 if (ulp_id == BNXT_ROCE_ULP && ulp->msix_requested) 102 edev->en_ops->bnxt_free_msix(edev, ulp_id); 103 104 mtx_lock(&bp->en_ops_lock); 105 RCU_INIT_POINTER(ulp->ulp_ops, NULL); 106 synchronize_rcu(); 107 ulp->max_async_event_id = 0; 108 ulp->async_events_bmap = NULL; 109 while (atomic_read(&ulp->ref_count) != 0 && i < 10) { 110 msleep(100); 111 i++; 112 } 113 mtx_unlock(&bp->en_ops_lock); 114 return 0; 115 } 116 117 static void bnxt_fill_msix_vecs(struct bnxt_softc *bp, struct bnxt_msix_entry *ent) 118 { 119 struct bnxt_en_dev *edev = bp->edev; 120 int num_msix, idx, i; 121 122 num_msix = edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested; 123 idx = edev->ulp_tbl[BNXT_ROCE_ULP].msix_base; 124 for (i = 0; i < num_msix; i++) { 125 ent[i].vector = bp->irq_tbl[idx + i].vector; 126 ent[i].ring_idx = idx + i; 127 if (BNXT_CHIP_P5(bp)) 128 ent[i].db_offset = DB_PF_OFFSET_P5; 129 else 130 ent[i].db_offset = (idx + i) * 0x80; 131 132 } 133 } 134 135 static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, int ulp_id, 136 struct bnxt_msix_entry *ent, int num_msix) 137 { 138 struct bnxt_softc *bp = edev->softc; 139 int avail_msix, idx; 140 141 if (ulp_id != BNXT_ROCE_ULP) 142 return -EINVAL; 143 144 if (edev->ulp_tbl[ulp_id].msix_requested) 145 return -EAGAIN; 146 147 idx = bp->total_irqs - BNXT_ROCE_IRQ_COUNT; 148 avail_msix = BNXT_ROCE_IRQ_COUNT; 149 150 mtx_lock(&bp->en_ops_lock); 151 edev->ulp_tbl[ulp_id].msix_base = idx; 152 edev->ulp_tbl[ulp_id].msix_requested = avail_msix; 153 154 bnxt_fill_msix_vecs(bp, ent); 155 edev->flags |= BNXT_EN_FLAG_MSIX_REQUESTED; 156 mtx_unlock(&bp->en_ops_lock); 157 return avail_msix; 158 } 159 160 static int bnxt_free_msix_vecs(struct bnxt_en_dev *edev, int ulp_id) 161 { 162 struct bnxt_softc *bp = edev->softc; 163 164 if (ulp_id != BNXT_ROCE_ULP) 165 return -EINVAL; 166 167 if (!(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED)) 168 return 0; 169 170 mtx_lock(&bp->en_ops_lock); 171 edev->ulp_tbl[ulp_id].msix_requested = 0; 172 edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED; 173 if (edev->flags & BNXT_EN_FLAG_ULP_STOPPED) 174 goto stopped; 175 176 stopped: 177 mtx_unlock(&bp->en_ops_lock); 178 179 return 0; 180 } 181 182 int bnxt_get_ulp_msix_num(struct bnxt_softc *bp) 183 { 184 if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { 185 struct bnxt_en_dev *edev = bp->edev; 186 187 return edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested; 188 } 189 return 0; 190 } 191 192 int bnxt_get_ulp_msix_base(struct bnxt_softc *bp) 193 { 194 if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { 195 struct bnxt_en_dev *edev = bp->edev; 196 197 if (edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested) 198 return edev->ulp_tbl[BNXT_ROCE_ULP].msix_base; 199 } 200 return 0; 201 } 202 203 static int bnxt_send_msg(struct bnxt_en_dev *edev, int ulp_id, 204 struct bnxt_fw_msg *fw_msg) 205 { 206 struct bnxt_softc *softc = edev->softc; 207 int rc; 208 209 if ((ulp_id != BNXT_ROCE_ULP) && softc->fw_reset_state) 210 return -EBUSY; 211 212 rc = bnxt_hwrm_passthrough(softc, fw_msg->msg, fw_msg->msg_len, fw_msg->resp, 213 fw_msg->resp_max_len, fw_msg->timeout); 214 return rc; 215 } 216 217 static void bnxt_ulp_get(struct bnxt_ulp *ulp) 218 { 219 atomic_inc(&ulp->ref_count); 220 } 221 222 static void bnxt_ulp_put(struct bnxt_ulp *ulp) 223 { 224 atomic_dec(&ulp->ref_count); 225 } 226 227 void bnxt_ulp_stop(struct bnxt_softc *bp) 228 { 229 struct bnxt_en_dev *edev = bp->edev; 230 struct bnxt_ulp_ops *ops; 231 int i; 232 233 if (!edev) 234 return; 235 236 edev->flags |= BNXT_EN_FLAG_ULP_STOPPED; 237 edev->en_state = bp->state; 238 for (i = 0; i < BNXT_MAX_ULP; i++) { 239 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 240 241 ops = ulp->ulp_ops; 242 if (!ops || !ops->ulp_stop) 243 continue; 244 ops->ulp_stop(ulp->handle); 245 } 246 } 247 248 void bnxt_ulp_start(struct bnxt_softc *bp, int err) 249 { 250 struct bnxt_en_dev *edev = bp->edev; 251 struct bnxt_ulp_ops *ops; 252 int i; 253 254 if (!edev) 255 return; 256 257 edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED; 258 edev->en_state = bp->state; 259 260 if (err) 261 return; 262 263 for (i = 0; i < BNXT_MAX_ULP; i++) { 264 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 265 266 ops = ulp->ulp_ops; 267 if (!ops || !ops->ulp_start) 268 continue; 269 ops->ulp_start(ulp->handle); 270 } 271 } 272 273 void bnxt_ulp_sriov_cfg(struct bnxt_softc *bp, int num_vfs) 274 { 275 struct bnxt_en_dev *edev = bp->edev; 276 struct bnxt_ulp_ops *ops; 277 int i; 278 279 if (!edev) 280 return; 281 282 for (i = 0; i < BNXT_MAX_ULP; i++) { 283 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 284 285 rcu_read_lock(); 286 ops = rcu_dereference(ulp->ulp_ops); 287 if (!ops || !ops->ulp_sriov_config) { 288 rcu_read_unlock(); 289 continue; 290 } 291 bnxt_ulp_get(ulp); 292 rcu_read_unlock(); 293 ops->ulp_sriov_config(ulp->handle, num_vfs); 294 bnxt_ulp_put(ulp); 295 } 296 } 297 298 void bnxt_ulp_shutdown(struct bnxt_softc *bp) 299 { 300 struct bnxt_en_dev *edev = bp->edev; 301 struct bnxt_ulp_ops *ops; 302 int i; 303 304 if (!edev) 305 return; 306 307 for (i = 0; i < BNXT_MAX_ULP; i++) { 308 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 309 310 ops = ulp->ulp_ops; 311 if (!ops || !ops->ulp_shutdown) 312 continue; 313 ops->ulp_shutdown(ulp->handle); 314 } 315 } 316 317 void bnxt_ulp_irq_stop(struct bnxt_softc *bp) 318 { 319 struct bnxt_en_dev *edev = bp->edev; 320 struct bnxt_ulp_ops *ops; 321 322 if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED)) 323 return; 324 325 if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) { 326 struct bnxt_ulp *ulp = &edev->ulp_tbl[BNXT_ROCE_ULP]; 327 328 if (!ulp->msix_requested) 329 return; 330 331 ops = ulp->ulp_ops; 332 if (!ops || !ops->ulp_irq_stop) 333 return; 334 ops->ulp_irq_stop(ulp->handle); 335 } 336 } 337 338 void bnxt_ulp_async_events(struct bnxt_softc *bp, struct hwrm_async_event_cmpl *cmpl) 339 { 340 u16 event_id = le16_to_cpu(cmpl->event_id); 341 struct bnxt_en_dev *edev = bp->edev; 342 struct bnxt_ulp_ops *ops; 343 int i; 344 345 if (!edev) 346 return; 347 348 rcu_read_lock(); 349 for (i = 0; i < BNXT_MAX_ULP; i++) { 350 struct bnxt_ulp *ulp = &edev->ulp_tbl[i]; 351 352 ops = rcu_dereference(ulp->ulp_ops); 353 if (!ops || !ops->ulp_async_notifier) 354 continue; 355 if (!ulp->async_events_bmap || 356 event_id > ulp->max_async_event_id) 357 continue; 358 359 /* Read max_async_event_id first before testing the bitmap. */ 360 rmb(); 361 if (edev->flags & BNXT_EN_FLAG_ULP_STOPPED) 362 continue; 363 364 if (test_bit(event_id, ulp->async_events_bmap)) 365 ops->ulp_async_notifier(ulp->handle, cmpl); 366 } 367 rcu_read_unlock(); 368 } 369 370 static int bnxt_register_async_events(struct bnxt_en_dev *edev, int ulp_id, 371 unsigned long *events_bmap, u16 max_id) 372 { 373 struct bnxt_softc *bp = edev->softc; 374 struct bnxt_ulp *ulp; 375 376 if (ulp_id >= BNXT_MAX_ULP) 377 return -EINVAL; 378 379 mtx_lock(&bp->en_ops_lock); 380 ulp = &edev->ulp_tbl[ulp_id]; 381 ulp->async_events_bmap = events_bmap; 382 wmb(); 383 ulp->max_async_event_id = max_id; 384 bnxt_hwrm_func_drv_rgtr(bp, events_bmap, max_id + 1, true); 385 mtx_unlock(&bp->en_ops_lock); 386 return 0; 387 } 388 389 void bnxt_destroy_irq(struct bnxt_softc *softc) 390 { 391 kfree(softc->irq_tbl); 392 } 393 394 static int bnxt_populate_irq(struct bnxt_softc *softc) 395 { 396 struct resource_list *rl = NULL; 397 struct resource_list_entry *rle = NULL; 398 struct bnxt_msix_tbl *irq_tbl = NULL; 399 struct pci_devinfo *dinfo = NULL; 400 int i; 401 402 softc->total_irqs = softc->scctx->isc_nrxqsets + BNXT_ROCE_IRQ_COUNT; 403 irq_tbl = kzalloc(softc->total_irqs * sizeof(*softc->irq_tbl), GFP_KERNEL); 404 405 if (!irq_tbl) { 406 device_printf(softc->dev, "Failed to allocate IRQ table\n"); 407 return -1; 408 } 409 dinfo = device_get_ivars(softc->pdev->dev.bsddev); 410 rl = &dinfo->resources; 411 rle = resource_list_find(rl, SYS_RES_IRQ, 1); 412 softc->pdev->dev.irq_start = rle->start; 413 softc->pdev->dev.irq_end = rle->start + softc->total_irqs; 414 415 for (i = 0; i < softc->total_irqs; i++) { 416 irq_tbl[i].entry = i; 417 irq_tbl[i].vector = softc->pdev->dev.irq_start + i; 418 } 419 420 softc->irq_tbl = irq_tbl; 421 422 return 0; 423 } 424 425 static const struct bnxt_en_ops bnxt_en_ops_tbl = { 426 .bnxt_register_device = bnxt_register_dev, 427 .bnxt_unregister_device = bnxt_unregister_dev, 428 .bnxt_request_msix = bnxt_req_msix_vecs, 429 .bnxt_free_msix = bnxt_free_msix_vecs, 430 .bnxt_send_fw_msg = bnxt_send_msg, 431 .bnxt_register_fw_async_events = bnxt_register_async_events, 432 }; 433 434 void bnxt_aux_dev_release(struct device *dev) 435 { 436 struct bnxt_aux_dev *bnxt_adev = 437 container_of(dev, struct bnxt_aux_dev, aux_dev.dev); 438 struct bnxt_softc *bp = bnxt_adev->edev->softc; 439 440 kfree(bnxt_adev->edev); 441 bnxt_adev->edev = NULL; 442 bp->edev = NULL; 443 } 444 445 static inline void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt_softc *bp) 446 { 447 edev->en_ops = &bnxt_en_ops_tbl; 448 edev->net = bp->ifp; 449 edev->pdev = bp->pdev; 450 edev->softc = bp; 451 edev->l2_db_size = bp->db_size; 452 mtx_init(&bp->en_ops_lock, "Ethernet ops lock", NULL, MTX_DEF); 453 454 if (bp->flags & BNXT_FLAG_ROCEV1_CAP) 455 edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP; 456 if (bp->flags & BNXT_FLAG_ROCEV2_CAP) 457 edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP; 458 if (bp->is_asym_q) 459 edev->flags |= BNXT_EN_FLAG_ASYM_Q; 460 edev->hwrm_bar = bp->hwrm_bar; 461 edev->port_partition_type = bp->port_partition_type; 462 edev->ulp_version = BNXT_ULP_VERSION; 463 } 464 465 int bnxt_rdma_aux_device_del(struct bnxt_softc *softc) 466 { 467 struct bnxt_aux_dev *bnxt_adev = softc->aux_dev; 468 struct auxiliary_device *adev; 469 470 adev = &bnxt_adev->aux_dev; 471 auxiliary_device_delete(adev); 472 auxiliary_device_uninit(adev); 473 bnxt_destroy_irq(softc); 474 475 return 0; 476 } 477 478 int bnxt_rdma_aux_device_add(struct bnxt_softc *bp) 479 { 480 struct bnxt_aux_dev *bnxt_adev = bp->aux_dev; 481 struct bnxt_en_dev *edev = bnxt_adev->edev; 482 struct auxiliary_device *aux_dev; 483 int ret = -1; 484 485 if (bnxt_populate_irq(bp)) 486 return ret; 487 488 device_printf(bp->dev, "V:D:SV:SD %x:%x:%x:%x, irq 0x%x, " 489 "devfn 0x%x, cla 0x%x, rev 0x%x, msi_en 0x%x\n", 490 bp->pdev->vendor, bp->pdev->device, bp->pdev->subsystem_vendor, 491 bp->pdev->subsystem_device, bp->pdev->irq, bp->pdev->devfn, 492 bp->pdev->class, bp->pdev->revision, bp->pdev->msi_enabled); 493 494 aux_dev = &bnxt_adev->aux_dev; 495 aux_dev->id = bnxt_adev->id; 496 aux_dev->name = "rdma"; 497 aux_dev->dev.parent = &bp->pdev->dev; 498 aux_dev->dev.release = bnxt_aux_dev_release; 499 500 if (!edev) { 501 edev = kzalloc(sizeof(*edev), GFP_KERNEL); 502 if (!edev) 503 return -ENOMEM; 504 } 505 506 bnxt_set_edev_info(edev, bp); 507 bnxt_adev->edev = edev; 508 bp->edev = edev; 509 510 ret = auxiliary_device_init(aux_dev); 511 if (ret) 512 goto err_free_edev; 513 514 ret = auxiliary_device_add(aux_dev); 515 if (ret) 516 goto err_dev_uninit; 517 518 return 0; 519 err_dev_uninit: 520 auxiliary_device_uninit(aux_dev); 521 err_free_edev: 522 kfree(edev); 523 bnxt_adev->edev = NULL; 524 bp->edev = NULL; 525 return ret; 526 } 527