1 /* 2 * Copyright (c) 2015-2024, Broadcom. All rights reserved. The term 3 * Broadcom refers to Broadcom Limited and/or its subsidiaries. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' 17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 18 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR 23 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 24 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 25 * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 26 * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * Description: RDMA Controller HW interface 29 */ 30 31 #include <linux/interrupt.h> 32 #include <linux/spinlock.h> 33 #include <linux/dma-mapping.h> 34 #include <linux/sched.h> 35 #include <linux/pci.h> 36 #include <linux/delay.h> 37 #include <linux/hardirq.h> 38 #include <linux/device.h> 39 40 #include "hsi_struct_def.h" 41 #include "qplib_tlv.h" 42 #include "qplib_res.h" 43 #include "qplib_sp.h" 44 #include "qplib_rcfw.h" 45 #include "bnxt_re.h" 46 47 static void bnxt_qplib_service_creq(unsigned long data); 48 49 int __check_cmdq_stall(struct bnxt_qplib_rcfw *rcfw, 50 u32 *cur_prod, u32 *cur_cons) 51 { 52 struct bnxt_qplib_cmdq_ctx *cmdq; 53 54 cmdq = &rcfw->cmdq; 55 56 if (*cur_prod == cmdq->hwq.prod && 57 *cur_cons == cmdq->hwq.cons) 58 /* No activity on CMDQ or CREQ. FW down */ 59 return -ETIMEDOUT; 60 61 *cur_prod = cmdq->hwq.prod; 62 *cur_cons = cmdq->hwq.cons; 63 return 0; 64 } 65 66 static int bnxt_qplib_map_rc(u8 opcode) 67 { 68 switch (opcode) { 69 case CMDQ_BASE_OPCODE_DESTROY_QP: 70 case CMDQ_BASE_OPCODE_DESTROY_SRQ: 71 case CMDQ_BASE_OPCODE_DESTROY_CQ: 72 case CMDQ_BASE_OPCODE_DEALLOCATE_KEY: 73 case CMDQ_BASE_OPCODE_DEREGISTER_MR: 74 case CMDQ_BASE_OPCODE_DELETE_GID: 75 case CMDQ_BASE_OPCODE_DESTROY_QP1: 76 case CMDQ_BASE_OPCODE_DESTROY_AH: 77 case CMDQ_BASE_OPCODE_DEINITIALIZE_FW: 78 case CMDQ_BASE_OPCODE_MODIFY_ROCE_CC: 79 case CMDQ_BASE_OPCODE_SET_LINK_AGGR_MODE: 80 return 0; 81 default: 82 return -ETIMEDOUT; 83 } 84 } 85 86 /** 87 * bnxt_re_is_fw_stalled - Check firmware health 88 * @rcfw - rcfw channel instance of rdev 89 * @cookie - cookie to track the command 90 * 91 * If firmware has not responded any rcfw command within 92 * rcfw->max_timeout, consider firmware as stalled. 93 * 94 * Returns: 95 * 0 if firmware is responding 96 * -ENODEV if firmware is not responding 97 */ 98 static int bnxt_re_is_fw_stalled(struct bnxt_qplib_rcfw *rcfw, u16 cookie) 99 { 100 struct bnxt_qplib_cmdq_ctx *cmdq; 101 struct bnxt_qplib_crsqe *crsqe; 102 103 crsqe = &rcfw->crsqe_tbl[cookie]; 104 cmdq = &rcfw->cmdq; 105 106 if (time_after(jiffies, cmdq->last_seen + 107 (rcfw->max_timeout * HZ))) { 108 dev_warn_ratelimited(&rcfw->pdev->dev, 109 "%s: FW STALL Detected. cmdq[%#x]=%#x waited (%ld > %d) msec active %d\n", 110 __func__, cookie, crsqe->opcode, 111 (long)jiffies_to_msecs(jiffies - cmdq->last_seen), 112 rcfw->max_timeout * 1000, 113 crsqe->is_in_used); 114 return -ENODEV; 115 } 116 117 return 0; 118 } 119 /** 120 * __wait_for_resp - Don't hold the cpu context and wait for response 121 * @rcfw - rcfw channel instance of rdev 122 * @cookie - cookie to track the command 123 * 124 * Wait for command completion in sleepable context. 125 * 126 * Returns: 127 * 0 if command is completed by firmware. 128 * Non zero error code for rest of the case. 129 */ 130 static int __wait_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) 131 { 132 struct bnxt_qplib_cmdq_ctx *cmdq; 133 struct bnxt_qplib_crsqe *crsqe; 134 unsigned long issue_time; 135 int ret; 136 137 cmdq = &rcfw->cmdq; 138 issue_time = jiffies; 139 crsqe = &rcfw->crsqe_tbl[cookie]; 140 141 do { 142 if (RCFW_NO_FW_ACCESS(rcfw)) 143 return bnxt_qplib_map_rc(crsqe->opcode); 144 if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) 145 return -ETIMEDOUT; 146 147 /* Non zero means command completed */ 148 ret = wait_event_timeout(cmdq->waitq, 149 !crsqe->is_in_used || 150 RCFW_NO_FW_ACCESS(rcfw), 151 msecs_to_jiffies(rcfw->max_timeout * 1000)); 152 153 if (!crsqe->is_in_used) 154 return 0; 155 /* 156 * Take care if interrupt miss or other cases like DBR drop 157 */ 158 bnxt_qplib_service_creq((unsigned long)rcfw); 159 dev_warn_ratelimited(&rcfw->pdev->dev, 160 "Non-Blocking QPLIB: cmdq[%#x]=%#x waited (%lu) msec bit %d\n", 161 cookie, crsqe->opcode, 162 (long)jiffies_to_msecs(jiffies - issue_time), 163 crsqe->is_in_used); 164 165 if (!crsqe->is_in_used) 166 return 0; 167 168 ret = bnxt_re_is_fw_stalled(rcfw, cookie); 169 if (ret) 170 return ret; 171 172 } while (true); 173 }; 174 175 /** 176 * __block_for_resp - hold the cpu context and wait for response 177 * @rcfw - rcfw channel instance of rdev 178 * @cookie - cookie to track the command 179 * 180 * This function will hold the cpu (non-sleepable context) and 181 * wait for command completion. Maximum holding interval is 8 second. 182 * 183 * Returns: 184 * -ETIMEOUT if command is not completed in specific time interval. 185 * 0 if command is completed by firmware. 186 */ 187 static int __block_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) 188 { 189 struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq; 190 struct bnxt_qplib_crsqe *crsqe; 191 unsigned long issue_time = 0; 192 193 issue_time = jiffies; 194 crsqe = &rcfw->crsqe_tbl[cookie]; 195 196 do { 197 if (RCFW_NO_FW_ACCESS(rcfw)) 198 return bnxt_qplib_map_rc(crsqe->opcode); 199 if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) 200 return -ETIMEDOUT; 201 202 udelay(1); 203 204 /* Below call is must since there can be a deadlock 205 * if interrupt is mapped to the same cpu 206 */ 207 bnxt_qplib_service_creq((unsigned long)rcfw); 208 if (!crsqe->is_in_used) 209 return 0; 210 211 } while (time_before(jiffies, issue_time + (8 * HZ))); 212 213 dev_warn_ratelimited(&rcfw->pdev->dev, 214 "Blocking QPLIB: cmdq[%#x]=%#x taken (%lu) msec", 215 cookie, crsqe->opcode, 216 (long)jiffies_to_msecs(jiffies - issue_time)); 217 218 return -ETIMEDOUT; 219 }; 220 221 /* __send_message_no_waiter - get cookie and post the message. 222 * @rcfw - rcfw channel instance of rdev 223 * @msg - qplib message internal 224 * 225 * This function will just post and don't bother about completion. 226 * Current design of this function is - 227 * user must hold the completion queue hwq->lock. 228 * user must have used existing completion and free the resources. 229 * this function will not check queue full condition. 230 * this function will explicitly set is_waiter_alive=false. 231 * current use case is - send destroy_ah if create_ah is return 232 * after waiter of create_ah is lost. It can be extended for other 233 * use case as well. 234 * 235 * Returns: Nothing 236 * 237 */ 238 static void __send_message_no_waiter(struct bnxt_qplib_rcfw *rcfw, 239 struct bnxt_qplib_cmdqmsg *msg) 240 { 241 struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq; 242 struct bnxt_qplib_hwq *cmdq_hwq = &cmdq->hwq; 243 struct bnxt_qplib_crsqe *crsqe; 244 struct bnxt_qplib_cmdqe *cmdqe; 245 u32 sw_prod, cmdq_prod, bsize; 246 u16 cookie; 247 u8 *preq; 248 249 cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE; 250 __set_cmdq_base_cookie(msg->req, msg->req_sz, cpu_to_le16(cookie)); 251 crsqe = &rcfw->crsqe_tbl[cookie]; 252 253 /* Set cmd_size in terms of 16B slots in req. */ 254 bsize = bnxt_qplib_set_cmd_slots(msg->req); 255 /* GET_CMD_SIZE would return number of slots in either case of tlv 256 * and non-tlv commands after call to bnxt_qplib_set_cmd_slots() 257 */ 258 crsqe->send_timestamp = jiffies; 259 crsqe->is_internal_cmd = true; 260 crsqe->is_waiter_alive = false; 261 crsqe->is_in_used = true; 262 crsqe->req_size = __get_cmdq_base_cmd_size(msg->req, msg->req_sz); 263 264 preq = (u8 *)msg->req; 265 do { 266 /* Locate the next cmdq slot */ 267 sw_prod = HWQ_CMP(cmdq_hwq->prod, cmdq_hwq); 268 cmdqe = bnxt_qplib_get_qe(cmdq_hwq, sw_prod, NULL); 269 /* Copy a segment of the req cmd to the cmdq */ 270 memset(cmdqe, 0, sizeof(*cmdqe)); 271 memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe))); 272 preq += min_t(u32, bsize, sizeof(*cmdqe)); 273 bsize -= min_t(u32, bsize, sizeof(*cmdqe)); 274 cmdq_hwq->prod++; 275 } while (bsize > 0); 276 cmdq->seq_num++; 277 278 cmdq_prod = cmdq_hwq->prod & 0xFFFF; 279 atomic_inc(&rcfw->timeout_send); 280 /* ring CMDQ DB */ 281 wmb(); 282 writel(cmdq_prod, cmdq->cmdq_mbox.prod); 283 writel(RCFW_CMDQ_TRIG_VAL, cmdq->cmdq_mbox.db); 284 } 285 286 static int __send_message(struct bnxt_qplib_rcfw *rcfw, 287 struct bnxt_qplib_cmdqmsg *msg) 288 { 289 u32 bsize, free_slots, required_slots; 290 struct bnxt_qplib_cmdq_ctx *cmdq; 291 struct bnxt_qplib_crsqe *crsqe; 292 struct bnxt_qplib_cmdqe *cmdqe; 293 struct bnxt_qplib_hwq *cmdq_hwq; 294 u32 sw_prod, cmdq_prod; 295 struct pci_dev *pdev; 296 unsigned long flags; 297 u16 cookie; 298 u8 opcode; 299 u8 *preq; 300 301 cmdq = &rcfw->cmdq; 302 cmdq_hwq = &cmdq->hwq; 303 pdev = rcfw->pdev; 304 opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz); 305 306 /* Cmdq are in 16-byte units, each request can consume 1 or more 307 cmdqe */ 308 spin_lock_irqsave(&cmdq_hwq->lock, flags); 309 required_slots = bnxt_qplib_get_cmd_slots(msg->req); 310 free_slots = HWQ_FREE_SLOTS(cmdq_hwq); 311 cookie = cmdq->seq_num & RCFW_MAX_COOKIE_VALUE; 312 crsqe = &rcfw->crsqe_tbl[cookie]; 313 314 if (required_slots >= free_slots) { 315 dev_warn_ratelimited(&pdev->dev, 316 "QPLIB: RCFW: CMDQ is full req/free %d/%d!\n", 317 required_slots, free_slots); 318 rcfw->cmdq_full_dbg++; 319 spin_unlock_irqrestore(&cmdq_hwq->lock, flags); 320 return -EAGAIN; 321 } 322 323 if (crsqe->is_in_used) 324 panic("QPLIB: Cookie was not requested %d\n", 325 cookie); 326 327 if (msg->block) 328 cookie |= RCFW_CMD_IS_BLOCKING; 329 __set_cmdq_base_cookie(msg->req, msg->req_sz, cpu_to_le16(cookie)); 330 331 /* Set cmd_size in terms of 16B slots in req. */ 332 bsize = bnxt_qplib_set_cmd_slots(msg->req); 333 /* GET_CMD_SIZE would return number of slots in either case of tlv 334 * and non-tlv commands after call to bnxt_qplib_set_cmd_slots() 335 */ 336 crsqe->send_timestamp = jiffies; 337 crsqe->free_slots = free_slots; 338 crsqe->resp = (struct creq_qp_event *)msg->resp; 339 crsqe->resp->cookie = cpu_to_le16(cookie); 340 crsqe->is_internal_cmd = false; 341 crsqe->is_waiter_alive = true; 342 crsqe->is_in_used = true; 343 crsqe->opcode = opcode; 344 crsqe->requested_qp_state = msg->qp_state; 345 346 crsqe->req_size = __get_cmdq_base_cmd_size(msg->req, msg->req_sz); 347 if (__get_cmdq_base_resp_size(msg->req, msg->req_sz) && msg->sb) { 348 struct bnxt_qplib_rcfw_sbuf *sbuf = msg->sb; 349 350 __set_cmdq_base_resp_addr(msg->req, msg->req_sz, 351 cpu_to_le64(sbuf->dma_addr)); 352 __set_cmdq_base_resp_size(msg->req, msg->req_sz, 353 ALIGN(sbuf->size, BNXT_QPLIB_CMDQE_UNITS) / 354 BNXT_QPLIB_CMDQE_UNITS); 355 } 356 357 preq = (u8 *)msg->req; 358 do { 359 /* Locate the next cmdq slot */ 360 sw_prod = HWQ_CMP(cmdq_hwq->prod, cmdq_hwq); 361 cmdqe = bnxt_qplib_get_qe(cmdq_hwq, sw_prod, NULL); 362 /* Copy a segment of the req cmd to the cmdq */ 363 memset(cmdqe, 0, sizeof(*cmdqe)); 364 memcpy(cmdqe, preq, min_t(u32, bsize, sizeof(*cmdqe))); 365 preq += min_t(u32, bsize, sizeof(*cmdqe)); 366 bsize -= min_t(u32, bsize, sizeof(*cmdqe)); 367 cmdq_hwq->prod++; 368 } while (bsize > 0); 369 cmdq->seq_num++; 370 371 cmdq_prod = cmdq_hwq->prod & 0xFFFF; 372 if (test_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags)) { 373 /* The very first doorbell write 374 * is required to set this flag 375 * which prompts the FW to reset 376 * its internal pointers 377 */ 378 cmdq_prod |= BIT(FIRMWARE_FIRST_FLAG); 379 clear_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags); 380 } 381 /* ring CMDQ DB */ 382 wmb(); 383 writel(cmdq_prod, cmdq->cmdq_mbox.prod); 384 writel(RCFW_CMDQ_TRIG_VAL, cmdq->cmdq_mbox.db); 385 386 dev_dbg(&pdev->dev, "QPLIB: RCFW sent request with 0x%x 0x%x 0x%x\n", 387 cmdq_prod, cmdq_hwq->prod, crsqe->req_size); 388 dev_dbg(&pdev->dev, 389 "QPLIB: opcode 0x%x with cookie 0x%x at cmdq/crsq 0x%p/0x%p\n", 390 opcode, 391 __get_cmdq_base_cookie(msg->req, msg->req_sz), 392 cmdqe, crsqe); 393 spin_unlock_irqrestore(&cmdq_hwq->lock, flags); 394 /* Return the CREQ response pointer */ 395 return 0; 396 } 397 398 /** 399 * __poll_for_resp - self poll completion for rcfw command 400 * @rcfw - rcfw channel instance of rdev 401 * @cookie - cookie to track the command 402 * 403 * It works same as __wait_for_resp except this function will 404 * do self polling in sort interval since interrupt is disabled. 405 * This function can not be called from non-sleepable context. 406 * 407 * Returns: 408 * -ETIMEOUT if command is not completed in specific time interval. 409 * 0 if command is completed by firmware. 410 */ 411 static int __poll_for_resp(struct bnxt_qplib_rcfw *rcfw, u16 cookie) 412 { 413 struct bnxt_qplib_cmdq_ctx *cmdq = &rcfw->cmdq; 414 struct bnxt_qplib_crsqe *crsqe; 415 unsigned long issue_time; 416 int ret; 417 418 issue_time = jiffies; 419 crsqe = &rcfw->crsqe_tbl[cookie]; 420 421 do { 422 if (RCFW_NO_FW_ACCESS(rcfw)) 423 return bnxt_qplib_map_rc(crsqe->opcode); 424 if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) 425 return -ETIMEDOUT; 426 427 usleep_range(1000, 1001); 428 429 bnxt_qplib_service_creq((unsigned long)rcfw); 430 if (!crsqe->is_in_used) 431 return 0; 432 433 if (jiffies_to_msecs(jiffies - issue_time) > 434 (rcfw->max_timeout * 1000)) { 435 dev_warn_ratelimited(&rcfw->pdev->dev, 436 "Self Polling QPLIB: cmdq[%#x]=%#x taken (%lu) msec", 437 cookie, crsqe->opcode, 438 (long)jiffies_to_msecs(jiffies - issue_time)); 439 ret = bnxt_re_is_fw_stalled(rcfw, cookie); 440 if (ret) 441 return ret; 442 } 443 } while (true); 444 445 }; 446 447 static int __send_message_basic_sanity(struct bnxt_qplib_rcfw *rcfw, 448 struct bnxt_qplib_cmdqmsg *msg, u8 opcode) 449 { 450 struct bnxt_qplib_cmdq_ctx *cmdq; 451 452 cmdq = &rcfw->cmdq; 453 454 /* Prevent posting if f/w is not in a state to process */ 455 if (RCFW_NO_FW_ACCESS(rcfw)) 456 return -ENXIO; 457 458 if (test_bit(FIRMWARE_STALL_DETECTED, &cmdq->flags)) 459 return -ETIMEDOUT; 460 461 if (test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) && 462 opcode == CMDQ_BASE_OPCODE_INITIALIZE_FW) { 463 dev_err(&rcfw->pdev->dev, "QPLIB: RCFW already initialized!\n"); 464 return -EINVAL; 465 } 466 467 if (!test_bit(FIRMWARE_INITIALIZED_FLAG, &cmdq->flags) && 468 (opcode != CMDQ_BASE_OPCODE_QUERY_FUNC && 469 opcode != CMDQ_BASE_OPCODE_INITIALIZE_FW && 470 opcode != CMDQ_BASE_OPCODE_QUERY_VERSION)) { 471 dev_err(&rcfw->pdev->dev, 472 "QPLIB: RCFW not initialized, reject opcode 0x%x\n", 473 opcode); 474 return -ENOTSUPP; 475 } 476 477 return 0; 478 } 479 480 /* This function will just post and do not bother about completion */ 481 static void __destroy_timedout_ah(struct bnxt_qplib_rcfw *rcfw, 482 struct creq_create_ah_resp *create_ah_resp) 483 { 484 struct bnxt_qplib_cmdqmsg msg = {}; 485 struct cmdq_destroy_ah req = {}; 486 487 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DESTROY_AH, 488 sizeof(req)); 489 req.ah_cid = create_ah_resp->xid; 490 msg.req = (struct cmdq_base *)&req; 491 msg.req_sz = sizeof(req); 492 __send_message_no_waiter(rcfw, &msg); 493 dev_warn_ratelimited(&rcfw->pdev->dev, 494 "From %s: ah_cid = %d timeout_send %d\n", __func__, 495 req.ah_cid, 496 atomic_read(&rcfw->timeout_send)); 497 } 498 499 /** 500 * __bnxt_qplib_rcfw_send_message - qplib interface to send 501 * and complete rcfw command. 502 * @rcfw - rcfw channel instance of rdev 503 * @msg - qplib message internal 504 * 505 * This function does not account shadow queue depth. It will send 506 * all the command unconditionally as long as send queue is not full. 507 * 508 * Returns: 509 * 0 if command completed by firmware. 510 * Non zero if the command is not completed by firmware. 511 */ 512 static int __bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, 513 struct bnxt_qplib_cmdqmsg *msg) 514 { 515 struct bnxt_qplib_crsqe *crsqe; 516 struct creq_qp_event *event; 517 unsigned long flags; 518 u16 cookie; 519 int rc = 0; 520 u8 opcode; 521 522 opcode = __get_cmdq_base_opcode(msg->req, msg->req_sz); 523 524 rc = __send_message_basic_sanity(rcfw, msg, opcode); 525 if (rc) 526 return rc == -ENXIO ? bnxt_qplib_map_rc(opcode) : rc; 527 528 rc = __send_message(rcfw, msg); 529 if (rc) 530 return rc; 531 532 cookie = le16_to_cpu(__get_cmdq_base_cookie(msg->req, 533 msg->req_sz)) & RCFW_MAX_COOKIE_VALUE; 534 535 536 if (msg->block) 537 rc = __block_for_resp(rcfw, cookie); 538 else if (atomic_read(&rcfw->rcfw_intr_enabled)) 539 rc = __wait_for_resp(rcfw, cookie); 540 else 541 rc = __poll_for_resp(rcfw, cookie); 542 543 if (rc) { 544 /* First check if it is FW stall. 545 * Use hwq.lock to avoid race with actual completion. 546 */ 547 spin_lock_irqsave(&rcfw->cmdq.hwq.lock, flags); 548 crsqe = &rcfw->crsqe_tbl[cookie]; 549 crsqe->is_waiter_alive = false; 550 if (rc == -ENODEV) 551 set_bit(FIRMWARE_STALL_DETECTED, &rcfw->cmdq.flags); 552 spin_unlock_irqrestore(&rcfw->cmdq.hwq.lock, flags); 553 554 return -ETIMEDOUT; 555 } 556 557 event = (struct creq_qp_event *)msg->resp; 558 if (event->status) { 559 /* failed with status */ 560 dev_err(&rcfw->pdev->dev, "QPLIB: cmdq[%#x]=%#x (%s) status %d\n", 561 cookie, opcode, GET_OPCODE_TYPE(opcode), event->status); 562 rc = -EFAULT; 563 /* 564 * Workaround to avoid errors in the stack during bond 565 * creation and deletion. 566 * Disable error returned for ADD_GID/DEL_GID 567 */ 568 if (opcode == CMDQ_BASE_OPCODE_ADD_GID || 569 opcode == CMDQ_BASE_OPCODE_DELETE_GID) 570 rc = 0; 571 } 572 573 dev_dbg(&pdev->dev, "QPLIB: %s:%d - op 0x%x (%s), cookie 0x%x -- Return: e->status 0x%x, rc = 0x%x\n", 574 __func__, __LINE__, opcode, GET_OPCODE_TYPE(opcode), cookie, event->status, rc); 575 return rc; 576 } 577 578 /** 579 * bnxt_qplib_rcfw_send_message - qplib interface to send 580 * and complete rcfw command. 581 * @rcfw - rcfw channel instance of rdev 582 * @msg - qplib message internal 583 * 584 * Driver interact with Firmware through rcfw channel/slow path in two ways. 585 * a. Blocking rcfw command send. In this path, driver cannot hold 586 * the context for longer period since it is holding cpu until 587 * command is not completed. 588 * b. Non-blocking rcfw command send. In this path, driver can hold the 589 * context for longer period. There may be many pending command waiting 590 * for completion because of non-blocking nature. 591 * 592 * Driver will use shadow queue depth. Current queue depth of 8K 593 * (due to size of rcfw message it can be actual ~4K rcfw outstanding) 594 * is not optimal for rcfw command processing in firmware. 595 * RCFW_CMD_NON_BLOCKING_SHADOW_QD is defined as 64. 596 * Restrict at max 64 Non-Blocking rcfw commands. 597 * Do not allow more than 64 non-blocking command to the Firmware. 598 * Allow all blocking commands until there is no queue full. 599 * 600 * Returns: 601 * 0 if command completed by firmware. 602 * Non zero if the command is not completed by firmware. 603 */ 604 int bnxt_qplib_rcfw_send_message(struct bnxt_qplib_rcfw *rcfw, 605 struct bnxt_qplib_cmdqmsg *msg) 606 { 607 int ret; 608 609 if (!msg->block) { 610 down(&rcfw->rcfw_inflight); 611 ret = __bnxt_qplib_rcfw_send_message(rcfw, msg); 612 up(&rcfw->rcfw_inflight); 613 } else { 614 ret = __bnxt_qplib_rcfw_send_message(rcfw, msg); 615 } 616 617 return ret; 618 } 619 620 static void bnxt_re_add_perf_stats(struct bnxt_qplib_rcfw *rcfw, 621 struct bnxt_qplib_crsqe *crsqe) 622 { 623 u32 latency_msec, dest_stats_id; 624 u64 *dest_stats_ptr = NULL; 625 626 latency_msec = jiffies_to_msecs(rcfw->cmdq.last_seen - 627 crsqe->send_timestamp); 628 if (latency_msec/1000 < RCFW_MAX_LATENCY_SEC_SLAB_INDEX) 629 rcfw->rcfw_lat_slab_sec[latency_msec/1000]++; 630 631 if (!rcfw->sp_perf_stats_enabled) 632 return; 633 634 if (latency_msec < RCFW_MAX_LATENCY_MSEC_SLAB_INDEX) 635 rcfw->rcfw_lat_slab_msec[latency_msec]++; 636 637 switch (crsqe->opcode) { 638 case CMDQ_BASE_OPCODE_CREATE_QP: 639 dest_stats_id = rcfw->qp_create_stats_id++; 640 dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; 641 dest_stats_ptr = &rcfw->qp_create_stats[dest_stats_id]; 642 break; 643 case CMDQ_BASE_OPCODE_DESTROY_QP: 644 dest_stats_id = rcfw->qp_destroy_stats_id++; 645 dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; 646 dest_stats_ptr = &rcfw->qp_destroy_stats[dest_stats_id]; 647 break; 648 case CMDQ_BASE_OPCODE_REGISTER_MR: 649 dest_stats_id = rcfw->mr_create_stats_id++; 650 dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; 651 dest_stats_ptr = &rcfw->mr_create_stats[dest_stats_id]; 652 break; 653 case CMDQ_BASE_OPCODE_DEREGISTER_MR: 654 case CMDQ_BASE_OPCODE_DEALLOCATE_KEY: 655 dest_stats_id = rcfw->mr_destroy_stats_id++; 656 dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; 657 dest_stats_ptr = &rcfw->mr_destroy_stats[dest_stats_id]; 658 break; 659 case CMDQ_BASE_OPCODE_MODIFY_QP: 660 if (crsqe->requested_qp_state != IB_QPS_ERR) 661 break; 662 dest_stats_id = rcfw->qp_modify_stats_id++; 663 dest_stats_id = dest_stats_id % RCFW_MAX_STAT_INDEX; 664 dest_stats_ptr = &rcfw->qp_modify_stats[dest_stats_id]; 665 break; 666 default: 667 break; 668 } 669 if (dest_stats_ptr) 670 *dest_stats_ptr = max_t(unsigned long, 671 (rcfw->cmdq.last_seen - crsqe->send_timestamp), 1); 672 673 } 674 675 /* Completions */ 676 static int bnxt_qplib_process_qp_event(struct bnxt_qplib_rcfw *rcfw, 677 struct creq_qp_event *event, 678 u32 *num_wait) 679 { 680 struct bnxt_qplib_hwq *cmdq_hwq = &rcfw->cmdq.hwq; 681 struct creq_cq_error_notification *cqerr; 682 struct creq_qp_error_notification *qperr; 683 struct bnxt_qplib_crsqe *crsqe; 684 struct bnxt_qplib_reftbl *tbl; 685 struct bnxt_qplib_qp *qp; 686 struct bnxt_qplib_cq *cq; 687 u16 cookie, blocked = 0; 688 struct pci_dev *pdev; 689 bool is_waiter_alive; 690 unsigned long flags; 691 u32 wait_cmds = 0; 692 u32 xid, qp_idx; 693 u32 req_size; 694 int rc = 0; 695 696 pdev = rcfw->pdev; 697 switch (event->event) { 698 case CREQ_QP_EVENT_EVENT_QP_ERROR_NOTIFICATION: 699 tbl = &rcfw->res->reftbl.qpref; 700 qperr = (struct creq_qp_error_notification *)event; 701 xid = le32_to_cpu(qperr->xid); 702 qp_idx = map_qp_id_to_tbl_indx(xid, tbl); 703 spin_lock(&tbl->lock); 704 qp = tbl->rec[qp_idx].handle; 705 if (!qp) { 706 spin_unlock(&tbl->lock); 707 break; 708 } 709 bnxt_qplib_mark_qp_error(qp); 710 rc = rcfw->creq.aeq_handler(rcfw, event, qp); 711 spin_unlock(&tbl->lock); 712 /* 713 * Keeping these prints as debug to avoid flooding of log 714 * messages during modify QP to error state by applications 715 */ 716 dev_dbg(&pdev->dev, "QPLIB: QP Error encountered!\n"); 717 dev_dbg(&pdev->dev, 718 "QPLIB: qpid 0x%x, req_err=0x%x, resp_err=0x%x\n", 719 xid, qperr->req_err_state_reason, 720 qperr->res_err_state_reason); 721 break; 722 case CREQ_QP_EVENT_EVENT_CQ_ERROR_NOTIFICATION: 723 tbl = &rcfw->res->reftbl.cqref; 724 cqerr = (struct creq_cq_error_notification *)event; 725 xid = le32_to_cpu(cqerr->xid); 726 spin_lock(&tbl->lock); 727 cq = tbl->rec[GET_TBL_INDEX(xid, tbl)].handle; 728 if (!cq) { 729 spin_unlock(&tbl->lock); 730 break; 731 } 732 rc = rcfw->creq.aeq_handler(rcfw, event, cq); 733 spin_unlock(&tbl->lock); 734 dev_dbg(&pdev->dev, "QPLIB: CQ error encountered!\n"); 735 break; 736 default: 737 /* 738 * Command Response 739 * cmdq hwq lock needs to be acquired to synchronize 740 * the command send and completion reaping. This function 741 * is always called with creq hwq lock held. So there is no 742 * chance of deadlock here as the locking is in correct sequence. 743 * Using the nested variant of spin_lock to annotate 744 */ 745 spin_lock_irqsave_nested(&cmdq_hwq->lock, flags, 746 SINGLE_DEPTH_NESTING); 747 cookie = le16_to_cpu(event->cookie); 748 blocked = cookie & RCFW_CMD_IS_BLOCKING; 749 cookie &= RCFW_MAX_COOKIE_VALUE; 750 751 crsqe = &rcfw->crsqe_tbl[cookie]; 752 753 bnxt_re_add_perf_stats(rcfw, crsqe); 754 755 if (WARN_ONCE(test_bit(FIRMWARE_STALL_DETECTED, 756 &rcfw->cmdq.flags), 757 "QPLIB: Unreponsive rcfw channel detected.!!")) { 758 dev_info(&pdev->dev, "rcfw timedout: cookie = %#x," 759 " latency_msec = %ld free_slots = %d\n", cookie, 760 (long)jiffies_to_msecs(rcfw->cmdq.last_seen - 761 crsqe->send_timestamp), 762 crsqe->free_slots); 763 spin_unlock_irqrestore(&cmdq_hwq->lock, flags); 764 return rc; 765 } 766 767 if (crsqe->is_internal_cmd && !event->status) 768 atomic_dec(&rcfw->timeout_send); 769 770 if (crsqe->is_waiter_alive) { 771 if (crsqe->resp) 772 memcpy(crsqe->resp, event, sizeof(*event)); 773 if (!blocked) 774 wait_cmds++; 775 } 776 777 req_size = crsqe->req_size; 778 is_waiter_alive = crsqe->is_waiter_alive; 779 780 crsqe->req_size = 0; 781 if (!crsqe->is_waiter_alive) 782 crsqe->resp = NULL; 783 crsqe->is_in_used = false; 784 /* Consumer is updated so that __send_message_no_waiter 785 * can never see queue full. 786 * It is safe since we are still holding cmdq_hwq->lock. 787 */ 788 cmdq_hwq->cons += req_size; 789 790 /* This is a case to handle below scenario - 791 * Create AH is completed successfully by firmware, 792 * but completion took more time and driver already lost 793 * the context of create_ah from caller. 794 * We have already return failure for create_ah verbs, 795 * so let's destroy the same address vector since it is 796 * no more used in stack. We don't care about completion 797 * in __send_message_no_waiter. 798 * If destroy_ah is failued by firmware, there will be AH 799 * resource leak and relatively not critical + unlikely 800 * scenario. Current design is not to handle such case. 801 */ 802 if (!is_waiter_alive && !event->status && 803 event->event == CREQ_QP_EVENT_EVENT_CREATE_AH) 804 __destroy_timedout_ah(rcfw, 805 (struct creq_create_ah_resp *) 806 event); 807 808 spin_unlock_irqrestore(&cmdq_hwq->lock, flags); 809 } 810 *num_wait += wait_cmds; 811 return rc; 812 } 813 814 /* SP - CREQ Completion handlers */ 815 static void bnxt_qplib_service_creq(unsigned long data) 816 { 817 struct bnxt_qplib_rcfw *rcfw = (struct bnxt_qplib_rcfw *)data; 818 struct bnxt_qplib_creq_ctx *creq = &rcfw->creq; 819 struct bnxt_qplib_res *res; 820 u32 type, budget = CREQ_ENTRY_POLL_BUDGET; 821 struct bnxt_qplib_hwq *creq_hwq = &creq->hwq; 822 struct creq_base *creqe; 823 struct pci_dev *pdev; 824 unsigned long flags; 825 u32 num_wakeup = 0; 826 int rc; 827 828 pdev = rcfw->pdev; 829 res = rcfw->res; 830 /* Service the CREQ until empty */ 831 spin_lock_irqsave(&creq_hwq->lock, flags); 832 while (budget > 0) { 833 if (RCFW_NO_FW_ACCESS(rcfw)) { 834 spin_unlock_irqrestore(&creq_hwq->lock, flags); 835 return; 836 } 837 creqe = bnxt_qplib_get_qe(creq_hwq, creq_hwq->cons, NULL); 838 if (!CREQ_CMP_VALID(creqe, creq->creq_db.dbinfo.flags)) 839 break; 840 /* The valid test of the entry must be done first before 841 * reading any further. 842 */ 843 dma_rmb(); 844 type = creqe->type & CREQ_BASE_TYPE_MASK; 845 rcfw->cmdq.last_seen = jiffies; 846 847 switch (type) { 848 case CREQ_BASE_TYPE_QP_EVENT: 849 bnxt_qplib_process_qp_event 850 (rcfw,(struct creq_qp_event *)creqe, 851 &num_wakeup); 852 creq->stats.creq_qp_event_processed++; 853 break; 854 case CREQ_BASE_TYPE_FUNC_EVENT: 855 rc = rcfw->creq.aeq_handler(rcfw, creqe, NULL); 856 if (rc) 857 dev_warn(&pdev->dev, 858 "QPLIB: async event type = 0x%x not handled", 859 type); 860 creq->stats.creq_func_event_processed++; 861 break; 862 default: 863 if (type != HWRM_ASYNC_EVENT_CMPL_TYPE_HWRM_ASYNC_EVENT) { 864 dev_warn(&pdev->dev, 865 "QPLIB: op_event = 0x%x not handled\n", 866 type); 867 } 868 break; 869 } 870 budget--; 871 bnxt_qplib_hwq_incr_cons(creq_hwq->max_elements, &creq_hwq->cons, 872 1, &creq->creq_db.dbinfo.flags); 873 } 874 if (budget == CREQ_ENTRY_POLL_BUDGET && 875 !CREQ_CMP_VALID(creqe, creq->creq_db.dbinfo.flags)) { 876 /* No completions received during this poll. Enable interrupt now */ 877 bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true); 878 creq->stats.creq_arm_count++; 879 dev_dbg(&pdev->dev, "QPLIB: Num of Func (0x%llx) \n", 880 creq->stats.creq_func_event_processed); 881 dev_dbg(&pdev->dev, "QPLIB: QP (0x%llx) events processed\n", 882 creq->stats.creq_qp_event_processed); 883 dev_dbg(&pdev->dev, "QPLIB: Armed:%#llx resched:%#llx \n", 884 creq->stats.creq_arm_count, 885 creq->stats.creq_tasklet_schedule_count); 886 } else if (creq->requested) { 887 /* 888 * Currently there is no bottom half implementation to process 889 * completions, all completions are processed in interrupt context 890 * only. So enable interrupts. 891 */ 892 bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true); 893 creq->stats.creq_tasklet_schedule_count++; 894 } 895 spin_unlock_irqrestore(&creq_hwq->lock, flags); 896 if (num_wakeup) 897 wake_up_all(&rcfw->cmdq.waitq); 898 } 899 900 static irqreturn_t bnxt_qplib_creq_irq(int irq, void *dev_instance) 901 { 902 struct bnxt_qplib_rcfw *rcfw = dev_instance; 903 904 bnxt_qplib_service_creq((unsigned long)rcfw); 905 return IRQ_HANDLED; 906 } 907 908 /* RCFW */ 909 int bnxt_qplib_deinit_rcfw(struct bnxt_qplib_rcfw *rcfw) 910 { 911 struct creq_deinitialize_fw_resp resp = {}; 912 struct cmdq_deinitialize_fw req = {}; 913 struct bnxt_qplib_cmdqmsg msg = {}; 914 int rc; 915 916 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_DEINITIALIZE_FW, 917 sizeof(req)); 918 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, 919 sizeof(req), sizeof(resp), 0); 920 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); 921 if (rc) 922 return rc; 923 clear_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags); 924 return 0; 925 } 926 927 int bnxt_qplib_init_rcfw(struct bnxt_qplib_rcfw *rcfw, int is_virtfn) 928 { 929 struct creq_initialize_fw_resp resp = {}; 930 struct cmdq_initialize_fw req = {}; 931 struct bnxt_qplib_cmdqmsg msg = {}; 932 struct bnxt_qplib_chip_ctx *cctx; 933 struct bnxt_qplib_ctx *hctx; 934 struct bnxt_qplib_res *res; 935 struct bnxt_qplib_hwq *hwq; 936 int rc; 937 938 res = rcfw->res; 939 cctx = res->cctx; 940 hctx = res->hctx; 941 942 bnxt_qplib_rcfw_cmd_prep(&req, CMDQ_BASE_OPCODE_INITIALIZE_FW, 943 sizeof(req)); 944 /* Supply (log-base-2-of-host-page-size - base-page-shift) 945 * to bono to adjust the doorbell page sizes. 946 */ 947 req.log2_dbr_pg_size = cpu_to_le16(PAGE_SHIFT - 948 RCFW_DBR_BASE_PAGE_SHIFT); 949 /* 950 * VFs need not setup the HW context area, PF 951 * shall setup this area for VF. Skipping the 952 * HW programming 953 */ 954 if (is_virtfn || _is_chip_gen_p5_p7(cctx)) 955 goto skip_ctx_setup; 956 957 hwq = &hctx->qp_ctx.hwq; 958 req.qpc_page_dir = cpu_to_le64(_get_base_addr(hwq)); 959 req.number_of_qp = cpu_to_le32(hwq->max_elements); 960 req.qpc_pg_size_qpc_lvl = (_get_pte_pg_size(hwq) << 961 CMDQ_INITIALIZE_FW_QPC_PG_SIZE_SFT) | 962 (u8)hwq->level; 963 964 hwq = &hctx->mrw_ctx.hwq; 965 req.mrw_page_dir = cpu_to_le64(_get_base_addr(hwq)); 966 req.number_of_mrw = cpu_to_le32(hwq->max_elements); 967 req.mrw_pg_size_mrw_lvl = (_get_pte_pg_size(hwq) << 968 CMDQ_INITIALIZE_FW_MRW_PG_SIZE_SFT) | 969 (u8)hwq->level; 970 971 hwq = &hctx->srq_ctx.hwq; 972 req.srq_page_dir = cpu_to_le64(_get_base_addr(hwq)); 973 req.number_of_srq = cpu_to_le32(hwq->max_elements); 974 req.srq_pg_size_srq_lvl = (_get_pte_pg_size(hwq) << 975 CMDQ_INITIALIZE_FW_SRQ_PG_SIZE_SFT) | 976 (u8)hwq->level; 977 978 hwq = &hctx->cq_ctx.hwq; 979 req.cq_page_dir = cpu_to_le64(_get_base_addr(hwq)); 980 req.number_of_cq = cpu_to_le32(hwq->max_elements); 981 req.cq_pg_size_cq_lvl = (_get_pte_pg_size(hwq) << 982 CMDQ_INITIALIZE_FW_CQ_PG_SIZE_SFT) | 983 (u8)hwq->level; 984 985 hwq = &hctx->tim_ctx.hwq; 986 req.tim_page_dir = cpu_to_le64(_get_base_addr(hwq)); 987 req.tim_pg_size_tim_lvl = (_get_pte_pg_size(hwq) << 988 CMDQ_INITIALIZE_FW_TIM_PG_SIZE_SFT) | 989 (u8)hwq->level; 990 hwq = &hctx->tqm_ctx.pde; 991 req.tqm_page_dir = cpu_to_le64(_get_base_addr(hwq)); 992 req.tqm_pg_size_tqm_lvl = (_get_pte_pg_size(hwq) << 993 CMDQ_INITIALIZE_FW_TQM_PG_SIZE_SFT) | 994 (u8)hwq->level; 995 skip_ctx_setup: 996 if (BNXT_RE_HW_RETX(res->dattr->dev_cap_flags)) 997 req.flags |= CMDQ_INITIALIZE_FW_FLAGS_HW_REQUESTER_RETX_SUPPORTED; 998 req.stat_ctx_id = cpu_to_le32(hctx->stats.fw_id); 999 bnxt_qplib_fill_cmdqmsg(&msg, &req, &resp, NULL, 1000 sizeof(req), sizeof(resp), 0); 1001 rc = bnxt_qplib_rcfw_send_message(rcfw, &msg); 1002 if (rc) 1003 return rc; 1004 set_bit(FIRMWARE_INITIALIZED_FLAG, &rcfw->cmdq.flags); 1005 1006 return 0; 1007 } 1008 1009 void bnxt_qplib_free_rcfw_channel(struct bnxt_qplib_res *res) 1010 { 1011 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1012 1013 vfree(rcfw->rcfw_lat_slab_msec); 1014 rcfw->rcfw_lat_slab_msec = NULL; 1015 vfree(rcfw->qp_create_stats); 1016 rcfw->qp_create_stats = NULL; 1017 vfree(rcfw->qp_destroy_stats); 1018 rcfw->qp_destroy_stats = NULL; 1019 vfree(rcfw->mr_create_stats); 1020 rcfw->mr_create_stats = NULL; 1021 vfree(rcfw->mr_destroy_stats); 1022 rcfw->mr_destroy_stats = NULL; 1023 vfree(rcfw->qp_modify_stats); 1024 rcfw->qp_modify_stats = NULL; 1025 rcfw->sp_perf_stats_enabled = false; 1026 1027 kfree(rcfw->crsqe_tbl); 1028 rcfw->crsqe_tbl = NULL; 1029 1030 bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq); 1031 bnxt_qplib_free_hwq(res, &rcfw->creq.hwq); 1032 rcfw->pdev = NULL; 1033 } 1034 1035 int bnxt_qplib_alloc_rcfw_channel(struct bnxt_qplib_res *res) 1036 { 1037 struct bnxt_qplib_hwq_attr hwq_attr = {}; 1038 struct bnxt_qplib_rcfw *rcfw = res->rcfw; 1039 struct bnxt_qplib_sg_info sginfo = {}; 1040 struct bnxt_qplib_cmdq_ctx *cmdq; 1041 struct bnxt_qplib_creq_ctx *creq; 1042 1043 rcfw->pdev = res->pdev; 1044 rcfw->res = res; 1045 cmdq = &rcfw->cmdq; 1046 creq = &rcfw->creq; 1047 1048 sginfo.pgsize = PAGE_SIZE; 1049 sginfo.pgshft = PAGE_SHIFT; 1050 1051 hwq_attr.sginfo = &sginfo; 1052 hwq_attr.res = rcfw->res; 1053 hwq_attr.depth = BNXT_QPLIB_CREQE_MAX_CNT; 1054 hwq_attr.stride = BNXT_QPLIB_CREQE_UNITS; 1055 hwq_attr.type = _get_hwq_type(res); 1056 1057 if (bnxt_qplib_alloc_init_hwq(&creq->hwq, &hwq_attr)) { 1058 dev_err(&rcfw->pdev->dev, 1059 "QPLIB: HW channel CREQ allocation failed\n"); 1060 return -ENOMEM; 1061 } 1062 1063 sginfo.pgsize = BNXT_QPLIB_CMDQE_PAGE_SIZE; 1064 hwq_attr.depth = BNXT_QPLIB_CMDQE_MAX_CNT & 0x7FFFFFFF; 1065 hwq_attr.stride = BNXT_QPLIB_CMDQE_UNITS; 1066 hwq_attr.type = HWQ_TYPE_CTX; 1067 if (bnxt_qplib_alloc_init_hwq(&cmdq->hwq, &hwq_attr)) { 1068 dev_err(&rcfw->pdev->dev, 1069 "QPLIB: HW channel CMDQ allocation failed\n"); 1070 goto fail_free_creq_hwq; 1071 } 1072 1073 rcfw->crsqe_tbl = kcalloc(cmdq->hwq.max_elements, 1074 sizeof(*rcfw->crsqe_tbl), GFP_KERNEL); 1075 if (!rcfw->crsqe_tbl) { 1076 dev_err(&rcfw->pdev->dev, 1077 "QPLIB: HW channel CRSQ allocation failed\n"); 1078 goto fail_free_cmdq_hwq; 1079 } 1080 1081 rcfw->max_timeout = res->cctx->hwrm_cmd_max_timeout; 1082 1083 rcfw->sp_perf_stats_enabled = false; 1084 rcfw->rcfw_lat_slab_msec = vzalloc(sizeof(u32) * 1085 RCFW_MAX_LATENCY_MSEC_SLAB_INDEX); 1086 rcfw->qp_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); 1087 rcfw->qp_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); 1088 rcfw->mr_create_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); 1089 rcfw->mr_destroy_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); 1090 rcfw->qp_modify_stats = vzalloc(sizeof(u64) * RCFW_MAX_STAT_INDEX); 1091 1092 if (rcfw->rcfw_lat_slab_msec && 1093 rcfw->qp_create_stats && 1094 rcfw->qp_destroy_stats && 1095 rcfw->mr_create_stats && 1096 rcfw->mr_destroy_stats && 1097 rcfw->qp_modify_stats) 1098 rcfw->sp_perf_stats_enabled = true; 1099 1100 return 0; 1101 fail_free_cmdq_hwq: 1102 bnxt_qplib_free_hwq(res, &rcfw->cmdq.hwq); 1103 fail_free_creq_hwq: 1104 bnxt_qplib_free_hwq(res, &rcfw->creq.hwq); 1105 return -ENOMEM; 1106 } 1107 1108 void bnxt_qplib_rcfw_stop_irq(struct bnxt_qplib_rcfw *rcfw, bool kill) 1109 { 1110 struct bnxt_qplib_creq_ctx *creq; 1111 struct bnxt_qplib_res *res; 1112 1113 creq = &rcfw->creq; 1114 res = rcfw->res; 1115 1116 if (!creq->requested) 1117 return; 1118 1119 creq->requested = false; 1120 /* Mask h/w interrupts */ 1121 bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, false); 1122 /* Sync with last running IRQ-handler */ 1123 synchronize_irq(creq->msix_vec); 1124 free_irq(creq->msix_vec, rcfw); 1125 kfree(creq->irq_name); 1126 creq->irq_name = NULL; 1127 /* rcfw_intr_enabled should not be greater than 1. Debug 1128 * print to check if that is the case 1129 */ 1130 if (atomic_read(&rcfw->rcfw_intr_enabled) > 1) { 1131 dev_err(&rcfw->pdev->dev, 1132 "%s: rcfw->rcfw_intr_enabled = 0x%x\n", __func__, 1133 atomic_read(&rcfw->rcfw_intr_enabled)); 1134 } 1135 atomic_set(&rcfw->rcfw_intr_enabled, 0); 1136 rcfw->num_irq_stopped++; 1137 } 1138 1139 void bnxt_qplib_disable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw) 1140 { 1141 struct bnxt_qplib_creq_ctx *creq; 1142 struct bnxt_qplib_cmdq_ctx *cmdq; 1143 1144 creq = &rcfw->creq; 1145 cmdq = &rcfw->cmdq; 1146 /* Make sure the HW channel is stopped! */ 1147 bnxt_qplib_rcfw_stop_irq(rcfw, true); 1148 1149 creq->creq_db.reg.bar_reg = NULL; 1150 creq->creq_db.db = NULL; 1151 1152 if (cmdq->cmdq_mbox.reg.bar_reg) { 1153 iounmap(cmdq->cmdq_mbox.reg.bar_reg); 1154 cmdq->cmdq_mbox.reg.bar_reg = NULL; 1155 cmdq->cmdq_mbox.prod = NULL; 1156 cmdq->cmdq_mbox.db = NULL; 1157 } 1158 1159 creq->aeq_handler = NULL; 1160 creq->msix_vec = 0; 1161 } 1162 1163 int bnxt_qplib_rcfw_start_irq(struct bnxt_qplib_rcfw *rcfw, int msix_vector, 1164 bool need_init) 1165 { 1166 struct bnxt_qplib_creq_ctx *creq; 1167 struct bnxt_qplib_res *res; 1168 int rc; 1169 1170 creq = &rcfw->creq; 1171 res = rcfw->res; 1172 1173 if (creq->requested) 1174 return -EFAULT; 1175 1176 creq->msix_vec = msix_vector; 1177 1178 creq->irq_name = kasprintf(GFP_KERNEL, "bnxt_re-creq@pci:%s\n", 1179 pci_name(res->pdev)); 1180 if (!creq->irq_name) 1181 return -ENOMEM; 1182 1183 rc = request_irq(creq->msix_vec, bnxt_qplib_creq_irq, 0, 1184 creq->irq_name, rcfw); 1185 if (rc) { 1186 kfree(creq->irq_name); 1187 creq->irq_name = NULL; 1188 return rc; 1189 } 1190 creq->requested = true; 1191 1192 bnxt_qplib_ring_nq_db(&creq->creq_db.dbinfo, res->cctx, true); 1193 1194 rcfw->num_irq_started++; 1195 /* Debug print to check rcfw interrupt enable/disable is invoked 1196 * out of sequence 1197 */ 1198 if (atomic_read(&rcfw->rcfw_intr_enabled) > 0) { 1199 dev_err(&rcfw->pdev->dev, 1200 "%s: rcfw->rcfw_intr_enabled = 0x%x\n", __func__, 1201 atomic_read(&rcfw->rcfw_intr_enabled)); 1202 } 1203 atomic_inc(&rcfw->rcfw_intr_enabled); 1204 return 0; 1205 } 1206 1207 static int bnxt_qplib_map_cmdq_mbox(struct bnxt_qplib_rcfw *rcfw) 1208 { 1209 struct bnxt_qplib_cmdq_mbox *mbox; 1210 resource_size_t bar_reg; 1211 struct pci_dev *pdev; 1212 1213 pdev = rcfw->pdev; 1214 mbox = &rcfw->cmdq.cmdq_mbox; 1215 1216 mbox->reg.bar_id = RCFW_COMM_PCI_BAR_REGION; 1217 mbox->reg.len = RCFW_COMM_SIZE; 1218 mbox->reg.bar_base = pci_resource_start(pdev, mbox->reg.bar_id); 1219 if (!mbox->reg.bar_base) { 1220 dev_err(&pdev->dev, 1221 "QPLIB: CMDQ BAR region %d resc start is 0!\n", 1222 mbox->reg.bar_id); 1223 return -ENOMEM; 1224 } 1225 1226 bar_reg = mbox->reg.bar_base + RCFW_COMM_BASE_OFFSET; 1227 mbox->reg.len = RCFW_COMM_SIZE; 1228 mbox->reg.bar_reg = ioremap(bar_reg, mbox->reg.len); 1229 if (!mbox->reg.bar_reg) { 1230 dev_err(&pdev->dev, 1231 "QPLIB: CMDQ BAR region %d mapping failed\n", 1232 mbox->reg.bar_id); 1233 return -ENOMEM; 1234 } 1235 1236 mbox->prod = (void __iomem *)((char *)mbox->reg.bar_reg + 1237 RCFW_PF_VF_COMM_PROD_OFFSET); 1238 mbox->db = (void __iomem *)((char *)mbox->reg.bar_reg + 1239 RCFW_COMM_TRIG_OFFSET); 1240 return 0; 1241 } 1242 1243 static int bnxt_qplib_map_creq_db(struct bnxt_qplib_rcfw *rcfw, u32 reg_offt) 1244 { 1245 struct bnxt_qplib_creq_db *creq_db; 1246 struct bnxt_qplib_reg_desc *dbreg; 1247 struct bnxt_qplib_res *res; 1248 1249 res = rcfw->res; 1250 creq_db = &rcfw->creq.creq_db; 1251 dbreg = &res->dpi_tbl.ucreg; 1252 1253 creq_db->reg.bar_id = dbreg->bar_id; 1254 creq_db->reg.bar_base = dbreg->bar_base; 1255 creq_db->reg.bar_reg = dbreg->bar_reg + reg_offt; 1256 creq_db->reg.len = _is_chip_gen_p5_p7(res->cctx) ? sizeof(u64) : 1257 sizeof(u32); 1258 1259 creq_db->dbinfo.db = creq_db->reg.bar_reg; 1260 creq_db->dbinfo.hwq = &rcfw->creq.hwq; 1261 creq_db->dbinfo.xid = rcfw->creq.ring_id; 1262 creq_db->dbinfo.seed = rcfw->creq.ring_id; 1263 creq_db->dbinfo.flags = 0; 1264 spin_lock_init(&creq_db->dbinfo.lock); 1265 creq_db->dbinfo.shadow_key = BNXT_QPLIB_DBR_KEY_INVALID; 1266 creq_db->dbinfo.res = rcfw->res; 1267 1268 return 0; 1269 } 1270 1271 static void bnxt_qplib_start_rcfw(struct bnxt_qplib_rcfw *rcfw) 1272 { 1273 struct bnxt_qplib_cmdq_ctx *cmdq; 1274 struct bnxt_qplib_creq_ctx *creq; 1275 struct bnxt_qplib_cmdq_mbox *mbox; 1276 struct cmdq_init init = {0}; 1277 1278 cmdq = &rcfw->cmdq; 1279 creq = &rcfw->creq; 1280 mbox = &cmdq->cmdq_mbox; 1281 1282 init.cmdq_pbl = cpu_to_le64(cmdq->hwq.pbl[PBL_LVL_0].pg_map_arr[0]); 1283 init.cmdq_size_cmdq_lvl = cpu_to_le16( 1284 ((BNXT_QPLIB_CMDQE_MAX_CNT << CMDQ_INIT_CMDQ_SIZE_SFT) & 1285 CMDQ_INIT_CMDQ_SIZE_MASK) | 1286 ((cmdq->hwq.level << CMDQ_INIT_CMDQ_LVL_SFT) & 1287 CMDQ_INIT_CMDQ_LVL_MASK)); 1288 init.creq_ring_id = cpu_to_le16(creq->ring_id); 1289 /* Write to the Bono mailbox register */ 1290 __iowrite32_copy(mbox->reg.bar_reg, &init, sizeof(init) / 4); 1291 } 1292 1293 int bnxt_qplib_enable_rcfw_channel(struct bnxt_qplib_rcfw *rcfw, 1294 int msix_vector, 1295 int cp_bar_reg_off, 1296 aeq_handler_t aeq_handler) 1297 { 1298 struct bnxt_qplib_cmdq_ctx *cmdq; 1299 struct bnxt_qplib_creq_ctx *creq; 1300 int rc; 1301 1302 cmdq = &rcfw->cmdq; 1303 creq = &rcfw->creq; 1304 1305 /* Clear to defaults */ 1306 cmdq->seq_num = 0; 1307 set_bit(FIRMWARE_FIRST_FLAG, &cmdq->flags); 1308 init_waitqueue_head(&cmdq->waitq); 1309 1310 creq->stats.creq_qp_event_processed = 0; 1311 creq->stats.creq_func_event_processed = 0; 1312 creq->aeq_handler = aeq_handler; 1313 1314 rc = bnxt_qplib_map_cmdq_mbox(rcfw); 1315 if (rc) 1316 return rc; 1317 1318 rc = bnxt_qplib_map_creq_db(rcfw, cp_bar_reg_off); 1319 if (rc) 1320 return rc; 1321 1322 rc = bnxt_qplib_rcfw_start_irq(rcfw, msix_vector, true); 1323 if (rc) { 1324 dev_err(&rcfw->pdev->dev, 1325 "QPLIB: Failed to request IRQ for CREQ rc = 0x%x\n", rc); 1326 bnxt_qplib_disable_rcfw_channel(rcfw); 1327 return rc; 1328 } 1329 1330 rcfw->curr_shadow_qd = min_not_zero(cmdq_shadow_qd, 1331 (unsigned int)RCFW_CMD_NON_BLOCKING_SHADOW_QD); 1332 sema_init(&rcfw->rcfw_inflight, rcfw->curr_shadow_qd); 1333 dev_dbg(&rcfw->pdev->dev, 1334 "Perf Debug: shadow qd %d\n", rcfw->curr_shadow_qd); 1335 bnxt_qplib_start_rcfw(rcfw); 1336 1337 return 0; 1338 } 1339