1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) 2 /* QLogic qed NIC Driver 3 * Copyright (c) 2015-2017 QLogic Corporation 4 * Copyright (c) 2019-2020 Marvell International Ltd. 5 */ 6 7 #include <linux/types.h> 8 #include <asm/byteorder.h> 9 #include <linux/io.h> 10 #include <linux/delay.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/errno.h> 13 #include <linux/kernel.h> 14 #include <linux/list.h> 15 #include <linux/pci.h> 16 #include <linux/slab.h> 17 #include <linux/spinlock.h> 18 #include <linux/string.h> 19 #include "qed.h" 20 #include "qed_cxt.h" 21 #include "qed_dev_api.h" 22 #include "qed_hsi.h" 23 #include "qed_iro_hsi.h" 24 #include "qed_hw.h" 25 #include "qed_int.h" 26 #include "qed_iscsi.h" 27 #include "qed_mcp.h" 28 #include "qed_ooo.h" 29 #include "qed_reg_addr.h" 30 #include "qed_sp.h" 31 #include "qed_sriov.h" 32 #include "qed_rdma.h" 33 34 /*************************************************************************** 35 * Structures & Definitions 36 ***************************************************************************/ 37 38 #define SPQ_HIGH_PRI_RESERVE_DEFAULT (1) 39 40 #define SPQ_BLOCK_DELAY_MAX_ITER (10) 41 #define SPQ_BLOCK_DELAY_US (10) 42 #define SPQ_BLOCK_SLEEP_MAX_ITER (1000) 43 #define SPQ_BLOCK_SLEEP_MS (5) 44 45 /*************************************************************************** 46 * Blocking Imp. (BLOCK/EBLOCK mode) 47 ***************************************************************************/ 48 static void qed_spq_blocking_cb(struct qed_hwfn *p_hwfn, 49 void *cookie, 50 union event_ring_data *data, u8 fw_return_code) 51 { 52 struct qed_spq_comp_done *comp_done; 53 54 comp_done = (struct qed_spq_comp_done *)cookie; 55 56 comp_done->fw_return_code = fw_return_code; 57 58 /* Make sure completion done is visible on waiting thread */ 59 smp_store_release(&comp_done->done, 0x1); 60 } 61 62 static int __qed_spq_block(struct qed_hwfn *p_hwfn, 63 struct qed_spq_entry *p_ent, 64 u8 *p_fw_ret, bool sleep_between_iter) 65 { 66 struct qed_spq_comp_done *comp_done; 67 u32 iter_cnt; 68 69 comp_done = (struct qed_spq_comp_done *)p_ent->comp_cb.cookie; 70 iter_cnt = sleep_between_iter ? SPQ_BLOCK_SLEEP_MAX_ITER 71 : SPQ_BLOCK_DELAY_MAX_ITER; 72 73 while (iter_cnt--) { 74 /* Validate we receive completion update */ 75 if (smp_load_acquire(&comp_done->done) == 1) { /* ^^^ */ 76 if (p_fw_ret) 77 *p_fw_ret = comp_done->fw_return_code; 78 return 0; 79 } 80 81 if (sleep_between_iter) 82 msleep(SPQ_BLOCK_SLEEP_MS); 83 else 84 udelay(SPQ_BLOCK_DELAY_US); 85 } 86 87 return -EBUSY; 88 } 89 90 static int qed_spq_block(struct qed_hwfn *p_hwfn, 91 struct qed_spq_entry *p_ent, 92 u8 *p_fw_ret, bool skip_quick_poll) 93 { 94 struct qed_spq_comp_done *comp_done; 95 struct qed_ptt *p_ptt; 96 int rc; 97 98 /* A relatively short polling period w/o sleeping, to allow the FW to 99 * complete the ramrod and thus possibly to avoid the following sleeps. 100 */ 101 if (!skip_quick_poll) { 102 rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, false); 103 if (!rc) 104 return 0; 105 } 106 107 /* Move to polling with a sleeping period between iterations */ 108 rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, true); 109 if (!rc) 110 return 0; 111 112 p_ptt = qed_ptt_acquire(p_hwfn); 113 if (!p_ptt) { 114 DP_NOTICE(p_hwfn, "ptt, failed to acquire\n"); 115 return -EAGAIN; 116 } 117 118 DP_INFO(p_hwfn, "Ramrod is stuck, requesting MCP drain\n"); 119 rc = qed_mcp_drain(p_hwfn, p_ptt); 120 qed_ptt_release(p_hwfn, p_ptt); 121 if (rc) { 122 DP_NOTICE(p_hwfn, "MCP drain failed\n"); 123 goto err; 124 } 125 126 /* Retry after drain */ 127 rc = __qed_spq_block(p_hwfn, p_ent, p_fw_ret, true); 128 if (!rc) 129 return 0; 130 131 comp_done = (struct qed_spq_comp_done *)p_ent->comp_cb.cookie; 132 if (comp_done->done == 1) { 133 if (p_fw_ret) 134 *p_fw_ret = comp_done->fw_return_code; 135 return 0; 136 } 137 err: 138 p_ptt = qed_ptt_acquire(p_hwfn); 139 if (!p_ptt) 140 return -EBUSY; 141 qed_hw_err_notify(p_hwfn, p_ptt, QED_HW_ERR_RAMROD_FAIL, 142 "Ramrod is stuck [CID %08x %s:%02x %s:%02x echo %04x]\n", 143 le32_to_cpu(p_ent->elem.hdr.cid), 144 qed_get_ramrod_cmd_id_str(p_ent->elem.hdr.protocol_id, 145 p_ent->elem.hdr.cmd_id), 146 p_ent->elem.hdr.cmd_id, 147 qed_get_protocol_type_str(p_ent->elem.hdr.protocol_id), 148 p_ent->elem.hdr.protocol_id, 149 le16_to_cpu(p_ent->elem.hdr.echo)); 150 qed_ptt_release(p_hwfn, p_ptt); 151 152 return -EBUSY; 153 } 154 155 /*************************************************************************** 156 * SPQ entries inner API 157 ***************************************************************************/ 158 static int qed_spq_fill_entry(struct qed_hwfn *p_hwfn, 159 struct qed_spq_entry *p_ent) 160 { 161 p_ent->flags = 0; 162 163 switch (p_ent->comp_mode) { 164 case QED_SPQ_MODE_EBLOCK: 165 case QED_SPQ_MODE_BLOCK: 166 p_ent->comp_cb.function = qed_spq_blocking_cb; 167 break; 168 case QED_SPQ_MODE_CB: 169 break; 170 default: 171 DP_NOTICE(p_hwfn, "Unknown SPQE completion mode %d\n", 172 p_ent->comp_mode); 173 return -EINVAL; 174 } 175 176 DP_VERBOSE(p_hwfn, 177 QED_MSG_SPQ, 178 "Ramrod hdr: [CID 0x%08x %s:0x%02x %s:0x%02x] Data ptr: [%08x:%08x] Cmpltion Mode: %s\n", 179 p_ent->elem.hdr.cid, 180 qed_get_ramrod_cmd_id_str(p_ent->elem.hdr.protocol_id, 181 p_ent->elem.hdr.cmd_id), 182 p_ent->elem.hdr.cmd_id, 183 qed_get_protocol_type_str(p_ent->elem.hdr.protocol_id), 184 p_ent->elem.hdr.protocol_id, 185 p_ent->elem.data_ptr.hi, p_ent->elem.data_ptr.lo, 186 D_TRINE(p_ent->comp_mode, QED_SPQ_MODE_EBLOCK, 187 QED_SPQ_MODE_BLOCK, "MODE_EBLOCK", "MODE_BLOCK", 188 "MODE_CB")); 189 190 return 0; 191 } 192 193 /*************************************************************************** 194 * HSI access 195 ***************************************************************************/ 196 static void qed_spq_hw_initialize(struct qed_hwfn *p_hwfn, 197 struct qed_spq *p_spq) 198 { 199 struct core_conn_context *p_cxt; 200 struct qed_cxt_info cxt_info; 201 u16 physical_q; 202 int rc; 203 204 cxt_info.iid = p_spq->cid; 205 206 rc = qed_cxt_get_cid_info(p_hwfn, &cxt_info); 207 208 if (rc < 0) { 209 DP_NOTICE(p_hwfn, "Cannot find context info for cid=%d\n", 210 p_spq->cid); 211 return; 212 } 213 214 p_cxt = cxt_info.p_cxt; 215 216 SET_FIELD(p_cxt->xstorm_ag_context.flags10, 217 XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN, 1); 218 SET_FIELD(p_cxt->xstorm_ag_context.flags1, 219 XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE, 1); 220 SET_FIELD(p_cxt->xstorm_ag_context.flags9, 221 XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN, 1); 222 223 /* QM physical queue */ 224 physical_q = qed_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LB); 225 p_cxt->xstorm_ag_context.physical_q0 = cpu_to_le16(physical_q); 226 227 p_cxt->xstorm_st_context.spq_base_addr.lo = 228 DMA_LO_LE(p_spq->chain.p_phys_addr); 229 p_cxt->xstorm_st_context.spq_base_addr.hi = 230 DMA_HI_LE(p_spq->chain.p_phys_addr); 231 } 232 233 static int qed_spq_hw_post(struct qed_hwfn *p_hwfn, 234 struct qed_spq *p_spq, struct qed_spq_entry *p_ent) 235 { 236 struct qed_chain *p_chain = &p_hwfn->p_spq->chain; 237 struct core_db_data *p_db_data = &p_spq->db_data; 238 u16 echo = qed_chain_get_prod_idx(p_chain); 239 struct slow_path_element *elem; 240 241 p_ent->elem.hdr.echo = cpu_to_le16(echo); 242 elem = qed_chain_produce(p_chain); 243 if (!elem) { 244 DP_NOTICE(p_hwfn, "Failed to produce from SPQ chain\n"); 245 return -EINVAL; 246 } 247 248 *elem = p_ent->elem; /* struct assignment */ 249 250 /* send a doorbell on the slow hwfn session */ 251 p_db_data->spq_prod = cpu_to_le16(qed_chain_get_prod_idx(p_chain)); 252 253 /* make sure the SPQE is updated before the doorbell */ 254 wmb(); 255 256 DOORBELL(p_hwfn, p_spq->db_addr_offset, *(u32 *)p_db_data); 257 258 /* make sure doorbell is rang */ 259 wmb(); 260 261 DP_VERBOSE(p_hwfn, QED_MSG_SPQ, 262 "Doorbelled [0x%08x, CID 0x%08x] with Flags: %02x agg_params: %02x, prod: %04x\n", 263 p_spq->db_addr_offset, 264 p_spq->cid, 265 p_db_data->params, 266 p_db_data->agg_flags, qed_chain_get_prod_idx(p_chain)); 267 268 return 0; 269 } 270 271 /*************************************************************************** 272 * Asynchronous events 273 ***************************************************************************/ 274 static int 275 qed_async_event_completion(struct qed_hwfn *p_hwfn, 276 struct event_ring_entry *p_eqe) 277 { 278 qed_spq_async_comp_cb cb; 279 280 if (!p_hwfn->p_spq) 281 return -EINVAL; 282 283 if (p_eqe->protocol_id >= MAX_PROTOCOL_TYPE) { 284 DP_ERR(p_hwfn, "Wrong protocol: %s:%d\n", 285 qed_get_protocol_type_str(p_eqe->protocol_id), 286 p_eqe->protocol_id); 287 288 return -EINVAL; 289 } 290 291 cb = p_hwfn->p_spq->async_comp_cb[p_eqe->protocol_id]; 292 if (cb) { 293 return cb(p_hwfn, p_eqe->opcode, p_eqe->echo, 294 &p_eqe->data, p_eqe->fw_return_code); 295 } else { 296 DP_NOTICE(p_hwfn, 297 "Unknown Async completion for %s:%d\n", 298 qed_get_protocol_type_str(p_eqe->protocol_id), 299 p_eqe->protocol_id); 300 301 return -EINVAL; 302 } 303 } 304 305 int 306 qed_spq_register_async_cb(struct qed_hwfn *p_hwfn, 307 enum protocol_type protocol_id, 308 qed_spq_async_comp_cb cb) 309 { 310 if (!p_hwfn->p_spq || (protocol_id >= MAX_PROTOCOL_TYPE)) 311 return -EINVAL; 312 313 p_hwfn->p_spq->async_comp_cb[protocol_id] = cb; 314 return 0; 315 } 316 317 void 318 qed_spq_unregister_async_cb(struct qed_hwfn *p_hwfn, 319 enum protocol_type protocol_id) 320 { 321 if (!p_hwfn->p_spq || (protocol_id >= MAX_PROTOCOL_TYPE)) 322 return; 323 324 p_hwfn->p_spq->async_comp_cb[protocol_id] = NULL; 325 } 326 327 /*************************************************************************** 328 * EQ API 329 ***************************************************************************/ 330 void qed_eq_prod_update(struct qed_hwfn *p_hwfn, u16 prod) 331 { 332 u32 addr = GET_GTT_REG_ADDR(GTT_BAR0_MAP_REG_USDM_RAM, 333 USTORM_EQE_CONS, p_hwfn->rel_pf_id); 334 335 REG_WR16(p_hwfn, addr, prod); 336 } 337 338 int qed_eq_completion(struct qed_hwfn *p_hwfn, void *cookie) 339 { 340 struct qed_eq *p_eq = cookie; 341 struct qed_chain *p_chain = &p_eq->chain; 342 int rc = 0; 343 344 /* take a snapshot of the FW consumer */ 345 u16 fw_cons_idx = le16_to_cpu(*p_eq->p_fw_cons); 346 347 DP_VERBOSE(p_hwfn, QED_MSG_SPQ, "fw_cons_idx %x\n", fw_cons_idx); 348 349 /* Need to guarantee the fw_cons index we use points to a usuable 350 * element (to comply with our chain), so our macros would comply 351 */ 352 if ((fw_cons_idx & qed_chain_get_usable_per_page(p_chain)) == 353 qed_chain_get_usable_per_page(p_chain)) 354 fw_cons_idx += qed_chain_get_unusable_per_page(p_chain); 355 356 /* Complete current segment of eq entries */ 357 while (fw_cons_idx != qed_chain_get_cons_idx(p_chain)) { 358 struct event_ring_entry *p_eqe = qed_chain_consume(p_chain); 359 360 if (!p_eqe) { 361 rc = -EINVAL; 362 break; 363 } 364 365 DP_VERBOSE(p_hwfn, QED_MSG_SPQ, 366 "op %x prot %x res0 %x echo %x fwret %x flags %x\n", 367 p_eqe->opcode, 368 p_eqe->protocol_id, 369 p_eqe->reserved0, 370 le16_to_cpu(p_eqe->echo), 371 p_eqe->fw_return_code, 372 p_eqe->flags); 373 374 if (GET_FIELD(p_eqe->flags, EVENT_RING_ENTRY_ASYNC)) { 375 if (qed_async_event_completion(p_hwfn, p_eqe)) 376 rc = -EINVAL; 377 } else if (qed_spq_completion(p_hwfn, 378 p_eqe->echo, 379 p_eqe->fw_return_code, 380 &p_eqe->data)) { 381 rc = -EINVAL; 382 } 383 384 qed_chain_recycle_consumed(p_chain); 385 } 386 387 qed_eq_prod_update(p_hwfn, qed_chain_get_prod_idx(p_chain)); 388 389 /* Attempt to post pending requests */ 390 spin_lock_bh(&p_hwfn->p_spq->lock); 391 rc = qed_spq_pend_post(p_hwfn); 392 spin_unlock_bh(&p_hwfn->p_spq->lock); 393 394 return rc; 395 } 396 397 int qed_eq_alloc(struct qed_hwfn *p_hwfn, u16 num_elem) 398 { 399 struct qed_chain_init_params params = { 400 .mode = QED_CHAIN_MODE_PBL, 401 .intended_use = QED_CHAIN_USE_TO_PRODUCE, 402 .cnt_type = QED_CHAIN_CNT_TYPE_U16, 403 .num_elems = num_elem, 404 .elem_size = sizeof(union event_ring_element), 405 }; 406 struct qed_eq *p_eq; 407 int ret; 408 409 /* Allocate EQ struct */ 410 p_eq = kzalloc(sizeof(*p_eq), GFP_KERNEL); 411 if (!p_eq) 412 return -ENOMEM; 413 414 ret = qed_chain_alloc(p_hwfn->cdev, &p_eq->chain, ¶ms); 415 if (ret) { 416 DP_NOTICE(p_hwfn, "Failed to allocate EQ chain\n"); 417 goto eq_allocate_fail; 418 } 419 420 /* register EQ completion on the SP SB */ 421 qed_int_register_cb(p_hwfn, qed_eq_completion, 422 p_eq, &p_eq->eq_sb_index, &p_eq->p_fw_cons); 423 424 p_hwfn->p_eq = p_eq; 425 return 0; 426 427 eq_allocate_fail: 428 kfree(p_eq); 429 430 return ret; 431 } 432 433 void qed_eq_setup(struct qed_hwfn *p_hwfn) 434 { 435 qed_chain_reset(&p_hwfn->p_eq->chain); 436 } 437 438 void qed_eq_free(struct qed_hwfn *p_hwfn) 439 { 440 if (!p_hwfn->p_eq) 441 return; 442 443 qed_chain_free(p_hwfn->cdev, &p_hwfn->p_eq->chain); 444 445 kfree(p_hwfn->p_eq); 446 p_hwfn->p_eq = NULL; 447 } 448 449 /*************************************************************************** 450 * CQE API - manipulate EQ functionality 451 ***************************************************************************/ 452 static int qed_cqe_completion(struct qed_hwfn *p_hwfn, 453 struct eth_slow_path_rx_cqe *cqe, 454 enum protocol_type protocol) 455 { 456 if (IS_VF(p_hwfn->cdev)) 457 return 0; 458 459 /* @@@tmp - it's possible we'll eventually want to handle some 460 * actual commands that can arrive here, but for now this is only 461 * used to complete the ramrod using the echo value on the cqe 462 */ 463 return qed_spq_completion(p_hwfn, cqe->echo, 0, NULL); 464 } 465 466 int qed_eth_cqe_completion(struct qed_hwfn *p_hwfn, 467 struct eth_slow_path_rx_cqe *cqe) 468 { 469 int rc; 470 471 rc = qed_cqe_completion(p_hwfn, cqe, PROTOCOLID_ETH); 472 if (rc) 473 DP_NOTICE(p_hwfn, 474 "Failed to handle RXQ CQE [cmd 0x%02x]\n", 475 cqe->ramrod_cmd_id); 476 477 return rc; 478 } 479 480 /*************************************************************************** 481 * Slow hwfn Queue (spq) 482 ***************************************************************************/ 483 void qed_spq_setup(struct qed_hwfn *p_hwfn) 484 { 485 struct qed_spq *p_spq = p_hwfn->p_spq; 486 struct qed_spq_entry *p_virt = NULL; 487 struct core_db_data *p_db_data; 488 void __iomem *db_addr; 489 dma_addr_t p_phys = 0; 490 u32 i, capacity; 491 int rc; 492 493 INIT_LIST_HEAD(&p_spq->pending); 494 INIT_LIST_HEAD(&p_spq->completion_pending); 495 INIT_LIST_HEAD(&p_spq->free_pool); 496 INIT_LIST_HEAD(&p_spq->unlimited_pending); 497 spin_lock_init(&p_spq->lock); 498 499 /* SPQ empty pool */ 500 p_phys = p_spq->p_phys + offsetof(struct qed_spq_entry, ramrod); 501 p_virt = p_spq->p_virt; 502 503 capacity = qed_chain_get_capacity(&p_spq->chain); 504 for (i = 0; i < capacity; i++) { 505 DMA_REGPAIR_LE(p_virt->elem.data_ptr, p_phys); 506 507 list_add_tail(&p_virt->list, &p_spq->free_pool); 508 509 p_virt++; 510 p_phys += sizeof(struct qed_spq_entry); 511 } 512 513 /* Statistics */ 514 p_spq->normal_count = 0; 515 p_spq->comp_count = 0; 516 p_spq->comp_sent_count = 0; 517 p_spq->unlimited_pending_count = 0; 518 519 bitmap_zero(p_spq->p_comp_bitmap, SPQ_RING_SIZE); 520 p_spq->comp_bitmap_idx = 0; 521 522 /* SPQ cid, cannot fail */ 523 qed_cxt_acquire_cid(p_hwfn, PROTOCOLID_CORE, &p_spq->cid); 524 qed_spq_hw_initialize(p_hwfn, p_spq); 525 526 /* reset the chain itself */ 527 qed_chain_reset(&p_spq->chain); 528 529 /* Initialize the address/data of the SPQ doorbell */ 530 p_spq->db_addr_offset = qed_db_addr(p_spq->cid, DQ_DEMS_LEGACY); 531 p_db_data = &p_spq->db_data; 532 memset(p_db_data, 0, sizeof(*p_db_data)); 533 SET_FIELD(p_db_data->params, CORE_DB_DATA_DEST, DB_DEST_XCM); 534 SET_FIELD(p_db_data->params, CORE_DB_DATA_AGG_CMD, DB_AGG_CMD_MAX); 535 SET_FIELD(p_db_data->params, CORE_DB_DATA_AGG_VAL_SEL, 536 DQ_XCM_CORE_SPQ_PROD_CMD); 537 p_db_data->agg_flags = DQ_XCM_CORE_DQ_CF_CMD; 538 539 /* Register the SPQ doorbell with the doorbell recovery mechanism */ 540 db_addr = (void __iomem *)((u8 __iomem *)p_hwfn->doorbells + 541 p_spq->db_addr_offset); 542 rc = qed_db_recovery_add(p_hwfn->cdev, db_addr, &p_spq->db_data, 543 DB_REC_WIDTH_32B, DB_REC_KERNEL); 544 if (rc) 545 DP_INFO(p_hwfn, 546 "Failed to register the SPQ doorbell with the doorbell recovery mechanism\n"); 547 } 548 549 int qed_spq_alloc(struct qed_hwfn *p_hwfn) 550 { 551 struct qed_chain_init_params params = { 552 .mode = QED_CHAIN_MODE_SINGLE, 553 .intended_use = QED_CHAIN_USE_TO_PRODUCE, 554 .cnt_type = QED_CHAIN_CNT_TYPE_U16, 555 .elem_size = sizeof(struct slow_path_element), 556 }; 557 struct qed_dev *cdev = p_hwfn->cdev; 558 struct qed_spq_entry *p_virt = NULL; 559 struct qed_spq *p_spq = NULL; 560 dma_addr_t p_phys = 0; 561 u32 capacity; 562 int ret; 563 564 /* SPQ struct */ 565 p_spq = kzalloc(sizeof(*p_spq), GFP_KERNEL); 566 if (!p_spq) 567 return -ENOMEM; 568 569 /* SPQ ring */ 570 ret = qed_chain_alloc(cdev, &p_spq->chain, ¶ms); 571 if (ret) { 572 DP_NOTICE(p_hwfn, "Failed to allocate SPQ chain\n"); 573 goto spq_chain_alloc_fail; 574 } 575 576 /* allocate and fill the SPQ elements (incl. ramrod data list) */ 577 capacity = qed_chain_get_capacity(&p_spq->chain); 578 ret = -ENOMEM; 579 580 p_virt = dma_alloc_coherent(&cdev->pdev->dev, 581 capacity * sizeof(struct qed_spq_entry), 582 &p_phys, GFP_KERNEL); 583 if (!p_virt) 584 goto spq_alloc_fail; 585 586 p_spq->p_virt = p_virt; 587 p_spq->p_phys = p_phys; 588 p_hwfn->p_spq = p_spq; 589 590 return 0; 591 592 spq_alloc_fail: 593 qed_chain_free(cdev, &p_spq->chain); 594 spq_chain_alloc_fail: 595 kfree(p_spq); 596 597 return ret; 598 } 599 600 void qed_spq_free(struct qed_hwfn *p_hwfn) 601 { 602 struct qed_spq *p_spq = p_hwfn->p_spq; 603 void __iomem *db_addr; 604 u32 capacity; 605 606 if (!p_spq) 607 return; 608 609 /* Delete the SPQ doorbell from the doorbell recovery mechanism */ 610 db_addr = (void __iomem *)((u8 __iomem *)p_hwfn->doorbells + 611 p_spq->db_addr_offset); 612 qed_db_recovery_del(p_hwfn->cdev, db_addr, &p_spq->db_data); 613 614 if (p_spq->p_virt) { 615 capacity = qed_chain_get_capacity(&p_spq->chain); 616 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 617 capacity * 618 sizeof(struct qed_spq_entry), 619 p_spq->p_virt, p_spq->p_phys); 620 } 621 622 qed_chain_free(p_hwfn->cdev, &p_spq->chain); 623 kfree(p_spq); 624 p_hwfn->p_spq = NULL; 625 } 626 627 int qed_spq_get_entry(struct qed_hwfn *p_hwfn, struct qed_spq_entry **pp_ent) 628 { 629 struct qed_spq *p_spq = p_hwfn->p_spq; 630 struct qed_spq_entry *p_ent = NULL; 631 int rc = 0; 632 633 spin_lock_bh(&p_spq->lock); 634 635 if (list_empty(&p_spq->free_pool)) { 636 p_ent = kzalloc(sizeof(*p_ent), GFP_ATOMIC); 637 if (!p_ent) { 638 DP_NOTICE(p_hwfn, 639 "Failed to allocate an SPQ entry for a pending ramrod\n"); 640 rc = -ENOMEM; 641 goto out_unlock; 642 } 643 p_ent->queue = &p_spq->unlimited_pending; 644 } else { 645 p_ent = list_first_entry(&p_spq->free_pool, 646 struct qed_spq_entry, list); 647 list_del(&p_ent->list); 648 p_ent->queue = &p_spq->pending; 649 } 650 651 *pp_ent = p_ent; 652 653 out_unlock: 654 spin_unlock_bh(&p_spq->lock); 655 return rc; 656 } 657 658 /* Locked variant; Should be called while the SPQ lock is taken */ 659 static void __qed_spq_return_entry(struct qed_hwfn *p_hwfn, 660 struct qed_spq_entry *p_ent) 661 { 662 list_add_tail(&p_ent->list, &p_hwfn->p_spq->free_pool); 663 } 664 665 void qed_spq_return_entry(struct qed_hwfn *p_hwfn, struct qed_spq_entry *p_ent) 666 { 667 spin_lock_bh(&p_hwfn->p_spq->lock); 668 __qed_spq_return_entry(p_hwfn, p_ent); 669 spin_unlock_bh(&p_hwfn->p_spq->lock); 670 } 671 672 /** 673 * qed_spq_add_entry() - Add a new entry to the pending list. 674 * Should be used while lock is being held. 675 * 676 * @p_hwfn: HW device data. 677 * @p_ent: An entry to add. 678 * @priority: Desired priority. 679 * 680 * Adds an entry to the pending list is there is room (an empty 681 * element is available in the free_pool), or else places the 682 * entry in the unlimited_pending pool. 683 * 684 * Return: zero on success, -EINVAL on invalid @priority. 685 */ 686 static int qed_spq_add_entry(struct qed_hwfn *p_hwfn, 687 struct qed_spq_entry *p_ent, 688 enum spq_priority priority) 689 { 690 struct qed_spq *p_spq = p_hwfn->p_spq; 691 692 if (p_ent->queue == &p_spq->unlimited_pending) { 693 if (list_empty(&p_spq->free_pool)) { 694 list_add_tail(&p_ent->list, &p_spq->unlimited_pending); 695 p_spq->unlimited_pending_count++; 696 697 return 0; 698 } else { 699 struct qed_spq_entry *p_en2; 700 701 p_en2 = list_first_entry(&p_spq->free_pool, 702 struct qed_spq_entry, list); 703 list_del(&p_en2->list); 704 705 /* Copy the ring element physical pointer to the new 706 * entry, since we are about to override the entire ring 707 * entry and don't want to lose the pointer. 708 */ 709 p_ent->elem.data_ptr = p_en2->elem.data_ptr; 710 711 *p_en2 = *p_ent; 712 713 /* EBLOCK responsible to free the allocated p_ent */ 714 if (p_ent->comp_mode != QED_SPQ_MODE_EBLOCK) 715 kfree(p_ent); 716 else 717 p_ent->post_ent = p_en2; 718 719 p_ent = p_en2; 720 } 721 } 722 723 /* entry is to be placed in 'pending' queue */ 724 switch (priority) { 725 case QED_SPQ_PRIORITY_NORMAL: 726 list_add_tail(&p_ent->list, &p_spq->pending); 727 p_spq->normal_count++; 728 break; 729 case QED_SPQ_PRIORITY_HIGH: 730 list_add(&p_ent->list, &p_spq->pending); 731 p_spq->high_count++; 732 break; 733 default: 734 return -EINVAL; 735 } 736 737 return 0; 738 } 739 740 /*************************************************************************** 741 * Accessor 742 ***************************************************************************/ 743 u32 qed_spq_get_cid(struct qed_hwfn *p_hwfn) 744 { 745 if (!p_hwfn->p_spq) 746 return 0xffffffff; /* illegal */ 747 return p_hwfn->p_spq->cid; 748 } 749 750 /*************************************************************************** 751 * Posting new Ramrods 752 ***************************************************************************/ 753 static int qed_spq_post_list(struct qed_hwfn *p_hwfn, 754 struct list_head *head, u32 keep_reserve) 755 { 756 struct qed_spq *p_spq = p_hwfn->p_spq; 757 int rc; 758 759 while (qed_chain_get_elem_left(&p_spq->chain) > keep_reserve && 760 !list_empty(head)) { 761 struct qed_spq_entry *p_ent = 762 list_first_entry(head, struct qed_spq_entry, list); 763 list_move_tail(&p_ent->list, &p_spq->completion_pending); 764 p_spq->comp_sent_count++; 765 766 rc = qed_spq_hw_post(p_hwfn, p_spq, p_ent); 767 if (rc) { 768 list_del(&p_ent->list); 769 __qed_spq_return_entry(p_hwfn, p_ent); 770 return rc; 771 } 772 } 773 774 return 0; 775 } 776 777 int qed_spq_pend_post(struct qed_hwfn *p_hwfn) 778 { 779 struct qed_spq *p_spq = p_hwfn->p_spq; 780 struct qed_spq_entry *p_ent = NULL; 781 782 while (!list_empty(&p_spq->free_pool)) { 783 if (list_empty(&p_spq->unlimited_pending)) 784 break; 785 786 p_ent = list_first_entry(&p_spq->unlimited_pending, 787 struct qed_spq_entry, list); 788 if (!p_ent) 789 return -EINVAL; 790 791 list_del(&p_ent->list); 792 793 qed_spq_add_entry(p_hwfn, p_ent, p_ent->priority); 794 } 795 796 return qed_spq_post_list(p_hwfn, &p_spq->pending, 797 SPQ_HIGH_PRI_RESERVE_DEFAULT); 798 } 799 800 static void qed_spq_recov_set_ret_code(struct qed_spq_entry *p_ent, 801 u8 *fw_return_code) 802 { 803 if (!fw_return_code) 804 return; 805 806 if (p_ent->elem.hdr.protocol_id == PROTOCOLID_ROCE || 807 p_ent->elem.hdr.protocol_id == PROTOCOLID_IWARP) 808 *fw_return_code = RDMA_RETURN_OK; 809 } 810 811 /* Avoid overriding of SPQ entries when getting out-of-order completions, by 812 * marking the completions in a bitmap and increasing the chain consumer only 813 * for the first successive completed entries. 814 */ 815 static void qed_spq_comp_bmap_update(struct qed_hwfn *p_hwfn, __le16 echo) 816 { 817 u16 pos = le16_to_cpu(echo) % SPQ_RING_SIZE; 818 struct qed_spq *p_spq = p_hwfn->p_spq; 819 820 __set_bit(pos, p_spq->p_comp_bitmap); 821 while (test_bit(p_spq->comp_bitmap_idx, 822 p_spq->p_comp_bitmap)) { 823 __clear_bit(p_spq->comp_bitmap_idx, 824 p_spq->p_comp_bitmap); 825 p_spq->comp_bitmap_idx++; 826 qed_chain_return_produced(&p_spq->chain); 827 } 828 } 829 830 int qed_spq_post(struct qed_hwfn *p_hwfn, 831 struct qed_spq_entry *p_ent, u8 *fw_return_code) 832 { 833 int rc = 0; 834 struct qed_spq *p_spq = p_hwfn ? p_hwfn->p_spq : NULL; 835 bool b_ret_ent = true; 836 bool eblock; 837 838 if (!p_hwfn) 839 return -EINVAL; 840 841 if (!p_ent) { 842 DP_NOTICE(p_hwfn, "Got a NULL pointer\n"); 843 return -EINVAL; 844 } 845 846 if (p_hwfn->cdev->recov_in_prog) { 847 DP_VERBOSE(p_hwfn, 848 QED_MSG_SPQ, 849 "Recovery is in progress. Skip spq post [%s:%02x %s:%02x]\n", 850 qed_get_ramrod_cmd_id_str(p_ent->elem.hdr.protocol_id, 851 p_ent->elem.hdr.cmd_id), 852 p_ent->elem.hdr.cmd_id, 853 qed_get_protocol_type_str(p_ent->elem.hdr.protocol_id), 854 p_ent->elem.hdr.protocol_id); 855 856 /* Let the flow complete w/o any error handling */ 857 qed_spq_recov_set_ret_code(p_ent, fw_return_code); 858 return 0; 859 } 860 861 /* Complete the entry */ 862 rc = qed_spq_fill_entry(p_hwfn, p_ent); 863 864 spin_lock_bh(&p_spq->lock); 865 866 /* Check return value after LOCK is taken for cleaner error flow */ 867 if (rc) 868 goto spq_post_fail; 869 870 /* Check if entry is in block mode before qed_spq_add_entry, 871 * which might kfree p_ent. 872 */ 873 eblock = (p_ent->comp_mode == QED_SPQ_MODE_EBLOCK); 874 875 /* Add the request to the pending queue */ 876 rc = qed_spq_add_entry(p_hwfn, p_ent, p_ent->priority); 877 if (rc) 878 goto spq_post_fail; 879 880 rc = qed_spq_pend_post(p_hwfn); 881 if (rc) { 882 /* Since it's possible that pending failed for a different 883 * entry [although unlikely], the failed entry was already 884 * dealt with; No need to return it here. 885 */ 886 b_ret_ent = false; 887 goto spq_post_fail; 888 } 889 890 spin_unlock_bh(&p_spq->lock); 891 892 if (eblock) { 893 /* For entries in QED BLOCK mode, the completion code cannot 894 * perform the necessary cleanup - if it did, we couldn't 895 * access p_ent here to see whether it's successful or not. 896 * Thus, after gaining the answer perform the cleanup here. 897 */ 898 rc = qed_spq_block(p_hwfn, p_ent, fw_return_code, 899 p_ent->queue == &p_spq->unlimited_pending); 900 901 if (p_ent->queue == &p_spq->unlimited_pending) { 902 struct qed_spq_entry *p_post_ent = p_ent->post_ent; 903 904 kfree(p_ent); 905 906 /* Return the entry which was actually posted */ 907 p_ent = p_post_ent; 908 } 909 910 if (rc) 911 goto spq_post_fail2; 912 913 /* return to pool */ 914 qed_spq_return_entry(p_hwfn, p_ent); 915 } 916 return rc; 917 918 spq_post_fail2: 919 spin_lock_bh(&p_spq->lock); 920 list_del(&p_ent->list); 921 qed_spq_comp_bmap_update(p_hwfn, p_ent->elem.hdr.echo); 922 923 spq_post_fail: 924 /* return to the free pool */ 925 if (b_ret_ent) 926 __qed_spq_return_entry(p_hwfn, p_ent); 927 spin_unlock_bh(&p_spq->lock); 928 929 return rc; 930 } 931 932 int qed_spq_completion(struct qed_hwfn *p_hwfn, 933 __le16 echo, 934 u8 fw_return_code, 935 union event_ring_data *p_data) 936 { 937 struct qed_spq *p_spq; 938 struct qed_spq_entry *p_ent = NULL; 939 struct qed_spq_entry *tmp; 940 struct qed_spq_entry *found = NULL; 941 942 if (!p_hwfn) 943 return -EINVAL; 944 945 p_spq = p_hwfn->p_spq; 946 if (!p_spq) 947 return -EINVAL; 948 949 spin_lock_bh(&p_spq->lock); 950 list_for_each_entry_safe(p_ent, tmp, &p_spq->completion_pending, list) { 951 if (p_ent->elem.hdr.echo == echo) { 952 list_del(&p_ent->list); 953 qed_spq_comp_bmap_update(p_hwfn, echo); 954 p_spq->comp_count++; 955 found = p_ent; 956 break; 957 } 958 959 /* This is relatively uncommon - depends on scenarios 960 * which have mutliple per-PF sent ramrods. 961 */ 962 DP_VERBOSE(p_hwfn, QED_MSG_SPQ, 963 "Got completion for echo %04x - doesn't match echo %04x in completion pending list\n", 964 le16_to_cpu(echo), 965 le16_to_cpu(p_ent->elem.hdr.echo)); 966 } 967 968 /* Release lock before callback, as callback may post 969 * an additional ramrod. 970 */ 971 spin_unlock_bh(&p_spq->lock); 972 973 if (!found) { 974 DP_NOTICE(p_hwfn, 975 "Failed to find an entry this EQE [echo %04x] completes\n", 976 le16_to_cpu(echo)); 977 return -EEXIST; 978 } 979 980 DP_VERBOSE(p_hwfn, QED_MSG_SPQ, 981 "Complete EQE [echo %04x]: func %p cookie %p)\n", 982 le16_to_cpu(echo), 983 p_ent->comp_cb.function, p_ent->comp_cb.cookie); 984 if (found->comp_cb.function) 985 found->comp_cb.function(p_hwfn, found->comp_cb.cookie, p_data, 986 fw_return_code); 987 else 988 DP_VERBOSE(p_hwfn, 989 QED_MSG_SPQ, 990 "Got a completion without a callback function\n"); 991 992 if (found->comp_mode != QED_SPQ_MODE_EBLOCK) 993 /* EBLOCK is responsible for returning its own entry into the 994 * free list. 995 */ 996 qed_spq_return_entry(p_hwfn, found); 997 998 return 0; 999 } 1000 1001 #define QED_SPQ_CONSQ_ELEM_SIZE 0x80 1002 1003 int qed_consq_alloc(struct qed_hwfn *p_hwfn) 1004 { 1005 struct qed_chain_init_params params = { 1006 .mode = QED_CHAIN_MODE_PBL, 1007 .intended_use = QED_CHAIN_USE_TO_PRODUCE, 1008 .cnt_type = QED_CHAIN_CNT_TYPE_U16, 1009 .num_elems = QED_CHAIN_PAGE_SIZE / QED_SPQ_CONSQ_ELEM_SIZE, 1010 .elem_size = QED_SPQ_CONSQ_ELEM_SIZE, 1011 }; 1012 struct qed_consq *p_consq; 1013 int ret; 1014 1015 /* Allocate ConsQ struct */ 1016 p_consq = kzalloc(sizeof(*p_consq), GFP_KERNEL); 1017 if (!p_consq) 1018 return -ENOMEM; 1019 1020 /* Allocate and initialize ConsQ chain */ 1021 ret = qed_chain_alloc(p_hwfn->cdev, &p_consq->chain, ¶ms); 1022 if (ret) { 1023 DP_NOTICE(p_hwfn, "Failed to allocate ConsQ chain"); 1024 goto consq_alloc_fail; 1025 } 1026 1027 p_hwfn->p_consq = p_consq; 1028 1029 return 0; 1030 1031 consq_alloc_fail: 1032 kfree(p_consq); 1033 1034 return ret; 1035 } 1036 1037 void qed_consq_setup(struct qed_hwfn *p_hwfn) 1038 { 1039 qed_chain_reset(&p_hwfn->p_consq->chain); 1040 } 1041 1042 void qed_consq_free(struct qed_hwfn *p_hwfn) 1043 { 1044 if (!p_hwfn->p_consq) 1045 return; 1046 1047 qed_chain_free(p_hwfn->cdev, &p_hwfn->p_consq->chain); 1048 1049 kfree(p_hwfn->p_consq); 1050 p_hwfn->p_consq = NULL; 1051 } 1052