1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018, Intel Corporation. */ 3 4 #include "ice_common.h" 5 6 #define ICE_CQ_INIT_REGS(qinfo, prefix) \ 7 do { \ 8 (qinfo)->sq.head = prefix##_ATQH; \ 9 (qinfo)->sq.tail = prefix##_ATQT; \ 10 (qinfo)->sq.len = prefix##_ATQLEN; \ 11 (qinfo)->sq.bah = prefix##_ATQBAH; \ 12 (qinfo)->sq.bal = prefix##_ATQBAL; \ 13 (qinfo)->sq.len_mask = prefix##_ATQLEN_ATQLEN_M; \ 14 (qinfo)->sq.len_ena_mask = prefix##_ATQLEN_ATQENABLE_M; \ 15 (qinfo)->sq.len_crit_mask = prefix##_ATQLEN_ATQCRIT_M; \ 16 (qinfo)->sq.head_mask = prefix##_ATQH_ATQH_M; \ 17 (qinfo)->rq.head = prefix##_ARQH; \ 18 (qinfo)->rq.tail = prefix##_ARQT; \ 19 (qinfo)->rq.len = prefix##_ARQLEN; \ 20 (qinfo)->rq.bah = prefix##_ARQBAH; \ 21 (qinfo)->rq.bal = prefix##_ARQBAL; \ 22 (qinfo)->rq.len_mask = prefix##_ARQLEN_ARQLEN_M; \ 23 (qinfo)->rq.len_ena_mask = prefix##_ARQLEN_ARQENABLE_M; \ 24 (qinfo)->rq.len_crit_mask = prefix##_ARQLEN_ARQCRIT_M; \ 25 (qinfo)->rq.head_mask = prefix##_ARQH_ARQH_M; \ 26 } while (0) 27 28 /** 29 * ice_adminq_init_regs - Initialize AdminQ registers 30 * @hw: pointer to the hardware structure 31 * 32 * This assumes the alloc_sq and alloc_rq functions have already been called 33 */ 34 static void ice_adminq_init_regs(struct ice_hw *hw) 35 { 36 struct ice_ctl_q_info *cq = &hw->adminq; 37 38 ICE_CQ_INIT_REGS(cq, PF_FW); 39 } 40 41 /** 42 * ice_mailbox_init_regs - Initialize Mailbox registers 43 * @hw: pointer to the hardware structure 44 * 45 * This assumes the alloc_sq and alloc_rq functions have already been called 46 */ 47 static void ice_mailbox_init_regs(struct ice_hw *hw) 48 { 49 struct ice_ctl_q_info *cq = &hw->mailboxq; 50 51 ICE_CQ_INIT_REGS(cq, PF_MBX); 52 } 53 54 /** 55 * ice_sb_init_regs - Initialize Sideband registers 56 * @hw: pointer to the hardware structure 57 * 58 * This assumes the alloc_sq and alloc_rq functions have already been called 59 */ 60 static void ice_sb_init_regs(struct ice_hw *hw) 61 { 62 struct ice_ctl_q_info *cq = &hw->sbq; 63 64 ICE_CQ_INIT_REGS(cq, PF_SB); 65 } 66 67 /** 68 * ice_check_sq_alive 69 * @hw: pointer to the HW struct 70 * @cq: pointer to the specific Control queue 71 * 72 * Returns true if Queue is enabled else false. 73 */ 74 bool ice_check_sq_alive(struct ice_hw *hw, struct ice_ctl_q_info *cq) 75 { 76 /* check both queue-length and queue-enable fields */ 77 if (cq->sq.len && cq->sq.len_mask && cq->sq.len_ena_mask) 78 return (rd32(hw, cq->sq.len) & (cq->sq.len_mask | 79 cq->sq.len_ena_mask)) == 80 (cq->num_sq_entries | cq->sq.len_ena_mask); 81 82 return false; 83 } 84 85 /** 86 * ice_alloc_ctrlq_sq_ring - Allocate Control Transmit Queue (ATQ) rings 87 * @hw: pointer to the hardware structure 88 * @cq: pointer to the specific Control queue 89 */ 90 static int 91 ice_alloc_ctrlq_sq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq) 92 { 93 size_t size = cq->num_sq_entries * sizeof(struct ice_aq_desc); 94 95 cq->sq.desc_buf.va = dmam_alloc_coherent(ice_hw_to_dev(hw), size, 96 &cq->sq.desc_buf.pa, 97 GFP_KERNEL | __GFP_ZERO); 98 if (!cq->sq.desc_buf.va) 99 return -ENOMEM; 100 cq->sq.desc_buf.size = size; 101 102 return 0; 103 } 104 105 /** 106 * ice_alloc_ctrlq_rq_ring - Allocate Control Receive Queue (ARQ) rings 107 * @hw: pointer to the hardware structure 108 * @cq: pointer to the specific Control queue 109 */ 110 static int 111 ice_alloc_ctrlq_rq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq) 112 { 113 size_t size = cq->num_rq_entries * sizeof(struct ice_aq_desc); 114 115 cq->rq.desc_buf.va = dmam_alloc_coherent(ice_hw_to_dev(hw), size, 116 &cq->rq.desc_buf.pa, 117 GFP_KERNEL | __GFP_ZERO); 118 if (!cq->rq.desc_buf.va) 119 return -ENOMEM; 120 cq->rq.desc_buf.size = size; 121 return 0; 122 } 123 124 /** 125 * ice_free_cq_ring - Free control queue ring 126 * @hw: pointer to the hardware structure 127 * @ring: pointer to the specific control queue ring 128 * 129 * This assumes the posted buffers have already been cleaned 130 * and de-allocated 131 */ 132 static void ice_free_cq_ring(struct ice_hw *hw, struct ice_ctl_q_ring *ring) 133 { 134 dmam_free_coherent(ice_hw_to_dev(hw), ring->desc_buf.size, 135 ring->desc_buf.va, ring->desc_buf.pa); 136 ring->desc_buf.va = NULL; 137 ring->desc_buf.pa = 0; 138 ring->desc_buf.size = 0; 139 } 140 141 /** 142 * ice_alloc_rq_bufs - Allocate pre-posted buffers for the ARQ 143 * @hw: pointer to the hardware structure 144 * @cq: pointer to the specific Control queue 145 */ 146 static int 147 ice_alloc_rq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq) 148 { 149 int i; 150 151 /* We'll be allocating the buffer info memory first, then we can 152 * allocate the mapped buffers for the event processing 153 */ 154 cq->rq.dma_head = devm_kcalloc(ice_hw_to_dev(hw), cq->num_rq_entries, 155 sizeof(cq->rq.desc_buf), GFP_KERNEL); 156 if (!cq->rq.dma_head) 157 return -ENOMEM; 158 cq->rq.r.rq_bi = (struct ice_dma_mem *)cq->rq.dma_head; 159 160 /* allocate the mapped buffers */ 161 for (i = 0; i < cq->num_rq_entries; i++) { 162 struct ice_aq_desc *desc; 163 struct ice_dma_mem *bi; 164 165 bi = &cq->rq.r.rq_bi[i]; 166 bi->va = dmam_alloc_coherent(ice_hw_to_dev(hw), 167 cq->rq_buf_size, &bi->pa, 168 GFP_KERNEL | __GFP_ZERO); 169 if (!bi->va) 170 goto unwind_alloc_rq_bufs; 171 bi->size = cq->rq_buf_size; 172 173 /* now configure the descriptors for use */ 174 desc = ICE_CTL_Q_DESC(cq->rq, i); 175 176 desc->flags = cpu_to_le16(ICE_AQ_FLAG_BUF); 177 if (cq->rq_buf_size > ICE_AQ_LG_BUF) 178 desc->flags |= cpu_to_le16(ICE_AQ_FLAG_LB); 179 desc->opcode = 0; 180 /* This is in accordance with control queue design, there is no 181 * register for buffer size configuration 182 */ 183 desc->datalen = cpu_to_le16(bi->size); 184 desc->retval = 0; 185 desc->cookie_high = 0; 186 desc->cookie_low = 0; 187 desc->params.generic.addr_high = 188 cpu_to_le32(upper_32_bits(bi->pa)); 189 desc->params.generic.addr_low = 190 cpu_to_le32(lower_32_bits(bi->pa)); 191 desc->params.generic.param0 = 0; 192 desc->params.generic.param1 = 0; 193 } 194 return 0; 195 196 unwind_alloc_rq_bufs: 197 /* don't try to free the one that failed... */ 198 i--; 199 for (; i >= 0; i--) { 200 dmam_free_coherent(ice_hw_to_dev(hw), cq->rq.r.rq_bi[i].size, 201 cq->rq.r.rq_bi[i].va, cq->rq.r.rq_bi[i].pa); 202 cq->rq.r.rq_bi[i].va = NULL; 203 cq->rq.r.rq_bi[i].pa = 0; 204 cq->rq.r.rq_bi[i].size = 0; 205 } 206 cq->rq.r.rq_bi = NULL; 207 devm_kfree(ice_hw_to_dev(hw), cq->rq.dma_head); 208 cq->rq.dma_head = NULL; 209 210 return -ENOMEM; 211 } 212 213 /** 214 * ice_alloc_sq_bufs - Allocate empty buffer structs for the ATQ 215 * @hw: pointer to the hardware structure 216 * @cq: pointer to the specific Control queue 217 */ 218 static int 219 ice_alloc_sq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq) 220 { 221 int i; 222 223 /* No mapped memory needed yet, just the buffer info structures */ 224 cq->sq.dma_head = devm_kcalloc(ice_hw_to_dev(hw), cq->num_sq_entries, 225 sizeof(cq->sq.desc_buf), GFP_KERNEL); 226 if (!cq->sq.dma_head) 227 return -ENOMEM; 228 cq->sq.r.sq_bi = (struct ice_dma_mem *)cq->sq.dma_head; 229 230 /* allocate the mapped buffers */ 231 for (i = 0; i < cq->num_sq_entries; i++) { 232 struct ice_dma_mem *bi; 233 234 bi = &cq->sq.r.sq_bi[i]; 235 bi->va = dmam_alloc_coherent(ice_hw_to_dev(hw), 236 cq->sq_buf_size, &bi->pa, 237 GFP_KERNEL | __GFP_ZERO); 238 if (!bi->va) 239 goto unwind_alloc_sq_bufs; 240 bi->size = cq->sq_buf_size; 241 } 242 return 0; 243 244 unwind_alloc_sq_bufs: 245 /* don't try to free the one that failed... */ 246 i--; 247 for (; i >= 0; i--) { 248 dmam_free_coherent(ice_hw_to_dev(hw), cq->sq.r.sq_bi[i].size, 249 cq->sq.r.sq_bi[i].va, cq->sq.r.sq_bi[i].pa); 250 cq->sq.r.sq_bi[i].va = NULL; 251 cq->sq.r.sq_bi[i].pa = 0; 252 cq->sq.r.sq_bi[i].size = 0; 253 } 254 cq->sq.r.sq_bi = NULL; 255 devm_kfree(ice_hw_to_dev(hw), cq->sq.dma_head); 256 cq->sq.dma_head = NULL; 257 258 return -ENOMEM; 259 } 260 261 static int 262 ice_cfg_cq_regs(struct ice_hw *hw, struct ice_ctl_q_ring *ring, u16 num_entries) 263 { 264 /* Clear Head and Tail */ 265 wr32(hw, ring->head, 0); 266 wr32(hw, ring->tail, 0); 267 268 /* set starting point */ 269 wr32(hw, ring->len, (num_entries | ring->len_ena_mask)); 270 wr32(hw, ring->bal, lower_32_bits(ring->desc_buf.pa)); 271 wr32(hw, ring->bah, upper_32_bits(ring->desc_buf.pa)); 272 273 /* Check one register to verify that config was applied */ 274 if (rd32(hw, ring->bal) != lower_32_bits(ring->desc_buf.pa)) 275 return -EIO; 276 277 return 0; 278 } 279 280 /** 281 * ice_cfg_sq_regs - configure Control ATQ registers 282 * @hw: pointer to the hardware structure 283 * @cq: pointer to the specific Control queue 284 * 285 * Configure base address and length registers for the transmit queue 286 */ 287 static int ice_cfg_sq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq) 288 { 289 return ice_cfg_cq_regs(hw, &cq->sq, cq->num_sq_entries); 290 } 291 292 /** 293 * ice_cfg_rq_regs - configure Control ARQ register 294 * @hw: pointer to the hardware structure 295 * @cq: pointer to the specific Control queue 296 * 297 * Configure base address and length registers for the receive (event queue) 298 */ 299 static int ice_cfg_rq_regs(struct ice_hw *hw, struct ice_ctl_q_info *cq) 300 { 301 int status; 302 303 status = ice_cfg_cq_regs(hw, &cq->rq, cq->num_rq_entries); 304 if (status) 305 return status; 306 307 /* Update tail in the HW to post pre-allocated buffers */ 308 wr32(hw, cq->rq.tail, (u32)(cq->num_rq_entries - 1)); 309 310 return 0; 311 } 312 313 #define ICE_FREE_CQ_BUFS(hw, qi, ring) \ 314 do { \ 315 /* free descriptors */ \ 316 if ((qi)->ring.r.ring##_bi) { \ 317 int i; \ 318 \ 319 for (i = 0; i < (qi)->num_##ring##_entries; i++) \ 320 if ((qi)->ring.r.ring##_bi[i].pa) { \ 321 dmam_free_coherent(ice_hw_to_dev(hw), \ 322 (qi)->ring.r.ring##_bi[i].size, \ 323 (qi)->ring.r.ring##_bi[i].va, \ 324 (qi)->ring.r.ring##_bi[i].pa); \ 325 (qi)->ring.r.ring##_bi[i].va = NULL;\ 326 (qi)->ring.r.ring##_bi[i].pa = 0;\ 327 (qi)->ring.r.ring##_bi[i].size = 0;\ 328 } \ 329 } \ 330 /* free DMA head */ \ 331 devm_kfree(ice_hw_to_dev(hw), (qi)->ring.dma_head); \ 332 } while (0) 333 334 /** 335 * ice_init_sq - main initialization routine for Control ATQ 336 * @hw: pointer to the hardware structure 337 * @cq: pointer to the specific Control queue 338 * 339 * This is the main initialization routine for the Control Send Queue 340 * Prior to calling this function, the driver *MUST* set the following fields 341 * in the cq->structure: 342 * - cq->num_sq_entries 343 * - cq->sq_buf_size 344 * 345 * Do *NOT* hold the lock when calling this as the memory allocation routines 346 * called are not going to be atomic context safe 347 */ 348 static int ice_init_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq) 349 { 350 int ret_code; 351 352 if (cq->sq.count > 0) { 353 /* queue already initialized */ 354 ret_code = -EBUSY; 355 goto init_ctrlq_exit; 356 } 357 358 /* verify input for valid configuration */ 359 if (!cq->num_sq_entries || !cq->sq_buf_size) { 360 ret_code = -EIO; 361 goto init_ctrlq_exit; 362 } 363 364 cq->sq.next_to_use = 0; 365 cq->sq.next_to_clean = 0; 366 367 /* allocate the ring memory */ 368 ret_code = ice_alloc_ctrlq_sq_ring(hw, cq); 369 if (ret_code) 370 goto init_ctrlq_exit; 371 372 /* allocate buffers in the rings */ 373 ret_code = ice_alloc_sq_bufs(hw, cq); 374 if (ret_code) 375 goto init_ctrlq_free_rings; 376 377 /* initialize base registers */ 378 ret_code = ice_cfg_sq_regs(hw, cq); 379 if (ret_code) 380 goto init_ctrlq_free_rings; 381 382 /* success! */ 383 cq->sq.count = cq->num_sq_entries; 384 goto init_ctrlq_exit; 385 386 init_ctrlq_free_rings: 387 ICE_FREE_CQ_BUFS(hw, cq, sq); 388 ice_free_cq_ring(hw, &cq->sq); 389 390 init_ctrlq_exit: 391 return ret_code; 392 } 393 394 /** 395 * ice_init_rq - initialize receive side of a control queue 396 * @hw: pointer to the hardware structure 397 * @cq: pointer to the specific Control queue 398 * 399 * The main initialization routine for Receive side of a control queue. 400 * Prior to calling this function, the driver *MUST* set the following fields 401 * in the cq->structure: 402 * - cq->num_rq_entries 403 * - cq->rq_buf_size 404 * 405 * Do *NOT* hold the lock when calling this as the memory allocation routines 406 * called are not going to be atomic context safe 407 */ 408 static int ice_init_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq) 409 { 410 int ret_code; 411 412 if (cq->rq.count > 0) { 413 /* queue already initialized */ 414 ret_code = -EBUSY; 415 goto init_ctrlq_exit; 416 } 417 418 /* verify input for valid configuration */ 419 if (!cq->num_rq_entries || !cq->rq_buf_size) { 420 ret_code = -EIO; 421 goto init_ctrlq_exit; 422 } 423 424 cq->rq.next_to_use = 0; 425 cq->rq.next_to_clean = 0; 426 427 /* allocate the ring memory */ 428 ret_code = ice_alloc_ctrlq_rq_ring(hw, cq); 429 if (ret_code) 430 goto init_ctrlq_exit; 431 432 /* allocate buffers in the rings */ 433 ret_code = ice_alloc_rq_bufs(hw, cq); 434 if (ret_code) 435 goto init_ctrlq_free_rings; 436 437 /* initialize base registers */ 438 ret_code = ice_cfg_rq_regs(hw, cq); 439 if (ret_code) 440 goto init_ctrlq_free_rings; 441 442 /* success! */ 443 cq->rq.count = cq->num_rq_entries; 444 goto init_ctrlq_exit; 445 446 init_ctrlq_free_rings: 447 ICE_FREE_CQ_BUFS(hw, cq, rq); 448 ice_free_cq_ring(hw, &cq->rq); 449 450 init_ctrlq_exit: 451 return ret_code; 452 } 453 454 /** 455 * ice_shutdown_sq - shutdown the transmit side of a control queue 456 * @hw: pointer to the hardware structure 457 * @cq: pointer to the specific Control queue 458 * 459 * The main shutdown routine for the Control Transmit Queue 460 */ 461 static int ice_shutdown_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq) 462 { 463 int ret_code = 0; 464 465 mutex_lock(&cq->sq_lock); 466 467 if (!cq->sq.count) { 468 ret_code = -EBUSY; 469 goto shutdown_sq_out; 470 } 471 472 /* Stop processing of the control queue */ 473 wr32(hw, cq->sq.head, 0); 474 wr32(hw, cq->sq.tail, 0); 475 wr32(hw, cq->sq.len, 0); 476 wr32(hw, cq->sq.bal, 0); 477 wr32(hw, cq->sq.bah, 0); 478 479 cq->sq.count = 0; /* to indicate uninitialized queue */ 480 481 /* free ring buffers and the ring itself */ 482 ICE_FREE_CQ_BUFS(hw, cq, sq); 483 ice_free_cq_ring(hw, &cq->sq); 484 485 shutdown_sq_out: 486 mutex_unlock(&cq->sq_lock); 487 return ret_code; 488 } 489 490 /** 491 * ice_aq_ver_check - Check the reported AQ API version 492 * @hw: pointer to the hardware structure 493 * 494 * Checks if the driver should load on a given AQ API version. 495 * 496 * Return: 'true' iff the driver should attempt to load. 'false' otherwise. 497 */ 498 static bool ice_aq_ver_check(struct ice_hw *hw) 499 { 500 u8 exp_fw_api_ver_major = EXP_FW_API_VER_MAJOR_BY_MAC(hw); 501 u8 exp_fw_api_ver_minor = EXP_FW_API_VER_MINOR_BY_MAC(hw); 502 503 if (hw->api_maj_ver > exp_fw_api_ver_major) { 504 /* Major API version is newer than expected, don't load */ 505 dev_warn(ice_hw_to_dev(hw), 506 "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n"); 507 return false; 508 } else if (hw->api_maj_ver == exp_fw_api_ver_major) { 509 if (hw->api_min_ver > (exp_fw_api_ver_minor + 2)) 510 dev_info(ice_hw_to_dev(hw), 511 "The driver for the device detected a newer version (%u.%u) of the NVM image than expected (%u.%u). Please install the most recent version of the network driver.\n", 512 hw->api_maj_ver, hw->api_min_ver, 513 exp_fw_api_ver_major, exp_fw_api_ver_minor); 514 else if ((hw->api_min_ver + 2) < exp_fw_api_ver_minor) 515 dev_info(ice_hw_to_dev(hw), 516 "The driver for the device detected an older version (%u.%u) of the NVM image than expected (%u.%u). Please update the NVM image.\n", 517 hw->api_maj_ver, hw->api_min_ver, 518 exp_fw_api_ver_major, exp_fw_api_ver_minor); 519 } else { 520 /* Major API version is older than expected, log a warning */ 521 dev_info(ice_hw_to_dev(hw), 522 "The driver for the device detected an older version (%u.%u) of the NVM image than expected (%u.%u). Please update the NVM image.\n", 523 hw->api_maj_ver, hw->api_min_ver, 524 exp_fw_api_ver_major, exp_fw_api_ver_minor); 525 } 526 return true; 527 } 528 529 /** 530 * ice_shutdown_rq - shutdown Control ARQ 531 * @hw: pointer to the hardware structure 532 * @cq: pointer to the specific Control queue 533 * 534 * The main shutdown routine for the Control Receive Queue 535 */ 536 static int ice_shutdown_rq(struct ice_hw *hw, struct ice_ctl_q_info *cq) 537 { 538 int ret_code = 0; 539 540 mutex_lock(&cq->rq_lock); 541 542 if (!cq->rq.count) { 543 ret_code = -EBUSY; 544 goto shutdown_rq_out; 545 } 546 547 /* Stop Control Queue processing */ 548 wr32(hw, cq->rq.head, 0); 549 wr32(hw, cq->rq.tail, 0); 550 wr32(hw, cq->rq.len, 0); 551 wr32(hw, cq->rq.bal, 0); 552 wr32(hw, cq->rq.bah, 0); 553 554 /* set rq.count to 0 to indicate uninitialized queue */ 555 cq->rq.count = 0; 556 557 /* free ring buffers and the ring itself */ 558 ICE_FREE_CQ_BUFS(hw, cq, rq); 559 ice_free_cq_ring(hw, &cq->rq); 560 561 shutdown_rq_out: 562 mutex_unlock(&cq->rq_lock); 563 return ret_code; 564 } 565 566 /** 567 * ice_init_check_adminq - Check version for Admin Queue to know if its alive 568 * @hw: pointer to the hardware structure 569 */ 570 static int ice_init_check_adminq(struct ice_hw *hw) 571 { 572 struct ice_ctl_q_info *cq = &hw->adminq; 573 int status; 574 575 status = ice_aq_get_fw_ver(hw, NULL); 576 if (status) 577 goto init_ctrlq_free_rq; 578 579 if (!ice_aq_ver_check(hw)) { 580 status = -EIO; 581 goto init_ctrlq_free_rq; 582 } 583 584 return 0; 585 586 init_ctrlq_free_rq: 587 ice_shutdown_rq(hw, cq); 588 ice_shutdown_sq(hw, cq); 589 return status; 590 } 591 592 /** 593 * ice_init_ctrlq - main initialization routine for any control Queue 594 * @hw: pointer to the hardware structure 595 * @q_type: specific Control queue type 596 * 597 * Prior to calling this function, the driver *MUST* set the following fields 598 * in the cq->structure: 599 * - cq->num_sq_entries 600 * - cq->num_rq_entries 601 * - cq->rq_buf_size 602 * - cq->sq_buf_size 603 * 604 * NOTE: this function does not initialize the controlq locks 605 */ 606 static int ice_init_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type) 607 { 608 struct ice_ctl_q_info *cq; 609 int ret_code; 610 611 switch (q_type) { 612 case ICE_CTL_Q_ADMIN: 613 ice_adminq_init_regs(hw); 614 cq = &hw->adminq; 615 break; 616 case ICE_CTL_Q_SB: 617 ice_sb_init_regs(hw); 618 cq = &hw->sbq; 619 break; 620 case ICE_CTL_Q_MAILBOX: 621 ice_mailbox_init_regs(hw); 622 cq = &hw->mailboxq; 623 break; 624 default: 625 return -EINVAL; 626 } 627 cq->qtype = q_type; 628 629 /* verify input for valid configuration */ 630 if (!cq->num_rq_entries || !cq->num_sq_entries || 631 !cq->rq_buf_size || !cq->sq_buf_size) { 632 return -EIO; 633 } 634 635 /* allocate the ATQ */ 636 ret_code = ice_init_sq(hw, cq); 637 if (ret_code) 638 return ret_code; 639 640 /* allocate the ARQ */ 641 ret_code = ice_init_rq(hw, cq); 642 if (ret_code) 643 goto init_ctrlq_free_sq; 644 645 /* success! */ 646 return 0; 647 648 init_ctrlq_free_sq: 649 ice_shutdown_sq(hw, cq); 650 return ret_code; 651 } 652 653 /** 654 * ice_is_sbq_supported - is the sideband queue supported 655 * @hw: pointer to the hardware structure 656 * 657 * Returns true if the sideband control queue interface is 658 * supported for the device, false otherwise 659 */ 660 bool ice_is_sbq_supported(struct ice_hw *hw) 661 { 662 /* The device sideband queue is only supported on devices with the 663 * generic MAC type. 664 */ 665 return ice_is_generic_mac(hw); 666 } 667 668 /** 669 * ice_get_sbq - returns the right control queue to use for sideband 670 * @hw: pointer to the hardware structure 671 */ 672 struct ice_ctl_q_info *ice_get_sbq(struct ice_hw *hw) 673 { 674 if (ice_is_sbq_supported(hw)) 675 return &hw->sbq; 676 return &hw->adminq; 677 } 678 679 /** 680 * ice_shutdown_ctrlq - shutdown routine for any control queue 681 * @hw: pointer to the hardware structure 682 * @q_type: specific Control queue type 683 * @unloading: is the driver unloading itself 684 * 685 * NOTE: this function does not destroy the control queue locks. 686 */ 687 static void ice_shutdown_ctrlq(struct ice_hw *hw, enum ice_ctl_q q_type, 688 bool unloading) 689 { 690 struct ice_ctl_q_info *cq; 691 692 switch (q_type) { 693 case ICE_CTL_Q_ADMIN: 694 cq = &hw->adminq; 695 if (ice_check_sq_alive(hw, cq)) 696 ice_aq_q_shutdown(hw, unloading); 697 break; 698 case ICE_CTL_Q_SB: 699 cq = &hw->sbq; 700 break; 701 case ICE_CTL_Q_MAILBOX: 702 cq = &hw->mailboxq; 703 break; 704 default: 705 return; 706 } 707 708 ice_shutdown_sq(hw, cq); 709 ice_shutdown_rq(hw, cq); 710 } 711 712 /** 713 * ice_shutdown_all_ctrlq - shutdown routine for all control queues 714 * @hw: pointer to the hardware structure 715 * @unloading: is the driver unloading itself 716 * 717 * NOTE: this function does not destroy the control queue locks. The driver 718 * may call this at runtime to shutdown and later restart control queues, such 719 * as in response to a reset event. 720 */ 721 void ice_shutdown_all_ctrlq(struct ice_hw *hw, bool unloading) 722 { 723 /* Shutdown FW admin queue */ 724 ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN, unloading); 725 /* Shutdown PHY Sideband */ 726 if (ice_is_sbq_supported(hw)) 727 ice_shutdown_ctrlq(hw, ICE_CTL_Q_SB, unloading); 728 /* Shutdown PF-VF Mailbox */ 729 ice_shutdown_ctrlq(hw, ICE_CTL_Q_MAILBOX, unloading); 730 } 731 732 /** 733 * ice_init_all_ctrlq - main initialization routine for all control queues 734 * @hw: pointer to the hardware structure 735 * 736 * Prior to calling this function, the driver MUST* set the following fields 737 * in the cq->structure for all control queues: 738 * - cq->num_sq_entries 739 * - cq->num_rq_entries 740 * - cq->rq_buf_size 741 * - cq->sq_buf_size 742 * 743 * NOTE: this function does not initialize the controlq locks. 744 */ 745 int ice_init_all_ctrlq(struct ice_hw *hw) 746 { 747 u32 retry = 0; 748 int status; 749 750 /* Init FW admin queue */ 751 do { 752 status = ice_init_ctrlq(hw, ICE_CTL_Q_ADMIN); 753 if (status) 754 return status; 755 756 status = ice_init_check_adminq(hw); 757 if (status != -EIO) 758 break; 759 760 ice_debug(hw, ICE_DBG_AQ_MSG, "Retry Admin Queue init due to FW critical error\n"); 761 ice_shutdown_ctrlq(hw, ICE_CTL_Q_ADMIN, true); 762 msleep(ICE_CTL_Q_ADMIN_INIT_MSEC); 763 } while (retry++ < ICE_CTL_Q_ADMIN_INIT_TIMEOUT); 764 765 if (status) 766 return status; 767 /* sideband control queue (SBQ) interface is not supported on some 768 * devices. Initialize if supported, else fallback to the admin queue 769 * interface 770 */ 771 if (ice_is_sbq_supported(hw)) { 772 status = ice_init_ctrlq(hw, ICE_CTL_Q_SB); 773 if (status) 774 return status; 775 } 776 /* Init Mailbox queue */ 777 return ice_init_ctrlq(hw, ICE_CTL_Q_MAILBOX); 778 } 779 780 /** 781 * ice_init_ctrlq_locks - Initialize locks for a control queue 782 * @cq: pointer to the control queue 783 * 784 * Initializes the send and receive queue locks for a given control queue. 785 */ 786 static void ice_init_ctrlq_locks(struct ice_ctl_q_info *cq) 787 { 788 mutex_init(&cq->sq_lock); 789 mutex_init(&cq->rq_lock); 790 } 791 792 /** 793 * ice_create_all_ctrlq - main initialization routine for all control queues 794 * @hw: pointer to the hardware structure 795 * 796 * Prior to calling this function, the driver *MUST* set the following fields 797 * in the cq->structure for all control queues: 798 * - cq->num_sq_entries 799 * - cq->num_rq_entries 800 * - cq->rq_buf_size 801 * - cq->sq_buf_size 802 * 803 * This function creates all the control queue locks and then calls 804 * ice_init_all_ctrlq. It should be called once during driver load. If the 805 * driver needs to re-initialize control queues at run time it should call 806 * ice_init_all_ctrlq instead. 807 */ 808 int ice_create_all_ctrlq(struct ice_hw *hw) 809 { 810 ice_init_ctrlq_locks(&hw->adminq); 811 if (ice_is_sbq_supported(hw)) 812 ice_init_ctrlq_locks(&hw->sbq); 813 ice_init_ctrlq_locks(&hw->mailboxq); 814 815 return ice_init_all_ctrlq(hw); 816 } 817 818 /** 819 * ice_destroy_ctrlq_locks - Destroy locks for a control queue 820 * @cq: pointer to the control queue 821 * 822 * Destroys the send and receive queue locks for a given control queue. 823 */ 824 static void ice_destroy_ctrlq_locks(struct ice_ctl_q_info *cq) 825 { 826 mutex_destroy(&cq->sq_lock); 827 mutex_destroy(&cq->rq_lock); 828 } 829 830 /** 831 * ice_destroy_all_ctrlq - exit routine for all control queues 832 * @hw: pointer to the hardware structure 833 * 834 * This function shuts down all the control queues and then destroys the 835 * control queue locks. It should be called once during driver unload. The 836 * driver should call ice_shutdown_all_ctrlq if it needs to shut down and 837 * reinitialize control queues, such as in response to a reset event. 838 */ 839 void ice_destroy_all_ctrlq(struct ice_hw *hw) 840 { 841 /* shut down all the control queues first */ 842 ice_shutdown_all_ctrlq(hw, true); 843 844 ice_destroy_ctrlq_locks(&hw->adminq); 845 if (ice_is_sbq_supported(hw)) 846 ice_destroy_ctrlq_locks(&hw->sbq); 847 ice_destroy_ctrlq_locks(&hw->mailboxq); 848 } 849 850 /** 851 * ice_clean_sq - cleans send side of a control queue 852 * @hw: pointer to the hardware structure 853 * @cq: pointer to the specific Control queue 854 * 855 * returns the number of free desc 856 */ 857 static u16 ice_clean_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq) 858 { 859 struct ice_ctl_q_ring *sq = &cq->sq; 860 u16 ntc = sq->next_to_clean; 861 struct ice_aq_desc *desc; 862 863 desc = ICE_CTL_Q_DESC(*sq, ntc); 864 865 while (rd32(hw, cq->sq.head) != ntc) { 866 ice_debug(hw, ICE_DBG_AQ_MSG, "ntc %d head %d.\n", ntc, rd32(hw, cq->sq.head)); 867 memset(desc, 0, sizeof(*desc)); 868 ntc++; 869 if (ntc == sq->count) 870 ntc = 0; 871 desc = ICE_CTL_Q_DESC(*sq, ntc); 872 } 873 874 sq->next_to_clean = ntc; 875 876 return ICE_CTL_Q_DESC_UNUSED(sq); 877 } 878 879 /** 880 * ice_ctl_q_str - Convert control queue type to string 881 * @qtype: the control queue type 882 * 883 * Return: A string name for the given control queue type. 884 */ 885 static const char *ice_ctl_q_str(enum ice_ctl_q qtype) 886 { 887 switch (qtype) { 888 case ICE_CTL_Q_UNKNOWN: 889 return "Unknown CQ"; 890 case ICE_CTL_Q_ADMIN: 891 return "AQ"; 892 case ICE_CTL_Q_MAILBOX: 893 return "MBXQ"; 894 case ICE_CTL_Q_SB: 895 return "SBQ"; 896 default: 897 return "Unrecognized CQ"; 898 } 899 } 900 901 /** 902 * ice_debug_cq 903 * @hw: pointer to the hardware structure 904 * @cq: pointer to the specific Control queue 905 * @desc: pointer to control queue descriptor 906 * @buf: pointer to command buffer 907 * @buf_len: max length of buf 908 * @response: true if this is the writeback response 909 * 910 * Dumps debug log about control command with descriptor contents. 911 */ 912 static void ice_debug_cq(struct ice_hw *hw, struct ice_ctl_q_info *cq, 913 void *desc, void *buf, u16 buf_len, bool response) 914 { 915 struct ice_aq_desc *cq_desc = desc; 916 u16 datalen, flags; 917 918 if (!IS_ENABLED(CONFIG_DYNAMIC_DEBUG) && 919 !((ICE_DBG_AQ_DESC | ICE_DBG_AQ_DESC_BUF) & hw->debug_mask)) 920 return; 921 922 if (!desc) 923 return; 924 925 datalen = le16_to_cpu(cq_desc->datalen); 926 flags = le16_to_cpu(cq_desc->flags); 927 928 ice_debug(hw, ICE_DBG_AQ_DESC, "%s %s: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n\tcookie (h,l) 0x%08X 0x%08X\n\tparam (0,1) 0x%08X 0x%08X\n\taddr (h,l) 0x%08X 0x%08X\n", 929 ice_ctl_q_str(cq->qtype), response ? "Response" : "Command", 930 le16_to_cpu(cq_desc->opcode), flags, datalen, 931 le16_to_cpu(cq_desc->retval), 932 le32_to_cpu(cq_desc->cookie_high), 933 le32_to_cpu(cq_desc->cookie_low), 934 le32_to_cpu(cq_desc->params.generic.param0), 935 le32_to_cpu(cq_desc->params.generic.param1), 936 le32_to_cpu(cq_desc->params.generic.addr_high), 937 le32_to_cpu(cq_desc->params.generic.addr_low)); 938 /* Dump buffer iff 1) one exists and 2) is either a response indicated 939 * by the DD and/or CMP flag set or a command with the RD flag set. 940 */ 941 if (buf && cq_desc->datalen && 942 (flags & (ICE_AQ_FLAG_DD | ICE_AQ_FLAG_CMP | ICE_AQ_FLAG_RD))) { 943 char prefix[] = KBUILD_MODNAME " 0x12341234 0x12341234 "; 944 945 sprintf(prefix, KBUILD_MODNAME " 0x%08X 0x%08X ", 946 le32_to_cpu(cq_desc->params.generic.addr_high), 947 le32_to_cpu(cq_desc->params.generic.addr_low)); 948 ice_debug_array_w_prefix(hw, ICE_DBG_AQ_DESC_BUF, prefix, 949 buf, 950 min_t(u16, buf_len, datalen)); 951 } 952 } 953 954 /** 955 * ice_sq_done - poll until the last send on a control queue has completed 956 * @hw: pointer to the HW struct 957 * @cq: pointer to the specific Control queue 958 * 959 * Use read_poll_timeout to poll the control queue head, checking until it 960 * matches next_to_use. According to the control queue designers, this has 961 * better timing reliability than the DD bit. 962 * 963 * Return: true if all the descriptors on the send side of a control queue 964 * are finished processing, false otherwise. 965 */ 966 static bool ice_sq_done(struct ice_hw *hw, struct ice_ctl_q_info *cq) 967 { 968 u32 head; 969 970 /* Wait a short time before the initial check, to allow hardware time 971 * for completion. 972 */ 973 udelay(5); 974 975 return !rd32_poll_timeout(hw, cq->sq.head, 976 head, head == cq->sq.next_to_use, 977 20, ICE_CTL_Q_SQ_CMD_TIMEOUT); 978 } 979 980 /** 981 * ice_sq_send_cmd - send command to a control queue 982 * @hw: pointer to the HW struct 983 * @cq: pointer to the specific Control queue 984 * @desc: prefilled descriptor describing the command 985 * @buf: buffer to use for indirect commands (or NULL for direct commands) 986 * @buf_size: size of buffer for indirect commands (or 0 for direct commands) 987 * @cd: pointer to command details structure 988 * 989 * Main command for the transmit side of a control queue. It puts the command 990 * on the queue, bumps the tail, waits for processing of the command, captures 991 * command status and results, etc. 992 */ 993 int 994 ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq, 995 struct ice_aq_desc *desc, void *buf, u16 buf_size, 996 struct ice_sq_cd *cd) 997 { 998 struct ice_dma_mem *dma_buf = NULL; 999 struct ice_aq_desc *desc_on_ring; 1000 bool cmd_completed = false; 1001 int status = 0; 1002 u16 retval = 0; 1003 u32 val = 0; 1004 1005 /* if reset is in progress return a soft error */ 1006 if (hw->reset_ongoing) 1007 return -EBUSY; 1008 mutex_lock(&cq->sq_lock); 1009 1010 cq->sq_last_status = ICE_AQ_RC_OK; 1011 1012 if (!cq->sq.count) { 1013 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send queue not initialized.\n"); 1014 status = -EIO; 1015 goto sq_send_command_error; 1016 } 1017 1018 if ((buf && !buf_size) || (!buf && buf_size)) { 1019 status = -EINVAL; 1020 goto sq_send_command_error; 1021 } 1022 1023 if (buf) { 1024 if (buf_size > cq->sq_buf_size) { 1025 ice_debug(hw, ICE_DBG_AQ_MSG, "Invalid buffer size for Control Send queue: %d.\n", 1026 buf_size); 1027 status = -EINVAL; 1028 goto sq_send_command_error; 1029 } 1030 1031 desc->flags |= cpu_to_le16(ICE_AQ_FLAG_BUF); 1032 if (buf_size > ICE_AQ_LG_BUF) 1033 desc->flags |= cpu_to_le16(ICE_AQ_FLAG_LB); 1034 } 1035 1036 val = rd32(hw, cq->sq.head); 1037 if (val >= cq->num_sq_entries) { 1038 ice_debug(hw, ICE_DBG_AQ_MSG, "head overrun at %d in the Control Send Queue ring\n", 1039 val); 1040 status = -EIO; 1041 goto sq_send_command_error; 1042 } 1043 1044 /* Call clean and check queue available function to reclaim the 1045 * descriptors that were processed by FW/MBX; the function returns the 1046 * number of desc available. The clean function called here could be 1047 * called in a separate thread in case of asynchronous completions. 1048 */ 1049 if (ice_clean_sq(hw, cq) == 0) { 1050 ice_debug(hw, ICE_DBG_AQ_MSG, "Error: Control Send Queue is full.\n"); 1051 status = -ENOSPC; 1052 goto sq_send_command_error; 1053 } 1054 1055 /* initialize the temp desc pointer with the right desc */ 1056 desc_on_ring = ICE_CTL_Q_DESC(cq->sq, cq->sq.next_to_use); 1057 1058 /* if the desc is available copy the temp desc to the right place */ 1059 memcpy(desc_on_ring, desc, sizeof(*desc_on_ring)); 1060 1061 /* if buf is not NULL assume indirect command */ 1062 if (buf) { 1063 dma_buf = &cq->sq.r.sq_bi[cq->sq.next_to_use]; 1064 /* copy the user buf into the respective DMA buf */ 1065 memcpy(dma_buf->va, buf, buf_size); 1066 desc_on_ring->datalen = cpu_to_le16(buf_size); 1067 1068 /* Update the address values in the desc with the pa value 1069 * for respective buffer 1070 */ 1071 desc_on_ring->params.generic.addr_high = 1072 cpu_to_le32(upper_32_bits(dma_buf->pa)); 1073 desc_on_ring->params.generic.addr_low = 1074 cpu_to_le32(lower_32_bits(dma_buf->pa)); 1075 } 1076 1077 /* Debug desc and buffer */ 1078 ice_debug(hw, ICE_DBG_AQ_DESC, "ATQ: Control Send queue desc and buffer:\n"); 1079 1080 ice_debug_cq(hw, cq, (void *)desc_on_ring, buf, buf_size, false); 1081 1082 (cq->sq.next_to_use)++; 1083 if (cq->sq.next_to_use == cq->sq.count) 1084 cq->sq.next_to_use = 0; 1085 wr32(hw, cq->sq.tail, cq->sq.next_to_use); 1086 ice_flush(hw); 1087 1088 /* Wait for the command to complete. If it finishes within the 1089 * timeout, copy the descriptor back to temp. 1090 */ 1091 if (ice_sq_done(hw, cq)) { 1092 memcpy(desc, desc_on_ring, sizeof(*desc)); 1093 if (buf) { 1094 /* get returned length to copy */ 1095 u16 copy_size = le16_to_cpu(desc->datalen); 1096 1097 if (copy_size > buf_size) { 1098 ice_debug(hw, ICE_DBG_AQ_MSG, "Return len %d > than buf len %d\n", 1099 copy_size, buf_size); 1100 status = -EIO; 1101 } else { 1102 memcpy(buf, dma_buf->va, copy_size); 1103 } 1104 } 1105 retval = le16_to_cpu(desc->retval); 1106 if (retval) { 1107 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send Queue command 0x%04X completed with error 0x%X\n", 1108 le16_to_cpu(desc->opcode), 1109 retval); 1110 1111 /* strip off FW internal code */ 1112 retval &= 0xff; 1113 } 1114 cmd_completed = true; 1115 if (!status && retval != ICE_AQ_RC_OK) 1116 status = -EIO; 1117 cq->sq_last_status = (enum ice_aq_err)retval; 1118 } 1119 1120 ice_debug(hw, ICE_DBG_AQ_MSG, "ATQ: desc and buffer writeback:\n"); 1121 1122 ice_debug_cq(hw, cq, (void *)desc, buf, buf_size, true); 1123 1124 /* save writeback AQ if requested */ 1125 if (cd && cd->wb_desc) 1126 memcpy(cd->wb_desc, desc_on_ring, sizeof(*cd->wb_desc)); 1127 1128 /* update the error if time out occurred */ 1129 if (!cmd_completed) { 1130 if (rd32(hw, cq->rq.len) & cq->rq.len_crit_mask || 1131 rd32(hw, cq->sq.len) & cq->sq.len_crit_mask) { 1132 ice_debug(hw, ICE_DBG_AQ_MSG, "Critical FW error.\n"); 1133 status = -EIO; 1134 } else { 1135 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Send Queue Writeback timeout.\n"); 1136 status = -EIO; 1137 } 1138 } 1139 1140 sq_send_command_error: 1141 mutex_unlock(&cq->sq_lock); 1142 return status; 1143 } 1144 1145 /** 1146 * ice_fill_dflt_direct_cmd_desc - AQ descriptor helper function 1147 * @desc: pointer to the temp descriptor (non DMA mem) 1148 * @opcode: the opcode can be used to decide which flags to turn off or on 1149 * 1150 * Fill the desc with default values 1151 */ 1152 void ice_fill_dflt_direct_cmd_desc(struct ice_aq_desc *desc, u16 opcode) 1153 { 1154 /* zero out the desc */ 1155 memset(desc, 0, sizeof(*desc)); 1156 desc->opcode = cpu_to_le16(opcode); 1157 desc->flags = cpu_to_le16(ICE_AQ_FLAG_SI); 1158 } 1159 1160 /** 1161 * ice_clean_rq_elem 1162 * @hw: pointer to the HW struct 1163 * @cq: pointer to the specific Control queue 1164 * @e: event info from the receive descriptor, includes any buffers 1165 * @pending: number of events that could be left to process 1166 * 1167 * Clean one element from the receive side of a control queue. On return 'e' 1168 * contains contents of the message, and 'pending' contains the number of 1169 * events left to process. 1170 */ 1171 int 1172 ice_clean_rq_elem(struct ice_hw *hw, struct ice_ctl_q_info *cq, 1173 struct ice_rq_event_info *e, u16 *pending) 1174 { 1175 u16 ntc = cq->rq.next_to_clean; 1176 enum ice_aq_err rq_last_status; 1177 struct ice_aq_desc *desc; 1178 struct ice_dma_mem *bi; 1179 int ret_code = 0; 1180 u16 desc_idx; 1181 u16 datalen; 1182 u16 flags; 1183 u16 ntu; 1184 1185 /* pre-clean the event info */ 1186 memset(&e->desc, 0, sizeof(e->desc)); 1187 1188 /* take the lock before we start messing with the ring */ 1189 mutex_lock(&cq->rq_lock); 1190 1191 if (!cq->rq.count) { 1192 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Receive queue not initialized.\n"); 1193 ret_code = -EIO; 1194 goto clean_rq_elem_err; 1195 } 1196 1197 /* set next_to_use to head */ 1198 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask); 1199 1200 if (ntu == ntc) { 1201 /* nothing to do - shouldn't need to update ring's values */ 1202 ret_code = -EALREADY; 1203 goto clean_rq_elem_out; 1204 } 1205 1206 /* now clean the next descriptor */ 1207 desc = ICE_CTL_Q_DESC(cq->rq, ntc); 1208 desc_idx = ntc; 1209 1210 rq_last_status = (enum ice_aq_err)le16_to_cpu(desc->retval); 1211 flags = le16_to_cpu(desc->flags); 1212 if (flags & ICE_AQ_FLAG_ERR) { 1213 ret_code = -EIO; 1214 ice_debug(hw, ICE_DBG_AQ_MSG, "Control Receive Queue Event 0x%04X received with error 0x%X\n", 1215 le16_to_cpu(desc->opcode), rq_last_status); 1216 } 1217 memcpy(&e->desc, desc, sizeof(e->desc)); 1218 datalen = le16_to_cpu(desc->datalen); 1219 e->msg_len = min_t(u16, datalen, e->buf_len); 1220 if (e->msg_buf && e->msg_len) 1221 memcpy(e->msg_buf, cq->rq.r.rq_bi[desc_idx].va, e->msg_len); 1222 1223 ice_debug(hw, ICE_DBG_AQ_DESC, "ARQ: desc and buffer:\n"); 1224 1225 ice_debug_cq(hw, cq, (void *)desc, e->msg_buf, cq->rq_buf_size, true); 1226 1227 /* Restore the original datalen and buffer address in the desc, 1228 * FW updates datalen to indicate the event message size 1229 */ 1230 bi = &cq->rq.r.rq_bi[ntc]; 1231 memset(desc, 0, sizeof(*desc)); 1232 1233 desc->flags = cpu_to_le16(ICE_AQ_FLAG_BUF); 1234 if (cq->rq_buf_size > ICE_AQ_LG_BUF) 1235 desc->flags |= cpu_to_le16(ICE_AQ_FLAG_LB); 1236 desc->datalen = cpu_to_le16(bi->size); 1237 desc->params.generic.addr_high = cpu_to_le32(upper_32_bits(bi->pa)); 1238 desc->params.generic.addr_low = cpu_to_le32(lower_32_bits(bi->pa)); 1239 1240 /* set tail = the last cleaned desc index. */ 1241 wr32(hw, cq->rq.tail, ntc); 1242 /* ntc is updated to tail + 1 */ 1243 ntc++; 1244 if (ntc == cq->num_rq_entries) 1245 ntc = 0; 1246 cq->rq.next_to_clean = ntc; 1247 cq->rq.next_to_use = ntu; 1248 1249 clean_rq_elem_out: 1250 /* Set pending if needed, unlock and return */ 1251 if (pending) { 1252 /* re-read HW head to calculate actual pending messages */ 1253 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask); 1254 *pending = (u16)((ntc > ntu ? cq->rq.count : 0) + (ntu - ntc)); 1255 } 1256 clean_rq_elem_err: 1257 mutex_unlock(&cq->rq_lock); 1258 1259 return ret_code; 1260 } 1261