1 /*- 2 * Copyright (c) 2017 Broadcom. All rights reserved. 3 * The term "Broadcom" refers to Broadcom Limited and/or its subsidiaries. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 11 * 2. Redistributions in binary form must reproduce the above copyright notice, 12 * this list of conditions and the following disclaimer in the documentation 13 * and/or other materials provided with the distribution. 14 * 15 * 3. Neither the name of the copyright holder nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE 23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /** 33 * @file 34 * 35 */ 36 37 #include "ocs_os.h" 38 #include "ocs_hw.h" 39 #include "ocs_hw_queues.h" 40 41 #define HW_QTOP_DEBUG 0 42 43 /** 44 * @brief Initialize queues 45 * 46 * Given the parsed queue topology spec, the SLI queues are created and 47 * initialized 48 * 49 * @param hw pointer to HW object 50 * @param qtop pointer to queue topology 51 * 52 * @return returns 0 for success, an error code value for failure. 53 */ 54 ocs_hw_rtn_e 55 ocs_hw_init_queues(ocs_hw_t *hw, ocs_hw_qtop_t *qtop) 56 { 57 uint32_t i, j; 58 uint32_t default_lengths[QTOP_LAST], len; 59 uint32_t rqset_len = 0, rqset_ulp = 0, rqset_count = 0; 60 uint8_t rqset_filter_mask = 0; 61 hw_eq_t *eqs[hw->config.n_rq]; 62 hw_cq_t *cqs[hw->config.n_rq]; 63 hw_rq_t *rqs[hw->config.n_rq]; 64 ocs_hw_qtop_entry_t *qt, *next_qt; 65 ocs_hw_mrq_t mrq; 66 bool use_mrq = FALSE; 67 68 hw_eq_t *eq = NULL; 69 hw_cq_t *cq = NULL; 70 hw_wq_t *wq = NULL; 71 hw_rq_t *rq = NULL; 72 hw_mq_t *mq = NULL; 73 74 mrq.num_pairs = 0; 75 default_lengths[QTOP_EQ] = 1024; 76 default_lengths[QTOP_CQ] = hw->num_qentries[SLI_QTYPE_CQ]; 77 default_lengths[QTOP_WQ] = hw->num_qentries[SLI_QTYPE_WQ]; 78 default_lengths[QTOP_RQ] = hw->num_qentries[SLI_QTYPE_RQ]; 79 default_lengths[QTOP_MQ] = OCS_HW_MQ_DEPTH; 80 81 ocs_hw_verify(hw != NULL, OCS_HW_RTN_INVALID_ARG); 82 83 hw->eq_count = 0; 84 hw->cq_count = 0; 85 hw->mq_count = 0; 86 hw->wq_count = 0; 87 hw->rq_count = 0; 88 hw->hw_rq_count = 0; 89 ocs_list_init(&hw->eq_list, hw_eq_t, link); 90 91 /* If MRQ is requested, Check if it is supported by SLI. */ 92 if ((hw->config.n_rq > 1 ) && !hw->sli.config.features.flag.mrqp) { 93 ocs_log_err(hw->os, "MRQ topology not supported by SLI4.\n"); 94 return OCS_HW_RTN_ERROR; 95 } 96 97 if (hw->config.n_rq > 1) 98 use_mrq = TRUE; 99 100 /* Allocate class WQ pools */ 101 for (i = 0; i < ARRAY_SIZE(hw->wq_class_array); i++) { 102 hw->wq_class_array[i] = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ); 103 if (hw->wq_class_array[i] == NULL) { 104 ocs_log_err(hw->os, "ocs_varray_alloc for wq_class failed\n"); 105 return OCS_HW_RTN_NO_MEMORY; 106 } 107 } 108 109 /* Allocate per CPU WQ pools */ 110 for (i = 0; i < ARRAY_SIZE(hw->wq_cpu_array); i++) { 111 hw->wq_cpu_array[i] = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ); 112 if (hw->wq_cpu_array[i] == NULL) { 113 ocs_log_err(hw->os, "ocs_varray_alloc for wq_class failed\n"); 114 return OCS_HW_RTN_NO_MEMORY; 115 } 116 } 117 118 ocs_hw_assert(qtop != NULL); 119 120 for (i = 0, qt = qtop->entries; i < qtop->inuse_count; i++, qt++) { 121 if (i == qtop->inuse_count - 1) 122 next_qt = NULL; 123 else 124 next_qt = qt + 1; 125 126 switch(qt->entry) { 127 case QTOP_EQ: 128 len = (qt->len) ? qt->len : default_lengths[QTOP_EQ]; 129 130 if (qt->set_default) { 131 default_lengths[QTOP_EQ] = len; 132 break; 133 } 134 135 eq = hw_new_eq(hw, len); 136 if (eq == NULL) { 137 hw_queue_teardown(hw); 138 return OCS_HW_RTN_NO_MEMORY; 139 } 140 break; 141 142 case QTOP_CQ: 143 len = (qt->len) ? qt->len : default_lengths[QTOP_CQ]; 144 145 if (qt->set_default) { 146 default_lengths[QTOP_CQ] = len; 147 break; 148 } 149 150 if (!eq || !next_qt) { 151 goto fail; 152 } 153 154 /* If this CQ is for MRQ, then delay the creation */ 155 if (!use_mrq || next_qt->entry != QTOP_RQ) { 156 cq = hw_new_cq(eq, len); 157 if (cq == NULL) { 158 goto fail; 159 } 160 } 161 break; 162 163 case QTOP_WQ: { 164 len = (qt->len) ? qt->len : default_lengths[QTOP_WQ]; 165 if (qt->set_default) { 166 default_lengths[QTOP_WQ] = len; 167 break; 168 } 169 170 if ((hw->ulp_start + qt->ulp) > hw->ulp_max) { 171 ocs_log_err(hw->os, "invalid ULP %d for WQ\n", qt->ulp); 172 hw_queue_teardown(hw); 173 return OCS_HW_RTN_NO_MEMORY; 174 } 175 176 if (cq == NULL) 177 goto fail; 178 179 wq = hw_new_wq(cq, len, qt->class, hw->ulp_start + qt->ulp); 180 if (wq == NULL) { 181 goto fail; 182 } 183 184 /* Place this WQ on the EQ WQ array */ 185 if (ocs_varray_add(eq->wq_array, wq)) { 186 ocs_log_err(hw->os, "QTOP_WQ: EQ ocs_varray_add failed\n"); 187 hw_queue_teardown(hw); 188 return OCS_HW_RTN_ERROR; 189 } 190 191 /* Place this WQ on the HW class array */ 192 if (qt->class < ARRAY_SIZE(hw->wq_class_array)) { 193 if (ocs_varray_add(hw->wq_class_array[qt->class], wq)) { 194 ocs_log_err(hw->os, "HW wq_class_array ocs_varray_add failed\n"); 195 hw_queue_teardown(hw); 196 return OCS_HW_RTN_ERROR; 197 } 198 } else { 199 ocs_log_err(hw->os, "Invalid class value: %d\n", qt->class); 200 hw_queue_teardown(hw); 201 return OCS_HW_RTN_ERROR; 202 } 203 204 /* 205 * Place this WQ on the per CPU list, asumming that EQs are mapped to cpu given 206 * by the EQ instance modulo number of CPUs 207 */ 208 if (ocs_varray_add(hw->wq_cpu_array[eq->instance % ocs_get_num_cpus()], wq)) { 209 ocs_log_err(hw->os, "HW wq_cpu_array ocs_varray_add failed\n"); 210 hw_queue_teardown(hw); 211 return OCS_HW_RTN_ERROR; 212 } 213 214 break; 215 } 216 case QTOP_RQ: { 217 len = (qt->len) ? qt->len : default_lengths[QTOP_RQ]; 218 if (qt->set_default) { 219 default_lengths[QTOP_RQ] = len; 220 break; 221 } 222 223 if ((hw->ulp_start + qt->ulp) > hw->ulp_max) { 224 ocs_log_err(hw->os, "invalid ULP %d for RQ\n", qt->ulp); 225 hw_queue_teardown(hw); 226 return OCS_HW_RTN_NO_MEMORY; 227 } 228 229 if (use_mrq) { 230 mrq.rq_cfg[mrq.num_pairs].len = len; 231 mrq.rq_cfg[mrq.num_pairs].ulp = hw->ulp_start + qt->ulp; 232 mrq.rq_cfg[mrq.num_pairs].filter_mask = qt->filter_mask; 233 mrq.rq_cfg[mrq.num_pairs].eq = eq; 234 mrq.num_pairs ++; 235 } else { 236 rq = hw_new_rq(cq, len, hw->ulp_start + qt->ulp); 237 if (rq == NULL) { 238 hw_queue_teardown(hw); 239 return OCS_HW_RTN_NO_MEMORY; 240 } 241 rq->filter_mask = qt->filter_mask; 242 } 243 break; 244 } 245 246 case QTOP_MQ: 247 len = (qt->len) ? qt->len : default_lengths[QTOP_MQ]; 248 if (qt->set_default) { 249 default_lengths[QTOP_MQ] = len; 250 break; 251 } 252 253 if (cq == NULL) 254 goto fail; 255 256 mq = hw_new_mq(cq, len); 257 if (mq == NULL) { 258 goto fail; 259 } 260 break; 261 262 default: 263 ocs_hw_assert(0); 264 break; 265 } 266 } 267 268 if (mrq.num_pairs) { 269 /* First create normal RQs. */ 270 for (i = 0; i < mrq.num_pairs; i++) { 271 for (j = 0; j < mrq.num_pairs; j++) { 272 if ((i != j) && (mrq.rq_cfg[i].filter_mask == mrq.rq_cfg[j].filter_mask)) { 273 /* This should be created using set */ 274 if (rqset_filter_mask && (rqset_filter_mask != mrq.rq_cfg[i].filter_mask)) { 275 ocs_log_crit(hw->os, "Cant create morethan one RQ Set\n"); 276 hw_queue_teardown(hw); 277 return OCS_HW_RTN_ERROR; 278 } else if (!rqset_filter_mask){ 279 rqset_filter_mask = mrq.rq_cfg[i].filter_mask; 280 rqset_len = mrq.rq_cfg[i].len; 281 rqset_ulp = mrq.rq_cfg[i].ulp; 282 } 283 eqs[rqset_count] = mrq.rq_cfg[i].eq; 284 rqset_count++; 285 break; 286 } 287 } 288 if (j == mrq.num_pairs) { 289 /* Normal RQ */ 290 cq = hw_new_cq(mrq.rq_cfg[i].eq, default_lengths[QTOP_CQ]); 291 if (cq == NULL) { 292 hw_queue_teardown(hw); 293 return OCS_HW_RTN_NO_MEMORY; 294 } 295 296 rq = hw_new_rq(cq, mrq.rq_cfg[i].len, mrq.rq_cfg[i].ulp); 297 if (rq == NULL) { 298 hw_queue_teardown(hw); 299 return OCS_HW_RTN_NO_MEMORY; 300 } 301 rq->filter_mask = mrq.rq_cfg[i].filter_mask; 302 } 303 } 304 305 /* Now create RQ Set */ 306 if (rqset_count) { 307 if (rqset_count > OCE_HW_MAX_NUM_MRQ_PAIRS) { 308 ocs_log_crit(hw->os, 309 "Max Supported MRQ pairs = %d\n", 310 OCE_HW_MAX_NUM_MRQ_PAIRS); 311 hw_queue_teardown(hw); 312 return OCS_HW_RTN_ERROR; 313 } 314 315 /* Create CQ set */ 316 if (hw_new_cq_set(eqs, cqs, rqset_count, default_lengths[QTOP_CQ])) { 317 hw_queue_teardown(hw); 318 return OCS_HW_RTN_ERROR; 319 } 320 321 /* Create RQ set */ 322 if (hw_new_rq_set(cqs, rqs, rqset_count, rqset_len, rqset_ulp)) { 323 hw_queue_teardown(hw); 324 return OCS_HW_RTN_ERROR; 325 } 326 327 for (i = 0; i < rqset_count ; i++) { 328 rqs[i]->filter_mask = rqset_filter_mask; 329 rqs[i]->is_mrq = TRUE; 330 rqs[i]->base_mrq_id = rqs[0]->hdr->id; 331 } 332 333 hw->hw_mrq_count = rqset_count; 334 } 335 } 336 337 return OCS_HW_RTN_SUCCESS; 338 fail: 339 hw_queue_teardown(hw); 340 return OCS_HW_RTN_NO_MEMORY; 341 342 } 343 344 /** 345 * @brief Allocate a new EQ object 346 * 347 * A new EQ object is instantiated 348 * 349 * @param hw pointer to HW object 350 * @param entry_count number of entries in the EQ 351 * 352 * @return pointer to allocated EQ object 353 */ 354 hw_eq_t* 355 hw_new_eq(ocs_hw_t *hw, uint32_t entry_count) 356 { 357 hw_eq_t *eq = ocs_malloc(hw->os, sizeof(*eq), OCS_M_ZERO | OCS_M_NOWAIT); 358 359 if (eq != NULL) { 360 eq->type = SLI_QTYPE_EQ; 361 eq->hw = hw; 362 eq->entry_count = entry_count; 363 eq->instance = hw->eq_count++; 364 eq->queue = &hw->eq[eq->instance]; 365 ocs_list_init(&eq->cq_list, hw_cq_t, link); 366 367 eq->wq_array = ocs_varray_alloc(hw->os, OCS_HW_MAX_NUM_WQ); 368 if (eq->wq_array == NULL) { 369 ocs_free(hw->os, eq, sizeof(*eq)); 370 eq = NULL; 371 } else { 372 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_EQ, eq->queue, entry_count, NULL, 0)) { 373 ocs_log_err(hw->os, "EQ[%d] allocation failure\n", eq->instance); 374 ocs_free(hw->os, eq, sizeof(*eq)); 375 eq = NULL; 376 } else { 377 sli_eq_modify_delay(&hw->sli, eq->queue, 1, 0, 8); 378 hw->hw_eq[eq->instance] = eq; 379 ocs_list_add_tail(&hw->eq_list, eq); 380 ocs_log_debug(hw->os, "create eq[%2d] id %3d len %4d\n", eq->instance, eq->queue->id, 381 eq->entry_count); 382 } 383 } 384 } 385 return eq; 386 } 387 388 /** 389 * @brief Allocate a new CQ object 390 * 391 * A new CQ object is instantiated 392 * 393 * @param eq pointer to parent EQ object 394 * @param entry_count number of entries in the CQ 395 * 396 * @return pointer to allocated CQ object 397 */ 398 hw_cq_t* 399 hw_new_cq(hw_eq_t *eq, uint32_t entry_count) 400 { 401 ocs_hw_t *hw = eq->hw; 402 hw_cq_t *cq = ocs_malloc(hw->os, sizeof(*cq), OCS_M_ZERO | OCS_M_NOWAIT); 403 404 if (cq != NULL) { 405 cq->eq = eq; 406 cq->type = SLI_QTYPE_CQ; 407 cq->instance = eq->hw->cq_count++; 408 cq->entry_count = entry_count; 409 cq->queue = &hw->cq[cq->instance]; 410 411 ocs_list_init(&cq->q_list, hw_q_t, link); 412 413 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_CQ, cq->queue, cq->entry_count, eq->queue, 0)) { 414 ocs_log_err(hw->os, "CQ[%d] allocation failure len=%d\n", 415 eq->instance, 416 eq->entry_count); 417 ocs_free(hw->os, cq, sizeof(*cq)); 418 cq = NULL; 419 } else { 420 hw->hw_cq[cq->instance] = cq; 421 ocs_list_add_tail(&eq->cq_list, cq); 422 ocs_log_debug(hw->os, "create cq[%2d] id %3d len %4d\n", cq->instance, cq->queue->id, 423 cq->entry_count); 424 } 425 } 426 return cq; 427 } 428 429 /** 430 * @brief Allocate a new CQ Set of objects. 431 * 432 * @param eqs pointer to a set of EQ objects. 433 * @param cqs pointer to a set of CQ objects to be returned. 434 * @param num_cqs number of CQ queues in the set. 435 * @param entry_count number of entries in the CQ. 436 * 437 * @return 0 on success and -1 on failure. 438 */ 439 uint32_t 440 hw_new_cq_set(hw_eq_t *eqs[], hw_cq_t *cqs[], uint32_t num_cqs, uint32_t entry_count) 441 { 442 uint32_t i; 443 ocs_hw_t *hw = eqs[0]->hw; 444 sli4_t *sli4 = &hw->sli; 445 hw_cq_t *cq = NULL; 446 sli4_queue_t *qs[SLI_MAX_CQ_SET_COUNT], *assocs[SLI_MAX_CQ_SET_COUNT]; 447 448 /* Initialise CQS pointers to NULL */ 449 for (i = 0; i < num_cqs; i++) { 450 cqs[i] = NULL; 451 } 452 453 for (i = 0; i < num_cqs; i++) { 454 cq = ocs_malloc(hw->os, sizeof(*cq), OCS_M_ZERO | OCS_M_NOWAIT); 455 if (cq == NULL) 456 goto error; 457 458 cqs[i] = cq; 459 cq->eq = eqs[i]; 460 cq->type = SLI_QTYPE_CQ; 461 cq->instance = hw->cq_count++; 462 cq->entry_count = entry_count; 463 cq->queue = &hw->cq[cq->instance]; 464 qs[i] = cq->queue; 465 assocs[i] = eqs[i]->queue; 466 ocs_list_init(&cq->q_list, hw_q_t, link); 467 } 468 469 if (sli_cq_alloc_set(sli4, qs, num_cqs, entry_count, assocs)) { 470 ocs_log_err(NULL, "Failed to create CQ Set. \n"); 471 goto error; 472 } 473 474 for (i = 0; i < num_cqs; i++) { 475 hw->hw_cq[cqs[i]->instance] = cqs[i]; 476 ocs_list_add_tail(&cqs[i]->eq->cq_list, cqs[i]); 477 } 478 479 return 0; 480 481 error: 482 for (i = 0; i < num_cqs; i++) { 483 if (cqs[i]) { 484 ocs_free(hw->os, cqs[i], sizeof(*cqs[i])); 485 cqs[i] = NULL; 486 } 487 } 488 return -1; 489 } 490 491 /** 492 * @brief Allocate a new MQ object 493 * 494 * A new MQ object is instantiated 495 * 496 * @param cq pointer to parent CQ object 497 * @param entry_count number of entries in the MQ 498 * 499 * @return pointer to allocated MQ object 500 */ 501 hw_mq_t* 502 hw_new_mq(hw_cq_t *cq, uint32_t entry_count) 503 { 504 ocs_hw_t *hw = cq->eq->hw; 505 hw_mq_t *mq = ocs_malloc(hw->os, sizeof(*mq), OCS_M_ZERO | OCS_M_NOWAIT); 506 507 if (mq != NULL) { 508 mq->cq = cq; 509 mq->type = SLI_QTYPE_MQ; 510 mq->instance = cq->eq->hw->mq_count++; 511 mq->entry_count = entry_count; 512 mq->entry_size = OCS_HW_MQ_DEPTH; 513 mq->queue = &hw->mq[mq->instance]; 514 515 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_MQ, 516 mq->queue, 517 mq->entry_size, 518 cq->queue, 0)) { 519 ocs_log_err(hw->os, "MQ allocation failure\n"); 520 ocs_free(hw->os, mq, sizeof(*mq)); 521 mq = NULL; 522 } else { 523 hw->hw_mq[mq->instance] = mq; 524 ocs_list_add_tail(&cq->q_list, mq); 525 ocs_log_debug(hw->os, "create mq[%2d] id %3d len %4d\n", mq->instance, mq->queue->id, 526 mq->entry_count); 527 } 528 } 529 return mq; 530 } 531 532 /** 533 * @brief Allocate a new WQ object 534 * 535 * A new WQ object is instantiated 536 * 537 * @param cq pointer to parent CQ object 538 * @param entry_count number of entries in the WQ 539 * @param class WQ class 540 * @param ulp index of chute 541 * 542 * @return pointer to allocated WQ object 543 */ 544 hw_wq_t* 545 hw_new_wq(hw_cq_t *cq, uint32_t entry_count, uint32_t class, uint32_t ulp) 546 { 547 ocs_hw_t *hw = cq->eq->hw; 548 hw_wq_t *wq = ocs_malloc(hw->os, sizeof(*wq), OCS_M_ZERO | OCS_M_NOWAIT); 549 550 if (wq != NULL) { 551 wq->hw = cq->eq->hw; 552 wq->cq = cq; 553 wq->type = SLI_QTYPE_WQ; 554 wq->instance = cq->eq->hw->wq_count++; 555 wq->entry_count = entry_count; 556 wq->queue = &hw->wq[wq->instance]; 557 wq->ulp = ulp; 558 wq->wqec_set_count = OCS_HW_WQEC_SET_COUNT; 559 wq->wqec_count = wq->wqec_set_count; 560 wq->free_count = wq->entry_count - 1; 561 wq->class = class; 562 ocs_list_init(&wq->pending_list, ocs_hw_wqe_t, link); 563 564 if (sli_queue_alloc(&hw->sli, SLI_QTYPE_WQ, wq->queue, wq->entry_count, cq->queue, ulp)) { 565 ocs_log_err(hw->os, "WQ allocation failure\n"); 566 ocs_free(hw->os, wq, sizeof(*wq)); 567 wq = NULL; 568 } else { 569 hw->hw_wq[wq->instance] = wq; 570 ocs_list_add_tail(&cq->q_list, wq); 571 ocs_log_debug(hw->os, "create wq[%2d] id %3d len %4d cls %d ulp %d\n", wq->instance, wq->queue->id, 572 wq->entry_count, wq->class, wq->ulp); 573 } 574 } 575 return wq; 576 } 577 578 /** 579 * @brief Allocate a hw_rq_t object 580 * 581 * Allocate an RQ object, which encapsulates 2 SLI queues (for rq pair) 582 * 583 * @param cq pointer to parent CQ object 584 * @param entry_count number of entries in the RQs 585 * @param ulp ULP index for this RQ 586 * 587 * @return pointer to newly allocated hw_rq_t 588 */ 589 hw_rq_t* 590 hw_new_rq(hw_cq_t *cq, uint32_t entry_count, uint32_t ulp) 591 { 592 ocs_hw_t *hw = cq->eq->hw; 593 hw_rq_t *rq = ocs_malloc(hw->os, sizeof(*rq), OCS_M_ZERO | OCS_M_NOWAIT); 594 uint32_t max_hw_rq; 595 596 ocs_hw_get(hw, OCS_HW_MAX_RQ_ENTRIES, &max_hw_rq); 597 598 if (rq != NULL) { 599 rq->instance = hw->hw_rq_count++; 600 rq->cq = cq; 601 rq->type = SLI_QTYPE_RQ; 602 rq->ulp = ulp; 603 604 rq->entry_count = OCS_MIN(entry_count, OCS_MIN(max_hw_rq, OCS_HW_RQ_NUM_HDR)); 605 606 /* Create the header RQ */ 607 ocs_hw_assert(hw->rq_count < ARRAY_SIZE(hw->rq)); 608 rq->hdr = &hw->rq[hw->rq_count]; 609 rq->hdr_entry_size = OCS_HW_RQ_HEADER_SIZE; 610 611 if (sli_fc_rq_alloc(&hw->sli, rq->hdr, 612 rq->entry_count, 613 rq->hdr_entry_size, 614 cq->queue, 615 ulp, TRUE)) { 616 ocs_log_err(hw->os, "RQ allocation failure - header\n"); 617 ocs_free(hw->os, rq, sizeof(*rq)); 618 return NULL; 619 } 620 hw->hw_rq_lookup[hw->rq_count] = rq->instance; /* Update hw_rq_lookup[] */ 621 hw->rq_count++; 622 ocs_log_debug(hw->os, "create rq[%2d] id %3d len %4d hdr size %4d ulp %d\n", 623 rq->instance, rq->hdr->id, rq->entry_count, rq->hdr_entry_size, rq->ulp); 624 625 /* Create the default data RQ */ 626 ocs_hw_assert(hw->rq_count < ARRAY_SIZE(hw->rq)); 627 rq->data = &hw->rq[hw->rq_count]; 628 rq->data_entry_size = hw->config.rq_default_buffer_size; 629 630 if (sli_fc_rq_alloc(&hw->sli, rq->data, 631 rq->entry_count, 632 rq->data_entry_size, 633 cq->queue, 634 ulp, FALSE)) { 635 ocs_log_err(hw->os, "RQ allocation failure - first burst\n"); 636 ocs_free(hw->os, rq, sizeof(*rq)); 637 return NULL; 638 } 639 hw->hw_rq_lookup[hw->rq_count] = rq->instance; /* Update hw_rq_lookup[] */ 640 hw->rq_count++; 641 ocs_log_debug(hw->os, "create rq[%2d] id %3d len %4d data size %4d ulp %d\n", rq->instance, 642 rq->data->id, rq->entry_count, rq->data_entry_size, rq->ulp); 643 644 hw->hw_rq[rq->instance] = rq; 645 ocs_list_add_tail(&cq->q_list, rq); 646 647 rq->rq_tracker = ocs_malloc(hw->os, sizeof(ocs_hw_sequence_t*) * 648 rq->entry_count, OCS_M_ZERO | OCS_M_NOWAIT); 649 if (rq->rq_tracker == NULL) { 650 ocs_log_err(hw->os, "RQ tracker buf allocation failure\n"); 651 return NULL; 652 } 653 } 654 return rq; 655 } 656 657 /** 658 * @brief Allocate a hw_rq_t object SET 659 * 660 * Allocate an RQ object SET, where each element in set 661 * encapsulates 2 SLI queues (for rq pair) 662 * 663 * @param cqs pointers to be associated with RQs. 664 * @param rqs RQ pointers to be returned on success. 665 * @param num_rq_pairs number of rq pairs in the Set. 666 * @param entry_count number of entries in the RQs 667 * @param ulp ULP index for this RQ 668 * 669 * @return 0 in success and -1 on failure. 670 */ 671 uint32_t 672 hw_new_rq_set(hw_cq_t *cqs[], hw_rq_t *rqs[], uint32_t num_rq_pairs, uint32_t entry_count, uint32_t ulp) 673 { 674 ocs_hw_t *hw = cqs[0]->eq->hw; 675 hw_rq_t *rq = NULL; 676 sli4_queue_t *qs[SLI_MAX_RQ_SET_COUNT * 2] = { NULL }; 677 uint32_t max_hw_rq, i, q_count; 678 679 ocs_hw_get(hw, OCS_HW_MAX_RQ_ENTRIES, &max_hw_rq); 680 681 /* Initialise RQS pointers */ 682 for (i = 0; i < num_rq_pairs; i++) { 683 rqs[i] = NULL; 684 } 685 686 for (i = 0, q_count = 0; i < num_rq_pairs; i++, q_count += 2) { 687 rq = ocs_malloc(hw->os, sizeof(*rq), OCS_M_ZERO | OCS_M_NOWAIT); 688 if (rq == NULL) 689 goto error; 690 691 rqs[i] = rq; 692 rq->instance = hw->hw_rq_count++; 693 rq->cq = cqs[i]; 694 rq->type = SLI_QTYPE_RQ; 695 rq->ulp = ulp; 696 rq->entry_count = OCS_MIN(entry_count, OCS_MIN(max_hw_rq, OCS_HW_RQ_NUM_HDR)); 697 698 /* Header RQ */ 699 rq->hdr = &hw->rq[hw->rq_count]; 700 rq->hdr_entry_size = OCS_HW_RQ_HEADER_SIZE; 701 hw->hw_rq_lookup[hw->rq_count] = rq->instance; 702 hw->rq_count++; 703 qs[q_count] = rq->hdr; 704 705 /* Data RQ */ 706 rq->data = &hw->rq[hw->rq_count]; 707 rq->data_entry_size = hw->config.rq_default_buffer_size; 708 hw->hw_rq_lookup[hw->rq_count] = rq->instance; 709 hw->rq_count++; 710 qs[q_count + 1] = rq->data; 711 712 rq->rq_tracker = NULL; 713 } 714 715 if (sli_fc_rq_set_alloc(&hw->sli, num_rq_pairs, qs, 716 cqs[0]->queue->id, 717 rqs[0]->entry_count, 718 rqs[0]->hdr_entry_size, 719 rqs[0]->data_entry_size, 720 ulp)) { 721 ocs_log_err(hw->os, "RQ Set allocation failure for base CQ=%d\n", cqs[0]->queue->id); 722 goto error; 723 } 724 725 for (i = 0; i < num_rq_pairs; i++) { 726 hw->hw_rq[rqs[i]->instance] = rqs[i]; 727 ocs_list_add_tail(&cqs[i]->q_list, rqs[i]); 728 rqs[i]->rq_tracker = ocs_malloc(hw->os, sizeof(ocs_hw_sequence_t*) * 729 rqs[i]->entry_count, OCS_M_ZERO | OCS_M_NOWAIT); 730 if (rqs[i]->rq_tracker == NULL) { 731 ocs_log_err(hw->os, "RQ tracker buf allocation failure\n"); 732 goto error; 733 } 734 } 735 736 return 0; 737 738 error: 739 for (i = 0; i < num_rq_pairs; i++) { 740 if (rqs[i] != NULL) { 741 if (rqs[i]->rq_tracker != NULL) { 742 ocs_free(hw->os, rqs[i]->rq_tracker, 743 sizeof(ocs_hw_sequence_t*) * 744 rqs[i]->entry_count); 745 } 746 ocs_free(hw->os, rqs[i], sizeof(*rqs[i])); 747 } 748 } 749 750 return -1; 751 } 752 753 /** 754 * @brief Free an EQ object 755 * 756 * The EQ object and any child queue objects are freed 757 * 758 * @param eq pointer to EQ object 759 * 760 * @return none 761 */ 762 void 763 hw_del_eq(hw_eq_t *eq) 764 { 765 if (eq != NULL) { 766 hw_cq_t *cq; 767 hw_cq_t *cq_next; 768 769 ocs_list_foreach_safe(&eq->cq_list, cq, cq_next) { 770 hw_del_cq(cq); 771 } 772 ocs_varray_free(eq->wq_array); 773 ocs_list_remove(&eq->hw->eq_list, eq); 774 eq->hw->hw_eq[eq->instance] = NULL; 775 ocs_free(eq->hw->os, eq, sizeof(*eq)); 776 } 777 } 778 779 /** 780 * @brief Free a CQ object 781 * 782 * The CQ object and any child queue objects are freed 783 * 784 * @param cq pointer to CQ object 785 * 786 * @return none 787 */ 788 void 789 hw_del_cq(hw_cq_t *cq) 790 { 791 if (cq != NULL) { 792 hw_q_t *q; 793 hw_q_t *q_next; 794 795 ocs_list_foreach_safe(&cq->q_list, q, q_next) { 796 switch(q->type) { 797 case SLI_QTYPE_MQ: 798 hw_del_mq((hw_mq_t*) q); 799 break; 800 case SLI_QTYPE_WQ: 801 hw_del_wq((hw_wq_t*) q); 802 break; 803 case SLI_QTYPE_RQ: 804 hw_del_rq((hw_rq_t*) q); 805 break; 806 default: 807 break; 808 } 809 } 810 ocs_list_remove(&cq->eq->cq_list, cq); 811 cq->eq->hw->hw_cq[cq->instance] = NULL; 812 ocs_free(cq->eq->hw->os, cq, sizeof(*cq)); 813 } 814 } 815 816 /** 817 * @brief Free a MQ object 818 * 819 * The MQ object is freed 820 * 821 * @param mq pointer to MQ object 822 * 823 * @return none 824 */ 825 void 826 hw_del_mq(hw_mq_t *mq) 827 { 828 if (mq != NULL) { 829 ocs_list_remove(&mq->cq->q_list, mq); 830 mq->cq->eq->hw->hw_mq[mq->instance] = NULL; 831 ocs_free(mq->cq->eq->hw->os, mq, sizeof(*mq)); 832 } 833 } 834 835 /** 836 * @brief Free a WQ object 837 * 838 * The WQ object is freed 839 * 840 * @param wq pointer to WQ object 841 * 842 * @return none 843 */ 844 void 845 hw_del_wq(hw_wq_t *wq) 846 { 847 if (wq != NULL) { 848 ocs_list_remove(&wq->cq->q_list, wq); 849 wq->cq->eq->hw->hw_wq[wq->instance] = NULL; 850 ocs_free(wq->cq->eq->hw->os, wq, sizeof(*wq)); 851 } 852 } 853 854 /** 855 * @brief Free an RQ object 856 * 857 * The RQ object is freed 858 * 859 * @param rq pointer to RQ object 860 * 861 * @return none 862 */ 863 void 864 hw_del_rq(hw_rq_t *rq) 865 { 866 867 if (rq != NULL) { 868 ocs_hw_t *hw = rq->cq->eq->hw; 869 /* Free RQ tracker */ 870 if (rq->rq_tracker != NULL) { 871 ocs_free(hw->os, rq->rq_tracker, sizeof(ocs_hw_sequence_t*) * rq->entry_count); 872 rq->rq_tracker = NULL; 873 } 874 ocs_list_remove(&rq->cq->q_list, rq); 875 hw->hw_rq[rq->instance] = NULL; 876 ocs_free(hw->os, rq, sizeof(*rq)); 877 } 878 } 879 880 /** 881 * @brief Display HW queue objects 882 * 883 * The HW queue objects are displayed using ocs_log 884 * 885 * @param hw pointer to HW object 886 * 887 * @return none 888 */ 889 void 890 hw_queue_dump(ocs_hw_t *hw) 891 { 892 hw_eq_t *eq; 893 hw_cq_t *cq; 894 hw_q_t *q; 895 hw_mq_t *mq; 896 hw_wq_t *wq; 897 hw_rq_t *rq; 898 899 ocs_list_foreach(&hw->eq_list, eq) { 900 ocs_printf("eq[%d] id %2d\n", eq->instance, eq->queue->id); 901 ocs_list_foreach(&eq->cq_list, cq) { 902 ocs_printf(" cq[%d] id %2d current\n", cq->instance, cq->queue->id); 903 ocs_list_foreach(&cq->q_list, q) { 904 switch(q->type) { 905 case SLI_QTYPE_MQ: 906 mq = (hw_mq_t *) q; 907 ocs_printf(" mq[%d] id %2d\n", mq->instance, mq->queue->id); 908 break; 909 case SLI_QTYPE_WQ: 910 wq = (hw_wq_t *) q; 911 ocs_printf(" wq[%d] id %2d\n", wq->instance, wq->queue->id); 912 break; 913 case SLI_QTYPE_RQ: 914 rq = (hw_rq_t *) q; 915 ocs_printf(" rq[%d] hdr id %2d\n", rq->instance, rq->hdr->id); 916 break; 917 default: 918 break; 919 } 920 } 921 } 922 } 923 } 924 925 /** 926 * @brief Teardown HW queue objects 927 * 928 * The HW queue objects are freed 929 * 930 * @param hw pointer to HW object 931 * 932 * @return none 933 */ 934 void 935 hw_queue_teardown(ocs_hw_t *hw) 936 { 937 uint32_t i; 938 hw_eq_t *eq; 939 hw_eq_t *eq_next; 940 941 if (ocs_list_valid(&hw->eq_list)) { 942 ocs_list_foreach_safe(&hw->eq_list, eq, eq_next) { 943 hw_del_eq(eq); 944 } 945 } 946 for (i = 0; i < ARRAY_SIZE(hw->wq_cpu_array); i++) { 947 ocs_varray_free(hw->wq_cpu_array[i]); 948 hw->wq_cpu_array[i] = NULL; 949 } 950 for (i = 0; i < ARRAY_SIZE(hw->wq_class_array); i++) { 951 ocs_varray_free(hw->wq_class_array[i]); 952 hw->wq_class_array[i] = NULL; 953 } 954 } 955 956 /** 957 * @brief Allocate a WQ to an IO object 958 * 959 * The next work queue index is used to assign a WQ to an IO. 960 * 961 * If wq_steering is OCS_HW_WQ_STEERING_CLASS, a WQ from io->wq_class is 962 * selected. 963 * 964 * If wq_steering is OCS_HW_WQ_STEERING_REQUEST, then a WQ from the EQ that 965 * the IO request came in on is selected. 966 * 967 * If wq_steering is OCS_HW_WQ_STEERING_CPU, then a WQ associated with the 968 * CPU the request is made on is selected. 969 * 970 * @param hw pointer to HW object 971 * @param io pointer to IO object 972 * 973 * @return Return pointer to next WQ 974 */ 975 hw_wq_t * 976 ocs_hw_queue_next_wq(ocs_hw_t *hw, ocs_hw_io_t *io) 977 { 978 hw_eq_t *eq; 979 hw_wq_t *wq = NULL; 980 981 switch(io->wq_steering) { 982 case OCS_HW_WQ_STEERING_CLASS: 983 if (likely(io->wq_class < ARRAY_SIZE(hw->wq_class_array))) { 984 wq = ocs_varray_iter_next(hw->wq_class_array[io->wq_class]); 985 } 986 break; 987 case OCS_HW_WQ_STEERING_REQUEST: 988 eq = io->eq; 989 if (likely(eq != NULL)) { 990 wq = ocs_varray_iter_next(eq->wq_array); 991 } 992 break; 993 case OCS_HW_WQ_STEERING_CPU: { 994 uint32_t cpuidx = ocs_thread_getcpu(); 995 996 if (likely(cpuidx < ARRAY_SIZE(hw->wq_cpu_array))) { 997 wq = ocs_varray_iter_next(hw->wq_cpu_array[cpuidx]); 998 } 999 break; 1000 } 1001 } 1002 1003 if (unlikely(wq == NULL)) { 1004 wq = hw->hw_wq[0]; 1005 } 1006 1007 return wq; 1008 } 1009 1010 /** 1011 * @brief Return count of EQs for a queue topology object 1012 * 1013 * The EQ count for in the HWs queue topology (hw->qtop) object is returned 1014 * 1015 * @param hw pointer to HW object 1016 * 1017 * @return count of EQs 1018 */ 1019 uint32_t 1020 ocs_hw_qtop_eq_count(ocs_hw_t *hw) 1021 { 1022 return hw->qtop->entry_counts[QTOP_EQ]; 1023 } 1024 1025 #define TOKEN_LEN 32 1026 1027 /** 1028 * @brief return string given a QTOP entry 1029 * 1030 * @param entry QTOP entry 1031 * 1032 * @return returns string or "unknown" 1033 */ 1034 #if HW_QTOP_DEBUG 1035 static char * 1036 qtopentry2s(ocs_hw_qtop_entry_e entry) { 1037 switch(entry) { 1038 #define P(x) case x: return #x; 1039 P(QTOP_EQ) 1040 P(QTOP_CQ) 1041 P(QTOP_WQ) 1042 P(QTOP_RQ) 1043 P(QTOP_MQ) 1044 P(QTOP_THREAD_START) 1045 P(QTOP_THREAD_END) 1046 P(QTOP_LAST) 1047 #undef P 1048 } 1049 return "unknown"; 1050 } 1051 #endif 1052 1053 /** 1054 * @brief Declare token types 1055 */ 1056 typedef enum { 1057 TOK_LPAREN = 1, 1058 TOK_RPAREN, 1059 TOK_COLON, 1060 TOK_EQUALS, 1061 TOK_QUEUE, 1062 TOK_ATTR_NAME, 1063 TOK_NUMBER, 1064 TOK_NUMBER_VALUE, 1065 TOK_NUMBER_LIST, 1066 } tok_type_e; 1067 1068 /** 1069 * @brief Declare token sub-types 1070 */ 1071 typedef enum { 1072 TOK_SUB_EQ = 100, 1073 TOK_SUB_CQ, 1074 TOK_SUB_RQ, 1075 TOK_SUB_MQ, 1076 TOK_SUB_WQ, 1077 TOK_SUB_LEN, 1078 TOK_SUB_CLASS, 1079 TOK_SUB_ULP, 1080 TOK_SUB_FILTER, 1081 } tok_subtype_e; 1082 1083 /** 1084 * @brief convert queue subtype to QTOP entry 1085 * 1086 * @param q queue subtype 1087 * 1088 * @return QTOP entry or 0 1089 */ 1090 static ocs_hw_qtop_entry_e 1091 subtype2qtop(tok_subtype_e q) 1092 { 1093 switch(q) { 1094 case TOK_SUB_EQ: return QTOP_EQ; 1095 case TOK_SUB_CQ: return QTOP_CQ; 1096 case TOK_SUB_RQ: return QTOP_RQ; 1097 case TOK_SUB_MQ: return QTOP_MQ; 1098 case TOK_SUB_WQ: return QTOP_WQ; 1099 default: 1100 break; 1101 } 1102 return 0; 1103 } 1104 1105 /** 1106 * @brief Declare token object 1107 */ 1108 typedef struct { 1109 tok_type_e type; 1110 tok_subtype_e subtype; 1111 char string[TOKEN_LEN]; 1112 } tok_t; 1113 1114 /** 1115 * @brief Declare token array object 1116 */ 1117 typedef struct { 1118 tok_t *tokens; /* Pointer to array of tokens */ 1119 uint32_t alloc_count; /* Number of tokens in the array */ 1120 uint32_t inuse_count; /* Number of tokens posted to array */ 1121 uint32_t iter_idx; /* Iterator index */ 1122 } tokarray_t; 1123 1124 /** 1125 * @brief Declare token match structure 1126 */ 1127 typedef struct { 1128 char *s; 1129 tok_type_e type; 1130 tok_subtype_e subtype; 1131 } tokmatch_t; 1132 1133 /** 1134 * @brief test if character is ID start character 1135 * 1136 * @param c character to test 1137 * 1138 * @return TRUE if character is an ID start character 1139 */ 1140 static int32_t 1141 idstart(int c) 1142 { 1143 return isalpha(c) || (c == '_') || (c == '$'); 1144 } 1145 1146 /** 1147 * @brief test if character is an ID character 1148 * 1149 * @param c character to test 1150 * 1151 * @return TRUE if character is an ID character 1152 */ 1153 static int32_t 1154 idchar(int c) 1155 { 1156 return idstart(c) || ocs_isdigit(c); 1157 } 1158 1159 /** 1160 * @brief Declare single character matches 1161 */ 1162 static tokmatch_t cmatches[] = { 1163 {"(", TOK_LPAREN}, 1164 {")", TOK_RPAREN}, 1165 {":", TOK_COLON}, 1166 {"=", TOK_EQUALS}, 1167 }; 1168 1169 /** 1170 * @brief Declare identifier match strings 1171 */ 1172 static tokmatch_t smatches[] = { 1173 {"eq", TOK_QUEUE, TOK_SUB_EQ}, 1174 {"cq", TOK_QUEUE, TOK_SUB_CQ}, 1175 {"rq", TOK_QUEUE, TOK_SUB_RQ}, 1176 {"mq", TOK_QUEUE, TOK_SUB_MQ}, 1177 {"wq", TOK_QUEUE, TOK_SUB_WQ}, 1178 {"len", TOK_ATTR_NAME, TOK_SUB_LEN}, 1179 {"class", TOK_ATTR_NAME, TOK_SUB_CLASS}, 1180 {"ulp", TOK_ATTR_NAME, TOK_SUB_ULP}, 1181 {"filter", TOK_ATTR_NAME, TOK_SUB_FILTER}, 1182 }; 1183 1184 /** 1185 * @brief Scan string and return next token 1186 * 1187 * The string is scanned and the next token is returned 1188 * 1189 * @param s input string to scan 1190 * @param tok pointer to place scanned token 1191 * 1192 * @return pointer to input string following scanned token, or NULL 1193 */ 1194 static const char * 1195 tokenize(const char *s, tok_t *tok) 1196 { 1197 uint32_t i; 1198 1199 memset(tok, 0, sizeof(*tok)); 1200 1201 /* Skip over whitespace */ 1202 while (*s && ocs_isspace(*s)) { 1203 s++; 1204 } 1205 1206 /* Return if nothing left in this string */ 1207 if (*s == 0) { 1208 return NULL; 1209 } 1210 1211 /* Look for single character matches */ 1212 for (i = 0; i < ARRAY_SIZE(cmatches); i++) { 1213 if (cmatches[i].s[0] == *s) { 1214 tok->type = cmatches[i].type; 1215 tok->subtype = cmatches[i].subtype; 1216 tok->string[0] = *s++; 1217 return s; 1218 } 1219 } 1220 1221 /* Scan for a hex number or decimal */ 1222 if ((s[0] == '0') && ((s[1] == 'x') || (s[1] == 'X'))) { 1223 char *p = tok->string; 1224 1225 tok->type = TOK_NUMBER; 1226 1227 *p++ = *s++; 1228 *p++ = *s++; 1229 while ((*s == '.') || ocs_isxdigit(*s)) { 1230 if ((p - tok->string) < (int32_t)sizeof(tok->string)) { 1231 *p++ = *s; 1232 } 1233 if (*s == ',') { 1234 tok->type = TOK_NUMBER_LIST; 1235 } 1236 s++; 1237 } 1238 *p = 0; 1239 return s; 1240 } else if (ocs_isdigit(*s)) { 1241 char *p = tok->string; 1242 1243 tok->type = TOK_NUMBER; 1244 while ((*s == ',') || ocs_isdigit(*s)) { 1245 if ((p - tok->string) < (int32_t)sizeof(tok->string)) { 1246 *p++ = *s; 1247 } 1248 if (*s == ',') { 1249 tok->type = TOK_NUMBER_LIST; 1250 } 1251 s++; 1252 } 1253 *p = 0; 1254 return s; 1255 } 1256 1257 /* Scan for an ID */ 1258 if (idstart(*s)) { 1259 char *p = tok->string; 1260 1261 for (*p++ = *s++; idchar(*s); s++) { 1262 if ((p - tok->string) < TOKEN_LEN) { 1263 *p++ = *s; 1264 } 1265 } 1266 1267 /* See if this is a $ number value */ 1268 if (tok->string[0] == '$') { 1269 tok->type = TOK_NUMBER_VALUE; 1270 } else { 1271 /* Look for a string match */ 1272 for (i = 0; i < ARRAY_SIZE(smatches); i++) { 1273 if (strcmp(smatches[i].s, tok->string) == 0) { 1274 tok->type = smatches[i].type; 1275 tok->subtype = smatches[i].subtype; 1276 return s; 1277 } 1278 } 1279 } 1280 } 1281 return s; 1282 } 1283 1284 /** 1285 * @brief convert token type to string 1286 * 1287 * @param type token type 1288 * 1289 * @return string, or "unknown" 1290 */ 1291 static const char * 1292 token_type2s(tok_type_e type) 1293 { 1294 switch(type) { 1295 #define P(x) case x: return #x; 1296 P(TOK_LPAREN) 1297 P(TOK_RPAREN) 1298 P(TOK_COLON) 1299 P(TOK_EQUALS) 1300 P(TOK_QUEUE) 1301 P(TOK_ATTR_NAME) 1302 P(TOK_NUMBER) 1303 P(TOK_NUMBER_VALUE) 1304 P(TOK_NUMBER_LIST) 1305 #undef P 1306 } 1307 return "unknown"; 1308 } 1309 1310 /** 1311 * @brief convert token sub-type to string 1312 * 1313 * @param subtype token sub-type 1314 * 1315 * @return string, or "unknown" 1316 */ 1317 static const char * 1318 token_subtype2s(tok_subtype_e subtype) 1319 { 1320 switch(subtype) { 1321 #define P(x) case x: return #x; 1322 P(TOK_SUB_EQ) 1323 P(TOK_SUB_CQ) 1324 P(TOK_SUB_RQ) 1325 P(TOK_SUB_MQ) 1326 P(TOK_SUB_WQ) 1327 P(TOK_SUB_LEN) 1328 P(TOK_SUB_CLASS) 1329 P(TOK_SUB_ULP) 1330 P(TOK_SUB_FILTER) 1331 #undef P 1332 } 1333 return ""; 1334 } 1335 1336 /** 1337 * @brief Generate syntax error message 1338 * 1339 * A syntax error message is found, the input tokens are dumped up to and including 1340 * the token that failed as indicated by the current iterator index. 1341 * 1342 * @param hw pointer to HW object 1343 * @param tokarray pointer to token array object 1344 * 1345 * @return none 1346 */ 1347 static void 1348 tok_syntax(ocs_hw_t *hw, tokarray_t *tokarray) 1349 { 1350 uint32_t i; 1351 tok_t *tok; 1352 1353 ocs_log_test(hw->os, "Syntax error:\n"); 1354 1355 for (i = 0, tok = tokarray->tokens; (i <= tokarray->inuse_count); i++, tok++) { 1356 ocs_log_test(hw->os, "%s [%2d] %-16s %-16s %s\n", (i == tokarray->iter_idx) ? ">>>" : " ", i, 1357 token_type2s(tok->type), token_subtype2s(tok->subtype), tok->string); 1358 } 1359 } 1360 1361 /** 1362 * @brief parse a number 1363 * 1364 * Parses tokens of type TOK_NUMBER and TOK_NUMBER_VALUE, returning a numeric value 1365 * 1366 * @param hw pointer to HW object 1367 * @param qtop pointer to QTOP object 1368 * @param tok pointer to token to parse 1369 * 1370 * @return numeric value 1371 */ 1372 static uint32_t 1373 tok_getnumber(ocs_hw_t *hw, ocs_hw_qtop_t *qtop, tok_t *tok) 1374 { 1375 uint32_t rval = 0; 1376 uint32_t num_cpus = ocs_get_num_cpus(); 1377 1378 switch(tok->type) { 1379 case TOK_NUMBER_VALUE: 1380 if (ocs_strcmp(tok->string, "$ncpu") == 0) { 1381 rval = num_cpus; 1382 } else if (ocs_strcmp(tok->string, "$ncpu1") == 0) { 1383 rval = num_cpus - 1; 1384 } else if (ocs_strcmp(tok->string, "$nwq") == 0) { 1385 if (hw != NULL) { 1386 rval = hw->config.n_wq; 1387 } 1388 } else if (ocs_strcmp(tok->string, "$maxmrq") == 0) { 1389 rval = MIN(num_cpus, OCS_HW_MAX_MRQS); 1390 } else if (ocs_strcmp(tok->string, "$nulp") == 0) { 1391 rval = hw->ulp_max - hw->ulp_start + 1; 1392 } else if ((qtop->rptcount_idx > 0) && ocs_strcmp(tok->string, "$rpt0") == 0) { 1393 rval = qtop->rptcount[qtop->rptcount_idx-1]; 1394 } else if ((qtop->rptcount_idx > 1) && ocs_strcmp(tok->string, "$rpt1") == 0) { 1395 rval = qtop->rptcount[qtop->rptcount_idx-2]; 1396 } else if ((qtop->rptcount_idx > 2) && ocs_strcmp(tok->string, "$rpt2") == 0) { 1397 rval = qtop->rptcount[qtop->rptcount_idx-3]; 1398 } else if ((qtop->rptcount_idx > 3) && ocs_strcmp(tok->string, "$rpt3") == 0) { 1399 rval = qtop->rptcount[qtop->rptcount_idx-4]; 1400 } else { 1401 rval = ocs_strtoul(tok->string, 0, 0); 1402 } 1403 break; 1404 case TOK_NUMBER: 1405 rval = ocs_strtoul(tok->string, 0, 0); 1406 break; 1407 default: 1408 break; 1409 } 1410 return rval; 1411 } 1412 1413 /** 1414 * @brief parse an array of tokens 1415 * 1416 * The tokens are semantically parsed, to generate QTOP entries. 1417 * 1418 * @param hw pointer to HW object 1419 * @param tokarray array array of tokens 1420 * @param qtop ouptut QTOP object 1421 * 1422 * @return returns 0 for success, a negative error code value for failure. 1423 */ 1424 static int32_t 1425 parse_topology(ocs_hw_t *hw, tokarray_t *tokarray, ocs_hw_qtop_t *qtop) 1426 { 1427 ocs_hw_qtop_entry_t *qt = qtop->entries + qtop->inuse_count; 1428 tok_t *tok; 1429 1430 for (; (tokarray->iter_idx < tokarray->inuse_count) && 1431 ((tok = &tokarray->tokens[tokarray->iter_idx]) != NULL); ) { 1432 if (qtop->inuse_count >= qtop->alloc_count) { 1433 return -1; 1434 } 1435 1436 qt = qtop->entries + qtop->inuse_count; 1437 1438 switch (tok[0].type) 1439 { 1440 case TOK_QUEUE: 1441 qt->entry = subtype2qtop(tok[0].subtype); 1442 qt->set_default = FALSE; 1443 qt->len = 0; 1444 qt->class = 0; 1445 qtop->inuse_count++; 1446 1447 tokarray->iter_idx++; /* Advance current token index */ 1448 1449 /* Parse for queue attributes, possibly multiple instances */ 1450 while ((tokarray->iter_idx + 4) <= tokarray->inuse_count) { 1451 tok = &tokarray->tokens[tokarray->iter_idx]; 1452 if( (tok[0].type == TOK_COLON) && 1453 (tok[1].type == TOK_ATTR_NAME) && 1454 (tok[2].type == TOK_EQUALS) && 1455 ((tok[3].type == TOK_NUMBER) || 1456 (tok[3].type == TOK_NUMBER_VALUE) || 1457 (tok[3].type == TOK_NUMBER_LIST))) { 1458 switch (tok[1].subtype) { 1459 case TOK_SUB_LEN: 1460 qt->len = tok_getnumber(hw, qtop, &tok[3]); 1461 break; 1462 1463 case TOK_SUB_CLASS: 1464 qt->class = tok_getnumber(hw, qtop, &tok[3]); 1465 break; 1466 1467 case TOK_SUB_ULP: 1468 qt->ulp = tok_getnumber(hw, qtop, &tok[3]); 1469 break; 1470 1471 case TOK_SUB_FILTER: 1472 if (tok[3].type == TOK_NUMBER_LIST) { 1473 uint32_t mask = 0; 1474 char *p = tok[3].string; 1475 1476 while ((p != NULL) && *p) { 1477 uint32_t v; 1478 1479 v = ocs_strtoul(p, 0, 0); 1480 if (v < 32) { 1481 mask |= (1U << v); 1482 } 1483 1484 p = ocs_strchr(p, ','); 1485 if (p != NULL) { 1486 p++; 1487 } 1488 } 1489 qt->filter_mask = mask; 1490 } else { 1491 qt->filter_mask = (1U << tok_getnumber(hw, qtop, &tok[3])); 1492 } 1493 break; 1494 default: 1495 break; 1496 } 1497 /* Advance current token index */ 1498 tokarray->iter_idx += 4; 1499 } else { 1500 break; 1501 } 1502 } 1503 qtop->entry_counts[qt->entry]++; 1504 break; 1505 1506 case TOK_ATTR_NAME: 1507 if ( ((tokarray->iter_idx + 5) <= tokarray->inuse_count) && 1508 (tok[1].type == TOK_COLON) && 1509 (tok[2].type == TOK_QUEUE) && 1510 (tok[3].type == TOK_EQUALS) && 1511 ((tok[4].type == TOK_NUMBER) || (tok[4].type == TOK_NUMBER_VALUE))) { 1512 qt->entry = subtype2qtop(tok[2].subtype); 1513 qt->set_default = TRUE; 1514 switch(tok[0].subtype) { 1515 case TOK_SUB_LEN: 1516 qt->len = tok_getnumber(hw, qtop, &tok[4]); 1517 break; 1518 case TOK_SUB_CLASS: 1519 qt->class = tok_getnumber(hw, qtop, &tok[4]); 1520 break; 1521 case TOK_SUB_ULP: 1522 qt->ulp = tok_getnumber(hw, qtop, &tok[4]); 1523 break; 1524 default: 1525 break; 1526 } 1527 qtop->inuse_count++; 1528 tokarray->iter_idx += 5; 1529 } else { 1530 tok_syntax(hw, tokarray); 1531 return -1; 1532 } 1533 break; 1534 1535 case TOK_NUMBER: 1536 case TOK_NUMBER_VALUE: { 1537 uint32_t rpt_count = 1; 1538 uint32_t i; 1539 1540 rpt_count = tok_getnumber(hw, qtop, tok); 1541 1542 if (tok[1].type == TOK_LPAREN) { 1543 uint32_t iter_idx_save; 1544 1545 tokarray->iter_idx += 2; 1546 1547 /* save token array iteration index */ 1548 iter_idx_save = tokarray->iter_idx; 1549 1550 for (i = 0; i < rpt_count; i++) { 1551 uint32_t rptcount_idx = qtop->rptcount_idx; 1552 1553 if (qtop->rptcount_idx < ARRAY_SIZE(qtop->rptcount)) { 1554 qtop->rptcount[qtop->rptcount_idx++] = i; 1555 } 1556 1557 /* restore token array iteration index */ 1558 tokarray->iter_idx = iter_idx_save; 1559 1560 /* parse, append to qtop */ 1561 parse_topology(hw, tokarray, qtop); 1562 1563 qtop->rptcount_idx = rptcount_idx; 1564 } 1565 } 1566 break; 1567 } 1568 1569 case TOK_RPAREN: 1570 tokarray->iter_idx++; 1571 return 0; 1572 1573 default: 1574 tok_syntax(hw, tokarray); 1575 return -1; 1576 } 1577 } 1578 return 0; 1579 } 1580 1581 /** 1582 * @brief Parse queue topology string 1583 * 1584 * The queue topology object is allocated, and filled with the results of parsing the 1585 * passed in queue topology string 1586 * 1587 * @param hw pointer to HW object 1588 * @param qtop_string input queue topology string 1589 * 1590 * @return pointer to allocated QTOP object, or NULL if there was an error 1591 */ 1592 ocs_hw_qtop_t * 1593 ocs_hw_qtop_parse(ocs_hw_t *hw, const char *qtop_string) 1594 { 1595 ocs_hw_qtop_t *qtop; 1596 tokarray_t tokarray; 1597 const char *s; 1598 #if HW_QTOP_DEBUG 1599 uint32_t i; 1600 ocs_hw_qtop_entry_t *qt; 1601 #endif 1602 1603 ocs_log_debug(hw->os, "queue topology: %s\n", qtop_string); 1604 1605 /* Allocate a token array */ 1606 tokarray.tokens = ocs_malloc(hw->os, MAX_TOKENS * sizeof(*tokarray.tokens), OCS_M_ZERO | OCS_M_NOWAIT); 1607 if (tokarray.tokens == NULL) { 1608 return NULL; 1609 } 1610 tokarray.alloc_count = MAX_TOKENS; 1611 tokarray.inuse_count = 0; 1612 tokarray.iter_idx = 0; 1613 1614 /* Parse the tokens */ 1615 for (s = qtop_string; (tokarray.inuse_count < tokarray.alloc_count) && 1616 ((s = tokenize(s, &tokarray.tokens[tokarray.inuse_count]))) != NULL; ) { 1617 tokarray.inuse_count++; 1618 } 1619 1620 /* Allocate a queue topology structure */ 1621 qtop = ocs_malloc(hw->os, sizeof(*qtop), OCS_M_ZERO | OCS_M_NOWAIT); 1622 if (qtop == NULL) { 1623 ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens)); 1624 ocs_log_err(hw->os, "malloc qtop failed\n"); 1625 return NULL; 1626 } 1627 qtop->os = hw->os; 1628 1629 /* Allocate queue topology entries */ 1630 qtop->entries = ocs_malloc(hw->os, OCS_HW_MAX_QTOP_ENTRIES*sizeof(*qtop->entries), OCS_M_ZERO | OCS_M_NOWAIT); 1631 if (qtop->entries == NULL) { 1632 ocs_log_err(hw->os, "malloc qtop entries failed\n"); 1633 ocs_free(hw->os, qtop, sizeof(*qtop)); 1634 ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens)); 1635 return NULL; 1636 } 1637 qtop->alloc_count = OCS_HW_MAX_QTOP_ENTRIES; 1638 qtop->inuse_count = 0; 1639 1640 /* Parse the tokens */ 1641 parse_topology(hw, &tokarray, qtop); 1642 #if HW_QTOP_DEBUG 1643 for (i = 0, qt = qtop->entries; i < qtop->inuse_count; i++, qt++) { 1644 ocs_log_debug(hw->os, "entry %s set_df %d len %4d class %d ulp %d\n", qtopentry2s(qt->entry), qt->set_default, qt->len, 1645 qt->class, qt->ulp); 1646 } 1647 #endif 1648 1649 /* Free the tokens array */ 1650 ocs_free(hw->os, tokarray.tokens, MAX_TOKENS * sizeof(*tokarray.tokens)); 1651 1652 return qtop; 1653 } 1654 1655 /** 1656 * @brief free queue topology object 1657 * 1658 * @param qtop pointer to QTOP object 1659 * 1660 * @return none 1661 */ 1662 void 1663 ocs_hw_qtop_free(ocs_hw_qtop_t *qtop) 1664 { 1665 if (qtop != NULL) { 1666 if (qtop->entries != NULL) { 1667 ocs_free(qtop->os, qtop->entries, qtop->alloc_count*sizeof(*qtop->entries)); 1668 } 1669 ocs_free(qtop->os, qtop, sizeof(*qtop)); 1670 } 1671 } 1672 1673 /* Uncomment this to turn on RQ debug */ 1674 // #define ENABLE_DEBUG_RQBUF 1675 1676 static int32_t ocs_hw_rqpair_find(ocs_hw_t *hw, uint16_t rq_id); 1677 static ocs_hw_sequence_t * ocs_hw_rqpair_get(ocs_hw_t *hw, uint16_t rqindex, uint16_t bufindex); 1678 static int32_t ocs_hw_rqpair_put(ocs_hw_t *hw, ocs_hw_sequence_t *seq); 1679 static ocs_hw_rtn_e ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(ocs_hw_t *hw, ocs_hw_sequence_t *seq); 1680 1681 /** 1682 * @brief Process receive queue completions for RQ Pair mode. 1683 * 1684 * @par Description 1685 * RQ completions are processed. In RQ pair mode, a single header and single payload 1686 * buffer are received, and passed to the function that has registered for unsolicited 1687 * callbacks. 1688 * 1689 * @param hw Hardware context. 1690 * @param cq Pointer to HW completion queue. 1691 * @param cqe Completion queue entry. 1692 * 1693 * @return Returns 0 for success, or a negative error code value for failure. 1694 */ 1695 1696 int32_t 1697 ocs_hw_rqpair_process_rq(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe) 1698 { 1699 uint16_t rq_id; 1700 uint32_t index; 1701 int32_t rqindex; 1702 int32_t rq_status; 1703 uint32_t h_len; 1704 uint32_t p_len; 1705 ocs_hw_sequence_t *seq; 1706 1707 rq_status = sli_fc_rqe_rqid_and_index(&hw->sli, cqe, &rq_id, &index); 1708 if (0 != rq_status) { 1709 switch (rq_status) { 1710 case SLI4_FC_ASYNC_RQ_BUF_LEN_EXCEEDED: 1711 case SLI4_FC_ASYNC_RQ_DMA_FAILURE: 1712 /* just get RQ buffer then return to chip */ 1713 rqindex = ocs_hw_rqpair_find(hw, rq_id); 1714 if (rqindex < 0) { 1715 ocs_log_test(hw->os, "status=%#x: rq_id lookup failed for id=%#x\n", 1716 rq_status, rq_id); 1717 break; 1718 } 1719 1720 /* get RQ buffer */ 1721 seq = ocs_hw_rqpair_get(hw, rqindex, index); 1722 1723 /* return to chip */ 1724 if (ocs_hw_rqpair_sequence_free(hw, seq)) { 1725 ocs_log_test(hw->os, "status=%#x, failed to return buffers to RQ\n", 1726 rq_status); 1727 break; 1728 } 1729 break; 1730 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_NEEDED: 1731 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_FRM_DISC: 1732 /* since RQ buffers were not consumed, cannot return them to chip */ 1733 /* fall through */ 1734 ocs_log_debug(hw->os, "Warning: RCQE status=%#x, \n", rq_status); 1735 default: 1736 break; 1737 } 1738 return -1; 1739 } 1740 1741 rqindex = ocs_hw_rqpair_find(hw, rq_id); 1742 if (rqindex < 0) { 1743 ocs_log_test(hw->os, "Error: rq_id lookup failed for id=%#x\n", rq_id); 1744 return -1; 1745 } 1746 1747 OCS_STAT({ hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]]; rq->use_count++; rq->hdr_use_count++; 1748 rq->payload_use_count++;}) 1749 1750 seq = ocs_hw_rqpair_get(hw, rqindex, index); 1751 ocs_hw_assert(seq != NULL); 1752 1753 seq->hw = hw; 1754 seq->auto_xrdy = 0; 1755 seq->out_of_xris = 0; 1756 seq->xri = 0; 1757 seq->hio = NULL; 1758 1759 sli_fc_rqe_length(&hw->sli, cqe, &h_len, &p_len); 1760 seq->header->dma.len = h_len; 1761 seq->payload->dma.len = p_len; 1762 seq->fcfi = sli_fc_rqe_fcfi(&hw->sli, cqe); 1763 seq->hw_priv = cq->eq; 1764 1765 /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */ 1766 if (hw->config.bounce) { 1767 fc_header_t *hdr = seq->header->dma.virt; 1768 uint32_t s_id = fc_be24toh(hdr->s_id); 1769 uint32_t d_id = fc_be24toh(hdr->d_id); 1770 uint32_t ox_id = ocs_be16toh(hdr->ox_id); 1771 if (hw->callback.bounce != NULL) { 1772 (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id); 1773 } 1774 } else { 1775 hw->callback.unsolicited(hw->args.unsolicited, seq); 1776 } 1777 1778 return 0; 1779 } 1780 1781 /** 1782 * @brief Process receive queue completions for RQ Pair mode - Auto xfer rdy 1783 * 1784 * @par Description 1785 * RQ completions are processed. In RQ pair mode, a single header and single payload 1786 * buffer are received, and passed to the function that has registered for unsolicited 1787 * callbacks. 1788 * 1789 * @param hw Hardware context. 1790 * @param cq Pointer to HW completion queue. 1791 * @param cqe Completion queue entry. 1792 * 1793 * @return Returns 0 for success, or a negative error code value for failure. 1794 */ 1795 1796 int32_t 1797 ocs_hw_rqpair_process_auto_xfr_rdy_cmd(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe) 1798 { 1799 /* Seems silly to call a SLI function to decode - use the structure directly for performance */ 1800 sli4_fc_optimized_write_cmd_cqe_t *opt_wr = (sli4_fc_optimized_write_cmd_cqe_t*)cqe; 1801 uint16_t rq_id; 1802 uint32_t index; 1803 int32_t rqindex; 1804 int32_t rq_status; 1805 uint32_t h_len; 1806 uint32_t p_len; 1807 ocs_hw_sequence_t *seq; 1808 uint8_t axr_lock_taken = 0; 1809 #if defined(OCS_DISC_SPIN_DELAY) 1810 uint32_t delay = 0; 1811 char prop_buf[32]; 1812 #endif 1813 1814 rq_status = sli_fc_rqe_rqid_and_index(&hw->sli, cqe, &rq_id, &index); 1815 if (0 != rq_status) { 1816 switch (rq_status) { 1817 case SLI4_FC_ASYNC_RQ_BUF_LEN_EXCEEDED: 1818 case SLI4_FC_ASYNC_RQ_DMA_FAILURE: 1819 /* just get RQ buffer then return to chip */ 1820 rqindex = ocs_hw_rqpair_find(hw, rq_id); 1821 if (rqindex < 0) { 1822 ocs_log_err(hw->os, "status=%#x: rq_id lookup failed for id=%#x\n", 1823 rq_status, rq_id); 1824 break; 1825 } 1826 1827 /* get RQ buffer */ 1828 seq = ocs_hw_rqpair_get(hw, rqindex, index); 1829 1830 /* return to chip */ 1831 if (ocs_hw_rqpair_sequence_free(hw, seq)) { 1832 ocs_log_err(hw->os, "status=%#x, failed to return buffers to RQ\n", 1833 rq_status); 1834 break; 1835 } 1836 break; 1837 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_NEEDED: 1838 case SLI4_FC_ASYNC_RQ_INSUFF_BUF_FRM_DISC: 1839 /* since RQ buffers were not consumed, cannot return them to chip */ 1840 ocs_log_debug(hw->os, "Warning: RCQE status=%#x, \n", rq_status); 1841 /* fall through */ 1842 default: 1843 break; 1844 } 1845 return -1; 1846 } 1847 1848 rqindex = ocs_hw_rqpair_find(hw, rq_id); 1849 if (rqindex < 0) { 1850 ocs_log_err(hw->os, "Error: rq_id lookup failed for id=%#x\n", rq_id); 1851 return -1; 1852 } 1853 1854 OCS_STAT({ hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]]; rq->use_count++; rq->hdr_use_count++; 1855 rq->payload_use_count++;}) 1856 1857 seq = ocs_hw_rqpair_get(hw, rqindex, index); 1858 ocs_hw_assert(seq != NULL); 1859 1860 seq->hw = hw; 1861 seq->auto_xrdy = opt_wr->agxr; 1862 seq->out_of_xris = opt_wr->oox; 1863 seq->xri = opt_wr->xri; 1864 seq->hio = NULL; 1865 1866 sli_fc_rqe_length(&hw->sli, cqe, &h_len, &p_len); 1867 seq->header->dma.len = h_len; 1868 seq->payload->dma.len = p_len; 1869 seq->fcfi = sli_fc_rqe_fcfi(&hw->sli, cqe); 1870 seq->hw_priv = cq->eq; 1871 1872 if (seq->auto_xrdy) { 1873 fc_header_t *fc_hdr = seq->header->dma.virt; 1874 1875 seq->hio = ocs_hw_io_lookup(hw, seq->xri); 1876 ocs_lock(&seq->hio->axr_lock); 1877 axr_lock_taken = 1; 1878 1879 /* save the FCFI, src_id, dest_id and ox_id because we need it for the sequence object when the data comes. */ 1880 seq->hio->axr_buf->fcfi = seq->fcfi; 1881 seq->hio->axr_buf->hdr.ox_id = fc_hdr->ox_id; 1882 seq->hio->axr_buf->hdr.s_id = fc_hdr->s_id; 1883 seq->hio->axr_buf->hdr.d_id = fc_hdr->d_id; 1884 seq->hio->axr_buf->cmd_cqe = 1; 1885 1886 /* 1887 * Since auto xfer rdy is used for this IO, then clear the sequence 1888 * initiative bit in the header so that the upper layers wait for the 1889 * data. This should flow exactly like the first burst case. 1890 */ 1891 fc_hdr->f_ctl &= fc_htobe24(~FC_FCTL_SEQUENCE_INITIATIVE); 1892 1893 /* If AXR CMD CQE came before previous TRSP CQE of same XRI */ 1894 if (seq->hio->type == OCS_HW_IO_TARGET_RSP) { 1895 seq->hio->axr_buf->call_axr_cmd = 1; 1896 seq->hio->axr_buf->cmd_seq = seq; 1897 goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_cmd; 1898 } 1899 } 1900 1901 /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */ 1902 if (hw->config.bounce) { 1903 fc_header_t *hdr = seq->header->dma.virt; 1904 uint32_t s_id = fc_be24toh(hdr->s_id); 1905 uint32_t d_id = fc_be24toh(hdr->d_id); 1906 uint32_t ox_id = ocs_be16toh(hdr->ox_id); 1907 if (hw->callback.bounce != NULL) { 1908 (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id); 1909 } 1910 } else { 1911 hw->callback.unsolicited(hw->args.unsolicited, seq); 1912 } 1913 1914 if (seq->auto_xrdy) { 1915 /* If data cqe came before cmd cqe in out of order in case of AXR */ 1916 if(seq->hio->axr_buf->data_cqe == 1) { 1917 #if defined(OCS_DISC_SPIN_DELAY) 1918 if (ocs_get_property("disk_spin_delay", prop_buf, sizeof(prop_buf)) == 0) { 1919 delay = ocs_strtoul(prop_buf, 0, 0); 1920 ocs_udelay(delay); 1921 } 1922 #endif 1923 /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */ 1924 if (hw->config.bounce) { 1925 fc_header_t *hdr = seq->header->dma.virt; 1926 uint32_t s_id = fc_be24toh(hdr->s_id); 1927 uint32_t d_id = fc_be24toh(hdr->d_id); 1928 uint32_t ox_id = ocs_be16toh(hdr->ox_id); 1929 if (hw->callback.bounce != NULL) { 1930 (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, &seq->hio->axr_buf->seq, s_id, d_id, ox_id); 1931 } 1932 } else { 1933 hw->callback.unsolicited(hw->args.unsolicited, &seq->hio->axr_buf->seq); 1934 } 1935 } 1936 } 1937 1938 exit_ocs_hw_rqpair_process_auto_xfr_rdy_cmd: 1939 if(axr_lock_taken) { 1940 ocs_unlock(&seq->hio->axr_lock); 1941 } 1942 return 0; 1943 } 1944 1945 /** 1946 * @brief Process CQ completions for Auto xfer rdy data phases. 1947 * 1948 * @par Description 1949 * The data is DMA'd into the data buffer posted to the SGL prior to the XRI 1950 * being assigned to an IO. When the completion is received, All of the data 1951 * is in the single buffer. 1952 * 1953 * @param hw Hardware context. 1954 * @param cq Pointer to HW completion queue. 1955 * @param cqe Completion queue entry. 1956 * 1957 * @return Returns 0 for success, or a negative error code value for failure. 1958 */ 1959 1960 int32_t 1961 ocs_hw_rqpair_process_auto_xfr_rdy_data(ocs_hw_t *hw, hw_cq_t *cq, uint8_t *cqe) 1962 { 1963 /* Seems silly to call a SLI function to decode - use the structure directly for performance */ 1964 sli4_fc_optimized_write_data_cqe_t *opt_wr = (sli4_fc_optimized_write_data_cqe_t*)cqe; 1965 ocs_hw_sequence_t *seq; 1966 ocs_hw_io_t *io; 1967 ocs_hw_auto_xfer_rdy_buffer_t *buf; 1968 #if defined(OCS_DISC_SPIN_DELAY) 1969 uint32_t delay = 0; 1970 char prop_buf[32]; 1971 #endif 1972 /* Look up the IO */ 1973 io = ocs_hw_io_lookup(hw, opt_wr->xri); 1974 ocs_lock(&io->axr_lock); 1975 buf = io->axr_buf; 1976 buf->data_cqe = 1; 1977 seq = &buf->seq; 1978 seq->hw = hw; 1979 seq->auto_xrdy = 1; 1980 seq->out_of_xris = 0; 1981 seq->xri = opt_wr->xri; 1982 seq->hio = io; 1983 seq->header = &buf->header; 1984 seq->payload = &buf->payload; 1985 1986 seq->header->dma.len = sizeof(fc_header_t); 1987 seq->payload->dma.len = opt_wr->total_data_placed; 1988 seq->fcfi = buf->fcfi; 1989 seq->hw_priv = cq->eq; 1990 1991 if (opt_wr->status == SLI4_FC_WCQE_STATUS_SUCCESS) { 1992 seq->status = OCS_HW_UNSOL_SUCCESS; 1993 } else if (opt_wr->status == SLI4_FC_WCQE_STATUS_REMOTE_STOP) { 1994 seq->status = OCS_HW_UNSOL_ABTS_RCVD; 1995 } else { 1996 seq->status = OCS_HW_UNSOL_ERROR; 1997 } 1998 1999 /* If AXR CMD CQE came before previous TRSP CQE of same XRI */ 2000 if(io->type == OCS_HW_IO_TARGET_RSP) { 2001 io->axr_buf->call_axr_data = 1; 2002 goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_data; 2003 } 2004 2005 if(!buf->cmd_cqe) { 2006 /* if data cqe came before cmd cqe, return here, cmd cqe will handle */ 2007 goto exit_ocs_hw_rqpair_process_auto_xfr_rdy_data; 2008 } 2009 #if defined(OCS_DISC_SPIN_DELAY) 2010 if (ocs_get_property("disk_spin_delay", prop_buf, sizeof(prop_buf)) == 0) { 2011 delay = ocs_strtoul(prop_buf, 0, 0); 2012 ocs_udelay(delay); 2013 } 2014 #endif 2015 2016 /* bounce enabled, single RQ, we snoop the ox_id to choose the cpuidx */ 2017 if (hw->config.bounce) { 2018 fc_header_t *hdr = seq->header->dma.virt; 2019 uint32_t s_id = fc_be24toh(hdr->s_id); 2020 uint32_t d_id = fc_be24toh(hdr->d_id); 2021 uint32_t ox_id = ocs_be16toh(hdr->ox_id); 2022 if (hw->callback.bounce != NULL) { 2023 (*hw->callback.bounce)(ocs_hw_unsol_process_bounce, seq, s_id, d_id, ox_id); 2024 } 2025 } else { 2026 hw->callback.unsolicited(hw->args.unsolicited, seq); 2027 } 2028 2029 exit_ocs_hw_rqpair_process_auto_xfr_rdy_data: 2030 ocs_unlock(&io->axr_lock); 2031 return 0; 2032 } 2033 2034 /** 2035 * @brief Return pointer to RQ buffer entry. 2036 * 2037 * @par Description 2038 * Returns a pointer to the RQ buffer entry given by @c rqindex and @c bufindex. 2039 * 2040 * @param hw Hardware context. 2041 * @param rqindex Index of the RQ that is being processed. 2042 * @param bufindex Index into the RQ that is being processed. 2043 * 2044 * @return Pointer to the sequence structure, or NULL otherwise. 2045 */ 2046 static ocs_hw_sequence_t * 2047 ocs_hw_rqpair_get(ocs_hw_t *hw, uint16_t rqindex, uint16_t bufindex) 2048 { 2049 sli4_queue_t *rq_hdr = &hw->rq[rqindex]; 2050 sli4_queue_t *rq_payload = &hw->rq[rqindex+1]; 2051 ocs_hw_sequence_t *seq = NULL; 2052 hw_rq_t *rq = hw->hw_rq[hw->hw_rq_lookup[rqindex]]; 2053 2054 #if defined(ENABLE_DEBUG_RQBUF) 2055 uint64_t rqbuf_debug_value = 0xdead0000 | ((rq->id & 0xf) << 12) | (bufindex & 0xfff); 2056 #endif 2057 2058 if (bufindex >= rq_hdr->length) { 2059 ocs_log_err(hw->os, "RQ index %d bufindex %d exceed ring length %d for id %d\n", 2060 rqindex, bufindex, rq_hdr->length, rq_hdr->id); 2061 return NULL; 2062 } 2063 2064 sli_queue_lock(rq_hdr); 2065 sli_queue_lock(rq_payload); 2066 2067 #if defined(ENABLE_DEBUG_RQBUF) 2068 /* Put a debug value into the rq, to track which entries are still valid */ 2069 _sli_queue_poke(&hw->sli, rq_hdr, bufindex, (uint8_t *)&rqbuf_debug_value); 2070 _sli_queue_poke(&hw->sli, rq_payload, bufindex, (uint8_t *)&rqbuf_debug_value); 2071 #endif 2072 2073 seq = rq->rq_tracker[bufindex]; 2074 rq->rq_tracker[bufindex] = NULL; 2075 2076 if (seq == NULL ) { 2077 ocs_log_err(hw->os, "RQ buffer NULL, rqindex %d, bufindex %d, current q index = %d\n", 2078 rqindex, bufindex, rq_hdr->index); 2079 } 2080 2081 sli_queue_unlock(rq_payload); 2082 sli_queue_unlock(rq_hdr); 2083 return seq; 2084 } 2085 2086 /** 2087 * @brief Posts an RQ buffer to a queue and update the verification structures 2088 * 2089 * @param hw hardware context 2090 * @param seq Pointer to sequence object. 2091 * 2092 * @return Returns 0 on success, or a non-zero value otherwise. 2093 */ 2094 static int32_t 2095 ocs_hw_rqpair_put(ocs_hw_t *hw, ocs_hw_sequence_t *seq) 2096 { 2097 sli4_queue_t *rq_hdr = &hw->rq[seq->header->rqindex]; 2098 sli4_queue_t *rq_payload = &hw->rq[seq->payload->rqindex]; 2099 uint32_t hw_rq_index = hw->hw_rq_lookup[seq->header->rqindex]; 2100 hw_rq_t *rq = hw->hw_rq[hw_rq_index]; 2101 uint32_t phys_hdr[2]; 2102 uint32_t phys_payload[2]; 2103 int32_t qindex_hdr; 2104 int32_t qindex_payload; 2105 2106 /* Update the RQ verification lookup tables */ 2107 phys_hdr[0] = ocs_addr32_hi(seq->header->dma.phys); 2108 phys_hdr[1] = ocs_addr32_lo(seq->header->dma.phys); 2109 phys_payload[0] = ocs_addr32_hi(seq->payload->dma.phys); 2110 phys_payload[1] = ocs_addr32_lo(seq->payload->dma.phys); 2111 2112 sli_queue_lock(rq_hdr); 2113 sli_queue_lock(rq_payload); 2114 2115 /* 2116 * Note: The header must be posted last for buffer pair mode because 2117 * posting on the header queue posts the payload queue as well. 2118 * We do not ring the payload queue independently in RQ pair mode. 2119 */ 2120 qindex_payload = _sli_queue_write(&hw->sli, rq_payload, (void *)phys_payload); 2121 qindex_hdr = _sli_queue_write(&hw->sli, rq_hdr, (void *)phys_hdr); 2122 if (qindex_hdr < 0 || 2123 qindex_payload < 0) { 2124 ocs_log_err(hw->os, "RQ_ID=%#x write failed\n", rq_hdr->id); 2125 sli_queue_unlock(rq_payload); 2126 sli_queue_unlock(rq_hdr); 2127 return OCS_HW_RTN_ERROR; 2128 } 2129 2130 /* ensure the indexes are the same */ 2131 ocs_hw_assert(qindex_hdr == qindex_payload); 2132 2133 /* Update the lookup table */ 2134 if (rq->rq_tracker[qindex_hdr] == NULL) { 2135 rq->rq_tracker[qindex_hdr] = seq; 2136 } else { 2137 ocs_log_test(hw->os, "expected rq_tracker[%d][%d] buffer to be NULL\n", 2138 hw_rq_index, qindex_hdr); 2139 } 2140 2141 sli_queue_unlock(rq_payload); 2142 sli_queue_unlock(rq_hdr); 2143 return OCS_HW_RTN_SUCCESS; 2144 } 2145 2146 /** 2147 * @brief Return RQ buffers (while in RQ pair mode). 2148 * 2149 * @par Description 2150 * The header and payload buffers are returned to the Receive Queue. 2151 * 2152 * @param hw Hardware context. 2153 * @param seq Header/payload sequence buffers. 2154 * 2155 * @return Returns OCS_HW_RTN_SUCCESS on success, or an error code value on failure. 2156 */ 2157 2158 ocs_hw_rtn_e 2159 ocs_hw_rqpair_sequence_free(ocs_hw_t *hw, ocs_hw_sequence_t *seq) 2160 { 2161 ocs_hw_rtn_e rc = OCS_HW_RTN_SUCCESS; 2162 2163 /* Check for auto xfer rdy dummy buffers and call the proper release function. */ 2164 if (seq->header->rqindex == OCS_HW_RQ_INDEX_DUMMY_HDR) { 2165 return ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(hw, seq); 2166 } 2167 2168 /* 2169 * Post the data buffer first. Because in RQ pair mode, ringing the 2170 * doorbell of the header ring will post the data buffer as well. 2171 */ 2172 if (ocs_hw_rqpair_put(hw, seq)) { 2173 ocs_log_err(hw->os, "error writing buffers\n"); 2174 return OCS_HW_RTN_ERROR; 2175 } 2176 2177 return rc; 2178 } 2179 2180 /** 2181 * @brief Find the RQ index of RQ_ID. 2182 * 2183 * @param hw Hardware context. 2184 * @param rq_id RQ ID to find. 2185 * 2186 * @return Returns the RQ index, or -1 if not found 2187 */ 2188 static inline int32_t 2189 ocs_hw_rqpair_find(ocs_hw_t *hw, uint16_t rq_id) 2190 { 2191 return ocs_hw_queue_hash_find(hw->rq_hash, rq_id); 2192 } 2193 2194 /** 2195 * @ingroup devInitShutdown 2196 * @brief Allocate auto xfer rdy buffers. 2197 * 2198 * @par Description 2199 * Allocates the auto xfer rdy buffers and places them on the free list. 2200 * 2201 * @param hw Hardware context allocated by the caller. 2202 * @param num_buffers Number of buffers to allocate. 2203 * 2204 * @return Returns 0 on success, or a non-zero value on failure. 2205 */ 2206 ocs_hw_rtn_e 2207 ocs_hw_rqpair_auto_xfer_rdy_buffer_alloc(ocs_hw_t *hw, uint32_t num_buffers) 2208 { 2209 ocs_hw_auto_xfer_rdy_buffer_t *buf; 2210 uint32_t i; 2211 2212 hw->auto_xfer_rdy_buf_pool = ocs_pool_alloc(hw->os, sizeof(ocs_hw_auto_xfer_rdy_buffer_t), num_buffers, FALSE); 2213 if (hw->auto_xfer_rdy_buf_pool == NULL) { 2214 ocs_log_err(hw->os, "Failure to allocate auto xfer ready buffer pool\n"); 2215 return OCS_HW_RTN_NO_MEMORY; 2216 } 2217 2218 for (i = 0; i < num_buffers; i++) { 2219 /* allocate the wrapper object */ 2220 buf = ocs_pool_get_instance(hw->auto_xfer_rdy_buf_pool, i); 2221 ocs_hw_assert(buf != NULL); 2222 2223 /* allocate the auto xfer ready buffer */ 2224 if (ocs_dma_alloc(hw->os, &buf->payload.dma, hw->config.auto_xfer_rdy_size, OCS_MIN_DMA_ALIGNMENT)) { 2225 ocs_log_err(hw->os, "DMA allocation failed\n"); 2226 ocs_free(hw->os, buf, sizeof(*buf)); 2227 return OCS_HW_RTN_NO_MEMORY; 2228 } 2229 2230 /* build a fake data header in big endian */ 2231 buf->hdr.info = FC_RCTL_INFO_SOL_DATA; 2232 buf->hdr.r_ctl = FC_RCTL_FC4_DATA; 2233 buf->hdr.type = FC_TYPE_FCP; 2234 buf->hdr.f_ctl = fc_htobe24(FC_FCTL_EXCHANGE_RESPONDER | 2235 FC_FCTL_FIRST_SEQUENCE | 2236 FC_FCTL_LAST_SEQUENCE | 2237 FC_FCTL_END_SEQUENCE | 2238 FC_FCTL_SEQUENCE_INITIATIVE); 2239 2240 /* build the fake header DMA object */ 2241 buf->header.rqindex = OCS_HW_RQ_INDEX_DUMMY_HDR; 2242 buf->header.dma.virt = &buf->hdr; 2243 buf->header.dma.alloc = buf; 2244 buf->header.dma.size = sizeof(buf->hdr); 2245 buf->header.dma.len = sizeof(buf->hdr); 2246 2247 buf->payload.rqindex = OCS_HW_RQ_INDEX_DUMMY_DATA; 2248 } 2249 return OCS_HW_RTN_SUCCESS; 2250 } 2251 2252 /** 2253 * @ingroup devInitShutdown 2254 * @brief Post Auto xfer rdy buffers to the XRIs posted with DNRX. 2255 * 2256 * @par Description 2257 * When new buffers are freed, check existing XRIs waiting for buffers. 2258 * 2259 * @param hw Hardware context allocated by the caller. 2260 */ 2261 static void 2262 ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(ocs_hw_t *hw) 2263 { 2264 ocs_hw_io_t *io; 2265 int32_t rc; 2266 2267 ocs_lock(&hw->io_lock); 2268 2269 while (!ocs_list_empty(&hw->io_port_dnrx)) { 2270 io = ocs_list_remove_head(&hw->io_port_dnrx); 2271 rc = ocs_hw_reque_xri(hw, io); 2272 if(rc) { 2273 break; 2274 } 2275 } 2276 2277 ocs_unlock(&hw->io_lock); 2278 } 2279 2280 /** 2281 * @brief Called when the POST_SGL_PAGE command completes. 2282 * 2283 * @par Description 2284 * Free the mailbox command buffer. 2285 * 2286 * @param hw Hardware context. 2287 * @param status Status field from the mbox completion. 2288 * @param mqe Mailbox response structure. 2289 * @param arg Pointer to a callback function that signals the caller that the command is done. 2290 * 2291 * @return Returns 0. 2292 */ 2293 static int32_t 2294 ocs_hw_rqpair_auto_xfer_rdy_move_to_port_cb(ocs_hw_t *hw, int32_t status, uint8_t *mqe, void *arg) 2295 { 2296 if (status != 0) { 2297 ocs_log_debug(hw->os, "Status 0x%x\n", status); 2298 } 2299 2300 ocs_free(hw->os, mqe, SLI4_BMBX_SIZE); 2301 return 0; 2302 } 2303 2304 /** 2305 * @brief Prepares an XRI to move to the chip. 2306 * 2307 * @par Description 2308 * Puts the data SGL into the SGL list for the IO object and possibly registers 2309 * an SGL list for the XRI. Since both the POST_XRI and POST_SGL_PAGES commands are 2310 * mailbox commands, we don't need to wait for completion before preceding. 2311 * 2312 * @param hw Hardware context allocated by the caller. 2313 * @param io Pointer to the IO object. 2314 * 2315 * @return Returns OCS_HW_RTN_SUCCESS for success, or an error code value for failure. 2316 */ 2317 ocs_hw_rtn_e 2318 ocs_hw_rqpair_auto_xfer_rdy_move_to_port(ocs_hw_t *hw, ocs_hw_io_t *io) 2319 { 2320 /* We only need to preregister the SGL if it has not yet been done. */ 2321 if (!sli_get_sgl_preregister(&hw->sli)) { 2322 uint8_t *post_sgl; 2323 ocs_dma_t *psgls = &io->def_sgl; 2324 ocs_dma_t **sgls = &psgls; 2325 2326 /* non-local buffer required for mailbox queue */ 2327 post_sgl = ocs_malloc(hw->os, SLI4_BMBX_SIZE, OCS_M_NOWAIT); 2328 if (post_sgl == NULL) { 2329 ocs_log_err(hw->os, "no buffer for command\n"); 2330 return OCS_HW_RTN_NO_MEMORY; 2331 } 2332 if (sli_cmd_fcoe_post_sgl_pages(&hw->sli, post_sgl, SLI4_BMBX_SIZE, 2333 io->indicator, 1, sgls, NULL, NULL)) { 2334 if (ocs_hw_command(hw, post_sgl, OCS_CMD_NOWAIT, 2335 ocs_hw_rqpair_auto_xfer_rdy_move_to_port_cb, NULL)) { 2336 ocs_free(hw->os, post_sgl, SLI4_BMBX_SIZE); 2337 ocs_log_err(hw->os, "SGL post failed\n"); 2338 return OCS_HW_RTN_ERROR; 2339 } 2340 } 2341 } 2342 2343 ocs_lock(&hw->io_lock); 2344 if (ocs_hw_rqpair_auto_xfer_rdy_buffer_post(hw, io, 0) != 0) { /* DNRX set - no buffer */ 2345 ocs_unlock(&hw->io_lock); 2346 return OCS_HW_RTN_ERROR; 2347 } 2348 ocs_unlock(&hw->io_lock); 2349 return OCS_HW_RTN_SUCCESS; 2350 } 2351 2352 /** 2353 * @brief Prepares an XRI to move back to the host. 2354 * 2355 * @par Description 2356 * Releases any attached buffer back to the pool. 2357 * 2358 * @param hw Hardware context allocated by the caller. 2359 * @param io Pointer to the IO object. 2360 */ 2361 void 2362 ocs_hw_rqpair_auto_xfer_rdy_move_to_host(ocs_hw_t *hw, ocs_hw_io_t *io) 2363 { 2364 if (io->axr_buf != NULL) { 2365 ocs_lock(&hw->io_lock); 2366 /* check list and remove if there */ 2367 if (ocs_list_on_list(&io->dnrx_link)) { 2368 ocs_list_remove(&hw->io_port_dnrx, io); 2369 io->auto_xfer_rdy_dnrx = 0; 2370 2371 /* release the count for waiting for a buffer */ 2372 ocs_hw_io_free(hw, io); 2373 } 2374 2375 ocs_pool_put(hw->auto_xfer_rdy_buf_pool, io->axr_buf); 2376 io->axr_buf = NULL; 2377 ocs_unlock(&hw->io_lock); 2378 2379 ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(hw); 2380 } 2381 return; 2382 } 2383 2384 /** 2385 * @brief Posts an auto xfer rdy buffer to an IO. 2386 * 2387 * @par Description 2388 * Puts the data SGL into the SGL list for the IO object 2389 * @n @name 2390 * @b Note: io_lock must be held. 2391 * 2392 * @param hw Hardware context allocated by the caller. 2393 * @param io Pointer to the IO object. 2394 * 2395 * @return Returns the value of DNRX bit in the TRSP and ABORT WQEs. 2396 */ 2397 uint8_t 2398 ocs_hw_rqpair_auto_xfer_rdy_buffer_post(ocs_hw_t *hw, ocs_hw_io_t *io, int reuse_buf) 2399 { 2400 ocs_hw_auto_xfer_rdy_buffer_t *buf; 2401 sli4_sge_t *data; 2402 2403 if(!reuse_buf) { 2404 buf = ocs_pool_get(hw->auto_xfer_rdy_buf_pool); 2405 io->axr_buf = buf; 2406 } 2407 2408 data = io->def_sgl.virt; 2409 data[0].sge_type = SLI4_SGE_TYPE_SKIP; 2410 data[0].last = 0; 2411 2412 /* 2413 * Note: if we are doing DIF assists, then the SGE[1] must contain the 2414 * DI_SEED SGE. The host is responsible for programming: 2415 * SGE Type (Word 2, bits 30:27) 2416 * Replacement App Tag (Word 2 bits 15:0) 2417 * App Tag (Word 3 bits 15:0) 2418 * New Ref Tag (Word 3 bit 23) 2419 * Metadata Enable (Word 3 bit 20) 2420 * Auto-Increment RefTag (Word 3 bit 19) 2421 * Block Size (Word 3 bits 18:16) 2422 * The following fields are managed by the SLI Port: 2423 * Ref Tag Compare (Word 0) 2424 * Replacement Ref Tag (Word 1) - In not the LBA 2425 * NA (Word 2 bit 25) 2426 * Opcode RX (Word 3 bits 27:24) 2427 * Checksum Enable (Word 3 bit 22) 2428 * RefTag Enable (Word 3 bit 21) 2429 * 2430 * The first two SGLs are cleared by ocs_hw_io_init_sges(), so assume eveything is cleared. 2431 */ 2432 if (hw->config.auto_xfer_rdy_p_type) { 2433 sli4_diseed_sge_t *diseed = (sli4_diseed_sge_t*)&data[1]; 2434 2435 diseed->sge_type = SLI4_SGE_TYPE_DISEED; 2436 diseed->repl_app_tag = hw->config.auto_xfer_rdy_app_tag_value; 2437 diseed->app_tag_cmp = hw->config.auto_xfer_rdy_app_tag_value; 2438 diseed->check_app_tag = hw->config.auto_xfer_rdy_app_tag_valid; 2439 diseed->auto_incr_ref_tag = TRUE; /* Always the LBA */ 2440 diseed->dif_blk_size = hw->config.auto_xfer_rdy_blk_size_chip; 2441 } else { 2442 data[1].sge_type = SLI4_SGE_TYPE_SKIP; 2443 data[1].last = 0; 2444 } 2445 2446 data[2].sge_type = SLI4_SGE_TYPE_DATA; 2447 data[2].buffer_address_high = ocs_addr32_hi(io->axr_buf->payload.dma.phys); 2448 data[2].buffer_address_low = ocs_addr32_lo(io->axr_buf->payload.dma.phys); 2449 data[2].buffer_length = io->axr_buf->payload.dma.size; 2450 data[2].last = TRUE; 2451 data[3].sge_type = SLI4_SGE_TYPE_SKIP; 2452 2453 return 0; 2454 } 2455 2456 /** 2457 * @brief Return auto xfer ready buffers (while in RQ pair mode). 2458 * 2459 * @par Description 2460 * The header and payload buffers are returned to the auto xfer rdy pool. 2461 * 2462 * @param hw Hardware context. 2463 * @param seq Header/payload sequence buffers. 2464 * 2465 * @return Returns OCS_HW_RTN_SUCCESS for success, an error code value for failure. 2466 */ 2467 2468 static ocs_hw_rtn_e 2469 ocs_hw_rqpair_auto_xfer_rdy_buffer_sequence_reset(ocs_hw_t *hw, ocs_hw_sequence_t *seq) 2470 { 2471 ocs_hw_auto_xfer_rdy_buffer_t *buf = seq->header->dma.alloc; 2472 2473 buf->data_cqe = 0; 2474 buf->cmd_cqe = 0; 2475 buf->fcfi = 0; 2476 buf->call_axr_cmd = 0; 2477 buf->call_axr_data = 0; 2478 2479 /* build a fake data header in big endian */ 2480 buf->hdr.info = FC_RCTL_INFO_SOL_DATA; 2481 buf->hdr.r_ctl = FC_RCTL_FC4_DATA; 2482 buf->hdr.type = FC_TYPE_FCP; 2483 buf->hdr.f_ctl = fc_htobe24(FC_FCTL_EXCHANGE_RESPONDER | 2484 FC_FCTL_FIRST_SEQUENCE | 2485 FC_FCTL_LAST_SEQUENCE | 2486 FC_FCTL_END_SEQUENCE | 2487 FC_FCTL_SEQUENCE_INITIATIVE); 2488 2489 /* build the fake header DMA object */ 2490 buf->header.rqindex = OCS_HW_RQ_INDEX_DUMMY_HDR; 2491 buf->header.dma.virt = &buf->hdr; 2492 buf->header.dma.alloc = buf; 2493 buf->header.dma.size = sizeof(buf->hdr); 2494 buf->header.dma.len = sizeof(buf->hdr); 2495 buf->payload.rqindex = OCS_HW_RQ_INDEX_DUMMY_DATA; 2496 2497 ocs_hw_rqpair_auto_xfer_rdy_dnrx_check(hw); 2498 2499 return OCS_HW_RTN_SUCCESS; 2500 } 2501 2502 /** 2503 * @ingroup devInitShutdown 2504 * @brief Free auto xfer rdy buffers. 2505 * 2506 * @par Description 2507 * Frees the auto xfer rdy buffers. 2508 * 2509 * @param hw Hardware context allocated by the caller. 2510 * 2511 * @return Returns 0 on success, or a non-zero value on failure. 2512 */ 2513 static void 2514 ocs_hw_rqpair_auto_xfer_rdy_buffer_free(ocs_hw_t *hw) 2515 { 2516 ocs_hw_auto_xfer_rdy_buffer_t *buf; 2517 uint32_t i; 2518 2519 if (hw->auto_xfer_rdy_buf_pool != NULL) { 2520 ocs_lock(&hw->io_lock); 2521 for (i = 0; i < ocs_pool_get_count(hw->auto_xfer_rdy_buf_pool); i++) { 2522 buf = ocs_pool_get_instance(hw->auto_xfer_rdy_buf_pool, i); 2523 if (buf != NULL) { 2524 ocs_dma_free(hw->os, &buf->payload.dma); 2525 } 2526 } 2527 ocs_unlock(&hw->io_lock); 2528 2529 ocs_pool_free(hw->auto_xfer_rdy_buf_pool); 2530 hw->auto_xfer_rdy_buf_pool = NULL; 2531 } 2532 } 2533 2534 /** 2535 * @ingroup devInitShutdown 2536 * @brief Configure the rq_pair function from ocs_hw_init(). 2537 * 2538 * @par Description 2539 * Allocates the buffers to auto xfer rdy and posts initial XRIs for this feature. 2540 * 2541 * @param hw Hardware context allocated by the caller. 2542 * 2543 * @return Returns 0 on success, or a non-zero value on failure. 2544 */ 2545 ocs_hw_rtn_e 2546 ocs_hw_rqpair_init(ocs_hw_t *hw) 2547 { 2548 ocs_hw_rtn_e rc; 2549 uint32_t xris_posted; 2550 2551 ocs_log_debug(hw->os, "RQ Pair mode\n"); 2552 2553 /* 2554 * If we get this far, the auto XFR_RDY feature was enabled successfully, otherwise ocs_hw_init() would 2555 * return with an error. So allocate the buffers based on the initial XRI pool required to support this 2556 * feature. 2557 */ 2558 if (sli_get_auto_xfer_rdy_capable(&hw->sli) && 2559 hw->config.auto_xfer_rdy_size > 0) { 2560 if (hw->auto_xfer_rdy_buf_pool == NULL) { 2561 /* 2562 * Allocate one more buffer than XRIs so that when all the XRIs are in use, we still have 2563 * one to post back for the case where the response phase is started in the context of 2564 * the data completion. 2565 */ 2566 rc = ocs_hw_rqpair_auto_xfer_rdy_buffer_alloc(hw, hw->config.auto_xfer_rdy_xri_cnt + 1); 2567 if (rc != OCS_HW_RTN_SUCCESS) { 2568 return rc; 2569 } 2570 } else { 2571 ocs_pool_reset(hw->auto_xfer_rdy_buf_pool); 2572 } 2573 2574 /* Post the auto XFR_RDY XRIs */ 2575 xris_posted = ocs_hw_xri_move_to_port_owned(hw, hw->config.auto_xfer_rdy_xri_cnt); 2576 if (xris_posted != hw->config.auto_xfer_rdy_xri_cnt) { 2577 ocs_log_err(hw->os, "post_xri failed, only posted %d XRIs\n", xris_posted); 2578 return OCS_HW_RTN_ERROR; 2579 } 2580 } 2581 2582 return 0; 2583 } 2584 2585 /** 2586 * @ingroup devInitShutdown 2587 * @brief Tear down the rq_pair function from ocs_hw_teardown(). 2588 * 2589 * @par Description 2590 * Frees the buffers to auto xfer rdy. 2591 * 2592 * @param hw Hardware context allocated by the caller. 2593 */ 2594 void 2595 ocs_hw_rqpair_teardown(ocs_hw_t *hw) 2596 { 2597 /* We need to free any auto xfer ready buffers */ 2598 ocs_hw_rqpair_auto_xfer_rdy_buffer_free(hw); 2599 } 2600