1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (C) 2013 Emulex 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions are met: 9 * 10 * 1. Redistributions of source code must retain the above copyright notice, 11 * this list of conditions and the following disclaimer. 12 * 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * 3. Neither the name of the Emulex Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived from 19 * this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 * POSSIBILITY OF SUCH DAMAGE. 32 * 33 * Contact Information: 34 * freebsd-drivers@emulex.com 35 * 36 * Emulex 37 * 3333 Susan Street 38 * Costa Mesa, CA 92626 39 */ 40 41 /* $FreeBSD$ */ 42 43 #include "oce_if.h" 44 45 /***************************************************** 46 * local queue functions 47 *****************************************************/ 48 49 static struct oce_wq *oce_wq_init(POCE_SOFTC sc, 50 uint32_t q_len, uint32_t wq_type); 51 static int oce_wq_create(struct oce_wq *wq, struct oce_eq *eq); 52 static void oce_wq_free(struct oce_wq *wq); 53 static void oce_wq_del(struct oce_wq *wq); 54 static struct oce_rq *oce_rq_init(POCE_SOFTC sc, 55 uint32_t q_len, 56 uint32_t frag_size, 57 uint32_t mtu, uint32_t rss); 58 static int oce_rq_create(struct oce_rq *rq, uint32_t if_id, struct oce_eq *eq); 59 static void oce_rq_free(struct oce_rq *rq); 60 static void oce_rq_del(struct oce_rq *rq); 61 static struct oce_eq *oce_eq_create(POCE_SOFTC sc, 62 uint32_t q_len, 63 uint32_t item_size, 64 uint32_t eq_delay, 65 uint32_t vector); 66 static void oce_eq_del(struct oce_eq *eq); 67 static struct oce_mq *oce_mq_create(POCE_SOFTC sc, 68 struct oce_eq *eq, uint32_t q_len); 69 static void oce_mq_free(struct oce_mq *mq); 70 static int oce_destroy_q(POCE_SOFTC sc, struct oce_mbx 71 *mbx, size_t req_size, enum qtype qtype, int version); 72 struct oce_cq *oce_cq_create(POCE_SOFTC sc, 73 struct oce_eq *eq, 74 uint32_t q_len, 75 uint32_t item_size, 76 uint32_t sol_event, 77 uint32_t is_eventable, 78 uint32_t nodelay, uint32_t ncoalesce); 79 static void oce_cq_del(POCE_SOFTC sc, struct oce_cq *cq); 80 81 /** 82 * @brief Create and initialize all the queues on the board 83 * @param sc software handle to the device 84 * @returns 0 if successful, or error 85 **/ 86 int 87 oce_queue_init_all(POCE_SOFTC sc) 88 { 89 int rc = 0, i, vector; 90 struct oce_wq *wq; 91 struct oce_rq *rq; 92 struct oce_aic_obj *aic; 93 94 /* alloc TX/RX queues */ 95 for_all_wq_queues(sc, wq, i) { 96 sc->wq[i] = oce_wq_init(sc, sc->tx_ring_size, 97 NIC_WQ_TYPE_STANDARD); 98 if (!sc->wq[i]) 99 goto error; 100 101 } 102 103 for_all_rq_queues(sc, rq, i) { 104 sc->rq[i] = oce_rq_init(sc, sc->rx_ring_size, sc->rq_frag_size, 105 OCE_MAX_JUMBO_FRAME_SIZE, 106 (i == 0) ? 0 : is_rss_enabled(sc)); 107 if (!sc->rq[i]) 108 goto error; 109 } 110 111 /* Create network interface on card */ 112 if (oce_create_nw_interface(sc)) 113 goto error; 114 115 /* create all of the event queues */ 116 for (vector = 0; vector < sc->intr_count; vector++) { 117 /* setup aic defaults for each event queue */ 118 aic = &sc->aic_obj[vector]; 119 aic->max_eqd = OCE_MAX_EQD; 120 aic->min_eqd = OCE_MIN_EQD; 121 aic->et_eqd = OCE_MIN_EQD; 122 aic->enable = TRUE; 123 124 sc->eq[vector] = oce_eq_create(sc, sc->enable_hwlro ? EQ_LEN_2048 : EQ_LEN_1024, 125 EQE_SIZE_4,0, vector); 126 127 if (!sc->eq[vector]) 128 goto error; 129 } 130 131 /* create Tx, Rx and mcc queues */ 132 for_all_wq_queues(sc, wq, i) { 133 rc = oce_wq_create(wq, sc->eq[i]); 134 if (rc) 135 goto error; 136 wq->queue_index = i; 137 TASK_INIT(&wq->txtask, 1, oce_tx_task, wq); 138 } 139 140 for_all_rq_queues(sc, rq, i) { 141 rc = oce_rq_create(rq, sc->if_id, 142 sc->eq[(i == 0) ? 0:(i-1)]); 143 if (rc) 144 goto error; 145 rq->queue_index = i; 146 } 147 148 sc->mq = oce_mq_create(sc, sc->eq[0], 64); 149 if (!sc->mq) 150 goto error; 151 152 return rc; 153 154 error: 155 oce_queue_release_all(sc); 156 return 1; 157 } 158 159 /** 160 * @brief Releases all mailbox queues created 161 * @param sc software handle to the device 162 */ 163 void 164 oce_queue_release_all(POCE_SOFTC sc) 165 { 166 int i = 0; 167 struct oce_wq *wq; 168 struct oce_rq *rq; 169 struct oce_eq *eq; 170 171 /* before deleting lro queues, we have to disable hwlro */ 172 if(sc->enable_hwlro) 173 oce_mbox_nic_set_iface_lro_config(sc, 0); 174 175 for_all_rq_queues(sc, rq, i) { 176 if (rq) { 177 oce_rq_del(sc->rq[i]); 178 oce_rq_free(sc->rq[i]); 179 } 180 } 181 182 for_all_wq_queues(sc, wq, i) { 183 if (wq) { 184 oce_wq_del(sc->wq[i]); 185 oce_wq_free(sc->wq[i]); 186 } 187 } 188 189 if (sc->mq) 190 oce_mq_free(sc->mq); 191 192 for_all_evnt_queues(sc, eq, i) { 193 if (eq) 194 oce_eq_del(sc->eq[i]); 195 } 196 } 197 198 /** 199 * @brief Function to create a WQ for NIC Tx 200 * @param sc software handle to the device 201 * @param qlen number of entries in the queue 202 * @param wq_type work queue type 203 * @returns the pointer to the WQ created or NULL on failure 204 */ 205 static struct 206 oce_wq *oce_wq_init(POCE_SOFTC sc, uint32_t q_len, uint32_t wq_type) 207 { 208 struct oce_wq *wq; 209 int rc = 0, i; 210 211 /* q_len must be min 256 and max 2k */ 212 if (q_len < 256 || q_len > 2048) { 213 device_printf(sc->dev, 214 "Invalid q length. Must be " 215 "[256, 2000]: 0x%x\n", q_len); 216 return NULL; 217 } 218 219 /* allocate wq */ 220 wq = malloc(sizeof(struct oce_wq), M_DEVBUF, M_NOWAIT | M_ZERO); 221 if (!wq) 222 return NULL; 223 224 /* Set the wq config */ 225 wq->cfg.q_len = q_len; 226 wq->cfg.wq_type = (uint8_t) wq_type; 227 wq->cfg.eqd = OCE_DEFAULT_WQ_EQD; 228 wq->cfg.nbufs = 2 * wq->cfg.q_len; 229 wq->cfg.nhdl = 2 * wq->cfg.q_len; 230 231 wq->parent = (void *)sc; 232 233 rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 234 1, 0, 235 BUS_SPACE_MAXADDR, 236 BUS_SPACE_MAXADDR, 237 NULL, NULL, 238 OCE_MAX_TX_SIZE, 239 OCE_MAX_TX_ELEMENTS, 240 PAGE_SIZE, 0, NULL, NULL, &wq->tag); 241 242 if (rc) 243 goto free_wq; 244 245 for (i = 0; i < OCE_WQ_PACKET_ARRAY_SIZE; i++) { 246 rc = bus_dmamap_create(wq->tag, 0, &wq->pckts[i].map); 247 if (rc) 248 goto free_wq; 249 } 250 251 wq->ring = oce_create_ring_buffer(sc, q_len, NIC_WQE_SIZE); 252 if (!wq->ring) 253 goto free_wq; 254 255 LOCK_CREATE(&wq->tx_lock, "TX_lock"); 256 LOCK_CREATE(&wq->tx_compl_lock, "WQ_HANDLER_LOCK"); 257 258 /* Allocate buf ring for multiqueue*/ 259 wq->br = buf_ring_alloc(4096, M_DEVBUF, 260 M_WAITOK, &wq->tx_lock.mutex); 261 if (!wq->br) 262 goto free_wq; 263 return wq; 264 265 free_wq: 266 device_printf(sc->dev, "Create WQ failed\n"); 267 oce_wq_free(wq); 268 return NULL; 269 } 270 271 /** 272 * @brief Frees the work queue 273 * @param wq pointer to work queue to free 274 */ 275 static void 276 oce_wq_free(struct oce_wq *wq) 277 { 278 POCE_SOFTC sc = (POCE_SOFTC) wq->parent; 279 int i; 280 281 taskqueue_drain(taskqueue_swi, &wq->txtask); 282 283 if (wq->ring != NULL) { 284 oce_destroy_ring_buffer(sc, wq->ring); 285 wq->ring = NULL; 286 } 287 288 for (i = 0; i < OCE_WQ_PACKET_ARRAY_SIZE; i++) { 289 if (wq->pckts[i].map != NULL) { 290 bus_dmamap_unload(wq->tag, wq->pckts[i].map); 291 bus_dmamap_destroy(wq->tag, wq->pckts[i].map); 292 wq->pckts[i].map = NULL; 293 } 294 } 295 296 if (wq->tag != NULL) 297 bus_dma_tag_destroy(wq->tag); 298 if (wq->br != NULL) 299 buf_ring_free(wq->br, M_DEVBUF); 300 301 LOCK_DESTROY(&wq->tx_lock); 302 LOCK_DESTROY(&wq->tx_compl_lock); 303 free(wq, M_DEVBUF); 304 } 305 306 /** 307 * @brief Create a work queue 308 * @param wq pointer to work queue 309 * @param eq pointer to associated event queue 310 */ 311 static int 312 oce_wq_create(struct oce_wq *wq, struct oce_eq *eq) 313 { 314 POCE_SOFTC sc = wq->parent; 315 struct oce_cq *cq; 316 int rc = 0; 317 318 /* create the CQ */ 319 cq = oce_cq_create(sc, 320 eq, 321 CQ_LEN_1024, 322 sizeof(struct oce_nic_tx_cqe), 0, 1, 0, 3); 323 if (!cq) 324 return ENXIO; 325 326 wq->cq = cq; 327 328 rc = oce_mbox_create_wq(wq); 329 if (rc) 330 goto error; 331 332 wq->qstate = QCREATED; 333 wq->wq_free = wq->cfg.q_len; 334 wq->ring->cidx = 0; 335 wq->ring->pidx = 0; 336 337 eq->cq[eq->cq_valid] = cq; 338 eq->cq_valid++; 339 cq->cb_arg = wq; 340 cq->cq_handler = oce_wq_handler; 341 342 return 0; 343 344 error: 345 device_printf(sc->dev, "WQ create failed\n"); 346 oce_wq_del(wq); 347 return rc; 348 } 349 350 /** 351 * @brief Delete a work queue 352 * @param wq pointer to work queue 353 */ 354 static void 355 oce_wq_del(struct oce_wq *wq) 356 { 357 struct oce_mbx mbx; 358 struct mbx_delete_nic_wq *fwcmd; 359 POCE_SOFTC sc = (POCE_SOFTC) wq->parent; 360 361 if (wq->qstate == QCREATED) { 362 bzero(&mbx, sizeof(struct oce_mbx)); 363 /* now fill the command */ 364 fwcmd = (struct mbx_delete_nic_wq *)&mbx.payload; 365 fwcmd->params.req.wq_id = wq->wq_id; 366 (void)oce_destroy_q(sc, &mbx, 367 sizeof(struct mbx_delete_nic_wq), QTYPE_WQ, 0); 368 wq->qstate = QDELETED; 369 } 370 371 if (wq->cq != NULL) { 372 oce_cq_del(sc, wq->cq); 373 wq->cq = NULL; 374 } 375 } 376 377 /** 378 * @brief function to allocate receive queue resources 379 * @param sc software handle to the device 380 * @param q_len length of receive queue 381 * @param frag_size size of an receive queue fragment 382 * @param mtu maximum transmission unit 383 * @param rss is-rss-queue flag 384 * @returns the pointer to the RQ created or NULL on failure 385 */ 386 static struct 387 oce_rq *oce_rq_init(POCE_SOFTC sc, 388 uint32_t q_len, 389 uint32_t frag_size, 390 uint32_t mtu, uint32_t rss) 391 { 392 struct oce_rq *rq; 393 int rc = 0, i; 394 395 if (OCE_LOG2(frag_size) <= 0) 396 return NULL; 397 398 if ((q_len == 0) || (q_len > 1024)) 399 return NULL; 400 401 /* allocate the rq */ 402 rq = malloc(sizeof(struct oce_rq), M_DEVBUF, M_NOWAIT | M_ZERO); 403 if (!rq) 404 return NULL; 405 406 rq->cfg.q_len = q_len; 407 rq->cfg.frag_size = frag_size; 408 rq->cfg.mtu = mtu; 409 rq->cfg.eqd = 0; 410 rq->lro_pkts_queued = 0; 411 rq->cfg.is_rss_queue = rss; 412 rq->pending = 0; 413 414 rq->parent = (void *)sc; 415 416 rc = bus_dma_tag_create(bus_get_dma_tag(sc->dev), 417 1, 0, 418 BUS_SPACE_MAXADDR, 419 BUS_SPACE_MAXADDR, 420 NULL, NULL, 421 oce_rq_buf_size, 422 1, oce_rq_buf_size, 0, NULL, NULL, &rq->tag); 423 if (rc) 424 goto free_rq; 425 426 for (i = 0; i < OCE_RQ_PACKET_ARRAY_SIZE; i++) { 427 rc = bus_dmamap_create(rq->tag, 0, &rq->pckts[i].map); 428 if (rc) 429 goto free_rq; 430 } 431 432 /* create the ring buffer */ 433 rq->ring = oce_create_ring_buffer(sc, q_len, 434 sizeof(struct oce_nic_rqe)); 435 if (!rq->ring) 436 goto free_rq; 437 438 LOCK_CREATE(&rq->rx_lock, "RX_lock"); 439 440 return rq; 441 442 free_rq: 443 device_printf(sc->dev, "Create RQ failed\n"); 444 oce_rq_free(rq); 445 return NULL; 446 } 447 448 /** 449 * @brief Free a receive queue 450 * @param rq pointer to receive queue 451 */ 452 static void 453 oce_rq_free(struct oce_rq *rq) 454 { 455 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 456 int i = 0 ; 457 458 if (rq->ring != NULL) { 459 oce_destroy_ring_buffer(sc, rq->ring); 460 rq->ring = NULL; 461 } 462 for (i = 0; i < OCE_RQ_PACKET_ARRAY_SIZE; i++) { 463 if (rq->pckts[i].map != NULL) { 464 bus_dmamap_unload(rq->tag, rq->pckts[i].map); 465 bus_dmamap_destroy(rq->tag, rq->pckts[i].map); 466 rq->pckts[i].map = NULL; 467 } 468 if (rq->pckts[i].mbuf) { 469 m_free(rq->pckts[i].mbuf); 470 rq->pckts[i].mbuf = NULL; 471 } 472 } 473 474 if (rq->tag != NULL) 475 bus_dma_tag_destroy(rq->tag); 476 477 LOCK_DESTROY(&rq->rx_lock); 478 free(rq, M_DEVBUF); 479 } 480 481 /** 482 * @brief Create a receive queue 483 * @param rq receive queue 484 * @param if_id interface identifier index` 485 * @param eq pointer to event queue 486 */ 487 static int 488 oce_rq_create(struct oce_rq *rq, uint32_t if_id, struct oce_eq *eq) 489 { 490 POCE_SOFTC sc = rq->parent; 491 struct oce_cq *cq; 492 493 cq = oce_cq_create(sc, eq, 494 sc->enable_hwlro ? CQ_LEN_2048 : CQ_LEN_1024, 495 sizeof(struct oce_nic_rx_cqe), 0, 1, 0, 3); 496 497 if (!cq) 498 return ENXIO; 499 500 rq->cq = cq; 501 rq->cfg.if_id = if_id; 502 503 /* Dont create RQ here. Create in if_activate */ 504 rq->qstate = 0; 505 rq->ring->cidx = 0; 506 rq->ring->pidx = 0; 507 eq->cq[eq->cq_valid] = cq; 508 eq->cq_valid++; 509 cq->cb_arg = rq; 510 cq->cq_handler = oce_rq_handler; 511 512 return 0; 513 514 } 515 516 /** 517 * @brief Delete a receive queue 518 * @param rq receive queue 519 */ 520 static void 521 oce_rq_del(struct oce_rq *rq) 522 { 523 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 524 struct oce_mbx mbx; 525 struct mbx_delete_nic_rq *fwcmd; 526 struct mbx_delete_nic_rq_v1 *fwcmd1; 527 528 if (rq->qstate == QCREATED) { 529 bzero(&mbx, sizeof(mbx)); 530 if(!rq->islro) { 531 fwcmd = (struct mbx_delete_nic_rq *)&mbx.payload; 532 fwcmd->params.req.rq_id = rq->rq_id; 533 (void)oce_destroy_q(sc, &mbx, sizeof(struct mbx_delete_nic_rq), QTYPE_RQ, 0); 534 }else { 535 fwcmd1 = (struct mbx_delete_nic_rq_v1 *)&mbx.payload; 536 fwcmd1->params.req.rq_id = rq->rq_id; 537 fwcmd1->params.req.rq_flags = (NIC_RQ_FLAGS_RSS | NIC_RQ_FLAGS_LRO); 538 (void)oce_destroy_q(sc, &mbx, sizeof(struct mbx_delete_nic_rq_v1), QTYPE_RQ, 1); 539 } 540 rq->qstate = QDELETED; 541 } 542 543 if (rq->cq != NULL) { 544 oce_cq_del(sc, rq->cq); 545 rq->cq = NULL; 546 } 547 } 548 549 /** 550 * @brief function to create an event queue 551 * @param sc software handle to the device 552 * @param q_len length of event queue 553 * @param item_size size of an event queue item 554 * @param eq_delay event queue delay 555 * @retval eq success, pointer to event queue 556 * @retval NULL failure 557 */ 558 static struct 559 oce_eq *oce_eq_create(POCE_SOFTC sc, uint32_t q_len, 560 uint32_t item_size, 561 uint32_t eq_delay, 562 uint32_t vector) 563 { 564 struct oce_eq *eq; 565 int rc = 0; 566 567 /* allocate an eq */ 568 eq = malloc(sizeof(struct oce_eq), M_DEVBUF, M_NOWAIT | M_ZERO); 569 if (eq == NULL) 570 return NULL; 571 572 eq->parent = (void *)sc; 573 eq->eq_id = 0xffff; 574 eq->ring = oce_create_ring_buffer(sc, q_len, item_size); 575 if (!eq->ring) 576 goto free_eq; 577 578 eq->eq_cfg.q_len = q_len; 579 eq->eq_cfg.item_size = item_size; 580 eq->eq_cfg.cur_eqd = (uint8_t) eq_delay; 581 582 rc = oce_mbox_create_eq(eq); 583 if (rc) 584 goto free_eq; 585 586 sc->intrs[sc->neqs++].eq = eq; 587 588 return eq; 589 590 free_eq: 591 oce_eq_del(eq); 592 return NULL; 593 } 594 595 /** 596 * @brief Function to delete an event queue 597 * @param eq pointer to an event queue 598 */ 599 static void 600 oce_eq_del(struct oce_eq *eq) 601 { 602 struct oce_mbx mbx; 603 struct mbx_destroy_common_eq *fwcmd; 604 POCE_SOFTC sc = (POCE_SOFTC) eq->parent; 605 606 if (eq->eq_id != 0xffff) { 607 bzero(&mbx, sizeof(mbx)); 608 fwcmd = (struct mbx_destroy_common_eq *)&mbx.payload; 609 fwcmd->params.req.id = eq->eq_id; 610 (void)oce_destroy_q(sc, &mbx, 611 sizeof(struct mbx_destroy_common_eq), QTYPE_EQ, 0); 612 } 613 614 if (eq->ring != NULL) { 615 oce_destroy_ring_buffer(sc, eq->ring); 616 eq->ring = NULL; 617 } 618 619 free(eq, M_DEVBUF); 620 621 } 622 623 /** 624 * @brief Function to create an MQ 625 * @param sc software handle to the device 626 * @param eq the EQ to associate with the MQ for event notification 627 * @param q_len the number of entries to create in the MQ 628 * @returns pointer to the created MQ, failure otherwise 629 */ 630 static struct oce_mq * 631 oce_mq_create(POCE_SOFTC sc, struct oce_eq *eq, uint32_t q_len) 632 { 633 struct oce_mbx mbx; 634 struct mbx_create_common_mq_ex *fwcmd = NULL; 635 struct oce_mq *mq = NULL; 636 int rc = 0; 637 struct oce_cq *cq; 638 oce_mq_ext_ctx_t *ctx; 639 uint32_t num_pages; 640 int version; 641 642 cq = oce_cq_create(sc, eq, CQ_LEN_256, 643 sizeof(struct oce_mq_cqe), 1, 1, 0, 0); 644 if (!cq) 645 return NULL; 646 647 /* allocate the mq */ 648 mq = malloc(sizeof(struct oce_mq), M_DEVBUF, M_NOWAIT | M_ZERO); 649 if (!mq) { 650 oce_cq_del(sc, cq); 651 goto error; 652 } 653 654 mq->parent = sc; 655 656 mq->ring = oce_create_ring_buffer(sc, q_len, sizeof(struct oce_mbx)); 657 if (!mq->ring) 658 goto error; 659 660 bzero(&mbx, sizeof(struct oce_mbx)); 661 662 IS_XE201(sc) ? (version = OCE_MBX_VER_V1) : (version = OCE_MBX_VER_V0); 663 fwcmd = (struct mbx_create_common_mq_ex *)&mbx.payload; 664 mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0, 665 MBX_SUBSYSTEM_COMMON, 666 OPCODE_COMMON_CREATE_MQ_EXT, 667 MBX_TIMEOUT_SEC, 668 sizeof(struct mbx_create_common_mq_ex), 669 version); 670 671 num_pages = oce_page_list(mq->ring, &fwcmd->params.req.pages[0]); 672 673 ctx = &fwcmd->params.req.context; 674 675 if (IS_XE201(sc)) { 676 ctx->v1.num_pages = num_pages; 677 ctx->v1.ring_size = OCE_LOG2(q_len) + 1; 678 ctx->v1.cq_id = cq->cq_id; 679 ctx->v1.valid = 1; 680 ctx->v1.async_cq_id = cq->cq_id; 681 ctx->v1.async_cq_valid = 1; 682 /* Subscribe to Link State and Group 5 Events(bits 1 & 5 set) */ 683 ctx->v1.async_evt_bitmap |= LE_32(0x00000022); 684 ctx->v1.async_evt_bitmap |= LE_32(1 << ASYNC_EVENT_CODE_DEBUG); 685 ctx->v1.async_evt_bitmap |= 686 LE_32(1 << ASYNC_EVENT_CODE_SLIPORT); 687 } 688 else { 689 ctx->v0.num_pages = num_pages; 690 ctx->v0.cq_id = cq->cq_id; 691 ctx->v0.ring_size = OCE_LOG2(q_len) + 1; 692 ctx->v0.valid = 1; 693 /* Subscribe to Link State and Group5 Events(bits 1 & 5 set) */ 694 ctx->v0.async_evt_bitmap = 0xffffffff; 695 } 696 697 mbx.u0.s.embedded = 1; 698 mbx.payload_length = sizeof(struct mbx_create_common_mq_ex); 699 DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ); 700 701 rc = oce_mbox_post(sc, &mbx, NULL); 702 if (!rc) 703 rc = fwcmd->hdr.u0.rsp.status; 704 if (rc) { 705 device_printf(sc->dev,"%s failed - cmd status: %d\n", 706 __FUNCTION__, rc); 707 goto error; 708 } 709 mq->mq_id = LE_16(fwcmd->params.rsp.mq_id); 710 mq->cq = cq; 711 eq->cq[eq->cq_valid] = cq; 712 eq->cq_valid++; 713 mq->cq->eq = eq; 714 mq->cfg.q_len = (uint8_t) q_len; 715 mq->cfg.eqd = 0; 716 mq->qstate = QCREATED; 717 718 mq->cq->cb_arg = mq; 719 mq->cq->cq_handler = oce_mq_handler; 720 721 return mq; 722 723 error: 724 device_printf(sc->dev, "MQ create failed\n"); 725 oce_mq_free(mq); 726 mq = NULL; 727 return mq; 728 } 729 730 /** 731 * @brief Function to free a mailbox queue 732 * @param mq pointer to a mailbox queue 733 */ 734 static void 735 oce_mq_free(struct oce_mq *mq) 736 { 737 POCE_SOFTC sc = (POCE_SOFTC) mq->parent; 738 struct oce_mbx mbx; 739 struct mbx_destroy_common_mq *fwcmd; 740 741 if (!mq) 742 return; 743 744 if (mq->ring != NULL) { 745 oce_destroy_ring_buffer(sc, mq->ring); 746 mq->ring = NULL; 747 if (mq->qstate == QCREATED) { 748 bzero(&mbx, sizeof (struct oce_mbx)); 749 fwcmd = (struct mbx_destroy_common_mq *)&mbx.payload; 750 fwcmd->params.req.id = mq->mq_id; 751 (void) oce_destroy_q(sc, &mbx, 752 sizeof (struct mbx_destroy_common_mq), 753 QTYPE_MQ, 0); 754 } 755 mq->qstate = QDELETED; 756 } 757 758 if (mq->cq != NULL) { 759 oce_cq_del(sc, mq->cq); 760 mq->cq = NULL; 761 } 762 763 free(mq, M_DEVBUF); 764 mq = NULL; 765 } 766 767 /** 768 * @brief Function to delete a EQ, CQ, MQ, WQ or RQ 769 * @param sc sofware handle to the device 770 * @param mbx mailbox command to send to the fw to delete the queue 771 * (mbx contains the queue information to delete) 772 * @param req_size the size of the mbx payload dependent on the qtype 773 * @param qtype the type of queue i.e. EQ, CQ, MQ, WQ or RQ 774 * @returns 0 on success, failure otherwise 775 */ 776 static int 777 oce_destroy_q(POCE_SOFTC sc, struct oce_mbx *mbx, size_t req_size, 778 enum qtype qtype, int version) 779 { 780 struct mbx_hdr *hdr = (struct mbx_hdr *)&mbx->payload; 781 int opcode; 782 int subsys; 783 int rc = 0; 784 785 switch (qtype) { 786 case QTYPE_EQ: 787 opcode = OPCODE_COMMON_DESTROY_EQ; 788 subsys = MBX_SUBSYSTEM_COMMON; 789 break; 790 case QTYPE_CQ: 791 opcode = OPCODE_COMMON_DESTROY_CQ; 792 subsys = MBX_SUBSYSTEM_COMMON; 793 break; 794 case QTYPE_MQ: 795 opcode = OPCODE_COMMON_DESTROY_MQ; 796 subsys = MBX_SUBSYSTEM_COMMON; 797 break; 798 case QTYPE_WQ: 799 opcode = NIC_DELETE_WQ; 800 subsys = MBX_SUBSYSTEM_NIC; 801 break; 802 case QTYPE_RQ: 803 opcode = NIC_DELETE_RQ; 804 subsys = MBX_SUBSYSTEM_NIC; 805 break; 806 default: 807 return EINVAL; 808 } 809 810 mbx_common_req_hdr_init(hdr, 0, 0, subsys, 811 opcode, MBX_TIMEOUT_SEC, req_size, 812 version); 813 814 mbx->u0.s.embedded = 1; 815 mbx->payload_length = (uint32_t) req_size; 816 DW_SWAP(u32ptr(mbx), mbx->payload_length + OCE_BMBX_RHDR_SZ); 817 818 rc = oce_mbox_post(sc, mbx, NULL); 819 if (!rc) 820 rc = hdr->u0.rsp.status; 821 if (rc) 822 device_printf(sc->dev,"%s failed - cmd status: %d\n", 823 __FUNCTION__, rc); 824 return rc; 825 } 826 827 /** 828 * @brief Function to create a completion queue 829 * @param sc software handle to the device 830 * @param eq optional eq to be associated with to the cq 831 * @param q_len length of completion queue 832 * @param item_size size of completion queue items 833 * @param sol_event command context event 834 * @param is_eventable event table 835 * @param nodelay no delay flag 836 * @param ncoalesce no coalescence flag 837 * @returns pointer to the cq created, NULL on failure 838 */ 839 struct oce_cq * 840 oce_cq_create(POCE_SOFTC sc, struct oce_eq *eq, 841 uint32_t q_len, 842 uint32_t item_size, 843 uint32_t sol_event, 844 uint32_t is_eventable, 845 uint32_t nodelay, uint32_t ncoalesce) 846 { 847 struct oce_cq *cq = NULL; 848 int rc = 0; 849 850 cq = malloc(sizeof(struct oce_cq), M_DEVBUF, M_NOWAIT | M_ZERO); 851 if (!cq) 852 return NULL; 853 854 cq->ring = oce_create_ring_buffer(sc, q_len, item_size); 855 if (!cq->ring) 856 goto error; 857 858 cq->parent = sc; 859 cq->eq = eq; 860 cq->cq_cfg.q_len = q_len; 861 cq->cq_cfg.item_size = item_size; 862 cq->cq_cfg.nodelay = (uint8_t) nodelay; 863 864 rc = oce_mbox_cq_create(cq, ncoalesce, is_eventable); 865 if (rc) 866 goto error; 867 868 sc->cq[sc->ncqs++] = cq; 869 870 return cq; 871 872 error: 873 device_printf(sc->dev, "CQ create failed\n"); 874 oce_cq_del(sc, cq); 875 return NULL; 876 } 877 878 /** 879 * @brief Deletes the completion queue 880 * @param sc software handle to the device 881 * @param cq pointer to a completion queue 882 */ 883 static void 884 oce_cq_del(POCE_SOFTC sc, struct oce_cq *cq) 885 { 886 struct oce_mbx mbx; 887 struct mbx_destroy_common_cq *fwcmd; 888 889 if (cq->ring != NULL) { 890 bzero(&mbx, sizeof(struct oce_mbx)); 891 /* now fill the command */ 892 fwcmd = (struct mbx_destroy_common_cq *)&mbx.payload; 893 fwcmd->params.req.id = cq->cq_id; 894 (void)oce_destroy_q(sc, &mbx, 895 sizeof(struct mbx_destroy_common_cq), QTYPE_CQ, 0); 896 /*NOW destroy the ring */ 897 oce_destroy_ring_buffer(sc, cq->ring); 898 cq->ring = NULL; 899 } 900 901 free(cq, M_DEVBUF); 902 cq = NULL; 903 } 904 905 /** 906 * @brief Start a receive queue 907 * @param rq pointer to a receive queue 908 */ 909 int 910 oce_start_rq(struct oce_rq *rq) 911 { 912 POCE_SOFTC sc = (POCE_SOFTC) rq->parent; 913 int rc; 914 915 if(sc->enable_hwlro) 916 rc = oce_alloc_rx_bufs(rq, 960); 917 else 918 rc = oce_alloc_rx_bufs(rq, rq->cfg.q_len - 1); 919 920 if (rc == 0) 921 oce_arm_cq(rq->parent, rq->cq->cq_id, 0, TRUE); 922 923 return rc; 924 } 925 926 /** 927 * @brief Start a work queue 928 * @param wq pointer to a work queue 929 */ 930 int 931 oce_start_wq(struct oce_wq *wq) 932 { 933 oce_arm_cq(wq->parent, wq->cq->cq_id, 0, TRUE); 934 return 0; 935 } 936 937 /** 938 * @brief Start a mailbox queue 939 * @param mq pointer to a mailbox queue 940 */ 941 int 942 oce_start_mq(struct oce_mq *mq) 943 { 944 oce_arm_cq(mq->parent, mq->cq->cq_id, 0, TRUE); 945 return 0; 946 } 947 948 /** 949 * @brief Function to arm an EQ so that it can generate events 950 * @param sc software handle to the device 951 * @param qid id of the EQ returned by the fw at the time of creation 952 * @param npopped number of EQEs to arm 953 * @param rearm rearm bit enable/disable 954 * @param clearint bit to clear the interrupt condition because of which 955 * EQEs are generated 956 */ 957 void 958 oce_arm_eq(POCE_SOFTC sc, 959 int16_t qid, int npopped, uint32_t rearm, uint32_t clearint) 960 { 961 eq_db_t eq_db = { 0 }; 962 963 eq_db.bits.rearm = rearm; 964 eq_db.bits.event = 1; 965 eq_db.bits.num_popped = npopped; 966 eq_db.bits.clrint = clearint; 967 eq_db.bits.qid = qid; 968 OCE_WRITE_REG32(sc, db, PD_EQ_DB, eq_db.dw0); 969 970 } 971 972 /** 973 * @brief Function to arm a CQ with CQEs 974 * @param sc software handle to the device 975 * @param qid id of the CQ returned by the fw at the time of creation 976 * @param npopped number of CQEs to arm 977 * @param rearm rearm bit enable/disable 978 */ 979 void oce_arm_cq(POCE_SOFTC sc, int16_t qid, int npopped, uint32_t rearm) 980 { 981 cq_db_t cq_db = { 0 }; 982 983 cq_db.bits.rearm = rearm; 984 cq_db.bits.num_popped = npopped; 985 cq_db.bits.event = 0; 986 cq_db.bits.qid = qid; 987 OCE_WRITE_REG32(sc, db, PD_CQ_DB, cq_db.dw0); 988 989 } 990 991 /* 992 * @brief function to cleanup the eqs used during stop 993 * @param eq pointer to event queue structure 994 * @returns the number of EQs processed 995 */ 996 void 997 oce_drain_eq(struct oce_eq *eq) 998 { 999 1000 struct oce_eqe *eqe; 1001 uint16_t num_eqe = 0; 1002 POCE_SOFTC sc = eq->parent; 1003 1004 do { 1005 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe); 1006 if (eqe->evnt == 0) 1007 break; 1008 eqe->evnt = 0; 1009 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map, 1010 BUS_DMASYNC_POSTWRITE); 1011 num_eqe++; 1012 RING_GET(eq->ring, 1); 1013 1014 } while (TRUE); 1015 1016 oce_arm_eq(sc, eq->eq_id, num_eqe, FALSE, TRUE); 1017 1018 } 1019 1020 void 1021 oce_drain_wq_cq(struct oce_wq *wq) 1022 { 1023 POCE_SOFTC sc = wq->parent; 1024 struct oce_cq *cq = wq->cq; 1025 struct oce_nic_tx_cqe *cqe; 1026 int num_cqes = 0; 1027 1028 bus_dmamap_sync(cq->ring->dma.tag, cq->ring->dma.map, 1029 BUS_DMASYNC_POSTWRITE); 1030 1031 do { 1032 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe); 1033 if (cqe->u0.dw[3] == 0) 1034 break; 1035 cqe->u0.dw[3] = 0; 1036 bus_dmamap_sync(cq->ring->dma.tag, cq->ring->dma.map, 1037 BUS_DMASYNC_POSTWRITE); 1038 RING_GET(cq->ring, 1); 1039 num_cqes++; 1040 1041 } while (TRUE); 1042 1043 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE); 1044 1045 } 1046 1047 /* 1048 * @brief function to drain a MCQ and process its CQEs 1049 * @param dev software handle to the device 1050 * @param cq pointer to the cq to drain 1051 * @returns the number of CQEs processed 1052 */ 1053 void 1054 oce_drain_mq_cq(void *arg) 1055 { 1056 /* TODO: additional code. */ 1057 return; 1058 } 1059 1060 /** 1061 * @brief function to process a Recieve queue 1062 * @param arg pointer to the RQ to charge 1063 * @return number of cqes processed 1064 */ 1065 void 1066 oce_drain_rq_cq(struct oce_rq *rq) 1067 { 1068 struct oce_nic_rx_cqe *cqe; 1069 uint16_t num_cqe = 0; 1070 struct oce_cq *cq; 1071 POCE_SOFTC sc; 1072 1073 sc = rq->parent; 1074 cq = rq->cq; 1075 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 1076 /* dequeue till you reach an invalid cqe */ 1077 while (RQ_CQE_VALID(cqe)) { 1078 RQ_CQE_INVALIDATE(cqe); 1079 RING_GET(cq->ring, 1); 1080 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, 1081 struct oce_nic_rx_cqe); 1082 num_cqe++; 1083 } 1084 oce_arm_cq(sc, cq->cq_id, num_cqe, FALSE); 1085 1086 return; 1087 } 1088 1089 void 1090 oce_free_posted_rxbuf(struct oce_rq *rq) 1091 { 1092 struct oce_packet_desc *pd; 1093 1094 while (rq->pending) { 1095 pd = &rq->pckts[rq->ring->cidx]; 1096 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE); 1097 bus_dmamap_unload(rq->tag, pd->map); 1098 if (pd->mbuf != NULL) { 1099 m_freem(pd->mbuf); 1100 pd->mbuf = NULL; 1101 } 1102 1103 RING_GET(rq->ring,1); 1104 rq->pending--; 1105 } 1106 1107 } 1108 1109 void 1110 oce_rx_cq_clean_hwlro(struct oce_rq *rq) 1111 { 1112 struct oce_cq *cq = rq->cq; 1113 POCE_SOFTC sc = rq->parent; 1114 struct nic_hwlro_singleton_cqe *cqe; 1115 struct nic_hwlro_cqe_part2 *cqe2; 1116 int flush_wait = 0; 1117 int flush_compl = 0; 1118 int num_frags = 0; 1119 1120 for (;;) { 1121 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1122 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe); 1123 if(cqe->valid) { 1124 if(cqe->cqe_type == 0) { /* singleton cqe */ 1125 /* we should not get singleton cqe after cqe1 on same rq */ 1126 if(rq->cqe_firstpart != NULL) { 1127 device_printf(sc->dev, "Got singleton cqe after cqe1 \n"); 1128 goto exit_rx_cq_clean_hwlro; 1129 } 1130 num_frags = cqe->pkt_size / rq->cfg.frag_size; 1131 if(cqe->pkt_size % rq->cfg.frag_size) 1132 num_frags++; 1133 oce_discard_rx_comp(rq, num_frags); 1134 /* Check if CQE is flush completion */ 1135 if(!cqe->pkt_size) 1136 flush_compl = 1; 1137 cqe->valid = 0; 1138 RING_GET(cq->ring, 1); 1139 }else if(cqe->cqe_type == 0x1) { /* first part */ 1140 /* we should not get cqe1 after cqe1 on same rq */ 1141 if(rq->cqe_firstpart != NULL) { 1142 device_printf(sc->dev, "Got cqe1 after cqe1 \n"); 1143 goto exit_rx_cq_clean_hwlro; 1144 } 1145 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe; 1146 RING_GET(cq->ring, 1); 1147 }else if(cqe->cqe_type == 0x2) { /* second part */ 1148 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe; 1149 /* We should not get cqe2 without cqe1 */ 1150 if(rq->cqe_firstpart == NULL) { 1151 device_printf(sc->dev, "Got cqe2 without cqe1 \n"); 1152 goto exit_rx_cq_clean_hwlro; 1153 } 1154 num_frags = cqe2->coalesced_size / rq->cfg.frag_size; 1155 if(cqe2->coalesced_size % rq->cfg.frag_size) 1156 num_frags++; 1157 1158 /* Flush completion will always come in singleton CQE */ 1159 oce_discard_rx_comp(rq, num_frags); 1160 1161 rq->cqe_firstpart->valid = 0; 1162 cqe2->valid = 0; 1163 rq->cqe_firstpart = NULL; 1164 RING_GET(cq->ring, 1); 1165 } 1166 oce_arm_cq(sc, cq->cq_id, 1, FALSE); 1167 if(flush_compl) 1168 break; 1169 }else { 1170 if (flush_wait++ > 100) { 1171 device_printf(sc->dev, "did not receive hwlro flush compl\n"); 1172 break; 1173 } 1174 oce_arm_cq(sc, cq->cq_id, 0, TRUE); 1175 DELAY(1000); 1176 } 1177 } 1178 1179 /* After cleanup, leave the CQ in unarmed state */ 1180 oce_arm_cq(sc, cq->cq_id, 0, FALSE); 1181 1182 exit_rx_cq_clean_hwlro: 1183 return; 1184 } 1185 1186 void 1187 oce_rx_cq_clean(struct oce_rq *rq) 1188 { 1189 struct oce_nic_rx_cqe *cqe; 1190 struct oce_cq *cq; 1191 POCE_SOFTC sc; 1192 int flush_wait = 0; 1193 int flush_compl = 0; 1194 sc = rq->parent; 1195 cq = rq->cq; 1196 1197 for (;;) { 1198 bus_dmamap_sync(cq->ring->dma.tag, 1199 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE); 1200 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe); 1201 if(RQ_CQE_VALID(cqe)) { 1202 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe)); 1203 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments); 1204 /* Check if CQE is flush completion */ 1205 if((cqe->u0.s.num_fragments==0)&&(cqe->u0.s.pkt_size == 0)&&(cqe->u0.s.error == 0)) 1206 flush_compl = 1; 1207 1208 RQ_CQE_INVALIDATE(cqe); 1209 RING_GET(cq->ring, 1); 1210 #if defined(INET6) || defined(INET) 1211 if (IF_LRO_ENABLED(sc)) 1212 oce_rx_flush_lro(rq); 1213 #endif 1214 oce_arm_cq(sc, cq->cq_id, 1, FALSE); 1215 if(flush_compl) 1216 break; 1217 }else { 1218 if (flush_wait++ > 100) { 1219 device_printf(sc->dev, "did not receive flush compl\n"); 1220 break; 1221 } 1222 oce_arm_cq(sc, cq->cq_id, 0, TRUE); 1223 DELAY(1000); 1224 } 1225 } 1226 1227 /* After cleanup, leave the CQ in unarmed state */ 1228 oce_arm_cq(sc, cq->cq_id, 0, FALSE); 1229 } 1230 1231 void 1232 oce_stop_rx(POCE_SOFTC sc) 1233 { 1234 struct epoch_tracker et; 1235 struct oce_mbx mbx; 1236 struct mbx_delete_nic_rq *fwcmd; 1237 struct mbx_delete_nic_rq_v1 *fwcmd1; 1238 struct oce_rq *rq; 1239 int i = 0; 1240 1241 NET_EPOCH_ENTER(et); 1242 /* before deleting disable hwlro */ 1243 if(sc->enable_hwlro) 1244 oce_mbox_nic_set_iface_lro_config(sc, 0); 1245 1246 for_all_rq_queues(sc, rq, i) { 1247 if (rq->qstate == QCREATED) { 1248 /* Delete rxq in firmware */ 1249 LOCK(&rq->rx_lock); 1250 1251 bzero(&mbx, sizeof(mbx)); 1252 if(!rq->islro) { 1253 fwcmd = (struct mbx_delete_nic_rq *)&mbx.payload; 1254 fwcmd->params.req.rq_id = rq->rq_id; 1255 (void)oce_destroy_q(sc, &mbx, sizeof(struct mbx_delete_nic_rq), QTYPE_RQ, 0); 1256 }else { 1257 fwcmd1 = (struct mbx_delete_nic_rq_v1 *)&mbx.payload; 1258 fwcmd1->params.req.rq_id = rq->rq_id; 1259 fwcmd1->params.req.rq_flags = (NIC_RQ_FLAGS_RSS | NIC_RQ_FLAGS_LRO); 1260 1261 (void)oce_destroy_q(sc,&mbx,sizeof(struct mbx_delete_nic_rq_v1),QTYPE_RQ,1); 1262 } 1263 rq->qstate = QDELETED; 1264 1265 DELAY(1000); 1266 1267 if(!rq->islro) 1268 oce_rx_cq_clean(rq); 1269 else 1270 oce_rx_cq_clean_hwlro(rq); 1271 1272 /* Free posted RX buffers that are not used */ 1273 oce_free_posted_rxbuf(rq); 1274 UNLOCK(&rq->rx_lock); 1275 } 1276 } 1277 NET_EPOCH_EXIT(et); 1278 } 1279 1280 int 1281 oce_start_rx(POCE_SOFTC sc) 1282 { 1283 struct oce_rq *rq; 1284 int rc = 0, i; 1285 1286 for_all_rq_queues(sc, rq, i) { 1287 if (rq->qstate == QCREATED) 1288 continue; 1289 if((i == 0) || (!sc->enable_hwlro)) { 1290 rc = oce_mbox_create_rq(rq); 1291 if (rc) 1292 goto error; 1293 rq->islro = 0; 1294 }else { 1295 rc = oce_mbox_create_rq_v2(rq); 1296 if (rc) 1297 goto error; 1298 rq->islro = 1; 1299 } 1300 /* reset queue pointers */ 1301 rq->qstate = QCREATED; 1302 rq->pending = 0; 1303 rq->ring->cidx = 0; 1304 rq->ring->pidx = 0; 1305 } 1306 1307 if(sc->enable_hwlro) { 1308 rc = oce_mbox_nic_set_iface_lro_config(sc, 1); 1309 if (rc) 1310 goto error; 1311 } 1312 1313 DELAY(1); 1314 1315 /* RSS config */ 1316 if (is_rss_enabled(sc)) { 1317 rc = oce_config_nic_rss(sc, (uint8_t) sc->if_id, RSS_ENABLE); 1318 if (rc) 1319 goto error; 1320 } 1321 1322 DELAY(1); 1323 return rc; 1324 error: 1325 device_printf(sc->dev, "Start RX failed\n"); 1326 return rc; 1327 1328 } 1329