1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * sun4v LDC Link Layer 31 */ 32 #include <sys/types.h> 33 #include <sys/file.h> 34 #include <sys/errno.h> 35 #include <sys/open.h> 36 #include <sys/cred.h> 37 #include <sys/kmem.h> 38 #include <sys/conf.h> 39 #include <sys/cmn_err.h> 40 #include <sys/ksynch.h> 41 #include <sys/modctl.h> 42 #include <sys/stat.h> /* needed for S_IFBLK and S_IFCHR */ 43 #include <sys/debug.h> 44 #include <sys/types.h> 45 #include <sys/cred.h> 46 #include <sys/promif.h> 47 #include <sys/ddi.h> 48 #include <sys/sunddi.h> 49 #include <sys/cyclic.h> 50 #include <sys/machsystm.h> 51 #include <sys/vm.h> 52 #include <sys/cpu.h> 53 #include <sys/intreg.h> 54 #include <sys/machcpuvar.h> 55 #include <sys/mmu.h> 56 #include <sys/pte.h> 57 #include <vm/hat.h> 58 #include <vm/as.h> 59 #include <vm/hat_sfmmu.h> 60 #include <sys/vm_machparam.h> 61 #include <vm/seg_kmem.h> 62 #include <vm/seg_kpm.h> 63 #include <sys/note.h> 64 #include <sys/ivintr.h> 65 #include <sys/hypervisor_api.h> 66 #include <sys/ldc.h> 67 #include <sys/ldc_impl.h> 68 #include <sys/cnex.h> 69 #include <sys/hsvc.h> 70 71 /* Core internal functions */ 72 static int i_ldc_h2v_error(int h_error); 73 static int i_ldc_txq_reconf(ldc_chan_t *ldcp); 74 static int i_ldc_rxq_reconf(ldc_chan_t *ldcp, boolean_t force_reset); 75 static int i_ldc_rxq_drain(ldc_chan_t *ldcp); 76 static void i_ldc_reset_state(ldc_chan_t *ldcp); 77 static void i_ldc_reset(ldc_chan_t *ldcp, boolean_t force_reset); 78 79 static int i_ldc_get_tx_tail(ldc_chan_t *ldcp, uint64_t *tail); 80 static void i_ldc_get_tx_head(ldc_chan_t *ldcp, uint64_t *head); 81 static int i_ldc_set_tx_tail(ldc_chan_t *ldcp, uint64_t tail); 82 static int i_ldc_set_rx_head(ldc_chan_t *ldcp, uint64_t head); 83 static int i_ldc_send_pkt(ldc_chan_t *ldcp, uint8_t pkttype, uint8_t subtype, 84 uint8_t ctrlmsg); 85 86 /* Interrupt handling functions */ 87 static uint_t i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2); 88 static uint_t i_ldc_rx_hdlr(caddr_t arg1, caddr_t arg2); 89 static void i_ldc_clear_intr(ldc_chan_t *ldcp, cnex_intrtype_t itype); 90 91 /* Read method functions */ 92 static int i_ldc_read_raw(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep); 93 static int i_ldc_read_packet(ldc_chan_t *ldcp, caddr_t target_bufp, 94 size_t *sizep); 95 static int i_ldc_read_stream(ldc_chan_t *ldcp, caddr_t target_bufp, 96 size_t *sizep); 97 98 /* Write method functions */ 99 static int i_ldc_write_raw(ldc_chan_t *ldcp, caddr_t target_bufp, 100 size_t *sizep); 101 static int i_ldc_write_packet(ldc_chan_t *ldcp, caddr_t target_bufp, 102 size_t *sizep); 103 static int i_ldc_write_stream(ldc_chan_t *ldcp, caddr_t target_bufp, 104 size_t *sizep); 105 106 /* Pkt processing internal functions */ 107 static int i_ldc_check_seqid(ldc_chan_t *ldcp, ldc_msg_t *ldcmsg); 108 static int i_ldc_ctrlmsg(ldc_chan_t *ldcp, ldc_msg_t *ldcmsg); 109 static int i_ldc_process_VER(ldc_chan_t *ldcp, ldc_msg_t *msg); 110 static int i_ldc_process_RTS(ldc_chan_t *ldcp, ldc_msg_t *msg); 111 static int i_ldc_process_RTR(ldc_chan_t *ldcp, ldc_msg_t *msg); 112 static int i_ldc_process_RDX(ldc_chan_t *ldcp, ldc_msg_t *msg); 113 static int i_ldc_process_data_ACK(ldc_chan_t *ldcp, ldc_msg_t *msg); 114 115 /* Memory synchronization internal functions */ 116 static int i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, 117 uint8_t direction, uint64_t offset, size_t size); 118 static int i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle, 119 uint8_t direction, uint64_t start, uint64_t end); 120 121 /* LDC Version */ 122 static ldc_ver_t ldc_versions[] = { {1, 0} }; 123 124 /* number of supported versions */ 125 #define LDC_NUM_VERS (sizeof (ldc_versions) / sizeof (ldc_versions[0])) 126 127 /* Module State Pointer */ 128 static ldc_soft_state_t *ldcssp; 129 130 static struct modldrv md = { 131 &mod_miscops, /* This is a misc module */ 132 "sun4v LDC module v%I%", /* Name of the module */ 133 }; 134 135 static struct modlinkage ml = { 136 MODREV_1, 137 &md, 138 NULL 139 }; 140 141 static uint64_t ldc_sup_minor; /* Supported minor number */ 142 static hsvc_info_t ldc_hsvc = { 143 HSVC_REV_1, NULL, HSVC_GROUP_LDC, 1, 0, "ldc" 144 }; 145 146 /* 147 * LDC framework supports mapping remote domain's memory 148 * either directly or via shadow memory pages. Default 149 * support is currently implemented via shadow copy. 150 * Direct map can be enabled by setting 'ldc_shmem_enabled' 151 */ 152 int ldc_shmem_enabled = 0; 153 154 /* 155 * The no. of MTU size messages that can be stored in 156 * the LDC Tx queue. The number of Tx queue entries is 157 * then computed as (mtu * mtu_msgs)/sizeof(queue_entry) 158 */ 159 uint64_t ldc_mtu_msgs = LDC_MTU_MSGS; 160 161 /* 162 * The minimum queue length. This is the size of the smallest 163 * LDC queue. If the computed value is less than this default, 164 * the queue length is rounded up to 'ldc_queue_entries'. 165 */ 166 uint64_t ldc_queue_entries = LDC_QUEUE_ENTRIES; 167 168 /* 169 * Pages exported for remote access over each channel is 170 * maintained in a table registered with the Hypervisor. 171 * The default number of entries in the table is set to 172 * 'ldc_mtbl_entries'. 173 */ 174 uint64_t ldc_maptable_entries = LDC_MTBL_ENTRIES; 175 176 /* 177 * LDC retry count and delay - when the HV returns EWOULDBLOCK 178 * the operation is retried 'ldc_max_retries' times with a 179 * wait of 'ldc_delay' usecs between each retry. 180 */ 181 int ldc_max_retries = LDC_MAX_RETRIES; 182 clock_t ldc_delay = LDC_DELAY; 183 184 /* 185 * delay between each retry of channel unregistration in 186 * ldc_close(), to wait for pending interrupts to complete. 187 */ 188 clock_t ldc_close_delay = LDC_CLOSE_DELAY; 189 190 #ifdef DEBUG 191 192 /* 193 * Print debug messages 194 * 195 * set ldcdbg to 0x7 for enabling all msgs 196 * 0x4 - Warnings 197 * 0x2 - All debug messages 198 * 0x1 - Minimal debug messages 199 * 200 * set ldcdbgchan to the channel number you want to debug 201 * setting it to -1 prints debug messages for all channels 202 * NOTE: ldcdbgchan has no effect on error messages 203 */ 204 205 #define DBG_ALL_LDCS -1 206 207 int ldcdbg = 0x0; 208 int64_t ldcdbgchan = DBG_ALL_LDCS; 209 uint64_t ldc_inject_err_flag = 0; 210 211 static void 212 ldcdebug(int64_t id, const char *fmt, ...) 213 { 214 char buf[512]; 215 va_list ap; 216 217 /* 218 * Do not return if, 219 * caller wants to print it anyway - (id == DBG_ALL_LDCS) 220 * debug channel is set to all LDCs - (ldcdbgchan == DBG_ALL_LDCS) 221 * debug channel = caller specified channel 222 */ 223 if ((id != DBG_ALL_LDCS) && 224 (ldcdbgchan != DBG_ALL_LDCS) && 225 (ldcdbgchan != id)) { 226 return; 227 } 228 229 va_start(ap, fmt); 230 (void) vsprintf(buf, fmt, ap); 231 va_end(ap); 232 233 cmn_err(CE_CONT, "?%s", buf); 234 } 235 236 #define LDC_ERR_RESET 0x1 237 #define LDC_ERR_PKTLOSS 0x2 238 239 static boolean_t 240 ldc_inject_error(ldc_chan_t *ldcp, uint64_t error) 241 { 242 if ((ldcdbgchan != DBG_ALL_LDCS) && (ldcdbgchan != ldcp->id)) 243 return (B_FALSE); 244 245 if ((ldc_inject_err_flag & error) == 0) 246 return (B_FALSE); 247 248 /* clear the injection state */ 249 ldc_inject_err_flag &= ~error; 250 251 return (B_TRUE); 252 } 253 254 #define D1 \ 255 if (ldcdbg & 0x01) \ 256 ldcdebug 257 258 #define D2 \ 259 if (ldcdbg & 0x02) \ 260 ldcdebug 261 262 #define DWARN \ 263 if (ldcdbg & 0x04) \ 264 ldcdebug 265 266 #define DUMP_PAYLOAD(id, addr) \ 267 { \ 268 char buf[65*3]; \ 269 int i; \ 270 uint8_t *src = (uint8_t *)addr; \ 271 for (i = 0; i < 64; i++, src++) \ 272 (void) sprintf(&buf[i * 3], "|%02x", *src); \ 273 (void) sprintf(&buf[i * 3], "|\n"); \ 274 D2((id), "payload: %s", buf); \ 275 } 276 277 #define DUMP_LDC_PKT(c, s, addr) \ 278 { \ 279 ldc_msg_t *msg = (ldc_msg_t *)(addr); \ 280 uint32_t mid = ((c)->mode != LDC_MODE_RAW) ? msg->seqid : 0; \ 281 if (msg->type == LDC_DATA) { \ 282 D2((c)->id, "%s: msg%d (/%x/%x/%x/,env[%c%c,sz=%d])", \ 283 (s), mid, msg->type, msg->stype, msg->ctrl, \ 284 (msg->env & LDC_FRAG_START) ? 'B' : ' ', \ 285 (msg->env & LDC_FRAG_STOP) ? 'E' : ' ', \ 286 (msg->env & LDC_LEN_MASK)); \ 287 } else { \ 288 D2((c)->id, "%s: msg%d (/%x/%x/%x/,env=%x)", (s), \ 289 mid, msg->type, msg->stype, msg->ctrl, msg->env); \ 290 } \ 291 } 292 293 #define LDC_INJECT_RESET(_ldcp) ldc_inject_error(_ldcp, LDC_ERR_RESET) 294 #define LDC_INJECT_PKTLOSS(_ldcp) ldc_inject_error(_ldcp, LDC_ERR_PKTLOSS) 295 296 #else 297 298 #define DBG_ALL_LDCS -1 299 300 #define D1 301 #define D2 302 #define DWARN 303 304 #define DUMP_PAYLOAD(id, addr) 305 #define DUMP_LDC_PKT(c, s, addr) 306 307 #define LDC_INJECT_RESET(_ldcp) (B_FALSE) 308 #define LDC_INJECT_PKTLOSS(_ldcp) (B_FALSE) 309 310 #endif 311 312 #define ZERO_PKT(p) \ 313 bzero((p), sizeof (ldc_msg_t)); 314 315 #define IDX2COOKIE(idx, pg_szc, pg_shift) \ 316 (((pg_szc) << LDC_COOKIE_PGSZC_SHIFT) | ((idx) << (pg_shift))) 317 318 int 319 _init(void) 320 { 321 int status; 322 323 status = hsvc_register(&ldc_hsvc, &ldc_sup_minor); 324 if (status != 0) { 325 cmn_err(CE_NOTE, "!%s: cannot negotiate hypervisor LDC services" 326 " group: 0x%lx major: %ld minor: %ld errno: %d", 327 ldc_hsvc.hsvc_modname, ldc_hsvc.hsvc_group, 328 ldc_hsvc.hsvc_major, ldc_hsvc.hsvc_minor, status); 329 return (-1); 330 } 331 332 /* allocate soft state structure */ 333 ldcssp = kmem_zalloc(sizeof (ldc_soft_state_t), KM_SLEEP); 334 335 /* Link the module into the system */ 336 status = mod_install(&ml); 337 if (status != 0) { 338 kmem_free(ldcssp, sizeof (ldc_soft_state_t)); 339 return (status); 340 } 341 342 /* Initialize the LDC state structure */ 343 mutex_init(&ldcssp->lock, NULL, MUTEX_DRIVER, NULL); 344 345 mutex_enter(&ldcssp->lock); 346 347 /* Create a cache for memory handles */ 348 ldcssp->memhdl_cache = kmem_cache_create("ldc_memhdl_cache", 349 sizeof (ldc_mhdl_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 350 if (ldcssp->memhdl_cache == NULL) { 351 DWARN(DBG_ALL_LDCS, "_init: ldc_memhdl cache create failed\n"); 352 mutex_exit(&ldcssp->lock); 353 return (-1); 354 } 355 356 /* Create cache for memory segment structures */ 357 ldcssp->memseg_cache = kmem_cache_create("ldc_memseg_cache", 358 sizeof (ldc_memseg_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 359 if (ldcssp->memseg_cache == NULL) { 360 DWARN(DBG_ALL_LDCS, "_init: ldc_memseg cache create failed\n"); 361 mutex_exit(&ldcssp->lock); 362 return (-1); 363 } 364 365 366 ldcssp->channel_count = 0; 367 ldcssp->channels_open = 0; 368 ldcssp->chan_list = NULL; 369 ldcssp->dring_list = NULL; 370 371 mutex_exit(&ldcssp->lock); 372 373 return (0); 374 } 375 376 int 377 _info(struct modinfo *modinfop) 378 { 379 /* Report status of the dynamically loadable driver module */ 380 return (mod_info(&ml, modinfop)); 381 } 382 383 int 384 _fini(void) 385 { 386 int rv, status; 387 ldc_chan_t *tmp_ldcp, *ldcp; 388 ldc_dring_t *tmp_dringp, *dringp; 389 ldc_mem_info_t minfo; 390 391 /* Unlink the driver module from the system */ 392 status = mod_remove(&ml); 393 if (status) { 394 DWARN(DBG_ALL_LDCS, "_fini: mod_remove failed\n"); 395 return (EIO); 396 } 397 398 /* Free descriptor rings */ 399 dringp = ldcssp->dring_list; 400 while (dringp != NULL) { 401 tmp_dringp = dringp->next; 402 403 rv = ldc_mem_dring_info((ldc_dring_handle_t)dringp, &minfo); 404 if (rv == 0 && minfo.status != LDC_UNBOUND) { 405 if (minfo.status == LDC_BOUND) { 406 (void) ldc_mem_dring_unbind( 407 (ldc_dring_handle_t)dringp); 408 } 409 if (minfo.status == LDC_MAPPED) { 410 (void) ldc_mem_dring_unmap( 411 (ldc_dring_handle_t)dringp); 412 } 413 } 414 415 (void) ldc_mem_dring_destroy((ldc_dring_handle_t)dringp); 416 dringp = tmp_dringp; 417 } 418 ldcssp->dring_list = NULL; 419 420 /* close and finalize channels */ 421 ldcp = ldcssp->chan_list; 422 while (ldcp != NULL) { 423 tmp_ldcp = ldcp->next; 424 425 (void) ldc_close((ldc_handle_t)ldcp); 426 (void) ldc_fini((ldc_handle_t)ldcp); 427 428 ldcp = tmp_ldcp; 429 } 430 ldcssp->chan_list = NULL; 431 432 /* Destroy kmem caches */ 433 kmem_cache_destroy(ldcssp->memhdl_cache); 434 kmem_cache_destroy(ldcssp->memseg_cache); 435 436 /* 437 * We have successfully "removed" the driver. 438 * Destroying soft states 439 */ 440 mutex_destroy(&ldcssp->lock); 441 kmem_free(ldcssp, sizeof (ldc_soft_state_t)); 442 443 (void) hsvc_unregister(&ldc_hsvc); 444 445 return (status); 446 } 447 448 /* -------------------------------------------------------------------------- */ 449 450 /* 451 * LDC Link Layer Internal Functions 452 */ 453 454 /* 455 * Translate HV Errors to sun4v error codes 456 */ 457 static int 458 i_ldc_h2v_error(int h_error) 459 { 460 switch (h_error) { 461 462 case H_EOK: 463 return (0); 464 465 case H_ENORADDR: 466 return (EFAULT); 467 468 case H_EBADPGSZ: 469 case H_EINVAL: 470 return (EINVAL); 471 472 case H_EWOULDBLOCK: 473 return (EWOULDBLOCK); 474 475 case H_ENOACCESS: 476 case H_ENOMAP: 477 return (EACCES); 478 479 case H_EIO: 480 case H_ECPUERROR: 481 return (EIO); 482 483 case H_ENOTSUPPORTED: 484 return (ENOTSUP); 485 486 case H_ETOOMANY: 487 return (ENOSPC); 488 489 case H_ECHANNEL: 490 return (ECHRNG); 491 default: 492 break; 493 } 494 495 return (EIO); 496 } 497 498 /* 499 * Reconfigure the transmit queue 500 */ 501 static int 502 i_ldc_txq_reconf(ldc_chan_t *ldcp) 503 { 504 int rv; 505 506 ASSERT(MUTEX_HELD(&ldcp->lock)); 507 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 508 509 rv = hv_ldc_tx_qconf(ldcp->id, ldcp->tx_q_ra, ldcp->tx_q_entries); 510 if (rv) { 511 cmn_err(CE_WARN, 512 "i_ldc_txq_reconf: (0x%lx) cannot set qconf", ldcp->id); 513 return (EIO); 514 } 515 rv = hv_ldc_tx_get_state(ldcp->id, &(ldcp->tx_head), 516 &(ldcp->tx_tail), &(ldcp->link_state)); 517 if (rv) { 518 cmn_err(CE_WARN, 519 "i_ldc_txq_reconf: (0x%lx) cannot get qptrs", ldcp->id); 520 return (EIO); 521 } 522 D1(ldcp->id, "i_ldc_txq_reconf: (0x%llx) h=0x%llx,t=0x%llx," 523 "s=0x%llx\n", ldcp->id, ldcp->tx_head, ldcp->tx_tail, 524 ldcp->link_state); 525 526 return (0); 527 } 528 529 /* 530 * Reconfigure the receive queue 531 */ 532 static int 533 i_ldc_rxq_reconf(ldc_chan_t *ldcp, boolean_t force_reset) 534 { 535 int rv; 536 uint64_t rx_head, rx_tail; 537 538 ASSERT(MUTEX_HELD(&ldcp->lock)); 539 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 540 &(ldcp->link_state)); 541 if (rv) { 542 cmn_err(CE_WARN, 543 "i_ldc_rxq_reconf: (0x%lx) cannot get state", 544 ldcp->id); 545 return (EIO); 546 } 547 548 if (force_reset || (ldcp->tstate & ~TS_IN_RESET) == TS_UP) { 549 rv = hv_ldc_rx_qconf(ldcp->id, ldcp->rx_q_ra, 550 ldcp->rx_q_entries); 551 if (rv) { 552 cmn_err(CE_WARN, 553 "i_ldc_rxq_reconf: (0x%lx) cannot set qconf", 554 ldcp->id); 555 return (EIO); 556 } 557 D1(ldcp->id, "i_ldc_rxq_reconf: (0x%llx) completed q reconf", 558 ldcp->id); 559 } 560 561 return (0); 562 } 563 564 565 /* 566 * Drain the contents of the receive queue 567 */ 568 static int 569 i_ldc_rxq_drain(ldc_chan_t *ldcp) 570 { 571 int rv; 572 uint64_t rx_head, rx_tail; 573 574 ASSERT(MUTEX_HELD(&ldcp->lock)); 575 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 576 &(ldcp->link_state)); 577 if (rv) { 578 cmn_err(CE_WARN, "i_ldc_rxq_drain: (0x%lx) cannot get state", 579 ldcp->id); 580 return (EIO); 581 } 582 583 /* flush contents by setting the head = tail */ 584 return (i_ldc_set_rx_head(ldcp, rx_tail)); 585 } 586 587 588 /* 589 * Reset LDC state structure and its contents 590 */ 591 static void 592 i_ldc_reset_state(ldc_chan_t *ldcp) 593 { 594 ASSERT(MUTEX_HELD(&ldcp->lock)); 595 ldcp->last_msg_snt = LDC_INIT_SEQID; 596 ldcp->last_ack_rcd = 0; 597 ldcp->last_msg_rcd = 0; 598 ldcp->tx_ackd_head = ldcp->tx_head; 599 ldcp->next_vidx = 0; 600 ldcp->hstate = 0; 601 ldcp->tstate = TS_OPEN; 602 ldcp->status = LDC_OPEN; 603 604 if (ldcp->link_state == LDC_CHANNEL_UP || 605 ldcp->link_state == LDC_CHANNEL_RESET) { 606 607 if (ldcp->mode == LDC_MODE_RAW) { 608 ldcp->status = LDC_UP; 609 ldcp->tstate = TS_UP; 610 } else { 611 ldcp->status = LDC_READY; 612 ldcp->tstate |= TS_LINK_READY; 613 } 614 } 615 } 616 617 /* 618 * Reset a LDC channel 619 */ 620 static void 621 i_ldc_reset(ldc_chan_t *ldcp, boolean_t force_reset) 622 { 623 DWARN(ldcp->id, "i_ldc_reset: (0x%llx) channel reset\n", ldcp->id); 624 625 ASSERT(MUTEX_HELD(&ldcp->lock)); 626 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 627 628 /* reconfig Tx and Rx queues */ 629 (void) i_ldc_txq_reconf(ldcp); 630 (void) i_ldc_rxq_reconf(ldcp, force_reset); 631 632 /* Clear Tx and Rx interrupts */ 633 (void) i_ldc_clear_intr(ldcp, CNEX_TX_INTR); 634 (void) i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 635 636 /* Reset channel state */ 637 i_ldc_reset_state(ldcp); 638 639 /* Mark channel in reset */ 640 ldcp->tstate |= TS_IN_RESET; 641 } 642 643 644 /* 645 * Clear pending interrupts 646 */ 647 static void 648 i_ldc_clear_intr(ldc_chan_t *ldcp, cnex_intrtype_t itype) 649 { 650 ldc_cnex_t *cinfo = &ldcssp->cinfo; 651 652 ASSERT(MUTEX_HELD(&ldcp->lock)); 653 ASSERT(cinfo->dip != NULL); 654 655 switch (itype) { 656 case CNEX_TX_INTR: 657 /* check Tx interrupt */ 658 if (ldcp->tx_intr_state) 659 ldcp->tx_intr_state = LDC_INTR_NONE; 660 else 661 return; 662 break; 663 664 case CNEX_RX_INTR: 665 /* check Rx interrupt */ 666 if (ldcp->rx_intr_state) 667 ldcp->rx_intr_state = LDC_INTR_NONE; 668 else 669 return; 670 break; 671 } 672 673 (void) cinfo->clr_intr(cinfo->dip, ldcp->id, itype); 674 D2(ldcp->id, 675 "i_ldc_clear_intr: (0x%llx) cleared 0x%x intr\n", 676 ldcp->id, itype); 677 } 678 679 /* 680 * Set the receive queue head 681 * Resets connection and returns an error if it fails. 682 */ 683 static int 684 i_ldc_set_rx_head(ldc_chan_t *ldcp, uint64_t head) 685 { 686 int rv; 687 int retries; 688 689 ASSERT(MUTEX_HELD(&ldcp->lock)); 690 for (retries = 0; retries < ldc_max_retries; retries++) { 691 692 if ((rv = hv_ldc_rx_set_qhead(ldcp->id, head)) == 0) 693 return (0); 694 695 if (rv != H_EWOULDBLOCK) 696 break; 697 698 /* wait for ldc_delay usecs */ 699 drv_usecwait(ldc_delay); 700 } 701 702 cmn_err(CE_WARN, "ldc_rx_set_qhead: (0x%lx) cannot set qhead 0x%lx", 703 ldcp->id, head); 704 mutex_enter(&ldcp->tx_lock); 705 i_ldc_reset(ldcp, B_TRUE); 706 mutex_exit(&ldcp->tx_lock); 707 708 return (ECONNRESET); 709 } 710 711 /* 712 * Returns the tx_head to be used for transfer 713 */ 714 static void 715 i_ldc_get_tx_head(ldc_chan_t *ldcp, uint64_t *head) 716 { 717 ldc_msg_t *pkt; 718 719 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 720 721 /* get current Tx head */ 722 *head = ldcp->tx_head; 723 724 /* 725 * Reliable mode will use the ACKd head instead of the regular tx_head. 726 * Also in Reliable mode, advance ackd_head for all non DATA/INFO pkts, 727 * up to the current location of tx_head. This needs to be done 728 * as the peer will only ACK DATA/INFO pkts. 729 */ 730 if (ldcp->mode == LDC_MODE_RELIABLE || ldcp->mode == LDC_MODE_STREAM) { 731 while (ldcp->tx_ackd_head != ldcp->tx_head) { 732 pkt = (ldc_msg_t *)(ldcp->tx_q_va + ldcp->tx_ackd_head); 733 if ((pkt->type & LDC_DATA) && (pkt->stype & LDC_INFO)) { 734 break; 735 } 736 /* advance ACKd head */ 737 ldcp->tx_ackd_head = 738 (ldcp->tx_ackd_head + LDC_PACKET_SIZE) % 739 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 740 } 741 *head = ldcp->tx_ackd_head; 742 } 743 } 744 745 /* 746 * Returns the tx_tail to be used for transfer 747 * Re-reads the TX queue ptrs if and only if the 748 * the cached head and tail are equal (queue is full) 749 */ 750 static int 751 i_ldc_get_tx_tail(ldc_chan_t *ldcp, uint64_t *tail) 752 { 753 int rv; 754 uint64_t current_head, new_tail; 755 756 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 757 /* Read the head and tail ptrs from HV */ 758 rv = hv_ldc_tx_get_state(ldcp->id, 759 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 760 if (rv) { 761 cmn_err(CE_WARN, 762 "i_ldc_get_tx_tail: (0x%lx) cannot read qptrs\n", 763 ldcp->id); 764 return (EIO); 765 } 766 if (ldcp->link_state == LDC_CHANNEL_DOWN) { 767 D1(ldcp->id, "i_ldc_get_tx_tail: (0x%llx) channel not ready\n", 768 ldcp->id); 769 return (ECONNRESET); 770 } 771 772 i_ldc_get_tx_head(ldcp, ¤t_head); 773 774 /* increment the tail */ 775 new_tail = (ldcp->tx_tail + LDC_PACKET_SIZE) % 776 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 777 778 if (new_tail == current_head) { 779 DWARN(ldcp->id, 780 "i_ldc_get_tx_tail: (0x%llx) TX queue is full\n", 781 ldcp->id); 782 return (EWOULDBLOCK); 783 } 784 785 D2(ldcp->id, "i_ldc_get_tx_tail: (0x%llx) head=0x%llx, tail=0x%llx\n", 786 ldcp->id, ldcp->tx_head, ldcp->tx_tail); 787 788 *tail = ldcp->tx_tail; 789 return (0); 790 } 791 792 /* 793 * Set the tail pointer. If HV returns EWOULDBLOCK, it will back off 794 * and retry ldc_max_retries times before returning an error. 795 * Returns 0, EWOULDBLOCK or EIO 796 */ 797 static int 798 i_ldc_set_tx_tail(ldc_chan_t *ldcp, uint64_t tail) 799 { 800 int rv, retval = EWOULDBLOCK; 801 int retries; 802 803 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 804 for (retries = 0; retries < ldc_max_retries; retries++) { 805 806 if ((rv = hv_ldc_tx_set_qtail(ldcp->id, tail)) == 0) { 807 retval = 0; 808 break; 809 } 810 if (rv != H_EWOULDBLOCK) { 811 DWARN(ldcp->id, "i_ldc_set_tx_tail: (0x%llx) set " 812 "qtail=0x%llx failed, rv=%d\n", ldcp->id, tail, rv); 813 retval = EIO; 814 break; 815 } 816 817 /* wait for ldc_delay usecs */ 818 drv_usecwait(ldc_delay); 819 } 820 return (retval); 821 } 822 823 /* 824 * Send a LDC message 825 */ 826 static int 827 i_ldc_send_pkt(ldc_chan_t *ldcp, uint8_t pkttype, uint8_t subtype, 828 uint8_t ctrlmsg) 829 { 830 int rv; 831 ldc_msg_t *pkt; 832 uint64_t tx_tail; 833 uint32_t curr_seqid; 834 835 /* Obtain Tx lock */ 836 mutex_enter(&ldcp->tx_lock); 837 838 curr_seqid = ldcp->last_msg_snt; 839 840 /* get the current tail for the message */ 841 rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 842 if (rv) { 843 DWARN(ldcp->id, 844 "i_ldc_send_pkt: (0x%llx) error sending pkt, " 845 "type=0x%x,subtype=0x%x,ctrl=0x%x\n", 846 ldcp->id, pkttype, subtype, ctrlmsg); 847 mutex_exit(&ldcp->tx_lock); 848 return (rv); 849 } 850 851 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 852 ZERO_PKT(pkt); 853 854 /* Initialize the packet */ 855 pkt->type = pkttype; 856 pkt->stype = subtype; 857 pkt->ctrl = ctrlmsg; 858 859 /* Store ackid/seqid iff it is RELIABLE mode & not a RTS/RTR message */ 860 if (((ctrlmsg & LDC_CTRL_MASK) != LDC_RTS) && 861 ((ctrlmsg & LDC_CTRL_MASK) != LDC_RTR)) { 862 curr_seqid++; 863 if (ldcp->mode != LDC_MODE_RAW) { 864 pkt->seqid = curr_seqid; 865 pkt->ackid = ldcp->last_msg_rcd; 866 } 867 } 868 DUMP_LDC_PKT(ldcp, "i_ldc_send_pkt", (uint64_t)pkt); 869 870 /* initiate the send by calling into HV and set the new tail */ 871 tx_tail = (tx_tail + LDC_PACKET_SIZE) % 872 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 873 874 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 875 if (rv) { 876 DWARN(ldcp->id, 877 "i_ldc_send_pkt:(0x%llx) error sending pkt, " 878 "type=0x%x,stype=0x%x,ctrl=0x%x\n", 879 ldcp->id, pkttype, subtype, ctrlmsg); 880 mutex_exit(&ldcp->tx_lock); 881 return (EIO); 882 } 883 884 ldcp->last_msg_snt = curr_seqid; 885 ldcp->tx_tail = tx_tail; 886 887 mutex_exit(&ldcp->tx_lock); 888 return (0); 889 } 890 891 /* 892 * Checks if packet was received in right order 893 * in the case of a reliable link. 894 * Returns 0 if in order, else EIO 895 */ 896 static int 897 i_ldc_check_seqid(ldc_chan_t *ldcp, ldc_msg_t *msg) 898 { 899 /* No seqid checking for RAW mode */ 900 if (ldcp->mode == LDC_MODE_RAW) 901 return (0); 902 903 /* No seqid checking for version, RTS, RTR message */ 904 if (msg->ctrl == LDC_VER || 905 msg->ctrl == LDC_RTS || 906 msg->ctrl == LDC_RTR) 907 return (0); 908 909 /* Initial seqid to use is sent in RTS/RTR and saved in last_msg_rcd */ 910 if (msg->seqid != (ldcp->last_msg_rcd + 1)) { 911 DWARN(ldcp->id, 912 "i_ldc_check_seqid: (0x%llx) out-of-order pkt, got 0x%x, " 913 "expecting 0x%x\n", ldcp->id, msg->seqid, 914 (ldcp->last_msg_rcd + 1)); 915 return (EIO); 916 } 917 918 #ifdef DEBUG 919 if (LDC_INJECT_PKTLOSS(ldcp)) { 920 DWARN(ldcp->id, 921 "i_ldc_check_seqid: (0x%llx) inject pkt loss\n", ldcp->id); 922 return (EIO); 923 } 924 #endif 925 926 return (0); 927 } 928 929 930 /* 931 * Process an incoming version ctrl message 932 */ 933 static int 934 i_ldc_process_VER(ldc_chan_t *ldcp, ldc_msg_t *msg) 935 { 936 int rv = 0, idx = ldcp->next_vidx; 937 ldc_msg_t *pkt; 938 uint64_t tx_tail; 939 ldc_ver_t *rcvd_ver; 940 941 /* get the received version */ 942 rcvd_ver = (ldc_ver_t *)((uint64_t)msg + LDC_PAYLOAD_VER_OFF); 943 944 D2(ldcp->id, "i_ldc_process_VER: (0x%llx) received VER v%u.%u\n", 945 ldcp->id, rcvd_ver->major, rcvd_ver->minor); 946 947 /* Obtain Tx lock */ 948 mutex_enter(&ldcp->tx_lock); 949 950 switch (msg->stype) { 951 case LDC_INFO: 952 953 if ((ldcp->tstate & ~TS_IN_RESET) == TS_VREADY) { 954 (void) i_ldc_txq_reconf(ldcp); 955 i_ldc_reset_state(ldcp); 956 mutex_exit(&ldcp->tx_lock); 957 return (EAGAIN); 958 } 959 960 /* get the current tail and pkt for the response */ 961 rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 962 if (rv != 0) { 963 DWARN(ldcp->id, 964 "i_ldc_process_VER: (0x%llx) err sending " 965 "version ACK/NACK\n", ldcp->id); 966 i_ldc_reset(ldcp, B_TRUE); 967 mutex_exit(&ldcp->tx_lock); 968 return (ECONNRESET); 969 } 970 971 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 972 ZERO_PKT(pkt); 973 974 /* initialize the packet */ 975 pkt->type = LDC_CTRL; 976 pkt->ctrl = LDC_VER; 977 978 for (;;) { 979 980 D1(ldcp->id, "i_ldc_process_VER: got %u.%u chk %u.%u\n", 981 rcvd_ver->major, rcvd_ver->minor, 982 ldc_versions[idx].major, ldc_versions[idx].minor); 983 984 if (rcvd_ver->major == ldc_versions[idx].major) { 985 /* major version match - ACK version */ 986 pkt->stype = LDC_ACK; 987 988 /* 989 * lower minor version to the one this endpt 990 * supports, if necessary 991 */ 992 if (rcvd_ver->minor > ldc_versions[idx].minor) 993 rcvd_ver->minor = 994 ldc_versions[idx].minor; 995 bcopy(rcvd_ver, pkt->udata, sizeof (*rcvd_ver)); 996 997 break; 998 } 999 1000 if (rcvd_ver->major > ldc_versions[idx].major) { 1001 1002 D1(ldcp->id, "i_ldc_process_VER: using next" 1003 " lower idx=%d, v%u.%u\n", idx, 1004 ldc_versions[idx].major, 1005 ldc_versions[idx].minor); 1006 1007 /* nack with next lower version */ 1008 pkt->stype = LDC_NACK; 1009 bcopy(&ldc_versions[idx], pkt->udata, 1010 sizeof (ldc_versions[idx])); 1011 ldcp->next_vidx = idx; 1012 break; 1013 } 1014 1015 /* next major version */ 1016 idx++; 1017 1018 D1(ldcp->id, "i_ldc_process_VER: inc idx %x\n", idx); 1019 1020 if (idx == LDC_NUM_VERS) { 1021 /* no version match - send NACK */ 1022 pkt->stype = LDC_NACK; 1023 bzero(pkt->udata, sizeof (ldc_ver_t)); 1024 ldcp->next_vidx = 0; 1025 break; 1026 } 1027 } 1028 1029 /* initiate the send by calling into HV and set the new tail */ 1030 tx_tail = (tx_tail + LDC_PACKET_SIZE) % 1031 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 1032 1033 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 1034 if (rv == 0) { 1035 ldcp->tx_tail = tx_tail; 1036 if (pkt->stype == LDC_ACK) { 1037 D2(ldcp->id, "i_ldc_process_VER: (0x%llx) sent" 1038 " version ACK\n", ldcp->id); 1039 /* Save the ACK'd version */ 1040 ldcp->version.major = rcvd_ver->major; 1041 ldcp->version.minor = rcvd_ver->minor; 1042 ldcp->hstate |= TS_RCVD_VER; 1043 ldcp->tstate |= TS_VER_DONE; 1044 D1(DBG_ALL_LDCS, 1045 "(0x%llx) Sent ACK, " 1046 "Agreed on version v%u.%u\n", 1047 ldcp->id, rcvd_ver->major, rcvd_ver->minor); 1048 } 1049 } else { 1050 DWARN(ldcp->id, 1051 "i_ldc_process_VER: (0x%llx) error sending " 1052 "ACK/NACK\n", ldcp->id); 1053 i_ldc_reset(ldcp, B_TRUE); 1054 mutex_exit(&ldcp->tx_lock); 1055 return (ECONNRESET); 1056 } 1057 1058 break; 1059 1060 case LDC_ACK: 1061 if ((ldcp->tstate & ~TS_IN_RESET) == TS_VREADY) { 1062 if (ldcp->version.major != rcvd_ver->major || 1063 ldcp->version.minor != rcvd_ver->minor) { 1064 1065 /* mismatched version - reset connection */ 1066 DWARN(ldcp->id, 1067 "i_ldc_process_VER: (0x%llx) recvd" 1068 " ACK ver != sent ACK ver\n", ldcp->id); 1069 i_ldc_reset(ldcp, B_TRUE); 1070 mutex_exit(&ldcp->tx_lock); 1071 return (ECONNRESET); 1072 } 1073 } else { 1074 /* SUCCESS - we have agreed on a version */ 1075 ldcp->version.major = rcvd_ver->major; 1076 ldcp->version.minor = rcvd_ver->minor; 1077 ldcp->tstate |= TS_VER_DONE; 1078 } 1079 1080 D1(ldcp->id, "(0x%llx) Got ACK, Agreed on version v%u.%u\n", 1081 ldcp->id, rcvd_ver->major, rcvd_ver->minor); 1082 1083 /* initiate RTS-RTR-RDX handshake */ 1084 rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 1085 if (rv) { 1086 DWARN(ldcp->id, 1087 "i_ldc_process_VER: (0x%llx) cannot send RTS\n", 1088 ldcp->id); 1089 i_ldc_reset(ldcp, B_TRUE); 1090 mutex_exit(&ldcp->tx_lock); 1091 return (ECONNRESET); 1092 } 1093 1094 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 1095 ZERO_PKT(pkt); 1096 1097 pkt->type = LDC_CTRL; 1098 pkt->stype = LDC_INFO; 1099 pkt->ctrl = LDC_RTS; 1100 pkt->env = ldcp->mode; 1101 if (ldcp->mode != LDC_MODE_RAW) 1102 pkt->seqid = LDC_INIT_SEQID; 1103 1104 ldcp->last_msg_rcd = LDC_INIT_SEQID; 1105 1106 DUMP_LDC_PKT(ldcp, "i_ldc_process_VER snd rts", (uint64_t)pkt); 1107 1108 /* initiate the send by calling into HV and set the new tail */ 1109 tx_tail = (tx_tail + LDC_PACKET_SIZE) % 1110 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 1111 1112 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 1113 if (rv) { 1114 D2(ldcp->id, 1115 "i_ldc_process_VER: (0x%llx) no listener\n", 1116 ldcp->id); 1117 i_ldc_reset(ldcp, B_TRUE); 1118 mutex_exit(&ldcp->tx_lock); 1119 return (ECONNRESET); 1120 } 1121 1122 ldcp->tx_tail = tx_tail; 1123 ldcp->hstate |= TS_SENT_RTS; 1124 1125 break; 1126 1127 case LDC_NACK: 1128 /* check if version in NACK is zero */ 1129 if (rcvd_ver->major == 0 && rcvd_ver->minor == 0) { 1130 /* version handshake failure */ 1131 DWARN(DBG_ALL_LDCS, 1132 "i_ldc_process_VER: (0x%llx) no version match\n", 1133 ldcp->id); 1134 i_ldc_reset(ldcp, B_TRUE); 1135 mutex_exit(&ldcp->tx_lock); 1136 return (ECONNRESET); 1137 } 1138 1139 /* get the current tail and pkt for the response */ 1140 rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 1141 if (rv != 0) { 1142 cmn_err(CE_NOTE, 1143 "i_ldc_process_VER: (0x%lx) err sending " 1144 "version ACK/NACK\n", ldcp->id); 1145 i_ldc_reset(ldcp, B_TRUE); 1146 mutex_exit(&ldcp->tx_lock); 1147 return (ECONNRESET); 1148 } 1149 1150 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 1151 ZERO_PKT(pkt); 1152 1153 /* initialize the packet */ 1154 pkt->type = LDC_CTRL; 1155 pkt->ctrl = LDC_VER; 1156 pkt->stype = LDC_INFO; 1157 1158 /* check ver in NACK msg has a match */ 1159 for (;;) { 1160 if (rcvd_ver->major == ldc_versions[idx].major) { 1161 /* 1162 * major version match - resubmit request 1163 * if lower minor version to the one this endpt 1164 * supports, if necessary 1165 */ 1166 if (rcvd_ver->minor > ldc_versions[idx].minor) 1167 rcvd_ver->minor = 1168 ldc_versions[idx].minor; 1169 bcopy(rcvd_ver, pkt->udata, sizeof (*rcvd_ver)); 1170 break; 1171 } 1172 1173 if (rcvd_ver->major > ldc_versions[idx].major) { 1174 1175 D1(ldcp->id, "i_ldc_process_VER: using next" 1176 " lower idx=%d, v%u.%u\n", idx, 1177 ldc_versions[idx].major, 1178 ldc_versions[idx].minor); 1179 1180 /* send next lower version */ 1181 bcopy(&ldc_versions[idx], pkt->udata, 1182 sizeof (ldc_versions[idx])); 1183 ldcp->next_vidx = idx; 1184 break; 1185 } 1186 1187 /* next version */ 1188 idx++; 1189 1190 D1(ldcp->id, "i_ldc_process_VER: inc idx %x\n", idx); 1191 1192 if (idx == LDC_NUM_VERS) { 1193 /* no version match - terminate */ 1194 ldcp->next_vidx = 0; 1195 mutex_exit(&ldcp->tx_lock); 1196 return (ECONNRESET); 1197 } 1198 } 1199 1200 /* initiate the send by calling into HV and set the new tail */ 1201 tx_tail = (tx_tail + LDC_PACKET_SIZE) % 1202 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 1203 1204 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 1205 if (rv == 0) { 1206 D2(ldcp->id, "i_ldc_process_VER: (0x%llx) sent version" 1207 "INFO v%u.%u\n", ldcp->id, ldc_versions[idx].major, 1208 ldc_versions[idx].minor); 1209 ldcp->tx_tail = tx_tail; 1210 } else { 1211 cmn_err(CE_NOTE, 1212 "i_ldc_process_VER: (0x%lx) error sending version" 1213 "INFO\n", ldcp->id); 1214 i_ldc_reset(ldcp, B_TRUE); 1215 mutex_exit(&ldcp->tx_lock); 1216 return (ECONNRESET); 1217 } 1218 1219 break; 1220 } 1221 1222 mutex_exit(&ldcp->tx_lock); 1223 return (rv); 1224 } 1225 1226 1227 /* 1228 * Process an incoming RTS ctrl message 1229 */ 1230 static int 1231 i_ldc_process_RTS(ldc_chan_t *ldcp, ldc_msg_t *msg) 1232 { 1233 int rv = 0; 1234 ldc_msg_t *pkt; 1235 uint64_t tx_tail; 1236 boolean_t sent_NACK = B_FALSE; 1237 1238 D2(ldcp->id, "i_ldc_process_RTS: (0x%llx) received RTS\n", ldcp->id); 1239 1240 switch (msg->stype) { 1241 case LDC_NACK: 1242 DWARN(ldcp->id, 1243 "i_ldc_process_RTS: (0x%llx) RTS NACK received\n", 1244 ldcp->id); 1245 1246 /* Reset the channel -- as we cannot continue */ 1247 mutex_enter(&ldcp->tx_lock); 1248 i_ldc_reset(ldcp, B_TRUE); 1249 mutex_exit(&ldcp->tx_lock); 1250 rv = ECONNRESET; 1251 break; 1252 1253 case LDC_INFO: 1254 1255 /* check mode */ 1256 if (ldcp->mode != (ldc_mode_t)msg->env) { 1257 cmn_err(CE_NOTE, 1258 "i_ldc_process_RTS: (0x%lx) mode mismatch\n", 1259 ldcp->id); 1260 /* 1261 * send NACK in response to MODE message 1262 * get the current tail for the response 1263 */ 1264 rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, LDC_RTS); 1265 if (rv) { 1266 /* if cannot send NACK - reset channel */ 1267 mutex_enter(&ldcp->tx_lock); 1268 i_ldc_reset(ldcp, B_TRUE); 1269 mutex_exit(&ldcp->tx_lock); 1270 rv = ECONNRESET; 1271 break; 1272 } 1273 sent_NACK = B_TRUE; 1274 } 1275 break; 1276 default: 1277 DWARN(ldcp->id, "i_ldc_process_RTS: (0x%llx) unexp ACK\n", 1278 ldcp->id); 1279 mutex_enter(&ldcp->tx_lock); 1280 i_ldc_reset(ldcp, B_TRUE); 1281 mutex_exit(&ldcp->tx_lock); 1282 rv = ECONNRESET; 1283 break; 1284 } 1285 1286 /* 1287 * If either the connection was reset (when rv != 0) or 1288 * a NACK was sent, we return. In the case of a NACK 1289 * we dont want to consume the packet that came in but 1290 * not record that we received the RTS 1291 */ 1292 if (rv || sent_NACK) 1293 return (rv); 1294 1295 /* record RTS received */ 1296 ldcp->hstate |= TS_RCVD_RTS; 1297 1298 /* store initial SEQID info */ 1299 ldcp->last_msg_snt = msg->seqid; 1300 1301 /* Obtain Tx lock */ 1302 mutex_enter(&ldcp->tx_lock); 1303 1304 /* get the current tail for the response */ 1305 rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 1306 if (rv != 0) { 1307 cmn_err(CE_NOTE, 1308 "i_ldc_process_RTS: (0x%lx) err sending RTR\n", 1309 ldcp->id); 1310 i_ldc_reset(ldcp, B_TRUE); 1311 mutex_exit(&ldcp->tx_lock); 1312 return (ECONNRESET); 1313 } 1314 1315 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 1316 ZERO_PKT(pkt); 1317 1318 /* initialize the packet */ 1319 pkt->type = LDC_CTRL; 1320 pkt->stype = LDC_INFO; 1321 pkt->ctrl = LDC_RTR; 1322 pkt->env = ldcp->mode; 1323 if (ldcp->mode != LDC_MODE_RAW) 1324 pkt->seqid = LDC_INIT_SEQID; 1325 1326 ldcp->last_msg_rcd = msg->seqid; 1327 1328 /* initiate the send by calling into HV and set the new tail */ 1329 tx_tail = (tx_tail + LDC_PACKET_SIZE) % 1330 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 1331 1332 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 1333 if (rv == 0) { 1334 D2(ldcp->id, 1335 "i_ldc_process_RTS: (0x%llx) sent RTR\n", ldcp->id); 1336 DUMP_LDC_PKT(ldcp, "i_ldc_process_RTS sent rtr", (uint64_t)pkt); 1337 1338 ldcp->tx_tail = tx_tail; 1339 ldcp->hstate |= TS_SENT_RTR; 1340 1341 } else { 1342 cmn_err(CE_NOTE, 1343 "i_ldc_process_RTS: (0x%lx) error sending RTR\n", 1344 ldcp->id); 1345 i_ldc_reset(ldcp, B_TRUE); 1346 mutex_exit(&ldcp->tx_lock); 1347 return (ECONNRESET); 1348 } 1349 1350 mutex_exit(&ldcp->tx_lock); 1351 return (0); 1352 } 1353 1354 /* 1355 * Process an incoming RTR ctrl message 1356 */ 1357 static int 1358 i_ldc_process_RTR(ldc_chan_t *ldcp, ldc_msg_t *msg) 1359 { 1360 int rv = 0; 1361 boolean_t sent_NACK = B_FALSE; 1362 1363 D2(ldcp->id, "i_ldc_process_RTR: (0x%llx) received RTR\n", ldcp->id); 1364 1365 switch (msg->stype) { 1366 case LDC_NACK: 1367 /* RTR NACK received */ 1368 DWARN(ldcp->id, 1369 "i_ldc_process_RTR: (0x%llx) RTR NACK received\n", 1370 ldcp->id); 1371 1372 /* Reset the channel -- as we cannot continue */ 1373 mutex_enter(&ldcp->tx_lock); 1374 i_ldc_reset(ldcp, B_TRUE); 1375 mutex_exit(&ldcp->tx_lock); 1376 rv = ECONNRESET; 1377 1378 break; 1379 1380 case LDC_INFO: 1381 1382 /* check mode */ 1383 if (ldcp->mode != (ldc_mode_t)msg->env) { 1384 DWARN(ldcp->id, 1385 "i_ldc_process_RTR: (0x%llx) mode mismatch, " 1386 "expecting 0x%x, got 0x%x\n", 1387 ldcp->id, ldcp->mode, (ldc_mode_t)msg->env); 1388 /* 1389 * send NACK in response to MODE message 1390 * get the current tail for the response 1391 */ 1392 rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, LDC_RTR); 1393 if (rv) { 1394 /* if cannot send NACK - reset channel */ 1395 mutex_enter(&ldcp->tx_lock); 1396 i_ldc_reset(ldcp, B_TRUE); 1397 mutex_exit(&ldcp->tx_lock); 1398 rv = ECONNRESET; 1399 break; 1400 } 1401 sent_NACK = B_TRUE; 1402 } 1403 break; 1404 1405 default: 1406 DWARN(ldcp->id, "i_ldc_process_RTR: (0x%llx) unexp ACK\n", 1407 ldcp->id); 1408 1409 /* Reset the channel -- as we cannot continue */ 1410 mutex_enter(&ldcp->tx_lock); 1411 i_ldc_reset(ldcp, B_TRUE); 1412 mutex_exit(&ldcp->tx_lock); 1413 rv = ECONNRESET; 1414 break; 1415 } 1416 1417 /* 1418 * If either the connection was reset (when rv != 0) or 1419 * a NACK was sent, we return. In the case of a NACK 1420 * we dont want to consume the packet that came in but 1421 * not record that we received the RTR 1422 */ 1423 if (rv || sent_NACK) 1424 return (rv); 1425 1426 ldcp->last_msg_snt = msg->seqid; 1427 ldcp->hstate |= TS_RCVD_RTR; 1428 1429 rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_INFO, LDC_RDX); 1430 if (rv) { 1431 cmn_err(CE_NOTE, 1432 "i_ldc_process_RTR: (0x%lx) cannot send RDX\n", 1433 ldcp->id); 1434 mutex_enter(&ldcp->tx_lock); 1435 i_ldc_reset(ldcp, B_TRUE); 1436 mutex_exit(&ldcp->tx_lock); 1437 return (ECONNRESET); 1438 } 1439 D2(ldcp->id, 1440 "i_ldc_process_RTR: (0x%llx) sent RDX\n", ldcp->id); 1441 1442 ldcp->hstate |= TS_SENT_RDX; 1443 ldcp->tstate |= TS_HSHAKE_DONE; 1444 if ((ldcp->tstate & TS_IN_RESET) == 0) 1445 ldcp->status = LDC_UP; 1446 1447 D1(ldcp->id, "(0x%llx) Handshake Complete\n", ldcp->id); 1448 1449 return (0); 1450 } 1451 1452 1453 /* 1454 * Process an incoming RDX ctrl message 1455 */ 1456 static int 1457 i_ldc_process_RDX(ldc_chan_t *ldcp, ldc_msg_t *msg) 1458 { 1459 int rv = 0; 1460 1461 D2(ldcp->id, "i_ldc_process_RDX: (0x%llx) received RDX\n", ldcp->id); 1462 1463 switch (msg->stype) { 1464 case LDC_NACK: 1465 /* RDX NACK received */ 1466 DWARN(ldcp->id, 1467 "i_ldc_process_RDX: (0x%llx) RDX NACK received\n", 1468 ldcp->id); 1469 1470 /* Reset the channel -- as we cannot continue */ 1471 mutex_enter(&ldcp->tx_lock); 1472 i_ldc_reset(ldcp, B_TRUE); 1473 mutex_exit(&ldcp->tx_lock); 1474 rv = ECONNRESET; 1475 1476 break; 1477 1478 case LDC_INFO: 1479 1480 /* 1481 * if channel is UP and a RDX received after data transmission 1482 * has commenced it is an error 1483 */ 1484 if ((ldcp->tstate == TS_UP) && (ldcp->hstate & TS_RCVD_RDX)) { 1485 DWARN(DBG_ALL_LDCS, 1486 "i_ldc_process_RDX: (0x%llx) unexpected RDX" 1487 " - LDC reset\n", ldcp->id); 1488 mutex_enter(&ldcp->tx_lock); 1489 i_ldc_reset(ldcp, B_TRUE); 1490 mutex_exit(&ldcp->tx_lock); 1491 return (ECONNRESET); 1492 } 1493 1494 ldcp->hstate |= TS_RCVD_RDX; 1495 ldcp->tstate |= TS_HSHAKE_DONE; 1496 if ((ldcp->tstate & TS_IN_RESET) == 0) 1497 ldcp->status = LDC_UP; 1498 1499 D1(DBG_ALL_LDCS, "(0x%llx) Handshake Complete\n", ldcp->id); 1500 break; 1501 1502 default: 1503 DWARN(ldcp->id, "i_ldc_process_RDX: (0x%llx) unexp ACK\n", 1504 ldcp->id); 1505 1506 /* Reset the channel -- as we cannot continue */ 1507 mutex_enter(&ldcp->tx_lock); 1508 i_ldc_reset(ldcp, B_TRUE); 1509 mutex_exit(&ldcp->tx_lock); 1510 rv = ECONNRESET; 1511 break; 1512 } 1513 1514 return (rv); 1515 } 1516 1517 /* 1518 * Process an incoming ACK for a data packet 1519 */ 1520 static int 1521 i_ldc_process_data_ACK(ldc_chan_t *ldcp, ldc_msg_t *msg) 1522 { 1523 int rv; 1524 uint64_t tx_head; 1525 ldc_msg_t *pkt; 1526 1527 /* Obtain Tx lock */ 1528 mutex_enter(&ldcp->tx_lock); 1529 1530 /* 1531 * Read the current Tx head and tail 1532 */ 1533 rv = hv_ldc_tx_get_state(ldcp->id, 1534 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 1535 if (rv != 0) { 1536 cmn_err(CE_WARN, 1537 "i_ldc_process_data_ACK: (0x%lx) cannot read qptrs\n", 1538 ldcp->id); 1539 1540 /* Reset the channel -- as we cannot continue */ 1541 i_ldc_reset(ldcp, B_TRUE); 1542 mutex_exit(&ldcp->tx_lock); 1543 return (ECONNRESET); 1544 } 1545 1546 /* 1547 * loop from where the previous ACK location was to the 1548 * current head location. This is how far the HV has 1549 * actually send pkts. Pkts between head and tail are 1550 * yet to be sent by HV. 1551 */ 1552 tx_head = ldcp->tx_ackd_head; 1553 for (;;) { 1554 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_head); 1555 tx_head = (tx_head + LDC_PACKET_SIZE) % 1556 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 1557 1558 if (pkt->seqid == msg->ackid) { 1559 D2(ldcp->id, 1560 "i_ldc_process_data_ACK: (0x%llx) found packet\n", 1561 ldcp->id); 1562 ldcp->last_ack_rcd = msg->ackid; 1563 ldcp->tx_ackd_head = tx_head; 1564 break; 1565 } 1566 if (tx_head == ldcp->tx_head) { 1567 /* could not find packet */ 1568 DWARN(ldcp->id, 1569 "i_ldc_process_data_ACK: (0x%llx) invalid ACKid\n", 1570 ldcp->id); 1571 1572 /* Reset the channel -- as we cannot continue */ 1573 i_ldc_reset(ldcp, B_TRUE); 1574 mutex_exit(&ldcp->tx_lock); 1575 return (ECONNRESET); 1576 } 1577 } 1578 1579 mutex_exit(&ldcp->tx_lock); 1580 return (0); 1581 } 1582 1583 /* 1584 * Process incoming control message 1585 * Return 0 - session can continue 1586 * EAGAIN - reprocess packet - state was changed 1587 * ECONNRESET - channel was reset 1588 */ 1589 static int 1590 i_ldc_ctrlmsg(ldc_chan_t *ldcp, ldc_msg_t *msg) 1591 { 1592 int rv = 0; 1593 1594 D1(ldcp->id, "i_ldc_ctrlmsg: (%llx) tstate = %lx, hstate = %lx\n", 1595 ldcp->id, ldcp->tstate, ldcp->hstate); 1596 1597 switch (ldcp->tstate & ~TS_IN_RESET) { 1598 1599 case TS_OPEN: 1600 case TS_READY: 1601 1602 switch (msg->ctrl & LDC_CTRL_MASK) { 1603 case LDC_VER: 1604 /* process version message */ 1605 rv = i_ldc_process_VER(ldcp, msg); 1606 break; 1607 default: 1608 DWARN(ldcp->id, 1609 "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 1610 "tstate=0x%x\n", ldcp->id, 1611 (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 1612 break; 1613 } 1614 1615 break; 1616 1617 case TS_VREADY: 1618 1619 switch (msg->ctrl & LDC_CTRL_MASK) { 1620 case LDC_VER: 1621 /* process version message */ 1622 rv = i_ldc_process_VER(ldcp, msg); 1623 break; 1624 case LDC_RTS: 1625 /* process RTS message */ 1626 rv = i_ldc_process_RTS(ldcp, msg); 1627 break; 1628 case LDC_RTR: 1629 /* process RTR message */ 1630 rv = i_ldc_process_RTR(ldcp, msg); 1631 break; 1632 case LDC_RDX: 1633 /* process RDX message */ 1634 rv = i_ldc_process_RDX(ldcp, msg); 1635 break; 1636 default: 1637 DWARN(ldcp->id, 1638 "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 1639 "tstate=0x%x\n", ldcp->id, 1640 (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 1641 break; 1642 } 1643 1644 break; 1645 1646 case TS_UP: 1647 1648 switch (msg->ctrl & LDC_CTRL_MASK) { 1649 case LDC_VER: 1650 DWARN(ldcp->id, 1651 "i_ldc_ctrlmsg: (0x%llx) unexpected VER " 1652 "- LDC reset\n", ldcp->id); 1653 /* peer is redoing version negotiation */ 1654 mutex_enter(&ldcp->tx_lock); 1655 (void) i_ldc_txq_reconf(ldcp); 1656 i_ldc_reset_state(ldcp); 1657 mutex_exit(&ldcp->tx_lock); 1658 rv = EAGAIN; 1659 break; 1660 1661 case LDC_RDX: 1662 /* process RDX message */ 1663 rv = i_ldc_process_RDX(ldcp, msg); 1664 break; 1665 1666 default: 1667 DWARN(ldcp->id, 1668 "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 1669 "tstate=0x%x\n", ldcp->id, 1670 (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 1671 break; 1672 } 1673 } 1674 1675 return (rv); 1676 } 1677 1678 /* 1679 * Register channel with the channel nexus 1680 */ 1681 static int 1682 i_ldc_register_channel(ldc_chan_t *ldcp) 1683 { 1684 int rv = 0; 1685 ldc_cnex_t *cinfo = &ldcssp->cinfo; 1686 1687 if (cinfo->dip == NULL) { 1688 DWARN(ldcp->id, 1689 "i_ldc_register_channel: cnex has not registered\n"); 1690 return (EAGAIN); 1691 } 1692 1693 rv = cinfo->reg_chan(cinfo->dip, ldcp->id, ldcp->devclass); 1694 if (rv) { 1695 DWARN(ldcp->id, 1696 "i_ldc_register_channel: cannot register channel\n"); 1697 return (rv); 1698 } 1699 1700 rv = cinfo->add_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR, 1701 i_ldc_tx_hdlr, ldcp, NULL); 1702 if (rv) { 1703 DWARN(ldcp->id, 1704 "i_ldc_register_channel: cannot add Tx interrupt\n"); 1705 (void) cinfo->unreg_chan(cinfo->dip, ldcp->id); 1706 return (rv); 1707 } 1708 1709 rv = cinfo->add_intr(cinfo->dip, ldcp->id, CNEX_RX_INTR, 1710 i_ldc_rx_hdlr, ldcp, NULL); 1711 if (rv) { 1712 DWARN(ldcp->id, 1713 "i_ldc_register_channel: cannot add Rx interrupt\n"); 1714 (void) cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR); 1715 (void) cinfo->unreg_chan(cinfo->dip, ldcp->id); 1716 return (rv); 1717 } 1718 1719 ldcp->tstate |= TS_CNEX_RDY; 1720 1721 return (0); 1722 } 1723 1724 /* 1725 * Unregister a channel with the channel nexus 1726 */ 1727 static int 1728 i_ldc_unregister_channel(ldc_chan_t *ldcp) 1729 { 1730 int rv = 0; 1731 ldc_cnex_t *cinfo = &ldcssp->cinfo; 1732 1733 if (cinfo->dip == NULL) { 1734 DWARN(ldcp->id, 1735 "i_ldc_unregister_channel: cnex has not registered\n"); 1736 return (EAGAIN); 1737 } 1738 1739 if (ldcp->tstate & TS_CNEX_RDY) { 1740 1741 /* Remove the Rx interrupt */ 1742 rv = cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_RX_INTR); 1743 if (rv) { 1744 if (rv != EAGAIN) { 1745 DWARN(ldcp->id, 1746 "i_ldc_unregister_channel: err removing " 1747 "Rx intr\n"); 1748 return (rv); 1749 } 1750 1751 /* 1752 * If interrupts are pending and handler has 1753 * finished running, clear interrupt and try 1754 * again 1755 */ 1756 if (ldcp->rx_intr_state != LDC_INTR_PEND) 1757 return (rv); 1758 1759 (void) i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 1760 rv = cinfo->rem_intr(cinfo->dip, ldcp->id, 1761 CNEX_RX_INTR); 1762 if (rv) { 1763 DWARN(ldcp->id, "i_ldc_unregister_channel: " 1764 "err removing Rx interrupt\n"); 1765 return (rv); 1766 } 1767 } 1768 1769 /* Remove the Tx interrupt */ 1770 rv = cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR); 1771 if (rv) { 1772 DWARN(ldcp->id, 1773 "i_ldc_unregister_channel: err removing Tx intr\n"); 1774 return (rv); 1775 } 1776 1777 /* Unregister the channel */ 1778 rv = cinfo->unreg_chan(ldcssp->cinfo.dip, ldcp->id); 1779 if (rv) { 1780 DWARN(ldcp->id, 1781 "i_ldc_unregister_channel: cannot unreg channel\n"); 1782 return (rv); 1783 } 1784 1785 ldcp->tstate &= ~TS_CNEX_RDY; 1786 } 1787 1788 return (0); 1789 } 1790 1791 1792 /* 1793 * LDC transmit interrupt handler 1794 * triggered for chanel up/down/reset events 1795 * and Tx queue content changes 1796 */ 1797 static uint_t 1798 i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2) 1799 { 1800 _NOTE(ARGUNUSED(arg2)) 1801 1802 int rv; 1803 ldc_chan_t *ldcp; 1804 boolean_t notify_client = B_FALSE; 1805 uint64_t notify_event = 0, link_state; 1806 1807 /* Get the channel for which interrupt was received */ 1808 ASSERT(arg1 != NULL); 1809 ldcp = (ldc_chan_t *)arg1; 1810 1811 D1(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n", 1812 ldcp->id, ldcp); 1813 1814 /* Lock channel */ 1815 mutex_enter(&ldcp->lock); 1816 1817 /* Obtain Tx lock */ 1818 mutex_enter(&ldcp->tx_lock); 1819 1820 /* mark interrupt as pending */ 1821 ldcp->tx_intr_state = LDC_INTR_ACTIVE; 1822 1823 /* save current link state */ 1824 link_state = ldcp->link_state; 1825 1826 rv = hv_ldc_tx_get_state(ldcp->id, &ldcp->tx_head, &ldcp->tx_tail, 1827 &ldcp->link_state); 1828 if (rv) { 1829 cmn_err(CE_WARN, 1830 "i_ldc_tx_hdlr: (0x%lx) cannot read queue ptrs rv=0x%d\n", 1831 ldcp->id, rv); 1832 i_ldc_clear_intr(ldcp, CNEX_TX_INTR); 1833 mutex_exit(&ldcp->tx_lock); 1834 mutex_exit(&ldcp->lock); 1835 return (DDI_INTR_CLAIMED); 1836 } 1837 1838 /* 1839 * reset the channel state if the channel went down 1840 * (other side unconfigured queue) or channel was reset 1841 * (other side reconfigured its queue) 1842 */ 1843 if (link_state != ldcp->link_state && 1844 ldcp->link_state == LDC_CHANNEL_DOWN) { 1845 D1(ldcp->id, "i_ldc_tx_hdlr: channel link down\n", ldcp->id); 1846 i_ldc_reset(ldcp, B_FALSE); 1847 notify_client = B_TRUE; 1848 notify_event = LDC_EVT_DOWN; 1849 } 1850 1851 if (link_state != ldcp->link_state && 1852 ldcp->link_state == LDC_CHANNEL_RESET) { 1853 D1(ldcp->id, "i_ldc_tx_hdlr: channel link reset\n", ldcp->id); 1854 i_ldc_reset(ldcp, B_FALSE); 1855 notify_client = B_TRUE; 1856 notify_event = LDC_EVT_RESET; 1857 } 1858 1859 if (link_state != ldcp->link_state && 1860 (ldcp->tstate & ~TS_IN_RESET) == TS_OPEN && 1861 ldcp->link_state == LDC_CHANNEL_UP) { 1862 D1(ldcp->id, "i_ldc_tx_hdlr: channel link up\n", ldcp->id); 1863 notify_client = B_TRUE; 1864 notify_event = LDC_EVT_RESET; 1865 ldcp->tstate |= TS_LINK_READY; 1866 ldcp->status = LDC_READY; 1867 } 1868 1869 /* if callbacks are disabled, do not notify */ 1870 if (!ldcp->cb_enabled) 1871 notify_client = B_FALSE; 1872 1873 i_ldc_clear_intr(ldcp, CNEX_TX_INTR); 1874 mutex_exit(&ldcp->tx_lock); 1875 1876 if (notify_client) { 1877 ldcp->cb_inprogress = B_TRUE; 1878 mutex_exit(&ldcp->lock); 1879 rv = ldcp->cb(notify_event, ldcp->cb_arg); 1880 if (rv) { 1881 DWARN(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) callback " 1882 "failure", ldcp->id); 1883 } 1884 mutex_enter(&ldcp->lock); 1885 ldcp->cb_inprogress = B_FALSE; 1886 } 1887 1888 mutex_exit(&ldcp->lock); 1889 1890 D1(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) exiting handler", ldcp->id); 1891 1892 return (DDI_INTR_CLAIMED); 1893 } 1894 1895 /* 1896 * LDC receive interrupt handler 1897 * triggered for channel with data pending to read 1898 * i.e. Rx queue content changes 1899 */ 1900 static uint_t 1901 i_ldc_rx_hdlr(caddr_t arg1, caddr_t arg2) 1902 { 1903 _NOTE(ARGUNUSED(arg2)) 1904 1905 int rv; 1906 uint64_t rx_head, rx_tail; 1907 ldc_msg_t *msg; 1908 ldc_chan_t *ldcp; 1909 boolean_t notify_client = B_FALSE; 1910 uint64_t notify_event = 0; 1911 uint64_t link_state, first_fragment = 0; 1912 1913 1914 /* Get the channel for which interrupt was received */ 1915 if (arg1 == NULL) { 1916 cmn_err(CE_WARN, "i_ldc_rx_hdlr: invalid arg\n"); 1917 return (DDI_INTR_UNCLAIMED); 1918 } 1919 1920 ldcp = (ldc_chan_t *)arg1; 1921 1922 D1(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n", 1923 ldcp->id, ldcp); 1924 D1(ldcp->id, "i_ldc_rx_hdlr: (%llx) USR%lx/TS%lx/HS%lx, LSTATE=%lx\n", 1925 ldcp->id, ldcp->status, ldcp->tstate, ldcp->hstate, 1926 ldcp->link_state); 1927 1928 /* Lock channel */ 1929 mutex_enter(&ldcp->lock); 1930 1931 /* mark interrupt as pending */ 1932 ldcp->rx_intr_state = LDC_INTR_ACTIVE; 1933 1934 /* 1935 * Read packet(s) from the queue 1936 */ 1937 for (;;) { 1938 1939 link_state = ldcp->link_state; 1940 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 1941 &ldcp->link_state); 1942 if (rv) { 1943 cmn_err(CE_WARN, 1944 "i_ldc_rx_hdlr: (0x%lx) cannot read " 1945 "queue ptrs, rv=0x%d\n", ldcp->id, rv); 1946 i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 1947 mutex_exit(&ldcp->lock); 1948 return (DDI_INTR_CLAIMED); 1949 } 1950 1951 /* 1952 * reset the channel state if the channel went down 1953 * (other side unconfigured queue) or channel was reset 1954 * (other side reconfigured its queue) 1955 */ 1956 1957 if (link_state != ldcp->link_state) { 1958 1959 switch (ldcp->link_state) { 1960 case LDC_CHANNEL_DOWN: 1961 D1(ldcp->id, "i_ldc_rx_hdlr: channel " 1962 "link down\n", ldcp->id); 1963 mutex_enter(&ldcp->tx_lock); 1964 i_ldc_reset(ldcp, B_FALSE); 1965 mutex_exit(&ldcp->tx_lock); 1966 notify_client = B_TRUE; 1967 notify_event = LDC_EVT_DOWN; 1968 goto loop_exit; 1969 1970 case LDC_CHANNEL_UP: 1971 D1(ldcp->id, "i_ldc_rx_hdlr: " 1972 "channel link up\n", ldcp->id); 1973 1974 if ((ldcp->tstate & ~TS_IN_RESET) == TS_OPEN) { 1975 notify_client = B_TRUE; 1976 notify_event = LDC_EVT_RESET; 1977 ldcp->tstate |= TS_LINK_READY; 1978 ldcp->status = LDC_READY; 1979 } 1980 break; 1981 1982 case LDC_CHANNEL_RESET: 1983 default: 1984 #ifdef DEBUG 1985 force_reset: 1986 #endif 1987 D1(ldcp->id, "i_ldc_rx_hdlr: channel " 1988 "link reset\n", ldcp->id); 1989 mutex_enter(&ldcp->tx_lock); 1990 i_ldc_reset(ldcp, B_FALSE); 1991 mutex_exit(&ldcp->tx_lock); 1992 notify_client = B_TRUE; 1993 notify_event = LDC_EVT_RESET; 1994 break; 1995 } 1996 } 1997 1998 #ifdef DEBUG 1999 if (LDC_INJECT_RESET(ldcp)) 2000 goto force_reset; 2001 #endif 2002 2003 if (rx_head == rx_tail) { 2004 D2(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) No packets\n", 2005 ldcp->id); 2006 break; 2007 } 2008 2009 D2(ldcp->id, "i_ldc_rx_hdlr: head=0x%llx, tail=0x%llx\n", 2010 rx_head, rx_tail); 2011 DUMP_LDC_PKT(ldcp, "i_ldc_rx_hdlr rcd", 2012 ldcp->rx_q_va + rx_head); 2013 2014 /* get the message */ 2015 msg = (ldc_msg_t *)(ldcp->rx_q_va + rx_head); 2016 2017 /* if channel is in RAW mode or data pkt, notify and return */ 2018 if (ldcp->mode == LDC_MODE_RAW) { 2019 notify_client = B_TRUE; 2020 notify_event |= LDC_EVT_READ; 2021 break; 2022 } 2023 2024 if ((msg->type & LDC_DATA) && (msg->stype & LDC_INFO)) { 2025 2026 /* discard packet if channel is not up */ 2027 if ((ldcp->tstate & ~TS_IN_RESET) != TS_UP) { 2028 2029 /* move the head one position */ 2030 rx_head = (rx_head + LDC_PACKET_SIZE) % 2031 (ldcp->rx_q_entries << LDC_PACKET_SHIFT); 2032 2033 if (rv = i_ldc_set_rx_head(ldcp, rx_head)) 2034 break; 2035 2036 continue; 2037 } else { 2038 if ((ldcp->tstate & TS_IN_RESET) == 0) 2039 notify_client = B_TRUE; 2040 notify_event |= LDC_EVT_READ; 2041 break; 2042 } 2043 } 2044 2045 /* Check the sequence ID for the message received */ 2046 rv = i_ldc_check_seqid(ldcp, msg); 2047 if (rv != 0) { 2048 2049 DWARN(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) seqid error, " 2050 "q_ptrs=0x%lx,0x%lx", ldcp->id, rx_head, rx_tail); 2051 2052 /* Reset last_msg_rcd to start of message */ 2053 if (first_fragment != 0) { 2054 ldcp->last_msg_rcd = first_fragment - 1; 2055 first_fragment = 0; 2056 } 2057 2058 /* 2059 * Send a NACK due to seqid mismatch 2060 */ 2061 rv = i_ldc_send_pkt(ldcp, msg->type, LDC_NACK, 2062 (msg->ctrl & LDC_CTRL_MASK)); 2063 2064 if (rv) { 2065 cmn_err(CE_NOTE, 2066 "i_ldc_rx_hdlr: (0x%lx) err sending " 2067 "CTRL/DATA NACK msg\n", ldcp->id); 2068 2069 /* if cannot send NACK - reset channel */ 2070 mutex_enter(&ldcp->tx_lock); 2071 i_ldc_reset(ldcp, B_TRUE); 2072 mutex_exit(&ldcp->tx_lock); 2073 2074 notify_client = B_TRUE; 2075 notify_event = LDC_EVT_RESET; 2076 break; 2077 } 2078 2079 /* purge receive queue */ 2080 (void) i_ldc_set_rx_head(ldcp, rx_tail); 2081 break; 2082 } 2083 2084 /* record the message ID */ 2085 ldcp->last_msg_rcd = msg->seqid; 2086 2087 /* process control messages */ 2088 if (msg->type & LDC_CTRL) { 2089 /* save current internal state */ 2090 uint64_t tstate = ldcp->tstate; 2091 2092 rv = i_ldc_ctrlmsg(ldcp, msg); 2093 if (rv == EAGAIN) { 2094 /* re-process pkt - state was adjusted */ 2095 continue; 2096 } 2097 if (rv == ECONNRESET) { 2098 notify_client = B_TRUE; 2099 notify_event = LDC_EVT_RESET; 2100 break; 2101 } 2102 2103 /* 2104 * control message processing was successful 2105 * channel transitioned to ready for communication 2106 */ 2107 if (rv == 0 && ldcp->tstate == TS_UP && 2108 (tstate & ~TS_IN_RESET) != 2109 (ldcp->tstate & ~TS_IN_RESET)) { 2110 notify_client = B_TRUE; 2111 notify_event = LDC_EVT_UP; 2112 } 2113 } 2114 2115 /* process data NACKs */ 2116 if ((msg->type & LDC_DATA) && (msg->stype & LDC_NACK)) { 2117 DWARN(ldcp->id, 2118 "i_ldc_rx_hdlr: (0x%llx) received DATA/NACK", 2119 ldcp->id); 2120 mutex_enter(&ldcp->tx_lock); 2121 i_ldc_reset(ldcp, B_TRUE); 2122 mutex_exit(&ldcp->tx_lock); 2123 notify_client = B_TRUE; 2124 notify_event = LDC_EVT_RESET; 2125 break; 2126 } 2127 2128 /* process data ACKs */ 2129 if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 2130 if (rv = i_ldc_process_data_ACK(ldcp, msg)) { 2131 notify_client = B_TRUE; 2132 notify_event = LDC_EVT_RESET; 2133 break; 2134 } 2135 } 2136 2137 /* move the head one position */ 2138 rx_head = (rx_head + LDC_PACKET_SIZE) % 2139 (ldcp->rx_q_entries << LDC_PACKET_SHIFT); 2140 if (rv = i_ldc_set_rx_head(ldcp, rx_head)) { 2141 notify_client = B_TRUE; 2142 notify_event = LDC_EVT_RESET; 2143 break; 2144 } 2145 2146 } /* for */ 2147 2148 loop_exit: 2149 2150 /* if callbacks are disabled, do not notify */ 2151 if (!ldcp->cb_enabled) 2152 notify_client = B_FALSE; 2153 2154 /* 2155 * If there are data packets in the queue, the ldc_read will 2156 * clear interrupts after draining the queue, else clear interrupts 2157 */ 2158 if ((notify_event & LDC_EVT_READ) == 0) { 2159 i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 2160 } else 2161 ldcp->rx_intr_state = LDC_INTR_PEND; 2162 2163 2164 if (notify_client) { 2165 ldcp->cb_inprogress = B_TRUE; 2166 mutex_exit(&ldcp->lock); 2167 rv = ldcp->cb(notify_event, ldcp->cb_arg); 2168 if (rv) { 2169 DWARN(ldcp->id, 2170 "i_ldc_rx_hdlr: (0x%llx) callback failure", 2171 ldcp->id); 2172 } 2173 mutex_enter(&ldcp->lock); 2174 ldcp->cb_inprogress = B_FALSE; 2175 } 2176 2177 mutex_exit(&ldcp->lock); 2178 2179 D1(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) exiting handler", ldcp->id); 2180 return (DDI_INTR_CLAIMED); 2181 } 2182 2183 2184 /* -------------------------------------------------------------------------- */ 2185 2186 /* 2187 * LDC API functions 2188 */ 2189 2190 /* 2191 * Initialize the channel. Allocate internal structure and memory for 2192 * TX/RX queues, and initialize locks. 2193 */ 2194 int 2195 ldc_init(uint64_t id, ldc_attr_t *attr, ldc_handle_t *handle) 2196 { 2197 ldc_chan_t *ldcp; 2198 int rv, exit_val; 2199 uint64_t ra_base, nentries; 2200 uint64_t qlen; 2201 2202 exit_val = EINVAL; /* guarantee an error if exit on failure */ 2203 2204 if (attr == NULL) { 2205 DWARN(id, "ldc_init: (0x%llx) invalid attr\n", id); 2206 return (EINVAL); 2207 } 2208 if (handle == NULL) { 2209 DWARN(id, "ldc_init: (0x%llx) invalid handle\n", id); 2210 return (EINVAL); 2211 } 2212 2213 /* check if channel is valid */ 2214 rv = hv_ldc_tx_qinfo(id, &ra_base, &nentries); 2215 if (rv == H_ECHANNEL) { 2216 DWARN(id, "ldc_init: (0x%llx) invalid channel id\n", id); 2217 return (EINVAL); 2218 } 2219 2220 /* check if the channel has already been initialized */ 2221 mutex_enter(&ldcssp->lock); 2222 ldcp = ldcssp->chan_list; 2223 while (ldcp != NULL) { 2224 if (ldcp->id == id) { 2225 DWARN(id, "ldc_init: (0x%llx) already initialized\n", 2226 id); 2227 mutex_exit(&ldcssp->lock); 2228 return (EADDRINUSE); 2229 } 2230 ldcp = ldcp->next; 2231 } 2232 mutex_exit(&ldcssp->lock); 2233 2234 ASSERT(ldcp == NULL); 2235 2236 *handle = 0; 2237 2238 /* Allocate an ldcp structure */ 2239 ldcp = kmem_zalloc(sizeof (ldc_chan_t), KM_SLEEP); 2240 2241 /* 2242 * Initialize the channel and Tx lock 2243 * 2244 * The channel 'lock' protects the entire channel and 2245 * should be acquired before initializing, resetting, 2246 * destroying or reading from a channel. 2247 * 2248 * The 'tx_lock' should be acquired prior to transmitting 2249 * data over the channel. The lock should also be acquired 2250 * prior to channel reconfiguration (in order to prevent 2251 * concurrent writes). 2252 * 2253 * ORDERING: When both locks are being acquired, to prevent 2254 * deadlocks, the channel lock should be always acquired prior 2255 * to the tx_lock. 2256 */ 2257 mutex_init(&ldcp->lock, NULL, MUTEX_DRIVER, NULL); 2258 mutex_init(&ldcp->tx_lock, NULL, MUTEX_DRIVER, NULL); 2259 2260 /* Initialize the channel */ 2261 ldcp->id = id; 2262 ldcp->cb = NULL; 2263 ldcp->cb_arg = NULL; 2264 ldcp->cb_inprogress = B_FALSE; 2265 ldcp->cb_enabled = B_FALSE; 2266 ldcp->next = NULL; 2267 2268 /* Read attributes */ 2269 ldcp->mode = attr->mode; 2270 ldcp->devclass = attr->devclass; 2271 ldcp->devinst = attr->instance; 2272 ldcp->mtu = (attr->mtu > 0) ? attr->mtu : LDC_DEFAULT_MTU; 2273 2274 D1(ldcp->id, 2275 "ldc_init: (0x%llx) channel attributes, class=0x%x, " 2276 "instance=0x%llx, mode=%d, mtu=%d\n", 2277 ldcp->id, ldcp->devclass, ldcp->devinst, ldcp->mode, ldcp->mtu); 2278 2279 ldcp->next_vidx = 0; 2280 ldcp->tstate = TS_IN_RESET; 2281 ldcp->hstate = 0; 2282 ldcp->last_msg_snt = LDC_INIT_SEQID; 2283 ldcp->last_ack_rcd = 0; 2284 ldcp->last_msg_rcd = 0; 2285 2286 ldcp->stream_bufferp = NULL; 2287 ldcp->exp_dring_list = NULL; 2288 ldcp->imp_dring_list = NULL; 2289 ldcp->mhdl_list = NULL; 2290 2291 ldcp->tx_intr_state = LDC_INTR_NONE; 2292 ldcp->rx_intr_state = LDC_INTR_NONE; 2293 2294 /* Initialize payload size depending on whether channel is reliable */ 2295 switch (ldcp->mode) { 2296 case LDC_MODE_RAW: 2297 ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RAW; 2298 ldcp->read_p = i_ldc_read_raw; 2299 ldcp->write_p = i_ldc_write_raw; 2300 break; 2301 case LDC_MODE_UNRELIABLE: 2302 ldcp->pkt_payload = LDC_PAYLOAD_SIZE_UNRELIABLE; 2303 ldcp->read_p = i_ldc_read_packet; 2304 ldcp->write_p = i_ldc_write_packet; 2305 break; 2306 case LDC_MODE_RELIABLE: 2307 ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RELIABLE; 2308 ldcp->read_p = i_ldc_read_packet; 2309 ldcp->write_p = i_ldc_write_packet; 2310 break; 2311 case LDC_MODE_STREAM: 2312 ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RELIABLE; 2313 2314 ldcp->stream_remains = 0; 2315 ldcp->stream_offset = 0; 2316 ldcp->stream_bufferp = kmem_alloc(ldcp->mtu, KM_SLEEP); 2317 ldcp->read_p = i_ldc_read_stream; 2318 ldcp->write_p = i_ldc_write_stream; 2319 break; 2320 default: 2321 exit_val = EINVAL; 2322 goto cleanup_on_exit; 2323 } 2324 2325 /* 2326 * qlen is (mtu * ldc_mtu_msgs) / pkt_payload. If this 2327 * value is smaller than default length of ldc_queue_entries, 2328 * qlen is set to ldc_queue_entries. Ensure that computed 2329 * length is a power-of-two value. 2330 */ 2331 qlen = (ldcp->mtu * ldc_mtu_msgs) / ldcp->pkt_payload; 2332 if (!ISP2(qlen)) { 2333 uint64_t tmp = 1; 2334 while (qlen) { 2335 qlen >>= 1; tmp <<= 1; 2336 } 2337 qlen = tmp; 2338 } 2339 2340 ldcp->rx_q_entries = 2341 (qlen < ldc_queue_entries) ? ldc_queue_entries : qlen; 2342 ldcp->tx_q_entries = ldcp->rx_q_entries; 2343 2344 D1(ldcp->id, "ldc_init: queue length = 0x%llx\n", ldcp->rx_q_entries); 2345 2346 /* Create a transmit queue */ 2347 ldcp->tx_q_va = (uint64_t) 2348 contig_mem_alloc(ldcp->tx_q_entries << LDC_PACKET_SHIFT); 2349 if (ldcp->tx_q_va == NULL) { 2350 cmn_err(CE_WARN, 2351 "ldc_init: (0x%lx) TX queue allocation failed\n", 2352 ldcp->id); 2353 exit_val = ENOMEM; 2354 goto cleanup_on_exit; 2355 } 2356 ldcp->tx_q_ra = va_to_pa((caddr_t)ldcp->tx_q_va); 2357 2358 D2(ldcp->id, "ldc_init: txq_va=0x%llx, txq_ra=0x%llx, entries=0x%llx\n", 2359 ldcp->tx_q_va, ldcp->tx_q_ra, ldcp->tx_q_entries); 2360 2361 ldcp->tstate |= TS_TXQ_RDY; 2362 2363 /* Create a receive queue */ 2364 ldcp->rx_q_va = (uint64_t) 2365 contig_mem_alloc(ldcp->rx_q_entries << LDC_PACKET_SHIFT); 2366 if (ldcp->rx_q_va == NULL) { 2367 cmn_err(CE_WARN, 2368 "ldc_init: (0x%lx) RX queue allocation failed\n", 2369 ldcp->id); 2370 exit_val = ENOMEM; 2371 goto cleanup_on_exit; 2372 } 2373 ldcp->rx_q_ra = va_to_pa((caddr_t)ldcp->rx_q_va); 2374 2375 D2(ldcp->id, "ldc_init: rxq_va=0x%llx, rxq_ra=0x%llx, entries=0x%llx\n", 2376 ldcp->rx_q_va, ldcp->rx_q_ra, ldcp->rx_q_entries); 2377 2378 ldcp->tstate |= TS_RXQ_RDY; 2379 2380 /* Init descriptor ring and memory handle list lock */ 2381 mutex_init(&ldcp->exp_dlist_lock, NULL, MUTEX_DRIVER, NULL); 2382 mutex_init(&ldcp->imp_dlist_lock, NULL, MUTEX_DRIVER, NULL); 2383 mutex_init(&ldcp->mlist_lock, NULL, MUTEX_DRIVER, NULL); 2384 2385 /* mark status as INITialized */ 2386 ldcp->status = LDC_INIT; 2387 2388 /* Add to channel list */ 2389 mutex_enter(&ldcssp->lock); 2390 ldcp->next = ldcssp->chan_list; 2391 ldcssp->chan_list = ldcp; 2392 ldcssp->channel_count++; 2393 mutex_exit(&ldcssp->lock); 2394 2395 /* set the handle */ 2396 *handle = (ldc_handle_t)ldcp; 2397 2398 D1(ldcp->id, "ldc_init: (0x%llx) channel initialized\n", ldcp->id); 2399 2400 return (0); 2401 2402 cleanup_on_exit: 2403 2404 if (ldcp->mode == LDC_MODE_STREAM && ldcp->stream_bufferp) 2405 kmem_free(ldcp->stream_bufferp, ldcp->mtu); 2406 2407 if (ldcp->tstate & TS_TXQ_RDY) 2408 contig_mem_free((caddr_t)ldcp->tx_q_va, 2409 (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 2410 2411 if (ldcp->tstate & TS_RXQ_RDY) 2412 contig_mem_free((caddr_t)ldcp->rx_q_va, 2413 (ldcp->rx_q_entries << LDC_PACKET_SHIFT)); 2414 2415 mutex_destroy(&ldcp->tx_lock); 2416 mutex_destroy(&ldcp->lock); 2417 2418 if (ldcp) 2419 kmem_free(ldcp, sizeof (ldc_chan_t)); 2420 2421 return (exit_val); 2422 } 2423 2424 /* 2425 * Finalizes the LDC connection. It will return EBUSY if the 2426 * channel is open. A ldc_close() has to be done prior to 2427 * a ldc_fini operation. It frees TX/RX queues, associated 2428 * with the channel 2429 */ 2430 int 2431 ldc_fini(ldc_handle_t handle) 2432 { 2433 ldc_chan_t *ldcp; 2434 ldc_chan_t *tmp_ldcp; 2435 uint64_t id; 2436 2437 if (handle == NULL) { 2438 DWARN(DBG_ALL_LDCS, "ldc_fini: invalid channel handle\n"); 2439 return (EINVAL); 2440 } 2441 ldcp = (ldc_chan_t *)handle; 2442 id = ldcp->id; 2443 2444 mutex_enter(&ldcp->lock); 2445 2446 if ((ldcp->tstate & ~TS_IN_RESET) > TS_INIT) { 2447 DWARN(ldcp->id, "ldc_fini: (0x%llx) channel is open\n", 2448 ldcp->id); 2449 mutex_exit(&ldcp->lock); 2450 return (EBUSY); 2451 } 2452 2453 /* Remove from the channel list */ 2454 mutex_enter(&ldcssp->lock); 2455 tmp_ldcp = ldcssp->chan_list; 2456 if (tmp_ldcp == ldcp) { 2457 ldcssp->chan_list = ldcp->next; 2458 ldcp->next = NULL; 2459 } else { 2460 while (tmp_ldcp != NULL) { 2461 if (tmp_ldcp->next == ldcp) { 2462 tmp_ldcp->next = ldcp->next; 2463 ldcp->next = NULL; 2464 break; 2465 } 2466 tmp_ldcp = tmp_ldcp->next; 2467 } 2468 if (tmp_ldcp == NULL) { 2469 DWARN(DBG_ALL_LDCS, "ldc_fini: invalid channel hdl\n"); 2470 mutex_exit(&ldcssp->lock); 2471 mutex_exit(&ldcp->lock); 2472 return (EINVAL); 2473 } 2474 } 2475 2476 ldcssp->channel_count--; 2477 2478 mutex_exit(&ldcssp->lock); 2479 2480 /* Free the map table for this channel */ 2481 if (ldcp->mtbl) { 2482 (void) hv_ldc_set_map_table(ldcp->id, NULL, NULL); 2483 if (ldcp->mtbl->contigmem) 2484 contig_mem_free(ldcp->mtbl->table, ldcp->mtbl->size); 2485 else 2486 kmem_free(ldcp->mtbl->table, ldcp->mtbl->size); 2487 mutex_destroy(&ldcp->mtbl->lock); 2488 kmem_free(ldcp->mtbl, sizeof (ldc_mtbl_t)); 2489 } 2490 2491 /* Destroy descriptor ring and memory handle list lock */ 2492 mutex_destroy(&ldcp->exp_dlist_lock); 2493 mutex_destroy(&ldcp->imp_dlist_lock); 2494 mutex_destroy(&ldcp->mlist_lock); 2495 2496 /* Free the stream buffer for STREAM_MODE */ 2497 if (ldcp->mode == LDC_MODE_STREAM && ldcp->stream_bufferp) 2498 kmem_free(ldcp->stream_bufferp, ldcp->mtu); 2499 2500 /* Free the RX queue */ 2501 contig_mem_free((caddr_t)ldcp->rx_q_va, 2502 (ldcp->rx_q_entries << LDC_PACKET_SHIFT)); 2503 ldcp->tstate &= ~TS_RXQ_RDY; 2504 2505 /* Free the TX queue */ 2506 contig_mem_free((caddr_t)ldcp->tx_q_va, 2507 (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 2508 ldcp->tstate &= ~TS_TXQ_RDY; 2509 2510 mutex_exit(&ldcp->lock); 2511 2512 /* Destroy mutex */ 2513 mutex_destroy(&ldcp->tx_lock); 2514 mutex_destroy(&ldcp->lock); 2515 2516 /* free channel structure */ 2517 kmem_free(ldcp, sizeof (ldc_chan_t)); 2518 2519 D1(id, "ldc_fini: (0x%llx) channel finalized\n", id); 2520 2521 return (0); 2522 } 2523 2524 /* 2525 * Open the LDC channel for use. It registers the TX/RX queues 2526 * with the Hypervisor. It also specifies the interrupt number 2527 * and target CPU for this channel 2528 */ 2529 int 2530 ldc_open(ldc_handle_t handle) 2531 { 2532 ldc_chan_t *ldcp; 2533 int rv; 2534 2535 if (handle == NULL) { 2536 DWARN(DBG_ALL_LDCS, "ldc_open: invalid channel handle\n"); 2537 return (EINVAL); 2538 } 2539 2540 ldcp = (ldc_chan_t *)handle; 2541 2542 mutex_enter(&ldcp->lock); 2543 2544 if (ldcp->tstate < TS_INIT) { 2545 DWARN(ldcp->id, 2546 "ldc_open: (0x%llx) channel not initialized\n", ldcp->id); 2547 mutex_exit(&ldcp->lock); 2548 return (EFAULT); 2549 } 2550 if ((ldcp->tstate & ~TS_IN_RESET) >= TS_OPEN) { 2551 DWARN(ldcp->id, 2552 "ldc_open: (0x%llx) channel is already open\n", ldcp->id); 2553 mutex_exit(&ldcp->lock); 2554 return (EFAULT); 2555 } 2556 2557 /* 2558 * Unregister/Register the tx queue with the hypervisor 2559 */ 2560 rv = hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 2561 if (rv) { 2562 cmn_err(CE_WARN, 2563 "ldc_open: (0x%lx) channel tx queue unconf failed\n", 2564 ldcp->id); 2565 mutex_exit(&ldcp->lock); 2566 return (EIO); 2567 } 2568 2569 rv = hv_ldc_tx_qconf(ldcp->id, ldcp->tx_q_ra, ldcp->tx_q_entries); 2570 if (rv) { 2571 cmn_err(CE_WARN, 2572 "ldc_open: (0x%lx) channel tx queue conf failed\n", 2573 ldcp->id); 2574 mutex_exit(&ldcp->lock); 2575 return (EIO); 2576 } 2577 2578 D2(ldcp->id, "ldc_open: (0x%llx) registered tx queue with LDC\n", 2579 ldcp->id); 2580 2581 /* 2582 * Unregister/Register the rx queue with the hypervisor 2583 */ 2584 rv = hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 2585 if (rv) { 2586 cmn_err(CE_WARN, 2587 "ldc_open: (0x%lx) channel rx queue unconf failed\n", 2588 ldcp->id); 2589 mutex_exit(&ldcp->lock); 2590 return (EIO); 2591 } 2592 2593 rv = hv_ldc_rx_qconf(ldcp->id, ldcp->rx_q_ra, ldcp->rx_q_entries); 2594 if (rv) { 2595 cmn_err(CE_WARN, 2596 "ldc_open: (0x%lx) channel rx queue conf failed\n", 2597 ldcp->id); 2598 mutex_exit(&ldcp->lock); 2599 return (EIO); 2600 } 2601 2602 D2(ldcp->id, "ldc_open: (0x%llx) registered rx queue with LDC\n", 2603 ldcp->id); 2604 2605 ldcp->tstate |= TS_QCONF_RDY; 2606 2607 /* Register the channel with the channel nexus */ 2608 rv = i_ldc_register_channel(ldcp); 2609 if (rv && rv != EAGAIN) { 2610 cmn_err(CE_WARN, 2611 "ldc_open: (0x%lx) channel register failed\n", ldcp->id); 2612 (void) hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 2613 (void) hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 2614 mutex_exit(&ldcp->lock); 2615 return (EIO); 2616 } 2617 2618 /* mark channel in OPEN state */ 2619 ldcp->status = LDC_OPEN; 2620 2621 /* Read channel state */ 2622 rv = hv_ldc_tx_get_state(ldcp->id, 2623 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 2624 if (rv) { 2625 cmn_err(CE_WARN, 2626 "ldc_open: (0x%lx) cannot read channel state\n", 2627 ldcp->id); 2628 (void) i_ldc_unregister_channel(ldcp); 2629 (void) hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 2630 (void) hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 2631 mutex_exit(&ldcp->lock); 2632 return (EIO); 2633 } 2634 2635 /* 2636 * set the ACKd head to current head location for reliable & 2637 * streaming mode 2638 */ 2639 ldcp->tx_ackd_head = ldcp->tx_head; 2640 2641 /* mark channel ready if HV report link is UP (peer alloc'd Rx queue) */ 2642 if (ldcp->link_state == LDC_CHANNEL_UP || 2643 ldcp->link_state == LDC_CHANNEL_RESET) { 2644 ldcp->tstate |= TS_LINK_READY; 2645 ldcp->status = LDC_READY; 2646 } 2647 2648 /* 2649 * if channel is being opened in RAW mode - no handshake is needed 2650 * switch the channel READY and UP state 2651 */ 2652 if (ldcp->mode == LDC_MODE_RAW) { 2653 ldcp->tstate = TS_UP; /* set bits associated with LDC UP */ 2654 ldcp->status = LDC_UP; 2655 } 2656 2657 mutex_exit(&ldcp->lock); 2658 2659 /* 2660 * Increment number of open channels 2661 */ 2662 mutex_enter(&ldcssp->lock); 2663 ldcssp->channels_open++; 2664 mutex_exit(&ldcssp->lock); 2665 2666 D1(ldcp->id, 2667 "ldc_open: (0x%llx) channel (0x%p) open for use " 2668 "(tstate=0x%x, status=0x%x)\n", 2669 ldcp->id, ldcp, ldcp->tstate, ldcp->status); 2670 2671 return (0); 2672 } 2673 2674 /* 2675 * Close the LDC connection. It will return EBUSY if there 2676 * are memory segments or descriptor rings either bound to or 2677 * mapped over the channel 2678 */ 2679 int 2680 ldc_close(ldc_handle_t handle) 2681 { 2682 ldc_chan_t *ldcp; 2683 int rv = 0, retries = 0; 2684 boolean_t chk_done = B_FALSE; 2685 2686 if (handle == NULL) { 2687 DWARN(DBG_ALL_LDCS, "ldc_close: invalid channel handle\n"); 2688 return (EINVAL); 2689 } 2690 ldcp = (ldc_chan_t *)handle; 2691 2692 mutex_enter(&ldcp->lock); 2693 2694 /* return error if channel is not open */ 2695 if ((ldcp->tstate & ~TS_IN_RESET) < TS_OPEN) { 2696 DWARN(ldcp->id, 2697 "ldc_close: (0x%llx) channel is not open\n", ldcp->id); 2698 mutex_exit(&ldcp->lock); 2699 return (EFAULT); 2700 } 2701 2702 /* if any memory handles, drings, are bound or mapped cannot close */ 2703 if (ldcp->mhdl_list != NULL) { 2704 DWARN(ldcp->id, 2705 "ldc_close: (0x%llx) channel has bound memory handles\n", 2706 ldcp->id); 2707 mutex_exit(&ldcp->lock); 2708 return (EBUSY); 2709 } 2710 if (ldcp->exp_dring_list != NULL) { 2711 DWARN(ldcp->id, 2712 "ldc_close: (0x%llx) channel has bound descriptor rings\n", 2713 ldcp->id); 2714 mutex_exit(&ldcp->lock); 2715 return (EBUSY); 2716 } 2717 if (ldcp->imp_dring_list != NULL) { 2718 DWARN(ldcp->id, 2719 "ldc_close: (0x%llx) channel has mapped descriptor rings\n", 2720 ldcp->id); 2721 mutex_exit(&ldcp->lock); 2722 return (EBUSY); 2723 } 2724 2725 if (ldcp->cb_inprogress) { 2726 DWARN(ldcp->id, "ldc_close: (0x%llx) callback active\n", 2727 ldcp->id); 2728 mutex_exit(&ldcp->lock); 2729 return (EWOULDBLOCK); 2730 } 2731 2732 /* Obtain Tx lock */ 2733 mutex_enter(&ldcp->tx_lock); 2734 2735 /* 2736 * Wait for pending transmits to complete i.e Tx queue to drain 2737 * if there are pending pkts - wait 1 ms and retry again 2738 */ 2739 for (;;) { 2740 2741 rv = hv_ldc_tx_get_state(ldcp->id, 2742 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 2743 if (rv) { 2744 cmn_err(CE_WARN, 2745 "ldc_close: (0x%lx) cannot read qptrs\n", ldcp->id); 2746 mutex_exit(&ldcp->tx_lock); 2747 mutex_exit(&ldcp->lock); 2748 return (EIO); 2749 } 2750 2751 if (ldcp->tx_head == ldcp->tx_tail || 2752 ldcp->link_state != LDC_CHANNEL_UP) { 2753 break; 2754 } 2755 2756 if (chk_done) { 2757 DWARN(ldcp->id, 2758 "ldc_close: (0x%llx) Tx queue drain timeout\n", 2759 ldcp->id); 2760 break; 2761 } 2762 2763 /* wait for one ms and try again */ 2764 delay(drv_usectohz(1000)); 2765 chk_done = B_TRUE; 2766 } 2767 2768 /* 2769 * Drain the Tx and Rx queues as we are closing the 2770 * channel. We dont care about any pending packets. 2771 * We have to also drain the queue prior to clearing 2772 * pending interrupts, otherwise the HV will trigger 2773 * an interrupt the moment the interrupt state is 2774 * cleared. 2775 */ 2776 (void) i_ldc_txq_reconf(ldcp); 2777 (void) i_ldc_rxq_drain(ldcp); 2778 2779 /* 2780 * Unregister the channel with the nexus 2781 */ 2782 while ((rv = i_ldc_unregister_channel(ldcp)) != 0) { 2783 2784 mutex_exit(&ldcp->tx_lock); 2785 mutex_exit(&ldcp->lock); 2786 2787 /* if any error other than EAGAIN return back */ 2788 if (rv != EAGAIN || retries >= ldc_max_retries) { 2789 cmn_err(CE_WARN, 2790 "ldc_close: (0x%lx) unregister failed, %d\n", 2791 ldcp->id, rv); 2792 return (rv); 2793 } 2794 2795 /* 2796 * As there could be pending interrupts we need 2797 * to wait and try again 2798 */ 2799 drv_usecwait(ldc_close_delay); 2800 mutex_enter(&ldcp->lock); 2801 mutex_enter(&ldcp->tx_lock); 2802 retries++; 2803 } 2804 2805 /* 2806 * Unregister queues 2807 */ 2808 rv = hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 2809 if (rv) { 2810 cmn_err(CE_WARN, 2811 "ldc_close: (0x%lx) channel TX queue unconf failed\n", 2812 ldcp->id); 2813 mutex_exit(&ldcp->tx_lock); 2814 mutex_exit(&ldcp->lock); 2815 return (EIO); 2816 } 2817 rv = hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 2818 if (rv) { 2819 cmn_err(CE_WARN, 2820 "ldc_close: (0x%lx) channel RX queue unconf failed\n", 2821 ldcp->id); 2822 mutex_exit(&ldcp->tx_lock); 2823 mutex_exit(&ldcp->lock); 2824 return (EIO); 2825 } 2826 2827 ldcp->tstate &= ~TS_QCONF_RDY; 2828 2829 /* Reset channel state information */ 2830 i_ldc_reset_state(ldcp); 2831 2832 /* Mark channel as down and in initialized state */ 2833 ldcp->tx_ackd_head = 0; 2834 ldcp->tx_head = 0; 2835 ldcp->tstate = TS_IN_RESET|TS_INIT; 2836 ldcp->status = LDC_INIT; 2837 2838 mutex_exit(&ldcp->tx_lock); 2839 mutex_exit(&ldcp->lock); 2840 2841 /* Decrement number of open channels */ 2842 mutex_enter(&ldcssp->lock); 2843 ldcssp->channels_open--; 2844 mutex_exit(&ldcssp->lock); 2845 2846 D1(ldcp->id, "ldc_close: (0x%llx) channel closed\n", ldcp->id); 2847 2848 return (0); 2849 } 2850 2851 /* 2852 * Register channel callback 2853 */ 2854 int 2855 ldc_reg_callback(ldc_handle_t handle, 2856 uint_t(*cb)(uint64_t event, caddr_t arg), caddr_t arg) 2857 { 2858 ldc_chan_t *ldcp; 2859 2860 if (handle == NULL) { 2861 DWARN(DBG_ALL_LDCS, 2862 "ldc_reg_callback: invalid channel handle\n"); 2863 return (EINVAL); 2864 } 2865 if (((uint64_t)cb) < KERNELBASE) { 2866 DWARN(DBG_ALL_LDCS, "ldc_reg_callback: invalid callback\n"); 2867 return (EINVAL); 2868 } 2869 ldcp = (ldc_chan_t *)handle; 2870 2871 mutex_enter(&ldcp->lock); 2872 2873 if (ldcp->cb) { 2874 DWARN(ldcp->id, "ldc_reg_callback: (0x%llx) callback exists\n", 2875 ldcp->id); 2876 mutex_exit(&ldcp->lock); 2877 return (EIO); 2878 } 2879 if (ldcp->cb_inprogress) { 2880 DWARN(ldcp->id, "ldc_reg_callback: (0x%llx) callback active\n", 2881 ldcp->id); 2882 mutex_exit(&ldcp->lock); 2883 return (EWOULDBLOCK); 2884 } 2885 2886 ldcp->cb = cb; 2887 ldcp->cb_arg = arg; 2888 ldcp->cb_enabled = B_TRUE; 2889 2890 D1(ldcp->id, 2891 "ldc_reg_callback: (0x%llx) registered callback for channel\n", 2892 ldcp->id); 2893 2894 mutex_exit(&ldcp->lock); 2895 2896 return (0); 2897 } 2898 2899 /* 2900 * Unregister channel callback 2901 */ 2902 int 2903 ldc_unreg_callback(ldc_handle_t handle) 2904 { 2905 ldc_chan_t *ldcp; 2906 2907 if (handle == NULL) { 2908 DWARN(DBG_ALL_LDCS, 2909 "ldc_unreg_callback: invalid channel handle\n"); 2910 return (EINVAL); 2911 } 2912 ldcp = (ldc_chan_t *)handle; 2913 2914 mutex_enter(&ldcp->lock); 2915 2916 if (ldcp->cb == NULL) { 2917 DWARN(ldcp->id, 2918 "ldc_unreg_callback: (0x%llx) no callback exists\n", 2919 ldcp->id); 2920 mutex_exit(&ldcp->lock); 2921 return (EIO); 2922 } 2923 if (ldcp->cb_inprogress) { 2924 DWARN(ldcp->id, 2925 "ldc_unreg_callback: (0x%llx) callback active\n", 2926 ldcp->id); 2927 mutex_exit(&ldcp->lock); 2928 return (EWOULDBLOCK); 2929 } 2930 2931 ldcp->cb = NULL; 2932 ldcp->cb_arg = NULL; 2933 ldcp->cb_enabled = B_FALSE; 2934 2935 D1(ldcp->id, 2936 "ldc_unreg_callback: (0x%llx) unregistered callback for channel\n", 2937 ldcp->id); 2938 2939 mutex_exit(&ldcp->lock); 2940 2941 return (0); 2942 } 2943 2944 2945 /* 2946 * Bring a channel up by initiating a handshake with the peer 2947 * This call is asynchronous. It will complete at a later point 2948 * in time when the peer responds back with an RTR. 2949 */ 2950 int 2951 ldc_up(ldc_handle_t handle) 2952 { 2953 int rv; 2954 ldc_chan_t *ldcp; 2955 ldc_msg_t *ldcmsg; 2956 uint64_t tx_tail, tstate, link_state; 2957 2958 if (handle == NULL) { 2959 DWARN(DBG_ALL_LDCS, "ldc_up: invalid channel handle\n"); 2960 return (EINVAL); 2961 } 2962 ldcp = (ldc_chan_t *)handle; 2963 2964 mutex_enter(&ldcp->lock); 2965 2966 D1(ldcp->id, "ldc_up: (0x%llx) doing channel UP\n", ldcp->id); 2967 2968 /* clear the reset state */ 2969 tstate = ldcp->tstate; 2970 ldcp->tstate &= ~TS_IN_RESET; 2971 2972 if (ldcp->tstate == TS_UP) { 2973 DWARN(ldcp->id, 2974 "ldc_up: (0x%llx) channel is already in UP state\n", 2975 ldcp->id); 2976 2977 /* mark channel as up */ 2978 ldcp->status = LDC_UP; 2979 2980 /* 2981 * if channel was in reset state and there was 2982 * pending data clear interrupt state. this will 2983 * trigger an interrupt, causing the RX handler to 2984 * to invoke the client's callback 2985 */ 2986 if ((tstate & TS_IN_RESET) && 2987 ldcp->rx_intr_state == LDC_INTR_PEND) { 2988 D1(ldcp->id, 2989 "ldc_up: (0x%llx) channel has pending data, " 2990 "clearing interrupt\n", ldcp->id); 2991 i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 2992 } 2993 2994 mutex_exit(&ldcp->lock); 2995 return (0); 2996 } 2997 2998 /* if the channel is in RAW mode - mark it as UP, if READY */ 2999 if (ldcp->mode == LDC_MODE_RAW && ldcp->tstate >= TS_READY) { 3000 ldcp->tstate = TS_UP; 3001 mutex_exit(&ldcp->lock); 3002 return (0); 3003 } 3004 3005 /* Don't start another handshake if there is one in progress */ 3006 if (ldcp->hstate) { 3007 D1(ldcp->id, 3008 "ldc_up: (0x%llx) channel handshake in progress\n", 3009 ldcp->id); 3010 mutex_exit(&ldcp->lock); 3011 return (0); 3012 } 3013 3014 mutex_enter(&ldcp->tx_lock); 3015 3016 /* save current link state */ 3017 link_state = ldcp->link_state; 3018 3019 /* get the current tail for the LDC msg */ 3020 rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 3021 if (rv) { 3022 D1(ldcp->id, "ldc_up: (0x%llx) cannot initiate handshake\n", 3023 ldcp->id); 3024 mutex_exit(&ldcp->tx_lock); 3025 mutex_exit(&ldcp->lock); 3026 return (ECONNREFUSED); 3027 } 3028 3029 /* 3030 * If i_ldc_get_tx_tail() changed link_state to either RESET or UP, 3031 * from a previous state of DOWN, then mark the channel as 3032 * being ready for handshake. 3033 */ 3034 if ((link_state == LDC_CHANNEL_DOWN) && 3035 (link_state != ldcp->link_state)) { 3036 3037 ASSERT((ldcp->link_state == LDC_CHANNEL_RESET) || 3038 (ldcp->link_state == LDC_CHANNEL_UP)); 3039 3040 if (ldcp->mode == LDC_MODE_RAW) { 3041 ldcp->status = LDC_UP; 3042 ldcp->tstate = TS_UP; 3043 mutex_exit(&ldcp->tx_lock); 3044 mutex_exit(&ldcp->lock); 3045 return (0); 3046 } else { 3047 ldcp->status = LDC_READY; 3048 ldcp->tstate |= TS_LINK_READY; 3049 } 3050 3051 } 3052 3053 ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 3054 ZERO_PKT(ldcmsg); 3055 3056 ldcmsg->type = LDC_CTRL; 3057 ldcmsg->stype = LDC_INFO; 3058 ldcmsg->ctrl = LDC_VER; 3059 ldcp->next_vidx = 0; 3060 bcopy(&ldc_versions[0], ldcmsg->udata, sizeof (ldc_versions[0])); 3061 3062 DUMP_LDC_PKT(ldcp, "ldc_up snd ver", (uint64_t)ldcmsg); 3063 3064 /* initiate the send by calling into HV and set the new tail */ 3065 tx_tail = (tx_tail + LDC_PACKET_SIZE) % 3066 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 3067 3068 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 3069 if (rv) { 3070 DWARN(ldcp->id, 3071 "ldc_up: (0x%llx) cannot initiate handshake rv=%d\n", 3072 ldcp->id, rv); 3073 mutex_exit(&ldcp->tx_lock); 3074 mutex_exit(&ldcp->lock); 3075 return (rv); 3076 } 3077 3078 ldcp->hstate |= TS_SENT_VER; 3079 ldcp->tx_tail = tx_tail; 3080 D1(ldcp->id, "ldc_up: (0x%llx) channel up initiated\n", ldcp->id); 3081 3082 mutex_exit(&ldcp->tx_lock); 3083 mutex_exit(&ldcp->lock); 3084 3085 return (rv); 3086 } 3087 3088 3089 /* 3090 * Bring a channel down by resetting its state and queues 3091 */ 3092 int 3093 ldc_down(ldc_handle_t handle) 3094 { 3095 ldc_chan_t *ldcp; 3096 3097 if (handle == NULL) { 3098 DWARN(DBG_ALL_LDCS, "ldc_down: invalid channel handle\n"); 3099 return (EINVAL); 3100 } 3101 ldcp = (ldc_chan_t *)handle; 3102 mutex_enter(&ldcp->lock); 3103 mutex_enter(&ldcp->tx_lock); 3104 i_ldc_reset(ldcp, B_TRUE); 3105 mutex_exit(&ldcp->tx_lock); 3106 mutex_exit(&ldcp->lock); 3107 3108 return (0); 3109 } 3110 3111 /* 3112 * Get the current channel status 3113 */ 3114 int 3115 ldc_status(ldc_handle_t handle, ldc_status_t *status) 3116 { 3117 ldc_chan_t *ldcp; 3118 3119 if (handle == NULL || status == NULL) { 3120 DWARN(DBG_ALL_LDCS, "ldc_status: invalid argument\n"); 3121 return (EINVAL); 3122 } 3123 ldcp = (ldc_chan_t *)handle; 3124 3125 *status = ((ldc_chan_t *)handle)->status; 3126 3127 D1(ldcp->id, 3128 "ldc_status: (0x%llx) returned status %d\n", ldcp->id, *status); 3129 return (0); 3130 } 3131 3132 3133 /* 3134 * Set the channel's callback mode - enable/disable callbacks 3135 */ 3136 int 3137 ldc_set_cb_mode(ldc_handle_t handle, ldc_cb_mode_t cmode) 3138 { 3139 ldc_chan_t *ldcp; 3140 3141 if (handle == NULL) { 3142 DWARN(DBG_ALL_LDCS, 3143 "ldc_set_intr_mode: invalid channel handle\n"); 3144 return (EINVAL); 3145 } 3146 ldcp = (ldc_chan_t *)handle; 3147 3148 /* 3149 * Record no callbacks should be invoked 3150 */ 3151 mutex_enter(&ldcp->lock); 3152 3153 switch (cmode) { 3154 case LDC_CB_DISABLE: 3155 if (!ldcp->cb_enabled) { 3156 DWARN(ldcp->id, 3157 "ldc_set_cb_mode: (0x%llx) callbacks disabled\n", 3158 ldcp->id); 3159 break; 3160 } 3161 ldcp->cb_enabled = B_FALSE; 3162 3163 D1(ldcp->id, "ldc_set_cb_mode: (0x%llx) disabled callbacks\n", 3164 ldcp->id); 3165 break; 3166 3167 case LDC_CB_ENABLE: 3168 if (ldcp->cb_enabled) { 3169 DWARN(ldcp->id, 3170 "ldc_set_cb_mode: (0x%llx) callbacks enabled\n", 3171 ldcp->id); 3172 break; 3173 } 3174 ldcp->cb_enabled = B_TRUE; 3175 3176 D1(ldcp->id, "ldc_set_cb_mode: (0x%llx) enabled callbacks\n", 3177 ldcp->id); 3178 break; 3179 } 3180 3181 mutex_exit(&ldcp->lock); 3182 3183 return (0); 3184 } 3185 3186 /* 3187 * Check to see if there are packets on the incoming queue 3188 * Will return hasdata = B_FALSE if there are no packets 3189 */ 3190 int 3191 ldc_chkq(ldc_handle_t handle, boolean_t *hasdata) 3192 { 3193 int rv; 3194 uint64_t rx_head, rx_tail; 3195 ldc_chan_t *ldcp; 3196 3197 if (handle == NULL) { 3198 DWARN(DBG_ALL_LDCS, "ldc_chkq: invalid channel handle\n"); 3199 return (EINVAL); 3200 } 3201 ldcp = (ldc_chan_t *)handle; 3202 3203 *hasdata = B_FALSE; 3204 3205 mutex_enter(&ldcp->lock); 3206 3207 if (ldcp->tstate != TS_UP) { 3208 D1(ldcp->id, 3209 "ldc_chkq: (0x%llx) channel is not up\n", ldcp->id); 3210 mutex_exit(&ldcp->lock); 3211 return (ECONNRESET); 3212 } 3213 3214 /* Read packet(s) from the queue */ 3215 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 3216 &ldcp->link_state); 3217 if (rv != 0) { 3218 cmn_err(CE_WARN, 3219 "ldc_chkq: (0x%lx) unable to read queue ptrs", ldcp->id); 3220 mutex_exit(&ldcp->lock); 3221 return (EIO); 3222 } 3223 /* reset the channel state if the channel went down */ 3224 if (ldcp->link_state == LDC_CHANNEL_DOWN || 3225 ldcp->link_state == LDC_CHANNEL_RESET) { 3226 mutex_enter(&ldcp->tx_lock); 3227 i_ldc_reset(ldcp, B_FALSE); 3228 mutex_exit(&ldcp->tx_lock); 3229 mutex_exit(&ldcp->lock); 3230 return (ECONNRESET); 3231 } 3232 3233 if ((rx_head != rx_tail) || 3234 (ldcp->mode == LDC_MODE_STREAM && ldcp->stream_remains > 0)) { 3235 D1(ldcp->id, 3236 "ldc_chkq: (0x%llx) queue has pkt(s) or buffered data\n", 3237 ldcp->id); 3238 *hasdata = B_TRUE; 3239 } 3240 3241 mutex_exit(&ldcp->lock); 3242 3243 return (0); 3244 } 3245 3246 3247 /* 3248 * Read 'size' amount of bytes or less. If incoming buffer 3249 * is more than 'size', ENOBUFS is returned. 3250 * 3251 * On return, size contains the number of bytes read. 3252 */ 3253 int 3254 ldc_read(ldc_handle_t handle, caddr_t bufp, size_t *sizep) 3255 { 3256 ldc_chan_t *ldcp; 3257 uint64_t rx_head = 0, rx_tail = 0; 3258 int rv = 0, exit_val; 3259 3260 if (handle == NULL) { 3261 DWARN(DBG_ALL_LDCS, "ldc_read: invalid channel handle\n"); 3262 return (EINVAL); 3263 } 3264 3265 ldcp = (ldc_chan_t *)handle; 3266 3267 /* channel lock */ 3268 mutex_enter(&ldcp->lock); 3269 3270 if (ldcp->tstate != TS_UP) { 3271 DWARN(ldcp->id, 3272 "ldc_read: (0x%llx) channel is not in UP state\n", 3273 ldcp->id); 3274 exit_val = ECONNRESET; 3275 } else { 3276 exit_val = ldcp->read_p(ldcp, bufp, sizep); 3277 } 3278 3279 /* 3280 * if queue has been drained - clear interrupt 3281 */ 3282 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 3283 &ldcp->link_state); 3284 if (rv != 0) { 3285 cmn_err(CE_WARN, "ldc_read: (0x%lx) unable to read queue ptrs", 3286 ldcp->id); 3287 mutex_enter(&ldcp->tx_lock); 3288 i_ldc_reset(ldcp, B_TRUE); 3289 mutex_exit(&ldcp->tx_lock); 3290 mutex_exit(&ldcp->lock); 3291 return (ECONNRESET); 3292 } 3293 3294 if (exit_val == 0) { 3295 if (ldcp->link_state == LDC_CHANNEL_DOWN || 3296 ldcp->link_state == LDC_CHANNEL_RESET) { 3297 mutex_enter(&ldcp->tx_lock); 3298 i_ldc_reset(ldcp, B_FALSE); 3299 exit_val = ECONNRESET; 3300 mutex_exit(&ldcp->tx_lock); 3301 } 3302 if ((rv == 0) && 3303 (ldcp->rx_intr_state == LDC_INTR_PEND) && 3304 (rx_head == rx_tail)) { 3305 i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 3306 } 3307 } 3308 3309 mutex_exit(&ldcp->lock); 3310 return (exit_val); 3311 } 3312 3313 /* 3314 * Basic raw mondo read - 3315 * no interpretation of mondo contents at all. 3316 * 3317 * Enter and exit with ldcp->lock held by caller 3318 */ 3319 static int 3320 i_ldc_read_raw(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 3321 { 3322 uint64_t q_size_mask; 3323 ldc_msg_t *msgp; 3324 uint8_t *msgbufp; 3325 int rv = 0, space; 3326 uint64_t rx_head, rx_tail; 3327 3328 space = *sizep; 3329 3330 if (space < LDC_PAYLOAD_SIZE_RAW) 3331 return (ENOBUFS); 3332 3333 ASSERT(mutex_owned(&ldcp->lock)); 3334 3335 /* compute mask for increment */ 3336 q_size_mask = (ldcp->rx_q_entries-1)<<LDC_PACKET_SHIFT; 3337 3338 /* 3339 * Read packet(s) from the queue 3340 */ 3341 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 3342 &ldcp->link_state); 3343 if (rv != 0) { 3344 cmn_err(CE_WARN, 3345 "ldc_read_raw: (0x%lx) unable to read queue ptrs", 3346 ldcp->id); 3347 return (EIO); 3348 } 3349 D1(ldcp->id, "ldc_read_raw: (0x%llx) rxh=0x%llx," 3350 " rxt=0x%llx, st=0x%llx\n", 3351 ldcp->id, rx_head, rx_tail, ldcp->link_state); 3352 3353 /* reset the channel state if the channel went down */ 3354 if (ldcp->link_state == LDC_CHANNEL_DOWN || 3355 ldcp->link_state == LDC_CHANNEL_RESET) { 3356 mutex_enter(&ldcp->tx_lock); 3357 i_ldc_reset(ldcp, B_FALSE); 3358 mutex_exit(&ldcp->tx_lock); 3359 return (ECONNRESET); 3360 } 3361 3362 /* 3363 * Check for empty queue 3364 */ 3365 if (rx_head == rx_tail) { 3366 *sizep = 0; 3367 return (0); 3368 } 3369 3370 /* get the message */ 3371 msgp = (ldc_msg_t *)(ldcp->rx_q_va + rx_head); 3372 3373 /* if channel is in RAW mode, copy data and return */ 3374 msgbufp = (uint8_t *)&(msgp->raw[0]); 3375 3376 bcopy(msgbufp, target_bufp, LDC_PAYLOAD_SIZE_RAW); 3377 3378 DUMP_PAYLOAD(ldcp->id, msgbufp); 3379 3380 *sizep = LDC_PAYLOAD_SIZE_RAW; 3381 3382 rx_head = (rx_head + LDC_PACKET_SIZE) & q_size_mask; 3383 rv = i_ldc_set_rx_head(ldcp, rx_head); 3384 3385 return (rv); 3386 } 3387 3388 /* 3389 * Process LDC mondos to build larger packets 3390 * with either un-reliable or reliable delivery. 3391 * 3392 * Enter and exit with ldcp->lock held by caller 3393 */ 3394 static int 3395 i_ldc_read_packet(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 3396 { 3397 int rv = 0; 3398 uint64_t rx_head = 0, rx_tail = 0; 3399 uint64_t curr_head = 0; 3400 ldc_msg_t *msg; 3401 caddr_t target; 3402 size_t len = 0, bytes_read = 0; 3403 int retries = 0; 3404 uint64_t q_size_mask; 3405 uint64_t first_fragment = 0; 3406 3407 target = target_bufp; 3408 3409 ASSERT(mutex_owned(&ldcp->lock)); 3410 3411 /* check if the buffer and size are valid */ 3412 if (target_bufp == NULL || *sizep == 0) { 3413 DWARN(ldcp->id, "ldc_read: (0x%llx) invalid buffer/size\n", 3414 ldcp->id); 3415 return (EINVAL); 3416 } 3417 3418 /* compute mask for increment */ 3419 q_size_mask = (ldcp->rx_q_entries-1)<<LDC_PACKET_SHIFT; 3420 3421 /* 3422 * Read packet(s) from the queue 3423 */ 3424 rv = hv_ldc_rx_get_state(ldcp->id, &curr_head, &rx_tail, 3425 &ldcp->link_state); 3426 if (rv != 0) { 3427 cmn_err(CE_WARN, "ldc_read: (0x%lx) unable to read queue ptrs", 3428 ldcp->id); 3429 mutex_enter(&ldcp->tx_lock); 3430 i_ldc_reset(ldcp, B_TRUE); 3431 mutex_exit(&ldcp->tx_lock); 3432 return (ECONNRESET); 3433 } 3434 D1(ldcp->id, "ldc_read: (0x%llx) chd=0x%llx, tl=0x%llx, st=0x%llx\n", 3435 ldcp->id, curr_head, rx_tail, ldcp->link_state); 3436 3437 /* reset the channel state if the channel went down */ 3438 if (ldcp->link_state != LDC_CHANNEL_UP) 3439 goto channel_is_reset; 3440 3441 for (;;) { 3442 3443 if (curr_head == rx_tail) { 3444 rv = hv_ldc_rx_get_state(ldcp->id, 3445 &rx_head, &rx_tail, &ldcp->link_state); 3446 if (rv != 0) { 3447 cmn_err(CE_WARN, 3448 "ldc_read: (0x%lx) cannot read queue ptrs", 3449 ldcp->id); 3450 mutex_enter(&ldcp->tx_lock); 3451 i_ldc_reset(ldcp, B_TRUE); 3452 mutex_exit(&ldcp->tx_lock); 3453 return (ECONNRESET); 3454 } 3455 if (ldcp->link_state != LDC_CHANNEL_UP) 3456 goto channel_is_reset; 3457 3458 if (curr_head == rx_tail) { 3459 3460 /* If in the middle of a fragmented xfer */ 3461 if (first_fragment != 0) { 3462 3463 /* wait for ldc_delay usecs */ 3464 drv_usecwait(ldc_delay); 3465 3466 if (++retries < ldc_max_retries) 3467 continue; 3468 3469 *sizep = 0; 3470 ldcp->last_msg_rcd = first_fragment - 1; 3471 DWARN(DBG_ALL_LDCS, "ldc_read: " 3472 "(0x%llx) read timeout", ldcp->id); 3473 return (EAGAIN); 3474 } 3475 *sizep = 0; 3476 break; 3477 } 3478 } 3479 retries = 0; 3480 3481 D2(ldcp->id, 3482 "ldc_read: (0x%llx) chd=0x%llx, rxhd=0x%llx, rxtl=0x%llx\n", 3483 ldcp->id, curr_head, rx_head, rx_tail); 3484 3485 /* get the message */ 3486 msg = (ldc_msg_t *)(ldcp->rx_q_va + curr_head); 3487 3488 DUMP_LDC_PKT(ldcp, "ldc_read received pkt", 3489 ldcp->rx_q_va + curr_head); 3490 3491 /* Check the message ID for the message received */ 3492 if ((rv = i_ldc_check_seqid(ldcp, msg)) != 0) { 3493 3494 DWARN(ldcp->id, "ldc_read: (0x%llx) seqid error, " 3495 "q_ptrs=0x%lx,0x%lx", ldcp->id, rx_head, rx_tail); 3496 3497 /* throw away data */ 3498 bytes_read = 0; 3499 3500 /* Reset last_msg_rcd to start of message */ 3501 if (first_fragment != 0) { 3502 ldcp->last_msg_rcd = first_fragment - 1; 3503 first_fragment = 0; 3504 } 3505 /* 3506 * Send a NACK -- invalid seqid 3507 * get the current tail for the response 3508 */ 3509 rv = i_ldc_send_pkt(ldcp, msg->type, LDC_NACK, 3510 (msg->ctrl & LDC_CTRL_MASK)); 3511 if (rv) { 3512 cmn_err(CE_NOTE, 3513 "ldc_read: (0x%lx) err sending " 3514 "NACK msg\n", ldcp->id); 3515 3516 /* if cannot send NACK - reset channel */ 3517 mutex_enter(&ldcp->tx_lock); 3518 i_ldc_reset(ldcp, B_FALSE); 3519 mutex_exit(&ldcp->tx_lock); 3520 rv = ECONNRESET; 3521 break; 3522 } 3523 3524 /* purge receive queue */ 3525 rv = i_ldc_set_rx_head(ldcp, rx_tail); 3526 3527 break; 3528 } 3529 3530 /* 3531 * Process any messages of type CTRL messages 3532 * Future implementations should try to pass these 3533 * to LDC link by resetting the intr state. 3534 * 3535 * NOTE: not done as a switch() as type can be both ctrl+data 3536 */ 3537 if (msg->type & LDC_CTRL) { 3538 if (rv = i_ldc_ctrlmsg(ldcp, msg)) { 3539 if (rv == EAGAIN) 3540 continue; 3541 rv = i_ldc_set_rx_head(ldcp, rx_tail); 3542 *sizep = 0; 3543 bytes_read = 0; 3544 break; 3545 } 3546 } 3547 3548 /* process data ACKs */ 3549 if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 3550 if (rv = i_ldc_process_data_ACK(ldcp, msg)) { 3551 *sizep = 0; 3552 bytes_read = 0; 3553 break; 3554 } 3555 } 3556 3557 /* process data NACKs */ 3558 if ((msg->type & LDC_DATA) && (msg->stype & LDC_NACK)) { 3559 DWARN(ldcp->id, 3560 "ldc_read: (0x%llx) received DATA/NACK", ldcp->id); 3561 mutex_enter(&ldcp->tx_lock); 3562 i_ldc_reset(ldcp, B_TRUE); 3563 mutex_exit(&ldcp->tx_lock); 3564 return (ECONNRESET); 3565 } 3566 3567 /* process data messages */ 3568 if ((msg->type & LDC_DATA) && (msg->stype & LDC_INFO)) { 3569 3570 uint8_t *msgbuf = (uint8_t *)( 3571 (ldcp->mode == LDC_MODE_RELIABLE || 3572 ldcp->mode == LDC_MODE_STREAM) ? 3573 msg->rdata : msg->udata); 3574 3575 D2(ldcp->id, 3576 "ldc_read: (0x%llx) received data msg\n", ldcp->id); 3577 3578 /* get the packet length */ 3579 len = (msg->env & LDC_LEN_MASK); 3580 3581 /* 3582 * FUTURE OPTIMIZATION: 3583 * dont need to set q head for every 3584 * packet we read just need to do this when 3585 * we are done or need to wait for more 3586 * mondos to make a full packet - this is 3587 * currently expensive. 3588 */ 3589 3590 if (first_fragment == 0) { 3591 3592 /* 3593 * first packets should always have the start 3594 * bit set (even for a single packet). If not 3595 * throw away the packet 3596 */ 3597 if (!(msg->env & LDC_FRAG_START)) { 3598 3599 DWARN(DBG_ALL_LDCS, 3600 "ldc_read: (0x%llx) not start - " 3601 "frag=%x\n", ldcp->id, 3602 (msg->env) & LDC_FRAG_MASK); 3603 3604 /* toss pkt, inc head, cont reading */ 3605 bytes_read = 0; 3606 target = target_bufp; 3607 curr_head = 3608 (curr_head + LDC_PACKET_SIZE) 3609 & q_size_mask; 3610 if (rv = i_ldc_set_rx_head(ldcp, 3611 curr_head)) 3612 break; 3613 3614 continue; 3615 } 3616 3617 first_fragment = msg->seqid; 3618 } else { 3619 /* check to see if this is a pkt w/ START bit */ 3620 if (msg->env & LDC_FRAG_START) { 3621 DWARN(DBG_ALL_LDCS, 3622 "ldc_read:(0x%llx) unexpected pkt" 3623 " env=0x%x discarding %d bytes," 3624 " lastmsg=%d, currentmsg=%d\n", 3625 ldcp->id, msg->env&LDC_FRAG_MASK, 3626 bytes_read, ldcp->last_msg_rcd, 3627 msg->seqid); 3628 3629 /* throw data we have read so far */ 3630 bytes_read = 0; 3631 target = target_bufp; 3632 first_fragment = msg->seqid; 3633 3634 if (rv = i_ldc_set_rx_head(ldcp, 3635 curr_head)) 3636 break; 3637 } 3638 } 3639 3640 /* copy (next) pkt into buffer */ 3641 if (len <= (*sizep - bytes_read)) { 3642 bcopy(msgbuf, target, len); 3643 target += len; 3644 bytes_read += len; 3645 } else { 3646 /* 3647 * there is not enough space in the buffer to 3648 * read this pkt. throw message away & continue 3649 * reading data from queue 3650 */ 3651 DWARN(DBG_ALL_LDCS, 3652 "ldc_read: (0x%llx) buffer too small, " 3653 "head=0x%lx, expect=%d, got=%d\n", ldcp->id, 3654 curr_head, *sizep, bytes_read+len); 3655 3656 first_fragment = 0; 3657 target = target_bufp; 3658 bytes_read = 0; 3659 3660 /* throw away everything received so far */ 3661 if (rv = i_ldc_set_rx_head(ldcp, curr_head)) 3662 break; 3663 3664 /* continue reading remaining pkts */ 3665 continue; 3666 } 3667 } 3668 3669 /* set the message id */ 3670 ldcp->last_msg_rcd = msg->seqid; 3671 3672 /* move the head one position */ 3673 curr_head = (curr_head + LDC_PACKET_SIZE) & q_size_mask; 3674 3675 if (msg->env & LDC_FRAG_STOP) { 3676 3677 /* 3678 * All pkts that are part of this fragmented transfer 3679 * have been read or this was a single pkt read 3680 * or there was an error 3681 */ 3682 3683 /* set the queue head */ 3684 if (rv = i_ldc_set_rx_head(ldcp, curr_head)) 3685 bytes_read = 0; 3686 3687 *sizep = bytes_read; 3688 3689 break; 3690 } 3691 3692 /* advance head if it is a DATA ACK */ 3693 if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 3694 3695 /* set the queue head */ 3696 if (rv = i_ldc_set_rx_head(ldcp, curr_head)) { 3697 bytes_read = 0; 3698 break; 3699 } 3700 3701 D2(ldcp->id, "ldc_read: (0x%llx) set ACK qhead 0x%llx", 3702 ldcp->id, curr_head); 3703 } 3704 3705 } /* for (;;) */ 3706 3707 3708 /* 3709 * If useful data was read - Send msg ACK 3710 * OPTIMIZE: do not send ACK for all msgs - use some frequency 3711 */ 3712 if ((bytes_read > 0) && (ldcp->mode == LDC_MODE_RELIABLE || 3713 ldcp->mode == LDC_MODE_STREAM)) { 3714 3715 rv = i_ldc_send_pkt(ldcp, LDC_DATA, LDC_ACK, 0); 3716 if (rv && rv != EWOULDBLOCK) { 3717 cmn_err(CE_NOTE, 3718 "ldc_read: (0x%lx) cannot send ACK\n", ldcp->id); 3719 3720 /* if cannot send ACK - reset channel */ 3721 goto channel_is_reset; 3722 } 3723 } 3724 3725 D2(ldcp->id, "ldc_read: (0x%llx) end size=%d", ldcp->id, *sizep); 3726 3727 return (rv); 3728 3729 channel_is_reset: 3730 mutex_enter(&ldcp->tx_lock); 3731 i_ldc_reset(ldcp, B_FALSE); 3732 mutex_exit(&ldcp->tx_lock); 3733 return (ECONNRESET); 3734 } 3735 3736 /* 3737 * Use underlying reliable packet mechanism to fetch 3738 * and buffer incoming packets so we can hand them back as 3739 * a basic byte stream. 3740 * 3741 * Enter and exit with ldcp->lock held by caller 3742 */ 3743 static int 3744 i_ldc_read_stream(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 3745 { 3746 int rv; 3747 size_t size; 3748 3749 ASSERT(mutex_owned(&ldcp->lock)); 3750 3751 D2(ldcp->id, "i_ldc_read_stream: (0x%llx) buffer size=%d", 3752 ldcp->id, *sizep); 3753 3754 if (ldcp->stream_remains == 0) { 3755 size = ldcp->mtu; 3756 rv = i_ldc_read_packet(ldcp, 3757 (caddr_t)ldcp->stream_bufferp, &size); 3758 D2(ldcp->id, "i_ldc_read_stream: read packet (0x%llx) size=%d", 3759 ldcp->id, size); 3760 3761 if (rv != 0) 3762 return (rv); 3763 3764 ldcp->stream_remains = size; 3765 ldcp->stream_offset = 0; 3766 } 3767 3768 size = MIN(ldcp->stream_remains, *sizep); 3769 3770 bcopy(ldcp->stream_bufferp + ldcp->stream_offset, target_bufp, size); 3771 ldcp->stream_offset += size; 3772 ldcp->stream_remains -= size; 3773 3774 D2(ldcp->id, "i_ldc_read_stream: (0x%llx) fill from buffer size=%d", 3775 ldcp->id, size); 3776 3777 *sizep = size; 3778 return (0); 3779 } 3780 3781 /* 3782 * Write specified amount of bytes to the channel 3783 * in multiple pkts of pkt_payload size. Each 3784 * packet is tagged with an unique packet ID in 3785 * the case of a reliable link. 3786 * 3787 * On return, size contains the number of bytes written. 3788 */ 3789 int 3790 ldc_write(ldc_handle_t handle, caddr_t buf, size_t *sizep) 3791 { 3792 ldc_chan_t *ldcp; 3793 int rv = 0; 3794 3795 if (handle == NULL) { 3796 DWARN(DBG_ALL_LDCS, "ldc_write: invalid channel handle\n"); 3797 return (EINVAL); 3798 } 3799 ldcp = (ldc_chan_t *)handle; 3800 3801 /* check if writes can occur */ 3802 if (!mutex_tryenter(&ldcp->tx_lock)) { 3803 /* 3804 * Could not get the lock - channel could 3805 * be in the process of being unconfigured 3806 * or reader has encountered an error 3807 */ 3808 return (EAGAIN); 3809 } 3810 3811 /* check if non-zero data to write */ 3812 if (buf == NULL || sizep == NULL) { 3813 DWARN(ldcp->id, "ldc_write: (0x%llx) invalid data write\n", 3814 ldcp->id); 3815 mutex_exit(&ldcp->tx_lock); 3816 return (EINVAL); 3817 } 3818 3819 if (*sizep == 0) { 3820 DWARN(ldcp->id, "ldc_write: (0x%llx) write size of zero\n", 3821 ldcp->id); 3822 mutex_exit(&ldcp->tx_lock); 3823 return (0); 3824 } 3825 3826 /* Check if channel is UP for data exchange */ 3827 if (ldcp->tstate != TS_UP) { 3828 DWARN(ldcp->id, 3829 "ldc_write: (0x%llx) channel is not in UP state\n", 3830 ldcp->id); 3831 *sizep = 0; 3832 rv = ECONNRESET; 3833 } else { 3834 rv = ldcp->write_p(ldcp, buf, sizep); 3835 } 3836 3837 mutex_exit(&ldcp->tx_lock); 3838 3839 return (rv); 3840 } 3841 3842 /* 3843 * Write a raw packet to the channel 3844 * On return, size contains the number of bytes written. 3845 */ 3846 static int 3847 i_ldc_write_raw(ldc_chan_t *ldcp, caddr_t buf, size_t *sizep) 3848 { 3849 ldc_msg_t *ldcmsg; 3850 uint64_t tx_head, tx_tail, new_tail; 3851 int rv = 0; 3852 size_t size; 3853 3854 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 3855 ASSERT(ldcp->mode == LDC_MODE_RAW); 3856 3857 size = *sizep; 3858 3859 /* 3860 * Check to see if the packet size is less than or 3861 * equal to packet size support in raw mode 3862 */ 3863 if (size > ldcp->pkt_payload) { 3864 DWARN(ldcp->id, 3865 "ldc_write: (0x%llx) invalid size (0x%llx) for RAW mode\n", 3866 ldcp->id, *sizep); 3867 *sizep = 0; 3868 return (EMSGSIZE); 3869 } 3870 3871 /* get the qptrs for the tx queue */ 3872 rv = hv_ldc_tx_get_state(ldcp->id, 3873 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 3874 if (rv != 0) { 3875 cmn_err(CE_WARN, 3876 "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp->id); 3877 *sizep = 0; 3878 return (EIO); 3879 } 3880 3881 if (ldcp->link_state == LDC_CHANNEL_DOWN || 3882 ldcp->link_state == LDC_CHANNEL_RESET) { 3883 DWARN(ldcp->id, 3884 "ldc_write: (0x%llx) channel down/reset\n", ldcp->id); 3885 3886 *sizep = 0; 3887 if (mutex_tryenter(&ldcp->lock)) { 3888 i_ldc_reset(ldcp, B_FALSE); 3889 mutex_exit(&ldcp->lock); 3890 } else { 3891 /* 3892 * Release Tx lock, and then reacquire channel 3893 * and Tx lock in correct order 3894 */ 3895 mutex_exit(&ldcp->tx_lock); 3896 mutex_enter(&ldcp->lock); 3897 mutex_enter(&ldcp->tx_lock); 3898 i_ldc_reset(ldcp, B_FALSE); 3899 mutex_exit(&ldcp->lock); 3900 } 3901 return (ECONNRESET); 3902 } 3903 3904 tx_tail = ldcp->tx_tail; 3905 tx_head = ldcp->tx_head; 3906 new_tail = (tx_tail + LDC_PACKET_SIZE) & 3907 ((ldcp->tx_q_entries-1) << LDC_PACKET_SHIFT); 3908 3909 if (new_tail == tx_head) { 3910 DWARN(DBG_ALL_LDCS, 3911 "ldc_write: (0x%llx) TX queue is full\n", ldcp->id); 3912 *sizep = 0; 3913 return (EWOULDBLOCK); 3914 } 3915 3916 D2(ldcp->id, "ldc_write: (0x%llx) start xfer size=%d", 3917 ldcp->id, size); 3918 3919 /* Send the data now */ 3920 ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 3921 3922 /* copy the data into pkt */ 3923 bcopy((uint8_t *)buf, ldcmsg, size); 3924 3925 /* increment tail */ 3926 tx_tail = new_tail; 3927 3928 /* 3929 * All packets have been copied into the TX queue 3930 * update the tail ptr in the HV 3931 */ 3932 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 3933 if (rv) { 3934 if (rv == EWOULDBLOCK) { 3935 DWARN(ldcp->id, "ldc_write: (0x%llx) write timed out\n", 3936 ldcp->id); 3937 *sizep = 0; 3938 return (EWOULDBLOCK); 3939 } 3940 3941 *sizep = 0; 3942 if (mutex_tryenter(&ldcp->lock)) { 3943 i_ldc_reset(ldcp, B_FALSE); 3944 mutex_exit(&ldcp->lock); 3945 } else { 3946 /* 3947 * Release Tx lock, and then reacquire channel 3948 * and Tx lock in correct order 3949 */ 3950 mutex_exit(&ldcp->tx_lock); 3951 mutex_enter(&ldcp->lock); 3952 mutex_enter(&ldcp->tx_lock); 3953 i_ldc_reset(ldcp, B_FALSE); 3954 mutex_exit(&ldcp->lock); 3955 } 3956 return (ECONNRESET); 3957 } 3958 3959 ldcp->tx_tail = tx_tail; 3960 *sizep = size; 3961 3962 D2(ldcp->id, "ldc_write: (0x%llx) end xfer size=%d", ldcp->id, size); 3963 3964 return (rv); 3965 } 3966 3967 3968 /* 3969 * Write specified amount of bytes to the channel 3970 * in multiple pkts of pkt_payload size. Each 3971 * packet is tagged with an unique packet ID in 3972 * the case of a reliable link. 3973 * 3974 * On return, size contains the number of bytes written. 3975 * This function needs to ensure that the write size is < MTU size 3976 */ 3977 static int 3978 i_ldc_write_packet(ldc_chan_t *ldcp, caddr_t buf, size_t *size) 3979 { 3980 ldc_msg_t *ldcmsg; 3981 uint64_t tx_head, tx_tail, new_tail, start; 3982 uint64_t txq_size_mask, numavail; 3983 uint8_t *msgbuf, *source = (uint8_t *)buf; 3984 size_t len, bytes_written = 0, remaining; 3985 int rv; 3986 uint32_t curr_seqid; 3987 3988 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 3989 3990 ASSERT(ldcp->mode == LDC_MODE_RELIABLE || 3991 ldcp->mode == LDC_MODE_UNRELIABLE || 3992 ldcp->mode == LDC_MODE_STREAM); 3993 3994 /* compute mask for increment */ 3995 txq_size_mask = (ldcp->tx_q_entries - 1) << LDC_PACKET_SHIFT; 3996 3997 /* get the qptrs for the tx queue */ 3998 rv = hv_ldc_tx_get_state(ldcp->id, 3999 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 4000 if (rv != 0) { 4001 cmn_err(CE_WARN, 4002 "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp->id); 4003 *size = 0; 4004 return (EIO); 4005 } 4006 4007 if (ldcp->link_state == LDC_CHANNEL_DOWN || 4008 ldcp->link_state == LDC_CHANNEL_RESET) { 4009 DWARN(ldcp->id, 4010 "ldc_write: (0x%llx) channel down/reset\n", ldcp->id); 4011 *size = 0; 4012 if (mutex_tryenter(&ldcp->lock)) { 4013 i_ldc_reset(ldcp, B_FALSE); 4014 mutex_exit(&ldcp->lock); 4015 } else { 4016 /* 4017 * Release Tx lock, and then reacquire channel 4018 * and Tx lock in correct order 4019 */ 4020 mutex_exit(&ldcp->tx_lock); 4021 mutex_enter(&ldcp->lock); 4022 mutex_enter(&ldcp->tx_lock); 4023 i_ldc_reset(ldcp, B_FALSE); 4024 mutex_exit(&ldcp->lock); 4025 } 4026 return (ECONNRESET); 4027 } 4028 4029 tx_tail = ldcp->tx_tail; 4030 new_tail = (tx_tail + LDC_PACKET_SIZE) % 4031 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 4032 4033 /* 4034 * Check to see if the queue is full. The check is done using 4035 * the appropriate head based on the link mode. 4036 */ 4037 i_ldc_get_tx_head(ldcp, &tx_head); 4038 4039 if (new_tail == tx_head) { 4040 DWARN(DBG_ALL_LDCS, 4041 "ldc_write: (0x%llx) TX queue is full\n", ldcp->id); 4042 *size = 0; 4043 return (EWOULDBLOCK); 4044 } 4045 4046 /* 4047 * Make sure that the LDC Tx queue has enough space 4048 */ 4049 numavail = (tx_head >> LDC_PACKET_SHIFT) - (tx_tail >> LDC_PACKET_SHIFT) 4050 + ldcp->tx_q_entries - 1; 4051 numavail %= ldcp->tx_q_entries; 4052 4053 if (*size > (numavail * ldcp->pkt_payload)) { 4054 DWARN(DBG_ALL_LDCS, 4055 "ldc_write: (0x%llx) TX queue has no space\n", ldcp->id); 4056 return (EWOULDBLOCK); 4057 } 4058 4059 D2(ldcp->id, "ldc_write: (0x%llx) start xfer size=%d", 4060 ldcp->id, *size); 4061 4062 /* Send the data now */ 4063 bytes_written = 0; 4064 curr_seqid = ldcp->last_msg_snt; 4065 start = tx_tail; 4066 4067 while (*size > bytes_written) { 4068 4069 ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 4070 4071 msgbuf = (uint8_t *)((ldcp->mode == LDC_MODE_RELIABLE || 4072 ldcp->mode == LDC_MODE_STREAM) ? 4073 ldcmsg->rdata : ldcmsg->udata); 4074 4075 ldcmsg->type = LDC_DATA; 4076 ldcmsg->stype = LDC_INFO; 4077 ldcmsg->ctrl = 0; 4078 4079 remaining = *size - bytes_written; 4080 len = min(ldcp->pkt_payload, remaining); 4081 ldcmsg->env = (uint8_t)len; 4082 4083 curr_seqid++; 4084 ldcmsg->seqid = curr_seqid; 4085 4086 /* copy the data into pkt */ 4087 bcopy(source, msgbuf, len); 4088 4089 source += len; 4090 bytes_written += len; 4091 4092 /* increment tail */ 4093 tx_tail = (tx_tail + LDC_PACKET_SIZE) & txq_size_mask; 4094 4095 ASSERT(tx_tail != tx_head); 4096 } 4097 4098 /* Set the start and stop bits */ 4099 ldcmsg->env |= LDC_FRAG_STOP; 4100 ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + start); 4101 ldcmsg->env |= LDC_FRAG_START; 4102 4103 /* 4104 * All packets have been copied into the TX queue 4105 * update the tail ptr in the HV 4106 */ 4107 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 4108 if (rv == 0) { 4109 ldcp->tx_tail = tx_tail; 4110 ldcp->last_msg_snt = curr_seqid; 4111 *size = bytes_written; 4112 } else { 4113 int rv2; 4114 4115 if (rv != EWOULDBLOCK) { 4116 *size = 0; 4117 if (mutex_tryenter(&ldcp->lock)) { 4118 i_ldc_reset(ldcp, B_FALSE); 4119 mutex_exit(&ldcp->lock); 4120 } else { 4121 /* 4122 * Release Tx lock, and then reacquire channel 4123 * and Tx lock in correct order 4124 */ 4125 mutex_exit(&ldcp->tx_lock); 4126 mutex_enter(&ldcp->lock); 4127 mutex_enter(&ldcp->tx_lock); 4128 i_ldc_reset(ldcp, B_FALSE); 4129 mutex_exit(&ldcp->lock); 4130 } 4131 return (ECONNRESET); 4132 } 4133 4134 D1(ldcp->id, "hv_tx_set_tail returns 0x%x (head 0x%x, " 4135 "old tail 0x%x, new tail 0x%x, qsize=0x%x)\n", 4136 rv, ldcp->tx_head, ldcp->tx_tail, tx_tail, 4137 (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 4138 4139 rv2 = hv_ldc_tx_get_state(ldcp->id, 4140 &tx_head, &tx_tail, &ldcp->link_state); 4141 4142 D1(ldcp->id, "hv_ldc_tx_get_state returns 0x%x " 4143 "(head 0x%x, tail 0x%x state 0x%x)\n", 4144 rv2, tx_head, tx_tail, ldcp->link_state); 4145 4146 *size = 0; 4147 } 4148 4149 D2(ldcp->id, "ldc_write: (0x%llx) end xfer size=%d", ldcp->id, *size); 4150 4151 return (rv); 4152 } 4153 4154 /* 4155 * Write specified amount of bytes to the channel 4156 * in multiple pkts of pkt_payload size. Each 4157 * packet is tagged with an unique packet ID in 4158 * the case of a reliable link. 4159 * 4160 * On return, size contains the number of bytes written. 4161 * This function needs to ensure that the write size is < MTU size 4162 */ 4163 static int 4164 i_ldc_write_stream(ldc_chan_t *ldcp, caddr_t buf, size_t *sizep) 4165 { 4166 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 4167 ASSERT(ldcp->mode == LDC_MODE_STREAM); 4168 4169 /* Truncate packet to max of MTU size */ 4170 if (*sizep > ldcp->mtu) *sizep = ldcp->mtu; 4171 return (i_ldc_write_packet(ldcp, buf, sizep)); 4172 } 4173 4174 4175 /* 4176 * Interfaces for channel nexus to register/unregister with LDC module 4177 * The nexus will register functions to be used to register individual 4178 * channels with the nexus and enable interrupts for the channels 4179 */ 4180 int 4181 ldc_register(ldc_cnex_t *cinfo) 4182 { 4183 ldc_chan_t *ldcp; 4184 4185 if (cinfo == NULL || cinfo->dip == NULL || 4186 cinfo->reg_chan == NULL || cinfo->unreg_chan == NULL || 4187 cinfo->add_intr == NULL || cinfo->rem_intr == NULL || 4188 cinfo->clr_intr == NULL) { 4189 4190 DWARN(DBG_ALL_LDCS, "ldc_register: invalid nexus info\n"); 4191 return (EINVAL); 4192 } 4193 4194 mutex_enter(&ldcssp->lock); 4195 4196 /* nexus registration */ 4197 ldcssp->cinfo.dip = cinfo->dip; 4198 ldcssp->cinfo.reg_chan = cinfo->reg_chan; 4199 ldcssp->cinfo.unreg_chan = cinfo->unreg_chan; 4200 ldcssp->cinfo.add_intr = cinfo->add_intr; 4201 ldcssp->cinfo.rem_intr = cinfo->rem_intr; 4202 ldcssp->cinfo.clr_intr = cinfo->clr_intr; 4203 4204 /* register any channels that might have been previously initialized */ 4205 ldcp = ldcssp->chan_list; 4206 while (ldcp) { 4207 if ((ldcp->tstate & TS_QCONF_RDY) && 4208 (ldcp->tstate & TS_CNEX_RDY) == 0) 4209 (void) i_ldc_register_channel(ldcp); 4210 4211 ldcp = ldcp->next; 4212 } 4213 4214 mutex_exit(&ldcssp->lock); 4215 4216 return (0); 4217 } 4218 4219 int 4220 ldc_unregister(ldc_cnex_t *cinfo) 4221 { 4222 if (cinfo == NULL || cinfo->dip == NULL) { 4223 DWARN(DBG_ALL_LDCS, "ldc_unregister: invalid nexus info\n"); 4224 return (EINVAL); 4225 } 4226 4227 mutex_enter(&ldcssp->lock); 4228 4229 if (cinfo->dip != ldcssp->cinfo.dip) { 4230 DWARN(DBG_ALL_LDCS, "ldc_unregister: invalid dip\n"); 4231 mutex_exit(&ldcssp->lock); 4232 return (EINVAL); 4233 } 4234 4235 /* nexus unregister */ 4236 ldcssp->cinfo.dip = NULL; 4237 ldcssp->cinfo.reg_chan = NULL; 4238 ldcssp->cinfo.unreg_chan = NULL; 4239 ldcssp->cinfo.add_intr = NULL; 4240 ldcssp->cinfo.rem_intr = NULL; 4241 ldcssp->cinfo.clr_intr = NULL; 4242 4243 mutex_exit(&ldcssp->lock); 4244 4245 return (0); 4246 } 4247 4248 4249 /* ------------------------------------------------------------------------- */ 4250 4251 /* 4252 * Allocate a memory handle for the channel and link it into the list 4253 * Also choose which memory table to use if this is the first handle 4254 * being assigned to this channel 4255 */ 4256 int 4257 ldc_mem_alloc_handle(ldc_handle_t handle, ldc_mem_handle_t *mhandle) 4258 { 4259 ldc_chan_t *ldcp; 4260 ldc_mhdl_t *mhdl; 4261 4262 if (handle == NULL) { 4263 DWARN(DBG_ALL_LDCS, 4264 "ldc_mem_alloc_handle: invalid channel handle\n"); 4265 return (EINVAL); 4266 } 4267 ldcp = (ldc_chan_t *)handle; 4268 4269 mutex_enter(&ldcp->lock); 4270 4271 /* check to see if channel is initalized */ 4272 if ((ldcp->tstate & ~TS_IN_RESET) < TS_INIT) { 4273 DWARN(ldcp->id, 4274 "ldc_mem_alloc_handle: (0x%llx) channel not initialized\n", 4275 ldcp->id); 4276 mutex_exit(&ldcp->lock); 4277 return (EINVAL); 4278 } 4279 4280 /* allocate handle for channel */ 4281 mhdl = kmem_cache_alloc(ldcssp->memhdl_cache, KM_SLEEP); 4282 4283 /* initialize the lock */ 4284 mutex_init(&mhdl->lock, NULL, MUTEX_DRIVER, NULL); 4285 4286 mhdl->myshadow = B_FALSE; 4287 mhdl->memseg = NULL; 4288 mhdl->ldcp = ldcp; 4289 mhdl->status = LDC_UNBOUND; 4290 4291 /* insert memory handle (@ head) into list */ 4292 if (ldcp->mhdl_list == NULL) { 4293 ldcp->mhdl_list = mhdl; 4294 mhdl->next = NULL; 4295 } else { 4296 /* insert @ head */ 4297 mhdl->next = ldcp->mhdl_list; 4298 ldcp->mhdl_list = mhdl; 4299 } 4300 4301 /* return the handle */ 4302 *mhandle = (ldc_mem_handle_t)mhdl; 4303 4304 mutex_exit(&ldcp->lock); 4305 4306 D1(ldcp->id, "ldc_mem_alloc_handle: (0x%llx) allocated handle 0x%llx\n", 4307 ldcp->id, mhdl); 4308 4309 return (0); 4310 } 4311 4312 /* 4313 * Free memory handle for the channel and unlink it from the list 4314 */ 4315 int 4316 ldc_mem_free_handle(ldc_mem_handle_t mhandle) 4317 { 4318 ldc_mhdl_t *mhdl, *phdl; 4319 ldc_chan_t *ldcp; 4320 4321 if (mhandle == NULL) { 4322 DWARN(DBG_ALL_LDCS, 4323 "ldc_mem_free_handle: invalid memory handle\n"); 4324 return (EINVAL); 4325 } 4326 mhdl = (ldc_mhdl_t *)mhandle; 4327 4328 mutex_enter(&mhdl->lock); 4329 4330 ldcp = mhdl->ldcp; 4331 4332 if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) { 4333 DWARN(ldcp->id, 4334 "ldc_mem_free_handle: cannot free, 0x%llx hdl bound\n", 4335 mhdl); 4336 mutex_exit(&mhdl->lock); 4337 return (EINVAL); 4338 } 4339 mutex_exit(&mhdl->lock); 4340 4341 mutex_enter(&ldcp->mlist_lock); 4342 4343 phdl = ldcp->mhdl_list; 4344 4345 /* first handle */ 4346 if (phdl == mhdl) { 4347 ldcp->mhdl_list = mhdl->next; 4348 mutex_destroy(&mhdl->lock); 4349 kmem_cache_free(ldcssp->memhdl_cache, mhdl); 4350 4351 D1(ldcp->id, 4352 "ldc_mem_free_handle: (0x%llx) freed handle 0x%llx\n", 4353 ldcp->id, mhdl); 4354 } else { 4355 /* walk the list - unlink and free */ 4356 while (phdl != NULL) { 4357 if (phdl->next == mhdl) { 4358 phdl->next = mhdl->next; 4359 mutex_destroy(&mhdl->lock); 4360 kmem_cache_free(ldcssp->memhdl_cache, mhdl); 4361 D1(ldcp->id, 4362 "ldc_mem_free_handle: (0x%llx) freed " 4363 "handle 0x%llx\n", ldcp->id, mhdl); 4364 break; 4365 } 4366 phdl = phdl->next; 4367 } 4368 } 4369 4370 if (phdl == NULL) { 4371 DWARN(ldcp->id, 4372 "ldc_mem_free_handle: invalid handle 0x%llx\n", mhdl); 4373 mutex_exit(&ldcp->mlist_lock); 4374 return (EINVAL); 4375 } 4376 4377 mutex_exit(&ldcp->mlist_lock); 4378 4379 return (0); 4380 } 4381 4382 /* 4383 * Bind a memory handle to a virtual address. 4384 * The virtual address is converted to the corresponding real addresses. 4385 * Returns pointer to the first ldc_mem_cookie and the total number 4386 * of cookies for this virtual address. Other cookies can be obtained 4387 * using the ldc_mem_nextcookie() call. If the pages are stored in 4388 * consecutive locations in the table, a single cookie corresponding to 4389 * the first location is returned. The cookie size spans all the entries. 4390 * 4391 * If the VA corresponds to a page that is already being exported, reuse 4392 * the page and do not export it again. Bump the page's use count. 4393 */ 4394 int 4395 ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr, size_t len, 4396 uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount) 4397 { 4398 ldc_mhdl_t *mhdl; 4399 ldc_chan_t *ldcp; 4400 ldc_mtbl_t *mtbl; 4401 ldc_memseg_t *memseg; 4402 ldc_mte_t tmp_mte; 4403 uint64_t index, prev_index = 0; 4404 int64_t cookie_idx; 4405 uintptr_t raddr, ra_aligned; 4406 uint64_t psize, poffset, v_offset; 4407 uint64_t pg_shift, pg_size, pg_size_code, pg_mask; 4408 pgcnt_t npages; 4409 caddr_t v_align, addr; 4410 int i, rv; 4411 4412 if (mhandle == NULL) { 4413 DWARN(DBG_ALL_LDCS, 4414 "ldc_mem_bind_handle: invalid memory handle\n"); 4415 return (EINVAL); 4416 } 4417 mhdl = (ldc_mhdl_t *)mhandle; 4418 ldcp = mhdl->ldcp; 4419 4420 /* clear count */ 4421 *ccount = 0; 4422 4423 mutex_enter(&mhdl->lock); 4424 4425 if (mhdl->status == LDC_BOUND || mhdl->memseg != NULL) { 4426 DWARN(ldcp->id, 4427 "ldc_mem_bind_handle: (0x%x) handle already bound\n", 4428 mhandle); 4429 mutex_exit(&mhdl->lock); 4430 return (EINVAL); 4431 } 4432 4433 /* Force address and size to be 8-byte aligned */ 4434 if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 4435 DWARN(ldcp->id, 4436 "ldc_mem_bind_handle: addr/size is not 8-byte aligned\n"); 4437 mutex_exit(&mhdl->lock); 4438 return (EINVAL); 4439 } 4440 4441 /* 4442 * If this channel is binding a memory handle for the 4443 * first time allocate it a memory map table and initialize it 4444 */ 4445 if ((mtbl = ldcp->mtbl) == NULL) { 4446 4447 mutex_enter(&ldcp->lock); 4448 4449 /* Allocate and initialize the map table structure */ 4450 mtbl = kmem_zalloc(sizeof (ldc_mtbl_t), KM_SLEEP); 4451 mtbl->num_entries = mtbl->num_avail = ldc_maptable_entries; 4452 mtbl->size = ldc_maptable_entries * sizeof (ldc_mte_slot_t); 4453 mtbl->next_entry = NULL; 4454 mtbl->contigmem = B_TRUE; 4455 4456 /* Allocate the table itself */ 4457 mtbl->table = (ldc_mte_slot_t *) 4458 contig_mem_alloc_align(mtbl->size, MMU_PAGESIZE); 4459 if (mtbl->table == NULL) { 4460 4461 /* allocate a page of memory using kmem_alloc */ 4462 mtbl->table = kmem_alloc(MMU_PAGESIZE, KM_SLEEP); 4463 mtbl->size = MMU_PAGESIZE; 4464 mtbl->contigmem = B_FALSE; 4465 mtbl->num_entries = mtbl->num_avail = 4466 mtbl->size / sizeof (ldc_mte_slot_t); 4467 DWARN(ldcp->id, 4468 "ldc_mem_bind_handle: (0x%llx) reduced tbl size " 4469 "to %lx entries\n", ldcp->id, mtbl->num_entries); 4470 } 4471 4472 /* zero out the memory */ 4473 bzero(mtbl->table, mtbl->size); 4474 4475 /* initialize the lock */ 4476 mutex_init(&mtbl->lock, NULL, MUTEX_DRIVER, NULL); 4477 4478 /* register table for this channel */ 4479 rv = hv_ldc_set_map_table(ldcp->id, 4480 va_to_pa(mtbl->table), mtbl->num_entries); 4481 if (rv != 0) { 4482 cmn_err(CE_WARN, 4483 "ldc_mem_bind_handle: (0x%lx) err %d mapping tbl", 4484 ldcp->id, rv); 4485 if (mtbl->contigmem) 4486 contig_mem_free(mtbl->table, mtbl->size); 4487 else 4488 kmem_free(mtbl->table, mtbl->size); 4489 mutex_destroy(&mtbl->lock); 4490 kmem_free(mtbl, sizeof (ldc_mtbl_t)); 4491 mutex_exit(&ldcp->lock); 4492 mutex_exit(&mhdl->lock); 4493 return (EIO); 4494 } 4495 4496 ldcp->mtbl = mtbl; 4497 mutex_exit(&ldcp->lock); 4498 4499 D1(ldcp->id, 4500 "ldc_mem_bind_handle: (0x%llx) alloc'd map table 0x%llx\n", 4501 ldcp->id, ldcp->mtbl->table); 4502 } 4503 4504 /* FUTURE: get the page size, pgsz code, and shift */ 4505 pg_size = MMU_PAGESIZE; 4506 pg_size_code = page_szc(pg_size); 4507 pg_shift = page_get_shift(pg_size_code); 4508 pg_mask = ~(pg_size - 1); 4509 4510 D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) binding " 4511 "va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 4512 ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 4513 4514 /* aligned VA and its offset */ 4515 v_align = (caddr_t)(((uintptr_t)vaddr) & ~(pg_size - 1)); 4516 v_offset = ((uintptr_t)vaddr) & (pg_size - 1); 4517 4518 npages = (len+v_offset)/pg_size; 4519 npages = ((len+v_offset)%pg_size == 0) ? npages : npages+1; 4520 4521 D1(ldcp->id, "ldc_mem_bind_handle: binding " 4522 "(0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 4523 ldcp->id, vaddr, v_align, v_offset, npages); 4524 4525 /* lock the memory table - exclusive access to channel */ 4526 mutex_enter(&mtbl->lock); 4527 4528 if (npages > mtbl->num_avail) { 4529 D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) no table entries\n", 4530 ldcp->id); 4531 mutex_exit(&mtbl->lock); 4532 mutex_exit(&mhdl->lock); 4533 return (ENOMEM); 4534 } 4535 4536 /* Allocate a memseg structure */ 4537 memseg = mhdl->memseg = 4538 kmem_cache_alloc(ldcssp->memseg_cache, KM_SLEEP); 4539 4540 /* Allocate memory to store all pages and cookies */ 4541 memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP); 4542 memseg->cookies = 4543 kmem_zalloc((sizeof (ldc_mem_cookie_t) * npages), KM_SLEEP); 4544 4545 D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) processing 0x%llx pages\n", 4546 ldcp->id, npages); 4547 4548 addr = v_align; 4549 4550 /* 4551 * Check if direct shared memory map is enabled, if not change 4552 * the mapping type to include SHADOW_MAP. 4553 */ 4554 if (ldc_shmem_enabled == 0) 4555 mtype = LDC_SHADOW_MAP; 4556 4557 /* 4558 * Table slots are used in a round-robin manner. The algorithm permits 4559 * inserting duplicate entries. Slots allocated earlier will typically 4560 * get freed before we get back to reusing the slot.Inserting duplicate 4561 * entries should be OK as we only lookup entries using the cookie addr 4562 * i.e. tbl index, during export, unexport and copy operation. 4563 * 4564 * One implementation what was tried was to search for a duplicate 4565 * page entry first and reuse it. The search overhead is very high and 4566 * in the vnet case dropped the perf by almost half, 50 to 24 mbps. 4567 * So it does make sense to avoid searching for duplicates. 4568 * 4569 * But during the process of searching for a free slot, if we find a 4570 * duplicate entry we will go ahead and use it, and bump its use count. 4571 */ 4572 4573 /* index to start searching from */ 4574 index = mtbl->next_entry; 4575 cookie_idx = -1; 4576 4577 tmp_mte.ll = 0; /* initialise fields to 0 */ 4578 4579 if (mtype & LDC_DIRECT_MAP) { 4580 tmp_mte.mte_r = (perm & LDC_MEM_R) ? 1 : 0; 4581 tmp_mte.mte_w = (perm & LDC_MEM_W) ? 1 : 0; 4582 tmp_mte.mte_x = (perm & LDC_MEM_X) ? 1 : 0; 4583 } 4584 4585 if (mtype & LDC_SHADOW_MAP) { 4586 tmp_mte.mte_cr = (perm & LDC_MEM_R) ? 1 : 0; 4587 tmp_mte.mte_cw = (perm & LDC_MEM_W) ? 1 : 0; 4588 } 4589 4590 if (mtype & LDC_IO_MAP) { 4591 tmp_mte.mte_ir = (perm & LDC_MEM_R) ? 1 : 0; 4592 tmp_mte.mte_iw = (perm & LDC_MEM_W) ? 1 : 0; 4593 } 4594 4595 D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll); 4596 4597 tmp_mte.mte_pgszc = pg_size_code; 4598 4599 /* initialize each mem table entry */ 4600 for (i = 0; i < npages; i++) { 4601 4602 /* check if slot is available in the table */ 4603 while (mtbl->table[index].entry.ll != 0) { 4604 4605 index = (index + 1) % mtbl->num_entries; 4606 4607 if (index == mtbl->next_entry) { 4608 /* we have looped around */ 4609 DWARN(DBG_ALL_LDCS, 4610 "ldc_mem_bind_handle: (0x%llx) cannot find " 4611 "entry\n", ldcp->id); 4612 *ccount = 0; 4613 4614 /* NOTE: free memory, remove previous entries */ 4615 /* this shouldnt happen as num_avail was ok */ 4616 4617 mutex_exit(&mtbl->lock); 4618 mutex_exit(&mhdl->lock); 4619 return (ENOMEM); 4620 } 4621 } 4622 4623 /* get the real address */ 4624 raddr = va_to_pa((void *)addr); 4625 ra_aligned = ((uintptr_t)raddr & pg_mask); 4626 4627 /* build the mte */ 4628 tmp_mte.mte_rpfn = ra_aligned >> pg_shift; 4629 4630 D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll); 4631 4632 /* update entry in table */ 4633 mtbl->table[index].entry = tmp_mte; 4634 4635 D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) stored MTE 0x%llx" 4636 " into loc 0x%llx\n", ldcp->id, tmp_mte.ll, index); 4637 4638 /* calculate the size and offset for this export range */ 4639 if (i == 0) { 4640 /* first page */ 4641 psize = min((pg_size - v_offset), len); 4642 poffset = v_offset; 4643 4644 } else if (i == (npages - 1)) { 4645 /* last page */ 4646 psize = (((uintptr_t)(vaddr + len)) & 4647 ((uint64_t)(pg_size-1))); 4648 if (psize == 0) 4649 psize = pg_size; 4650 poffset = 0; 4651 4652 } else { 4653 /* middle pages */ 4654 psize = pg_size; 4655 poffset = 0; 4656 } 4657 4658 /* store entry for this page */ 4659 memseg->pages[i].index = index; 4660 memseg->pages[i].raddr = raddr; 4661 memseg->pages[i].offset = poffset; 4662 memseg->pages[i].size = psize; 4663 memseg->pages[i].mte = &(mtbl->table[index]); 4664 4665 /* create the cookie */ 4666 if (i == 0 || (index != prev_index + 1)) { 4667 cookie_idx++; 4668 memseg->cookies[cookie_idx].addr = 4669 IDX2COOKIE(index, pg_size_code, pg_shift); 4670 memseg->cookies[cookie_idx].addr |= poffset; 4671 memseg->cookies[cookie_idx].size = psize; 4672 4673 } else { 4674 memseg->cookies[cookie_idx].size += psize; 4675 } 4676 4677 D1(ldcp->id, "ldc_mem_bind_handle: bound " 4678 "(0x%llx) va=0x%llx, idx=0x%llx, " 4679 "ra=0x%llx(sz=0x%x,off=0x%x)\n", 4680 ldcp->id, addr, index, raddr, psize, poffset); 4681 4682 /* decrement number of available entries */ 4683 mtbl->num_avail--; 4684 4685 /* increment va by page size */ 4686 addr += pg_size; 4687 4688 /* increment index */ 4689 prev_index = index; 4690 index = (index + 1) % mtbl->num_entries; 4691 4692 /* save the next slot */ 4693 mtbl->next_entry = index; 4694 } 4695 4696 mutex_exit(&mtbl->lock); 4697 4698 /* memory handle = bound */ 4699 mhdl->mtype = mtype; 4700 mhdl->perm = perm; 4701 mhdl->status = LDC_BOUND; 4702 4703 /* update memseg_t */ 4704 memseg->vaddr = vaddr; 4705 memseg->raddr = memseg->pages[0].raddr; 4706 memseg->size = len; 4707 memseg->npages = npages; 4708 memseg->ncookies = cookie_idx + 1; 4709 memseg->next_cookie = (memseg->ncookies > 1) ? 1 : 0; 4710 4711 /* return count and first cookie */ 4712 *ccount = memseg->ncookies; 4713 cookie->addr = memseg->cookies[0].addr; 4714 cookie->size = memseg->cookies[0].size; 4715 4716 D1(ldcp->id, 4717 "ldc_mem_bind_handle: (0x%llx) bound 0x%llx, va=0x%llx, " 4718 "pgs=0x%llx cookies=0x%llx\n", 4719 ldcp->id, mhdl, vaddr, npages, memseg->ncookies); 4720 4721 mutex_exit(&mhdl->lock); 4722 return (0); 4723 } 4724 4725 /* 4726 * Return the next cookie associated with the specified memory handle 4727 */ 4728 int 4729 ldc_mem_nextcookie(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie) 4730 { 4731 ldc_mhdl_t *mhdl; 4732 ldc_chan_t *ldcp; 4733 ldc_memseg_t *memseg; 4734 4735 if (mhandle == NULL) { 4736 DWARN(DBG_ALL_LDCS, 4737 "ldc_mem_nextcookie: invalid memory handle\n"); 4738 return (EINVAL); 4739 } 4740 mhdl = (ldc_mhdl_t *)mhandle; 4741 4742 mutex_enter(&mhdl->lock); 4743 4744 ldcp = mhdl->ldcp; 4745 memseg = mhdl->memseg; 4746 4747 if (cookie == 0) { 4748 DWARN(ldcp->id, 4749 "ldc_mem_nextcookie:(0x%llx) invalid cookie arg\n", 4750 ldcp->id); 4751 mutex_exit(&mhdl->lock); 4752 return (EINVAL); 4753 } 4754 4755 if (memseg->next_cookie != 0) { 4756 cookie->addr = memseg->cookies[memseg->next_cookie].addr; 4757 cookie->size = memseg->cookies[memseg->next_cookie].size; 4758 memseg->next_cookie++; 4759 if (memseg->next_cookie == memseg->ncookies) 4760 memseg->next_cookie = 0; 4761 4762 } else { 4763 DWARN(ldcp->id, 4764 "ldc_mem_nextcookie:(0x%llx) no more cookies\n", ldcp->id); 4765 cookie->addr = 0; 4766 cookie->size = 0; 4767 mutex_exit(&mhdl->lock); 4768 return (EINVAL); 4769 } 4770 4771 D1(ldcp->id, 4772 "ldc_mem_nextcookie: (0x%llx) cookie addr=0x%llx,sz=0x%llx\n", 4773 ldcp->id, cookie->addr, cookie->size); 4774 4775 mutex_exit(&mhdl->lock); 4776 return (0); 4777 } 4778 4779 /* 4780 * Unbind the virtual memory region associated with the specified 4781 * memory handle. Allassociated cookies are freed and the corresponding 4782 * RA space is no longer exported. 4783 */ 4784 int 4785 ldc_mem_unbind_handle(ldc_mem_handle_t mhandle) 4786 { 4787 ldc_mhdl_t *mhdl; 4788 ldc_chan_t *ldcp; 4789 ldc_mtbl_t *mtbl; 4790 ldc_memseg_t *memseg; 4791 uint64_t cookie_addr; 4792 uint64_t pg_shift, pg_size_code; 4793 int i, rv; 4794 4795 if (mhandle == NULL) { 4796 DWARN(DBG_ALL_LDCS, 4797 "ldc_mem_unbind_handle: invalid memory handle\n"); 4798 return (EINVAL); 4799 } 4800 mhdl = (ldc_mhdl_t *)mhandle; 4801 4802 mutex_enter(&mhdl->lock); 4803 4804 if (mhdl->status == LDC_UNBOUND) { 4805 DWARN(DBG_ALL_LDCS, 4806 "ldc_mem_unbind_handle: (0x%x) handle is not bound\n", 4807 mhandle); 4808 mutex_exit(&mhdl->lock); 4809 return (EINVAL); 4810 } 4811 4812 ldcp = mhdl->ldcp; 4813 mtbl = ldcp->mtbl; 4814 4815 memseg = mhdl->memseg; 4816 4817 /* lock the memory table - exclusive access to channel */ 4818 mutex_enter(&mtbl->lock); 4819 4820 /* undo the pages exported */ 4821 for (i = 0; i < memseg->npages; i++) { 4822 4823 /* check for mapped pages, revocation cookie != 0 */ 4824 if (memseg->pages[i].mte->cookie) { 4825 4826 pg_size_code = page_szc(memseg->pages[i].size); 4827 pg_shift = page_get_shift(memseg->pages[i].size); 4828 cookie_addr = IDX2COOKIE(memseg->pages[i].index, 4829 pg_size_code, pg_shift); 4830 4831 D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) revoke " 4832 "cookie 0x%llx, rcookie 0x%llx\n", ldcp->id, 4833 cookie_addr, memseg->pages[i].mte->cookie); 4834 rv = hv_ldc_revoke(ldcp->id, cookie_addr, 4835 memseg->pages[i].mte->cookie); 4836 if (rv) { 4837 DWARN(ldcp->id, 4838 "ldc_mem_unbind_handle: (0x%llx) cannot " 4839 "revoke mapping, cookie %llx\n", ldcp->id, 4840 cookie_addr); 4841 } 4842 } 4843 4844 /* clear the entry from the table */ 4845 memseg->pages[i].mte->entry.ll = 0; 4846 mtbl->num_avail++; 4847 } 4848 mutex_exit(&mtbl->lock); 4849 4850 /* free the allocated memseg and page structures */ 4851 kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages)); 4852 kmem_free(memseg->cookies, 4853 (sizeof (ldc_mem_cookie_t) * memseg->npages)); 4854 kmem_cache_free(ldcssp->memseg_cache, memseg); 4855 4856 /* uninitialize the memory handle */ 4857 mhdl->memseg = NULL; 4858 mhdl->status = LDC_UNBOUND; 4859 4860 D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) unbound handle 0x%llx\n", 4861 ldcp->id, mhdl); 4862 4863 mutex_exit(&mhdl->lock); 4864 return (0); 4865 } 4866 4867 /* 4868 * Get information about the dring. The base address of the descriptor 4869 * ring along with the type and permission are returned back. 4870 */ 4871 int 4872 ldc_mem_info(ldc_mem_handle_t mhandle, ldc_mem_info_t *minfo) 4873 { 4874 ldc_mhdl_t *mhdl; 4875 4876 if (mhandle == NULL) { 4877 DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid memory handle\n"); 4878 return (EINVAL); 4879 } 4880 mhdl = (ldc_mhdl_t *)mhandle; 4881 4882 if (minfo == NULL) { 4883 DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid args\n"); 4884 return (EINVAL); 4885 } 4886 4887 mutex_enter(&mhdl->lock); 4888 4889 minfo->status = mhdl->status; 4890 if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) { 4891 minfo->vaddr = mhdl->memseg->vaddr; 4892 minfo->raddr = mhdl->memseg->raddr; 4893 minfo->mtype = mhdl->mtype; 4894 minfo->perm = mhdl->perm; 4895 } 4896 mutex_exit(&mhdl->lock); 4897 4898 return (0); 4899 } 4900 4901 /* 4902 * Copy data either from or to the client specified virtual address 4903 * space to or from the exported memory associated with the cookies. 4904 * The direction argument determines whether the data is read from or 4905 * written to exported memory. 4906 */ 4907 int 4908 ldc_mem_copy(ldc_handle_t handle, caddr_t vaddr, uint64_t off, size_t *size, 4909 ldc_mem_cookie_t *cookies, uint32_t ccount, uint8_t direction) 4910 { 4911 ldc_chan_t *ldcp; 4912 uint64_t local_voff, local_valign; 4913 uint64_t cookie_addr, cookie_size; 4914 uint64_t pg_shift, pg_size, pg_size_code; 4915 uint64_t export_caddr, export_poff, export_psize, export_size; 4916 uint64_t local_ra, local_poff, local_psize; 4917 uint64_t copy_size, copied_len = 0, total_bal = 0, idx = 0; 4918 pgcnt_t npages; 4919 size_t len = *size; 4920 int i, rv = 0; 4921 4922 uint64_t chid; 4923 4924 if (handle == NULL) { 4925 DWARN(DBG_ALL_LDCS, "ldc_mem_copy: invalid channel handle\n"); 4926 return (EINVAL); 4927 } 4928 ldcp = (ldc_chan_t *)handle; 4929 chid = ldcp->id; 4930 4931 /* check to see if channel is UP */ 4932 if (ldcp->tstate != TS_UP) { 4933 DWARN(chid, "ldc_mem_copy: (0x%llx) channel is not UP\n", 4934 chid); 4935 return (ECONNRESET); 4936 } 4937 4938 /* Force address and size to be 8-byte aligned */ 4939 if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 4940 DWARN(chid, 4941 "ldc_mem_copy: addr/sz is not 8-byte aligned\n"); 4942 return (EINVAL); 4943 } 4944 4945 /* Find the size of the exported memory */ 4946 export_size = 0; 4947 for (i = 0; i < ccount; i++) 4948 export_size += cookies[i].size; 4949 4950 /* check to see if offset is valid */ 4951 if (off > export_size) { 4952 DWARN(chid, 4953 "ldc_mem_copy: (0x%llx) start offset > export mem size\n", 4954 chid); 4955 return (EINVAL); 4956 } 4957 4958 /* 4959 * Check to see if the export size is smaller than the size we 4960 * are requesting to copy - if so flag an error 4961 */ 4962 if ((export_size - off) < *size) { 4963 DWARN(chid, 4964 "ldc_mem_copy: (0x%llx) copy size > export mem size\n", 4965 chid); 4966 return (EINVAL); 4967 } 4968 4969 total_bal = min(export_size, *size); 4970 4971 /* FUTURE: get the page size, pgsz code, and shift */ 4972 pg_size = MMU_PAGESIZE; 4973 pg_size_code = page_szc(pg_size); 4974 pg_shift = page_get_shift(pg_size_code); 4975 4976 D1(chid, "ldc_mem_copy: copying data " 4977 "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 4978 chid, vaddr, pg_size, pg_size_code, pg_shift); 4979 4980 /* aligned VA and its offset */ 4981 local_valign = (((uintptr_t)vaddr) & ~(pg_size - 1)); 4982 local_voff = ((uintptr_t)vaddr) & (pg_size - 1); 4983 4984 npages = (len+local_voff)/pg_size; 4985 npages = ((len+local_voff)%pg_size == 0) ? npages : npages+1; 4986 4987 D1(chid, 4988 "ldc_mem_copy: (0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 4989 chid, vaddr, local_valign, local_voff, npages); 4990 4991 local_ra = va_to_pa((void *)local_valign); 4992 local_poff = local_voff; 4993 local_psize = min(len, (pg_size - local_voff)); 4994 4995 len -= local_psize; 4996 4997 /* 4998 * find the first cookie in the list of cookies 4999 * if the offset passed in is not zero 5000 */ 5001 for (idx = 0; idx < ccount; idx++) { 5002 cookie_size = cookies[idx].size; 5003 if (off < cookie_size) 5004 break; 5005 off -= cookie_size; 5006 } 5007 5008 cookie_addr = cookies[idx].addr + off; 5009 cookie_size = cookies[idx].size - off; 5010 5011 export_caddr = cookie_addr & ~(pg_size - 1); 5012 export_poff = cookie_addr & (pg_size - 1); 5013 export_psize = min(cookie_size, (pg_size - export_poff)); 5014 5015 for (;;) { 5016 5017 copy_size = min(export_psize, local_psize); 5018 5019 D1(chid, 5020 "ldc_mem_copy:(0x%llx) dir=0x%x, caddr=0x%llx," 5021 " loc_ra=0x%llx, exp_poff=0x%llx, loc_poff=0x%llx," 5022 " exp_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 5023 " total_bal=0x%llx\n", 5024 chid, direction, export_caddr, local_ra, export_poff, 5025 local_poff, export_psize, local_psize, copy_size, 5026 total_bal); 5027 5028 rv = hv_ldc_copy(chid, direction, 5029 (export_caddr + export_poff), (local_ra + local_poff), 5030 copy_size, &copied_len); 5031 5032 if (rv != 0) { 5033 int error = EIO; 5034 uint64_t rx_hd, rx_tl; 5035 5036 DWARN(chid, 5037 "ldc_mem_copy: (0x%llx) err %d during copy\n", 5038 (unsigned long long)chid, rv); 5039 DWARN(chid, 5040 "ldc_mem_copy: (0x%llx) dir=0x%x, caddr=0x%lx, " 5041 "loc_ra=0x%lx, exp_poff=0x%lx, loc_poff=0x%lx," 5042 " exp_psz=0x%lx, loc_psz=0x%lx, copy_sz=0x%lx," 5043 " copied_len=0x%lx, total_bal=0x%lx\n", 5044 chid, direction, export_caddr, local_ra, 5045 export_poff, local_poff, export_psize, local_psize, 5046 copy_size, copied_len, total_bal); 5047 5048 *size = *size - total_bal; 5049 5050 /* 5051 * check if reason for copy error was due to 5052 * a channel reset. we need to grab the lock 5053 * just in case we have to do a reset. 5054 */ 5055 mutex_enter(&ldcp->lock); 5056 mutex_enter(&ldcp->tx_lock); 5057 5058 rv = hv_ldc_rx_get_state(ldcp->id, 5059 &rx_hd, &rx_tl, &(ldcp->link_state)); 5060 if (ldcp->link_state == LDC_CHANNEL_DOWN || 5061 ldcp->link_state == LDC_CHANNEL_RESET) { 5062 i_ldc_reset(ldcp, B_FALSE); 5063 error = ECONNRESET; 5064 } 5065 5066 mutex_exit(&ldcp->tx_lock); 5067 mutex_exit(&ldcp->lock); 5068 5069 return (error); 5070 } 5071 5072 ASSERT(copied_len <= copy_size); 5073 5074 D2(chid, "ldc_mem_copy: copied=0x%llx\n", copied_len); 5075 export_poff += copied_len; 5076 local_poff += copied_len; 5077 export_psize -= copied_len; 5078 local_psize -= copied_len; 5079 cookie_size -= copied_len; 5080 5081 total_bal -= copied_len; 5082 5083 if (copy_size != copied_len) 5084 continue; 5085 5086 if (export_psize == 0 && total_bal != 0) { 5087 5088 if (cookie_size == 0) { 5089 idx++; 5090 cookie_addr = cookies[idx].addr; 5091 cookie_size = cookies[idx].size; 5092 5093 export_caddr = cookie_addr & ~(pg_size - 1); 5094 export_poff = cookie_addr & (pg_size - 1); 5095 export_psize = 5096 min(cookie_size, (pg_size-export_poff)); 5097 } else { 5098 export_caddr += pg_size; 5099 export_poff = 0; 5100 export_psize = min(cookie_size, pg_size); 5101 } 5102 } 5103 5104 if (local_psize == 0 && total_bal != 0) { 5105 local_valign += pg_size; 5106 local_ra = va_to_pa((void *)local_valign); 5107 local_poff = 0; 5108 local_psize = min(pg_size, len); 5109 len -= local_psize; 5110 } 5111 5112 /* check if we are all done */ 5113 if (total_bal == 0) 5114 break; 5115 } 5116 5117 5118 D1(chid, 5119 "ldc_mem_copy: (0x%llx) done copying sz=0x%llx\n", 5120 chid, *size); 5121 5122 return (0); 5123 } 5124 5125 /* 5126 * Copy data either from or to the client specified virtual address 5127 * space to or from HV physical memory. 5128 * 5129 * The direction argument determines whether the data is read from or 5130 * written to HV memory. direction values are LDC_COPY_IN/OUT similar 5131 * to the ldc_mem_copy interface 5132 */ 5133 int 5134 ldc_mem_rdwr_cookie(ldc_handle_t handle, caddr_t vaddr, size_t *size, 5135 caddr_t paddr, uint8_t direction) 5136 { 5137 ldc_chan_t *ldcp; 5138 uint64_t local_voff, local_valign; 5139 uint64_t pg_shift, pg_size, pg_size_code; 5140 uint64_t target_pa, target_poff, target_psize, target_size; 5141 uint64_t local_ra, local_poff, local_psize; 5142 uint64_t copy_size, copied_len = 0; 5143 pgcnt_t npages; 5144 size_t len = *size; 5145 int rv = 0; 5146 5147 if (handle == NULL) { 5148 DWARN(DBG_ALL_LDCS, 5149 "ldc_mem_rdwr_cookie: invalid channel handle\n"); 5150 return (EINVAL); 5151 } 5152 ldcp = (ldc_chan_t *)handle; 5153 5154 mutex_enter(&ldcp->lock); 5155 5156 /* check to see if channel is UP */ 5157 if (ldcp->tstate != TS_UP) { 5158 DWARN(ldcp->id, 5159 "ldc_mem_rdwr_cookie: (0x%llx) channel is not UP\n", 5160 ldcp->id); 5161 mutex_exit(&ldcp->lock); 5162 return (ECONNRESET); 5163 } 5164 5165 /* Force address and size to be 8-byte aligned */ 5166 if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 5167 DWARN(ldcp->id, 5168 "ldc_mem_rdwr_cookie: addr/size is not 8-byte aligned\n"); 5169 mutex_exit(&ldcp->lock); 5170 return (EINVAL); 5171 } 5172 5173 target_size = *size; 5174 5175 /* FUTURE: get the page size, pgsz code, and shift */ 5176 pg_size = MMU_PAGESIZE; 5177 pg_size_code = page_szc(pg_size); 5178 pg_shift = page_get_shift(pg_size_code); 5179 5180 D1(ldcp->id, "ldc_mem_rdwr_cookie: copying data " 5181 "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 5182 ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 5183 5184 /* aligned VA and its offset */ 5185 local_valign = ((uintptr_t)vaddr) & ~(pg_size - 1); 5186 local_voff = ((uintptr_t)vaddr) & (pg_size - 1); 5187 5188 npages = (len + local_voff) / pg_size; 5189 npages = ((len + local_voff) % pg_size == 0) ? npages : npages+1; 5190 5191 D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) v=0x%llx, " 5192 "val=0x%llx,off=0x%x,pgs=0x%x\n", 5193 ldcp->id, vaddr, local_valign, local_voff, npages); 5194 5195 local_ra = va_to_pa((void *)local_valign); 5196 local_poff = local_voff; 5197 local_psize = min(len, (pg_size - local_voff)); 5198 5199 len -= local_psize; 5200 5201 target_pa = ((uintptr_t)paddr) & ~(pg_size - 1); 5202 target_poff = ((uintptr_t)paddr) & (pg_size - 1); 5203 target_psize = pg_size - target_poff; 5204 5205 for (;;) { 5206 5207 copy_size = min(target_psize, local_psize); 5208 5209 D1(ldcp->id, 5210 "ldc_mem_rdwr_cookie: (0x%llx) dir=0x%x, tar_pa=0x%llx," 5211 " loc_ra=0x%llx, tar_poff=0x%llx, loc_poff=0x%llx," 5212 " tar_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 5213 " total_bal=0x%llx\n", 5214 ldcp->id, direction, target_pa, local_ra, target_poff, 5215 local_poff, target_psize, local_psize, copy_size, 5216 target_size); 5217 5218 rv = hv_ldc_copy(ldcp->id, direction, 5219 (target_pa + target_poff), (local_ra + local_poff), 5220 copy_size, &copied_len); 5221 5222 if (rv != 0) { 5223 DWARN(DBG_ALL_LDCS, 5224 "ldc_mem_rdwr_cookie: (0x%lx) err %d during copy\n", 5225 ldcp->id, rv); 5226 DWARN(DBG_ALL_LDCS, 5227 "ldc_mem_rdwr_cookie: (0x%llx) dir=%lld, " 5228 "tar_pa=0x%llx, loc_ra=0x%llx, tar_poff=0x%llx, " 5229 "loc_poff=0x%llx, tar_psz=0x%llx, loc_psz=0x%llx, " 5230 "copy_sz=0x%llx, total_bal=0x%llx\n", 5231 ldcp->id, direction, target_pa, local_ra, 5232 target_poff, local_poff, target_psize, local_psize, 5233 copy_size, target_size); 5234 5235 *size = *size - target_size; 5236 mutex_exit(&ldcp->lock); 5237 return (i_ldc_h2v_error(rv)); 5238 } 5239 5240 D2(ldcp->id, "ldc_mem_rdwr_cookie: copied=0x%llx\n", 5241 copied_len); 5242 target_poff += copied_len; 5243 local_poff += copied_len; 5244 target_psize -= copied_len; 5245 local_psize -= copied_len; 5246 5247 target_size -= copied_len; 5248 5249 if (copy_size != copied_len) 5250 continue; 5251 5252 if (target_psize == 0 && target_size != 0) { 5253 target_pa += pg_size; 5254 target_poff = 0; 5255 target_psize = min(pg_size, target_size); 5256 } 5257 5258 if (local_psize == 0 && target_size != 0) { 5259 local_valign += pg_size; 5260 local_ra = va_to_pa((void *)local_valign); 5261 local_poff = 0; 5262 local_psize = min(pg_size, len); 5263 len -= local_psize; 5264 } 5265 5266 /* check if we are all done */ 5267 if (target_size == 0) 5268 break; 5269 } 5270 5271 mutex_exit(&ldcp->lock); 5272 5273 D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) done copying sz=0x%llx\n", 5274 ldcp->id, *size); 5275 5276 return (0); 5277 } 5278 5279 /* 5280 * Map an exported memory segment into the local address space. If the 5281 * memory range was exported for direct map access, a HV call is made 5282 * to allocate a RA range. If the map is done via a shadow copy, local 5283 * shadow memory is allocated and the base VA is returned in 'vaddr'. If 5284 * the mapping is a direct map then the RA is returned in 'raddr'. 5285 */ 5286 int 5287 ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie, uint32_t ccount, 5288 uint8_t mtype, uint8_t perm, caddr_t *vaddr, caddr_t *raddr) 5289 { 5290 int i, j, idx, rv, retries; 5291 ldc_chan_t *ldcp; 5292 ldc_mhdl_t *mhdl; 5293 ldc_memseg_t *memseg; 5294 caddr_t tmpaddr; 5295 uint64_t map_perm = perm; 5296 uint64_t pg_size, pg_shift, pg_size_code, pg_mask; 5297 uint64_t exp_size = 0, base_off, map_size, npages; 5298 uint64_t cookie_addr, cookie_off, cookie_size; 5299 tte_t ldc_tte; 5300 5301 if (mhandle == NULL) { 5302 DWARN(DBG_ALL_LDCS, "ldc_mem_map: invalid memory handle\n"); 5303 return (EINVAL); 5304 } 5305 mhdl = (ldc_mhdl_t *)mhandle; 5306 5307 mutex_enter(&mhdl->lock); 5308 5309 if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED || 5310 mhdl->memseg != NULL) { 5311 DWARN(DBG_ALL_LDCS, 5312 "ldc_mem_map: (0x%llx) handle bound/mapped\n", mhandle); 5313 mutex_exit(&mhdl->lock); 5314 return (EINVAL); 5315 } 5316 5317 ldcp = mhdl->ldcp; 5318 5319 mutex_enter(&ldcp->lock); 5320 5321 if (ldcp->tstate != TS_UP) { 5322 DWARN(ldcp->id, 5323 "ldc_mem_dring_map: (0x%llx) channel is not UP\n", 5324 ldcp->id); 5325 mutex_exit(&ldcp->lock); 5326 mutex_exit(&mhdl->lock); 5327 return (ECONNRESET); 5328 } 5329 5330 if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) { 5331 DWARN(ldcp->id, "ldc_mem_map: invalid map type\n"); 5332 mutex_exit(&ldcp->lock); 5333 mutex_exit(&mhdl->lock); 5334 return (EINVAL); 5335 } 5336 5337 D1(ldcp->id, "ldc_mem_map: (0x%llx) cookie = 0x%llx,0x%llx\n", 5338 ldcp->id, cookie->addr, cookie->size); 5339 5340 /* FUTURE: get the page size, pgsz code, and shift */ 5341 pg_size = MMU_PAGESIZE; 5342 pg_size_code = page_szc(pg_size); 5343 pg_shift = page_get_shift(pg_size_code); 5344 pg_mask = ~(pg_size - 1); 5345 5346 /* calculate the number of pages in the exported cookie */ 5347 base_off = cookie[0].addr & (pg_size - 1); 5348 for (idx = 0; idx < ccount; idx++) 5349 exp_size += cookie[idx].size; 5350 map_size = P2ROUNDUP((exp_size + base_off), pg_size); 5351 npages = (map_size >> pg_shift); 5352 5353 /* Allocate memseg structure */ 5354 memseg = mhdl->memseg = 5355 kmem_cache_alloc(ldcssp->memseg_cache, KM_SLEEP); 5356 5357 /* Allocate memory to store all pages and cookies */ 5358 memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP); 5359 memseg->cookies = 5360 kmem_zalloc((sizeof (ldc_mem_cookie_t) * ccount), KM_SLEEP); 5361 5362 D2(ldcp->id, "ldc_mem_map: (0x%llx) exp_size=0x%llx, map_size=0x%llx," 5363 "pages=0x%llx\n", ldcp->id, exp_size, map_size, npages); 5364 5365 /* 5366 * Check if direct map over shared memory is enabled, if not change 5367 * the mapping type to SHADOW_MAP. 5368 */ 5369 if (ldc_shmem_enabled == 0) 5370 mtype = LDC_SHADOW_MAP; 5371 5372 /* 5373 * Check to see if the client is requesting direct or shadow map 5374 * If direct map is requested, try to map remote memory first, 5375 * and if that fails, revert to shadow map 5376 */ 5377 if (mtype == LDC_DIRECT_MAP) { 5378 5379 /* Allocate kernel virtual space for mapping */ 5380 memseg->vaddr = vmem_xalloc(heap_arena, map_size, 5381 pg_size, 0, 0, NULL, NULL, VM_NOSLEEP); 5382 if (memseg->vaddr == NULL) { 5383 cmn_err(CE_WARN, 5384 "ldc_mem_map: (0x%lx) memory map failed\n", 5385 ldcp->id); 5386 kmem_free(memseg->cookies, 5387 (sizeof (ldc_mem_cookie_t) * ccount)); 5388 kmem_free(memseg->pages, 5389 (sizeof (ldc_page_t) * npages)); 5390 kmem_cache_free(ldcssp->memseg_cache, memseg); 5391 5392 mutex_exit(&ldcp->lock); 5393 mutex_exit(&mhdl->lock); 5394 return (ENOMEM); 5395 } 5396 5397 /* Unload previous mapping */ 5398 hat_unload(kas.a_hat, memseg->vaddr, map_size, 5399 HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK); 5400 5401 /* for each cookie passed in - map into address space */ 5402 idx = 0; 5403 cookie_size = 0; 5404 tmpaddr = memseg->vaddr; 5405 5406 for (i = 0; i < npages; i++) { 5407 5408 if (cookie_size == 0) { 5409 ASSERT(idx < ccount); 5410 cookie_addr = cookie[idx].addr & pg_mask; 5411 cookie_off = cookie[idx].addr & (pg_size - 1); 5412 cookie_size = 5413 P2ROUNDUP((cookie_off + cookie[idx].size), 5414 pg_size); 5415 idx++; 5416 } 5417 5418 D1(ldcp->id, "ldc_mem_map: (0x%llx) mapping " 5419 "cookie 0x%llx, bal=0x%llx\n", ldcp->id, 5420 cookie_addr, cookie_size); 5421 5422 /* map the cookie into address space */ 5423 for (retries = 0; retries < ldc_max_retries; 5424 retries++) { 5425 5426 rv = hv_ldc_mapin(ldcp->id, cookie_addr, 5427 &memseg->pages[i].raddr, &map_perm); 5428 if (rv != H_EWOULDBLOCK && rv != H_ETOOMANY) 5429 break; 5430 5431 drv_usecwait(ldc_delay); 5432 } 5433 5434 if (rv || memseg->pages[i].raddr == 0) { 5435 DWARN(ldcp->id, 5436 "ldc_mem_map: (0x%llx) hv mapin err %d\n", 5437 ldcp->id, rv); 5438 5439 /* remove previous mapins */ 5440 hat_unload(kas.a_hat, memseg->vaddr, map_size, 5441 HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK); 5442 for (j = 0; j < i; j++) { 5443 rv = hv_ldc_unmap( 5444 memseg->pages[j].raddr); 5445 if (rv) { 5446 DWARN(ldcp->id, 5447 "ldc_mem_map: (0x%llx) " 5448 "cannot unmap ra=0x%llx\n", 5449 ldcp->id, 5450 memseg->pages[j].raddr); 5451 } 5452 } 5453 5454 /* free kernel virtual space */ 5455 vmem_free(heap_arena, (void *)memseg->vaddr, 5456 map_size); 5457 5458 /* direct map failed - revert to shadow map */ 5459 mtype = LDC_SHADOW_MAP; 5460 break; 5461 5462 } else { 5463 5464 D1(ldcp->id, 5465 "ldc_mem_map: (0x%llx) vtop map 0x%llx -> " 5466 "0x%llx, cookie=0x%llx, perm=0x%llx\n", 5467 ldcp->id, tmpaddr, memseg->pages[i].raddr, 5468 cookie_addr, perm); 5469 5470 /* 5471 * NOTE: Calling hat_devload directly, causes it 5472 * to look for page_t using the pfn. Since this 5473 * addr is greater than the memlist, it treates 5474 * it as non-memory 5475 */ 5476 sfmmu_memtte(&ldc_tte, 5477 (pfn_t)(memseg->pages[i].raddr >> pg_shift), 5478 PROT_READ | PROT_WRITE | HAT_NOSYNC, TTE8K); 5479 5480 D1(ldcp->id, 5481 "ldc_mem_map: (0x%llx) ra 0x%llx -> " 5482 "tte 0x%llx\n", ldcp->id, 5483 memseg->pages[i].raddr, ldc_tte); 5484 5485 sfmmu_tteload(kas.a_hat, &ldc_tte, tmpaddr, 5486 NULL, HAT_LOAD_LOCK); 5487 5488 cookie_size -= pg_size; 5489 cookie_addr += pg_size; 5490 tmpaddr += pg_size; 5491 } 5492 } 5493 } 5494 5495 if (mtype == LDC_SHADOW_MAP) { 5496 if (*vaddr == NULL) { 5497 memseg->vaddr = kmem_zalloc(exp_size, KM_SLEEP); 5498 mhdl->myshadow = B_TRUE; 5499 5500 D1(ldcp->id, "ldc_mem_map: (0x%llx) allocated " 5501 "shadow page va=0x%llx\n", ldcp->id, memseg->vaddr); 5502 } else { 5503 /* 5504 * Use client supplied memory for memseg->vaddr 5505 * WARNING: assuming that client mem is >= exp_size 5506 */ 5507 memseg->vaddr = *vaddr; 5508 } 5509 5510 /* Save all page and cookie information */ 5511 for (i = 0, tmpaddr = memseg->vaddr; i < npages; i++) { 5512 memseg->pages[i].raddr = va_to_pa(tmpaddr); 5513 memseg->pages[i].size = pg_size; 5514 tmpaddr += pg_size; 5515 } 5516 5517 } 5518 5519 /* save all cookies */ 5520 bcopy(cookie, memseg->cookies, ccount * sizeof (ldc_mem_cookie_t)); 5521 5522 /* update memseg_t */ 5523 memseg->raddr = memseg->pages[0].raddr; 5524 memseg->size = (mtype == LDC_SHADOW_MAP) ? exp_size : map_size; 5525 memseg->npages = npages; 5526 memseg->ncookies = ccount; 5527 memseg->next_cookie = 0; 5528 5529 /* memory handle = mapped */ 5530 mhdl->mtype = mtype; 5531 mhdl->perm = perm; 5532 mhdl->status = LDC_MAPPED; 5533 5534 D1(ldcp->id, "ldc_mem_map: (0x%llx) mapped 0x%llx, ra=0x%llx, " 5535 "va=0x%llx, pgs=0x%llx cookies=0x%llx\n", 5536 ldcp->id, mhdl, memseg->raddr, memseg->vaddr, 5537 memseg->npages, memseg->ncookies); 5538 5539 if (mtype == LDC_SHADOW_MAP) 5540 base_off = 0; 5541 if (raddr) 5542 *raddr = (caddr_t)(memseg->raddr | base_off); 5543 if (vaddr) 5544 *vaddr = (caddr_t)((uintptr_t)memseg->vaddr | base_off); 5545 5546 mutex_exit(&ldcp->lock); 5547 mutex_exit(&mhdl->lock); 5548 return (0); 5549 } 5550 5551 /* 5552 * Unmap a memory segment. Free shadow memory (if any). 5553 */ 5554 int 5555 ldc_mem_unmap(ldc_mem_handle_t mhandle) 5556 { 5557 int i, rv; 5558 ldc_mhdl_t *mhdl = (ldc_mhdl_t *)mhandle; 5559 ldc_chan_t *ldcp; 5560 ldc_memseg_t *memseg; 5561 5562 if (mhdl == 0 || mhdl->status != LDC_MAPPED) { 5563 DWARN(DBG_ALL_LDCS, 5564 "ldc_mem_unmap: (0x%llx) handle is not mapped\n", 5565 mhandle); 5566 return (EINVAL); 5567 } 5568 5569 mutex_enter(&mhdl->lock); 5570 5571 ldcp = mhdl->ldcp; 5572 memseg = mhdl->memseg; 5573 5574 D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapping handle 0x%llx\n", 5575 ldcp->id, mhdl); 5576 5577 /* if we allocated shadow memory - free it */ 5578 if (mhdl->mtype == LDC_SHADOW_MAP && mhdl->myshadow) { 5579 kmem_free(memseg->vaddr, memseg->size); 5580 } else if (mhdl->mtype == LDC_DIRECT_MAP) { 5581 5582 /* unmap in the case of DIRECT_MAP */ 5583 hat_unload(kas.a_hat, memseg->vaddr, memseg->size, 5584 HAT_UNLOAD_UNLOCK); 5585 5586 for (i = 0; i < memseg->npages; i++) { 5587 rv = hv_ldc_unmap(memseg->pages[i].raddr); 5588 if (rv) { 5589 cmn_err(CE_WARN, 5590 "ldc_mem_map: (0x%lx) hv unmap err %d\n", 5591 ldcp->id, rv); 5592 } 5593 } 5594 5595 vmem_free(heap_arena, (void *)memseg->vaddr, memseg->size); 5596 } 5597 5598 /* free the allocated memseg and page structures */ 5599 kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages)); 5600 kmem_free(memseg->cookies, 5601 (sizeof (ldc_mem_cookie_t) * memseg->ncookies)); 5602 kmem_cache_free(ldcssp->memseg_cache, memseg); 5603 5604 /* uninitialize the memory handle */ 5605 mhdl->memseg = NULL; 5606 mhdl->status = LDC_UNBOUND; 5607 5608 D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapped handle 0x%llx\n", 5609 ldcp->id, mhdl); 5610 5611 mutex_exit(&mhdl->lock); 5612 return (0); 5613 } 5614 5615 /* 5616 * Internal entry point for LDC mapped memory entry consistency 5617 * semantics. Acquire copies the contents of the remote memory 5618 * into the local shadow copy. The release operation copies the local 5619 * contents into the remote memory. The offset and size specify the 5620 * bounds for the memory range being synchronized. 5621 */ 5622 static int 5623 i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, uint8_t direction, 5624 uint64_t offset, size_t size) 5625 { 5626 int err; 5627 ldc_mhdl_t *mhdl; 5628 ldc_chan_t *ldcp; 5629 ldc_memseg_t *memseg; 5630 caddr_t local_vaddr; 5631 size_t copy_size; 5632 5633 if (mhandle == NULL) { 5634 DWARN(DBG_ALL_LDCS, 5635 "i_ldc_mem_acquire_release: invalid memory handle\n"); 5636 return (EINVAL); 5637 } 5638 mhdl = (ldc_mhdl_t *)mhandle; 5639 5640 mutex_enter(&mhdl->lock); 5641 5642 if (mhdl->status != LDC_MAPPED || mhdl->ldcp == NULL) { 5643 DWARN(DBG_ALL_LDCS, 5644 "i_ldc_mem_acquire_release: not mapped memory\n"); 5645 mutex_exit(&mhdl->lock); 5646 return (EINVAL); 5647 } 5648 5649 /* do nothing for direct map */ 5650 if (mhdl->mtype == LDC_DIRECT_MAP) { 5651 mutex_exit(&mhdl->lock); 5652 return (0); 5653 } 5654 5655 /* do nothing if COPY_IN+MEM_W and COPY_OUT+MEM_R */ 5656 if ((direction == LDC_COPY_IN && (mhdl->perm & LDC_MEM_R) == 0) || 5657 (direction == LDC_COPY_OUT && (mhdl->perm & LDC_MEM_W) == 0)) { 5658 mutex_exit(&mhdl->lock); 5659 return (0); 5660 } 5661 5662 if (offset >= mhdl->memseg->size || 5663 (offset + size) > mhdl->memseg->size) { 5664 DWARN(DBG_ALL_LDCS, 5665 "i_ldc_mem_acquire_release: memory out of range\n"); 5666 mutex_exit(&mhdl->lock); 5667 return (EINVAL); 5668 } 5669 5670 /* get the channel handle and memory segment */ 5671 ldcp = mhdl->ldcp; 5672 memseg = mhdl->memseg; 5673 5674 if (mhdl->mtype == LDC_SHADOW_MAP) { 5675 5676 local_vaddr = memseg->vaddr + offset; 5677 copy_size = size; 5678 5679 /* copy to/from remote from/to local memory */ 5680 err = ldc_mem_copy((ldc_handle_t)ldcp, local_vaddr, offset, 5681 ©_size, memseg->cookies, memseg->ncookies, 5682 direction); 5683 if (err || copy_size != size) { 5684 DWARN(ldcp->id, 5685 "i_ldc_mem_acquire_release: copy failed\n"); 5686 mutex_exit(&mhdl->lock); 5687 return (err); 5688 } 5689 } 5690 5691 mutex_exit(&mhdl->lock); 5692 5693 return (0); 5694 } 5695 5696 /* 5697 * Ensure that the contents in the remote memory seg are consistent 5698 * with the contents if of local segment 5699 */ 5700 int 5701 ldc_mem_acquire(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size) 5702 { 5703 return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_IN, offset, size)); 5704 } 5705 5706 5707 /* 5708 * Ensure that the contents in the local memory seg are consistent 5709 * with the contents if of remote segment 5710 */ 5711 int 5712 ldc_mem_release(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size) 5713 { 5714 return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_OUT, offset, size)); 5715 } 5716 5717 /* 5718 * Allocate a descriptor ring. The size of each each descriptor 5719 * must be 8-byte aligned and the entire ring should be a multiple 5720 * of MMU_PAGESIZE. 5721 */ 5722 int 5723 ldc_mem_dring_create(uint32_t len, uint32_t dsize, ldc_dring_handle_t *dhandle) 5724 { 5725 ldc_dring_t *dringp; 5726 size_t size = (dsize * len); 5727 5728 D1(DBG_ALL_LDCS, "ldc_mem_dring_create: len=0x%x, size=0x%x\n", 5729 len, dsize); 5730 5731 if (dhandle == NULL) { 5732 DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid dhandle\n"); 5733 return (EINVAL); 5734 } 5735 5736 if (len == 0) { 5737 DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid length\n"); 5738 return (EINVAL); 5739 } 5740 5741 /* descriptor size should be 8-byte aligned */ 5742 if (dsize == 0 || (dsize & 0x7)) { 5743 DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid size\n"); 5744 return (EINVAL); 5745 } 5746 5747 *dhandle = 0; 5748 5749 /* Allocate a desc ring structure */ 5750 dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP); 5751 5752 /* Initialize dring */ 5753 dringp->length = len; 5754 dringp->dsize = dsize; 5755 5756 /* round off to multiple of pagesize */ 5757 dringp->size = (size & MMU_PAGEMASK); 5758 if (size & MMU_PAGEOFFSET) 5759 dringp->size += MMU_PAGESIZE; 5760 5761 dringp->status = LDC_UNBOUND; 5762 5763 /* allocate descriptor ring memory */ 5764 dringp->base = kmem_zalloc(dringp->size, KM_SLEEP); 5765 5766 /* initialize the desc ring lock */ 5767 mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL); 5768 5769 /* Add descriptor ring to the head of global list */ 5770 mutex_enter(&ldcssp->lock); 5771 dringp->next = ldcssp->dring_list; 5772 ldcssp->dring_list = dringp; 5773 mutex_exit(&ldcssp->lock); 5774 5775 *dhandle = (ldc_dring_handle_t)dringp; 5776 5777 D1(DBG_ALL_LDCS, "ldc_mem_dring_create: dring allocated\n"); 5778 5779 return (0); 5780 } 5781 5782 5783 /* 5784 * Destroy a descriptor ring. 5785 */ 5786 int 5787 ldc_mem_dring_destroy(ldc_dring_handle_t dhandle) 5788 { 5789 ldc_dring_t *dringp; 5790 ldc_dring_t *tmp_dringp; 5791 5792 D1(DBG_ALL_LDCS, "ldc_mem_dring_destroy: entered\n"); 5793 5794 if (dhandle == NULL) { 5795 DWARN(DBG_ALL_LDCS, 5796 "ldc_mem_dring_destroy: invalid desc ring handle\n"); 5797 return (EINVAL); 5798 } 5799 dringp = (ldc_dring_t *)dhandle; 5800 5801 if (dringp->status == LDC_BOUND) { 5802 DWARN(DBG_ALL_LDCS, 5803 "ldc_mem_dring_destroy: desc ring is bound\n"); 5804 return (EACCES); 5805 } 5806 5807 mutex_enter(&dringp->lock); 5808 mutex_enter(&ldcssp->lock); 5809 5810 /* remove from linked list - if not bound */ 5811 tmp_dringp = ldcssp->dring_list; 5812 if (tmp_dringp == dringp) { 5813 ldcssp->dring_list = dringp->next; 5814 dringp->next = NULL; 5815 5816 } else { 5817 while (tmp_dringp != NULL) { 5818 if (tmp_dringp->next == dringp) { 5819 tmp_dringp->next = dringp->next; 5820 dringp->next = NULL; 5821 break; 5822 } 5823 tmp_dringp = tmp_dringp->next; 5824 } 5825 if (tmp_dringp == NULL) { 5826 DWARN(DBG_ALL_LDCS, 5827 "ldc_mem_dring_destroy: invalid descriptor\n"); 5828 mutex_exit(&ldcssp->lock); 5829 mutex_exit(&dringp->lock); 5830 return (EINVAL); 5831 } 5832 } 5833 5834 mutex_exit(&ldcssp->lock); 5835 5836 /* free the descriptor ring */ 5837 kmem_free(dringp->base, dringp->size); 5838 5839 mutex_exit(&dringp->lock); 5840 5841 /* destroy dring lock */ 5842 mutex_destroy(&dringp->lock); 5843 5844 /* free desc ring object */ 5845 kmem_free(dringp, sizeof (ldc_dring_t)); 5846 5847 return (0); 5848 } 5849 5850 /* 5851 * Bind a previously allocated dring to a channel. The channel should 5852 * be OPEN in order to bind the ring to the channel. Returns back a 5853 * descriptor ring cookie. The descriptor ring is exported for remote 5854 * access by the client at the other end of the channel. An entry for 5855 * dring pages is stored in map table (via call to ldc_mem_bind_handle). 5856 */ 5857 int 5858 ldc_mem_dring_bind(ldc_handle_t handle, ldc_dring_handle_t dhandle, 5859 uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount) 5860 { 5861 int err; 5862 ldc_chan_t *ldcp; 5863 ldc_dring_t *dringp; 5864 ldc_mem_handle_t mhandle; 5865 5866 /* check to see if channel is initalized */ 5867 if (handle == NULL) { 5868 DWARN(DBG_ALL_LDCS, 5869 "ldc_mem_dring_bind: invalid channel handle\n"); 5870 return (EINVAL); 5871 } 5872 ldcp = (ldc_chan_t *)handle; 5873 5874 if (dhandle == NULL) { 5875 DWARN(DBG_ALL_LDCS, 5876 "ldc_mem_dring_bind: invalid desc ring handle\n"); 5877 return (EINVAL); 5878 } 5879 dringp = (ldc_dring_t *)dhandle; 5880 5881 if (cookie == NULL) { 5882 DWARN(ldcp->id, 5883 "ldc_mem_dring_bind: invalid cookie arg\n"); 5884 return (EINVAL); 5885 } 5886 5887 mutex_enter(&dringp->lock); 5888 5889 if (dringp->status == LDC_BOUND) { 5890 DWARN(DBG_ALL_LDCS, 5891 "ldc_mem_dring_bind: (0x%llx) descriptor ring is bound\n", 5892 ldcp->id); 5893 mutex_exit(&dringp->lock); 5894 return (EINVAL); 5895 } 5896 5897 if ((perm & LDC_MEM_RW) == 0) { 5898 DWARN(DBG_ALL_LDCS, 5899 "ldc_mem_dring_bind: invalid permissions\n"); 5900 mutex_exit(&dringp->lock); 5901 return (EINVAL); 5902 } 5903 5904 if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) { 5905 DWARN(DBG_ALL_LDCS, "ldc_mem_dring_bind: invalid type\n"); 5906 mutex_exit(&dringp->lock); 5907 return (EINVAL); 5908 } 5909 5910 dringp->ldcp = ldcp; 5911 5912 /* create an memory handle */ 5913 err = ldc_mem_alloc_handle(handle, &mhandle); 5914 if (err || mhandle == NULL) { 5915 DWARN(DBG_ALL_LDCS, 5916 "ldc_mem_dring_bind: (0x%llx) error allocating mhandle\n", 5917 ldcp->id); 5918 mutex_exit(&dringp->lock); 5919 return (err); 5920 } 5921 dringp->mhdl = mhandle; 5922 5923 /* bind the descriptor ring to channel */ 5924 err = ldc_mem_bind_handle(mhandle, dringp->base, dringp->size, 5925 mtype, perm, cookie, ccount); 5926 if (err) { 5927 DWARN(ldcp->id, 5928 "ldc_mem_dring_bind: (0x%llx) error binding mhandle\n", 5929 ldcp->id); 5930 mutex_exit(&dringp->lock); 5931 return (err); 5932 } 5933 5934 /* 5935 * For now return error if we get more than one cookie 5936 * FUTURE: Return multiple cookies .. 5937 */ 5938 if (*ccount > 1) { 5939 (void) ldc_mem_unbind_handle(mhandle); 5940 (void) ldc_mem_free_handle(mhandle); 5941 5942 dringp->ldcp = NULL; 5943 dringp->mhdl = NULL; 5944 *ccount = 0; 5945 5946 mutex_exit(&dringp->lock); 5947 return (EAGAIN); 5948 } 5949 5950 /* Add descriptor ring to channel's exported dring list */ 5951 mutex_enter(&ldcp->exp_dlist_lock); 5952 dringp->ch_next = ldcp->exp_dring_list; 5953 ldcp->exp_dring_list = dringp; 5954 mutex_exit(&ldcp->exp_dlist_lock); 5955 5956 dringp->status = LDC_BOUND; 5957 5958 mutex_exit(&dringp->lock); 5959 5960 return (0); 5961 } 5962 5963 /* 5964 * Return the next cookie associated with the specified dring handle 5965 */ 5966 int 5967 ldc_mem_dring_nextcookie(ldc_dring_handle_t dhandle, ldc_mem_cookie_t *cookie) 5968 { 5969 int rv = 0; 5970 ldc_dring_t *dringp; 5971 ldc_chan_t *ldcp; 5972 5973 if (dhandle == NULL) { 5974 DWARN(DBG_ALL_LDCS, 5975 "ldc_mem_dring_nextcookie: invalid desc ring handle\n"); 5976 return (EINVAL); 5977 } 5978 dringp = (ldc_dring_t *)dhandle; 5979 mutex_enter(&dringp->lock); 5980 5981 if (dringp->status != LDC_BOUND) { 5982 DWARN(DBG_ALL_LDCS, 5983 "ldc_mem_dring_nextcookie: descriptor ring 0x%llx " 5984 "is not bound\n", dringp); 5985 mutex_exit(&dringp->lock); 5986 return (EINVAL); 5987 } 5988 5989 ldcp = dringp->ldcp; 5990 5991 if (cookie == NULL) { 5992 DWARN(ldcp->id, 5993 "ldc_mem_dring_nextcookie:(0x%llx) invalid cookie arg\n", 5994 ldcp->id); 5995 mutex_exit(&dringp->lock); 5996 return (EINVAL); 5997 } 5998 5999 rv = ldc_mem_nextcookie((ldc_mem_handle_t)dringp->mhdl, cookie); 6000 mutex_exit(&dringp->lock); 6001 6002 return (rv); 6003 } 6004 /* 6005 * Unbind a previously bound dring from a channel. 6006 */ 6007 int 6008 ldc_mem_dring_unbind(ldc_dring_handle_t dhandle) 6009 { 6010 ldc_dring_t *dringp; 6011 ldc_dring_t *tmp_dringp; 6012 ldc_chan_t *ldcp; 6013 6014 if (dhandle == NULL) { 6015 DWARN(DBG_ALL_LDCS, 6016 "ldc_mem_dring_unbind: invalid desc ring handle\n"); 6017 return (EINVAL); 6018 } 6019 dringp = (ldc_dring_t *)dhandle; 6020 6021 mutex_enter(&dringp->lock); 6022 6023 if (dringp->status == LDC_UNBOUND) { 6024 DWARN(DBG_ALL_LDCS, 6025 "ldc_mem_dring_bind: descriptor ring 0x%llx is unbound\n", 6026 dringp); 6027 mutex_exit(&dringp->lock); 6028 return (EINVAL); 6029 } 6030 ldcp = dringp->ldcp; 6031 6032 mutex_enter(&ldcp->exp_dlist_lock); 6033 6034 tmp_dringp = ldcp->exp_dring_list; 6035 if (tmp_dringp == dringp) { 6036 ldcp->exp_dring_list = dringp->ch_next; 6037 dringp->ch_next = NULL; 6038 6039 } else { 6040 while (tmp_dringp != NULL) { 6041 if (tmp_dringp->ch_next == dringp) { 6042 tmp_dringp->ch_next = dringp->ch_next; 6043 dringp->ch_next = NULL; 6044 break; 6045 } 6046 tmp_dringp = tmp_dringp->ch_next; 6047 } 6048 if (tmp_dringp == NULL) { 6049 DWARN(DBG_ALL_LDCS, 6050 "ldc_mem_dring_unbind: invalid descriptor\n"); 6051 mutex_exit(&ldcp->exp_dlist_lock); 6052 mutex_exit(&dringp->lock); 6053 return (EINVAL); 6054 } 6055 } 6056 6057 mutex_exit(&ldcp->exp_dlist_lock); 6058 6059 (void) ldc_mem_unbind_handle((ldc_mem_handle_t)dringp->mhdl); 6060 (void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl); 6061 6062 dringp->ldcp = NULL; 6063 dringp->mhdl = NULL; 6064 dringp->status = LDC_UNBOUND; 6065 6066 mutex_exit(&dringp->lock); 6067 6068 return (0); 6069 } 6070 6071 /* 6072 * Get information about the dring. The base address of the descriptor 6073 * ring along with the type and permission are returned back. 6074 */ 6075 int 6076 ldc_mem_dring_info(ldc_dring_handle_t dhandle, ldc_mem_info_t *minfo) 6077 { 6078 ldc_dring_t *dringp; 6079 int rv; 6080 6081 if (dhandle == NULL) { 6082 DWARN(DBG_ALL_LDCS, 6083 "ldc_mem_dring_info: invalid desc ring handle\n"); 6084 return (EINVAL); 6085 } 6086 dringp = (ldc_dring_t *)dhandle; 6087 6088 mutex_enter(&dringp->lock); 6089 6090 if (dringp->mhdl) { 6091 rv = ldc_mem_info(dringp->mhdl, minfo); 6092 if (rv) { 6093 DWARN(DBG_ALL_LDCS, 6094 "ldc_mem_dring_info: error reading mem info\n"); 6095 mutex_exit(&dringp->lock); 6096 return (rv); 6097 } 6098 } else { 6099 minfo->vaddr = dringp->base; 6100 minfo->raddr = NULL; 6101 minfo->status = dringp->status; 6102 } 6103 6104 mutex_exit(&dringp->lock); 6105 6106 return (0); 6107 } 6108 6109 /* 6110 * Map an exported descriptor ring into the local address space. If the 6111 * descriptor ring was exported for direct map access, a HV call is made 6112 * to allocate a RA range. If the map is done via a shadow copy, local 6113 * shadow memory is allocated. 6114 */ 6115 int 6116 ldc_mem_dring_map(ldc_handle_t handle, ldc_mem_cookie_t *cookie, 6117 uint32_t ccount, uint32_t len, uint32_t dsize, uint8_t mtype, 6118 ldc_dring_handle_t *dhandle) 6119 { 6120 int err; 6121 ldc_chan_t *ldcp = (ldc_chan_t *)handle; 6122 ldc_mem_handle_t mhandle; 6123 ldc_dring_t *dringp; 6124 size_t dring_size; 6125 6126 if (dhandle == NULL) { 6127 DWARN(DBG_ALL_LDCS, 6128 "ldc_mem_dring_map: invalid dhandle\n"); 6129 return (EINVAL); 6130 } 6131 6132 /* check to see if channel is initalized */ 6133 if (handle == NULL) { 6134 DWARN(DBG_ALL_LDCS, 6135 "ldc_mem_dring_map: invalid channel handle\n"); 6136 return (EINVAL); 6137 } 6138 ldcp = (ldc_chan_t *)handle; 6139 6140 if (cookie == NULL) { 6141 DWARN(ldcp->id, 6142 "ldc_mem_dring_map: (0x%llx) invalid cookie\n", 6143 ldcp->id); 6144 return (EINVAL); 6145 } 6146 6147 /* FUTURE: For now we support only one cookie per dring */ 6148 ASSERT(ccount == 1); 6149 6150 if (cookie->size < (dsize * len)) { 6151 DWARN(ldcp->id, 6152 "ldc_mem_dring_map: (0x%llx) invalid dsize/len\n", 6153 ldcp->id); 6154 return (EINVAL); 6155 } 6156 6157 *dhandle = 0; 6158 6159 /* Allocate an dring structure */ 6160 dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP); 6161 6162 D1(ldcp->id, 6163 "ldc_mem_dring_map: 0x%x,0x%x,0x%x,0x%llx,0x%llx\n", 6164 mtype, len, dsize, cookie->addr, cookie->size); 6165 6166 /* Initialize dring */ 6167 dringp->length = len; 6168 dringp->dsize = dsize; 6169 6170 /* round of to multiple of page size */ 6171 dring_size = len * dsize; 6172 dringp->size = (dring_size & MMU_PAGEMASK); 6173 if (dring_size & MMU_PAGEOFFSET) 6174 dringp->size += MMU_PAGESIZE; 6175 6176 dringp->ldcp = ldcp; 6177 6178 /* create an memory handle */ 6179 err = ldc_mem_alloc_handle(handle, &mhandle); 6180 if (err || mhandle == NULL) { 6181 DWARN(DBG_ALL_LDCS, 6182 "ldc_mem_dring_map: cannot alloc hdl err=%d\n", 6183 err); 6184 kmem_free(dringp, sizeof (ldc_dring_t)); 6185 return (ENOMEM); 6186 } 6187 6188 dringp->mhdl = mhandle; 6189 dringp->base = NULL; 6190 6191 /* map the dring into local memory */ 6192 err = ldc_mem_map(mhandle, cookie, ccount, mtype, LDC_MEM_RW, 6193 &(dringp->base), NULL); 6194 if (err || dringp->base == NULL) { 6195 cmn_err(CE_WARN, 6196 "ldc_mem_dring_map: cannot map desc ring err=%d\n", err); 6197 (void) ldc_mem_free_handle(mhandle); 6198 kmem_free(dringp, sizeof (ldc_dring_t)); 6199 return (ENOMEM); 6200 } 6201 6202 /* initialize the desc ring lock */ 6203 mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL); 6204 6205 /* Add descriptor ring to channel's imported dring list */ 6206 mutex_enter(&ldcp->imp_dlist_lock); 6207 dringp->ch_next = ldcp->imp_dring_list; 6208 ldcp->imp_dring_list = dringp; 6209 mutex_exit(&ldcp->imp_dlist_lock); 6210 6211 dringp->status = LDC_MAPPED; 6212 6213 *dhandle = (ldc_dring_handle_t)dringp; 6214 6215 return (0); 6216 } 6217 6218 /* 6219 * Unmap a descriptor ring. Free shadow memory (if any). 6220 */ 6221 int 6222 ldc_mem_dring_unmap(ldc_dring_handle_t dhandle) 6223 { 6224 ldc_dring_t *dringp; 6225 ldc_dring_t *tmp_dringp; 6226 ldc_chan_t *ldcp; 6227 6228 if (dhandle == NULL) { 6229 DWARN(DBG_ALL_LDCS, 6230 "ldc_mem_dring_unmap: invalid desc ring handle\n"); 6231 return (EINVAL); 6232 } 6233 dringp = (ldc_dring_t *)dhandle; 6234 6235 if (dringp->status != LDC_MAPPED) { 6236 DWARN(DBG_ALL_LDCS, 6237 "ldc_mem_dring_unmap: not a mapped desc ring\n"); 6238 return (EINVAL); 6239 } 6240 6241 mutex_enter(&dringp->lock); 6242 6243 ldcp = dringp->ldcp; 6244 6245 mutex_enter(&ldcp->imp_dlist_lock); 6246 6247 /* find and unlink the desc ring from channel import list */ 6248 tmp_dringp = ldcp->imp_dring_list; 6249 if (tmp_dringp == dringp) { 6250 ldcp->imp_dring_list = dringp->ch_next; 6251 dringp->ch_next = NULL; 6252 6253 } else { 6254 while (tmp_dringp != NULL) { 6255 if (tmp_dringp->ch_next == dringp) { 6256 tmp_dringp->ch_next = dringp->ch_next; 6257 dringp->ch_next = NULL; 6258 break; 6259 } 6260 tmp_dringp = tmp_dringp->ch_next; 6261 } 6262 if (tmp_dringp == NULL) { 6263 DWARN(DBG_ALL_LDCS, 6264 "ldc_mem_dring_unmap: invalid descriptor\n"); 6265 mutex_exit(&ldcp->imp_dlist_lock); 6266 mutex_exit(&dringp->lock); 6267 return (EINVAL); 6268 } 6269 } 6270 6271 mutex_exit(&ldcp->imp_dlist_lock); 6272 6273 /* do a LDC memory handle unmap and free */ 6274 (void) ldc_mem_unmap(dringp->mhdl); 6275 (void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl); 6276 6277 dringp->status = 0; 6278 dringp->ldcp = NULL; 6279 6280 mutex_exit(&dringp->lock); 6281 6282 /* destroy dring lock */ 6283 mutex_destroy(&dringp->lock); 6284 6285 /* free desc ring object */ 6286 kmem_free(dringp, sizeof (ldc_dring_t)); 6287 6288 return (0); 6289 } 6290 6291 /* 6292 * Internal entry point for descriptor ring access entry consistency 6293 * semantics. Acquire copies the contents of the remote descriptor ring 6294 * into the local shadow copy. The release operation copies the local 6295 * contents into the remote dring. The start and end locations specify 6296 * bounds for the entries being synchronized. 6297 */ 6298 static int 6299 i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle, 6300 uint8_t direction, uint64_t start, uint64_t end) 6301 { 6302 int err; 6303 ldc_dring_t *dringp; 6304 ldc_chan_t *ldcp; 6305 uint64_t soff; 6306 size_t copy_size; 6307 6308 if (dhandle == NULL) { 6309 DWARN(DBG_ALL_LDCS, 6310 "i_ldc_dring_acquire_release: invalid desc ring handle\n"); 6311 return (EINVAL); 6312 } 6313 dringp = (ldc_dring_t *)dhandle; 6314 mutex_enter(&dringp->lock); 6315 6316 if (dringp->status != LDC_MAPPED || dringp->ldcp == NULL) { 6317 DWARN(DBG_ALL_LDCS, 6318 "i_ldc_dring_acquire_release: not a mapped desc ring\n"); 6319 mutex_exit(&dringp->lock); 6320 return (EINVAL); 6321 } 6322 6323 if (start >= dringp->length || end >= dringp->length) { 6324 DWARN(DBG_ALL_LDCS, 6325 "i_ldc_dring_acquire_release: index out of range\n"); 6326 mutex_exit(&dringp->lock); 6327 return (EINVAL); 6328 } 6329 6330 /* get the channel handle */ 6331 ldcp = dringp->ldcp; 6332 6333 copy_size = (start <= end) ? (((end - start) + 1) * dringp->dsize) : 6334 ((dringp->length - start) * dringp->dsize); 6335 6336 /* Calculate the relative offset for the first desc */ 6337 soff = (start * dringp->dsize); 6338 6339 /* copy to/from remote from/to local memory */ 6340 D1(ldcp->id, "i_ldc_dring_acquire_release: c1 off=0x%llx sz=0x%llx\n", 6341 soff, copy_size); 6342 err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl, 6343 direction, soff, copy_size); 6344 if (err) { 6345 DWARN(ldcp->id, 6346 "i_ldc_dring_acquire_release: copy failed\n"); 6347 mutex_exit(&dringp->lock); 6348 return (err); 6349 } 6350 6351 /* do the balance */ 6352 if (start > end) { 6353 copy_size = ((end + 1) * dringp->dsize); 6354 soff = 0; 6355 6356 /* copy to/from remote from/to local memory */ 6357 D1(ldcp->id, "i_ldc_dring_acquire_release: c2 " 6358 "off=0x%llx sz=0x%llx\n", soff, copy_size); 6359 err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl, 6360 direction, soff, copy_size); 6361 if (err) { 6362 DWARN(ldcp->id, 6363 "i_ldc_dring_acquire_release: copy failed\n"); 6364 mutex_exit(&dringp->lock); 6365 return (err); 6366 } 6367 } 6368 6369 mutex_exit(&dringp->lock); 6370 6371 return (0); 6372 } 6373 6374 /* 6375 * Ensure that the contents in the local dring are consistent 6376 * with the contents if of remote dring 6377 */ 6378 int 6379 ldc_mem_dring_acquire(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end) 6380 { 6381 return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_IN, start, end)); 6382 } 6383 6384 /* 6385 * Ensure that the contents in the remote dring are consistent 6386 * with the contents if of local dring 6387 */ 6388 int 6389 ldc_mem_dring_release(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end) 6390 { 6391 return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_OUT, start, end)); 6392 } 6393 6394 6395 /* ------------------------------------------------------------------------- */ 6396