1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * sun4v LDC Link Layer 31 */ 32 #include <sys/types.h> 33 #include <sys/file.h> 34 #include <sys/errno.h> 35 #include <sys/open.h> 36 #include <sys/cred.h> 37 #include <sys/kmem.h> 38 #include <sys/conf.h> 39 #include <sys/cmn_err.h> 40 #include <sys/ksynch.h> 41 #include <sys/modctl.h> 42 #include <sys/stat.h> /* needed for S_IFBLK and S_IFCHR */ 43 #include <sys/debug.h> 44 #include <sys/types.h> 45 #include <sys/cred.h> 46 #include <sys/promif.h> 47 #include <sys/ddi.h> 48 #include <sys/sunddi.h> 49 #include <sys/cyclic.h> 50 #include <sys/machsystm.h> 51 #include <sys/vm.h> 52 #include <sys/cpu.h> 53 #include <sys/intreg.h> 54 #include <sys/machcpuvar.h> 55 #include <sys/mmu.h> 56 #include <sys/pte.h> 57 #include <vm/hat.h> 58 #include <vm/as.h> 59 #include <vm/hat_sfmmu.h> 60 #include <sys/vm_machparam.h> 61 #include <vm/seg_kmem.h> 62 #include <vm/seg_kpm.h> 63 #include <sys/note.h> 64 #include <sys/ivintr.h> 65 #include <sys/hypervisor_api.h> 66 #include <sys/ldc.h> 67 #include <sys/ldc_impl.h> 68 #include <sys/cnex.h> 69 #include <sys/hsvc.h> 70 71 /* Core internal functions */ 72 static int i_ldc_h2v_error(int h_error); 73 static int i_ldc_txq_reconf(ldc_chan_t *ldcp); 74 static int i_ldc_rxq_reconf(ldc_chan_t *ldcp, boolean_t force_reset); 75 static int i_ldc_rxq_drain(ldc_chan_t *ldcp); 76 static void i_ldc_reset_state(ldc_chan_t *ldcp); 77 static void i_ldc_reset(ldc_chan_t *ldcp, boolean_t force_reset); 78 79 static int i_ldc_get_tx_tail(ldc_chan_t *ldcp, uint64_t *tail); 80 static int i_ldc_set_tx_tail(ldc_chan_t *ldcp, uint64_t tail); 81 static int i_ldc_set_rx_head(ldc_chan_t *ldcp, uint64_t head); 82 static int i_ldc_send_pkt(ldc_chan_t *ldcp, uint8_t pkttype, uint8_t subtype, 83 uint8_t ctrlmsg); 84 85 /* Interrupt handling functions */ 86 static uint_t i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2); 87 static uint_t i_ldc_rx_hdlr(caddr_t arg1, caddr_t arg2); 88 static void i_ldc_clear_intr(ldc_chan_t *ldcp, cnex_intrtype_t itype); 89 90 /* Read method functions */ 91 static int i_ldc_read_raw(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep); 92 static int i_ldc_read_packet(ldc_chan_t *ldcp, caddr_t target_bufp, 93 size_t *sizep); 94 static int i_ldc_read_stream(ldc_chan_t *ldcp, caddr_t target_bufp, 95 size_t *sizep); 96 97 /* Write method functions */ 98 static int i_ldc_write_raw(ldc_chan_t *ldcp, caddr_t target_bufp, 99 size_t *sizep); 100 static int i_ldc_write_packet(ldc_chan_t *ldcp, caddr_t target_bufp, 101 size_t *sizep); 102 static int i_ldc_write_stream(ldc_chan_t *ldcp, caddr_t target_bufp, 103 size_t *sizep); 104 105 /* Pkt processing internal functions */ 106 static int i_ldc_check_seqid(ldc_chan_t *ldcp, ldc_msg_t *ldcmsg); 107 static int i_ldc_ctrlmsg(ldc_chan_t *ldcp, ldc_msg_t *ldcmsg); 108 static int i_ldc_process_VER(ldc_chan_t *ldcp, ldc_msg_t *msg); 109 static int i_ldc_process_RTS(ldc_chan_t *ldcp, ldc_msg_t *msg); 110 static int i_ldc_process_RTR(ldc_chan_t *ldcp, ldc_msg_t *msg); 111 static int i_ldc_process_RDX(ldc_chan_t *ldcp, ldc_msg_t *msg); 112 static int i_ldc_process_data_ACK(ldc_chan_t *ldcp, ldc_msg_t *msg); 113 114 /* Memory synchronization internal functions */ 115 static int i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, 116 uint8_t direction, uint64_t offset, size_t size); 117 static int i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle, 118 uint8_t direction, uint64_t start, uint64_t end); 119 120 /* LDC Version */ 121 static ldc_ver_t ldc_versions[] = { {1, 0} }; 122 123 /* number of supported versions */ 124 #define LDC_NUM_VERS (sizeof (ldc_versions) / sizeof (ldc_versions[0])) 125 126 /* Module State Pointer */ 127 static ldc_soft_state_t *ldcssp; 128 129 static struct modldrv md = { 130 &mod_miscops, /* This is a misc module */ 131 "sun4v LDC module v%I%", /* Name of the module */ 132 }; 133 134 static struct modlinkage ml = { 135 MODREV_1, 136 &md, 137 NULL 138 }; 139 140 static uint64_t ldc_sup_minor; /* Supported minor number */ 141 static hsvc_info_t ldc_hsvc = { 142 HSVC_REV_1, NULL, HSVC_GROUP_LDC, 1, 0, "ldc" 143 }; 144 145 static uint64_t intr_sup_minor; /* Supported minor number */ 146 static hsvc_info_t intr_hsvc = { 147 HSVC_REV_1, NULL, HSVC_GROUP_INTR, 1, 0, "ldc" 148 }; 149 150 /* 151 * LDC framework supports mapping remote domain's memory 152 * either directly or via shadow memory pages. Default 153 * support is currently implemented via shadow copy. 154 * Direct map can be enabled by setting 'ldc_shmem_enabled' 155 */ 156 int ldc_shmem_enabled = 0; 157 158 /* 159 * The no. of MTU size messages that can be stored in 160 * the LDC Tx queue. The number of Tx queue entries is 161 * then computed as (mtu * mtu_msgs)/sizeof(queue_entry) 162 */ 163 uint64_t ldc_mtu_msgs = LDC_MTU_MSGS; 164 165 /* 166 * The minimum queue length. This is the size of the smallest 167 * LDC queue. If the computed value is less than this default, 168 * the queue length is rounded up to 'ldc_queue_entries'. 169 */ 170 uint64_t ldc_queue_entries = LDC_QUEUE_ENTRIES; 171 172 /* 173 * Pages exported for remote access over each channel is 174 * maintained in a table registered with the Hypervisor. 175 * The default number of entries in the table is set to 176 * 'ldc_mtbl_entries'. 177 */ 178 uint64_t ldc_maptable_entries = LDC_MTBL_ENTRIES; 179 180 /* 181 * LDC retry count and delay - when the HV returns EWOULDBLOCK 182 * the operation is retried 'ldc_max_retries' times with a 183 * wait of 'ldc_delay' usecs between each retry. 184 */ 185 int ldc_max_retries = LDC_MAX_RETRIES; 186 clock_t ldc_delay = LDC_DELAY; 187 188 #ifdef DEBUG 189 190 /* 191 * Print debug messages 192 * 193 * set ldcdbg to 0x7 for enabling all msgs 194 * 0x4 - Warnings 195 * 0x2 - All debug messages 196 * 0x1 - Minimal debug messages 197 * 198 * set ldcdbgchan to the channel number you want to debug 199 * setting it to -1 prints debug messages for all channels 200 * NOTE: ldcdbgchan has no effect on error messages 201 */ 202 203 #define DBG_ALL_LDCS -1 204 205 int ldcdbg = 0x0; 206 int64_t ldcdbgchan = DBG_ALL_LDCS; 207 boolean_t ldc_inject_reset_flag = B_FALSE; 208 209 static void 210 ldcdebug(int64_t id, const char *fmt, ...) 211 { 212 char buf[512]; 213 va_list ap; 214 215 /* 216 * Do not return if, 217 * caller wants to print it anyway - (id == DBG_ALL_LDCS) 218 * debug channel is set to all LDCs - (ldcdbgchan == DBG_ALL_LDCS) 219 * debug channel = caller specified channel 220 */ 221 if ((id != DBG_ALL_LDCS) && 222 (ldcdbgchan != DBG_ALL_LDCS) && 223 (ldcdbgchan != id)) { 224 return; 225 } 226 227 va_start(ap, fmt); 228 (void) vsprintf(buf, fmt, ap); 229 va_end(ap); 230 231 cmn_err(CE_CONT, "?%s", buf); 232 } 233 234 static boolean_t 235 ldc_inject_reset(ldc_chan_t *ldcp) 236 { 237 if ((ldcdbgchan != DBG_ALL_LDCS) && (ldcdbgchan != ldcp->id)) 238 return (B_FALSE); 239 240 if (!ldc_inject_reset_flag) 241 return (B_FALSE); 242 243 /* clear the injection state */ 244 ldc_inject_reset_flag = 0; 245 246 return (B_TRUE); 247 } 248 249 #define D1 \ 250 if (ldcdbg & 0x01) \ 251 ldcdebug 252 253 #define D2 \ 254 if (ldcdbg & 0x02) \ 255 ldcdebug 256 257 #define DWARN \ 258 if (ldcdbg & 0x04) \ 259 ldcdebug 260 261 #define DUMP_PAYLOAD(id, addr) \ 262 { \ 263 char buf[65*3]; \ 264 int i; \ 265 uint8_t *src = (uint8_t *)addr; \ 266 for (i = 0; i < 64; i++, src++) \ 267 (void) sprintf(&buf[i * 3], "|%02x", *src); \ 268 (void) sprintf(&buf[i * 3], "|\n"); \ 269 D2((id), "payload: %s", buf); \ 270 } 271 272 #define DUMP_LDC_PKT(c, s, addr) \ 273 { \ 274 ldc_msg_t *msg = (ldc_msg_t *)(addr); \ 275 uint32_t mid = ((c)->mode != LDC_MODE_RAW) ? msg->seqid : 0; \ 276 if (msg->type == LDC_DATA) { \ 277 D2((c)->id, "%s: msg%d (/%x/%x/%x/,env[%c%c,sz=%d])", \ 278 (s), mid, msg->type, msg->stype, msg->ctrl, \ 279 (msg->env & LDC_FRAG_START) ? 'B' : ' ', \ 280 (msg->env & LDC_FRAG_STOP) ? 'E' : ' ', \ 281 (msg->env & LDC_LEN_MASK)); \ 282 } else { \ 283 D2((c)->id, "%s: msg%d (/%x/%x/%x/,env=%x)", (s), \ 284 mid, msg->type, msg->stype, msg->ctrl, msg->env); \ 285 } \ 286 } 287 288 #define LDC_INJECT_RESET(_ldcp) ldc_inject_reset(_ldcp) 289 290 #else 291 292 #define DBG_ALL_LDCS -1 293 294 #define D1 295 #define D2 296 #define DWARN 297 298 #define DUMP_PAYLOAD(id, addr) 299 #define DUMP_LDC_PKT(c, s, addr) 300 301 #define LDC_INJECT_RESET(_ldcp) (B_FALSE) 302 303 #endif 304 305 #define ZERO_PKT(p) \ 306 bzero((p), sizeof (ldc_msg_t)); 307 308 #define IDX2COOKIE(idx, pg_szc, pg_shift) \ 309 (((pg_szc) << LDC_COOKIE_PGSZC_SHIFT) | ((idx) << (pg_shift))) 310 311 312 int 313 _init(void) 314 { 315 int status; 316 317 status = hsvc_register(&ldc_hsvc, &ldc_sup_minor); 318 if (status != 0) { 319 cmn_err(CE_WARN, "%s: cannot negotiate hypervisor LDC services" 320 " group: 0x%lx major: %ld minor: %ld errno: %d", 321 ldc_hsvc.hsvc_modname, ldc_hsvc.hsvc_group, 322 ldc_hsvc.hsvc_major, ldc_hsvc.hsvc_minor, status); 323 return (-1); 324 } 325 326 status = hsvc_register(&intr_hsvc, &intr_sup_minor); 327 if (status != 0) { 328 cmn_err(CE_WARN, "%s: cannot negotiate hypervisor interrupt " 329 "services group: 0x%lx major: %ld minor: %ld errno: %d", 330 intr_hsvc.hsvc_modname, intr_hsvc.hsvc_group, 331 intr_hsvc.hsvc_major, intr_hsvc.hsvc_minor, status); 332 (void) hsvc_unregister(&ldc_hsvc); 333 return (-1); 334 } 335 336 /* allocate soft state structure */ 337 ldcssp = kmem_zalloc(sizeof (ldc_soft_state_t), KM_SLEEP); 338 339 /* Link the module into the system */ 340 status = mod_install(&ml); 341 if (status != 0) { 342 kmem_free(ldcssp, sizeof (ldc_soft_state_t)); 343 return (status); 344 } 345 346 /* Initialize the LDC state structure */ 347 mutex_init(&ldcssp->lock, NULL, MUTEX_DRIVER, NULL); 348 349 mutex_enter(&ldcssp->lock); 350 351 /* Create a cache for memory handles */ 352 ldcssp->memhdl_cache = kmem_cache_create("ldc_memhdl_cache", 353 sizeof (ldc_mhdl_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 354 if (ldcssp->memhdl_cache == NULL) { 355 DWARN(DBG_ALL_LDCS, "_init: ldc_memhdl cache create failed\n"); 356 mutex_exit(&ldcssp->lock); 357 return (-1); 358 } 359 360 /* Create cache for memory segment structures */ 361 ldcssp->memseg_cache = kmem_cache_create("ldc_memseg_cache", 362 sizeof (ldc_memseg_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 363 if (ldcssp->memseg_cache == NULL) { 364 DWARN(DBG_ALL_LDCS, "_init: ldc_memseg cache create failed\n"); 365 mutex_exit(&ldcssp->lock); 366 return (-1); 367 } 368 369 370 ldcssp->channel_count = 0; 371 ldcssp->channels_open = 0; 372 ldcssp->chan_list = NULL; 373 ldcssp->dring_list = NULL; 374 375 mutex_exit(&ldcssp->lock); 376 377 return (0); 378 } 379 380 int 381 _info(struct modinfo *modinfop) 382 { 383 /* Report status of the dynamically loadable driver module */ 384 return (mod_info(&ml, modinfop)); 385 } 386 387 int 388 _fini(void) 389 { 390 int rv, status; 391 ldc_chan_t *ldcp; 392 ldc_dring_t *dringp; 393 ldc_mem_info_t minfo; 394 395 /* Unlink the driver module from the system */ 396 status = mod_remove(&ml); 397 if (status) { 398 DWARN(DBG_ALL_LDCS, "_fini: mod_remove failed\n"); 399 return (EIO); 400 } 401 402 /* close and finalize channels */ 403 ldcp = ldcssp->chan_list; 404 while (ldcp != NULL) { 405 (void) ldc_close((ldc_handle_t)ldcp); 406 (void) ldc_fini((ldc_handle_t)ldcp); 407 408 ldcp = ldcp->next; 409 } 410 411 /* Free descriptor rings */ 412 dringp = ldcssp->dring_list; 413 while (dringp != NULL) { 414 dringp = dringp->next; 415 416 rv = ldc_mem_dring_info((ldc_dring_handle_t)dringp, &minfo); 417 if (rv == 0 && minfo.status != LDC_UNBOUND) { 418 if (minfo.status == LDC_BOUND) { 419 (void) ldc_mem_dring_unbind( 420 (ldc_dring_handle_t)dringp); 421 } 422 if (minfo.status == LDC_MAPPED) { 423 (void) ldc_mem_dring_unmap( 424 (ldc_dring_handle_t)dringp); 425 } 426 } 427 428 (void) ldc_mem_dring_destroy((ldc_dring_handle_t)dringp); 429 } 430 ldcssp->dring_list = NULL; 431 432 /* Destroy kmem caches */ 433 kmem_cache_destroy(ldcssp->memhdl_cache); 434 kmem_cache_destroy(ldcssp->memseg_cache); 435 436 /* 437 * We have successfully "removed" the driver. 438 * Destroying soft states 439 */ 440 mutex_destroy(&ldcssp->lock); 441 kmem_free(ldcssp, sizeof (ldc_soft_state_t)); 442 443 (void) hsvc_unregister(&ldc_hsvc); 444 (void) hsvc_unregister(&intr_hsvc); 445 446 return (status); 447 } 448 449 /* -------------------------------------------------------------------------- */ 450 451 /* 452 * LDC Link Layer Internal Functions 453 */ 454 455 /* 456 * Translate HV Errors to sun4v error codes 457 */ 458 static int 459 i_ldc_h2v_error(int h_error) 460 { 461 switch (h_error) { 462 463 case H_EOK: 464 return (0); 465 466 case H_ENORADDR: 467 return (EFAULT); 468 469 case H_EBADPGSZ: 470 case H_EINVAL: 471 return (EINVAL); 472 473 case H_EWOULDBLOCK: 474 return (EWOULDBLOCK); 475 476 case H_ENOACCESS: 477 case H_ENOMAP: 478 return (EACCES); 479 480 case H_EIO: 481 case H_ECPUERROR: 482 return (EIO); 483 484 case H_ENOTSUPPORTED: 485 return (ENOTSUP); 486 487 case H_ETOOMANY: 488 return (ENOSPC); 489 490 case H_ECHANNEL: 491 return (ECHRNG); 492 default: 493 break; 494 } 495 496 return (EIO); 497 } 498 499 /* 500 * Reconfigure the transmit queue 501 */ 502 static int 503 i_ldc_txq_reconf(ldc_chan_t *ldcp) 504 { 505 int rv; 506 507 ASSERT(MUTEX_HELD(&ldcp->lock)); 508 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 509 510 rv = hv_ldc_tx_qconf(ldcp->id, ldcp->tx_q_ra, ldcp->tx_q_entries); 511 if (rv) { 512 cmn_err(CE_WARN, 513 "i_ldc_txq_reconf: (0x%lx) cannot set qconf", ldcp->id); 514 return (EIO); 515 } 516 rv = hv_ldc_tx_get_state(ldcp->id, &(ldcp->tx_head), 517 &(ldcp->tx_tail), &(ldcp->link_state)); 518 if (rv) { 519 cmn_err(CE_WARN, 520 "i_ldc_txq_reconf: (0x%lx) cannot get qptrs", ldcp->id); 521 return (EIO); 522 } 523 D1(ldcp->id, "i_ldc_txq_reconf: (0x%llx) h=0x%llx,t=0x%llx," 524 "s=0x%llx\n", ldcp->id, ldcp->tx_head, ldcp->tx_tail, 525 ldcp->link_state); 526 527 return (0); 528 } 529 530 /* 531 * Reconfigure the receive queue 532 */ 533 static int 534 i_ldc_rxq_reconf(ldc_chan_t *ldcp, boolean_t force_reset) 535 { 536 int rv; 537 uint64_t rx_head, rx_tail; 538 539 ASSERT(MUTEX_HELD(&ldcp->lock)); 540 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 541 &(ldcp->link_state)); 542 if (rv) { 543 cmn_err(CE_WARN, 544 "i_ldc_rxq_reconf: (0x%lx) cannot get state", 545 ldcp->id); 546 return (EIO); 547 } 548 549 if (force_reset || (ldcp->tstate & ~TS_IN_RESET) == TS_UP) { 550 rv = hv_ldc_rx_qconf(ldcp->id, ldcp->rx_q_ra, 551 ldcp->rx_q_entries); 552 if (rv) { 553 cmn_err(CE_WARN, 554 "i_ldc_rxq_reconf: (0x%lx) cannot set qconf", 555 ldcp->id); 556 return (EIO); 557 } 558 D1(ldcp->id, "i_ldc_rxq_reconf: (0x%llx) completed q reconf", 559 ldcp->id); 560 } 561 562 return (0); 563 } 564 565 566 /* 567 * Drain the contents of the receive queue 568 */ 569 static int 570 i_ldc_rxq_drain(ldc_chan_t *ldcp) 571 { 572 int rv; 573 uint64_t rx_head, rx_tail; 574 575 ASSERT(MUTEX_HELD(&ldcp->lock)); 576 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 577 &(ldcp->link_state)); 578 if (rv) { 579 cmn_err(CE_WARN, "i_ldc_rxq_drain: (0x%lx) cannot get state", 580 ldcp->id); 581 return (EIO); 582 } 583 584 /* flush contents by setting the head = tail */ 585 return (i_ldc_set_rx_head(ldcp, rx_tail)); 586 } 587 588 589 /* 590 * Reset LDC state structure and its contents 591 */ 592 static void 593 i_ldc_reset_state(ldc_chan_t *ldcp) 594 { 595 ASSERT(MUTEX_HELD(&ldcp->lock)); 596 ldcp->last_msg_snt = LDC_INIT_SEQID; 597 ldcp->last_ack_rcd = 0; 598 ldcp->last_msg_rcd = 0; 599 ldcp->tx_ackd_head = ldcp->tx_head; 600 ldcp->next_vidx = 0; 601 ldcp->hstate = 0; 602 ldcp->tstate = TS_OPEN; 603 ldcp->status = LDC_OPEN; 604 605 if (ldcp->link_state == LDC_CHANNEL_UP || 606 ldcp->link_state == LDC_CHANNEL_RESET) { 607 608 if (ldcp->mode == LDC_MODE_RAW) { 609 ldcp->status = LDC_UP; 610 ldcp->tstate = TS_UP; 611 } else { 612 ldcp->status = LDC_READY; 613 ldcp->tstate |= TS_LINK_READY; 614 } 615 } 616 } 617 618 /* 619 * Reset a LDC channel 620 */ 621 static void 622 i_ldc_reset(ldc_chan_t *ldcp, boolean_t force_reset) 623 { 624 DWARN(ldcp->id, "i_ldc_reset: (0x%llx) channel reset\n", ldcp->id); 625 626 ASSERT(MUTEX_HELD(&ldcp->lock)); 627 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 628 629 /* reconfig Tx and Rx queues */ 630 (void) i_ldc_txq_reconf(ldcp); 631 (void) i_ldc_rxq_reconf(ldcp, force_reset); 632 633 /* Clear Tx and Rx interrupts */ 634 (void) i_ldc_clear_intr(ldcp, CNEX_TX_INTR); 635 (void) i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 636 637 /* Reset channel state */ 638 i_ldc_reset_state(ldcp); 639 640 /* Mark channel in reset */ 641 ldcp->tstate |= TS_IN_RESET; 642 } 643 644 645 /* 646 * Clear pending interrupts 647 */ 648 static void 649 i_ldc_clear_intr(ldc_chan_t *ldcp, cnex_intrtype_t itype) 650 { 651 ldc_cnex_t *cinfo = &ldcssp->cinfo; 652 653 ASSERT(MUTEX_HELD(&ldcp->lock)); 654 ASSERT(cinfo->dip != NULL); 655 656 switch (itype) { 657 case CNEX_TX_INTR: 658 /* check Tx interrupt */ 659 if (ldcp->tx_intr_state) 660 ldcp->tx_intr_state = LDC_INTR_NONE; 661 else 662 return; 663 break; 664 665 case CNEX_RX_INTR: 666 /* check Rx interrupt */ 667 if (ldcp->rx_intr_state) 668 ldcp->rx_intr_state = LDC_INTR_NONE; 669 else 670 return; 671 break; 672 } 673 674 (void) cinfo->clr_intr(cinfo->dip, ldcp->id, itype); 675 D2(ldcp->id, 676 "i_ldc_clear_intr: (0x%llx) cleared 0x%x intr\n", 677 ldcp->id, itype); 678 } 679 680 /* 681 * Set the receive queue head 682 * Resets connection and returns an error if it fails. 683 */ 684 static int 685 i_ldc_set_rx_head(ldc_chan_t *ldcp, uint64_t head) 686 { 687 int rv; 688 int retries; 689 690 ASSERT(MUTEX_HELD(&ldcp->lock)); 691 for (retries = 0; retries < ldc_max_retries; retries++) { 692 693 if ((rv = hv_ldc_rx_set_qhead(ldcp->id, head)) == 0) 694 return (0); 695 696 if (rv != H_EWOULDBLOCK) 697 break; 698 699 /* wait for ldc_delay usecs */ 700 drv_usecwait(ldc_delay); 701 } 702 703 cmn_err(CE_WARN, "ldc_rx_set_qhead: (0x%lx) cannot set qhead 0x%lx", 704 ldcp->id, head); 705 mutex_enter(&ldcp->tx_lock); 706 i_ldc_reset(ldcp, B_TRUE); 707 mutex_exit(&ldcp->tx_lock); 708 709 return (ECONNRESET); 710 } 711 712 713 /* 714 * Returns the tx_tail to be used for transfer 715 * Re-reads the TX queue ptrs if and only if the 716 * the cached head and tail are equal (queue is full) 717 */ 718 static int 719 i_ldc_get_tx_tail(ldc_chan_t *ldcp, uint64_t *tail) 720 { 721 int rv; 722 uint64_t current_head, new_tail; 723 724 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 725 /* Read the head and tail ptrs from HV */ 726 rv = hv_ldc_tx_get_state(ldcp->id, 727 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 728 if (rv) { 729 cmn_err(CE_WARN, 730 "i_ldc_get_tx_tail: (0x%lx) cannot read qptrs\n", 731 ldcp->id); 732 return (EIO); 733 } 734 if (ldcp->link_state == LDC_CHANNEL_DOWN) { 735 DWARN(DBG_ALL_LDCS, 736 "i_ldc_get_tx_tail: (0x%llx) channel not ready\n", 737 ldcp->id); 738 return (ECONNRESET); 739 } 740 741 /* In reliable mode, check against last ACKd msg */ 742 current_head = (ldcp->mode == LDC_MODE_RELIABLE || 743 ldcp->mode == LDC_MODE_STREAM) 744 ? ldcp->tx_ackd_head : ldcp->tx_head; 745 746 /* increment the tail */ 747 new_tail = (ldcp->tx_tail + LDC_PACKET_SIZE) % 748 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 749 750 if (new_tail == current_head) { 751 DWARN(ldcp->id, 752 "i_ldc_get_tx_tail: (0x%llx) TX queue is full\n", 753 ldcp->id); 754 return (EWOULDBLOCK); 755 } 756 757 D2(ldcp->id, "i_ldc_get_tx_tail: (0x%llx) head=0x%llx, tail=0x%llx\n", 758 ldcp->id, ldcp->tx_head, ldcp->tx_tail); 759 760 *tail = ldcp->tx_tail; 761 return (0); 762 } 763 764 /* 765 * Set the tail pointer. If HV returns EWOULDBLOCK, it will back off 766 * and retry ldc_max_retries times before returning an error. 767 * Returns 0, EWOULDBLOCK or EIO 768 */ 769 static int 770 i_ldc_set_tx_tail(ldc_chan_t *ldcp, uint64_t tail) 771 { 772 int rv, retval = EWOULDBLOCK; 773 int retries; 774 775 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 776 for (retries = 0; retries < ldc_max_retries; retries++) { 777 778 if ((rv = hv_ldc_tx_set_qtail(ldcp->id, tail)) == 0) { 779 retval = 0; 780 break; 781 } 782 if (rv != H_EWOULDBLOCK) { 783 DWARN(ldcp->id, "i_ldc_set_tx_tail: (0x%llx) set " 784 "qtail=0x%llx failed, rv=%d\n", ldcp->id, tail, rv); 785 retval = EIO; 786 break; 787 } 788 789 /* wait for ldc_delay usecs */ 790 drv_usecwait(ldc_delay); 791 } 792 return (retval); 793 } 794 795 /* 796 * Send a LDC message 797 */ 798 static int 799 i_ldc_send_pkt(ldc_chan_t *ldcp, uint8_t pkttype, uint8_t subtype, 800 uint8_t ctrlmsg) 801 { 802 int rv; 803 ldc_msg_t *pkt; 804 uint64_t tx_tail; 805 uint32_t curr_seqid = ldcp->last_msg_snt; 806 807 /* Obtain Tx lock */ 808 mutex_enter(&ldcp->tx_lock); 809 810 /* get the current tail for the message */ 811 rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 812 if (rv) { 813 DWARN(ldcp->id, 814 "i_ldc_send_pkt: (0x%llx) error sending pkt, " 815 "type=0x%x,subtype=0x%x,ctrl=0x%x\n", 816 ldcp->id, pkttype, subtype, ctrlmsg); 817 mutex_exit(&ldcp->tx_lock); 818 return (rv); 819 } 820 821 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 822 ZERO_PKT(pkt); 823 824 /* Initialize the packet */ 825 pkt->type = pkttype; 826 pkt->stype = subtype; 827 pkt->ctrl = ctrlmsg; 828 829 /* Store ackid/seqid iff it is RELIABLE mode & not a RTS/RTR message */ 830 if (((ctrlmsg & LDC_CTRL_MASK) != LDC_RTS) && 831 ((ctrlmsg & LDC_CTRL_MASK) != LDC_RTR)) { 832 curr_seqid++; 833 if (ldcp->mode != LDC_MODE_RAW) { 834 pkt->seqid = curr_seqid; 835 pkt->ackid = ldcp->last_msg_rcd; 836 } 837 } 838 DUMP_LDC_PKT(ldcp, "i_ldc_send_pkt", (uint64_t)pkt); 839 840 /* initiate the send by calling into HV and set the new tail */ 841 tx_tail = (tx_tail + LDC_PACKET_SIZE) % 842 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 843 844 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 845 if (rv) { 846 DWARN(ldcp->id, 847 "i_ldc_send_pkt:(0x%llx) error sending pkt, " 848 "type=0x%x,stype=0x%x,ctrl=0x%x\n", 849 ldcp->id, pkttype, subtype, ctrlmsg); 850 mutex_exit(&ldcp->tx_lock); 851 return (EIO); 852 } 853 854 ldcp->last_msg_snt = curr_seqid; 855 ldcp->tx_tail = tx_tail; 856 857 mutex_exit(&ldcp->tx_lock); 858 return (0); 859 } 860 861 /* 862 * Checks if packet was received in right order 863 * in the case of a reliable link. 864 * Returns 0 if in order, else EIO 865 */ 866 static int 867 i_ldc_check_seqid(ldc_chan_t *ldcp, ldc_msg_t *msg) 868 { 869 /* No seqid checking for RAW mode */ 870 if (ldcp->mode == LDC_MODE_RAW) 871 return (0); 872 873 /* No seqid checking for version, RTS, RTR message */ 874 if (msg->ctrl == LDC_VER || 875 msg->ctrl == LDC_RTS || 876 msg->ctrl == LDC_RTR) 877 return (0); 878 879 /* Initial seqid to use is sent in RTS/RTR and saved in last_msg_rcd */ 880 if (msg->seqid != (ldcp->last_msg_rcd + 1)) { 881 DWARN(ldcp->id, 882 "i_ldc_check_seqid: (0x%llx) out-of-order pkt, got 0x%x, " 883 "expecting 0x%x\n", ldcp->id, msg->seqid, 884 (ldcp->last_msg_rcd + 1)); 885 return (EIO); 886 } 887 888 return (0); 889 } 890 891 892 /* 893 * Process an incoming version ctrl message 894 */ 895 static int 896 i_ldc_process_VER(ldc_chan_t *ldcp, ldc_msg_t *msg) 897 { 898 int rv = 0, idx = ldcp->next_vidx; 899 ldc_msg_t *pkt; 900 uint64_t tx_tail; 901 ldc_ver_t *rcvd_ver; 902 903 /* get the received version */ 904 rcvd_ver = (ldc_ver_t *)((uint64_t)msg + LDC_PAYLOAD_VER_OFF); 905 906 D2(ldcp->id, "i_ldc_process_VER: (0x%llx) received VER v%u.%u\n", 907 ldcp->id, rcvd_ver->major, rcvd_ver->minor); 908 909 /* Obtain Tx lock */ 910 mutex_enter(&ldcp->tx_lock); 911 912 switch (msg->stype) { 913 case LDC_INFO: 914 915 if ((ldcp->tstate & ~TS_IN_RESET) == TS_VREADY) { 916 (void) i_ldc_txq_reconf(ldcp); 917 i_ldc_reset_state(ldcp); 918 mutex_exit(&ldcp->tx_lock); 919 return (EAGAIN); 920 } 921 922 /* get the current tail and pkt for the response */ 923 rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 924 if (rv != 0) { 925 DWARN(ldcp->id, 926 "i_ldc_process_VER: (0x%llx) err sending " 927 "version ACK/NACK\n", ldcp->id); 928 i_ldc_reset(ldcp, B_TRUE); 929 mutex_exit(&ldcp->tx_lock); 930 return (ECONNRESET); 931 } 932 933 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 934 ZERO_PKT(pkt); 935 936 /* initialize the packet */ 937 pkt->type = LDC_CTRL; 938 pkt->ctrl = LDC_VER; 939 940 for (;;) { 941 942 D1(ldcp->id, "i_ldc_process_VER: got %u.%u chk %u.%u\n", 943 rcvd_ver->major, rcvd_ver->minor, 944 ldc_versions[idx].major, ldc_versions[idx].minor); 945 946 if (rcvd_ver->major == ldc_versions[idx].major) { 947 /* major version match - ACK version */ 948 pkt->stype = LDC_ACK; 949 950 /* 951 * lower minor version to the one this endpt 952 * supports, if necessary 953 */ 954 if (rcvd_ver->minor > ldc_versions[idx].minor) 955 rcvd_ver->minor = 956 ldc_versions[idx].minor; 957 bcopy(rcvd_ver, pkt->udata, sizeof (*rcvd_ver)); 958 959 break; 960 } 961 962 if (rcvd_ver->major > ldc_versions[idx].major) { 963 964 D1(ldcp->id, "i_ldc_process_VER: using next" 965 " lower idx=%d, v%u.%u\n", idx, 966 ldc_versions[idx].major, 967 ldc_versions[idx].minor); 968 969 /* nack with next lower version */ 970 pkt->stype = LDC_NACK; 971 bcopy(&ldc_versions[idx], pkt->udata, 972 sizeof (ldc_versions[idx])); 973 ldcp->next_vidx = idx; 974 break; 975 } 976 977 /* next major version */ 978 idx++; 979 980 D1(ldcp->id, "i_ldc_process_VER: inc idx %x\n", idx); 981 982 if (idx == LDC_NUM_VERS) { 983 /* no version match - send NACK */ 984 pkt->stype = LDC_NACK; 985 bzero(pkt->udata, sizeof (ldc_ver_t)); 986 ldcp->next_vidx = 0; 987 break; 988 } 989 } 990 991 /* initiate the send by calling into HV and set the new tail */ 992 tx_tail = (tx_tail + LDC_PACKET_SIZE) % 993 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 994 995 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 996 if (rv == 0) { 997 ldcp->tx_tail = tx_tail; 998 if (pkt->stype == LDC_ACK) { 999 D2(ldcp->id, "i_ldc_process_VER: (0x%llx) sent" 1000 " version ACK\n", ldcp->id); 1001 /* Save the ACK'd version */ 1002 ldcp->version.major = rcvd_ver->major; 1003 ldcp->version.minor = rcvd_ver->minor; 1004 ldcp->hstate |= TS_RCVD_VER; 1005 ldcp->tstate |= TS_VER_DONE; 1006 DWARN(DBG_ALL_LDCS, 1007 "(0x%llx) Sent ACK, " 1008 "Agreed on version v%u.%u\n", 1009 ldcp->id, rcvd_ver->major, rcvd_ver->minor); 1010 } 1011 } else { 1012 DWARN(ldcp->id, 1013 "i_ldc_process_VER: (0x%llx) error sending " 1014 "ACK/NACK\n", ldcp->id); 1015 i_ldc_reset(ldcp, B_TRUE); 1016 mutex_exit(&ldcp->tx_lock); 1017 return (ECONNRESET); 1018 } 1019 1020 break; 1021 1022 case LDC_ACK: 1023 if ((ldcp->tstate & ~TS_IN_RESET) == TS_VREADY) { 1024 if (ldcp->version.major != rcvd_ver->major || 1025 ldcp->version.minor != rcvd_ver->minor) { 1026 1027 /* mismatched version - reset connection */ 1028 DWARN(ldcp->id, 1029 "i_ldc_process_VER: (0x%llx) recvd" 1030 " ACK ver != sent ACK ver\n", ldcp->id); 1031 i_ldc_reset(ldcp, B_TRUE); 1032 mutex_exit(&ldcp->tx_lock); 1033 return (ECONNRESET); 1034 } 1035 } else { 1036 /* SUCCESS - we have agreed on a version */ 1037 ldcp->version.major = rcvd_ver->major; 1038 ldcp->version.minor = rcvd_ver->minor; 1039 ldcp->tstate |= TS_VER_DONE; 1040 } 1041 1042 DWARN(DBG_ALL_LDCS, 1043 "(0x%llx) Got ACK, Agreed on version v%u.%u\n", 1044 ldcp->id, rcvd_ver->major, rcvd_ver->minor); 1045 1046 /* initiate RTS-RTR-RDX handshake */ 1047 rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 1048 if (rv) { 1049 DWARN(ldcp->id, 1050 "i_ldc_process_VER: (0x%llx) cannot send RTS\n", 1051 ldcp->id); 1052 i_ldc_reset(ldcp, B_TRUE); 1053 mutex_exit(&ldcp->tx_lock); 1054 return (ECONNRESET); 1055 } 1056 1057 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 1058 ZERO_PKT(pkt); 1059 1060 pkt->type = LDC_CTRL; 1061 pkt->stype = LDC_INFO; 1062 pkt->ctrl = LDC_RTS; 1063 pkt->env = ldcp->mode; 1064 if (ldcp->mode != LDC_MODE_RAW) 1065 pkt->seqid = LDC_INIT_SEQID; 1066 1067 ldcp->last_msg_rcd = LDC_INIT_SEQID; 1068 1069 DUMP_LDC_PKT(ldcp, "i_ldc_process_VER snd rts", (uint64_t)pkt); 1070 1071 /* initiate the send by calling into HV and set the new tail */ 1072 tx_tail = (tx_tail + LDC_PACKET_SIZE) % 1073 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 1074 1075 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 1076 if (rv) { 1077 D2(ldcp->id, 1078 "i_ldc_process_VER: (0x%llx) no listener\n", 1079 ldcp->id); 1080 i_ldc_reset(ldcp, B_TRUE); 1081 mutex_exit(&ldcp->tx_lock); 1082 return (ECONNRESET); 1083 } 1084 1085 ldcp->tx_tail = tx_tail; 1086 ldcp->hstate |= TS_SENT_RTS; 1087 1088 break; 1089 1090 case LDC_NACK: 1091 /* check if version in NACK is zero */ 1092 if (rcvd_ver->major == 0 && rcvd_ver->minor == 0) { 1093 /* version handshake failure */ 1094 DWARN(DBG_ALL_LDCS, 1095 "i_ldc_process_VER: (0x%llx) no version match\n", 1096 ldcp->id); 1097 i_ldc_reset(ldcp, B_TRUE); 1098 mutex_exit(&ldcp->tx_lock); 1099 return (ECONNRESET); 1100 } 1101 1102 /* get the current tail and pkt for the response */ 1103 rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 1104 if (rv != 0) { 1105 cmn_err(CE_NOTE, 1106 "i_ldc_process_VER: (0x%lx) err sending " 1107 "version ACK/NACK\n", ldcp->id); 1108 i_ldc_reset(ldcp, B_TRUE); 1109 mutex_exit(&ldcp->tx_lock); 1110 return (ECONNRESET); 1111 } 1112 1113 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 1114 ZERO_PKT(pkt); 1115 1116 /* initialize the packet */ 1117 pkt->type = LDC_CTRL; 1118 pkt->ctrl = LDC_VER; 1119 pkt->stype = LDC_INFO; 1120 1121 /* check ver in NACK msg has a match */ 1122 for (;;) { 1123 if (rcvd_ver->major == ldc_versions[idx].major) { 1124 /* 1125 * major version match - resubmit request 1126 * if lower minor version to the one this endpt 1127 * supports, if necessary 1128 */ 1129 if (rcvd_ver->minor > ldc_versions[idx].minor) 1130 rcvd_ver->minor = 1131 ldc_versions[idx].minor; 1132 bcopy(rcvd_ver, pkt->udata, sizeof (*rcvd_ver)); 1133 break; 1134 1135 } 1136 1137 if (rcvd_ver->major > ldc_versions[idx].major) { 1138 1139 D1(ldcp->id, "i_ldc_process_VER: using next" 1140 " lower idx=%d, v%u.%u\n", idx, 1141 ldc_versions[idx].major, 1142 ldc_versions[idx].minor); 1143 1144 /* send next lower version */ 1145 bcopy(&ldc_versions[idx], pkt->udata, 1146 sizeof (ldc_versions[idx])); 1147 ldcp->next_vidx = idx; 1148 break; 1149 } 1150 1151 /* next version */ 1152 idx++; 1153 1154 D1(ldcp->id, "i_ldc_process_VER: inc idx %x\n", idx); 1155 1156 if (idx == LDC_NUM_VERS) { 1157 /* no version match - terminate */ 1158 ldcp->next_vidx = 0; 1159 mutex_exit(&ldcp->tx_lock); 1160 return (ECONNRESET); 1161 } 1162 } 1163 1164 /* initiate the send by calling into HV and set the new tail */ 1165 tx_tail = (tx_tail + LDC_PACKET_SIZE) % 1166 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 1167 1168 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 1169 if (rv == 0) { 1170 D2(ldcp->id, "i_ldc_process_VER: (0x%llx) sent version" 1171 "INFO v%u.%u\n", ldcp->id, ldc_versions[idx].major, 1172 ldc_versions[idx].minor); 1173 ldcp->tx_tail = tx_tail; 1174 } else { 1175 cmn_err(CE_NOTE, 1176 "i_ldc_process_VER: (0x%lx) error sending version" 1177 "INFO\n", ldcp->id); 1178 i_ldc_reset(ldcp, B_TRUE); 1179 mutex_exit(&ldcp->tx_lock); 1180 return (ECONNRESET); 1181 } 1182 1183 break; 1184 } 1185 1186 mutex_exit(&ldcp->tx_lock); 1187 return (rv); 1188 } 1189 1190 1191 /* 1192 * Process an incoming RTS ctrl message 1193 */ 1194 static int 1195 i_ldc_process_RTS(ldc_chan_t *ldcp, ldc_msg_t *msg) 1196 { 1197 int rv = 0; 1198 ldc_msg_t *pkt; 1199 uint64_t tx_tail; 1200 boolean_t sent_NACK = B_FALSE; 1201 1202 D2(ldcp->id, "i_ldc_process_RTS: (0x%llx) received RTS\n", ldcp->id); 1203 1204 switch (msg->stype) { 1205 case LDC_NACK: 1206 DWARN(ldcp->id, 1207 "i_ldc_process_RTS: (0x%llx) RTS NACK received\n", 1208 ldcp->id); 1209 1210 /* Reset the channel -- as we cannot continue */ 1211 mutex_enter(&ldcp->tx_lock); 1212 i_ldc_reset(ldcp, B_TRUE); 1213 mutex_exit(&ldcp->tx_lock); 1214 rv = ECONNRESET; 1215 break; 1216 1217 case LDC_INFO: 1218 1219 /* check mode */ 1220 if (ldcp->mode != (ldc_mode_t)msg->env) { 1221 cmn_err(CE_NOTE, 1222 "i_ldc_process_RTS: (0x%lx) mode mismatch\n", 1223 ldcp->id); 1224 /* 1225 * send NACK in response to MODE message 1226 * get the current tail for the response 1227 */ 1228 rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, LDC_RTS); 1229 if (rv) { 1230 /* if cannot send NACK - reset channel */ 1231 mutex_enter(&ldcp->tx_lock); 1232 i_ldc_reset(ldcp, B_TRUE); 1233 mutex_exit(&ldcp->tx_lock); 1234 rv = ECONNRESET; 1235 break; 1236 } 1237 sent_NACK = B_TRUE; 1238 } 1239 break; 1240 default: 1241 DWARN(ldcp->id, "i_ldc_process_RTS: (0x%llx) unexp ACK\n", 1242 ldcp->id); 1243 mutex_enter(&ldcp->tx_lock); 1244 i_ldc_reset(ldcp, B_TRUE); 1245 mutex_exit(&ldcp->tx_lock); 1246 rv = ECONNRESET; 1247 break; 1248 } 1249 1250 /* 1251 * If either the connection was reset (when rv != 0) or 1252 * a NACK was sent, we return. In the case of a NACK 1253 * we dont want to consume the packet that came in but 1254 * not record that we received the RTS 1255 */ 1256 if (rv || sent_NACK) 1257 return (rv); 1258 1259 /* record RTS received */ 1260 ldcp->hstate |= TS_RCVD_RTS; 1261 1262 /* store initial SEQID info */ 1263 ldcp->last_msg_snt = msg->seqid; 1264 1265 /* Obtain Tx lock */ 1266 mutex_enter(&ldcp->tx_lock); 1267 1268 /* get the current tail for the response */ 1269 rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 1270 if (rv != 0) { 1271 cmn_err(CE_NOTE, 1272 "i_ldc_process_RTS: (0x%lx) err sending RTR\n", 1273 ldcp->id); 1274 i_ldc_reset(ldcp, B_TRUE); 1275 mutex_exit(&ldcp->tx_lock); 1276 return (ECONNRESET); 1277 } 1278 1279 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 1280 ZERO_PKT(pkt); 1281 1282 /* initialize the packet */ 1283 pkt->type = LDC_CTRL; 1284 pkt->stype = LDC_INFO; 1285 pkt->ctrl = LDC_RTR; 1286 pkt->env = ldcp->mode; 1287 if (ldcp->mode != LDC_MODE_RAW) 1288 pkt->seqid = LDC_INIT_SEQID; 1289 1290 ldcp->last_msg_rcd = msg->seqid; 1291 1292 /* initiate the send by calling into HV and set the new tail */ 1293 tx_tail = (tx_tail + LDC_PACKET_SIZE) % 1294 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 1295 1296 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 1297 if (rv == 0) { 1298 D2(ldcp->id, 1299 "i_ldc_process_RTS: (0x%llx) sent RTR\n", ldcp->id); 1300 DUMP_LDC_PKT(ldcp, "i_ldc_process_RTS sent rtr", (uint64_t)pkt); 1301 1302 ldcp->tx_tail = tx_tail; 1303 ldcp->hstate |= TS_SENT_RTR; 1304 1305 } else { 1306 cmn_err(CE_NOTE, 1307 "i_ldc_process_RTS: (0x%lx) error sending RTR\n", 1308 ldcp->id); 1309 i_ldc_reset(ldcp, B_TRUE); 1310 mutex_exit(&ldcp->tx_lock); 1311 return (ECONNRESET); 1312 } 1313 1314 mutex_exit(&ldcp->tx_lock); 1315 return (0); 1316 } 1317 1318 /* 1319 * Process an incoming RTR ctrl message 1320 */ 1321 static int 1322 i_ldc_process_RTR(ldc_chan_t *ldcp, ldc_msg_t *msg) 1323 { 1324 int rv = 0; 1325 boolean_t sent_NACK = B_FALSE; 1326 1327 D2(ldcp->id, "i_ldc_process_RTR: (0x%llx) received RTR\n", ldcp->id); 1328 1329 switch (msg->stype) { 1330 case LDC_NACK: 1331 /* RTR NACK received */ 1332 DWARN(ldcp->id, 1333 "i_ldc_process_RTR: (0x%llx) RTR NACK received\n", 1334 ldcp->id); 1335 1336 /* Reset the channel -- as we cannot continue */ 1337 mutex_enter(&ldcp->tx_lock); 1338 i_ldc_reset(ldcp, B_TRUE); 1339 mutex_exit(&ldcp->tx_lock); 1340 rv = ECONNRESET; 1341 1342 break; 1343 1344 case LDC_INFO: 1345 1346 /* check mode */ 1347 if (ldcp->mode != (ldc_mode_t)msg->env) { 1348 DWARN(ldcp->id, 1349 "i_ldc_process_RTR: (0x%llx) mode mismatch\n", 1350 ldcp->id); 1351 /* 1352 * send NACK in response to MODE message 1353 * get the current tail for the response 1354 */ 1355 rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, LDC_RTR); 1356 if (rv) { 1357 /* if cannot send NACK - reset channel */ 1358 mutex_enter(&ldcp->tx_lock); 1359 i_ldc_reset(ldcp, B_TRUE); 1360 mutex_exit(&ldcp->tx_lock); 1361 rv = ECONNRESET; 1362 break; 1363 } 1364 sent_NACK = B_TRUE; 1365 } 1366 break; 1367 1368 default: 1369 DWARN(ldcp->id, "i_ldc_process_RTR: (0x%llx) unexp ACK\n", 1370 ldcp->id); 1371 1372 /* Reset the channel -- as we cannot continue */ 1373 mutex_enter(&ldcp->tx_lock); 1374 i_ldc_reset(ldcp, B_TRUE); 1375 mutex_exit(&ldcp->tx_lock); 1376 rv = ECONNRESET; 1377 break; 1378 } 1379 1380 /* 1381 * If either the connection was reset (when rv != 0) or 1382 * a NACK was sent, we return. In the case of a NACK 1383 * we dont want to consume the packet that came in but 1384 * not record that we received the RTR 1385 */ 1386 if (rv || sent_NACK) 1387 return (rv); 1388 1389 ldcp->last_msg_snt = msg->seqid; 1390 ldcp->hstate |= TS_RCVD_RTR; 1391 1392 rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_INFO, LDC_RDX); 1393 if (rv) { 1394 cmn_err(CE_NOTE, 1395 "i_ldc_process_RTR: (0x%lx) cannot send RDX\n", 1396 ldcp->id); 1397 mutex_enter(&ldcp->tx_lock); 1398 i_ldc_reset(ldcp, B_TRUE); 1399 mutex_exit(&ldcp->tx_lock); 1400 return (ECONNRESET); 1401 } 1402 D2(ldcp->id, 1403 "i_ldc_process_RTR: (0x%llx) sent RDX\n", ldcp->id); 1404 1405 ldcp->hstate |= TS_SENT_RDX; 1406 ldcp->tstate |= TS_HSHAKE_DONE; 1407 if ((ldcp->tstate & TS_IN_RESET) == 0) 1408 ldcp->status = LDC_UP; 1409 1410 DWARN(DBG_ALL_LDCS, "(0x%llx) Handshake Complete\n", ldcp->id); 1411 1412 return (0); 1413 } 1414 1415 1416 /* 1417 * Process an incoming RDX ctrl message 1418 */ 1419 static int 1420 i_ldc_process_RDX(ldc_chan_t *ldcp, ldc_msg_t *msg) 1421 { 1422 int rv = 0; 1423 1424 D2(ldcp->id, "i_ldc_process_RDX: (0x%llx) received RDX\n", ldcp->id); 1425 1426 switch (msg->stype) { 1427 case LDC_NACK: 1428 /* RDX NACK received */ 1429 DWARN(ldcp->id, 1430 "i_ldc_process_RDX: (0x%llx) RDX NACK received\n", 1431 ldcp->id); 1432 1433 /* Reset the channel -- as we cannot continue */ 1434 mutex_enter(&ldcp->tx_lock); 1435 i_ldc_reset(ldcp, B_TRUE); 1436 mutex_exit(&ldcp->tx_lock); 1437 rv = ECONNRESET; 1438 1439 break; 1440 1441 case LDC_INFO: 1442 1443 /* 1444 * if channel is UP and a RDX received after data transmission 1445 * has commenced it is an error 1446 */ 1447 if ((ldcp->tstate == TS_UP) && (ldcp->hstate & TS_RCVD_RDX)) { 1448 DWARN(DBG_ALL_LDCS, 1449 "i_ldc_process_RDX: (0x%llx) unexpected RDX" 1450 " - LDC reset\n", ldcp->id); 1451 mutex_enter(&ldcp->tx_lock); 1452 i_ldc_reset(ldcp, B_TRUE); 1453 mutex_exit(&ldcp->tx_lock); 1454 return (ECONNRESET); 1455 } 1456 1457 ldcp->hstate |= TS_RCVD_RDX; 1458 ldcp->tstate |= TS_HSHAKE_DONE; 1459 if ((ldcp->tstate & TS_IN_RESET) == 0) 1460 ldcp->status = LDC_UP; 1461 1462 D1(DBG_ALL_LDCS, "(0x%llx) Handshake Complete\n", ldcp->id); 1463 break; 1464 1465 default: 1466 DWARN(ldcp->id, "i_ldc_process_RDX: (0x%llx) unexp ACK\n", 1467 ldcp->id); 1468 1469 /* Reset the channel -- as we cannot continue */ 1470 mutex_enter(&ldcp->tx_lock); 1471 i_ldc_reset(ldcp, B_TRUE); 1472 mutex_exit(&ldcp->tx_lock); 1473 rv = ECONNRESET; 1474 break; 1475 } 1476 1477 return (rv); 1478 } 1479 1480 /* 1481 * Process an incoming ACK for a data packet 1482 */ 1483 static int 1484 i_ldc_process_data_ACK(ldc_chan_t *ldcp, ldc_msg_t *msg) 1485 { 1486 int rv; 1487 uint64_t tx_head; 1488 ldc_msg_t *pkt; 1489 1490 /* Obtain Tx lock */ 1491 mutex_enter(&ldcp->tx_lock); 1492 1493 /* 1494 * Read the current Tx head and tail 1495 */ 1496 rv = hv_ldc_tx_get_state(ldcp->id, 1497 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 1498 if (rv != 0) { 1499 cmn_err(CE_WARN, 1500 "i_ldc_process_data_ACK: (0x%lx) cannot read qptrs\n", 1501 ldcp->id); 1502 1503 /* Reset the channel -- as we cannot continue */ 1504 i_ldc_reset(ldcp, B_TRUE); 1505 mutex_exit(&ldcp->tx_lock); 1506 return (ECONNRESET); 1507 } 1508 1509 /* 1510 * loop from where the previous ACK location was to the 1511 * current head location. This is how far the HV has 1512 * actually send pkts. Pkts between head and tail are 1513 * yet to be sent by HV. 1514 */ 1515 tx_head = ldcp->tx_ackd_head; 1516 for (;;) { 1517 pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_head); 1518 tx_head = (tx_head + LDC_PACKET_SIZE) % 1519 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 1520 1521 if (pkt->seqid == msg->ackid) { 1522 D2(ldcp->id, 1523 "i_ldc_process_data_ACK: (0x%llx) found packet\n", 1524 ldcp->id); 1525 ldcp->last_ack_rcd = msg->ackid; 1526 ldcp->tx_ackd_head = tx_head; 1527 break; 1528 } 1529 if (tx_head == ldcp->tx_head) { 1530 /* could not find packet */ 1531 DWARN(ldcp->id, 1532 "i_ldc_process_data_ACK: (0x%llx) invalid ACKid\n", 1533 ldcp->id); 1534 1535 /* Reset the channel -- as we cannot continue */ 1536 i_ldc_reset(ldcp, B_TRUE); 1537 mutex_exit(&ldcp->tx_lock); 1538 return (ECONNRESET); 1539 } 1540 } 1541 1542 mutex_exit(&ldcp->tx_lock); 1543 return (0); 1544 } 1545 1546 /* 1547 * Process incoming control message 1548 * Return 0 - session can continue 1549 * EAGAIN - reprocess packet - state was changed 1550 * ECONNRESET - channel was reset 1551 */ 1552 static int 1553 i_ldc_ctrlmsg(ldc_chan_t *ldcp, ldc_msg_t *msg) 1554 { 1555 int rv = 0; 1556 1557 D1(ldcp->id, "i_ldc_ctrlmsg: (%llx) tstate = %lx, hstate = %lx\n", 1558 ldcp->id, ldcp->tstate, ldcp->hstate); 1559 1560 switch (ldcp->tstate & ~TS_IN_RESET) { 1561 1562 case TS_OPEN: 1563 case TS_READY: 1564 1565 switch (msg->ctrl & LDC_CTRL_MASK) { 1566 case LDC_VER: 1567 /* process version message */ 1568 rv = i_ldc_process_VER(ldcp, msg); 1569 break; 1570 default: 1571 DWARN(ldcp->id, 1572 "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 1573 "tstate=0x%x\n", ldcp->id, 1574 (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 1575 break; 1576 } 1577 1578 break; 1579 1580 case TS_VREADY: 1581 1582 switch (msg->ctrl & LDC_CTRL_MASK) { 1583 case LDC_VER: 1584 /* process version message */ 1585 rv = i_ldc_process_VER(ldcp, msg); 1586 break; 1587 case LDC_RTS: 1588 /* process RTS message */ 1589 rv = i_ldc_process_RTS(ldcp, msg); 1590 break; 1591 case LDC_RTR: 1592 /* process RTR message */ 1593 rv = i_ldc_process_RTR(ldcp, msg); 1594 break; 1595 case LDC_RDX: 1596 /* process RDX message */ 1597 rv = i_ldc_process_RDX(ldcp, msg); 1598 break; 1599 default: 1600 DWARN(ldcp->id, 1601 "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 1602 "tstate=0x%x\n", ldcp->id, 1603 (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 1604 break; 1605 } 1606 1607 break; 1608 1609 case TS_UP: 1610 1611 switch (msg->ctrl & LDC_CTRL_MASK) { 1612 case LDC_VER: 1613 DWARN(ldcp->id, 1614 "i_ldc_ctrlmsg: (0x%llx) unexpected VER " 1615 "- LDC reset\n", ldcp->id); 1616 /* peer is redoing version negotiation */ 1617 mutex_enter(&ldcp->tx_lock); 1618 (void) i_ldc_txq_reconf(ldcp); 1619 i_ldc_reset_state(ldcp); 1620 mutex_exit(&ldcp->tx_lock); 1621 rv = EAGAIN; 1622 break; 1623 1624 case LDC_RDX: 1625 /* process RDX message */ 1626 rv = i_ldc_process_RDX(ldcp, msg); 1627 break; 1628 1629 default: 1630 DWARN(ldcp->id, 1631 "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 1632 "tstate=0x%x\n", ldcp->id, 1633 (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 1634 break; 1635 } 1636 } 1637 1638 return (rv); 1639 } 1640 1641 /* 1642 * Register channel with the channel nexus 1643 */ 1644 static int 1645 i_ldc_register_channel(ldc_chan_t *ldcp) 1646 { 1647 int rv = 0; 1648 ldc_cnex_t *cinfo = &ldcssp->cinfo; 1649 1650 if (cinfo->dip == NULL) { 1651 DWARN(ldcp->id, 1652 "i_ldc_register_channel: cnex has not registered\n"); 1653 return (EAGAIN); 1654 } 1655 1656 rv = cinfo->reg_chan(cinfo->dip, ldcp->id, ldcp->devclass); 1657 if (rv) { 1658 DWARN(ldcp->id, 1659 "i_ldc_register_channel: cannot register channel\n"); 1660 return (rv); 1661 } 1662 1663 rv = cinfo->add_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR, 1664 i_ldc_tx_hdlr, ldcp, NULL); 1665 if (rv) { 1666 DWARN(ldcp->id, 1667 "i_ldc_register_channel: cannot add Tx interrupt\n"); 1668 (void) cinfo->unreg_chan(cinfo->dip, ldcp->id); 1669 return (rv); 1670 } 1671 1672 rv = cinfo->add_intr(cinfo->dip, ldcp->id, CNEX_RX_INTR, 1673 i_ldc_rx_hdlr, ldcp, NULL); 1674 if (rv) { 1675 DWARN(ldcp->id, 1676 "i_ldc_register_channel: cannot add Rx interrupt\n"); 1677 (void) cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR); 1678 (void) cinfo->unreg_chan(cinfo->dip, ldcp->id); 1679 return (rv); 1680 } 1681 1682 ldcp->tstate |= TS_CNEX_RDY; 1683 1684 return (0); 1685 } 1686 1687 /* 1688 * Unregister a channel with the channel nexus 1689 */ 1690 static int 1691 i_ldc_unregister_channel(ldc_chan_t *ldcp) 1692 { 1693 int rv = 0; 1694 ldc_cnex_t *cinfo = &ldcssp->cinfo; 1695 1696 if (cinfo->dip == NULL) { 1697 DWARN(ldcp->id, 1698 "i_ldc_unregister_channel: cnex has not registered\n"); 1699 return (EAGAIN); 1700 } 1701 1702 if (ldcp->tstate & TS_CNEX_RDY) { 1703 1704 /* Remove the Rx interrupt */ 1705 rv = cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_RX_INTR); 1706 if (rv) { 1707 if (rv != EAGAIN) { 1708 DWARN(ldcp->id, 1709 "i_ldc_unregister_channel: err removing " 1710 "Rx intr\n"); 1711 return (rv); 1712 } 1713 1714 /* 1715 * If interrupts are pending and handler has 1716 * finished running, clear interrupt and try 1717 * again 1718 */ 1719 if (ldcp->rx_intr_state != LDC_INTR_PEND) 1720 return (rv); 1721 1722 (void) i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 1723 rv = cinfo->rem_intr(cinfo->dip, ldcp->id, 1724 CNEX_RX_INTR); 1725 if (rv) { 1726 DWARN(ldcp->id, "i_ldc_unregister_channel: " 1727 "err removing Rx interrupt\n"); 1728 return (rv); 1729 } 1730 } 1731 1732 /* Remove the Tx interrupt */ 1733 rv = cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR); 1734 if (rv) { 1735 DWARN(ldcp->id, 1736 "i_ldc_unregister_channel: err removing Tx intr\n"); 1737 return (rv); 1738 } 1739 1740 /* Unregister the channel */ 1741 rv = cinfo->unreg_chan(ldcssp->cinfo.dip, ldcp->id); 1742 if (rv) { 1743 DWARN(ldcp->id, 1744 "i_ldc_unregister_channel: cannot unreg channel\n"); 1745 return (rv); 1746 } 1747 1748 ldcp->tstate &= ~TS_CNEX_RDY; 1749 } 1750 1751 return (0); 1752 } 1753 1754 1755 /* 1756 * LDC transmit interrupt handler 1757 * triggered for chanel up/down/reset events 1758 * and Tx queue content changes 1759 */ 1760 static uint_t 1761 i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2) 1762 { 1763 _NOTE(ARGUNUSED(arg2)) 1764 1765 int rv; 1766 ldc_chan_t *ldcp; 1767 boolean_t notify_client = B_FALSE; 1768 uint64_t notify_event = 0, link_state; 1769 1770 /* Get the channel for which interrupt was received */ 1771 ASSERT(arg1 != NULL); 1772 ldcp = (ldc_chan_t *)arg1; 1773 1774 D1(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n", 1775 ldcp->id, ldcp); 1776 1777 /* Lock channel */ 1778 mutex_enter(&ldcp->lock); 1779 1780 /* Obtain Tx lock */ 1781 mutex_enter(&ldcp->tx_lock); 1782 1783 /* mark interrupt as pending */ 1784 ldcp->tx_intr_state = LDC_INTR_ACTIVE; 1785 1786 /* save current link state */ 1787 link_state = ldcp->link_state; 1788 1789 rv = hv_ldc_tx_get_state(ldcp->id, &ldcp->tx_head, &ldcp->tx_tail, 1790 &ldcp->link_state); 1791 if (rv) { 1792 cmn_err(CE_WARN, 1793 "i_ldc_tx_hdlr: (0x%lx) cannot read queue ptrs rv=0x%d\n", 1794 ldcp->id, rv); 1795 i_ldc_clear_intr(ldcp, CNEX_TX_INTR); 1796 mutex_exit(&ldcp->tx_lock); 1797 mutex_exit(&ldcp->lock); 1798 return (DDI_INTR_CLAIMED); 1799 } 1800 1801 /* 1802 * reset the channel state if the channel went down 1803 * (other side unconfigured queue) or channel was reset 1804 * (other side reconfigured its queue) 1805 */ 1806 if (link_state != ldcp->link_state && 1807 ldcp->link_state == LDC_CHANNEL_DOWN) { 1808 D1(ldcp->id, "i_ldc_tx_hdlr: channel link down\n", ldcp->id); 1809 i_ldc_reset(ldcp, B_FALSE); 1810 notify_client = B_TRUE; 1811 notify_event = LDC_EVT_DOWN; 1812 } 1813 1814 if (link_state != ldcp->link_state && 1815 ldcp->link_state == LDC_CHANNEL_RESET) { 1816 D1(ldcp->id, "i_ldc_tx_hdlr: channel link reset\n", ldcp->id); 1817 i_ldc_reset(ldcp, B_FALSE); 1818 notify_client = B_TRUE; 1819 notify_event = LDC_EVT_RESET; 1820 } 1821 1822 if (link_state != ldcp->link_state && 1823 (ldcp->tstate & ~TS_IN_RESET) == TS_OPEN && 1824 ldcp->link_state == LDC_CHANNEL_UP) { 1825 D1(ldcp->id, "i_ldc_tx_hdlr: channel link up\n", ldcp->id); 1826 notify_client = B_TRUE; 1827 notify_event = LDC_EVT_RESET; 1828 ldcp->tstate |= TS_LINK_READY; 1829 ldcp->status = LDC_READY; 1830 } 1831 1832 /* if callbacks are disabled, do not notify */ 1833 if (!ldcp->cb_enabled) 1834 notify_client = B_FALSE; 1835 1836 /* Unlock channel */ 1837 1838 if (notify_client) { 1839 ldcp->cb_inprogress = B_TRUE; 1840 mutex_exit(&ldcp->tx_lock); 1841 mutex_exit(&ldcp->lock); 1842 rv = ldcp->cb(notify_event, ldcp->cb_arg); 1843 if (rv) { 1844 DWARN(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) callback " 1845 "failure", ldcp->id); 1846 } 1847 mutex_enter(&ldcp->lock); 1848 ldcp->cb_inprogress = B_FALSE; 1849 } 1850 1851 i_ldc_clear_intr(ldcp, CNEX_TX_INTR); 1852 mutex_exit(&ldcp->lock); 1853 1854 D1(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) exiting handler", ldcp->id); 1855 1856 return (DDI_INTR_CLAIMED); 1857 } 1858 1859 /* 1860 * LDC receive interrupt handler 1861 * triggered for channel with data pending to read 1862 * i.e. Rx queue content changes 1863 */ 1864 static uint_t 1865 i_ldc_rx_hdlr(caddr_t arg1, caddr_t arg2) 1866 { 1867 _NOTE(ARGUNUSED(arg2)) 1868 1869 int rv; 1870 uint64_t rx_head, rx_tail; 1871 ldc_msg_t *msg; 1872 ldc_chan_t *ldcp; 1873 boolean_t notify_client = B_FALSE; 1874 uint64_t notify_event = 0; 1875 uint64_t link_state, first_fragment = 0; 1876 1877 1878 /* Get the channel for which interrupt was received */ 1879 if (arg1 == NULL) { 1880 cmn_err(CE_WARN, "i_ldc_rx_hdlr: invalid arg\n"); 1881 return (DDI_INTR_UNCLAIMED); 1882 } 1883 1884 ldcp = (ldc_chan_t *)arg1; 1885 1886 D1(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n", 1887 ldcp->id, ldcp); 1888 D1(ldcp->id, "i_ldc_rx_hdlr: (%llx) USR%lx/TS%lx/HS%lx, LSTATE=%lx\n", 1889 ldcp->id, ldcp->status, ldcp->tstate, ldcp->hstate, 1890 ldcp->link_state); 1891 1892 /* Lock channel */ 1893 mutex_enter(&ldcp->lock); 1894 1895 /* mark interrupt as pending */ 1896 ldcp->rx_intr_state = LDC_INTR_ACTIVE; 1897 1898 /* 1899 * Read packet(s) from the queue 1900 */ 1901 for (;;) { 1902 1903 link_state = ldcp->link_state; 1904 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 1905 &ldcp->link_state); 1906 if (rv) { 1907 cmn_err(CE_WARN, 1908 "i_ldc_rx_hdlr: (0x%lx) cannot read " 1909 "queue ptrs, rv=0x%d\n", ldcp->id, rv); 1910 i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 1911 mutex_exit(&ldcp->lock); 1912 return (DDI_INTR_CLAIMED); 1913 } 1914 1915 /* 1916 * reset the channel state if the channel went down 1917 * (other side unconfigured queue) or channel was reset 1918 * (other side reconfigured its queue) 1919 */ 1920 1921 if (link_state != ldcp->link_state) { 1922 switch (ldcp->link_state) { 1923 case LDC_CHANNEL_DOWN: 1924 D1(ldcp->id, "i_ldc_rx_hdlr: channel " 1925 "link down\n", ldcp->id); 1926 mutex_enter(&ldcp->tx_lock); 1927 i_ldc_reset(ldcp, B_FALSE); 1928 mutex_exit(&ldcp->tx_lock); 1929 notify_client = B_TRUE; 1930 notify_event = LDC_EVT_DOWN; 1931 goto loop_exit; 1932 1933 case LDC_CHANNEL_UP: 1934 D1(ldcp->id, "i_ldc_rx_hdlr: " 1935 "channel link up\n", ldcp->id); 1936 1937 if ((ldcp->tstate & ~TS_IN_RESET) == TS_OPEN) { 1938 notify_client = B_TRUE; 1939 notify_event = LDC_EVT_RESET; 1940 ldcp->tstate |= TS_LINK_READY; 1941 ldcp->status = LDC_READY; 1942 } 1943 break; 1944 1945 case LDC_CHANNEL_RESET: 1946 default: 1947 #ifdef DEBUG 1948 force_reset: 1949 #endif 1950 D1(ldcp->id, "i_ldc_rx_hdlr: channel " 1951 "link reset\n", ldcp->id); 1952 mutex_enter(&ldcp->tx_lock); 1953 i_ldc_reset(ldcp, B_FALSE); 1954 mutex_exit(&ldcp->tx_lock); 1955 notify_client = B_TRUE; 1956 notify_event = LDC_EVT_RESET; 1957 break; 1958 } 1959 } 1960 1961 #ifdef DEBUG 1962 if (LDC_INJECT_RESET(ldcp)) 1963 goto force_reset; 1964 #endif 1965 1966 if (rx_head == rx_tail) { 1967 D2(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) No packets\n", 1968 ldcp->id); 1969 break; 1970 } 1971 1972 D2(ldcp->id, "i_ldc_rx_hdlr: head=0x%llx, tail=0x%llx\n", 1973 rx_head, rx_tail); 1974 DUMP_LDC_PKT(ldcp, "i_ldc_rx_hdlr rcd", 1975 ldcp->rx_q_va + rx_head); 1976 1977 /* get the message */ 1978 msg = (ldc_msg_t *)(ldcp->rx_q_va + rx_head); 1979 1980 /* if channel is in RAW mode or data pkt, notify and return */ 1981 if (ldcp->mode == LDC_MODE_RAW) { 1982 notify_client = B_TRUE; 1983 notify_event |= LDC_EVT_READ; 1984 break; 1985 } 1986 1987 if ((msg->type & LDC_DATA) && (msg->stype & LDC_INFO)) { 1988 1989 /* discard packet if channel is not up */ 1990 if ((ldcp->tstate & ~TS_IN_RESET) != TS_UP) { 1991 1992 /* move the head one position */ 1993 rx_head = (rx_head + LDC_PACKET_SIZE) % 1994 (ldcp->rx_q_entries << LDC_PACKET_SHIFT); 1995 1996 if (rv = i_ldc_set_rx_head(ldcp, rx_head)) 1997 break; 1998 1999 continue; 2000 } else { 2001 if ((ldcp->tstate & TS_IN_RESET) == 0) 2002 notify_client = B_TRUE; 2003 notify_event |= LDC_EVT_READ; 2004 break; 2005 } 2006 } 2007 2008 /* Check the sequence ID for the message received */ 2009 rv = i_ldc_check_seqid(ldcp, msg); 2010 if (rv != 0) { 2011 2012 DWARN(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) seqid error, " 2013 "q_ptrs=0x%lx,0x%lx", ldcp->id, rx_head, rx_tail); 2014 2015 /* Reset last_msg_rcd to start of message */ 2016 if (first_fragment != 0) { 2017 ldcp->last_msg_rcd = first_fragment - 1; 2018 first_fragment = 0; 2019 } 2020 2021 /* 2022 * Send a NACK due to seqid mismatch 2023 */ 2024 rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, 2025 (msg->ctrl & LDC_CTRL_MASK)); 2026 2027 if (rv) { 2028 cmn_err(CE_NOTE, 2029 "i_ldc_rx_hdlr: (0x%lx) err sending " 2030 "CTRL/NACK msg\n", ldcp->id); 2031 2032 /* if cannot send NACK - reset channel */ 2033 mutex_enter(&ldcp->tx_lock); 2034 i_ldc_reset(ldcp, B_TRUE); 2035 mutex_exit(&ldcp->tx_lock); 2036 rv = ECONNRESET; 2037 break; 2038 } 2039 2040 /* purge receive queue */ 2041 (void) i_ldc_set_rx_head(ldcp, rx_tail); 2042 break; 2043 } 2044 2045 /* record the message ID */ 2046 ldcp->last_msg_rcd = msg->seqid; 2047 2048 /* process control messages */ 2049 if (msg->type & LDC_CTRL) { 2050 /* save current internal state */ 2051 uint64_t tstate = ldcp->tstate; 2052 2053 rv = i_ldc_ctrlmsg(ldcp, msg); 2054 if (rv == EAGAIN) { 2055 /* re-process pkt - state was adjusted */ 2056 continue; 2057 } 2058 if (rv == ECONNRESET) { 2059 notify_client = B_TRUE; 2060 notify_event = LDC_EVT_RESET; 2061 break; 2062 } 2063 2064 /* 2065 * control message processing was successful 2066 * channel transitioned to ready for communication 2067 */ 2068 if (rv == 0 && ldcp->tstate == TS_UP && 2069 (tstate & ~TS_IN_RESET) != 2070 (ldcp->tstate & ~TS_IN_RESET)) { 2071 notify_client = B_TRUE; 2072 notify_event = LDC_EVT_UP; 2073 } 2074 } 2075 2076 /* process data ACKs */ 2077 if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 2078 if (rv = i_ldc_process_data_ACK(ldcp, msg)) { 2079 notify_client = B_TRUE; 2080 notify_event = LDC_EVT_RESET; 2081 break; 2082 } 2083 } 2084 2085 /* move the head one position */ 2086 rx_head = (rx_head + LDC_PACKET_SIZE) % 2087 (ldcp->rx_q_entries << LDC_PACKET_SHIFT); 2088 if (rv = i_ldc_set_rx_head(ldcp, rx_head)) { 2089 notify_client = B_TRUE; 2090 notify_event = LDC_EVT_RESET; 2091 break; 2092 } 2093 2094 } /* for */ 2095 2096 loop_exit: 2097 2098 /* if callbacks are disabled, do not notify */ 2099 if (!ldcp->cb_enabled) 2100 notify_client = B_FALSE; 2101 2102 /* 2103 * If there are data packets in the queue, the ldc_read will 2104 * clear interrupts after draining the queue, else clear interrupts 2105 */ 2106 if ((notify_event & LDC_EVT_READ) == 0) { 2107 i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 2108 } else 2109 ldcp->rx_intr_state = LDC_INTR_PEND; 2110 2111 mutex_exit(&ldcp->lock); 2112 2113 if (notify_client) { 2114 rv = ldcp->cb(notify_event, ldcp->cb_arg); 2115 if (rv) { 2116 DWARN(ldcp->id, 2117 "i_ldc_rx_hdlr: (0x%llx) callback failure", 2118 ldcp->id); 2119 } 2120 } 2121 2122 D1(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) exiting handler", ldcp->id); 2123 return (DDI_INTR_CLAIMED); 2124 } 2125 2126 2127 /* -------------------------------------------------------------------------- */ 2128 2129 /* 2130 * LDC API functions 2131 */ 2132 2133 /* 2134 * Initialize the channel. Allocate internal structure and memory for 2135 * TX/RX queues, and initialize locks. 2136 */ 2137 int 2138 ldc_init(uint64_t id, ldc_attr_t *attr, ldc_handle_t *handle) 2139 { 2140 ldc_chan_t *ldcp; 2141 int rv, exit_val; 2142 uint64_t ra_base, nentries; 2143 uint64_t qlen; 2144 2145 exit_val = EINVAL; /* guarantee an error if exit on failure */ 2146 2147 if (attr == NULL) { 2148 DWARN(id, "ldc_init: (0x%llx) invalid attr\n", id); 2149 return (EINVAL); 2150 } 2151 if (handle == NULL) { 2152 DWARN(id, "ldc_init: (0x%llx) invalid handle\n", id); 2153 return (EINVAL); 2154 } 2155 2156 /* check if channel is valid */ 2157 rv = hv_ldc_tx_qinfo(id, &ra_base, &nentries); 2158 if (rv == H_ECHANNEL) { 2159 DWARN(id, "ldc_init: (0x%llx) invalid channel id\n", id); 2160 return (EINVAL); 2161 } 2162 2163 /* check if the channel has already been initialized */ 2164 mutex_enter(&ldcssp->lock); 2165 ldcp = ldcssp->chan_list; 2166 while (ldcp != NULL) { 2167 if (ldcp->id == id) { 2168 DWARN(id, "ldc_init: (0x%llx) already initialized\n", 2169 id); 2170 mutex_exit(&ldcssp->lock); 2171 return (EADDRINUSE); 2172 } 2173 ldcp = ldcp->next; 2174 } 2175 mutex_exit(&ldcssp->lock); 2176 2177 ASSERT(ldcp == NULL); 2178 2179 *handle = 0; 2180 2181 /* Allocate an ldcp structure */ 2182 ldcp = kmem_zalloc(sizeof (ldc_chan_t), KM_SLEEP); 2183 2184 /* 2185 * Initialize the channel and Tx lock 2186 * 2187 * The channel 'lock' protects the entire channel and 2188 * should be acquired before initializing, resetting, 2189 * destroying or reading from a channel. 2190 * 2191 * The 'tx_lock' should be acquired prior to transmitting 2192 * data over the channel. The lock should also be acquired 2193 * prior to channel reconfiguration (in order to prevent 2194 * concurrent writes). 2195 * 2196 * ORDERING: When both locks are being acquired, to prevent 2197 * deadlocks, the channel lock should be always acquired prior 2198 * to the tx_lock. 2199 */ 2200 mutex_init(&ldcp->lock, NULL, MUTEX_DRIVER, NULL); 2201 mutex_init(&ldcp->tx_lock, NULL, MUTEX_DRIVER, NULL); 2202 2203 /* Initialize the channel */ 2204 ldcp->id = id; 2205 ldcp->cb = NULL; 2206 ldcp->cb_arg = NULL; 2207 ldcp->cb_inprogress = B_FALSE; 2208 ldcp->cb_enabled = B_FALSE; 2209 ldcp->next = NULL; 2210 2211 /* Read attributes */ 2212 ldcp->mode = attr->mode; 2213 ldcp->devclass = attr->devclass; 2214 ldcp->devinst = attr->instance; 2215 ldcp->mtu = (attr->mtu > 0) ? attr->mtu : LDC_DEFAULT_MTU; 2216 2217 D1(ldcp->id, 2218 "ldc_init: (0x%llx) channel attributes, class=0x%x, " 2219 "instance=0x%llx, mode=%d, mtu=%d\n", 2220 ldcp->id, ldcp->devclass, ldcp->devinst, ldcp->mode, ldcp->mtu); 2221 2222 ldcp->next_vidx = 0; 2223 ldcp->tstate = TS_IN_RESET; 2224 ldcp->hstate = 0; 2225 ldcp->last_msg_snt = LDC_INIT_SEQID; 2226 ldcp->last_ack_rcd = 0; 2227 ldcp->last_msg_rcd = 0; 2228 2229 ldcp->stream_bufferp = NULL; 2230 ldcp->exp_dring_list = NULL; 2231 ldcp->imp_dring_list = NULL; 2232 ldcp->mhdl_list = NULL; 2233 2234 ldcp->tx_intr_state = LDC_INTR_NONE; 2235 ldcp->rx_intr_state = LDC_INTR_NONE; 2236 2237 /* Initialize payload size depending on whether channel is reliable */ 2238 switch (ldcp->mode) { 2239 case LDC_MODE_RAW: 2240 ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RAW; 2241 ldcp->read_p = i_ldc_read_raw; 2242 ldcp->write_p = i_ldc_write_raw; 2243 break; 2244 case LDC_MODE_UNRELIABLE: 2245 ldcp->pkt_payload = LDC_PAYLOAD_SIZE_UNRELIABLE; 2246 ldcp->read_p = i_ldc_read_packet; 2247 ldcp->write_p = i_ldc_write_packet; 2248 break; 2249 case LDC_MODE_RELIABLE: 2250 ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RELIABLE; 2251 ldcp->read_p = i_ldc_read_packet; 2252 ldcp->write_p = i_ldc_write_packet; 2253 break; 2254 case LDC_MODE_STREAM: 2255 ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RELIABLE; 2256 2257 ldcp->stream_remains = 0; 2258 ldcp->stream_offset = 0; 2259 ldcp->stream_bufferp = kmem_alloc(ldcp->mtu, KM_SLEEP); 2260 ldcp->read_p = i_ldc_read_stream; 2261 ldcp->write_p = i_ldc_write_stream; 2262 break; 2263 default: 2264 exit_val = EINVAL; 2265 goto cleanup_on_exit; 2266 } 2267 2268 /* 2269 * qlen is (mtu * ldc_mtu_msgs) / pkt_payload. If this 2270 * value is smaller than default length of ldc_queue_entries, 2271 * qlen is set to ldc_queue_entries.. 2272 */ 2273 qlen = (ldcp->mtu * ldc_mtu_msgs) / ldcp->pkt_payload; 2274 ldcp->rx_q_entries = 2275 (qlen < ldc_queue_entries) ? ldc_queue_entries : qlen; 2276 ldcp->tx_q_entries = ldcp->rx_q_entries; 2277 2278 D1(ldcp->id, "ldc_init: queue length = 0x%llx\n", qlen); 2279 2280 /* Create a transmit queue */ 2281 ldcp->tx_q_va = (uint64_t) 2282 contig_mem_alloc(ldcp->tx_q_entries << LDC_PACKET_SHIFT); 2283 if (ldcp->tx_q_va == NULL) { 2284 cmn_err(CE_WARN, 2285 "ldc_init: (0x%lx) TX queue allocation failed\n", 2286 ldcp->id); 2287 exit_val = ENOMEM; 2288 goto cleanup_on_exit; 2289 } 2290 ldcp->tx_q_ra = va_to_pa((caddr_t)ldcp->tx_q_va); 2291 2292 D2(ldcp->id, "ldc_init: txq_va=0x%llx, txq_ra=0x%llx, entries=0x%llx\n", 2293 ldcp->tx_q_va, ldcp->tx_q_ra, ldcp->tx_q_entries); 2294 2295 ldcp->tstate |= TS_TXQ_RDY; 2296 2297 /* Create a receive queue */ 2298 ldcp->rx_q_va = (uint64_t) 2299 contig_mem_alloc(ldcp->rx_q_entries << LDC_PACKET_SHIFT); 2300 if (ldcp->rx_q_va == NULL) { 2301 cmn_err(CE_WARN, 2302 "ldc_init: (0x%lx) RX queue allocation failed\n", 2303 ldcp->id); 2304 exit_val = ENOMEM; 2305 goto cleanup_on_exit; 2306 } 2307 ldcp->rx_q_ra = va_to_pa((caddr_t)ldcp->rx_q_va); 2308 2309 D2(ldcp->id, "ldc_init: rxq_va=0x%llx, rxq_ra=0x%llx, entries=0x%llx\n", 2310 ldcp->rx_q_va, ldcp->rx_q_ra, ldcp->rx_q_entries); 2311 2312 ldcp->tstate |= TS_RXQ_RDY; 2313 2314 /* Init descriptor ring and memory handle list lock */ 2315 mutex_init(&ldcp->exp_dlist_lock, NULL, MUTEX_DRIVER, NULL); 2316 mutex_init(&ldcp->imp_dlist_lock, NULL, MUTEX_DRIVER, NULL); 2317 mutex_init(&ldcp->mlist_lock, NULL, MUTEX_DRIVER, NULL); 2318 2319 /* mark status as INITialized */ 2320 ldcp->status = LDC_INIT; 2321 2322 /* Add to channel list */ 2323 mutex_enter(&ldcssp->lock); 2324 ldcp->next = ldcssp->chan_list; 2325 ldcssp->chan_list = ldcp; 2326 ldcssp->channel_count++; 2327 mutex_exit(&ldcssp->lock); 2328 2329 /* set the handle */ 2330 *handle = (ldc_handle_t)ldcp; 2331 2332 D1(ldcp->id, "ldc_init: (0x%llx) channel initialized\n", ldcp->id); 2333 2334 return (0); 2335 2336 cleanup_on_exit: 2337 2338 if (ldcp->mode == LDC_MODE_STREAM && ldcp->stream_bufferp) 2339 kmem_free(ldcp->stream_bufferp, ldcp->mtu); 2340 2341 if (ldcp->tstate & TS_TXQ_RDY) 2342 contig_mem_free((caddr_t)ldcp->tx_q_va, 2343 (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 2344 2345 if (ldcp->tstate & TS_RXQ_RDY) 2346 contig_mem_free((caddr_t)ldcp->rx_q_va, 2347 (ldcp->rx_q_entries << LDC_PACKET_SHIFT)); 2348 2349 mutex_destroy(&ldcp->tx_lock); 2350 mutex_destroy(&ldcp->lock); 2351 2352 if (ldcp) 2353 kmem_free(ldcp, sizeof (ldc_chan_t)); 2354 2355 return (exit_val); 2356 } 2357 2358 /* 2359 * Finalizes the LDC connection. It will return EBUSY if the 2360 * channel is open. A ldc_close() has to be done prior to 2361 * a ldc_fini operation. It frees TX/RX queues, associated 2362 * with the channel 2363 */ 2364 int 2365 ldc_fini(ldc_handle_t handle) 2366 { 2367 ldc_chan_t *ldcp; 2368 ldc_chan_t *tmp_ldcp; 2369 uint64_t id; 2370 2371 if (handle == NULL) { 2372 DWARN(DBG_ALL_LDCS, "ldc_fini: invalid channel handle\n"); 2373 return (EINVAL); 2374 } 2375 ldcp = (ldc_chan_t *)handle; 2376 id = ldcp->id; 2377 2378 mutex_enter(&ldcp->lock); 2379 2380 if ((ldcp->tstate & ~TS_IN_RESET) > TS_INIT) { 2381 DWARN(ldcp->id, "ldc_fini: (0x%llx) channel is open\n", 2382 ldcp->id); 2383 mutex_exit(&ldcp->lock); 2384 return (EBUSY); 2385 } 2386 2387 /* Remove from the channel list */ 2388 mutex_enter(&ldcssp->lock); 2389 tmp_ldcp = ldcssp->chan_list; 2390 if (tmp_ldcp == ldcp) { 2391 ldcssp->chan_list = ldcp->next; 2392 ldcp->next = NULL; 2393 } else { 2394 while (tmp_ldcp != NULL) { 2395 if (tmp_ldcp->next == ldcp) { 2396 tmp_ldcp->next = ldcp->next; 2397 ldcp->next = NULL; 2398 break; 2399 } 2400 tmp_ldcp = tmp_ldcp->next; 2401 } 2402 if (tmp_ldcp == NULL) { 2403 DWARN(DBG_ALL_LDCS, "ldc_fini: invalid channel hdl\n"); 2404 mutex_exit(&ldcssp->lock); 2405 mutex_exit(&ldcp->lock); 2406 return (EINVAL); 2407 } 2408 } 2409 2410 ldcssp->channel_count--; 2411 2412 mutex_exit(&ldcssp->lock); 2413 2414 /* Free the map table for this channel */ 2415 if (ldcp->mtbl) { 2416 (void) hv_ldc_set_map_table(ldcp->id, NULL, NULL); 2417 if (ldcp->mtbl->contigmem) 2418 contig_mem_free(ldcp->mtbl->table, ldcp->mtbl->size); 2419 else 2420 kmem_free(ldcp->mtbl->table, ldcp->mtbl->size); 2421 mutex_destroy(&ldcp->mtbl->lock); 2422 kmem_free(ldcp->mtbl, sizeof (ldc_mtbl_t)); 2423 } 2424 2425 /* Destroy descriptor ring and memory handle list lock */ 2426 mutex_destroy(&ldcp->exp_dlist_lock); 2427 mutex_destroy(&ldcp->imp_dlist_lock); 2428 mutex_destroy(&ldcp->mlist_lock); 2429 2430 /* Free the stream buffer for STREAM_MODE */ 2431 if (ldcp->mode == LDC_MODE_STREAM && ldcp->stream_bufferp) 2432 kmem_free(ldcp->stream_bufferp, ldcp->mtu); 2433 2434 /* Free the RX queue */ 2435 contig_mem_free((caddr_t)ldcp->rx_q_va, 2436 (ldcp->rx_q_entries << LDC_PACKET_SHIFT)); 2437 ldcp->tstate &= ~TS_RXQ_RDY; 2438 2439 /* Free the TX queue */ 2440 contig_mem_free((caddr_t)ldcp->tx_q_va, 2441 (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 2442 ldcp->tstate &= ~TS_TXQ_RDY; 2443 2444 mutex_exit(&ldcp->lock); 2445 2446 /* Destroy mutex */ 2447 mutex_destroy(&ldcp->tx_lock); 2448 mutex_destroy(&ldcp->lock); 2449 2450 /* free channel structure */ 2451 kmem_free(ldcp, sizeof (ldc_chan_t)); 2452 2453 D1(id, "ldc_fini: (0x%llx) channel finalized\n", id); 2454 2455 return (0); 2456 } 2457 2458 /* 2459 * Open the LDC channel for use. It registers the TX/RX queues 2460 * with the Hypervisor. It also specifies the interrupt number 2461 * and target CPU for this channel 2462 */ 2463 int 2464 ldc_open(ldc_handle_t handle) 2465 { 2466 ldc_chan_t *ldcp; 2467 int rv; 2468 2469 if (handle == NULL) { 2470 DWARN(DBG_ALL_LDCS, "ldc_open: invalid channel handle\n"); 2471 return (EINVAL); 2472 } 2473 2474 ldcp = (ldc_chan_t *)handle; 2475 2476 mutex_enter(&ldcp->lock); 2477 2478 if (ldcp->tstate < TS_INIT) { 2479 DWARN(ldcp->id, 2480 "ldc_open: (0x%llx) channel not initialized\n", ldcp->id); 2481 mutex_exit(&ldcp->lock); 2482 return (EFAULT); 2483 } 2484 if ((ldcp->tstate & ~TS_IN_RESET) >= TS_OPEN) { 2485 DWARN(ldcp->id, 2486 "ldc_open: (0x%llx) channel is already open\n", ldcp->id); 2487 mutex_exit(&ldcp->lock); 2488 return (EFAULT); 2489 } 2490 2491 /* 2492 * Unregister/Register the tx queue with the hypervisor 2493 */ 2494 rv = hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 2495 if (rv) { 2496 cmn_err(CE_WARN, 2497 "ldc_open: (0x%lx) channel tx queue unconf failed\n", 2498 ldcp->id); 2499 mutex_exit(&ldcp->lock); 2500 return (EIO); 2501 } 2502 2503 rv = hv_ldc_tx_qconf(ldcp->id, ldcp->tx_q_ra, ldcp->tx_q_entries); 2504 if (rv) { 2505 cmn_err(CE_WARN, 2506 "ldc_open: (0x%lx) channel tx queue conf failed\n", 2507 ldcp->id); 2508 mutex_exit(&ldcp->lock); 2509 return (EIO); 2510 } 2511 2512 D2(ldcp->id, "ldc_open: (0x%llx) registered tx queue with LDC\n", 2513 ldcp->id); 2514 2515 /* 2516 * Unregister/Register the rx queue with the hypervisor 2517 */ 2518 rv = hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 2519 if (rv) { 2520 cmn_err(CE_WARN, 2521 "ldc_open: (0x%lx) channel rx queue unconf failed\n", 2522 ldcp->id); 2523 mutex_exit(&ldcp->lock); 2524 return (EIO); 2525 } 2526 2527 rv = hv_ldc_rx_qconf(ldcp->id, ldcp->rx_q_ra, ldcp->rx_q_entries); 2528 if (rv) { 2529 cmn_err(CE_WARN, 2530 "ldc_open: (0x%lx) channel rx queue conf failed\n", 2531 ldcp->id); 2532 mutex_exit(&ldcp->lock); 2533 return (EIO); 2534 } 2535 2536 D2(ldcp->id, "ldc_open: (0x%llx) registered rx queue with LDC\n", 2537 ldcp->id); 2538 2539 ldcp->tstate |= TS_QCONF_RDY; 2540 2541 /* Register the channel with the channel nexus */ 2542 rv = i_ldc_register_channel(ldcp); 2543 if (rv && rv != EAGAIN) { 2544 cmn_err(CE_WARN, 2545 "ldc_open: (0x%lx) channel register failed\n", ldcp->id); 2546 (void) hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 2547 (void) hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 2548 mutex_exit(&ldcp->lock); 2549 return (EIO); 2550 } 2551 2552 /* mark channel in OPEN state */ 2553 ldcp->status = LDC_OPEN; 2554 2555 /* Read channel state */ 2556 rv = hv_ldc_tx_get_state(ldcp->id, 2557 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 2558 if (rv) { 2559 cmn_err(CE_WARN, 2560 "ldc_open: (0x%lx) cannot read channel state\n", 2561 ldcp->id); 2562 (void) i_ldc_unregister_channel(ldcp); 2563 (void) hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 2564 (void) hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 2565 mutex_exit(&ldcp->lock); 2566 return (EIO); 2567 } 2568 2569 /* 2570 * set the ACKd head to current head location for reliable & 2571 * streaming mode 2572 */ 2573 ldcp->tx_ackd_head = ldcp->tx_head; 2574 2575 /* mark channel ready if HV report link is UP (peer alloc'd Rx queue) */ 2576 if (ldcp->link_state == LDC_CHANNEL_UP || 2577 ldcp->link_state == LDC_CHANNEL_RESET) { 2578 ldcp->tstate |= TS_LINK_READY; 2579 ldcp->status = LDC_READY; 2580 } 2581 2582 /* 2583 * if channel is being opened in RAW mode - no handshake is needed 2584 * switch the channel READY and UP state 2585 */ 2586 if (ldcp->mode == LDC_MODE_RAW) { 2587 ldcp->tstate = TS_UP; /* set bits associated with LDC UP */ 2588 ldcp->status = LDC_UP; 2589 } 2590 2591 mutex_exit(&ldcp->lock); 2592 2593 /* 2594 * Increment number of open channels 2595 */ 2596 mutex_enter(&ldcssp->lock); 2597 ldcssp->channels_open++; 2598 mutex_exit(&ldcssp->lock); 2599 2600 DWARN(ldcp->id, 2601 "ldc_open: (0x%llx) channel (0x%p) open for use " 2602 "(tstate=0x%x, status=0x%x)\n", 2603 ldcp->id, ldcp, ldcp->tstate, ldcp->status); 2604 2605 return (0); 2606 } 2607 2608 /* 2609 * Close the LDC connection. It will return EBUSY if there 2610 * are memory segments or descriptor rings either bound to or 2611 * mapped over the channel 2612 */ 2613 int 2614 ldc_close(ldc_handle_t handle) 2615 { 2616 ldc_chan_t *ldcp; 2617 int rv = 0, retries = 0; 2618 boolean_t chk_done = B_FALSE; 2619 2620 if (handle == NULL) { 2621 DWARN(DBG_ALL_LDCS, "ldc_close: invalid channel handle\n"); 2622 return (EINVAL); 2623 } 2624 ldcp = (ldc_chan_t *)handle; 2625 2626 mutex_enter(&ldcp->lock); 2627 2628 /* return error if channel is not open */ 2629 if ((ldcp->tstate & ~TS_IN_RESET) < TS_OPEN) { 2630 DWARN(ldcp->id, 2631 "ldc_close: (0x%llx) channel is not open\n", ldcp->id); 2632 mutex_exit(&ldcp->lock); 2633 return (EFAULT); 2634 } 2635 2636 /* if any memory handles, drings, are bound or mapped cannot close */ 2637 if (ldcp->mhdl_list != NULL) { 2638 DWARN(ldcp->id, 2639 "ldc_close: (0x%llx) channel has bound memory handles\n", 2640 ldcp->id); 2641 mutex_exit(&ldcp->lock); 2642 return (EBUSY); 2643 } 2644 if (ldcp->exp_dring_list != NULL) { 2645 DWARN(ldcp->id, 2646 "ldc_close: (0x%llx) channel has bound descriptor rings\n", 2647 ldcp->id); 2648 mutex_exit(&ldcp->lock); 2649 return (EBUSY); 2650 } 2651 if (ldcp->imp_dring_list != NULL) { 2652 DWARN(ldcp->id, 2653 "ldc_close: (0x%llx) channel has mapped descriptor rings\n", 2654 ldcp->id); 2655 mutex_exit(&ldcp->lock); 2656 return (EBUSY); 2657 } 2658 2659 /* Obtain Tx lock */ 2660 mutex_enter(&ldcp->tx_lock); 2661 2662 /* 2663 * Wait for pending transmits to complete i.e Tx queue to drain 2664 * if there are pending pkts - wait 1 ms and retry again 2665 */ 2666 for (;;) { 2667 2668 rv = hv_ldc_tx_get_state(ldcp->id, 2669 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 2670 if (rv) { 2671 cmn_err(CE_WARN, 2672 "ldc_close: (0x%lx) cannot read qptrs\n", ldcp->id); 2673 mutex_exit(&ldcp->tx_lock); 2674 mutex_exit(&ldcp->lock); 2675 return (EIO); 2676 } 2677 2678 if (ldcp->tx_head == ldcp->tx_tail || 2679 ldcp->link_state != LDC_CHANNEL_UP) { 2680 break; 2681 } 2682 2683 if (chk_done) { 2684 DWARN(ldcp->id, 2685 "ldc_close: (0x%llx) Tx queue drain timeout\n", 2686 ldcp->id); 2687 break; 2688 } 2689 2690 /* wait for one ms and try again */ 2691 delay(drv_usectohz(1000)); 2692 chk_done = B_TRUE; 2693 } 2694 2695 /* 2696 * Drain the Tx and Rx queues as we are closing the 2697 * channel. We dont care about any pending packets. 2698 * We have to also drain the queue prior to clearing 2699 * pending interrupts, otherwise the HV will trigger 2700 * an interrupt the moment the interrupt state is 2701 * cleared. 2702 */ 2703 (void) i_ldc_txq_reconf(ldcp); 2704 (void) i_ldc_rxq_drain(ldcp); 2705 2706 /* 2707 * Unregister the channel with the nexus 2708 */ 2709 while ((rv = i_ldc_unregister_channel(ldcp)) != 0) { 2710 2711 mutex_exit(&ldcp->tx_lock); 2712 mutex_exit(&ldcp->lock); 2713 2714 /* if any error other than EAGAIN return back */ 2715 if (rv != EAGAIN || retries >= ldc_max_retries) { 2716 cmn_err(CE_WARN, 2717 "ldc_close: (0x%lx) unregister failed, %d\n", 2718 ldcp->id, rv); 2719 return (rv); 2720 } 2721 2722 /* 2723 * As there could be pending interrupts we need 2724 * to wait and try again 2725 */ 2726 drv_usecwait(ldc_delay); 2727 mutex_enter(&ldcp->lock); 2728 mutex_enter(&ldcp->tx_lock); 2729 retries++; 2730 } 2731 2732 /* 2733 * Unregister queues 2734 */ 2735 rv = hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 2736 if (rv) { 2737 cmn_err(CE_WARN, 2738 "ldc_close: (0x%lx) channel TX queue unconf failed\n", 2739 ldcp->id); 2740 mutex_exit(&ldcp->tx_lock); 2741 mutex_exit(&ldcp->lock); 2742 return (EIO); 2743 } 2744 rv = hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 2745 if (rv) { 2746 cmn_err(CE_WARN, 2747 "ldc_close: (0x%lx) channel RX queue unconf failed\n", 2748 ldcp->id); 2749 mutex_exit(&ldcp->tx_lock); 2750 mutex_exit(&ldcp->lock); 2751 return (EIO); 2752 } 2753 2754 ldcp->tstate &= ~TS_QCONF_RDY; 2755 2756 /* Reset channel state information */ 2757 i_ldc_reset_state(ldcp); 2758 2759 /* Mark channel as down and in initialized state */ 2760 ldcp->tx_ackd_head = 0; 2761 ldcp->tx_head = 0; 2762 ldcp->tstate = TS_IN_RESET|TS_INIT; 2763 ldcp->status = LDC_INIT; 2764 2765 mutex_exit(&ldcp->tx_lock); 2766 mutex_exit(&ldcp->lock); 2767 2768 /* Decrement number of open channels */ 2769 mutex_enter(&ldcssp->lock); 2770 ldcssp->channels_open--; 2771 mutex_exit(&ldcssp->lock); 2772 2773 D1(ldcp->id, "ldc_close: (0x%llx) channel closed\n", ldcp->id); 2774 2775 return (0); 2776 } 2777 2778 /* 2779 * Register channel callback 2780 */ 2781 int 2782 ldc_reg_callback(ldc_handle_t handle, 2783 uint_t(*cb)(uint64_t event, caddr_t arg), caddr_t arg) 2784 { 2785 ldc_chan_t *ldcp; 2786 2787 if (handle == NULL) { 2788 DWARN(DBG_ALL_LDCS, 2789 "ldc_reg_callback: invalid channel handle\n"); 2790 return (EINVAL); 2791 } 2792 if (((uint64_t)cb) < KERNELBASE) { 2793 DWARN(DBG_ALL_LDCS, "ldc_reg_callback: invalid callback\n"); 2794 return (EINVAL); 2795 } 2796 ldcp = (ldc_chan_t *)handle; 2797 2798 mutex_enter(&ldcp->lock); 2799 2800 if (ldcp->cb) { 2801 DWARN(ldcp->id, "ldc_reg_callback: (0x%llx) callback exists\n", 2802 ldcp->id); 2803 mutex_exit(&ldcp->lock); 2804 return (EIO); 2805 } 2806 if (ldcp->cb_inprogress) { 2807 DWARN(ldcp->id, "ldc_reg_callback: (0x%llx) callback active\n", 2808 ldcp->id); 2809 mutex_exit(&ldcp->lock); 2810 return (EWOULDBLOCK); 2811 } 2812 2813 ldcp->cb = cb; 2814 ldcp->cb_arg = arg; 2815 ldcp->cb_enabled = B_TRUE; 2816 2817 D1(ldcp->id, 2818 "ldc_reg_callback: (0x%llx) registered callback for channel\n", 2819 ldcp->id); 2820 2821 mutex_exit(&ldcp->lock); 2822 2823 return (0); 2824 } 2825 2826 /* 2827 * Unregister channel callback 2828 */ 2829 int 2830 ldc_unreg_callback(ldc_handle_t handle) 2831 { 2832 ldc_chan_t *ldcp; 2833 2834 if (handle == NULL) { 2835 DWARN(DBG_ALL_LDCS, 2836 "ldc_unreg_callback: invalid channel handle\n"); 2837 return (EINVAL); 2838 } 2839 ldcp = (ldc_chan_t *)handle; 2840 2841 mutex_enter(&ldcp->lock); 2842 2843 if (ldcp->cb == NULL) { 2844 DWARN(ldcp->id, 2845 "ldc_unreg_callback: (0x%llx) no callback exists\n", 2846 ldcp->id); 2847 mutex_exit(&ldcp->lock); 2848 return (EIO); 2849 } 2850 if (ldcp->cb_inprogress) { 2851 DWARN(ldcp->id, 2852 "ldc_unreg_callback: (0x%llx) callback active\n", 2853 ldcp->id); 2854 mutex_exit(&ldcp->lock); 2855 return (EWOULDBLOCK); 2856 } 2857 2858 ldcp->cb = NULL; 2859 ldcp->cb_arg = NULL; 2860 ldcp->cb_enabled = B_FALSE; 2861 2862 D1(ldcp->id, 2863 "ldc_unreg_callback: (0x%llx) unregistered callback for channel\n", 2864 ldcp->id); 2865 2866 mutex_exit(&ldcp->lock); 2867 2868 return (0); 2869 } 2870 2871 2872 /* 2873 * Bring a channel up by initiating a handshake with the peer 2874 * This call is asynchronous. It will complete at a later point 2875 * in time when the peer responds back with an RTR. 2876 */ 2877 int 2878 ldc_up(ldc_handle_t handle) 2879 { 2880 int rv; 2881 ldc_chan_t *ldcp; 2882 ldc_msg_t *ldcmsg; 2883 uint64_t tx_tail, tstate; 2884 2885 if (handle == NULL) { 2886 DWARN(DBG_ALL_LDCS, "ldc_up: invalid channel handle\n"); 2887 return (EINVAL); 2888 } 2889 ldcp = (ldc_chan_t *)handle; 2890 2891 mutex_enter(&ldcp->lock); 2892 2893 D1(ldcp->id, "ldc_up: (0x%llx) doing channel UP\n", ldcp->id); 2894 2895 /* clear the reset state */ 2896 tstate = ldcp->tstate; 2897 ldcp->tstate &= ~TS_IN_RESET; 2898 2899 if (ldcp->tstate == TS_UP) { 2900 DWARN(ldcp->id, 2901 "ldc_up: (0x%llx) channel is already in UP state\n", 2902 ldcp->id); 2903 2904 /* mark channel as up */ 2905 ldcp->status = LDC_UP; 2906 2907 /* 2908 * if channel was in reset state and there was 2909 * pending data clear interrupt state. this will 2910 * trigger an interrupt, causing the RX handler to 2911 * to invoke the client's callback 2912 */ 2913 if ((tstate & TS_IN_RESET) && 2914 ldcp->rx_intr_state == LDC_INTR_PEND) { 2915 DWARN(ldcp->id, 2916 "ldc_up: (0x%llx) channel has pending data, " 2917 "clearing interrupt\n", ldcp->id); 2918 i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 2919 } 2920 2921 mutex_exit(&ldcp->lock); 2922 return (0); 2923 } 2924 2925 /* if the channel is in RAW mode - mark it as UP, if READY */ 2926 if (ldcp->mode == LDC_MODE_RAW && ldcp->tstate >= TS_READY) { 2927 ldcp->tstate = TS_UP; 2928 mutex_exit(&ldcp->lock); 2929 return (0); 2930 } 2931 2932 /* Don't start another handshake if there is one in progress */ 2933 if (ldcp->hstate) { 2934 D1(ldcp->id, 2935 "ldc_up: (0x%llx) channel handshake in progress\n", 2936 ldcp->id); 2937 mutex_exit(&ldcp->lock); 2938 return (0); 2939 } 2940 2941 mutex_enter(&ldcp->tx_lock); 2942 2943 /* get the current tail for the LDC msg */ 2944 rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 2945 if (rv) { 2946 DWARN(ldcp->id, "ldc_up: (0x%llx) cannot initiate handshake\n", 2947 ldcp->id); 2948 mutex_exit(&ldcp->tx_lock); 2949 mutex_exit(&ldcp->lock); 2950 return (ECONNREFUSED); 2951 } 2952 2953 ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 2954 ZERO_PKT(ldcmsg); 2955 2956 ldcmsg->type = LDC_CTRL; 2957 ldcmsg->stype = LDC_INFO; 2958 ldcmsg->ctrl = LDC_VER; 2959 ldcp->next_vidx = 0; 2960 bcopy(&ldc_versions[0], ldcmsg->udata, sizeof (ldc_versions[0])); 2961 2962 DUMP_LDC_PKT(ldcp, "ldc_up snd ver", (uint64_t)ldcmsg); 2963 2964 /* initiate the send by calling into HV and set the new tail */ 2965 tx_tail = (tx_tail + LDC_PACKET_SIZE) % 2966 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 2967 2968 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 2969 if (rv) { 2970 DWARN(ldcp->id, 2971 "ldc_up: (0x%llx) cannot initiate handshake rv=%d\n", 2972 ldcp->id, rv); 2973 mutex_exit(&ldcp->tx_lock); 2974 mutex_exit(&ldcp->lock); 2975 return (rv); 2976 } 2977 2978 ldcp->hstate |= TS_SENT_VER; 2979 ldcp->tx_tail = tx_tail; 2980 D1(ldcp->id, "ldc_up: (0x%llx) channel up initiated\n", ldcp->id); 2981 2982 mutex_exit(&ldcp->tx_lock); 2983 mutex_exit(&ldcp->lock); 2984 2985 return (rv); 2986 } 2987 2988 2989 /* 2990 * Bring a channel down by resetting its state and queues 2991 */ 2992 int 2993 ldc_down(ldc_handle_t handle) 2994 { 2995 ldc_chan_t *ldcp; 2996 2997 if (handle == NULL) { 2998 DWARN(DBG_ALL_LDCS, "ldc_down: invalid channel handle\n"); 2999 return (EINVAL); 3000 } 3001 ldcp = (ldc_chan_t *)handle; 3002 mutex_enter(&ldcp->lock); 3003 mutex_enter(&ldcp->tx_lock); 3004 i_ldc_reset(ldcp, B_TRUE); 3005 mutex_exit(&ldcp->tx_lock); 3006 mutex_exit(&ldcp->lock); 3007 3008 return (0); 3009 } 3010 3011 /* 3012 * Get the current channel status 3013 */ 3014 int 3015 ldc_status(ldc_handle_t handle, ldc_status_t *status) 3016 { 3017 ldc_chan_t *ldcp; 3018 3019 if (handle == NULL || status == NULL) { 3020 DWARN(DBG_ALL_LDCS, "ldc_status: invalid argument\n"); 3021 return (EINVAL); 3022 } 3023 ldcp = (ldc_chan_t *)handle; 3024 3025 *status = ((ldc_chan_t *)handle)->status; 3026 3027 DWARN(ldcp->id, 3028 "ldc_status: (0x%llx) returned status %d\n", ldcp->id, *status); 3029 return (0); 3030 } 3031 3032 3033 /* 3034 * Set the channel's callback mode - enable/disable callbacks 3035 */ 3036 int 3037 ldc_set_cb_mode(ldc_handle_t handle, ldc_cb_mode_t cmode) 3038 { 3039 ldc_chan_t *ldcp; 3040 3041 if (handle == NULL) { 3042 DWARN(DBG_ALL_LDCS, 3043 "ldc_set_intr_mode: invalid channel handle\n"); 3044 return (EINVAL); 3045 } 3046 ldcp = (ldc_chan_t *)handle; 3047 3048 /* 3049 * Record no callbacks should be invoked 3050 */ 3051 mutex_enter(&ldcp->lock); 3052 3053 switch (cmode) { 3054 case LDC_CB_DISABLE: 3055 if (!ldcp->cb_enabled) { 3056 DWARN(ldcp->id, 3057 "ldc_set_cb_mode: (0x%llx) callbacks disabled\n", 3058 ldcp->id); 3059 break; 3060 } 3061 ldcp->cb_enabled = B_FALSE; 3062 3063 D1(ldcp->id, "ldc_set_cb_mode: (0x%llx) disabled callbacks\n", 3064 ldcp->id); 3065 break; 3066 3067 case LDC_CB_ENABLE: 3068 if (ldcp->cb_enabled) { 3069 DWARN(ldcp->id, 3070 "ldc_set_cb_mode: (0x%llx) callbacks enabled\n", 3071 ldcp->id); 3072 break; 3073 } 3074 ldcp->cb_enabled = B_TRUE; 3075 3076 D1(ldcp->id, "ldc_set_cb_mode: (0x%llx) enabled callbacks\n", 3077 ldcp->id); 3078 break; 3079 } 3080 3081 mutex_exit(&ldcp->lock); 3082 3083 return (0); 3084 } 3085 3086 /* 3087 * Check to see if there are packets on the incoming queue 3088 * Will return hasdata = B_FALSE if there are no packets 3089 */ 3090 int 3091 ldc_chkq(ldc_handle_t handle, boolean_t *hasdata) 3092 { 3093 int rv; 3094 uint64_t rx_head, rx_tail; 3095 ldc_chan_t *ldcp; 3096 3097 if (handle == NULL) { 3098 DWARN(DBG_ALL_LDCS, "ldc_chkq: invalid channel handle\n"); 3099 return (EINVAL); 3100 } 3101 ldcp = (ldc_chan_t *)handle; 3102 3103 *hasdata = B_FALSE; 3104 3105 mutex_enter(&ldcp->lock); 3106 3107 if (ldcp->tstate != TS_UP) { 3108 D1(ldcp->id, 3109 "ldc_chkq: (0x%llx) channel is not up\n", ldcp->id); 3110 mutex_exit(&ldcp->lock); 3111 return (ECONNRESET); 3112 } 3113 3114 /* Read packet(s) from the queue */ 3115 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 3116 &ldcp->link_state); 3117 if (rv != 0) { 3118 cmn_err(CE_WARN, 3119 "ldc_chkq: (0x%lx) unable to read queue ptrs", ldcp->id); 3120 mutex_exit(&ldcp->lock); 3121 return (EIO); 3122 } 3123 /* reset the channel state if the channel went down */ 3124 if (ldcp->link_state == LDC_CHANNEL_DOWN || 3125 ldcp->link_state == LDC_CHANNEL_RESET) { 3126 mutex_enter(&ldcp->tx_lock); 3127 i_ldc_reset(ldcp, B_FALSE); 3128 mutex_exit(&ldcp->tx_lock); 3129 mutex_exit(&ldcp->lock); 3130 return (ECONNRESET); 3131 } 3132 3133 if ((rx_head != rx_tail) || 3134 (ldcp->mode == LDC_MODE_STREAM && ldcp->stream_remains > 0)) { 3135 D1(ldcp->id, 3136 "ldc_chkq: (0x%llx) queue has pkt(s) or buffered data\n", 3137 ldcp->id); 3138 *hasdata = B_TRUE; 3139 } 3140 3141 mutex_exit(&ldcp->lock); 3142 3143 return (0); 3144 } 3145 3146 3147 /* 3148 * Read 'size' amount of bytes or less. If incoming buffer 3149 * is more than 'size', ENOBUFS is returned. 3150 * 3151 * On return, size contains the number of bytes read. 3152 */ 3153 int 3154 ldc_read(ldc_handle_t handle, caddr_t bufp, size_t *sizep) 3155 { 3156 ldc_chan_t *ldcp; 3157 uint64_t rx_head = 0, rx_tail = 0; 3158 int rv = 0, exit_val; 3159 3160 if (handle == NULL) { 3161 DWARN(DBG_ALL_LDCS, "ldc_read: invalid channel handle\n"); 3162 return (EINVAL); 3163 } 3164 3165 ldcp = (ldc_chan_t *)handle; 3166 3167 /* channel lock */ 3168 mutex_enter(&ldcp->lock); 3169 3170 if (ldcp->tstate != TS_UP) { 3171 DWARN(ldcp->id, 3172 "ldc_read: (0x%llx) channel is not in UP state\n", 3173 ldcp->id); 3174 exit_val = ECONNRESET; 3175 } else { 3176 exit_val = ldcp->read_p(ldcp, bufp, sizep); 3177 } 3178 3179 /* 3180 * if queue has been drained - clear interrupt 3181 */ 3182 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 3183 &ldcp->link_state); 3184 3185 ASSERT(rv == 0); 3186 3187 if (exit_val == 0) { 3188 if (ldcp->link_state == LDC_CHANNEL_DOWN || 3189 ldcp->link_state == LDC_CHANNEL_RESET) { 3190 mutex_enter(&ldcp->tx_lock); 3191 i_ldc_reset(ldcp, B_FALSE); 3192 exit_val = ECONNRESET; 3193 mutex_exit(&ldcp->tx_lock); 3194 } 3195 if ((rv == 0) && 3196 (ldcp->rx_intr_state == LDC_INTR_PEND) && 3197 (rx_head == rx_tail)) { 3198 i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 3199 } 3200 } 3201 3202 mutex_exit(&ldcp->lock); 3203 return (exit_val); 3204 } 3205 3206 /* 3207 * Basic raw mondo read - 3208 * no interpretation of mondo contents at all. 3209 * 3210 * Enter and exit with ldcp->lock held by caller 3211 */ 3212 static int 3213 i_ldc_read_raw(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 3214 { 3215 uint64_t q_size_mask; 3216 ldc_msg_t *msgp; 3217 uint8_t *msgbufp; 3218 int rv = 0, space; 3219 uint64_t rx_head, rx_tail; 3220 3221 space = *sizep; 3222 3223 if (space < LDC_PAYLOAD_SIZE_RAW) 3224 return (ENOBUFS); 3225 3226 ASSERT(mutex_owned(&ldcp->lock)); 3227 3228 /* compute mask for increment */ 3229 q_size_mask = (ldcp->rx_q_entries-1)<<LDC_PACKET_SHIFT; 3230 3231 /* 3232 * Read packet(s) from the queue 3233 */ 3234 rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 3235 &ldcp->link_state); 3236 if (rv != 0) { 3237 cmn_err(CE_WARN, 3238 "ldc_read_raw: (0x%lx) unable to read queue ptrs", 3239 ldcp->id); 3240 return (EIO); 3241 } 3242 D1(ldcp->id, "ldc_read_raw: (0x%llx) rxh=0x%llx," 3243 " rxt=0x%llx, st=0x%llx\n", 3244 ldcp->id, rx_head, rx_tail, ldcp->link_state); 3245 3246 /* reset the channel state if the channel went down */ 3247 if (ldcp->link_state == LDC_CHANNEL_DOWN || 3248 ldcp->link_state == LDC_CHANNEL_RESET) { 3249 mutex_enter(&ldcp->tx_lock); 3250 i_ldc_reset(ldcp, B_FALSE); 3251 mutex_exit(&ldcp->tx_lock); 3252 return (ECONNRESET); 3253 } 3254 3255 /* 3256 * Check for empty queue 3257 */ 3258 if (rx_head == rx_tail) { 3259 *sizep = 0; 3260 return (0); 3261 } 3262 3263 /* get the message */ 3264 msgp = (ldc_msg_t *)(ldcp->rx_q_va + rx_head); 3265 3266 /* if channel is in RAW mode, copy data and return */ 3267 msgbufp = (uint8_t *)&(msgp->raw[0]); 3268 3269 bcopy(msgbufp, target_bufp, LDC_PAYLOAD_SIZE_RAW); 3270 3271 DUMP_PAYLOAD(ldcp->id, msgbufp); 3272 3273 *sizep = LDC_PAYLOAD_SIZE_RAW; 3274 3275 rx_head = (rx_head + LDC_PACKET_SIZE) & q_size_mask; 3276 rv = i_ldc_set_rx_head(ldcp, rx_head); 3277 3278 return (rv); 3279 } 3280 3281 /* 3282 * Process LDC mondos to build larger packets 3283 * with either un-reliable or reliable delivery. 3284 * 3285 * Enter and exit with ldcp->lock held by caller 3286 */ 3287 static int 3288 i_ldc_read_packet(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 3289 { 3290 int rv = 0; 3291 uint64_t rx_head = 0, rx_tail = 0; 3292 uint64_t curr_head = 0; 3293 ldc_msg_t *msg; 3294 caddr_t target; 3295 size_t len = 0, bytes_read = 0; 3296 int retries = 0; 3297 uint64_t q_size_mask; 3298 uint64_t first_fragment = 0; 3299 3300 target = target_bufp; 3301 3302 ASSERT(mutex_owned(&ldcp->lock)); 3303 3304 /* check if the buffer and size are valid */ 3305 if (target_bufp == NULL || *sizep == 0) { 3306 DWARN(ldcp->id, "ldc_read: (0x%llx) invalid buffer/size\n", 3307 ldcp->id); 3308 return (EINVAL); 3309 } 3310 3311 /* compute mask for increment */ 3312 q_size_mask = (ldcp->rx_q_entries-1)<<LDC_PACKET_SHIFT; 3313 3314 /* 3315 * Read packet(s) from the queue 3316 */ 3317 rv = hv_ldc_rx_get_state(ldcp->id, &curr_head, &rx_tail, 3318 &ldcp->link_state); 3319 if (rv != 0) { 3320 cmn_err(CE_WARN, "ldc_read: (0x%lx) unable to read queue ptrs", 3321 ldcp->id); 3322 mutex_enter(&ldcp->tx_lock); 3323 i_ldc_reset(ldcp, B_TRUE); 3324 mutex_exit(&ldcp->tx_lock); 3325 return (ECONNRESET); 3326 } 3327 D1(ldcp->id, "ldc_read: (0x%llx) chd=0x%llx, tl=0x%llx, st=0x%llx\n", 3328 ldcp->id, curr_head, rx_tail, ldcp->link_state); 3329 3330 /* reset the channel state if the channel went down */ 3331 if (ldcp->link_state != LDC_CHANNEL_UP) 3332 goto channel_is_reset; 3333 3334 for (;;) { 3335 3336 if (curr_head == rx_tail) { 3337 rv = hv_ldc_rx_get_state(ldcp->id, 3338 &rx_head, &rx_tail, &ldcp->link_state); 3339 if (rv != 0) { 3340 cmn_err(CE_WARN, 3341 "ldc_read: (0x%lx) cannot read queue ptrs", 3342 ldcp->id); 3343 mutex_enter(&ldcp->tx_lock); 3344 i_ldc_reset(ldcp, B_TRUE); 3345 mutex_exit(&ldcp->tx_lock); 3346 return (ECONNRESET); 3347 } 3348 if (ldcp->link_state != LDC_CHANNEL_UP) 3349 goto channel_is_reset; 3350 3351 if (curr_head == rx_tail) { 3352 3353 /* If in the middle of a fragmented xfer */ 3354 if (first_fragment != 0) { 3355 3356 /* wait for ldc_delay usecs */ 3357 drv_usecwait(ldc_delay); 3358 3359 if (++retries < ldc_max_retries) 3360 continue; 3361 3362 *sizep = 0; 3363 ldcp->last_msg_rcd = first_fragment - 1; 3364 DWARN(DBG_ALL_LDCS, "ldc_read: " 3365 "(0x%llx) read timeout", 3366 ldcp->id); 3367 return (EAGAIN); 3368 } 3369 *sizep = 0; 3370 break; 3371 } 3372 } 3373 retries = 0; 3374 3375 D2(ldcp->id, 3376 "ldc_read: (0x%llx) chd=0x%llx, rxhd=0x%llx, rxtl=0x%llx\n", 3377 ldcp->id, curr_head, rx_head, rx_tail); 3378 3379 /* get the message */ 3380 msg = (ldc_msg_t *)(ldcp->rx_q_va + curr_head); 3381 3382 DUMP_LDC_PKT(ldcp, "ldc_read received pkt", 3383 ldcp->rx_q_va + curr_head); 3384 3385 /* Check the message ID for the message received */ 3386 if ((rv = i_ldc_check_seqid(ldcp, msg)) != 0) { 3387 3388 DWARN(ldcp->id, "ldc_read: (0x%llx) seqid error, " 3389 "q_ptrs=0x%lx,0x%lx", ldcp->id, rx_head, rx_tail); 3390 3391 /* throw away data */ 3392 bytes_read = 0; 3393 3394 /* Reset last_msg_rcd to start of message */ 3395 if (first_fragment != 0) { 3396 ldcp->last_msg_rcd = first_fragment - 1; 3397 first_fragment = 0; 3398 } 3399 /* 3400 * Send a NACK -- invalid seqid 3401 * get the current tail for the response 3402 */ 3403 rv = i_ldc_send_pkt(ldcp, msg->type, LDC_NACK, 3404 (msg->ctrl & LDC_CTRL_MASK)); 3405 if (rv) { 3406 cmn_err(CE_NOTE, 3407 "ldc_read: (0x%lx) err sending " 3408 "NACK msg\n", ldcp->id); 3409 3410 /* if cannot send NACK - reset channel */ 3411 mutex_enter(&ldcp->tx_lock); 3412 i_ldc_reset(ldcp, B_FALSE); 3413 mutex_exit(&ldcp->tx_lock); 3414 rv = ECONNRESET; 3415 break; 3416 } 3417 3418 /* purge receive queue */ 3419 rv = i_ldc_set_rx_head(ldcp, rx_tail); 3420 3421 break; 3422 } 3423 3424 /* 3425 * Process any messages of type CTRL messages 3426 * Future implementations should try to pass these 3427 * to LDC link by resetting the intr state. 3428 * 3429 * NOTE: not done as a switch() as type can be both ctrl+data 3430 */ 3431 if (msg->type & LDC_CTRL) { 3432 if (rv = i_ldc_ctrlmsg(ldcp, msg)) { 3433 if (rv == EAGAIN) 3434 continue; 3435 rv = i_ldc_set_rx_head(ldcp, rx_tail); 3436 *sizep = 0; 3437 bytes_read = 0; 3438 break; 3439 } 3440 } 3441 3442 /* process data ACKs */ 3443 if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 3444 if (rv = i_ldc_process_data_ACK(ldcp, msg)) { 3445 *sizep = 0; 3446 bytes_read = 0; 3447 break; 3448 } 3449 } 3450 3451 /* process data messages */ 3452 if ((msg->type & LDC_DATA) && (msg->stype & LDC_INFO)) { 3453 3454 uint8_t *msgbuf = (uint8_t *)( 3455 (ldcp->mode == LDC_MODE_RELIABLE || 3456 ldcp->mode == LDC_MODE_STREAM) 3457 ? msg->rdata : msg->udata); 3458 3459 D2(ldcp->id, 3460 "ldc_read: (0x%llx) received data msg\n", ldcp->id); 3461 3462 /* get the packet length */ 3463 len = (msg->env & LDC_LEN_MASK); 3464 3465 /* 3466 * FUTURE OPTIMIZATION: 3467 * dont need to set q head for every 3468 * packet we read just need to do this when 3469 * we are done or need to wait for more 3470 * mondos to make a full packet - this is 3471 * currently expensive. 3472 */ 3473 3474 if (first_fragment == 0) { 3475 3476 /* 3477 * first packets should always have the start 3478 * bit set (even for a single packet). If not 3479 * throw away the packet 3480 */ 3481 if (!(msg->env & LDC_FRAG_START)) { 3482 3483 DWARN(DBG_ALL_LDCS, 3484 "ldc_read: (0x%llx) not start - " 3485 "frag=%x\n", ldcp->id, 3486 (msg->env) & LDC_FRAG_MASK); 3487 3488 /* toss pkt, inc head, cont reading */ 3489 bytes_read = 0; 3490 target = target_bufp; 3491 curr_head = 3492 (curr_head + LDC_PACKET_SIZE) 3493 & q_size_mask; 3494 if (rv = i_ldc_set_rx_head(ldcp, 3495 curr_head)) 3496 break; 3497 3498 continue; 3499 } 3500 3501 first_fragment = msg->seqid; 3502 } else { 3503 /* check to see if this is a pkt w/ START bit */ 3504 if (msg->env & LDC_FRAG_START) { 3505 DWARN(DBG_ALL_LDCS, 3506 "ldc_read:(0x%llx) unexpected pkt" 3507 " env=0x%x discarding %d bytes," 3508 " lastmsg=%d, currentmsg=%d\n", 3509 ldcp->id, msg->env&LDC_FRAG_MASK, 3510 bytes_read, ldcp->last_msg_rcd, 3511 msg->seqid); 3512 3513 /* throw data we have read so far */ 3514 bytes_read = 0; 3515 target = target_bufp; 3516 first_fragment = msg->seqid; 3517 3518 if (rv = i_ldc_set_rx_head(ldcp, 3519 curr_head)) 3520 break; 3521 } 3522 } 3523 3524 /* copy (next) pkt into buffer */ 3525 if (len <= (*sizep - bytes_read)) { 3526 bcopy(msgbuf, target, len); 3527 target += len; 3528 bytes_read += len; 3529 } else { 3530 /* 3531 * there is not enough space in the buffer to 3532 * read this pkt. throw message away & continue 3533 * reading data from queue 3534 */ 3535 DWARN(DBG_ALL_LDCS, 3536 "ldc_read: (0x%llx) buffer too small, " 3537 "head=0x%lx, expect=%d, got=%d\n", ldcp->id, 3538 curr_head, *sizep, bytes_read+len); 3539 3540 first_fragment = 0; 3541 target = target_bufp; 3542 bytes_read = 0; 3543 3544 /* throw away everything received so far */ 3545 if (rv = i_ldc_set_rx_head(ldcp, curr_head)) 3546 break; 3547 3548 /* continue reading remaining pkts */ 3549 continue; 3550 } 3551 } 3552 3553 /* set the message id */ 3554 ldcp->last_msg_rcd = msg->seqid; 3555 3556 /* move the head one position */ 3557 curr_head = (curr_head + LDC_PACKET_SIZE) & q_size_mask; 3558 3559 if (msg->env & LDC_FRAG_STOP) { 3560 3561 /* 3562 * All pkts that are part of this fragmented transfer 3563 * have been read or this was a single pkt read 3564 * or there was an error 3565 */ 3566 3567 /* set the queue head */ 3568 if (rv = i_ldc_set_rx_head(ldcp, curr_head)) 3569 bytes_read = 0; 3570 3571 *sizep = bytes_read; 3572 3573 break; 3574 } 3575 3576 /* advance head if it is a DATA ACK */ 3577 if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 3578 3579 /* set the queue head */ 3580 if (rv = i_ldc_set_rx_head(ldcp, curr_head)) { 3581 bytes_read = 0; 3582 break; 3583 } 3584 3585 D2(ldcp->id, "ldc_read: (0x%llx) set ACK qhead 0x%llx", 3586 ldcp->id, curr_head); 3587 } 3588 3589 } /* for (;;) */ 3590 3591 3592 /* 3593 * If useful data was read - Send msg ACK 3594 * OPTIMIZE: do not send ACK for all msgs - use some frequency 3595 */ 3596 if ((bytes_read > 0) && (ldcp->mode == LDC_MODE_RELIABLE || 3597 ldcp->mode == LDC_MODE_STREAM)) { 3598 3599 rv = i_ldc_send_pkt(ldcp, LDC_DATA, LDC_ACK, 0); 3600 if (rv && rv != EWOULDBLOCK) { 3601 cmn_err(CE_NOTE, 3602 "ldc_read: (0x%lx) cannot send ACK\n", ldcp->id); 3603 3604 /* if cannot send ACK - reset channel */ 3605 goto channel_is_reset; 3606 } 3607 } 3608 3609 D2(ldcp->id, "ldc_read: (0x%llx) end size=%d", ldcp->id, *sizep); 3610 3611 return (rv); 3612 3613 channel_is_reset: 3614 mutex_enter(&ldcp->tx_lock); 3615 i_ldc_reset(ldcp, B_FALSE); 3616 mutex_exit(&ldcp->tx_lock); 3617 return (ECONNRESET); 3618 } 3619 3620 /* 3621 * Use underlying reliable packet mechanism to fetch 3622 * and buffer incoming packets so we can hand them back as 3623 * a basic byte stream. 3624 * 3625 * Enter and exit with ldcp->lock held by caller 3626 */ 3627 static int 3628 i_ldc_read_stream(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 3629 { 3630 int rv; 3631 size_t size; 3632 3633 ASSERT(mutex_owned(&ldcp->lock)); 3634 3635 D2(ldcp->id, "i_ldc_read_stream: (0x%llx) buffer size=%d", 3636 ldcp->id, *sizep); 3637 3638 if (ldcp->stream_remains == 0) { 3639 size = ldcp->mtu; 3640 rv = i_ldc_read_packet(ldcp, 3641 (caddr_t)ldcp->stream_bufferp, &size); 3642 D2(ldcp->id, "i_ldc_read_stream: read packet (0x%llx) size=%d", 3643 ldcp->id, size); 3644 3645 if (rv != 0) 3646 return (rv); 3647 3648 ldcp->stream_remains = size; 3649 ldcp->stream_offset = 0; 3650 } 3651 3652 size = MIN(ldcp->stream_remains, *sizep); 3653 3654 bcopy(ldcp->stream_bufferp + ldcp->stream_offset, target_bufp, size); 3655 ldcp->stream_offset += size; 3656 ldcp->stream_remains -= size; 3657 3658 D2(ldcp->id, "i_ldc_read_stream: (0x%llx) fill from buffer size=%d", 3659 ldcp->id, size); 3660 3661 *sizep = size; 3662 return (0); 3663 } 3664 3665 /* 3666 * Write specified amount of bytes to the channel 3667 * in multiple pkts of pkt_payload size. Each 3668 * packet is tagged with an unique packet ID in 3669 * the case of a reliable link. 3670 * 3671 * On return, size contains the number of bytes written. 3672 */ 3673 int 3674 ldc_write(ldc_handle_t handle, caddr_t buf, size_t *sizep) 3675 { 3676 ldc_chan_t *ldcp; 3677 int rv = 0; 3678 3679 if (handle == NULL) { 3680 DWARN(DBG_ALL_LDCS, "ldc_write: invalid channel handle\n"); 3681 return (EINVAL); 3682 } 3683 ldcp = (ldc_chan_t *)handle; 3684 3685 /* check if writes can occur */ 3686 if (!mutex_tryenter(&ldcp->tx_lock)) { 3687 /* 3688 * Could not get the lock - channel could 3689 * be in the process of being unconfigured 3690 * or reader has encountered an error 3691 */ 3692 return (EAGAIN); 3693 } 3694 3695 /* check if non-zero data to write */ 3696 if (buf == NULL || sizep == NULL) { 3697 DWARN(ldcp->id, "ldc_write: (0x%llx) invalid data write\n", 3698 ldcp->id); 3699 mutex_exit(&ldcp->tx_lock); 3700 return (EINVAL); 3701 } 3702 3703 if (*sizep == 0) { 3704 DWARN(ldcp->id, "ldc_write: (0x%llx) write size of zero\n", 3705 ldcp->id); 3706 mutex_exit(&ldcp->tx_lock); 3707 return (0); 3708 } 3709 3710 /* Check if channel is UP for data exchange */ 3711 if (ldcp->tstate != TS_UP) { 3712 DWARN(ldcp->id, 3713 "ldc_write: (0x%llx) channel is not in UP state\n", 3714 ldcp->id); 3715 *sizep = 0; 3716 rv = ECONNRESET; 3717 } else { 3718 rv = ldcp->write_p(ldcp, buf, sizep); 3719 } 3720 3721 mutex_exit(&ldcp->tx_lock); 3722 3723 return (rv); 3724 } 3725 3726 /* 3727 * Write a raw packet to the channel 3728 * On return, size contains the number of bytes written. 3729 */ 3730 static int 3731 i_ldc_write_raw(ldc_chan_t *ldcp, caddr_t buf, size_t *sizep) 3732 { 3733 ldc_msg_t *ldcmsg; 3734 uint64_t tx_head, tx_tail, new_tail; 3735 int rv = 0; 3736 size_t size; 3737 3738 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 3739 ASSERT(ldcp->mode == LDC_MODE_RAW); 3740 3741 size = *sizep; 3742 3743 /* 3744 * Check to see if the packet size is less than or 3745 * equal to packet size support in raw mode 3746 */ 3747 if (size > ldcp->pkt_payload) { 3748 DWARN(ldcp->id, 3749 "ldc_write: (0x%llx) invalid size (0x%llx) for RAW mode\n", 3750 ldcp->id, *sizep); 3751 *sizep = 0; 3752 return (EMSGSIZE); 3753 } 3754 3755 /* get the qptrs for the tx queue */ 3756 rv = hv_ldc_tx_get_state(ldcp->id, 3757 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 3758 if (rv != 0) { 3759 cmn_err(CE_WARN, 3760 "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp->id); 3761 *sizep = 0; 3762 return (EIO); 3763 } 3764 3765 if (ldcp->link_state == LDC_CHANNEL_DOWN || 3766 ldcp->link_state == LDC_CHANNEL_RESET) { 3767 DWARN(ldcp->id, 3768 "ldc_write: (0x%llx) channel down/reset\n", ldcp->id); 3769 3770 *sizep = 0; 3771 if (mutex_tryenter(&ldcp->lock)) { 3772 i_ldc_reset(ldcp, B_FALSE); 3773 mutex_exit(&ldcp->lock); 3774 } else { 3775 /* 3776 * Release Tx lock, and then reacquire channel 3777 * and Tx lock in correct order 3778 */ 3779 mutex_exit(&ldcp->tx_lock); 3780 mutex_enter(&ldcp->lock); 3781 mutex_enter(&ldcp->tx_lock); 3782 i_ldc_reset(ldcp, B_FALSE); 3783 mutex_exit(&ldcp->lock); 3784 } 3785 return (ECONNRESET); 3786 } 3787 3788 tx_tail = ldcp->tx_tail; 3789 tx_head = ldcp->tx_head; 3790 new_tail = (tx_tail + LDC_PACKET_SIZE) & 3791 ((ldcp->tx_q_entries-1) << LDC_PACKET_SHIFT); 3792 3793 if (new_tail == tx_head) { 3794 DWARN(DBG_ALL_LDCS, 3795 "ldc_write: (0x%llx) TX queue is full\n", ldcp->id); 3796 *sizep = 0; 3797 return (EWOULDBLOCK); 3798 } 3799 3800 D2(ldcp->id, "ldc_write: (0x%llx) start xfer size=%d", 3801 ldcp->id, size); 3802 3803 /* Send the data now */ 3804 ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 3805 3806 /* copy the data into pkt */ 3807 bcopy((uint8_t *)buf, ldcmsg, size); 3808 3809 /* increment tail */ 3810 tx_tail = new_tail; 3811 3812 /* 3813 * All packets have been copied into the TX queue 3814 * update the tail ptr in the HV 3815 */ 3816 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 3817 if (rv) { 3818 if (rv == EWOULDBLOCK) { 3819 DWARN(ldcp->id, "ldc_write: (0x%llx) write timed out\n", 3820 ldcp->id); 3821 *sizep = 0; 3822 return (EWOULDBLOCK); 3823 } 3824 3825 *sizep = 0; 3826 if (mutex_tryenter(&ldcp->lock)) { 3827 i_ldc_reset(ldcp, B_FALSE); 3828 mutex_exit(&ldcp->lock); 3829 } else { 3830 /* 3831 * Release Tx lock, and then reacquire channel 3832 * and Tx lock in correct order 3833 */ 3834 mutex_exit(&ldcp->tx_lock); 3835 mutex_enter(&ldcp->lock); 3836 mutex_enter(&ldcp->tx_lock); 3837 i_ldc_reset(ldcp, B_FALSE); 3838 mutex_exit(&ldcp->lock); 3839 } 3840 return (ECONNRESET); 3841 } 3842 3843 ldcp->tx_tail = tx_tail; 3844 *sizep = size; 3845 3846 D2(ldcp->id, "ldc_write: (0x%llx) end xfer size=%d", ldcp->id, size); 3847 3848 return (rv); 3849 } 3850 3851 3852 /* 3853 * Write specified amount of bytes to the channel 3854 * in multiple pkts of pkt_payload size. Each 3855 * packet is tagged with an unique packet ID in 3856 * the case of a reliable link. 3857 * 3858 * On return, size contains the number of bytes written. 3859 * This function needs to ensure that the write size is < MTU size 3860 */ 3861 static int 3862 i_ldc_write_packet(ldc_chan_t *ldcp, caddr_t buf, size_t *size) 3863 { 3864 ldc_msg_t *ldcmsg; 3865 uint64_t tx_head, tx_tail, new_tail, start; 3866 uint64_t txq_size_mask, numavail; 3867 uint8_t *msgbuf, *source = (uint8_t *)buf; 3868 size_t len, bytes_written = 0, remaining; 3869 int rv; 3870 uint32_t curr_seqid; 3871 3872 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 3873 3874 ASSERT(ldcp->mode == LDC_MODE_RELIABLE || 3875 ldcp->mode == LDC_MODE_UNRELIABLE || 3876 ldcp->mode == LDC_MODE_STREAM); 3877 3878 /* compute mask for increment */ 3879 txq_size_mask = (ldcp->tx_q_entries - 1) << LDC_PACKET_SHIFT; 3880 3881 /* get the qptrs for the tx queue */ 3882 rv = hv_ldc_tx_get_state(ldcp->id, 3883 &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 3884 if (rv != 0) { 3885 cmn_err(CE_WARN, 3886 "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp->id); 3887 *size = 0; 3888 return (EIO); 3889 } 3890 3891 if (ldcp->link_state == LDC_CHANNEL_DOWN || 3892 ldcp->link_state == LDC_CHANNEL_RESET) { 3893 DWARN(ldcp->id, 3894 "ldc_write: (0x%llx) channel down/reset\n", ldcp->id); 3895 *size = 0; 3896 if (mutex_tryenter(&ldcp->lock)) { 3897 i_ldc_reset(ldcp, B_FALSE); 3898 mutex_exit(&ldcp->lock); 3899 } else { 3900 /* 3901 * Release Tx lock, and then reacquire channel 3902 * and Tx lock in correct order 3903 */ 3904 mutex_exit(&ldcp->tx_lock); 3905 mutex_enter(&ldcp->lock); 3906 mutex_enter(&ldcp->tx_lock); 3907 i_ldc_reset(ldcp, B_FALSE); 3908 mutex_exit(&ldcp->lock); 3909 } 3910 return (ECONNRESET); 3911 } 3912 3913 tx_tail = ldcp->tx_tail; 3914 new_tail = (tx_tail + LDC_PACKET_SIZE) % 3915 (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 3916 3917 /* 3918 * Link mode determines whether we use HV Tx head or the 3919 * private protocol head (corresponding to last ACKd pkt) for 3920 * determining how much we can write 3921 */ 3922 tx_head = (ldcp->mode == LDC_MODE_RELIABLE || 3923 ldcp->mode == LDC_MODE_STREAM) 3924 ? ldcp->tx_ackd_head : ldcp->tx_head; 3925 if (new_tail == tx_head) { 3926 DWARN(DBG_ALL_LDCS, 3927 "ldc_write: (0x%llx) TX queue is full\n", ldcp->id); 3928 *size = 0; 3929 return (EWOULDBLOCK); 3930 } 3931 3932 /* 3933 * Make sure that the LDC Tx queue has enough space 3934 */ 3935 numavail = (tx_head >> LDC_PACKET_SHIFT) - (tx_tail >> LDC_PACKET_SHIFT) 3936 + ldcp->tx_q_entries - 1; 3937 numavail %= ldcp->tx_q_entries; 3938 3939 if (*size > (numavail * ldcp->pkt_payload)) { 3940 DWARN(DBG_ALL_LDCS, 3941 "ldc_write: (0x%llx) TX queue has no space\n", ldcp->id); 3942 return (EWOULDBLOCK); 3943 } 3944 3945 D2(ldcp->id, "ldc_write: (0x%llx) start xfer size=%d", 3946 ldcp->id, *size); 3947 3948 /* Send the data now */ 3949 bytes_written = 0; 3950 curr_seqid = ldcp->last_msg_snt; 3951 start = tx_tail; 3952 3953 while (*size > bytes_written) { 3954 3955 ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 3956 3957 msgbuf = (uint8_t *)((ldcp->mode == LDC_MODE_RELIABLE || 3958 ldcp->mode == LDC_MODE_STREAM) 3959 ? ldcmsg->rdata : ldcmsg->udata); 3960 3961 ldcmsg->type = LDC_DATA; 3962 ldcmsg->stype = LDC_INFO; 3963 ldcmsg->ctrl = 0; 3964 3965 remaining = *size - bytes_written; 3966 len = min(ldcp->pkt_payload, remaining); 3967 ldcmsg->env = (uint8_t)len; 3968 3969 curr_seqid++; 3970 ldcmsg->seqid = curr_seqid; 3971 3972 /* copy the data into pkt */ 3973 bcopy(source, msgbuf, len); 3974 3975 source += len; 3976 bytes_written += len; 3977 3978 /* increment tail */ 3979 tx_tail = (tx_tail + LDC_PACKET_SIZE) & txq_size_mask; 3980 3981 ASSERT(tx_tail != tx_head); 3982 } 3983 3984 /* Set the start and stop bits */ 3985 ldcmsg->env |= LDC_FRAG_STOP; 3986 ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + start); 3987 ldcmsg->env |= LDC_FRAG_START; 3988 3989 /* 3990 * All packets have been copied into the TX queue 3991 * update the tail ptr in the HV 3992 */ 3993 rv = i_ldc_set_tx_tail(ldcp, tx_tail); 3994 if (rv == 0) { 3995 ldcp->tx_tail = tx_tail; 3996 ldcp->last_msg_snt = curr_seqid; 3997 *size = bytes_written; 3998 } else { 3999 int rv2; 4000 4001 if (rv != EWOULDBLOCK) { 4002 *size = 0; 4003 if (mutex_tryenter(&ldcp->lock)) { 4004 i_ldc_reset(ldcp, B_FALSE); 4005 mutex_exit(&ldcp->lock); 4006 } else { 4007 /* 4008 * Release Tx lock, and then reacquire channel 4009 * and Tx lock in correct order 4010 */ 4011 mutex_exit(&ldcp->tx_lock); 4012 mutex_enter(&ldcp->lock); 4013 mutex_enter(&ldcp->tx_lock); 4014 i_ldc_reset(ldcp, B_FALSE); 4015 mutex_exit(&ldcp->lock); 4016 } 4017 return (ECONNRESET); 4018 } 4019 4020 DWARN(ldcp->id, "hv_tx_set_tail returns 0x%x (head 0x%x, " 4021 "old tail 0x%x, new tail 0x%x, qsize=0x%x)\n", 4022 rv, ldcp->tx_head, ldcp->tx_tail, tx_tail, 4023 (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 4024 4025 rv2 = hv_ldc_tx_get_state(ldcp->id, 4026 &tx_head, &tx_tail, &ldcp->link_state); 4027 4028 DWARN(ldcp->id, "hv_ldc_tx_get_state returns 0x%x " 4029 "(head 0x%x, tail 0x%x state 0x%x)\n", 4030 rv2, tx_head, tx_tail, ldcp->link_state); 4031 4032 *size = 0; 4033 } 4034 4035 D2(ldcp->id, "ldc_write: (0x%llx) end xfer size=%d", ldcp->id, *size); 4036 4037 return (rv); 4038 } 4039 4040 /* 4041 * Write specified amount of bytes to the channel 4042 * in multiple pkts of pkt_payload size. Each 4043 * packet is tagged with an unique packet ID in 4044 * the case of a reliable link. 4045 * 4046 * On return, size contains the number of bytes written. 4047 * This function needs to ensure that the write size is < MTU size 4048 */ 4049 static int 4050 i_ldc_write_stream(ldc_chan_t *ldcp, caddr_t buf, size_t *sizep) 4051 { 4052 ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 4053 ASSERT(ldcp->mode == LDC_MODE_STREAM); 4054 4055 /* Truncate packet to max of MTU size */ 4056 if (*sizep > ldcp->mtu) *sizep = ldcp->mtu; 4057 return (i_ldc_write_packet(ldcp, buf, sizep)); 4058 } 4059 4060 4061 /* 4062 * Interfaces for channel nexus to register/unregister with LDC module 4063 * The nexus will register functions to be used to register individual 4064 * channels with the nexus and enable interrupts for the channels 4065 */ 4066 int 4067 ldc_register(ldc_cnex_t *cinfo) 4068 { 4069 ldc_chan_t *ldcp; 4070 4071 if (cinfo == NULL || cinfo->dip == NULL || 4072 cinfo->reg_chan == NULL || cinfo->unreg_chan == NULL || 4073 cinfo->add_intr == NULL || cinfo->rem_intr == NULL || 4074 cinfo->clr_intr == NULL) { 4075 4076 DWARN(DBG_ALL_LDCS, "ldc_register: invalid nexus info\n"); 4077 return (EINVAL); 4078 } 4079 4080 mutex_enter(&ldcssp->lock); 4081 4082 /* nexus registration */ 4083 ldcssp->cinfo.dip = cinfo->dip; 4084 ldcssp->cinfo.reg_chan = cinfo->reg_chan; 4085 ldcssp->cinfo.unreg_chan = cinfo->unreg_chan; 4086 ldcssp->cinfo.add_intr = cinfo->add_intr; 4087 ldcssp->cinfo.rem_intr = cinfo->rem_intr; 4088 ldcssp->cinfo.clr_intr = cinfo->clr_intr; 4089 4090 /* register any channels that might have been previously initialized */ 4091 ldcp = ldcssp->chan_list; 4092 while (ldcp) { 4093 if ((ldcp->tstate & TS_QCONF_RDY) && 4094 (ldcp->tstate & TS_CNEX_RDY) == 0) 4095 (void) i_ldc_register_channel(ldcp); 4096 4097 ldcp = ldcp->next; 4098 } 4099 4100 mutex_exit(&ldcssp->lock); 4101 4102 return (0); 4103 } 4104 4105 int 4106 ldc_unregister(ldc_cnex_t *cinfo) 4107 { 4108 if (cinfo == NULL || cinfo->dip == NULL) { 4109 DWARN(DBG_ALL_LDCS, "ldc_unregister: invalid nexus info\n"); 4110 return (EINVAL); 4111 } 4112 4113 mutex_enter(&ldcssp->lock); 4114 4115 if (cinfo->dip != ldcssp->cinfo.dip) { 4116 DWARN(DBG_ALL_LDCS, "ldc_unregister: invalid dip\n"); 4117 mutex_exit(&ldcssp->lock); 4118 return (EINVAL); 4119 } 4120 4121 /* nexus unregister */ 4122 ldcssp->cinfo.dip = NULL; 4123 ldcssp->cinfo.reg_chan = NULL; 4124 ldcssp->cinfo.unreg_chan = NULL; 4125 ldcssp->cinfo.add_intr = NULL; 4126 ldcssp->cinfo.rem_intr = NULL; 4127 ldcssp->cinfo.clr_intr = NULL; 4128 4129 mutex_exit(&ldcssp->lock); 4130 4131 return (0); 4132 } 4133 4134 4135 /* ------------------------------------------------------------------------- */ 4136 4137 /* 4138 * Allocate a memory handle for the channel and link it into the list 4139 * Also choose which memory table to use if this is the first handle 4140 * being assigned to this channel 4141 */ 4142 int 4143 ldc_mem_alloc_handle(ldc_handle_t handle, ldc_mem_handle_t *mhandle) 4144 { 4145 ldc_chan_t *ldcp; 4146 ldc_mhdl_t *mhdl; 4147 4148 if (handle == NULL) { 4149 DWARN(DBG_ALL_LDCS, 4150 "ldc_mem_alloc_handle: invalid channel handle\n"); 4151 return (EINVAL); 4152 } 4153 ldcp = (ldc_chan_t *)handle; 4154 4155 mutex_enter(&ldcp->lock); 4156 4157 /* check to see if channel is initalized */ 4158 if ((ldcp->tstate & ~TS_IN_RESET) < TS_INIT) { 4159 DWARN(ldcp->id, 4160 "ldc_mem_alloc_handle: (0x%llx) channel not initialized\n", 4161 ldcp->id); 4162 mutex_exit(&ldcp->lock); 4163 return (EINVAL); 4164 } 4165 4166 /* allocate handle for channel */ 4167 mhdl = kmem_cache_alloc(ldcssp->memhdl_cache, KM_SLEEP); 4168 4169 /* initialize the lock */ 4170 mutex_init(&mhdl->lock, NULL, MUTEX_DRIVER, NULL); 4171 4172 mhdl->myshadow = B_FALSE; 4173 mhdl->memseg = NULL; 4174 mhdl->ldcp = ldcp; 4175 mhdl->status = LDC_UNBOUND; 4176 4177 /* insert memory handle (@ head) into list */ 4178 if (ldcp->mhdl_list == NULL) { 4179 ldcp->mhdl_list = mhdl; 4180 mhdl->next = NULL; 4181 } else { 4182 /* insert @ head */ 4183 mhdl->next = ldcp->mhdl_list; 4184 ldcp->mhdl_list = mhdl; 4185 } 4186 4187 /* return the handle */ 4188 *mhandle = (ldc_mem_handle_t)mhdl; 4189 4190 mutex_exit(&ldcp->lock); 4191 4192 D1(ldcp->id, "ldc_mem_alloc_handle: (0x%llx) allocated handle 0x%llx\n", 4193 ldcp->id, mhdl); 4194 4195 return (0); 4196 } 4197 4198 /* 4199 * Free memory handle for the channel and unlink it from the list 4200 */ 4201 int 4202 ldc_mem_free_handle(ldc_mem_handle_t mhandle) 4203 { 4204 ldc_mhdl_t *mhdl, *phdl; 4205 ldc_chan_t *ldcp; 4206 4207 if (mhandle == NULL) { 4208 DWARN(DBG_ALL_LDCS, 4209 "ldc_mem_free_handle: invalid memory handle\n"); 4210 return (EINVAL); 4211 } 4212 mhdl = (ldc_mhdl_t *)mhandle; 4213 4214 mutex_enter(&mhdl->lock); 4215 4216 ldcp = mhdl->ldcp; 4217 4218 if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) { 4219 DWARN(ldcp->id, 4220 "ldc_mem_free_handle: cannot free, 0x%llx hdl bound\n", 4221 mhdl); 4222 mutex_exit(&mhdl->lock); 4223 return (EINVAL); 4224 } 4225 mutex_exit(&mhdl->lock); 4226 4227 mutex_enter(&ldcp->mlist_lock); 4228 4229 phdl = ldcp->mhdl_list; 4230 4231 /* first handle */ 4232 if (phdl == mhdl) { 4233 ldcp->mhdl_list = mhdl->next; 4234 mutex_destroy(&mhdl->lock); 4235 kmem_cache_free(ldcssp->memhdl_cache, mhdl); 4236 4237 D1(ldcp->id, 4238 "ldc_mem_free_handle: (0x%llx) freed handle 0x%llx\n", 4239 ldcp->id, mhdl); 4240 } else { 4241 /* walk the list - unlink and free */ 4242 while (phdl != NULL) { 4243 if (phdl->next == mhdl) { 4244 phdl->next = mhdl->next; 4245 mutex_destroy(&mhdl->lock); 4246 kmem_cache_free(ldcssp->memhdl_cache, mhdl); 4247 D1(ldcp->id, 4248 "ldc_mem_free_handle: (0x%llx) freed " 4249 "handle 0x%llx\n", ldcp->id, mhdl); 4250 break; 4251 } 4252 phdl = phdl->next; 4253 } 4254 } 4255 4256 if (phdl == NULL) { 4257 DWARN(ldcp->id, 4258 "ldc_mem_free_handle: invalid handle 0x%llx\n", mhdl); 4259 mutex_exit(&ldcp->mlist_lock); 4260 return (EINVAL); 4261 } 4262 4263 mutex_exit(&ldcp->mlist_lock); 4264 4265 return (0); 4266 } 4267 4268 /* 4269 * Bind a memory handle to a virtual address. 4270 * The virtual address is converted to the corresponding real addresses. 4271 * Returns pointer to the first ldc_mem_cookie and the total number 4272 * of cookies for this virtual address. Other cookies can be obtained 4273 * using the ldc_mem_nextcookie() call. If the pages are stored in 4274 * consecutive locations in the table, a single cookie corresponding to 4275 * the first location is returned. The cookie size spans all the entries. 4276 * 4277 * If the VA corresponds to a page that is already being exported, reuse 4278 * the page and do not export it again. Bump the page's use count. 4279 */ 4280 int 4281 ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr, size_t len, 4282 uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount) 4283 { 4284 ldc_mhdl_t *mhdl; 4285 ldc_chan_t *ldcp; 4286 ldc_mtbl_t *mtbl; 4287 ldc_memseg_t *memseg; 4288 ldc_mte_t tmp_mte; 4289 uint64_t index, prev_index = 0; 4290 int64_t cookie_idx; 4291 uintptr_t raddr, ra_aligned; 4292 uint64_t psize, poffset, v_offset; 4293 uint64_t pg_shift, pg_size, pg_size_code, pg_mask; 4294 pgcnt_t npages; 4295 caddr_t v_align, addr; 4296 int i, rv; 4297 4298 if (mhandle == NULL) { 4299 DWARN(DBG_ALL_LDCS, 4300 "ldc_mem_bind_handle: invalid memory handle\n"); 4301 return (EINVAL); 4302 } 4303 mhdl = (ldc_mhdl_t *)mhandle; 4304 ldcp = mhdl->ldcp; 4305 4306 /* clear count */ 4307 *ccount = 0; 4308 4309 mutex_enter(&mhdl->lock); 4310 4311 if (mhdl->status == LDC_BOUND || mhdl->memseg != NULL) { 4312 DWARN(ldcp->id, 4313 "ldc_mem_bind_handle: (0x%x) handle already bound\n", 4314 mhandle); 4315 mutex_exit(&mhdl->lock); 4316 return (EINVAL); 4317 } 4318 4319 /* Force address and size to be 8-byte aligned */ 4320 if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 4321 DWARN(ldcp->id, 4322 "ldc_mem_bind_handle: addr/size is not 8-byte aligned\n"); 4323 mutex_exit(&mhdl->lock); 4324 return (EINVAL); 4325 } 4326 4327 /* 4328 * If this channel is binding a memory handle for the 4329 * first time allocate it a memory map table and initialize it 4330 */ 4331 if ((mtbl = ldcp->mtbl) == NULL) { 4332 4333 mutex_enter(&ldcp->lock); 4334 4335 /* Allocate and initialize the map table structure */ 4336 mtbl = kmem_zalloc(sizeof (ldc_mtbl_t), KM_SLEEP); 4337 mtbl->num_entries = mtbl->num_avail = ldc_maptable_entries; 4338 mtbl->size = ldc_maptable_entries * sizeof (ldc_mte_slot_t); 4339 mtbl->next_entry = NULL; 4340 mtbl->contigmem = B_TRUE; 4341 4342 /* Allocate the table itself */ 4343 mtbl->table = (ldc_mte_slot_t *) 4344 contig_mem_alloc_align(mtbl->size, MMU_PAGESIZE); 4345 if (mtbl->table == NULL) { 4346 4347 /* allocate a page of memory using kmem_alloc */ 4348 mtbl->table = kmem_alloc(MMU_PAGESIZE, KM_SLEEP); 4349 mtbl->size = MMU_PAGESIZE; 4350 mtbl->contigmem = B_FALSE; 4351 mtbl->num_entries = mtbl->num_avail = 4352 mtbl->size / sizeof (ldc_mte_slot_t); 4353 DWARN(ldcp->id, 4354 "ldc_mem_bind_handle: (0x%llx) reduced tbl size " 4355 "to %lx entries\n", ldcp->id, mtbl->num_entries); 4356 } 4357 4358 /* zero out the memory */ 4359 bzero(mtbl->table, mtbl->size); 4360 4361 /* initialize the lock */ 4362 mutex_init(&mtbl->lock, NULL, MUTEX_DRIVER, NULL); 4363 4364 /* register table for this channel */ 4365 rv = hv_ldc_set_map_table(ldcp->id, 4366 va_to_pa(mtbl->table), mtbl->num_entries); 4367 if (rv != 0) { 4368 cmn_err(CE_WARN, 4369 "ldc_mem_bind_handle: (0x%lx) err %d mapping tbl", 4370 ldcp->id, rv); 4371 if (mtbl->contigmem) 4372 contig_mem_free(mtbl->table, mtbl->size); 4373 else 4374 kmem_free(mtbl->table, mtbl->size); 4375 mutex_destroy(&mtbl->lock); 4376 kmem_free(mtbl, sizeof (ldc_mtbl_t)); 4377 mutex_exit(&ldcp->lock); 4378 mutex_exit(&mhdl->lock); 4379 return (EIO); 4380 } 4381 4382 ldcp->mtbl = mtbl; 4383 mutex_exit(&ldcp->lock); 4384 4385 D1(ldcp->id, 4386 "ldc_mem_bind_handle: (0x%llx) alloc'd map table 0x%llx\n", 4387 ldcp->id, ldcp->mtbl->table); 4388 } 4389 4390 /* FUTURE: get the page size, pgsz code, and shift */ 4391 pg_size = MMU_PAGESIZE; 4392 pg_size_code = page_szc(pg_size); 4393 pg_shift = page_get_shift(pg_size_code); 4394 pg_mask = ~(pg_size - 1); 4395 4396 D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) binding " 4397 "va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 4398 ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 4399 4400 /* aligned VA and its offset */ 4401 v_align = (caddr_t)(((uintptr_t)vaddr) & ~(pg_size - 1)); 4402 v_offset = ((uintptr_t)vaddr) & (pg_size - 1); 4403 4404 npages = (len+v_offset)/pg_size; 4405 npages = ((len+v_offset)%pg_size == 0) ? npages : npages+1; 4406 4407 D1(ldcp->id, "ldc_mem_bind_handle: binding " 4408 "(0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 4409 ldcp->id, vaddr, v_align, v_offset, npages); 4410 4411 /* lock the memory table - exclusive access to channel */ 4412 mutex_enter(&mtbl->lock); 4413 4414 if (npages > mtbl->num_avail) { 4415 D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) no table entries\n", 4416 ldcp->id); 4417 mutex_exit(&mtbl->lock); 4418 mutex_exit(&mhdl->lock); 4419 return (ENOMEM); 4420 } 4421 4422 /* Allocate a memseg structure */ 4423 memseg = mhdl->memseg = 4424 kmem_cache_alloc(ldcssp->memseg_cache, KM_SLEEP); 4425 4426 /* Allocate memory to store all pages and cookies */ 4427 memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP); 4428 memseg->cookies = 4429 kmem_zalloc((sizeof (ldc_mem_cookie_t) * npages), KM_SLEEP); 4430 4431 D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) processing 0x%llx pages\n", 4432 ldcp->id, npages); 4433 4434 addr = v_align; 4435 4436 /* 4437 * Check if direct shared memory map is enabled, if not change 4438 * the mapping type to include SHADOW_MAP. 4439 */ 4440 if (ldc_shmem_enabled == 0) 4441 mtype = LDC_SHADOW_MAP; 4442 4443 /* 4444 * Table slots are used in a round-robin manner. The algorithm permits 4445 * inserting duplicate entries. Slots allocated earlier will typically 4446 * get freed before we get back to reusing the slot.Inserting duplicate 4447 * entries should be OK as we only lookup entries using the cookie addr 4448 * i.e. tbl index, during export, unexport and copy operation. 4449 * 4450 * One implementation what was tried was to search for a duplicate 4451 * page entry first and reuse it. The search overhead is very high and 4452 * in the vnet case dropped the perf by almost half, 50 to 24 mbps. 4453 * So it does make sense to avoid searching for duplicates. 4454 * 4455 * But during the process of searching for a free slot, if we find a 4456 * duplicate entry we will go ahead and use it, and bump its use count. 4457 */ 4458 4459 /* index to start searching from */ 4460 index = mtbl->next_entry; 4461 cookie_idx = -1; 4462 4463 tmp_mte.ll = 0; /* initialise fields to 0 */ 4464 4465 if (mtype & LDC_DIRECT_MAP) { 4466 tmp_mte.mte_r = (perm & LDC_MEM_R) ? 1 : 0; 4467 tmp_mte.mte_w = (perm & LDC_MEM_W) ? 1 : 0; 4468 tmp_mte.mte_x = (perm & LDC_MEM_X) ? 1 : 0; 4469 } 4470 4471 if (mtype & LDC_SHADOW_MAP) { 4472 tmp_mte.mte_cr = (perm & LDC_MEM_R) ? 1 : 0; 4473 tmp_mte.mte_cw = (perm & LDC_MEM_W) ? 1 : 0; 4474 } 4475 4476 if (mtype & LDC_IO_MAP) { 4477 tmp_mte.mte_ir = (perm & LDC_MEM_R) ? 1 : 0; 4478 tmp_mte.mte_iw = (perm & LDC_MEM_W) ? 1 : 0; 4479 } 4480 4481 D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll); 4482 4483 tmp_mte.mte_pgszc = pg_size_code; 4484 4485 /* initialize each mem table entry */ 4486 for (i = 0; i < npages; i++) { 4487 4488 /* check if slot is available in the table */ 4489 while (mtbl->table[index].entry.ll != 0) { 4490 4491 index = (index + 1) % mtbl->num_entries; 4492 4493 if (index == mtbl->next_entry) { 4494 /* we have looped around */ 4495 DWARN(DBG_ALL_LDCS, 4496 "ldc_mem_bind_handle: (0x%llx) cannot find " 4497 "entry\n", ldcp->id); 4498 *ccount = 0; 4499 4500 /* NOTE: free memory, remove previous entries */ 4501 /* this shouldnt happen as num_avail was ok */ 4502 4503 mutex_exit(&mtbl->lock); 4504 mutex_exit(&mhdl->lock); 4505 return (ENOMEM); 4506 } 4507 } 4508 4509 /* get the real address */ 4510 raddr = va_to_pa((void *)addr); 4511 ra_aligned = ((uintptr_t)raddr & pg_mask); 4512 4513 /* build the mte */ 4514 tmp_mte.mte_rpfn = ra_aligned >> pg_shift; 4515 4516 D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll); 4517 4518 /* update entry in table */ 4519 mtbl->table[index].entry = tmp_mte; 4520 4521 D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) stored MTE 0x%llx" 4522 " into loc 0x%llx\n", ldcp->id, tmp_mte.ll, index); 4523 4524 /* calculate the size and offset for this export range */ 4525 if (i == 0) { 4526 /* first page */ 4527 psize = min((pg_size - v_offset), len); 4528 poffset = v_offset; 4529 4530 } else if (i == (npages - 1)) { 4531 /* last page */ 4532 psize = (((uintptr_t)(vaddr + len)) & 4533 ((uint64_t)(pg_size-1))); 4534 if (psize == 0) 4535 psize = pg_size; 4536 poffset = 0; 4537 4538 } else { 4539 /* middle pages */ 4540 psize = pg_size; 4541 poffset = 0; 4542 } 4543 4544 /* store entry for this page */ 4545 memseg->pages[i].index = index; 4546 memseg->pages[i].raddr = raddr; 4547 memseg->pages[i].offset = poffset; 4548 memseg->pages[i].size = psize; 4549 memseg->pages[i].mte = &(mtbl->table[index]); 4550 4551 /* create the cookie */ 4552 if (i == 0 || (index != prev_index + 1)) { 4553 cookie_idx++; 4554 memseg->cookies[cookie_idx].addr = 4555 IDX2COOKIE(index, pg_size_code, pg_shift); 4556 memseg->cookies[cookie_idx].addr |= poffset; 4557 memseg->cookies[cookie_idx].size = psize; 4558 4559 } else { 4560 memseg->cookies[cookie_idx].size += psize; 4561 } 4562 4563 D1(ldcp->id, "ldc_mem_bind_handle: bound " 4564 "(0x%llx) va=0x%llx, idx=0x%llx, " 4565 "ra=0x%llx(sz=0x%x,off=0x%x)\n", 4566 ldcp->id, addr, index, raddr, psize, poffset); 4567 4568 /* decrement number of available entries */ 4569 mtbl->num_avail--; 4570 4571 /* increment va by page size */ 4572 addr += pg_size; 4573 4574 /* increment index */ 4575 prev_index = index; 4576 index = (index + 1) % mtbl->num_entries; 4577 4578 /* save the next slot */ 4579 mtbl->next_entry = index; 4580 } 4581 4582 mutex_exit(&mtbl->lock); 4583 4584 /* memory handle = bound */ 4585 mhdl->mtype = mtype; 4586 mhdl->perm = perm; 4587 mhdl->status = LDC_BOUND; 4588 4589 /* update memseg_t */ 4590 memseg->vaddr = vaddr; 4591 memseg->raddr = memseg->pages[0].raddr; 4592 memseg->size = len; 4593 memseg->npages = npages; 4594 memseg->ncookies = cookie_idx + 1; 4595 memseg->next_cookie = (memseg->ncookies > 1) ? 1 : 0; 4596 4597 /* return count and first cookie */ 4598 *ccount = memseg->ncookies; 4599 cookie->addr = memseg->cookies[0].addr; 4600 cookie->size = memseg->cookies[0].size; 4601 4602 D1(ldcp->id, 4603 "ldc_mem_bind_handle: (0x%llx) bound 0x%llx, va=0x%llx, " 4604 "pgs=0x%llx cookies=0x%llx\n", 4605 ldcp->id, mhdl, vaddr, npages, memseg->ncookies); 4606 4607 mutex_exit(&mhdl->lock); 4608 return (0); 4609 } 4610 4611 /* 4612 * Return the next cookie associated with the specified memory handle 4613 */ 4614 int 4615 ldc_mem_nextcookie(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie) 4616 { 4617 ldc_mhdl_t *mhdl; 4618 ldc_chan_t *ldcp; 4619 ldc_memseg_t *memseg; 4620 4621 if (mhandle == NULL) { 4622 DWARN(DBG_ALL_LDCS, 4623 "ldc_mem_nextcookie: invalid memory handle\n"); 4624 return (EINVAL); 4625 } 4626 mhdl = (ldc_mhdl_t *)mhandle; 4627 4628 mutex_enter(&mhdl->lock); 4629 4630 ldcp = mhdl->ldcp; 4631 memseg = mhdl->memseg; 4632 4633 if (cookie == 0) { 4634 DWARN(ldcp->id, 4635 "ldc_mem_nextcookie:(0x%llx) invalid cookie arg\n", 4636 ldcp->id); 4637 mutex_exit(&mhdl->lock); 4638 return (EINVAL); 4639 } 4640 4641 if (memseg->next_cookie != 0) { 4642 cookie->addr = memseg->cookies[memseg->next_cookie].addr; 4643 cookie->size = memseg->cookies[memseg->next_cookie].size; 4644 memseg->next_cookie++; 4645 if (memseg->next_cookie == memseg->ncookies) 4646 memseg->next_cookie = 0; 4647 4648 } else { 4649 DWARN(ldcp->id, 4650 "ldc_mem_nextcookie:(0x%llx) no more cookies\n", ldcp->id); 4651 cookie->addr = 0; 4652 cookie->size = 0; 4653 mutex_exit(&mhdl->lock); 4654 return (EINVAL); 4655 } 4656 4657 D1(ldcp->id, 4658 "ldc_mem_nextcookie: (0x%llx) cookie addr=0x%llx,sz=0x%llx\n", 4659 ldcp->id, cookie->addr, cookie->size); 4660 4661 mutex_exit(&mhdl->lock); 4662 return (0); 4663 } 4664 4665 /* 4666 * Unbind the virtual memory region associated with the specified 4667 * memory handle. Allassociated cookies are freed and the corresponding 4668 * RA space is no longer exported. 4669 */ 4670 int 4671 ldc_mem_unbind_handle(ldc_mem_handle_t mhandle) 4672 { 4673 ldc_mhdl_t *mhdl; 4674 ldc_chan_t *ldcp; 4675 ldc_mtbl_t *mtbl; 4676 ldc_memseg_t *memseg; 4677 uint64_t cookie_addr; 4678 uint64_t pg_shift, pg_size_code; 4679 int i, rv; 4680 4681 if (mhandle == NULL) { 4682 DWARN(DBG_ALL_LDCS, 4683 "ldc_mem_unbind_handle: invalid memory handle\n"); 4684 return (EINVAL); 4685 } 4686 mhdl = (ldc_mhdl_t *)mhandle; 4687 4688 mutex_enter(&mhdl->lock); 4689 4690 if (mhdl->status == LDC_UNBOUND) { 4691 DWARN(DBG_ALL_LDCS, 4692 "ldc_mem_unbind_handle: (0x%x) handle is not bound\n", 4693 mhandle); 4694 mutex_exit(&mhdl->lock); 4695 return (EINVAL); 4696 } 4697 4698 ldcp = mhdl->ldcp; 4699 mtbl = ldcp->mtbl; 4700 4701 memseg = mhdl->memseg; 4702 4703 /* lock the memory table - exclusive access to channel */ 4704 mutex_enter(&mtbl->lock); 4705 4706 /* undo the pages exported */ 4707 for (i = 0; i < memseg->npages; i++) { 4708 4709 /* check for mapped pages, revocation cookie != 0 */ 4710 if (memseg->pages[i].mte->cookie) { 4711 4712 pg_size_code = page_szc(memseg->pages[i].size); 4713 pg_shift = page_get_shift(memseg->pages[i].size); 4714 cookie_addr = IDX2COOKIE(memseg->pages[i].index, 4715 pg_size_code, pg_shift); 4716 4717 D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) revoke " 4718 "cookie 0x%llx, rcookie 0x%llx\n", ldcp->id, 4719 cookie_addr, memseg->pages[i].mte->cookie); 4720 rv = hv_ldc_revoke(ldcp->id, cookie_addr, 4721 memseg->pages[i].mte->cookie); 4722 if (rv) { 4723 DWARN(ldcp->id, 4724 "ldc_mem_unbind_handle: (0x%llx) cannot " 4725 "revoke mapping, cookie %llx\n", ldcp->id, 4726 cookie_addr); 4727 } 4728 } 4729 4730 /* clear the entry from the table */ 4731 memseg->pages[i].mte->entry.ll = 0; 4732 mtbl->num_avail++; 4733 } 4734 mutex_exit(&mtbl->lock); 4735 4736 /* free the allocated memseg and page structures */ 4737 kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages)); 4738 kmem_free(memseg->cookies, 4739 (sizeof (ldc_mem_cookie_t) * memseg->npages)); 4740 kmem_cache_free(ldcssp->memseg_cache, memseg); 4741 4742 /* uninitialize the memory handle */ 4743 mhdl->memseg = NULL; 4744 mhdl->status = LDC_UNBOUND; 4745 4746 D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) unbound handle 0x%llx\n", 4747 ldcp->id, mhdl); 4748 4749 mutex_exit(&mhdl->lock); 4750 return (0); 4751 } 4752 4753 /* 4754 * Get information about the dring. The base address of the descriptor 4755 * ring along with the type and permission are returned back. 4756 */ 4757 int 4758 ldc_mem_info(ldc_mem_handle_t mhandle, ldc_mem_info_t *minfo) 4759 { 4760 ldc_mhdl_t *mhdl; 4761 4762 if (mhandle == NULL) { 4763 DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid memory handle\n"); 4764 return (EINVAL); 4765 } 4766 mhdl = (ldc_mhdl_t *)mhandle; 4767 4768 if (minfo == NULL) { 4769 DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid args\n"); 4770 return (EINVAL); 4771 } 4772 4773 mutex_enter(&mhdl->lock); 4774 4775 minfo->status = mhdl->status; 4776 if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) { 4777 minfo->vaddr = mhdl->memseg->vaddr; 4778 minfo->raddr = mhdl->memseg->raddr; 4779 minfo->mtype = mhdl->mtype; 4780 minfo->perm = mhdl->perm; 4781 } 4782 mutex_exit(&mhdl->lock); 4783 4784 return (0); 4785 } 4786 4787 /* 4788 * Copy data either from or to the client specified virtual address 4789 * space to or from the exported memory associated with the cookies. 4790 * The direction argument determines whether the data is read from or 4791 * written to exported memory. 4792 */ 4793 int 4794 ldc_mem_copy(ldc_handle_t handle, caddr_t vaddr, uint64_t off, size_t *size, 4795 ldc_mem_cookie_t *cookies, uint32_t ccount, uint8_t direction) 4796 { 4797 ldc_chan_t *ldcp; 4798 uint64_t local_voff, local_valign; 4799 uint64_t cookie_addr, cookie_size; 4800 uint64_t pg_shift, pg_size, pg_size_code; 4801 uint64_t export_caddr, export_poff, export_psize, export_size; 4802 uint64_t local_ra, local_poff, local_psize; 4803 uint64_t copy_size, copied_len = 0, total_bal = 0, idx = 0; 4804 pgcnt_t npages; 4805 size_t len = *size; 4806 int i, rv = 0; 4807 4808 uint64_t chid; 4809 4810 if (handle == NULL) { 4811 DWARN(DBG_ALL_LDCS, "ldc_mem_copy: invalid channel handle\n"); 4812 return (EINVAL); 4813 } 4814 ldcp = (ldc_chan_t *)handle; 4815 chid = ldcp->id; 4816 4817 /* check to see if channel is UP */ 4818 if (ldcp->tstate != TS_UP) { 4819 DWARN(chid, "ldc_mem_copy: (0x%llx) channel is not UP\n", 4820 chid); 4821 return (ECONNRESET); 4822 } 4823 4824 /* Force address and size to be 8-byte aligned */ 4825 if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 4826 DWARN(chid, 4827 "ldc_mem_copy: addr/sz is not 8-byte aligned\n"); 4828 return (EINVAL); 4829 } 4830 4831 /* Find the size of the exported memory */ 4832 export_size = 0; 4833 for (i = 0; i < ccount; i++) 4834 export_size += cookies[i].size; 4835 4836 /* check to see if offset is valid */ 4837 if (off > export_size) { 4838 DWARN(chid, 4839 "ldc_mem_copy: (0x%llx) start offset > export mem size\n", 4840 chid); 4841 return (EINVAL); 4842 } 4843 4844 /* 4845 * Check to see if the export size is smaller than the size we 4846 * are requesting to copy - if so flag an error 4847 */ 4848 if ((export_size - off) < *size) { 4849 DWARN(chid, 4850 "ldc_mem_copy: (0x%llx) copy size > export mem size\n", 4851 chid); 4852 return (EINVAL); 4853 } 4854 4855 total_bal = min(export_size, *size); 4856 4857 /* FUTURE: get the page size, pgsz code, and shift */ 4858 pg_size = MMU_PAGESIZE; 4859 pg_size_code = page_szc(pg_size); 4860 pg_shift = page_get_shift(pg_size_code); 4861 4862 D1(chid, "ldc_mem_copy: copying data " 4863 "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 4864 chid, vaddr, pg_size, pg_size_code, pg_shift); 4865 4866 /* aligned VA and its offset */ 4867 local_valign = (((uintptr_t)vaddr) & ~(pg_size - 1)); 4868 local_voff = ((uintptr_t)vaddr) & (pg_size - 1); 4869 4870 npages = (len+local_voff)/pg_size; 4871 npages = ((len+local_voff)%pg_size == 0) ? npages : npages+1; 4872 4873 D1(chid, 4874 "ldc_mem_copy: (0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 4875 chid, vaddr, local_valign, local_voff, npages); 4876 4877 local_ra = va_to_pa((void *)local_valign); 4878 local_poff = local_voff; 4879 local_psize = min(len, (pg_size - local_voff)); 4880 4881 len -= local_psize; 4882 4883 /* 4884 * find the first cookie in the list of cookies 4885 * if the offset passed in is not zero 4886 */ 4887 for (idx = 0; idx < ccount; idx++) { 4888 cookie_size = cookies[idx].size; 4889 if (off < cookie_size) 4890 break; 4891 off -= cookie_size; 4892 } 4893 4894 cookie_addr = cookies[idx].addr + off; 4895 cookie_size = cookies[idx].size - off; 4896 4897 export_caddr = cookie_addr & ~(pg_size - 1); 4898 export_poff = cookie_addr & (pg_size - 1); 4899 export_psize = min(cookie_size, (pg_size - export_poff)); 4900 4901 for (;;) { 4902 4903 copy_size = min(export_psize, local_psize); 4904 4905 D1(chid, 4906 "ldc_mem_copy:(0x%llx) dir=0x%x, caddr=0x%llx," 4907 " loc_ra=0x%llx, exp_poff=0x%llx, loc_poff=0x%llx," 4908 " exp_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 4909 " total_bal=0x%llx\n", 4910 chid, direction, export_caddr, local_ra, export_poff, 4911 local_poff, export_psize, local_psize, copy_size, 4912 total_bal); 4913 4914 rv = hv_ldc_copy(chid, direction, 4915 (export_caddr + export_poff), (local_ra + local_poff), 4916 copy_size, &copied_len); 4917 4918 if (rv != 0) { 4919 int error = EIO; 4920 uint64_t rx_hd, rx_tl; 4921 4922 DWARN(chid, 4923 "ldc_mem_copy: (0x%llx) err %d during copy\n", 4924 (unsigned long long)chid, rv); 4925 DWARN(chid, 4926 "ldc_mem_copy: (0x%llx) dir=0x%x, caddr=0x%lx, " 4927 "loc_ra=0x%lx, exp_poff=0x%lx, loc_poff=0x%lx," 4928 " exp_psz=0x%lx, loc_psz=0x%lx, copy_sz=0x%lx," 4929 " copied_len=0x%lx, total_bal=0x%lx\n", 4930 chid, direction, export_caddr, local_ra, 4931 export_poff, local_poff, export_psize, local_psize, 4932 copy_size, copied_len, total_bal); 4933 4934 *size = *size - total_bal; 4935 4936 /* 4937 * check if reason for copy error was due to 4938 * a channel reset. we need to grab the lock 4939 * just in case we have to do a reset. 4940 */ 4941 mutex_enter(&ldcp->lock); 4942 mutex_enter(&ldcp->tx_lock); 4943 4944 rv = hv_ldc_rx_get_state(ldcp->id, 4945 &rx_hd, &rx_tl, &(ldcp->link_state)); 4946 if (ldcp->link_state == LDC_CHANNEL_DOWN || 4947 ldcp->link_state == LDC_CHANNEL_RESET) { 4948 i_ldc_reset(ldcp, B_FALSE); 4949 error = ECONNRESET; 4950 } 4951 4952 mutex_exit(&ldcp->tx_lock); 4953 mutex_exit(&ldcp->lock); 4954 4955 return (error); 4956 } 4957 4958 ASSERT(copied_len <= copy_size); 4959 4960 D2(chid, "ldc_mem_copy: copied=0x%llx\n", copied_len); 4961 export_poff += copied_len; 4962 local_poff += copied_len; 4963 export_psize -= copied_len; 4964 local_psize -= copied_len; 4965 cookie_size -= copied_len; 4966 4967 total_bal -= copied_len; 4968 4969 if (copy_size != copied_len) 4970 continue; 4971 4972 if (export_psize == 0 && total_bal != 0) { 4973 4974 if (cookie_size == 0) { 4975 idx++; 4976 cookie_addr = cookies[idx].addr; 4977 cookie_size = cookies[idx].size; 4978 4979 export_caddr = cookie_addr & ~(pg_size - 1); 4980 export_poff = cookie_addr & (pg_size - 1); 4981 export_psize = 4982 min(cookie_size, (pg_size-export_poff)); 4983 } else { 4984 export_caddr += pg_size; 4985 export_poff = 0; 4986 export_psize = min(cookie_size, pg_size); 4987 } 4988 } 4989 4990 if (local_psize == 0 && total_bal != 0) { 4991 local_valign += pg_size; 4992 local_ra = va_to_pa((void *)local_valign); 4993 local_poff = 0; 4994 local_psize = min(pg_size, len); 4995 len -= local_psize; 4996 } 4997 4998 /* check if we are all done */ 4999 if (total_bal == 0) 5000 break; 5001 } 5002 5003 5004 D1(chid, 5005 "ldc_mem_copy: (0x%llx) done copying sz=0x%llx\n", 5006 chid, *size); 5007 5008 return (0); 5009 } 5010 5011 /* 5012 * Copy data either from or to the client specified virtual address 5013 * space to or from HV physical memory. 5014 * 5015 * The direction argument determines whether the data is read from or 5016 * written to HV memory. direction values are LDC_COPY_IN/OUT similar 5017 * to the ldc_mem_copy interface 5018 */ 5019 int 5020 ldc_mem_rdwr_cookie(ldc_handle_t handle, caddr_t vaddr, size_t *size, 5021 caddr_t paddr, uint8_t direction) 5022 { 5023 ldc_chan_t *ldcp; 5024 uint64_t local_voff, local_valign; 5025 uint64_t pg_shift, pg_size, pg_size_code; 5026 uint64_t target_pa, target_poff, target_psize, target_size; 5027 uint64_t local_ra, local_poff, local_psize; 5028 uint64_t copy_size, copied_len = 0; 5029 pgcnt_t npages; 5030 size_t len = *size; 5031 int rv = 0; 5032 5033 if (handle == NULL) { 5034 DWARN(DBG_ALL_LDCS, 5035 "ldc_mem_rdwr_cookie: invalid channel handle\n"); 5036 return (EINVAL); 5037 } 5038 ldcp = (ldc_chan_t *)handle; 5039 5040 mutex_enter(&ldcp->lock); 5041 5042 /* check to see if channel is UP */ 5043 if (ldcp->tstate != TS_UP) { 5044 DWARN(ldcp->id, 5045 "ldc_mem_rdwr_cookie: (0x%llx) channel is not UP\n", 5046 ldcp->id); 5047 mutex_exit(&ldcp->lock); 5048 return (ECONNRESET); 5049 } 5050 5051 /* Force address and size to be 8-byte aligned */ 5052 if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 5053 DWARN(ldcp->id, 5054 "ldc_mem_rdwr_cookie: addr/size is not 8-byte aligned\n"); 5055 mutex_exit(&ldcp->lock); 5056 return (EINVAL); 5057 } 5058 5059 target_size = *size; 5060 5061 /* FUTURE: get the page size, pgsz code, and shift */ 5062 pg_size = MMU_PAGESIZE; 5063 pg_size_code = page_szc(pg_size); 5064 pg_shift = page_get_shift(pg_size_code); 5065 5066 D1(ldcp->id, "ldc_mem_rdwr_cookie: copying data " 5067 "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 5068 ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 5069 5070 /* aligned VA and its offset */ 5071 local_valign = ((uintptr_t)vaddr) & ~(pg_size - 1); 5072 local_voff = ((uintptr_t)vaddr) & (pg_size - 1); 5073 5074 npages = (len + local_voff) / pg_size; 5075 npages = ((len + local_voff) % pg_size == 0) ? npages : npages+1; 5076 5077 D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) v=0x%llx, " 5078 "val=0x%llx,off=0x%x,pgs=0x%x\n", 5079 ldcp->id, vaddr, local_valign, local_voff, npages); 5080 5081 local_ra = va_to_pa((void *)local_valign); 5082 local_poff = local_voff; 5083 local_psize = min(len, (pg_size - local_voff)); 5084 5085 len -= local_psize; 5086 5087 target_pa = ((uintptr_t)paddr) & ~(pg_size - 1); 5088 target_poff = ((uintptr_t)paddr) & (pg_size - 1); 5089 target_psize = pg_size - target_poff; 5090 5091 for (;;) { 5092 5093 copy_size = min(target_psize, local_psize); 5094 5095 D1(ldcp->id, 5096 "ldc_mem_rdwr_cookie: (0x%llx) dir=0x%x, tar_pa=0x%llx," 5097 " loc_ra=0x%llx, tar_poff=0x%llx, loc_poff=0x%llx," 5098 " tar_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 5099 " total_bal=0x%llx\n", 5100 ldcp->id, direction, target_pa, local_ra, target_poff, 5101 local_poff, target_psize, local_psize, copy_size, 5102 target_size); 5103 5104 rv = hv_ldc_copy(ldcp->id, direction, 5105 (target_pa + target_poff), (local_ra + local_poff), 5106 copy_size, &copied_len); 5107 5108 if (rv != 0) { 5109 DWARN(DBG_ALL_LDCS, 5110 "ldc_mem_rdwr_cookie: (0x%lx) err %d during copy\n", 5111 ldcp->id, rv); 5112 DWARN(DBG_ALL_LDCS, 5113 "ldc_mem_rdwr_cookie: (0x%llx) dir=%lld, " 5114 "tar_pa=0x%llx, loc_ra=0x%llx, tar_poff=0x%llx, " 5115 "loc_poff=0x%llx, tar_psz=0x%llx, loc_psz=0x%llx, " 5116 "copy_sz=0x%llx, total_bal=0x%llx\n", 5117 ldcp->id, direction, target_pa, local_ra, 5118 target_poff, local_poff, target_psize, local_psize, 5119 copy_size, target_size); 5120 5121 *size = *size - target_size; 5122 mutex_exit(&ldcp->lock); 5123 return (i_ldc_h2v_error(rv)); 5124 } 5125 5126 D2(ldcp->id, "ldc_mem_rdwr_cookie: copied=0x%llx\n", 5127 copied_len); 5128 target_poff += copied_len; 5129 local_poff += copied_len; 5130 target_psize -= copied_len; 5131 local_psize -= copied_len; 5132 5133 target_size -= copied_len; 5134 5135 if (copy_size != copied_len) 5136 continue; 5137 5138 if (target_psize == 0 && target_size != 0) { 5139 target_pa += pg_size; 5140 target_poff = 0; 5141 target_psize = min(pg_size, target_size); 5142 } 5143 5144 if (local_psize == 0 && target_size != 0) { 5145 local_valign += pg_size; 5146 local_ra = va_to_pa((void *)local_valign); 5147 local_poff = 0; 5148 local_psize = min(pg_size, len); 5149 len -= local_psize; 5150 } 5151 5152 /* check if we are all done */ 5153 if (target_size == 0) 5154 break; 5155 } 5156 5157 mutex_exit(&ldcp->lock); 5158 5159 D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) done copying sz=0x%llx\n", 5160 ldcp->id, *size); 5161 5162 return (0); 5163 } 5164 5165 /* 5166 * Map an exported memory segment into the local address space. If the 5167 * memory range was exported for direct map access, a HV call is made 5168 * to allocate a RA range. If the map is done via a shadow copy, local 5169 * shadow memory is allocated and the base VA is returned in 'vaddr'. If 5170 * the mapping is a direct map then the RA is returned in 'raddr'. 5171 */ 5172 int 5173 ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie, uint32_t ccount, 5174 uint8_t mtype, uint8_t perm, caddr_t *vaddr, caddr_t *raddr) 5175 { 5176 int i, j, idx, rv, retries; 5177 ldc_chan_t *ldcp; 5178 ldc_mhdl_t *mhdl; 5179 ldc_memseg_t *memseg; 5180 caddr_t tmpaddr; 5181 uint64_t map_perm = perm; 5182 uint64_t pg_size, pg_shift, pg_size_code, pg_mask; 5183 uint64_t exp_size = 0, base_off, map_size, npages; 5184 uint64_t cookie_addr, cookie_off, cookie_size; 5185 tte_t ldc_tte; 5186 5187 if (mhandle == NULL) { 5188 DWARN(DBG_ALL_LDCS, "ldc_mem_map: invalid memory handle\n"); 5189 return (EINVAL); 5190 } 5191 mhdl = (ldc_mhdl_t *)mhandle; 5192 5193 mutex_enter(&mhdl->lock); 5194 5195 if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED || 5196 mhdl->memseg != NULL) { 5197 DWARN(DBG_ALL_LDCS, 5198 "ldc_mem_map: (0x%llx) handle bound/mapped\n", mhandle); 5199 mutex_exit(&mhdl->lock); 5200 return (EINVAL); 5201 } 5202 5203 ldcp = mhdl->ldcp; 5204 5205 mutex_enter(&ldcp->lock); 5206 5207 if (ldcp->tstate != TS_UP) { 5208 DWARN(ldcp->id, 5209 "ldc_mem_dring_map: (0x%llx) channel is not UP\n", 5210 ldcp->id); 5211 mutex_exit(&ldcp->lock); 5212 mutex_exit(&mhdl->lock); 5213 return (ECONNRESET); 5214 } 5215 5216 if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) { 5217 DWARN(ldcp->id, "ldc_mem_map: invalid map type\n"); 5218 mutex_exit(&ldcp->lock); 5219 mutex_exit(&mhdl->lock); 5220 return (EINVAL); 5221 } 5222 5223 D1(ldcp->id, "ldc_mem_map: (0x%llx) cookie = 0x%llx,0x%llx\n", 5224 ldcp->id, cookie->addr, cookie->size); 5225 5226 /* FUTURE: get the page size, pgsz code, and shift */ 5227 pg_size = MMU_PAGESIZE; 5228 pg_size_code = page_szc(pg_size); 5229 pg_shift = page_get_shift(pg_size_code); 5230 pg_mask = ~(pg_size - 1); 5231 5232 /* calculate the number of pages in the exported cookie */ 5233 base_off = cookie[0].addr & (pg_size - 1); 5234 for (idx = 0; idx < ccount; idx++) 5235 exp_size += cookie[idx].size; 5236 map_size = P2ROUNDUP((exp_size + base_off), pg_size); 5237 npages = (map_size >> pg_shift); 5238 5239 /* Allocate memseg structure */ 5240 memseg = mhdl->memseg = 5241 kmem_cache_alloc(ldcssp->memseg_cache, KM_SLEEP); 5242 5243 /* Allocate memory to store all pages and cookies */ 5244 memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP); 5245 memseg->cookies = 5246 kmem_zalloc((sizeof (ldc_mem_cookie_t) * ccount), KM_SLEEP); 5247 5248 D2(ldcp->id, "ldc_mem_map: (0x%llx) exp_size=0x%llx, map_size=0x%llx," 5249 "pages=0x%llx\n", ldcp->id, exp_size, map_size, npages); 5250 5251 /* 5252 * Check if direct map over shared memory is enabled, if not change 5253 * the mapping type to SHADOW_MAP. 5254 */ 5255 if (ldc_shmem_enabled == 0) 5256 mtype = LDC_SHADOW_MAP; 5257 5258 /* 5259 * Check to see if the client is requesting direct or shadow map 5260 * If direct map is requested, try to map remote memory first, 5261 * and if that fails, revert to shadow map 5262 */ 5263 if (mtype == LDC_DIRECT_MAP) { 5264 5265 /* Allocate kernel virtual space for mapping */ 5266 memseg->vaddr = vmem_xalloc(heap_arena, map_size, 5267 pg_size, 0, 0, NULL, NULL, VM_NOSLEEP); 5268 if (memseg->vaddr == NULL) { 5269 cmn_err(CE_WARN, 5270 "ldc_mem_map: (0x%lx) memory map failed\n", 5271 ldcp->id); 5272 kmem_free(memseg->cookies, 5273 (sizeof (ldc_mem_cookie_t) * ccount)); 5274 kmem_free(memseg->pages, 5275 (sizeof (ldc_page_t) * npages)); 5276 kmem_cache_free(ldcssp->memseg_cache, memseg); 5277 5278 mutex_exit(&ldcp->lock); 5279 mutex_exit(&mhdl->lock); 5280 return (ENOMEM); 5281 } 5282 5283 /* Unload previous mapping */ 5284 hat_unload(kas.a_hat, memseg->vaddr, map_size, 5285 HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK); 5286 5287 /* for each cookie passed in - map into address space */ 5288 idx = 0; 5289 cookie_size = 0; 5290 tmpaddr = memseg->vaddr; 5291 5292 for (i = 0; i < npages; i++) { 5293 5294 if (cookie_size == 0) { 5295 ASSERT(idx < ccount); 5296 cookie_addr = cookie[idx].addr & pg_mask; 5297 cookie_off = cookie[idx].addr & (pg_size - 1); 5298 cookie_size = 5299 P2ROUNDUP((cookie_off + cookie[idx].size), 5300 pg_size); 5301 idx++; 5302 } 5303 5304 D1(ldcp->id, "ldc_mem_map: (0x%llx) mapping " 5305 "cookie 0x%llx, bal=0x%llx\n", ldcp->id, 5306 cookie_addr, cookie_size); 5307 5308 /* map the cookie into address space */ 5309 for (retries = 0; retries < ldc_max_retries; 5310 retries++) { 5311 5312 rv = hv_ldc_mapin(ldcp->id, cookie_addr, 5313 &memseg->pages[i].raddr, &map_perm); 5314 if (rv != H_EWOULDBLOCK && rv != H_ETOOMANY) 5315 break; 5316 5317 drv_usecwait(ldc_delay); 5318 } 5319 5320 if (rv || memseg->pages[i].raddr == 0) { 5321 DWARN(ldcp->id, 5322 "ldc_mem_map: (0x%llx) hv mapin err %d\n", 5323 ldcp->id, rv); 5324 5325 /* remove previous mapins */ 5326 hat_unload(kas.a_hat, memseg->vaddr, map_size, 5327 HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK); 5328 for (j = 0; j < i; j++) { 5329 rv = hv_ldc_unmap( 5330 memseg->pages[j].raddr); 5331 if (rv) { 5332 DWARN(ldcp->id, 5333 "ldc_mem_map: (0x%llx) " 5334 "cannot unmap ra=0x%llx\n", 5335 ldcp->id, 5336 memseg->pages[j].raddr); 5337 } 5338 } 5339 5340 /* free kernel virtual space */ 5341 vmem_free(heap_arena, (void *)memseg->vaddr, 5342 map_size); 5343 5344 /* direct map failed - revert to shadow map */ 5345 mtype = LDC_SHADOW_MAP; 5346 break; 5347 5348 } else { 5349 5350 D1(ldcp->id, 5351 "ldc_mem_map: (0x%llx) vtop map 0x%llx -> " 5352 "0x%llx, cookie=0x%llx, perm=0x%llx\n", 5353 ldcp->id, tmpaddr, memseg->pages[i].raddr, 5354 cookie_addr, perm); 5355 5356 /* 5357 * NOTE: Calling hat_devload directly, causes it 5358 * to look for page_t using the pfn. Since this 5359 * addr is greater than the memlist, it treates 5360 * it as non-memory 5361 */ 5362 sfmmu_memtte(&ldc_tte, 5363 (pfn_t)(memseg->pages[i].raddr >> pg_shift), 5364 PROT_READ | PROT_WRITE | HAT_NOSYNC, TTE8K); 5365 5366 D1(ldcp->id, 5367 "ldc_mem_map: (0x%llx) ra 0x%llx -> " 5368 "tte 0x%llx\n", ldcp->id, 5369 memseg->pages[i].raddr, ldc_tte); 5370 5371 sfmmu_tteload(kas.a_hat, &ldc_tte, tmpaddr, 5372 NULL, HAT_LOAD_LOCK); 5373 5374 cookie_size -= pg_size; 5375 cookie_addr += pg_size; 5376 tmpaddr += pg_size; 5377 } 5378 } 5379 } 5380 5381 if (mtype == LDC_SHADOW_MAP) { 5382 if (*vaddr == NULL) { 5383 memseg->vaddr = kmem_zalloc(exp_size, KM_SLEEP); 5384 mhdl->myshadow = B_TRUE; 5385 5386 D1(ldcp->id, "ldc_mem_map: (0x%llx) allocated " 5387 "shadow page va=0x%llx\n", ldcp->id, memseg->vaddr); 5388 } else { 5389 /* 5390 * Use client supplied memory for memseg->vaddr 5391 * WARNING: assuming that client mem is >= exp_size 5392 */ 5393 memseg->vaddr = *vaddr; 5394 } 5395 5396 /* Save all page and cookie information */ 5397 for (i = 0, tmpaddr = memseg->vaddr; i < npages; i++) { 5398 memseg->pages[i].raddr = va_to_pa(tmpaddr); 5399 memseg->pages[i].size = pg_size; 5400 tmpaddr += pg_size; 5401 } 5402 5403 } 5404 5405 /* save all cookies */ 5406 bcopy(cookie, memseg->cookies, ccount * sizeof (ldc_mem_cookie_t)); 5407 5408 /* update memseg_t */ 5409 memseg->raddr = memseg->pages[0].raddr; 5410 memseg->size = (mtype == LDC_SHADOW_MAP) ? exp_size : map_size; 5411 memseg->npages = npages; 5412 memseg->ncookies = ccount; 5413 memseg->next_cookie = 0; 5414 5415 /* memory handle = mapped */ 5416 mhdl->mtype = mtype; 5417 mhdl->perm = perm; 5418 mhdl->status = LDC_MAPPED; 5419 5420 D1(ldcp->id, "ldc_mem_map: (0x%llx) mapped 0x%llx, ra=0x%llx, " 5421 "va=0x%llx, pgs=0x%llx cookies=0x%llx\n", 5422 ldcp->id, mhdl, memseg->raddr, memseg->vaddr, 5423 memseg->npages, memseg->ncookies); 5424 5425 if (mtype == LDC_SHADOW_MAP) 5426 base_off = 0; 5427 if (raddr) 5428 *raddr = (caddr_t)(memseg->raddr | base_off); 5429 if (vaddr) 5430 *vaddr = (caddr_t)((uintptr_t)memseg->vaddr | base_off); 5431 5432 mutex_exit(&ldcp->lock); 5433 mutex_exit(&mhdl->lock); 5434 return (0); 5435 } 5436 5437 /* 5438 * Unmap a memory segment. Free shadow memory (if any). 5439 */ 5440 int 5441 ldc_mem_unmap(ldc_mem_handle_t mhandle) 5442 { 5443 int i, rv; 5444 ldc_mhdl_t *mhdl = (ldc_mhdl_t *)mhandle; 5445 ldc_chan_t *ldcp; 5446 ldc_memseg_t *memseg; 5447 5448 if (mhdl == 0 || mhdl->status != LDC_MAPPED) { 5449 DWARN(DBG_ALL_LDCS, 5450 "ldc_mem_unmap: (0x%llx) handle is not mapped\n", 5451 mhandle); 5452 return (EINVAL); 5453 } 5454 5455 mutex_enter(&mhdl->lock); 5456 5457 ldcp = mhdl->ldcp; 5458 memseg = mhdl->memseg; 5459 5460 D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapping handle 0x%llx\n", 5461 ldcp->id, mhdl); 5462 5463 /* if we allocated shadow memory - free it */ 5464 if (mhdl->mtype == LDC_SHADOW_MAP && mhdl->myshadow) { 5465 kmem_free(memseg->vaddr, memseg->size); 5466 } else if (mhdl->mtype == LDC_DIRECT_MAP) { 5467 5468 /* unmap in the case of DIRECT_MAP */ 5469 hat_unload(kas.a_hat, memseg->vaddr, memseg->size, 5470 HAT_UNLOAD_UNLOCK); 5471 5472 for (i = 0; i < memseg->npages; i++) { 5473 rv = hv_ldc_unmap(memseg->pages[i].raddr); 5474 if (rv) { 5475 cmn_err(CE_WARN, 5476 "ldc_mem_map: (0x%lx) hv unmap err %d\n", 5477 ldcp->id, rv); 5478 } 5479 } 5480 5481 vmem_free(heap_arena, (void *)memseg->vaddr, memseg->size); 5482 } 5483 5484 /* free the allocated memseg and page structures */ 5485 kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages)); 5486 kmem_free(memseg->cookies, 5487 (sizeof (ldc_mem_cookie_t) * memseg->ncookies)); 5488 kmem_cache_free(ldcssp->memseg_cache, memseg); 5489 5490 /* uninitialize the memory handle */ 5491 mhdl->memseg = NULL; 5492 mhdl->status = LDC_UNBOUND; 5493 5494 D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapped handle 0x%llx\n", 5495 ldcp->id, mhdl); 5496 5497 mutex_exit(&mhdl->lock); 5498 return (0); 5499 } 5500 5501 /* 5502 * Internal entry point for LDC mapped memory entry consistency 5503 * semantics. Acquire copies the contents of the remote memory 5504 * into the local shadow copy. The release operation copies the local 5505 * contents into the remote memory. The offset and size specify the 5506 * bounds for the memory range being synchronized. 5507 */ 5508 static int 5509 i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, uint8_t direction, 5510 uint64_t offset, size_t size) 5511 { 5512 int err; 5513 ldc_mhdl_t *mhdl; 5514 ldc_chan_t *ldcp; 5515 ldc_memseg_t *memseg; 5516 caddr_t local_vaddr; 5517 size_t copy_size; 5518 5519 if (mhandle == NULL) { 5520 DWARN(DBG_ALL_LDCS, 5521 "i_ldc_mem_acquire_release: invalid memory handle\n"); 5522 return (EINVAL); 5523 } 5524 mhdl = (ldc_mhdl_t *)mhandle; 5525 5526 mutex_enter(&mhdl->lock); 5527 5528 if (mhdl->status != LDC_MAPPED || mhdl->ldcp == NULL) { 5529 DWARN(DBG_ALL_LDCS, 5530 "i_ldc_mem_acquire_release: not mapped memory\n"); 5531 mutex_exit(&mhdl->lock); 5532 return (EINVAL); 5533 } 5534 5535 /* do nothing for direct map */ 5536 if (mhdl->mtype == LDC_DIRECT_MAP) { 5537 mutex_exit(&mhdl->lock); 5538 return (0); 5539 } 5540 5541 /* do nothing if COPY_IN+MEM_W and COPY_OUT+MEM_R */ 5542 if ((direction == LDC_COPY_IN && (mhdl->perm & LDC_MEM_R) == 0) || 5543 (direction == LDC_COPY_OUT && (mhdl->perm & LDC_MEM_W) == 0)) { 5544 mutex_exit(&mhdl->lock); 5545 return (0); 5546 } 5547 5548 if (offset >= mhdl->memseg->size || 5549 (offset + size) > mhdl->memseg->size) { 5550 DWARN(DBG_ALL_LDCS, 5551 "i_ldc_mem_acquire_release: memory out of range\n"); 5552 mutex_exit(&mhdl->lock); 5553 return (EINVAL); 5554 } 5555 5556 /* get the channel handle and memory segment */ 5557 ldcp = mhdl->ldcp; 5558 memseg = mhdl->memseg; 5559 5560 if (mhdl->mtype == LDC_SHADOW_MAP) { 5561 5562 local_vaddr = memseg->vaddr + offset; 5563 copy_size = size; 5564 5565 /* copy to/from remote from/to local memory */ 5566 err = ldc_mem_copy((ldc_handle_t)ldcp, local_vaddr, offset, 5567 ©_size, memseg->cookies, memseg->ncookies, 5568 direction); 5569 if (err || copy_size != size) { 5570 cmn_err(CE_WARN, 5571 "i_ldc_mem_acquire_release: copy failed\n"); 5572 mutex_exit(&mhdl->lock); 5573 return (err); 5574 } 5575 } 5576 5577 mutex_exit(&mhdl->lock); 5578 5579 return (0); 5580 } 5581 5582 /* 5583 * Ensure that the contents in the remote memory seg are consistent 5584 * with the contents if of local segment 5585 */ 5586 int 5587 ldc_mem_acquire(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size) 5588 { 5589 return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_IN, offset, size)); 5590 } 5591 5592 5593 /* 5594 * Ensure that the contents in the local memory seg are consistent 5595 * with the contents if of remote segment 5596 */ 5597 int 5598 ldc_mem_release(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size) 5599 { 5600 return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_OUT, offset, size)); 5601 } 5602 5603 /* 5604 * Allocate a descriptor ring. The size of each each descriptor 5605 * must be 8-byte aligned and the entire ring should be a multiple 5606 * of MMU_PAGESIZE. 5607 */ 5608 int 5609 ldc_mem_dring_create(uint32_t len, uint32_t dsize, ldc_dring_handle_t *dhandle) 5610 { 5611 ldc_dring_t *dringp; 5612 size_t size = (dsize * len); 5613 5614 D1(DBG_ALL_LDCS, "ldc_mem_dring_create: len=0x%x, size=0x%x\n", 5615 len, dsize); 5616 5617 if (dhandle == NULL) { 5618 DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid dhandle\n"); 5619 return (EINVAL); 5620 } 5621 5622 if (len == 0) { 5623 DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid length\n"); 5624 return (EINVAL); 5625 } 5626 5627 /* descriptor size should be 8-byte aligned */ 5628 if (dsize == 0 || (dsize & 0x7)) { 5629 DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid size\n"); 5630 return (EINVAL); 5631 } 5632 5633 *dhandle = 0; 5634 5635 /* Allocate a desc ring structure */ 5636 dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP); 5637 5638 /* Initialize dring */ 5639 dringp->length = len; 5640 dringp->dsize = dsize; 5641 5642 /* round off to multiple of pagesize */ 5643 dringp->size = (size & MMU_PAGEMASK); 5644 if (size & MMU_PAGEOFFSET) 5645 dringp->size += MMU_PAGESIZE; 5646 5647 dringp->status = LDC_UNBOUND; 5648 5649 /* allocate descriptor ring memory */ 5650 dringp->base = kmem_zalloc(dringp->size, KM_SLEEP); 5651 5652 /* initialize the desc ring lock */ 5653 mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL); 5654 5655 /* Add descriptor ring to the head of global list */ 5656 mutex_enter(&ldcssp->lock); 5657 dringp->next = ldcssp->dring_list; 5658 ldcssp->dring_list = dringp; 5659 mutex_exit(&ldcssp->lock); 5660 5661 *dhandle = (ldc_dring_handle_t)dringp; 5662 5663 D1(DBG_ALL_LDCS, "ldc_mem_dring_create: dring allocated\n"); 5664 5665 return (0); 5666 } 5667 5668 5669 /* 5670 * Destroy a descriptor ring. 5671 */ 5672 int 5673 ldc_mem_dring_destroy(ldc_dring_handle_t dhandle) 5674 { 5675 ldc_dring_t *dringp; 5676 ldc_dring_t *tmp_dringp; 5677 5678 D1(DBG_ALL_LDCS, "ldc_mem_dring_destroy: entered\n"); 5679 5680 if (dhandle == NULL) { 5681 DWARN(DBG_ALL_LDCS, 5682 "ldc_mem_dring_destroy: invalid desc ring handle\n"); 5683 return (EINVAL); 5684 } 5685 dringp = (ldc_dring_t *)dhandle; 5686 5687 if (dringp->status == LDC_BOUND) { 5688 DWARN(DBG_ALL_LDCS, 5689 "ldc_mem_dring_destroy: desc ring is bound\n"); 5690 return (EACCES); 5691 } 5692 5693 mutex_enter(&dringp->lock); 5694 mutex_enter(&ldcssp->lock); 5695 5696 /* remove from linked list - if not bound */ 5697 tmp_dringp = ldcssp->dring_list; 5698 if (tmp_dringp == dringp) { 5699 ldcssp->dring_list = dringp->next; 5700 dringp->next = NULL; 5701 5702 } else { 5703 while (tmp_dringp != NULL) { 5704 if (tmp_dringp->next == dringp) { 5705 tmp_dringp->next = dringp->next; 5706 dringp->next = NULL; 5707 break; 5708 } 5709 tmp_dringp = tmp_dringp->next; 5710 } 5711 if (tmp_dringp == NULL) { 5712 DWARN(DBG_ALL_LDCS, 5713 "ldc_mem_dring_destroy: invalid descriptor\n"); 5714 mutex_exit(&ldcssp->lock); 5715 mutex_exit(&dringp->lock); 5716 return (EINVAL); 5717 } 5718 } 5719 5720 mutex_exit(&ldcssp->lock); 5721 5722 /* free the descriptor ring */ 5723 kmem_free(dringp->base, dringp->size); 5724 5725 mutex_exit(&dringp->lock); 5726 5727 /* destroy dring lock */ 5728 mutex_destroy(&dringp->lock); 5729 5730 /* free desc ring object */ 5731 kmem_free(dringp, sizeof (ldc_dring_t)); 5732 5733 return (0); 5734 } 5735 5736 /* 5737 * Bind a previously allocated dring to a channel. The channel should 5738 * be OPEN in order to bind the ring to the channel. Returns back a 5739 * descriptor ring cookie. The descriptor ring is exported for remote 5740 * access by the client at the other end of the channel. An entry for 5741 * dring pages is stored in map table (via call to ldc_mem_bind_handle). 5742 */ 5743 int 5744 ldc_mem_dring_bind(ldc_handle_t handle, ldc_dring_handle_t dhandle, 5745 uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount) 5746 { 5747 int err; 5748 ldc_chan_t *ldcp; 5749 ldc_dring_t *dringp; 5750 ldc_mem_handle_t mhandle; 5751 5752 /* check to see if channel is initalized */ 5753 if (handle == NULL) { 5754 DWARN(DBG_ALL_LDCS, 5755 "ldc_mem_dring_bind: invalid channel handle\n"); 5756 return (EINVAL); 5757 } 5758 ldcp = (ldc_chan_t *)handle; 5759 5760 if (dhandle == NULL) { 5761 DWARN(DBG_ALL_LDCS, 5762 "ldc_mem_dring_bind: invalid desc ring handle\n"); 5763 return (EINVAL); 5764 } 5765 dringp = (ldc_dring_t *)dhandle; 5766 5767 if (cookie == NULL) { 5768 DWARN(ldcp->id, 5769 "ldc_mem_dring_bind: invalid cookie arg\n"); 5770 return (EINVAL); 5771 } 5772 5773 mutex_enter(&dringp->lock); 5774 5775 if (dringp->status == LDC_BOUND) { 5776 DWARN(DBG_ALL_LDCS, 5777 "ldc_mem_dring_bind: (0x%llx) descriptor ring is bound\n", 5778 ldcp->id); 5779 mutex_exit(&dringp->lock); 5780 return (EINVAL); 5781 } 5782 5783 if ((perm & LDC_MEM_RW) == 0) { 5784 DWARN(DBG_ALL_LDCS, 5785 "ldc_mem_dring_bind: invalid permissions\n"); 5786 mutex_exit(&dringp->lock); 5787 return (EINVAL); 5788 } 5789 5790 if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) { 5791 DWARN(DBG_ALL_LDCS, "ldc_mem_dring_bind: invalid type\n"); 5792 mutex_exit(&dringp->lock); 5793 return (EINVAL); 5794 } 5795 5796 dringp->ldcp = ldcp; 5797 5798 /* create an memory handle */ 5799 err = ldc_mem_alloc_handle(handle, &mhandle); 5800 if (err || mhandle == NULL) { 5801 DWARN(DBG_ALL_LDCS, 5802 "ldc_mem_dring_bind: (0x%llx) error allocating mhandle\n", 5803 ldcp->id); 5804 mutex_exit(&dringp->lock); 5805 return (err); 5806 } 5807 dringp->mhdl = mhandle; 5808 5809 /* bind the descriptor ring to channel */ 5810 err = ldc_mem_bind_handle(mhandle, dringp->base, dringp->size, 5811 mtype, perm, cookie, ccount); 5812 if (err) { 5813 DWARN(ldcp->id, 5814 "ldc_mem_dring_bind: (0x%llx) error binding mhandle\n", 5815 ldcp->id); 5816 mutex_exit(&dringp->lock); 5817 return (err); 5818 } 5819 5820 /* 5821 * For now return error if we get more than one cookie 5822 * FUTURE: Return multiple cookies .. 5823 */ 5824 if (*ccount > 1) { 5825 (void) ldc_mem_unbind_handle(mhandle); 5826 (void) ldc_mem_free_handle(mhandle); 5827 5828 dringp->ldcp = NULL; 5829 dringp->mhdl = NULL; 5830 *ccount = 0; 5831 5832 mutex_exit(&dringp->lock); 5833 return (EAGAIN); 5834 } 5835 5836 /* Add descriptor ring to channel's exported dring list */ 5837 mutex_enter(&ldcp->exp_dlist_lock); 5838 dringp->ch_next = ldcp->exp_dring_list; 5839 ldcp->exp_dring_list = dringp; 5840 mutex_exit(&ldcp->exp_dlist_lock); 5841 5842 dringp->status = LDC_BOUND; 5843 5844 mutex_exit(&dringp->lock); 5845 5846 return (0); 5847 } 5848 5849 /* 5850 * Return the next cookie associated with the specified dring handle 5851 */ 5852 int 5853 ldc_mem_dring_nextcookie(ldc_dring_handle_t dhandle, ldc_mem_cookie_t *cookie) 5854 { 5855 int rv = 0; 5856 ldc_dring_t *dringp; 5857 ldc_chan_t *ldcp; 5858 5859 if (dhandle == NULL) { 5860 DWARN(DBG_ALL_LDCS, 5861 "ldc_mem_dring_nextcookie: invalid desc ring handle\n"); 5862 return (EINVAL); 5863 } 5864 dringp = (ldc_dring_t *)dhandle; 5865 mutex_enter(&dringp->lock); 5866 5867 if (dringp->status != LDC_BOUND) { 5868 DWARN(DBG_ALL_LDCS, 5869 "ldc_mem_dring_nextcookie: descriptor ring 0x%llx " 5870 "is not bound\n", dringp); 5871 mutex_exit(&dringp->lock); 5872 return (EINVAL); 5873 } 5874 5875 ldcp = dringp->ldcp; 5876 5877 if (cookie == NULL) { 5878 DWARN(ldcp->id, 5879 "ldc_mem_dring_nextcookie:(0x%llx) invalid cookie arg\n", 5880 ldcp->id); 5881 mutex_exit(&dringp->lock); 5882 return (EINVAL); 5883 } 5884 5885 rv = ldc_mem_nextcookie((ldc_mem_handle_t)dringp->mhdl, cookie); 5886 mutex_exit(&dringp->lock); 5887 5888 return (rv); 5889 } 5890 /* 5891 * Unbind a previously bound dring from a channel. 5892 */ 5893 int 5894 ldc_mem_dring_unbind(ldc_dring_handle_t dhandle) 5895 { 5896 ldc_dring_t *dringp; 5897 ldc_dring_t *tmp_dringp; 5898 ldc_chan_t *ldcp; 5899 5900 if (dhandle == NULL) { 5901 DWARN(DBG_ALL_LDCS, 5902 "ldc_mem_dring_unbind: invalid desc ring handle\n"); 5903 return (EINVAL); 5904 } 5905 dringp = (ldc_dring_t *)dhandle; 5906 5907 mutex_enter(&dringp->lock); 5908 5909 if (dringp->status == LDC_UNBOUND) { 5910 DWARN(DBG_ALL_LDCS, 5911 "ldc_mem_dring_bind: descriptor ring 0x%llx is unbound\n", 5912 dringp); 5913 mutex_exit(&dringp->lock); 5914 return (EINVAL); 5915 } 5916 ldcp = dringp->ldcp; 5917 5918 mutex_enter(&ldcp->exp_dlist_lock); 5919 5920 tmp_dringp = ldcp->exp_dring_list; 5921 if (tmp_dringp == dringp) { 5922 ldcp->exp_dring_list = dringp->ch_next; 5923 dringp->ch_next = NULL; 5924 5925 } else { 5926 while (tmp_dringp != NULL) { 5927 if (tmp_dringp->ch_next == dringp) { 5928 tmp_dringp->ch_next = dringp->ch_next; 5929 dringp->ch_next = NULL; 5930 break; 5931 } 5932 tmp_dringp = tmp_dringp->ch_next; 5933 } 5934 if (tmp_dringp == NULL) { 5935 DWARN(DBG_ALL_LDCS, 5936 "ldc_mem_dring_unbind: invalid descriptor\n"); 5937 mutex_exit(&ldcp->exp_dlist_lock); 5938 mutex_exit(&dringp->lock); 5939 return (EINVAL); 5940 } 5941 } 5942 5943 mutex_exit(&ldcp->exp_dlist_lock); 5944 5945 (void) ldc_mem_unbind_handle((ldc_mem_handle_t)dringp->mhdl); 5946 (void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl); 5947 5948 dringp->ldcp = NULL; 5949 dringp->mhdl = NULL; 5950 dringp->status = LDC_UNBOUND; 5951 5952 mutex_exit(&dringp->lock); 5953 5954 return (0); 5955 } 5956 5957 /* 5958 * Get information about the dring. The base address of the descriptor 5959 * ring along with the type and permission are returned back. 5960 */ 5961 int 5962 ldc_mem_dring_info(ldc_dring_handle_t dhandle, ldc_mem_info_t *minfo) 5963 { 5964 ldc_dring_t *dringp; 5965 int rv; 5966 5967 if (dhandle == NULL) { 5968 DWARN(DBG_ALL_LDCS, 5969 "ldc_mem_dring_info: invalid desc ring handle\n"); 5970 return (EINVAL); 5971 } 5972 dringp = (ldc_dring_t *)dhandle; 5973 5974 mutex_enter(&dringp->lock); 5975 5976 if (dringp->mhdl) { 5977 rv = ldc_mem_info(dringp->mhdl, minfo); 5978 if (rv) { 5979 DWARN(DBG_ALL_LDCS, 5980 "ldc_mem_dring_info: error reading mem info\n"); 5981 mutex_exit(&dringp->lock); 5982 return (rv); 5983 } 5984 } else { 5985 minfo->vaddr = dringp->base; 5986 minfo->raddr = NULL; 5987 minfo->status = dringp->status; 5988 } 5989 5990 mutex_exit(&dringp->lock); 5991 5992 return (0); 5993 } 5994 5995 /* 5996 * Map an exported descriptor ring into the local address space. If the 5997 * descriptor ring was exported for direct map access, a HV call is made 5998 * to allocate a RA range. If the map is done via a shadow copy, local 5999 * shadow memory is allocated. 6000 */ 6001 int 6002 ldc_mem_dring_map(ldc_handle_t handle, ldc_mem_cookie_t *cookie, 6003 uint32_t ccount, uint32_t len, uint32_t dsize, uint8_t mtype, 6004 ldc_dring_handle_t *dhandle) 6005 { 6006 int err; 6007 ldc_chan_t *ldcp = (ldc_chan_t *)handle; 6008 ldc_mem_handle_t mhandle; 6009 ldc_dring_t *dringp; 6010 size_t dring_size; 6011 6012 if (dhandle == NULL) { 6013 DWARN(DBG_ALL_LDCS, 6014 "ldc_mem_dring_map: invalid dhandle\n"); 6015 return (EINVAL); 6016 } 6017 6018 /* check to see if channel is initalized */ 6019 if (handle == NULL) { 6020 DWARN(DBG_ALL_LDCS, 6021 "ldc_mem_dring_map: invalid channel handle\n"); 6022 return (EINVAL); 6023 } 6024 ldcp = (ldc_chan_t *)handle; 6025 6026 if (cookie == NULL) { 6027 DWARN(ldcp->id, 6028 "ldc_mem_dring_map: (0x%llx) invalid cookie\n", 6029 ldcp->id); 6030 return (EINVAL); 6031 } 6032 6033 /* FUTURE: For now we support only one cookie per dring */ 6034 ASSERT(ccount == 1); 6035 6036 if (cookie->size < (dsize * len)) { 6037 DWARN(ldcp->id, 6038 "ldc_mem_dring_map: (0x%llx) invalid dsize/len\n", 6039 ldcp->id); 6040 return (EINVAL); 6041 } 6042 6043 *dhandle = 0; 6044 6045 /* Allocate an dring structure */ 6046 dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP); 6047 6048 D1(ldcp->id, 6049 "ldc_mem_dring_map: 0x%x,0x%x,0x%x,0x%llx,0x%llx\n", 6050 mtype, len, dsize, cookie->addr, cookie->size); 6051 6052 /* Initialize dring */ 6053 dringp->length = len; 6054 dringp->dsize = dsize; 6055 6056 /* round of to multiple of page size */ 6057 dring_size = len * dsize; 6058 dringp->size = (dring_size & MMU_PAGEMASK); 6059 if (dring_size & MMU_PAGEOFFSET) 6060 dringp->size += MMU_PAGESIZE; 6061 6062 dringp->ldcp = ldcp; 6063 6064 /* create an memory handle */ 6065 err = ldc_mem_alloc_handle(handle, &mhandle); 6066 if (err || mhandle == NULL) { 6067 DWARN(DBG_ALL_LDCS, 6068 "ldc_mem_dring_map: cannot alloc hdl err=%d\n", 6069 err); 6070 kmem_free(dringp, sizeof (ldc_dring_t)); 6071 return (ENOMEM); 6072 } 6073 6074 dringp->mhdl = mhandle; 6075 dringp->base = NULL; 6076 6077 /* map the dring into local memory */ 6078 err = ldc_mem_map(mhandle, cookie, ccount, mtype, LDC_MEM_RW, 6079 &(dringp->base), NULL); 6080 if (err || dringp->base == NULL) { 6081 cmn_err(CE_WARN, 6082 "ldc_mem_dring_map: cannot map desc ring err=%d\n", err); 6083 (void) ldc_mem_free_handle(mhandle); 6084 kmem_free(dringp, sizeof (ldc_dring_t)); 6085 return (ENOMEM); 6086 } 6087 6088 /* initialize the desc ring lock */ 6089 mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL); 6090 6091 /* Add descriptor ring to channel's imported dring list */ 6092 mutex_enter(&ldcp->imp_dlist_lock); 6093 dringp->ch_next = ldcp->imp_dring_list; 6094 ldcp->imp_dring_list = dringp; 6095 mutex_exit(&ldcp->imp_dlist_lock); 6096 6097 dringp->status = LDC_MAPPED; 6098 6099 *dhandle = (ldc_dring_handle_t)dringp; 6100 6101 return (0); 6102 } 6103 6104 /* 6105 * Unmap a descriptor ring. Free shadow memory (if any). 6106 */ 6107 int 6108 ldc_mem_dring_unmap(ldc_dring_handle_t dhandle) 6109 { 6110 ldc_dring_t *dringp; 6111 ldc_dring_t *tmp_dringp; 6112 ldc_chan_t *ldcp; 6113 6114 if (dhandle == NULL) { 6115 DWARN(DBG_ALL_LDCS, 6116 "ldc_mem_dring_unmap: invalid desc ring handle\n"); 6117 return (EINVAL); 6118 } 6119 dringp = (ldc_dring_t *)dhandle; 6120 6121 if (dringp->status != LDC_MAPPED) { 6122 DWARN(DBG_ALL_LDCS, 6123 "ldc_mem_dring_unmap: not a mapped desc ring\n"); 6124 return (EINVAL); 6125 } 6126 6127 mutex_enter(&dringp->lock); 6128 6129 ldcp = dringp->ldcp; 6130 6131 mutex_enter(&ldcp->imp_dlist_lock); 6132 6133 /* find and unlink the desc ring from channel import list */ 6134 tmp_dringp = ldcp->imp_dring_list; 6135 if (tmp_dringp == dringp) { 6136 ldcp->imp_dring_list = dringp->ch_next; 6137 dringp->ch_next = NULL; 6138 6139 } else { 6140 while (tmp_dringp != NULL) { 6141 if (tmp_dringp->ch_next == dringp) { 6142 tmp_dringp->ch_next = dringp->ch_next; 6143 dringp->ch_next = NULL; 6144 break; 6145 } 6146 tmp_dringp = tmp_dringp->ch_next; 6147 } 6148 if (tmp_dringp == NULL) { 6149 DWARN(DBG_ALL_LDCS, 6150 "ldc_mem_dring_unmap: invalid descriptor\n"); 6151 mutex_exit(&ldcp->imp_dlist_lock); 6152 mutex_exit(&dringp->lock); 6153 return (EINVAL); 6154 } 6155 } 6156 6157 mutex_exit(&ldcp->imp_dlist_lock); 6158 6159 /* do a LDC memory handle unmap and free */ 6160 (void) ldc_mem_unmap(dringp->mhdl); 6161 (void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl); 6162 6163 dringp->status = 0; 6164 dringp->ldcp = NULL; 6165 6166 mutex_exit(&dringp->lock); 6167 6168 /* destroy dring lock */ 6169 mutex_destroy(&dringp->lock); 6170 6171 /* free desc ring object */ 6172 kmem_free(dringp, sizeof (ldc_dring_t)); 6173 6174 return (0); 6175 } 6176 6177 /* 6178 * Internal entry point for descriptor ring access entry consistency 6179 * semantics. Acquire copies the contents of the remote descriptor ring 6180 * into the local shadow copy. The release operation copies the local 6181 * contents into the remote dring. The start and end locations specify 6182 * bounds for the entries being synchronized. 6183 */ 6184 static int 6185 i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle, 6186 uint8_t direction, uint64_t start, uint64_t end) 6187 { 6188 int err; 6189 ldc_dring_t *dringp; 6190 ldc_chan_t *ldcp; 6191 uint64_t soff; 6192 size_t copy_size; 6193 6194 if (dhandle == NULL) { 6195 DWARN(DBG_ALL_LDCS, 6196 "i_ldc_dring_acquire_release: invalid desc ring handle\n"); 6197 return (EINVAL); 6198 } 6199 dringp = (ldc_dring_t *)dhandle; 6200 mutex_enter(&dringp->lock); 6201 6202 if (dringp->status != LDC_MAPPED || dringp->ldcp == NULL) { 6203 DWARN(DBG_ALL_LDCS, 6204 "i_ldc_dring_acquire_release: not a mapped desc ring\n"); 6205 mutex_exit(&dringp->lock); 6206 return (EINVAL); 6207 } 6208 6209 if (start >= dringp->length || end >= dringp->length) { 6210 DWARN(DBG_ALL_LDCS, 6211 "i_ldc_dring_acquire_release: index out of range\n"); 6212 mutex_exit(&dringp->lock); 6213 return (EINVAL); 6214 } 6215 6216 /* get the channel handle */ 6217 ldcp = dringp->ldcp; 6218 6219 copy_size = (start <= end) ? (((end - start) + 1) * dringp->dsize) : 6220 ((dringp->length - start) * dringp->dsize); 6221 6222 /* Calculate the relative offset for the first desc */ 6223 soff = (start * dringp->dsize); 6224 6225 /* copy to/from remote from/to local memory */ 6226 D1(ldcp->id, "i_ldc_dring_acquire_release: c1 off=0x%llx sz=0x%llx\n", 6227 soff, copy_size); 6228 err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl, 6229 direction, soff, copy_size); 6230 if (err) { 6231 DWARN(ldcp->id, 6232 "i_ldc_dring_acquire_release: copy failed\n"); 6233 mutex_exit(&dringp->lock); 6234 return (err); 6235 } 6236 6237 /* do the balance */ 6238 if (start > end) { 6239 copy_size = ((end + 1) * dringp->dsize); 6240 soff = 0; 6241 6242 /* copy to/from remote from/to local memory */ 6243 D1(ldcp->id, "i_ldc_dring_acquire_release: c2 " 6244 "off=0x%llx sz=0x%llx\n", soff, copy_size); 6245 err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl, 6246 direction, soff, copy_size); 6247 if (err) { 6248 DWARN(ldcp->id, 6249 "i_ldc_dring_acquire_release: copy failed\n"); 6250 mutex_exit(&dringp->lock); 6251 return (err); 6252 } 6253 } 6254 6255 mutex_exit(&dringp->lock); 6256 6257 return (0); 6258 } 6259 6260 /* 6261 * Ensure that the contents in the local dring are consistent 6262 * with the contents if of remote dring 6263 */ 6264 int 6265 ldc_mem_dring_acquire(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end) 6266 { 6267 return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_IN, start, end)); 6268 } 6269 6270 /* 6271 * Ensure that the contents in the remote dring are consistent 6272 * with the contents if of local dring 6273 */ 6274 int 6275 ldc_mem_dring_release(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end) 6276 { 6277 return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_OUT, start, end)); 6278 } 6279 6280 6281 /* ------------------------------------------------------------------------- */ 6282