11ae08745Sheppo /* 21ae08745Sheppo * CDDL HEADER START 31ae08745Sheppo * 41ae08745Sheppo * The contents of this file are subject to the terms of the 51ae08745Sheppo * Common Development and Distribution License (the "License"). 61ae08745Sheppo * You may not use this file except in compliance with the License. 71ae08745Sheppo * 81ae08745Sheppo * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 91ae08745Sheppo * or http://www.opensolaris.org/os/licensing. 101ae08745Sheppo * See the License for the specific language governing permissions 111ae08745Sheppo * and limitations under the License. 121ae08745Sheppo * 131ae08745Sheppo * When distributing Covered Code, include this CDDL HEADER in each 141ae08745Sheppo * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 151ae08745Sheppo * If applicable, add the following below this CDDL HEADER, with the 161ae08745Sheppo * fields enclosed by brackets "[]" replaced with your own identifying 171ae08745Sheppo * information: Portions Copyright [yyyy] [name of copyright owner] 181ae08745Sheppo * 191ae08745Sheppo * CDDL HEADER END 201ae08745Sheppo */ 211ae08745Sheppo 221ae08745Sheppo /* 2383d3bc6fSnarayan * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 241ae08745Sheppo * Use is subject to license terms. 251ae08745Sheppo */ 261ae08745Sheppo 271ae08745Sheppo #pragma ident "%Z%%M% %I% %E% SMI" 281ae08745Sheppo 291ae08745Sheppo /* 30e1ebb9ecSlm66018 * sun4v LDC Link Layer 311ae08745Sheppo */ 321ae08745Sheppo #include <sys/types.h> 331ae08745Sheppo #include <sys/file.h> 341ae08745Sheppo #include <sys/errno.h> 351ae08745Sheppo #include <sys/open.h> 361ae08745Sheppo #include <sys/cred.h> 371ae08745Sheppo #include <sys/kmem.h> 381ae08745Sheppo #include <sys/conf.h> 391ae08745Sheppo #include <sys/cmn_err.h> 401ae08745Sheppo #include <sys/ksynch.h> 411ae08745Sheppo #include <sys/modctl.h> 421ae08745Sheppo #include <sys/stat.h> /* needed for S_IFBLK and S_IFCHR */ 431ae08745Sheppo #include <sys/debug.h> 441ae08745Sheppo #include <sys/types.h> 451ae08745Sheppo #include <sys/cred.h> 461ae08745Sheppo #include <sys/promif.h> 471ae08745Sheppo #include <sys/ddi.h> 481ae08745Sheppo #include <sys/sunddi.h> 491ae08745Sheppo #include <sys/cyclic.h> 501ae08745Sheppo #include <sys/machsystm.h> 511ae08745Sheppo #include <sys/vm.h> 521ae08745Sheppo #include <sys/cpu.h> 531ae08745Sheppo #include <sys/intreg.h> 541ae08745Sheppo #include <sys/machcpuvar.h> 554bac2208Snarayan #include <sys/mmu.h> 564bac2208Snarayan #include <sys/pte.h> 574bac2208Snarayan #include <vm/hat.h> 584bac2208Snarayan #include <vm/as.h> 594bac2208Snarayan #include <vm/hat_sfmmu.h> 604bac2208Snarayan #include <sys/vm_machparam.h> 614bac2208Snarayan #include <vm/seg_kmem.h> 624bac2208Snarayan #include <vm/seg_kpm.h> 631ae08745Sheppo #include <sys/note.h> 641ae08745Sheppo #include <sys/ivintr.h> 651ae08745Sheppo #include <sys/hypervisor_api.h> 661ae08745Sheppo #include <sys/ldc.h> 671ae08745Sheppo #include <sys/ldc_impl.h> 681ae08745Sheppo #include <sys/cnex.h> 691ae08745Sheppo #include <sys/hsvc.h> 701ae08745Sheppo 711ae08745Sheppo /* Core internal functions */ 721ae08745Sheppo static int i_ldc_h2v_error(int h_error); 731ae08745Sheppo static int i_ldc_txq_reconf(ldc_chan_t *ldcp); 743af08d82Slm66018 static int i_ldc_rxq_reconf(ldc_chan_t *ldcp, boolean_t force_reset); 75a8ea4edeSnarayan static int i_ldc_rxq_drain(ldc_chan_t *ldcp); 761ae08745Sheppo static void i_ldc_reset_state(ldc_chan_t *ldcp); 773af08d82Slm66018 static void i_ldc_reset(ldc_chan_t *ldcp, boolean_t force_reset); 781ae08745Sheppo 791ae08745Sheppo static int i_ldc_get_tx_tail(ldc_chan_t *ldcp, uint64_t *tail); 801ae08745Sheppo static int i_ldc_set_tx_tail(ldc_chan_t *ldcp, uint64_t tail); 811ae08745Sheppo static int i_ldc_set_rx_head(ldc_chan_t *ldcp, uint64_t head); 821ae08745Sheppo static int i_ldc_send_pkt(ldc_chan_t *ldcp, uint8_t pkttype, uint8_t subtype, 831ae08745Sheppo uint8_t ctrlmsg); 841ae08745Sheppo 851ae08745Sheppo /* Interrupt handling functions */ 861ae08745Sheppo static uint_t i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2); 871ae08745Sheppo static uint_t i_ldc_rx_hdlr(caddr_t arg1, caddr_t arg2); 881ae08745Sheppo static void i_ldc_clear_intr(ldc_chan_t *ldcp, cnex_intrtype_t itype); 891ae08745Sheppo 901ae08745Sheppo /* Read method functions */ 911ae08745Sheppo static int i_ldc_read_raw(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep); 921ae08745Sheppo static int i_ldc_read_packet(ldc_chan_t *ldcp, caddr_t target_bufp, 931ae08745Sheppo size_t *sizep); 941ae08745Sheppo static int i_ldc_read_stream(ldc_chan_t *ldcp, caddr_t target_bufp, 951ae08745Sheppo size_t *sizep); 961ae08745Sheppo 971ae08745Sheppo /* Write method functions */ 981ae08745Sheppo static int i_ldc_write_raw(ldc_chan_t *ldcp, caddr_t target_bufp, 991ae08745Sheppo size_t *sizep); 1001ae08745Sheppo static int i_ldc_write_packet(ldc_chan_t *ldcp, caddr_t target_bufp, 1011ae08745Sheppo size_t *sizep); 1021ae08745Sheppo static int i_ldc_write_stream(ldc_chan_t *ldcp, caddr_t target_bufp, 1031ae08745Sheppo size_t *sizep); 1041ae08745Sheppo 1051ae08745Sheppo /* Pkt processing internal functions */ 1061ae08745Sheppo static int i_ldc_check_seqid(ldc_chan_t *ldcp, ldc_msg_t *ldcmsg); 1071ae08745Sheppo static int i_ldc_ctrlmsg(ldc_chan_t *ldcp, ldc_msg_t *ldcmsg); 1081ae08745Sheppo static int i_ldc_process_VER(ldc_chan_t *ldcp, ldc_msg_t *msg); 1091ae08745Sheppo static int i_ldc_process_RTS(ldc_chan_t *ldcp, ldc_msg_t *msg); 1101ae08745Sheppo static int i_ldc_process_RTR(ldc_chan_t *ldcp, ldc_msg_t *msg); 1111ae08745Sheppo static int i_ldc_process_RDX(ldc_chan_t *ldcp, ldc_msg_t *msg); 1121ae08745Sheppo static int i_ldc_process_data_ACK(ldc_chan_t *ldcp, ldc_msg_t *msg); 1131ae08745Sheppo 1141ae08745Sheppo /* Memory synchronization internal functions */ 1151ae08745Sheppo static int i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, 1161ae08745Sheppo uint8_t direction, uint64_t offset, size_t size); 1171ae08745Sheppo static int i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle, 1181ae08745Sheppo uint8_t direction, uint64_t start, uint64_t end); 1191ae08745Sheppo 1201ae08745Sheppo /* LDC Version */ 1211ae08745Sheppo static ldc_ver_t ldc_versions[] = { {1, 0} }; 1221ae08745Sheppo 1231ae08745Sheppo /* number of supported versions */ 1241ae08745Sheppo #define LDC_NUM_VERS (sizeof (ldc_versions) / sizeof (ldc_versions[0])) 1251ae08745Sheppo 1261ae08745Sheppo /* Module State Pointer */ 1271ae08745Sheppo static ldc_soft_state_t *ldcssp; 1281ae08745Sheppo 1291ae08745Sheppo static struct modldrv md = { 1301ae08745Sheppo &mod_miscops, /* This is a misc module */ 1311ae08745Sheppo "sun4v LDC module v%I%", /* Name of the module */ 1321ae08745Sheppo }; 1331ae08745Sheppo 1341ae08745Sheppo static struct modlinkage ml = { 1351ae08745Sheppo MODREV_1, 1361ae08745Sheppo &md, 1371ae08745Sheppo NULL 1381ae08745Sheppo }; 1391ae08745Sheppo 1401ae08745Sheppo static uint64_t ldc_sup_minor; /* Supported minor number */ 1411ae08745Sheppo static hsvc_info_t ldc_hsvc = { 1421ae08745Sheppo HSVC_REV_1, NULL, HSVC_GROUP_LDC, 1, 0, "ldc" 1431ae08745Sheppo }; 1441ae08745Sheppo 1454bac2208Snarayan /* 1464bac2208Snarayan * LDC framework supports mapping remote domain's memory 1474bac2208Snarayan * either directly or via shadow memory pages. Default 1484bac2208Snarayan * support is currently implemented via shadow copy. 1494bac2208Snarayan * Direct map can be enabled by setting 'ldc_shmem_enabled' 1504bac2208Snarayan */ 1514bac2208Snarayan int ldc_shmem_enabled = 0; 152e1ebb9ecSlm66018 1530a55fbb7Slm66018 /* 154e1ebb9ecSlm66018 * The no. of MTU size messages that can be stored in 155e1ebb9ecSlm66018 * the LDC Tx queue. The number of Tx queue entries is 156e1ebb9ecSlm66018 * then computed as (mtu * mtu_msgs)/sizeof(queue_entry) 157e1ebb9ecSlm66018 */ 158e1ebb9ecSlm66018 uint64_t ldc_mtu_msgs = LDC_MTU_MSGS; 159e1ebb9ecSlm66018 160e1ebb9ecSlm66018 /* 161e1ebb9ecSlm66018 * The minimum queue length. This is the size of the smallest 162e1ebb9ecSlm66018 * LDC queue. If the computed value is less than this default, 163e1ebb9ecSlm66018 * the queue length is rounded up to 'ldc_queue_entries'. 164e1ebb9ecSlm66018 */ 165e1ebb9ecSlm66018 uint64_t ldc_queue_entries = LDC_QUEUE_ENTRIES; 166e1ebb9ecSlm66018 167e1ebb9ecSlm66018 /* 168e1ebb9ecSlm66018 * Pages exported for remote access over each channel is 169e1ebb9ecSlm66018 * maintained in a table registered with the Hypervisor. 170e1ebb9ecSlm66018 * The default number of entries in the table is set to 171e1ebb9ecSlm66018 * 'ldc_mtbl_entries'. 172e1ebb9ecSlm66018 */ 173e1ebb9ecSlm66018 uint64_t ldc_maptable_entries = LDC_MTBL_ENTRIES; 174e1ebb9ecSlm66018 175e1ebb9ecSlm66018 /* 176e1ebb9ecSlm66018 * LDC retry count and delay - when the HV returns EWOULDBLOCK 177e1ebb9ecSlm66018 * the operation is retried 'ldc_max_retries' times with a 178e1ebb9ecSlm66018 * wait of 'ldc_delay' usecs between each retry. 1790a55fbb7Slm66018 */ 1800a55fbb7Slm66018 int ldc_max_retries = LDC_MAX_RETRIES; 1810a55fbb7Slm66018 clock_t ldc_delay = LDC_DELAY; 1820a55fbb7Slm66018 1834d39be2bSsg70180 /* 1844d39be2bSsg70180 * delay between each retry of channel unregistration in 1854d39be2bSsg70180 * ldc_close(), to wait for pending interrupts to complete. 1864d39be2bSsg70180 */ 1874d39be2bSsg70180 clock_t ldc_close_delay = LDC_CLOSE_DELAY; 1884d39be2bSsg70180 1891ae08745Sheppo #ifdef DEBUG 1901ae08745Sheppo 1911ae08745Sheppo /* 1921ae08745Sheppo * Print debug messages 1931ae08745Sheppo * 1941ae08745Sheppo * set ldcdbg to 0x7 for enabling all msgs 1951ae08745Sheppo * 0x4 - Warnings 1961ae08745Sheppo * 0x2 - All debug messages 1971ae08745Sheppo * 0x1 - Minimal debug messages 1981ae08745Sheppo * 1991ae08745Sheppo * set ldcdbgchan to the channel number you want to debug 2001ae08745Sheppo * setting it to -1 prints debug messages for all channels 2011ae08745Sheppo * NOTE: ldcdbgchan has no effect on error messages 2021ae08745Sheppo */ 2031ae08745Sheppo 2041ae08745Sheppo #define DBG_ALL_LDCS -1 2051ae08745Sheppo 2061ae08745Sheppo int ldcdbg = 0x0; 2071ae08745Sheppo int64_t ldcdbgchan = DBG_ALL_LDCS; 20883d3bc6fSnarayan uint64_t ldc_inject_err_flag = 0; 2091ae08745Sheppo 2101ae08745Sheppo static void 2111ae08745Sheppo ldcdebug(int64_t id, const char *fmt, ...) 2121ae08745Sheppo { 2131ae08745Sheppo char buf[512]; 2141ae08745Sheppo va_list ap; 2151ae08745Sheppo 2161ae08745Sheppo /* 2171ae08745Sheppo * Do not return if, 2181ae08745Sheppo * caller wants to print it anyway - (id == DBG_ALL_LDCS) 2191ae08745Sheppo * debug channel is set to all LDCs - (ldcdbgchan == DBG_ALL_LDCS) 2201ae08745Sheppo * debug channel = caller specified channel 2211ae08745Sheppo */ 2221ae08745Sheppo if ((id != DBG_ALL_LDCS) && 2231ae08745Sheppo (ldcdbgchan != DBG_ALL_LDCS) && 2241ae08745Sheppo (ldcdbgchan != id)) { 2251ae08745Sheppo return; 2261ae08745Sheppo } 2271ae08745Sheppo 2281ae08745Sheppo va_start(ap, fmt); 2291ae08745Sheppo (void) vsprintf(buf, fmt, ap); 2301ae08745Sheppo va_end(ap); 2311ae08745Sheppo 2323af08d82Slm66018 cmn_err(CE_CONT, "?%s", buf); 2333af08d82Slm66018 } 2343af08d82Slm66018 23583d3bc6fSnarayan #define LDC_ERR_RESET 0x1 23683d3bc6fSnarayan #define LDC_ERR_PKTLOSS 0x2 23783d3bc6fSnarayan 2383af08d82Slm66018 static boolean_t 23983d3bc6fSnarayan ldc_inject_error(ldc_chan_t *ldcp, uint64_t error) 2403af08d82Slm66018 { 2413af08d82Slm66018 if ((ldcdbgchan != DBG_ALL_LDCS) && (ldcdbgchan != ldcp->id)) 2423af08d82Slm66018 return (B_FALSE); 2433af08d82Slm66018 24483d3bc6fSnarayan if ((ldc_inject_err_flag & error) == 0) 2453af08d82Slm66018 return (B_FALSE); 2463af08d82Slm66018 2473af08d82Slm66018 /* clear the injection state */ 24883d3bc6fSnarayan ldc_inject_err_flag &= ~error; 2493af08d82Slm66018 2503af08d82Slm66018 return (B_TRUE); 2511ae08745Sheppo } 2521ae08745Sheppo 2531ae08745Sheppo #define D1 \ 2541ae08745Sheppo if (ldcdbg & 0x01) \ 2551ae08745Sheppo ldcdebug 2561ae08745Sheppo 2571ae08745Sheppo #define D2 \ 2581ae08745Sheppo if (ldcdbg & 0x02) \ 2591ae08745Sheppo ldcdebug 2601ae08745Sheppo 2611ae08745Sheppo #define DWARN \ 2621ae08745Sheppo if (ldcdbg & 0x04) \ 2631ae08745Sheppo ldcdebug 2641ae08745Sheppo 2651ae08745Sheppo #define DUMP_PAYLOAD(id, addr) \ 2661ae08745Sheppo { \ 2671ae08745Sheppo char buf[65*3]; \ 2681ae08745Sheppo int i; \ 2691ae08745Sheppo uint8_t *src = (uint8_t *)addr; \ 2701ae08745Sheppo for (i = 0; i < 64; i++, src++) \ 2711ae08745Sheppo (void) sprintf(&buf[i * 3], "|%02x", *src); \ 2721ae08745Sheppo (void) sprintf(&buf[i * 3], "|\n"); \ 2731ae08745Sheppo D2((id), "payload: %s", buf); \ 2741ae08745Sheppo } 2751ae08745Sheppo 2761ae08745Sheppo #define DUMP_LDC_PKT(c, s, addr) \ 2771ae08745Sheppo { \ 2781ae08745Sheppo ldc_msg_t *msg = (ldc_msg_t *)(addr); \ 2791ae08745Sheppo uint32_t mid = ((c)->mode != LDC_MODE_RAW) ? msg->seqid : 0; \ 2801ae08745Sheppo if (msg->type == LDC_DATA) { \ 2811ae08745Sheppo D2((c)->id, "%s: msg%d (/%x/%x/%x/,env[%c%c,sz=%d])", \ 2821ae08745Sheppo (s), mid, msg->type, msg->stype, msg->ctrl, \ 2831ae08745Sheppo (msg->env & LDC_FRAG_START) ? 'B' : ' ', \ 2841ae08745Sheppo (msg->env & LDC_FRAG_STOP) ? 'E' : ' ', \ 2851ae08745Sheppo (msg->env & LDC_LEN_MASK)); \ 2861ae08745Sheppo } else { \ 2871ae08745Sheppo D2((c)->id, "%s: msg%d (/%x/%x/%x/,env=%x)", (s), \ 2881ae08745Sheppo mid, msg->type, msg->stype, msg->ctrl, msg->env); \ 2891ae08745Sheppo } \ 2901ae08745Sheppo } 2911ae08745Sheppo 29283d3bc6fSnarayan #define LDC_INJECT_RESET(_ldcp) ldc_inject_error(_ldcp, LDC_ERR_RESET) 29383d3bc6fSnarayan #define LDC_INJECT_PKTLOSS(_ldcp) ldc_inject_error(_ldcp, LDC_ERR_PKTLOSS) 2943af08d82Slm66018 2951ae08745Sheppo #else 2961ae08745Sheppo 2971ae08745Sheppo #define DBG_ALL_LDCS -1 2981ae08745Sheppo 2991ae08745Sheppo #define D1 3001ae08745Sheppo #define D2 3011ae08745Sheppo #define DWARN 3021ae08745Sheppo 3031ae08745Sheppo #define DUMP_PAYLOAD(id, addr) 3041ae08745Sheppo #define DUMP_LDC_PKT(c, s, addr) 3051ae08745Sheppo 3063af08d82Slm66018 #define LDC_INJECT_RESET(_ldcp) (B_FALSE) 30783d3bc6fSnarayan #define LDC_INJECT_PKTLOSS(_ldcp) (B_FALSE) 3083af08d82Slm66018 3091ae08745Sheppo #endif 3101ae08745Sheppo 3111ae08745Sheppo #define ZERO_PKT(p) \ 3121ae08745Sheppo bzero((p), sizeof (ldc_msg_t)); 3131ae08745Sheppo 3141ae08745Sheppo #define IDX2COOKIE(idx, pg_szc, pg_shift) \ 3151ae08745Sheppo (((pg_szc) << LDC_COOKIE_PGSZC_SHIFT) | ((idx) << (pg_shift))) 3161ae08745Sheppo 3171ae08745Sheppo 3181ae08745Sheppo int 3191ae08745Sheppo _init(void) 3201ae08745Sheppo { 3211ae08745Sheppo int status; 3221ae08745Sheppo 3231ae08745Sheppo status = hsvc_register(&ldc_hsvc, &ldc_sup_minor); 3241ae08745Sheppo if (status != 0) { 325*d66f8315Sjb145095 cmn_err(CE_NOTE, "!%s: cannot negotiate hypervisor LDC services" 3261ae08745Sheppo " group: 0x%lx major: %ld minor: %ld errno: %d", 3271ae08745Sheppo ldc_hsvc.hsvc_modname, ldc_hsvc.hsvc_group, 3281ae08745Sheppo ldc_hsvc.hsvc_major, ldc_hsvc.hsvc_minor, status); 3291ae08745Sheppo return (-1); 3301ae08745Sheppo } 3311ae08745Sheppo 3321ae08745Sheppo /* allocate soft state structure */ 3331ae08745Sheppo ldcssp = kmem_zalloc(sizeof (ldc_soft_state_t), KM_SLEEP); 3341ae08745Sheppo 3351ae08745Sheppo /* Link the module into the system */ 3361ae08745Sheppo status = mod_install(&ml); 3371ae08745Sheppo if (status != 0) { 3381ae08745Sheppo kmem_free(ldcssp, sizeof (ldc_soft_state_t)); 3391ae08745Sheppo return (status); 3401ae08745Sheppo } 3411ae08745Sheppo 3421ae08745Sheppo /* Initialize the LDC state structure */ 3431ae08745Sheppo mutex_init(&ldcssp->lock, NULL, MUTEX_DRIVER, NULL); 3441ae08745Sheppo 3451ae08745Sheppo mutex_enter(&ldcssp->lock); 3461ae08745Sheppo 3474bac2208Snarayan /* Create a cache for memory handles */ 3484bac2208Snarayan ldcssp->memhdl_cache = kmem_cache_create("ldc_memhdl_cache", 3494bac2208Snarayan sizeof (ldc_mhdl_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 3504bac2208Snarayan if (ldcssp->memhdl_cache == NULL) { 3514bac2208Snarayan DWARN(DBG_ALL_LDCS, "_init: ldc_memhdl cache create failed\n"); 3524bac2208Snarayan mutex_exit(&ldcssp->lock); 3534bac2208Snarayan return (-1); 3544bac2208Snarayan } 3554bac2208Snarayan 3564bac2208Snarayan /* Create cache for memory segment structures */ 3574bac2208Snarayan ldcssp->memseg_cache = kmem_cache_create("ldc_memseg_cache", 3584bac2208Snarayan sizeof (ldc_memseg_t), 0, NULL, NULL, NULL, NULL, NULL, 0); 3594bac2208Snarayan if (ldcssp->memseg_cache == NULL) { 3604bac2208Snarayan DWARN(DBG_ALL_LDCS, "_init: ldc_memseg cache create failed\n"); 3614bac2208Snarayan mutex_exit(&ldcssp->lock); 3624bac2208Snarayan return (-1); 3634bac2208Snarayan } 3644bac2208Snarayan 3654bac2208Snarayan 3661ae08745Sheppo ldcssp->channel_count = 0; 3671ae08745Sheppo ldcssp->channels_open = 0; 3681ae08745Sheppo ldcssp->chan_list = NULL; 3691ae08745Sheppo ldcssp->dring_list = NULL; 3701ae08745Sheppo 3711ae08745Sheppo mutex_exit(&ldcssp->lock); 3721ae08745Sheppo 3731ae08745Sheppo return (0); 3741ae08745Sheppo } 3751ae08745Sheppo 3761ae08745Sheppo int 3771ae08745Sheppo _info(struct modinfo *modinfop) 3781ae08745Sheppo { 3791ae08745Sheppo /* Report status of the dynamically loadable driver module */ 3801ae08745Sheppo return (mod_info(&ml, modinfop)); 3811ae08745Sheppo } 3821ae08745Sheppo 3831ae08745Sheppo int 3841ae08745Sheppo _fini(void) 3851ae08745Sheppo { 3861ae08745Sheppo int rv, status; 3871ae08745Sheppo ldc_chan_t *ldcp; 3881ae08745Sheppo ldc_dring_t *dringp; 3891ae08745Sheppo ldc_mem_info_t minfo; 3901ae08745Sheppo 3911ae08745Sheppo /* Unlink the driver module from the system */ 3921ae08745Sheppo status = mod_remove(&ml); 3931ae08745Sheppo if (status) { 3941ae08745Sheppo DWARN(DBG_ALL_LDCS, "_fini: mod_remove failed\n"); 3951ae08745Sheppo return (EIO); 3961ae08745Sheppo } 3971ae08745Sheppo 3981ae08745Sheppo /* close and finalize channels */ 3991ae08745Sheppo ldcp = ldcssp->chan_list; 4001ae08745Sheppo while (ldcp != NULL) { 4011ae08745Sheppo (void) ldc_close((ldc_handle_t)ldcp); 4021ae08745Sheppo (void) ldc_fini((ldc_handle_t)ldcp); 4031ae08745Sheppo 4041ae08745Sheppo ldcp = ldcp->next; 4051ae08745Sheppo } 4061ae08745Sheppo 4071ae08745Sheppo /* Free descriptor rings */ 4081ae08745Sheppo dringp = ldcssp->dring_list; 4091ae08745Sheppo while (dringp != NULL) { 4101ae08745Sheppo dringp = dringp->next; 4111ae08745Sheppo 4121ae08745Sheppo rv = ldc_mem_dring_info((ldc_dring_handle_t)dringp, &minfo); 4131ae08745Sheppo if (rv == 0 && minfo.status != LDC_UNBOUND) { 4141ae08745Sheppo if (minfo.status == LDC_BOUND) { 4151ae08745Sheppo (void) ldc_mem_dring_unbind( 4161ae08745Sheppo (ldc_dring_handle_t)dringp); 4171ae08745Sheppo } 4181ae08745Sheppo if (minfo.status == LDC_MAPPED) { 4191ae08745Sheppo (void) ldc_mem_dring_unmap( 4201ae08745Sheppo (ldc_dring_handle_t)dringp); 4211ae08745Sheppo } 4221ae08745Sheppo } 4231ae08745Sheppo 4241ae08745Sheppo (void) ldc_mem_dring_destroy((ldc_dring_handle_t)dringp); 4251ae08745Sheppo } 4261ae08745Sheppo ldcssp->dring_list = NULL; 4271ae08745Sheppo 4284bac2208Snarayan /* Destroy kmem caches */ 4294bac2208Snarayan kmem_cache_destroy(ldcssp->memhdl_cache); 4304bac2208Snarayan kmem_cache_destroy(ldcssp->memseg_cache); 4314bac2208Snarayan 4321ae08745Sheppo /* 4331ae08745Sheppo * We have successfully "removed" the driver. 4341ae08745Sheppo * Destroying soft states 4351ae08745Sheppo */ 4361ae08745Sheppo mutex_destroy(&ldcssp->lock); 4371ae08745Sheppo kmem_free(ldcssp, sizeof (ldc_soft_state_t)); 4381ae08745Sheppo 4391ae08745Sheppo (void) hsvc_unregister(&ldc_hsvc); 4401ae08745Sheppo 4411ae08745Sheppo return (status); 4421ae08745Sheppo } 4431ae08745Sheppo 4441ae08745Sheppo /* -------------------------------------------------------------------------- */ 4451ae08745Sheppo 4461ae08745Sheppo /* 447e1ebb9ecSlm66018 * LDC Link Layer Internal Functions 4481ae08745Sheppo */ 4491ae08745Sheppo 4501ae08745Sheppo /* 4511ae08745Sheppo * Translate HV Errors to sun4v error codes 4521ae08745Sheppo */ 4531ae08745Sheppo static int 4541ae08745Sheppo i_ldc_h2v_error(int h_error) 4551ae08745Sheppo { 4561ae08745Sheppo switch (h_error) { 4571ae08745Sheppo 4581ae08745Sheppo case H_EOK: 4591ae08745Sheppo return (0); 4601ae08745Sheppo 4611ae08745Sheppo case H_ENORADDR: 4621ae08745Sheppo return (EFAULT); 4631ae08745Sheppo 4641ae08745Sheppo case H_EBADPGSZ: 4651ae08745Sheppo case H_EINVAL: 4661ae08745Sheppo return (EINVAL); 4671ae08745Sheppo 4681ae08745Sheppo case H_EWOULDBLOCK: 4691ae08745Sheppo return (EWOULDBLOCK); 4701ae08745Sheppo 4711ae08745Sheppo case H_ENOACCESS: 4721ae08745Sheppo case H_ENOMAP: 4731ae08745Sheppo return (EACCES); 4741ae08745Sheppo 4751ae08745Sheppo case H_EIO: 4761ae08745Sheppo case H_ECPUERROR: 4771ae08745Sheppo return (EIO); 4781ae08745Sheppo 4791ae08745Sheppo case H_ENOTSUPPORTED: 4801ae08745Sheppo return (ENOTSUP); 4811ae08745Sheppo 4821ae08745Sheppo case H_ETOOMANY: 4831ae08745Sheppo return (ENOSPC); 4841ae08745Sheppo 4851ae08745Sheppo case H_ECHANNEL: 4861ae08745Sheppo return (ECHRNG); 4871ae08745Sheppo default: 4881ae08745Sheppo break; 4891ae08745Sheppo } 4901ae08745Sheppo 4911ae08745Sheppo return (EIO); 4921ae08745Sheppo } 4931ae08745Sheppo 4941ae08745Sheppo /* 4951ae08745Sheppo * Reconfigure the transmit queue 4961ae08745Sheppo */ 4971ae08745Sheppo static int 4981ae08745Sheppo i_ldc_txq_reconf(ldc_chan_t *ldcp) 4991ae08745Sheppo { 5001ae08745Sheppo int rv; 5011ae08745Sheppo 5021ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 503d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 504d10e4ef2Snarayan 5051ae08745Sheppo rv = hv_ldc_tx_qconf(ldcp->id, ldcp->tx_q_ra, ldcp->tx_q_entries); 5061ae08745Sheppo if (rv) { 5071ae08745Sheppo cmn_err(CE_WARN, 5083af08d82Slm66018 "i_ldc_txq_reconf: (0x%lx) cannot set qconf", ldcp->id); 5091ae08745Sheppo return (EIO); 5101ae08745Sheppo } 5111ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, &(ldcp->tx_head), 5121ae08745Sheppo &(ldcp->tx_tail), &(ldcp->link_state)); 5131ae08745Sheppo if (rv) { 5141ae08745Sheppo cmn_err(CE_WARN, 5153af08d82Slm66018 "i_ldc_txq_reconf: (0x%lx) cannot get qptrs", ldcp->id); 5161ae08745Sheppo return (EIO); 5171ae08745Sheppo } 5183af08d82Slm66018 D1(ldcp->id, "i_ldc_txq_reconf: (0x%llx) h=0x%llx,t=0x%llx," 5191ae08745Sheppo "s=0x%llx\n", ldcp->id, ldcp->tx_head, ldcp->tx_tail, 5201ae08745Sheppo ldcp->link_state); 5211ae08745Sheppo 5221ae08745Sheppo return (0); 5231ae08745Sheppo } 5241ae08745Sheppo 5251ae08745Sheppo /* 5261ae08745Sheppo * Reconfigure the receive queue 5271ae08745Sheppo */ 5281ae08745Sheppo static int 5293af08d82Slm66018 i_ldc_rxq_reconf(ldc_chan_t *ldcp, boolean_t force_reset) 5301ae08745Sheppo { 5311ae08745Sheppo int rv; 5321ae08745Sheppo uint64_t rx_head, rx_tail; 5331ae08745Sheppo 5341ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 5351ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 5361ae08745Sheppo &(ldcp->link_state)); 5371ae08745Sheppo if (rv) { 5381ae08745Sheppo cmn_err(CE_WARN, 5393af08d82Slm66018 "i_ldc_rxq_reconf: (0x%lx) cannot get state", 5401ae08745Sheppo ldcp->id); 5411ae08745Sheppo return (EIO); 5421ae08745Sheppo } 5431ae08745Sheppo 5443af08d82Slm66018 if (force_reset || (ldcp->tstate & ~TS_IN_RESET) == TS_UP) { 5451ae08745Sheppo rv = hv_ldc_rx_qconf(ldcp->id, ldcp->rx_q_ra, 5461ae08745Sheppo ldcp->rx_q_entries); 5471ae08745Sheppo if (rv) { 5481ae08745Sheppo cmn_err(CE_WARN, 5493af08d82Slm66018 "i_ldc_rxq_reconf: (0x%lx) cannot set qconf", 5501ae08745Sheppo ldcp->id); 5511ae08745Sheppo return (EIO); 5521ae08745Sheppo } 5533af08d82Slm66018 D1(ldcp->id, "i_ldc_rxq_reconf: (0x%llx) completed q reconf", 5541ae08745Sheppo ldcp->id); 5551ae08745Sheppo } 5561ae08745Sheppo 5571ae08745Sheppo return (0); 5581ae08745Sheppo } 5591ae08745Sheppo 560a8ea4edeSnarayan 561a8ea4edeSnarayan /* 562a8ea4edeSnarayan * Drain the contents of the receive queue 563a8ea4edeSnarayan */ 564a8ea4edeSnarayan static int 565a8ea4edeSnarayan i_ldc_rxq_drain(ldc_chan_t *ldcp) 566a8ea4edeSnarayan { 567a8ea4edeSnarayan int rv; 568a8ea4edeSnarayan uint64_t rx_head, rx_tail; 569a8ea4edeSnarayan 570a8ea4edeSnarayan ASSERT(MUTEX_HELD(&ldcp->lock)); 571a8ea4edeSnarayan rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 572a8ea4edeSnarayan &(ldcp->link_state)); 573a8ea4edeSnarayan if (rv) { 574a8ea4edeSnarayan cmn_err(CE_WARN, "i_ldc_rxq_drain: (0x%lx) cannot get state", 575a8ea4edeSnarayan ldcp->id); 576a8ea4edeSnarayan return (EIO); 577a8ea4edeSnarayan } 578a8ea4edeSnarayan 579a8ea4edeSnarayan /* flush contents by setting the head = tail */ 580a8ea4edeSnarayan return (i_ldc_set_rx_head(ldcp, rx_tail)); 581a8ea4edeSnarayan } 582a8ea4edeSnarayan 583a8ea4edeSnarayan 5841ae08745Sheppo /* 5851ae08745Sheppo * Reset LDC state structure and its contents 5861ae08745Sheppo */ 5871ae08745Sheppo static void 5881ae08745Sheppo i_ldc_reset_state(ldc_chan_t *ldcp) 5891ae08745Sheppo { 5901ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 5911ae08745Sheppo ldcp->last_msg_snt = LDC_INIT_SEQID; 5921ae08745Sheppo ldcp->last_ack_rcd = 0; 5931ae08745Sheppo ldcp->last_msg_rcd = 0; 5941ae08745Sheppo ldcp->tx_ackd_head = ldcp->tx_head; 5951ae08745Sheppo ldcp->next_vidx = 0; 5961ae08745Sheppo ldcp->hstate = 0; 5971ae08745Sheppo ldcp->tstate = TS_OPEN; 5981ae08745Sheppo ldcp->status = LDC_OPEN; 5991ae08745Sheppo 6001ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_UP || 6011ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 6021ae08745Sheppo 6031ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW) { 6041ae08745Sheppo ldcp->status = LDC_UP; 6051ae08745Sheppo ldcp->tstate = TS_UP; 6061ae08745Sheppo } else { 6071ae08745Sheppo ldcp->status = LDC_READY; 6081ae08745Sheppo ldcp->tstate |= TS_LINK_READY; 6091ae08745Sheppo } 6101ae08745Sheppo } 6111ae08745Sheppo } 6121ae08745Sheppo 6131ae08745Sheppo /* 6141ae08745Sheppo * Reset a LDC channel 6151ae08745Sheppo */ 6161ae08745Sheppo static void 6173af08d82Slm66018 i_ldc_reset(ldc_chan_t *ldcp, boolean_t force_reset) 6181ae08745Sheppo { 61983d3bc6fSnarayan DWARN(ldcp->id, "i_ldc_reset: (0x%llx) channel reset\n", ldcp->id); 6201ae08745Sheppo 621d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->lock)); 622d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 623d10e4ef2Snarayan 6243af08d82Slm66018 /* reconfig Tx and Rx queues */ 6251ae08745Sheppo (void) i_ldc_txq_reconf(ldcp); 6263af08d82Slm66018 (void) i_ldc_rxq_reconf(ldcp, force_reset); 6273af08d82Slm66018 6283af08d82Slm66018 /* Clear Tx and Rx interrupts */ 6293af08d82Slm66018 (void) i_ldc_clear_intr(ldcp, CNEX_TX_INTR); 6303af08d82Slm66018 (void) i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 6313af08d82Slm66018 6323af08d82Slm66018 /* Reset channel state */ 6331ae08745Sheppo i_ldc_reset_state(ldcp); 6343af08d82Slm66018 6353af08d82Slm66018 /* Mark channel in reset */ 6363af08d82Slm66018 ldcp->tstate |= TS_IN_RESET; 6371ae08745Sheppo } 6381ae08745Sheppo 6394bac2208Snarayan 6401ae08745Sheppo /* 6411ae08745Sheppo * Clear pending interrupts 6421ae08745Sheppo */ 6431ae08745Sheppo static void 6441ae08745Sheppo i_ldc_clear_intr(ldc_chan_t *ldcp, cnex_intrtype_t itype) 6451ae08745Sheppo { 6461ae08745Sheppo ldc_cnex_t *cinfo = &ldcssp->cinfo; 6471ae08745Sheppo 6481ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 6493af08d82Slm66018 ASSERT(cinfo->dip != NULL); 6504bac2208Snarayan 6513af08d82Slm66018 switch (itype) { 6523af08d82Slm66018 case CNEX_TX_INTR: 6534bac2208Snarayan /* check Tx interrupt */ 6543af08d82Slm66018 if (ldcp->tx_intr_state) 6553af08d82Slm66018 ldcp->tx_intr_state = LDC_INTR_NONE; 6564bac2208Snarayan else 6574bac2208Snarayan return; 6583af08d82Slm66018 break; 6593af08d82Slm66018 6603af08d82Slm66018 case CNEX_RX_INTR: 6614bac2208Snarayan /* check Rx interrupt */ 6623af08d82Slm66018 if (ldcp->rx_intr_state) 6633af08d82Slm66018 ldcp->rx_intr_state = LDC_INTR_NONE; 6644bac2208Snarayan else 6654bac2208Snarayan return; 6663af08d82Slm66018 break; 6674bac2208Snarayan } 6684bac2208Snarayan 6691ae08745Sheppo (void) cinfo->clr_intr(cinfo->dip, ldcp->id, itype); 6704bac2208Snarayan D2(ldcp->id, 6714bac2208Snarayan "i_ldc_clear_intr: (0x%llx) cleared 0x%x intr\n", 6724bac2208Snarayan ldcp->id, itype); 6731ae08745Sheppo } 6741ae08745Sheppo 6751ae08745Sheppo /* 6761ae08745Sheppo * Set the receive queue head 6770a55fbb7Slm66018 * Resets connection and returns an error if it fails. 6781ae08745Sheppo */ 6791ae08745Sheppo static int 6801ae08745Sheppo i_ldc_set_rx_head(ldc_chan_t *ldcp, uint64_t head) 6811ae08745Sheppo { 6821ae08745Sheppo int rv; 6830a55fbb7Slm66018 int retries; 6841ae08745Sheppo 6851ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 6860a55fbb7Slm66018 for (retries = 0; retries < ldc_max_retries; retries++) { 6870a55fbb7Slm66018 6880a55fbb7Slm66018 if ((rv = hv_ldc_rx_set_qhead(ldcp->id, head)) == 0) 6890a55fbb7Slm66018 return (0); 6900a55fbb7Slm66018 6910a55fbb7Slm66018 if (rv != H_EWOULDBLOCK) 6920a55fbb7Slm66018 break; 6930a55fbb7Slm66018 6940a55fbb7Slm66018 /* wait for ldc_delay usecs */ 6950a55fbb7Slm66018 drv_usecwait(ldc_delay); 6961ae08745Sheppo } 6971ae08745Sheppo 6980a55fbb7Slm66018 cmn_err(CE_WARN, "ldc_rx_set_qhead: (0x%lx) cannot set qhead 0x%lx", 6990a55fbb7Slm66018 ldcp->id, head); 700d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 7013af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 702d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 7030a55fbb7Slm66018 7040a55fbb7Slm66018 return (ECONNRESET); 7051ae08745Sheppo } 7061ae08745Sheppo 7071ae08745Sheppo 7081ae08745Sheppo /* 7091ae08745Sheppo * Returns the tx_tail to be used for transfer 7101ae08745Sheppo * Re-reads the TX queue ptrs if and only if the 7111ae08745Sheppo * the cached head and tail are equal (queue is full) 7121ae08745Sheppo */ 7131ae08745Sheppo static int 7141ae08745Sheppo i_ldc_get_tx_tail(ldc_chan_t *ldcp, uint64_t *tail) 7151ae08745Sheppo { 7161ae08745Sheppo int rv; 7171ae08745Sheppo uint64_t current_head, new_tail; 7181ae08745Sheppo 719d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 7201ae08745Sheppo /* Read the head and tail ptrs from HV */ 7211ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 7221ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 7231ae08745Sheppo if (rv) { 7241ae08745Sheppo cmn_err(CE_WARN, 7251ae08745Sheppo "i_ldc_get_tx_tail: (0x%lx) cannot read qptrs\n", 7261ae08745Sheppo ldcp->id); 7271ae08745Sheppo return (EIO); 7281ae08745Sheppo } 7291ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN) { 730cb112a14Slm66018 D1(ldcp->id, "i_ldc_get_tx_tail: (0x%llx) channel not ready\n", 7311ae08745Sheppo ldcp->id); 7321ae08745Sheppo return (ECONNRESET); 7331ae08745Sheppo } 7341ae08745Sheppo 7351ae08745Sheppo /* In reliable mode, check against last ACKd msg */ 7361ae08745Sheppo current_head = (ldcp->mode == LDC_MODE_RELIABLE || 7371ae08745Sheppo ldcp->mode == LDC_MODE_STREAM) 7381ae08745Sheppo ? ldcp->tx_ackd_head : ldcp->tx_head; 7391ae08745Sheppo 7401ae08745Sheppo /* increment the tail */ 7411ae08745Sheppo new_tail = (ldcp->tx_tail + LDC_PACKET_SIZE) % 7421ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 7431ae08745Sheppo 7441ae08745Sheppo if (new_tail == current_head) { 7451ae08745Sheppo DWARN(ldcp->id, 7461ae08745Sheppo "i_ldc_get_tx_tail: (0x%llx) TX queue is full\n", 7471ae08745Sheppo ldcp->id); 7481ae08745Sheppo return (EWOULDBLOCK); 7491ae08745Sheppo } 7501ae08745Sheppo 7511ae08745Sheppo D2(ldcp->id, "i_ldc_get_tx_tail: (0x%llx) head=0x%llx, tail=0x%llx\n", 7521ae08745Sheppo ldcp->id, ldcp->tx_head, ldcp->tx_tail); 7531ae08745Sheppo 7541ae08745Sheppo *tail = ldcp->tx_tail; 7551ae08745Sheppo return (0); 7561ae08745Sheppo } 7571ae08745Sheppo 7581ae08745Sheppo /* 7591ae08745Sheppo * Set the tail pointer. If HV returns EWOULDBLOCK, it will back off 7600a55fbb7Slm66018 * and retry ldc_max_retries times before returning an error. 7611ae08745Sheppo * Returns 0, EWOULDBLOCK or EIO 7621ae08745Sheppo */ 7631ae08745Sheppo static int 7641ae08745Sheppo i_ldc_set_tx_tail(ldc_chan_t *ldcp, uint64_t tail) 7651ae08745Sheppo { 7661ae08745Sheppo int rv, retval = EWOULDBLOCK; 7670a55fbb7Slm66018 int retries; 7681ae08745Sheppo 769d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 7700a55fbb7Slm66018 for (retries = 0; retries < ldc_max_retries; retries++) { 7711ae08745Sheppo 7721ae08745Sheppo if ((rv = hv_ldc_tx_set_qtail(ldcp->id, tail)) == 0) { 7731ae08745Sheppo retval = 0; 7741ae08745Sheppo break; 7751ae08745Sheppo } 7761ae08745Sheppo if (rv != H_EWOULDBLOCK) { 7771ae08745Sheppo DWARN(ldcp->id, "i_ldc_set_tx_tail: (0x%llx) set " 7781ae08745Sheppo "qtail=0x%llx failed, rv=%d\n", ldcp->id, tail, rv); 7791ae08745Sheppo retval = EIO; 7801ae08745Sheppo break; 7811ae08745Sheppo } 7821ae08745Sheppo 7830a55fbb7Slm66018 /* wait for ldc_delay usecs */ 7840a55fbb7Slm66018 drv_usecwait(ldc_delay); 7851ae08745Sheppo } 7861ae08745Sheppo return (retval); 7871ae08745Sheppo } 7881ae08745Sheppo 7891ae08745Sheppo /* 7901ae08745Sheppo * Send a LDC message 7911ae08745Sheppo */ 7921ae08745Sheppo static int 7931ae08745Sheppo i_ldc_send_pkt(ldc_chan_t *ldcp, uint8_t pkttype, uint8_t subtype, 7941ae08745Sheppo uint8_t ctrlmsg) 7951ae08745Sheppo { 7961ae08745Sheppo int rv; 7971ae08745Sheppo ldc_msg_t *pkt; 7981ae08745Sheppo uint64_t tx_tail; 7991ae08745Sheppo uint32_t curr_seqid = ldcp->last_msg_snt; 8001ae08745Sheppo 801d10e4ef2Snarayan /* Obtain Tx lock */ 802d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 803d10e4ef2Snarayan 8041ae08745Sheppo /* get the current tail for the message */ 8051ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 8061ae08745Sheppo if (rv) { 8071ae08745Sheppo DWARN(ldcp->id, 8081ae08745Sheppo "i_ldc_send_pkt: (0x%llx) error sending pkt, " 8091ae08745Sheppo "type=0x%x,subtype=0x%x,ctrl=0x%x\n", 8101ae08745Sheppo ldcp->id, pkttype, subtype, ctrlmsg); 811d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 8121ae08745Sheppo return (rv); 8131ae08745Sheppo } 8141ae08745Sheppo 8151ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 8161ae08745Sheppo ZERO_PKT(pkt); 8171ae08745Sheppo 8181ae08745Sheppo /* Initialize the packet */ 8191ae08745Sheppo pkt->type = pkttype; 8201ae08745Sheppo pkt->stype = subtype; 8211ae08745Sheppo pkt->ctrl = ctrlmsg; 8221ae08745Sheppo 8231ae08745Sheppo /* Store ackid/seqid iff it is RELIABLE mode & not a RTS/RTR message */ 8241ae08745Sheppo if (((ctrlmsg & LDC_CTRL_MASK) != LDC_RTS) && 8251ae08745Sheppo ((ctrlmsg & LDC_CTRL_MASK) != LDC_RTR)) { 8261ae08745Sheppo curr_seqid++; 8271ae08745Sheppo if (ldcp->mode != LDC_MODE_RAW) { 8281ae08745Sheppo pkt->seqid = curr_seqid; 8291ae08745Sheppo pkt->ackid = ldcp->last_msg_rcd; 8301ae08745Sheppo } 8311ae08745Sheppo } 8321ae08745Sheppo DUMP_LDC_PKT(ldcp, "i_ldc_send_pkt", (uint64_t)pkt); 8331ae08745Sheppo 8341ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 8351ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 8361ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 8371ae08745Sheppo 8381ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 8391ae08745Sheppo if (rv) { 8401ae08745Sheppo DWARN(ldcp->id, 8411ae08745Sheppo "i_ldc_send_pkt:(0x%llx) error sending pkt, " 8421ae08745Sheppo "type=0x%x,stype=0x%x,ctrl=0x%x\n", 8431ae08745Sheppo ldcp->id, pkttype, subtype, ctrlmsg); 844d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 8451ae08745Sheppo return (EIO); 8461ae08745Sheppo } 8471ae08745Sheppo 8481ae08745Sheppo ldcp->last_msg_snt = curr_seqid; 8491ae08745Sheppo ldcp->tx_tail = tx_tail; 8501ae08745Sheppo 851d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 8521ae08745Sheppo return (0); 8531ae08745Sheppo } 8541ae08745Sheppo 8551ae08745Sheppo /* 8561ae08745Sheppo * Checks if packet was received in right order 857e1ebb9ecSlm66018 * in the case of a reliable link. 8581ae08745Sheppo * Returns 0 if in order, else EIO 8591ae08745Sheppo */ 8601ae08745Sheppo static int 8611ae08745Sheppo i_ldc_check_seqid(ldc_chan_t *ldcp, ldc_msg_t *msg) 8621ae08745Sheppo { 8631ae08745Sheppo /* No seqid checking for RAW mode */ 8641ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW) 8651ae08745Sheppo return (0); 8661ae08745Sheppo 8671ae08745Sheppo /* No seqid checking for version, RTS, RTR message */ 8681ae08745Sheppo if (msg->ctrl == LDC_VER || 8691ae08745Sheppo msg->ctrl == LDC_RTS || 8701ae08745Sheppo msg->ctrl == LDC_RTR) 8711ae08745Sheppo return (0); 8721ae08745Sheppo 8731ae08745Sheppo /* Initial seqid to use is sent in RTS/RTR and saved in last_msg_rcd */ 8741ae08745Sheppo if (msg->seqid != (ldcp->last_msg_rcd + 1)) { 8751ae08745Sheppo DWARN(ldcp->id, 8761ae08745Sheppo "i_ldc_check_seqid: (0x%llx) out-of-order pkt, got 0x%x, " 8771ae08745Sheppo "expecting 0x%x\n", ldcp->id, msg->seqid, 8781ae08745Sheppo (ldcp->last_msg_rcd + 1)); 8791ae08745Sheppo return (EIO); 8801ae08745Sheppo } 8811ae08745Sheppo 88283d3bc6fSnarayan #ifdef DEBUG 88383d3bc6fSnarayan if (LDC_INJECT_PKTLOSS(ldcp)) { 88483d3bc6fSnarayan DWARN(ldcp->id, 88583d3bc6fSnarayan "i_ldc_check_seqid: (0x%llx) inject pkt loss\n", ldcp->id); 88683d3bc6fSnarayan return (EIO); 88783d3bc6fSnarayan } 88883d3bc6fSnarayan #endif 88983d3bc6fSnarayan 8901ae08745Sheppo return (0); 8911ae08745Sheppo } 8921ae08745Sheppo 8931ae08745Sheppo 8941ae08745Sheppo /* 8951ae08745Sheppo * Process an incoming version ctrl message 8961ae08745Sheppo */ 8971ae08745Sheppo static int 8981ae08745Sheppo i_ldc_process_VER(ldc_chan_t *ldcp, ldc_msg_t *msg) 8991ae08745Sheppo { 9001ae08745Sheppo int rv = 0, idx = ldcp->next_vidx; 9011ae08745Sheppo ldc_msg_t *pkt; 9021ae08745Sheppo uint64_t tx_tail; 9031ae08745Sheppo ldc_ver_t *rcvd_ver; 9041ae08745Sheppo 9051ae08745Sheppo /* get the received version */ 9061ae08745Sheppo rcvd_ver = (ldc_ver_t *)((uint64_t)msg + LDC_PAYLOAD_VER_OFF); 9071ae08745Sheppo 9081ae08745Sheppo D2(ldcp->id, "i_ldc_process_VER: (0x%llx) received VER v%u.%u\n", 9091ae08745Sheppo ldcp->id, rcvd_ver->major, rcvd_ver->minor); 9101ae08745Sheppo 911d10e4ef2Snarayan /* Obtain Tx lock */ 912d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 913d10e4ef2Snarayan 9141ae08745Sheppo switch (msg->stype) { 9151ae08745Sheppo case LDC_INFO: 9161ae08745Sheppo 9173af08d82Slm66018 if ((ldcp->tstate & ~TS_IN_RESET) == TS_VREADY) { 9183af08d82Slm66018 (void) i_ldc_txq_reconf(ldcp); 9193af08d82Slm66018 i_ldc_reset_state(ldcp); 9203af08d82Slm66018 mutex_exit(&ldcp->tx_lock); 9213af08d82Slm66018 return (EAGAIN); 9223af08d82Slm66018 } 9233af08d82Slm66018 9241ae08745Sheppo /* get the current tail and pkt for the response */ 9251ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 9261ae08745Sheppo if (rv != 0) { 9271ae08745Sheppo DWARN(ldcp->id, 9281ae08745Sheppo "i_ldc_process_VER: (0x%llx) err sending " 9291ae08745Sheppo "version ACK/NACK\n", ldcp->id); 9303af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 931d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 9321ae08745Sheppo return (ECONNRESET); 9331ae08745Sheppo } 9341ae08745Sheppo 9351ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 9361ae08745Sheppo ZERO_PKT(pkt); 9371ae08745Sheppo 9381ae08745Sheppo /* initialize the packet */ 9391ae08745Sheppo pkt->type = LDC_CTRL; 9401ae08745Sheppo pkt->ctrl = LDC_VER; 9411ae08745Sheppo 9421ae08745Sheppo for (;;) { 9431ae08745Sheppo 9441ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: got %u.%u chk %u.%u\n", 9451ae08745Sheppo rcvd_ver->major, rcvd_ver->minor, 9461ae08745Sheppo ldc_versions[idx].major, ldc_versions[idx].minor); 9471ae08745Sheppo 9481ae08745Sheppo if (rcvd_ver->major == ldc_versions[idx].major) { 9491ae08745Sheppo /* major version match - ACK version */ 9501ae08745Sheppo pkt->stype = LDC_ACK; 9511ae08745Sheppo 9521ae08745Sheppo /* 9531ae08745Sheppo * lower minor version to the one this endpt 9541ae08745Sheppo * supports, if necessary 9551ae08745Sheppo */ 9561ae08745Sheppo if (rcvd_ver->minor > ldc_versions[idx].minor) 9571ae08745Sheppo rcvd_ver->minor = 9581ae08745Sheppo ldc_versions[idx].minor; 9591ae08745Sheppo bcopy(rcvd_ver, pkt->udata, sizeof (*rcvd_ver)); 9601ae08745Sheppo 9611ae08745Sheppo break; 9621ae08745Sheppo } 9631ae08745Sheppo 9641ae08745Sheppo if (rcvd_ver->major > ldc_versions[idx].major) { 9651ae08745Sheppo 9661ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: using next" 9671ae08745Sheppo " lower idx=%d, v%u.%u\n", idx, 9681ae08745Sheppo ldc_versions[idx].major, 9691ae08745Sheppo ldc_versions[idx].minor); 9701ae08745Sheppo 9711ae08745Sheppo /* nack with next lower version */ 9721ae08745Sheppo pkt->stype = LDC_NACK; 9731ae08745Sheppo bcopy(&ldc_versions[idx], pkt->udata, 9741ae08745Sheppo sizeof (ldc_versions[idx])); 9751ae08745Sheppo ldcp->next_vidx = idx; 9761ae08745Sheppo break; 9771ae08745Sheppo } 9781ae08745Sheppo 9791ae08745Sheppo /* next major version */ 9801ae08745Sheppo idx++; 9811ae08745Sheppo 9821ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: inc idx %x\n", idx); 9831ae08745Sheppo 9841ae08745Sheppo if (idx == LDC_NUM_VERS) { 9851ae08745Sheppo /* no version match - send NACK */ 9861ae08745Sheppo pkt->stype = LDC_NACK; 9871ae08745Sheppo bzero(pkt->udata, sizeof (ldc_ver_t)); 9881ae08745Sheppo ldcp->next_vidx = 0; 9891ae08745Sheppo break; 9901ae08745Sheppo } 9911ae08745Sheppo } 9921ae08745Sheppo 9931ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 9941ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 9951ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 9961ae08745Sheppo 9971ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 9981ae08745Sheppo if (rv == 0) { 9991ae08745Sheppo ldcp->tx_tail = tx_tail; 10001ae08745Sheppo if (pkt->stype == LDC_ACK) { 10011ae08745Sheppo D2(ldcp->id, "i_ldc_process_VER: (0x%llx) sent" 10021ae08745Sheppo " version ACK\n", ldcp->id); 10031ae08745Sheppo /* Save the ACK'd version */ 10041ae08745Sheppo ldcp->version.major = rcvd_ver->major; 10051ae08745Sheppo ldcp->version.minor = rcvd_ver->minor; 10060a55fbb7Slm66018 ldcp->hstate |= TS_RCVD_VER; 10071ae08745Sheppo ldcp->tstate |= TS_VER_DONE; 100883d3bc6fSnarayan D1(DBG_ALL_LDCS, 10093af08d82Slm66018 "(0x%llx) Sent ACK, " 10103af08d82Slm66018 "Agreed on version v%u.%u\n", 10111ae08745Sheppo ldcp->id, rcvd_ver->major, rcvd_ver->minor); 10121ae08745Sheppo } 10131ae08745Sheppo } else { 10141ae08745Sheppo DWARN(ldcp->id, 10151ae08745Sheppo "i_ldc_process_VER: (0x%llx) error sending " 10161ae08745Sheppo "ACK/NACK\n", ldcp->id); 10173af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1018d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 10191ae08745Sheppo return (ECONNRESET); 10201ae08745Sheppo } 10211ae08745Sheppo 10221ae08745Sheppo break; 10231ae08745Sheppo 10241ae08745Sheppo case LDC_ACK: 10253af08d82Slm66018 if ((ldcp->tstate & ~TS_IN_RESET) == TS_VREADY) { 10263af08d82Slm66018 if (ldcp->version.major != rcvd_ver->major || 10273af08d82Slm66018 ldcp->version.minor != rcvd_ver->minor) { 10283af08d82Slm66018 10293af08d82Slm66018 /* mismatched version - reset connection */ 10303af08d82Slm66018 DWARN(ldcp->id, 10313af08d82Slm66018 "i_ldc_process_VER: (0x%llx) recvd" 10323af08d82Slm66018 " ACK ver != sent ACK ver\n", ldcp->id); 10333af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 10343af08d82Slm66018 mutex_exit(&ldcp->tx_lock); 10353af08d82Slm66018 return (ECONNRESET); 10363af08d82Slm66018 } 10373af08d82Slm66018 } else { 10381ae08745Sheppo /* SUCCESS - we have agreed on a version */ 10391ae08745Sheppo ldcp->version.major = rcvd_ver->major; 10401ae08745Sheppo ldcp->version.minor = rcvd_ver->minor; 10411ae08745Sheppo ldcp->tstate |= TS_VER_DONE; 10423af08d82Slm66018 } 10431ae08745Sheppo 1044cb112a14Slm66018 D1(ldcp->id, "(0x%llx) Got ACK, Agreed on version v%u.%u\n", 10451ae08745Sheppo ldcp->id, rcvd_ver->major, rcvd_ver->minor); 10461ae08745Sheppo 10471ae08745Sheppo /* initiate RTS-RTR-RDX handshake */ 10481ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 10491ae08745Sheppo if (rv) { 10501ae08745Sheppo DWARN(ldcp->id, 10511ae08745Sheppo "i_ldc_process_VER: (0x%llx) cannot send RTS\n", 10521ae08745Sheppo ldcp->id); 10533af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1054d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 10551ae08745Sheppo return (ECONNRESET); 10561ae08745Sheppo } 10571ae08745Sheppo 10581ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 10591ae08745Sheppo ZERO_PKT(pkt); 10601ae08745Sheppo 10611ae08745Sheppo pkt->type = LDC_CTRL; 10621ae08745Sheppo pkt->stype = LDC_INFO; 10631ae08745Sheppo pkt->ctrl = LDC_RTS; 10641ae08745Sheppo pkt->env = ldcp->mode; 10651ae08745Sheppo if (ldcp->mode != LDC_MODE_RAW) 10661ae08745Sheppo pkt->seqid = LDC_INIT_SEQID; 10671ae08745Sheppo 10681ae08745Sheppo ldcp->last_msg_rcd = LDC_INIT_SEQID; 10691ae08745Sheppo 10701ae08745Sheppo DUMP_LDC_PKT(ldcp, "i_ldc_process_VER snd rts", (uint64_t)pkt); 10711ae08745Sheppo 10721ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 10731ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 10741ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 10751ae08745Sheppo 10761ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 10771ae08745Sheppo if (rv) { 10781ae08745Sheppo D2(ldcp->id, 10791ae08745Sheppo "i_ldc_process_VER: (0x%llx) no listener\n", 10801ae08745Sheppo ldcp->id); 10813af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1082d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 10831ae08745Sheppo return (ECONNRESET); 10841ae08745Sheppo } 10851ae08745Sheppo 10861ae08745Sheppo ldcp->tx_tail = tx_tail; 10871ae08745Sheppo ldcp->hstate |= TS_SENT_RTS; 10881ae08745Sheppo 10891ae08745Sheppo break; 10901ae08745Sheppo 10911ae08745Sheppo case LDC_NACK: 10921ae08745Sheppo /* check if version in NACK is zero */ 10931ae08745Sheppo if (rcvd_ver->major == 0 && rcvd_ver->minor == 0) { 10941ae08745Sheppo /* version handshake failure */ 10951ae08745Sheppo DWARN(DBG_ALL_LDCS, 10961ae08745Sheppo "i_ldc_process_VER: (0x%llx) no version match\n", 10971ae08745Sheppo ldcp->id); 10983af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1099d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11001ae08745Sheppo return (ECONNRESET); 11011ae08745Sheppo } 11021ae08745Sheppo 11031ae08745Sheppo /* get the current tail and pkt for the response */ 11041ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 11051ae08745Sheppo if (rv != 0) { 11061ae08745Sheppo cmn_err(CE_NOTE, 11071ae08745Sheppo "i_ldc_process_VER: (0x%lx) err sending " 11081ae08745Sheppo "version ACK/NACK\n", ldcp->id); 11093af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1110d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11111ae08745Sheppo return (ECONNRESET); 11121ae08745Sheppo } 11131ae08745Sheppo 11141ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 11151ae08745Sheppo ZERO_PKT(pkt); 11161ae08745Sheppo 11171ae08745Sheppo /* initialize the packet */ 11181ae08745Sheppo pkt->type = LDC_CTRL; 11191ae08745Sheppo pkt->ctrl = LDC_VER; 11201ae08745Sheppo pkt->stype = LDC_INFO; 11211ae08745Sheppo 11221ae08745Sheppo /* check ver in NACK msg has a match */ 11231ae08745Sheppo for (;;) { 11241ae08745Sheppo if (rcvd_ver->major == ldc_versions[idx].major) { 11251ae08745Sheppo /* 11261ae08745Sheppo * major version match - resubmit request 11271ae08745Sheppo * if lower minor version to the one this endpt 11281ae08745Sheppo * supports, if necessary 11291ae08745Sheppo */ 11301ae08745Sheppo if (rcvd_ver->minor > ldc_versions[idx].minor) 11311ae08745Sheppo rcvd_ver->minor = 11321ae08745Sheppo ldc_versions[idx].minor; 11331ae08745Sheppo bcopy(rcvd_ver, pkt->udata, sizeof (*rcvd_ver)); 11341ae08745Sheppo break; 11351ae08745Sheppo 11361ae08745Sheppo } 11371ae08745Sheppo 11381ae08745Sheppo if (rcvd_ver->major > ldc_versions[idx].major) { 11391ae08745Sheppo 11401ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: using next" 11411ae08745Sheppo " lower idx=%d, v%u.%u\n", idx, 11421ae08745Sheppo ldc_versions[idx].major, 11431ae08745Sheppo ldc_versions[idx].minor); 11441ae08745Sheppo 11451ae08745Sheppo /* send next lower version */ 11461ae08745Sheppo bcopy(&ldc_versions[idx], pkt->udata, 11471ae08745Sheppo sizeof (ldc_versions[idx])); 11481ae08745Sheppo ldcp->next_vidx = idx; 11491ae08745Sheppo break; 11501ae08745Sheppo } 11511ae08745Sheppo 11521ae08745Sheppo /* next version */ 11531ae08745Sheppo idx++; 11541ae08745Sheppo 11551ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: inc idx %x\n", idx); 11561ae08745Sheppo 11571ae08745Sheppo if (idx == LDC_NUM_VERS) { 11581ae08745Sheppo /* no version match - terminate */ 11591ae08745Sheppo ldcp->next_vidx = 0; 1160d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11611ae08745Sheppo return (ECONNRESET); 11621ae08745Sheppo } 11631ae08745Sheppo } 11641ae08745Sheppo 11651ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 11661ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 11671ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 11681ae08745Sheppo 11691ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 11701ae08745Sheppo if (rv == 0) { 11711ae08745Sheppo D2(ldcp->id, "i_ldc_process_VER: (0x%llx) sent version" 11721ae08745Sheppo "INFO v%u.%u\n", ldcp->id, ldc_versions[idx].major, 11731ae08745Sheppo ldc_versions[idx].minor); 11741ae08745Sheppo ldcp->tx_tail = tx_tail; 11751ae08745Sheppo } else { 11761ae08745Sheppo cmn_err(CE_NOTE, 11771ae08745Sheppo "i_ldc_process_VER: (0x%lx) error sending version" 11781ae08745Sheppo "INFO\n", ldcp->id); 11793af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1180d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11811ae08745Sheppo return (ECONNRESET); 11821ae08745Sheppo } 11831ae08745Sheppo 11841ae08745Sheppo break; 11851ae08745Sheppo } 11861ae08745Sheppo 1187d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11881ae08745Sheppo return (rv); 11891ae08745Sheppo } 11901ae08745Sheppo 11911ae08745Sheppo 11921ae08745Sheppo /* 11931ae08745Sheppo * Process an incoming RTS ctrl message 11941ae08745Sheppo */ 11951ae08745Sheppo static int 11961ae08745Sheppo i_ldc_process_RTS(ldc_chan_t *ldcp, ldc_msg_t *msg) 11971ae08745Sheppo { 11981ae08745Sheppo int rv = 0; 11991ae08745Sheppo ldc_msg_t *pkt; 12001ae08745Sheppo uint64_t tx_tail; 12011ae08745Sheppo boolean_t sent_NACK = B_FALSE; 12021ae08745Sheppo 12031ae08745Sheppo D2(ldcp->id, "i_ldc_process_RTS: (0x%llx) received RTS\n", ldcp->id); 12041ae08745Sheppo 12051ae08745Sheppo switch (msg->stype) { 12061ae08745Sheppo case LDC_NACK: 12071ae08745Sheppo DWARN(ldcp->id, 12081ae08745Sheppo "i_ldc_process_RTS: (0x%llx) RTS NACK received\n", 12091ae08745Sheppo ldcp->id); 12101ae08745Sheppo 12111ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1212d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 12133af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1214d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 12151ae08745Sheppo rv = ECONNRESET; 12161ae08745Sheppo break; 12171ae08745Sheppo 12181ae08745Sheppo case LDC_INFO: 12191ae08745Sheppo 12201ae08745Sheppo /* check mode */ 12211ae08745Sheppo if (ldcp->mode != (ldc_mode_t)msg->env) { 12221ae08745Sheppo cmn_err(CE_NOTE, 12231ae08745Sheppo "i_ldc_process_RTS: (0x%lx) mode mismatch\n", 12241ae08745Sheppo ldcp->id); 12251ae08745Sheppo /* 12261ae08745Sheppo * send NACK in response to MODE message 12271ae08745Sheppo * get the current tail for the response 12281ae08745Sheppo */ 12291ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, LDC_RTS); 12301ae08745Sheppo if (rv) { 12311ae08745Sheppo /* if cannot send NACK - reset channel */ 1232d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 12333af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1234d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 12351ae08745Sheppo rv = ECONNRESET; 12361ae08745Sheppo break; 12371ae08745Sheppo } 12381ae08745Sheppo sent_NACK = B_TRUE; 12391ae08745Sheppo } 12401ae08745Sheppo break; 12411ae08745Sheppo default: 12421ae08745Sheppo DWARN(ldcp->id, "i_ldc_process_RTS: (0x%llx) unexp ACK\n", 12431ae08745Sheppo ldcp->id); 1244d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 12453af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1246d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 12471ae08745Sheppo rv = ECONNRESET; 12481ae08745Sheppo break; 12491ae08745Sheppo } 12501ae08745Sheppo 12511ae08745Sheppo /* 12521ae08745Sheppo * If either the connection was reset (when rv != 0) or 12531ae08745Sheppo * a NACK was sent, we return. In the case of a NACK 12541ae08745Sheppo * we dont want to consume the packet that came in but 12551ae08745Sheppo * not record that we received the RTS 12561ae08745Sheppo */ 12571ae08745Sheppo if (rv || sent_NACK) 12581ae08745Sheppo return (rv); 12591ae08745Sheppo 12601ae08745Sheppo /* record RTS received */ 12611ae08745Sheppo ldcp->hstate |= TS_RCVD_RTS; 12621ae08745Sheppo 12631ae08745Sheppo /* store initial SEQID info */ 12641ae08745Sheppo ldcp->last_msg_snt = msg->seqid; 12651ae08745Sheppo 1266d10e4ef2Snarayan /* Obtain Tx lock */ 1267d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 1268d10e4ef2Snarayan 12691ae08745Sheppo /* get the current tail for the response */ 12701ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 12711ae08745Sheppo if (rv != 0) { 12721ae08745Sheppo cmn_err(CE_NOTE, 12731ae08745Sheppo "i_ldc_process_RTS: (0x%lx) err sending RTR\n", 12741ae08745Sheppo ldcp->id); 12753af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1276d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 12771ae08745Sheppo return (ECONNRESET); 12781ae08745Sheppo } 12791ae08745Sheppo 12801ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 12811ae08745Sheppo ZERO_PKT(pkt); 12821ae08745Sheppo 12831ae08745Sheppo /* initialize the packet */ 12841ae08745Sheppo pkt->type = LDC_CTRL; 12851ae08745Sheppo pkt->stype = LDC_INFO; 12861ae08745Sheppo pkt->ctrl = LDC_RTR; 12871ae08745Sheppo pkt->env = ldcp->mode; 12881ae08745Sheppo if (ldcp->mode != LDC_MODE_RAW) 12891ae08745Sheppo pkt->seqid = LDC_INIT_SEQID; 12901ae08745Sheppo 12911ae08745Sheppo ldcp->last_msg_rcd = msg->seqid; 12921ae08745Sheppo 12931ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 12941ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 12951ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 12961ae08745Sheppo 12971ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 12981ae08745Sheppo if (rv == 0) { 12991ae08745Sheppo D2(ldcp->id, 13001ae08745Sheppo "i_ldc_process_RTS: (0x%llx) sent RTR\n", ldcp->id); 13011ae08745Sheppo DUMP_LDC_PKT(ldcp, "i_ldc_process_RTS sent rtr", (uint64_t)pkt); 13021ae08745Sheppo 13031ae08745Sheppo ldcp->tx_tail = tx_tail; 13041ae08745Sheppo ldcp->hstate |= TS_SENT_RTR; 13051ae08745Sheppo 13061ae08745Sheppo } else { 13071ae08745Sheppo cmn_err(CE_NOTE, 13081ae08745Sheppo "i_ldc_process_RTS: (0x%lx) error sending RTR\n", 13091ae08745Sheppo ldcp->id); 13103af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1311d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 13121ae08745Sheppo return (ECONNRESET); 13131ae08745Sheppo } 13141ae08745Sheppo 1315d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 13161ae08745Sheppo return (0); 13171ae08745Sheppo } 13181ae08745Sheppo 13191ae08745Sheppo /* 13201ae08745Sheppo * Process an incoming RTR ctrl message 13211ae08745Sheppo */ 13221ae08745Sheppo static int 13231ae08745Sheppo i_ldc_process_RTR(ldc_chan_t *ldcp, ldc_msg_t *msg) 13241ae08745Sheppo { 13251ae08745Sheppo int rv = 0; 13261ae08745Sheppo boolean_t sent_NACK = B_FALSE; 13271ae08745Sheppo 13281ae08745Sheppo D2(ldcp->id, "i_ldc_process_RTR: (0x%llx) received RTR\n", ldcp->id); 13291ae08745Sheppo 13301ae08745Sheppo switch (msg->stype) { 13311ae08745Sheppo case LDC_NACK: 13321ae08745Sheppo /* RTR NACK received */ 13331ae08745Sheppo DWARN(ldcp->id, 13341ae08745Sheppo "i_ldc_process_RTR: (0x%llx) RTR NACK received\n", 13351ae08745Sheppo ldcp->id); 13361ae08745Sheppo 13371ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1338d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 13393af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1340d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 13411ae08745Sheppo rv = ECONNRESET; 13421ae08745Sheppo 13431ae08745Sheppo break; 13441ae08745Sheppo 13451ae08745Sheppo case LDC_INFO: 13461ae08745Sheppo 13471ae08745Sheppo /* check mode */ 13481ae08745Sheppo if (ldcp->mode != (ldc_mode_t)msg->env) { 13491ae08745Sheppo DWARN(ldcp->id, 1350cb112a14Slm66018 "i_ldc_process_RTR: (0x%llx) mode mismatch, " 1351cb112a14Slm66018 "expecting 0x%x, got 0x%x\n", 1352cb112a14Slm66018 ldcp->id, ldcp->mode, (ldc_mode_t)msg->env); 13531ae08745Sheppo /* 13541ae08745Sheppo * send NACK in response to MODE message 13551ae08745Sheppo * get the current tail for the response 13561ae08745Sheppo */ 13571ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, LDC_RTR); 13581ae08745Sheppo if (rv) { 13591ae08745Sheppo /* if cannot send NACK - reset channel */ 1360d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 13613af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1362d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 13631ae08745Sheppo rv = ECONNRESET; 13641ae08745Sheppo break; 13651ae08745Sheppo } 13661ae08745Sheppo sent_NACK = B_TRUE; 13671ae08745Sheppo } 13681ae08745Sheppo break; 13691ae08745Sheppo 13701ae08745Sheppo default: 13711ae08745Sheppo DWARN(ldcp->id, "i_ldc_process_RTR: (0x%llx) unexp ACK\n", 13721ae08745Sheppo ldcp->id); 13731ae08745Sheppo 13741ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1375d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 13763af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1377d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 13781ae08745Sheppo rv = ECONNRESET; 13791ae08745Sheppo break; 13801ae08745Sheppo } 13811ae08745Sheppo 13821ae08745Sheppo /* 13831ae08745Sheppo * If either the connection was reset (when rv != 0) or 13841ae08745Sheppo * a NACK was sent, we return. In the case of a NACK 13851ae08745Sheppo * we dont want to consume the packet that came in but 13861ae08745Sheppo * not record that we received the RTR 13871ae08745Sheppo */ 13881ae08745Sheppo if (rv || sent_NACK) 13891ae08745Sheppo return (rv); 13901ae08745Sheppo 13911ae08745Sheppo ldcp->last_msg_snt = msg->seqid; 13921ae08745Sheppo ldcp->hstate |= TS_RCVD_RTR; 13931ae08745Sheppo 13941ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_INFO, LDC_RDX); 13951ae08745Sheppo if (rv) { 13961ae08745Sheppo cmn_err(CE_NOTE, 13971ae08745Sheppo "i_ldc_process_RTR: (0x%lx) cannot send RDX\n", 13981ae08745Sheppo ldcp->id); 1399d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 14003af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1401d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 14021ae08745Sheppo return (ECONNRESET); 14031ae08745Sheppo } 14041ae08745Sheppo D2(ldcp->id, 14051ae08745Sheppo "i_ldc_process_RTR: (0x%llx) sent RDX\n", ldcp->id); 14061ae08745Sheppo 14071ae08745Sheppo ldcp->hstate |= TS_SENT_RDX; 14081ae08745Sheppo ldcp->tstate |= TS_HSHAKE_DONE; 14093af08d82Slm66018 if ((ldcp->tstate & TS_IN_RESET) == 0) 14101ae08745Sheppo ldcp->status = LDC_UP; 14111ae08745Sheppo 1412cb112a14Slm66018 D1(ldcp->id, "(0x%llx) Handshake Complete\n", ldcp->id); 14131ae08745Sheppo 14141ae08745Sheppo return (0); 14151ae08745Sheppo } 14161ae08745Sheppo 14171ae08745Sheppo 14181ae08745Sheppo /* 14191ae08745Sheppo * Process an incoming RDX ctrl message 14201ae08745Sheppo */ 14211ae08745Sheppo static int 14221ae08745Sheppo i_ldc_process_RDX(ldc_chan_t *ldcp, ldc_msg_t *msg) 14231ae08745Sheppo { 14241ae08745Sheppo int rv = 0; 14251ae08745Sheppo 14261ae08745Sheppo D2(ldcp->id, "i_ldc_process_RDX: (0x%llx) received RDX\n", ldcp->id); 14271ae08745Sheppo 14281ae08745Sheppo switch (msg->stype) { 14291ae08745Sheppo case LDC_NACK: 14301ae08745Sheppo /* RDX NACK received */ 14311ae08745Sheppo DWARN(ldcp->id, 14321ae08745Sheppo "i_ldc_process_RDX: (0x%llx) RDX NACK received\n", 14331ae08745Sheppo ldcp->id); 14341ae08745Sheppo 14351ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1436d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 14373af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1438d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 14391ae08745Sheppo rv = ECONNRESET; 14401ae08745Sheppo 14411ae08745Sheppo break; 14421ae08745Sheppo 14431ae08745Sheppo case LDC_INFO: 14441ae08745Sheppo 14451ae08745Sheppo /* 14461ae08745Sheppo * if channel is UP and a RDX received after data transmission 14471ae08745Sheppo * has commenced it is an error 14481ae08745Sheppo */ 14491ae08745Sheppo if ((ldcp->tstate == TS_UP) && (ldcp->hstate & TS_RCVD_RDX)) { 14501ae08745Sheppo DWARN(DBG_ALL_LDCS, 14511ae08745Sheppo "i_ldc_process_RDX: (0x%llx) unexpected RDX" 14521ae08745Sheppo " - LDC reset\n", ldcp->id); 1453d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 14543af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1455d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 14561ae08745Sheppo return (ECONNRESET); 14571ae08745Sheppo } 14581ae08745Sheppo 14591ae08745Sheppo ldcp->hstate |= TS_RCVD_RDX; 14601ae08745Sheppo ldcp->tstate |= TS_HSHAKE_DONE; 14613af08d82Slm66018 if ((ldcp->tstate & TS_IN_RESET) == 0) 14621ae08745Sheppo ldcp->status = LDC_UP; 14631ae08745Sheppo 14641ae08745Sheppo D1(DBG_ALL_LDCS, "(0x%llx) Handshake Complete\n", ldcp->id); 14651ae08745Sheppo break; 14661ae08745Sheppo 14671ae08745Sheppo default: 14681ae08745Sheppo DWARN(ldcp->id, "i_ldc_process_RDX: (0x%llx) unexp ACK\n", 14691ae08745Sheppo ldcp->id); 14701ae08745Sheppo 14711ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1472d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 14733af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1474d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 14751ae08745Sheppo rv = ECONNRESET; 14761ae08745Sheppo break; 14771ae08745Sheppo } 14781ae08745Sheppo 14791ae08745Sheppo return (rv); 14801ae08745Sheppo } 14811ae08745Sheppo 14821ae08745Sheppo /* 14831ae08745Sheppo * Process an incoming ACK for a data packet 14841ae08745Sheppo */ 14851ae08745Sheppo static int 14861ae08745Sheppo i_ldc_process_data_ACK(ldc_chan_t *ldcp, ldc_msg_t *msg) 14871ae08745Sheppo { 14881ae08745Sheppo int rv; 14891ae08745Sheppo uint64_t tx_head; 14901ae08745Sheppo ldc_msg_t *pkt; 14911ae08745Sheppo 1492d10e4ef2Snarayan /* Obtain Tx lock */ 1493d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 1494d10e4ef2Snarayan 14951ae08745Sheppo /* 1496d10e4ef2Snarayan * Read the current Tx head and tail 14971ae08745Sheppo */ 14981ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 14991ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 15001ae08745Sheppo if (rv != 0) { 15011ae08745Sheppo cmn_err(CE_WARN, 15021ae08745Sheppo "i_ldc_process_data_ACK: (0x%lx) cannot read qptrs\n", 15031ae08745Sheppo ldcp->id); 1504d10e4ef2Snarayan 1505d10e4ef2Snarayan /* Reset the channel -- as we cannot continue */ 15063af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1507d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 1508d10e4ef2Snarayan return (ECONNRESET); 15091ae08745Sheppo } 15101ae08745Sheppo 15111ae08745Sheppo /* 15121ae08745Sheppo * loop from where the previous ACK location was to the 15131ae08745Sheppo * current head location. This is how far the HV has 15141ae08745Sheppo * actually send pkts. Pkts between head and tail are 15151ae08745Sheppo * yet to be sent by HV. 15161ae08745Sheppo */ 15171ae08745Sheppo tx_head = ldcp->tx_ackd_head; 15181ae08745Sheppo for (;;) { 15191ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_head); 15201ae08745Sheppo tx_head = (tx_head + LDC_PACKET_SIZE) % 15211ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 15221ae08745Sheppo 15231ae08745Sheppo if (pkt->seqid == msg->ackid) { 15241ae08745Sheppo D2(ldcp->id, 15251ae08745Sheppo "i_ldc_process_data_ACK: (0x%llx) found packet\n", 15261ae08745Sheppo ldcp->id); 15271ae08745Sheppo ldcp->last_ack_rcd = msg->ackid; 15281ae08745Sheppo ldcp->tx_ackd_head = tx_head; 15291ae08745Sheppo break; 15301ae08745Sheppo } 15311ae08745Sheppo if (tx_head == ldcp->tx_head) { 15321ae08745Sheppo /* could not find packet */ 15331ae08745Sheppo DWARN(ldcp->id, 15341ae08745Sheppo "i_ldc_process_data_ACK: (0x%llx) invalid ACKid\n", 15351ae08745Sheppo ldcp->id); 1536d10e4ef2Snarayan 1537d10e4ef2Snarayan /* Reset the channel -- as we cannot continue */ 15383af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 1539d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 1540d10e4ef2Snarayan return (ECONNRESET); 15411ae08745Sheppo } 15421ae08745Sheppo } 15431ae08745Sheppo 1544d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 15451ae08745Sheppo return (0); 15461ae08745Sheppo } 15471ae08745Sheppo 15481ae08745Sheppo /* 15491ae08745Sheppo * Process incoming control message 15501ae08745Sheppo * Return 0 - session can continue 15511ae08745Sheppo * EAGAIN - reprocess packet - state was changed 15521ae08745Sheppo * ECONNRESET - channel was reset 15531ae08745Sheppo */ 15541ae08745Sheppo static int 15551ae08745Sheppo i_ldc_ctrlmsg(ldc_chan_t *ldcp, ldc_msg_t *msg) 15561ae08745Sheppo { 15571ae08745Sheppo int rv = 0; 15581ae08745Sheppo 15593af08d82Slm66018 D1(ldcp->id, "i_ldc_ctrlmsg: (%llx) tstate = %lx, hstate = %lx\n", 15603af08d82Slm66018 ldcp->id, ldcp->tstate, ldcp->hstate); 15613af08d82Slm66018 15623af08d82Slm66018 switch (ldcp->tstate & ~TS_IN_RESET) { 15631ae08745Sheppo 15641ae08745Sheppo case TS_OPEN: 15651ae08745Sheppo case TS_READY: 15661ae08745Sheppo 15671ae08745Sheppo switch (msg->ctrl & LDC_CTRL_MASK) { 15681ae08745Sheppo case LDC_VER: 15691ae08745Sheppo /* process version message */ 15701ae08745Sheppo rv = i_ldc_process_VER(ldcp, msg); 15711ae08745Sheppo break; 15721ae08745Sheppo default: 15731ae08745Sheppo DWARN(ldcp->id, 15741ae08745Sheppo "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 15751ae08745Sheppo "tstate=0x%x\n", ldcp->id, 15761ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 15771ae08745Sheppo break; 15781ae08745Sheppo } 15791ae08745Sheppo 15801ae08745Sheppo break; 15811ae08745Sheppo 15821ae08745Sheppo case TS_VREADY: 15831ae08745Sheppo 15841ae08745Sheppo switch (msg->ctrl & LDC_CTRL_MASK) { 15851ae08745Sheppo case LDC_VER: 15863af08d82Slm66018 /* process version message */ 15873af08d82Slm66018 rv = i_ldc_process_VER(ldcp, msg); 15881ae08745Sheppo break; 15891ae08745Sheppo case LDC_RTS: 15901ae08745Sheppo /* process RTS message */ 15911ae08745Sheppo rv = i_ldc_process_RTS(ldcp, msg); 15921ae08745Sheppo break; 15931ae08745Sheppo case LDC_RTR: 15941ae08745Sheppo /* process RTR message */ 15951ae08745Sheppo rv = i_ldc_process_RTR(ldcp, msg); 15961ae08745Sheppo break; 15971ae08745Sheppo case LDC_RDX: 15981ae08745Sheppo /* process RDX message */ 15991ae08745Sheppo rv = i_ldc_process_RDX(ldcp, msg); 16001ae08745Sheppo break; 16011ae08745Sheppo default: 16021ae08745Sheppo DWARN(ldcp->id, 16031ae08745Sheppo "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 16041ae08745Sheppo "tstate=0x%x\n", ldcp->id, 16051ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 16061ae08745Sheppo break; 16071ae08745Sheppo } 16081ae08745Sheppo 16091ae08745Sheppo break; 16101ae08745Sheppo 16111ae08745Sheppo case TS_UP: 16121ae08745Sheppo 16131ae08745Sheppo switch (msg->ctrl & LDC_CTRL_MASK) { 16141ae08745Sheppo case LDC_VER: 16151ae08745Sheppo DWARN(ldcp->id, 16161ae08745Sheppo "i_ldc_ctrlmsg: (0x%llx) unexpected VER " 16171ae08745Sheppo "- LDC reset\n", ldcp->id); 16181ae08745Sheppo /* peer is redoing version negotiation */ 1619d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 16201ae08745Sheppo (void) i_ldc_txq_reconf(ldcp); 16211ae08745Sheppo i_ldc_reset_state(ldcp); 1622d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 16231ae08745Sheppo rv = EAGAIN; 16241ae08745Sheppo break; 16251ae08745Sheppo 16261ae08745Sheppo case LDC_RDX: 16271ae08745Sheppo /* process RDX message */ 16281ae08745Sheppo rv = i_ldc_process_RDX(ldcp, msg); 16291ae08745Sheppo break; 16301ae08745Sheppo 16311ae08745Sheppo default: 16321ae08745Sheppo DWARN(ldcp->id, 16331ae08745Sheppo "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 16341ae08745Sheppo "tstate=0x%x\n", ldcp->id, 16351ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 16361ae08745Sheppo break; 16371ae08745Sheppo } 16381ae08745Sheppo } 16391ae08745Sheppo 16401ae08745Sheppo return (rv); 16411ae08745Sheppo } 16421ae08745Sheppo 16431ae08745Sheppo /* 16441ae08745Sheppo * Register channel with the channel nexus 16451ae08745Sheppo */ 16461ae08745Sheppo static int 16471ae08745Sheppo i_ldc_register_channel(ldc_chan_t *ldcp) 16481ae08745Sheppo { 16491ae08745Sheppo int rv = 0; 16501ae08745Sheppo ldc_cnex_t *cinfo = &ldcssp->cinfo; 16511ae08745Sheppo 16521ae08745Sheppo if (cinfo->dip == NULL) { 16531ae08745Sheppo DWARN(ldcp->id, 16541ae08745Sheppo "i_ldc_register_channel: cnex has not registered\n"); 16551ae08745Sheppo return (EAGAIN); 16561ae08745Sheppo } 16571ae08745Sheppo 16581ae08745Sheppo rv = cinfo->reg_chan(cinfo->dip, ldcp->id, ldcp->devclass); 16591ae08745Sheppo if (rv) { 16601ae08745Sheppo DWARN(ldcp->id, 16611ae08745Sheppo "i_ldc_register_channel: cannot register channel\n"); 16621ae08745Sheppo return (rv); 16631ae08745Sheppo } 16641ae08745Sheppo 16651ae08745Sheppo rv = cinfo->add_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR, 16661ae08745Sheppo i_ldc_tx_hdlr, ldcp, NULL); 16671ae08745Sheppo if (rv) { 16681ae08745Sheppo DWARN(ldcp->id, 16691ae08745Sheppo "i_ldc_register_channel: cannot add Tx interrupt\n"); 16701ae08745Sheppo (void) cinfo->unreg_chan(cinfo->dip, ldcp->id); 16711ae08745Sheppo return (rv); 16721ae08745Sheppo } 16731ae08745Sheppo 16741ae08745Sheppo rv = cinfo->add_intr(cinfo->dip, ldcp->id, CNEX_RX_INTR, 16751ae08745Sheppo i_ldc_rx_hdlr, ldcp, NULL); 16761ae08745Sheppo if (rv) { 16771ae08745Sheppo DWARN(ldcp->id, 16781ae08745Sheppo "i_ldc_register_channel: cannot add Rx interrupt\n"); 16791ae08745Sheppo (void) cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR); 16801ae08745Sheppo (void) cinfo->unreg_chan(cinfo->dip, ldcp->id); 16811ae08745Sheppo return (rv); 16821ae08745Sheppo } 16831ae08745Sheppo 16841ae08745Sheppo ldcp->tstate |= TS_CNEX_RDY; 16851ae08745Sheppo 16861ae08745Sheppo return (0); 16871ae08745Sheppo } 16881ae08745Sheppo 16891ae08745Sheppo /* 16901ae08745Sheppo * Unregister a channel with the channel nexus 16911ae08745Sheppo */ 16921ae08745Sheppo static int 16931ae08745Sheppo i_ldc_unregister_channel(ldc_chan_t *ldcp) 16941ae08745Sheppo { 16951ae08745Sheppo int rv = 0; 16961ae08745Sheppo ldc_cnex_t *cinfo = &ldcssp->cinfo; 16971ae08745Sheppo 16981ae08745Sheppo if (cinfo->dip == NULL) { 16991ae08745Sheppo DWARN(ldcp->id, 17001ae08745Sheppo "i_ldc_unregister_channel: cnex has not registered\n"); 17011ae08745Sheppo return (EAGAIN); 17021ae08745Sheppo } 17031ae08745Sheppo 17041ae08745Sheppo if (ldcp->tstate & TS_CNEX_RDY) { 17051ae08745Sheppo 1706d10e4ef2Snarayan /* Remove the Rx interrupt */ 17071ae08745Sheppo rv = cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_RX_INTR); 17081ae08745Sheppo if (rv) { 17093af08d82Slm66018 if (rv != EAGAIN) { 17101ae08745Sheppo DWARN(ldcp->id, 17113af08d82Slm66018 "i_ldc_unregister_channel: err removing " 17123af08d82Slm66018 "Rx intr\n"); 1713d10e4ef2Snarayan return (rv); 17141ae08745Sheppo } 1715d10e4ef2Snarayan 17163af08d82Slm66018 /* 17173af08d82Slm66018 * If interrupts are pending and handler has 17183af08d82Slm66018 * finished running, clear interrupt and try 17193af08d82Slm66018 * again 17203af08d82Slm66018 */ 17213af08d82Slm66018 if (ldcp->rx_intr_state != LDC_INTR_PEND) 17223af08d82Slm66018 return (rv); 17233af08d82Slm66018 17243af08d82Slm66018 (void) i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 17253af08d82Slm66018 rv = cinfo->rem_intr(cinfo->dip, ldcp->id, 17263af08d82Slm66018 CNEX_RX_INTR); 17273af08d82Slm66018 if (rv) { 17283af08d82Slm66018 DWARN(ldcp->id, "i_ldc_unregister_channel: " 17293af08d82Slm66018 "err removing Rx interrupt\n"); 17303af08d82Slm66018 return (rv); 17313af08d82Slm66018 } 17323af08d82Slm66018 } 17333af08d82Slm66018 1734d10e4ef2Snarayan /* Remove the Tx interrupt */ 17351ae08745Sheppo rv = cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR); 17361ae08745Sheppo if (rv) { 17371ae08745Sheppo DWARN(ldcp->id, 17381ae08745Sheppo "i_ldc_unregister_channel: err removing Tx intr\n"); 1739d10e4ef2Snarayan return (rv); 17401ae08745Sheppo } 1741d10e4ef2Snarayan 1742d10e4ef2Snarayan /* Unregister the channel */ 17431ae08745Sheppo rv = cinfo->unreg_chan(ldcssp->cinfo.dip, ldcp->id); 17441ae08745Sheppo if (rv) { 17451ae08745Sheppo DWARN(ldcp->id, 17461ae08745Sheppo "i_ldc_unregister_channel: cannot unreg channel\n"); 1747d10e4ef2Snarayan return (rv); 17481ae08745Sheppo } 17491ae08745Sheppo 17501ae08745Sheppo ldcp->tstate &= ~TS_CNEX_RDY; 17511ae08745Sheppo } 17521ae08745Sheppo 17531ae08745Sheppo return (0); 17541ae08745Sheppo } 17551ae08745Sheppo 17561ae08745Sheppo 17571ae08745Sheppo /* 17581ae08745Sheppo * LDC transmit interrupt handler 17591ae08745Sheppo * triggered for chanel up/down/reset events 17601ae08745Sheppo * and Tx queue content changes 17611ae08745Sheppo */ 17621ae08745Sheppo static uint_t 17631ae08745Sheppo i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2) 17641ae08745Sheppo { 17651ae08745Sheppo _NOTE(ARGUNUSED(arg2)) 17661ae08745Sheppo 17671ae08745Sheppo int rv; 17681ae08745Sheppo ldc_chan_t *ldcp; 17691ae08745Sheppo boolean_t notify_client = B_FALSE; 17703af08d82Slm66018 uint64_t notify_event = 0, link_state; 17711ae08745Sheppo 17721ae08745Sheppo /* Get the channel for which interrupt was received */ 17731ae08745Sheppo ASSERT(arg1 != NULL); 17741ae08745Sheppo ldcp = (ldc_chan_t *)arg1; 17751ae08745Sheppo 17761ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n", 17771ae08745Sheppo ldcp->id, ldcp); 17781ae08745Sheppo 17791ae08745Sheppo /* Lock channel */ 17801ae08745Sheppo mutex_enter(&ldcp->lock); 17811ae08745Sheppo 1782d10e4ef2Snarayan /* Obtain Tx lock */ 1783d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 1784d10e4ef2Snarayan 17854bac2208Snarayan /* mark interrupt as pending */ 17863af08d82Slm66018 ldcp->tx_intr_state = LDC_INTR_ACTIVE; 17873af08d82Slm66018 17883af08d82Slm66018 /* save current link state */ 17893af08d82Slm66018 link_state = ldcp->link_state; 17904bac2208Snarayan 17911ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, &ldcp->tx_head, &ldcp->tx_tail, 17921ae08745Sheppo &ldcp->link_state); 17931ae08745Sheppo if (rv) { 17941ae08745Sheppo cmn_err(CE_WARN, 17951ae08745Sheppo "i_ldc_tx_hdlr: (0x%lx) cannot read queue ptrs rv=0x%d\n", 17961ae08745Sheppo ldcp->id, rv); 17974bac2208Snarayan i_ldc_clear_intr(ldcp, CNEX_TX_INTR); 1798d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 17991ae08745Sheppo mutex_exit(&ldcp->lock); 18001ae08745Sheppo return (DDI_INTR_CLAIMED); 18011ae08745Sheppo } 18021ae08745Sheppo 18031ae08745Sheppo /* 18041ae08745Sheppo * reset the channel state if the channel went down 18051ae08745Sheppo * (other side unconfigured queue) or channel was reset 18061ae08745Sheppo * (other side reconfigured its queue) 18071ae08745Sheppo */ 18083af08d82Slm66018 if (link_state != ldcp->link_state && 18093af08d82Slm66018 ldcp->link_state == LDC_CHANNEL_DOWN) { 18101ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: channel link down\n", ldcp->id); 18113af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 18121ae08745Sheppo notify_client = B_TRUE; 18131ae08745Sheppo notify_event = LDC_EVT_DOWN; 18141ae08745Sheppo } 18151ae08745Sheppo 18163af08d82Slm66018 if (link_state != ldcp->link_state && 18173af08d82Slm66018 ldcp->link_state == LDC_CHANNEL_RESET) { 18181ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: channel link reset\n", ldcp->id); 18193af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 18201ae08745Sheppo notify_client = B_TRUE; 18211ae08745Sheppo notify_event = LDC_EVT_RESET; 18221ae08745Sheppo } 18231ae08745Sheppo 18243af08d82Slm66018 if (link_state != ldcp->link_state && 18253af08d82Slm66018 (ldcp->tstate & ~TS_IN_RESET) == TS_OPEN && 18263af08d82Slm66018 ldcp->link_state == LDC_CHANNEL_UP) { 18271ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: channel link up\n", ldcp->id); 18281ae08745Sheppo notify_client = B_TRUE; 18291ae08745Sheppo notify_event = LDC_EVT_RESET; 18301ae08745Sheppo ldcp->tstate |= TS_LINK_READY; 18311ae08745Sheppo ldcp->status = LDC_READY; 18321ae08745Sheppo } 18331ae08745Sheppo 18341ae08745Sheppo /* if callbacks are disabled, do not notify */ 18351ae08745Sheppo if (!ldcp->cb_enabled) 18361ae08745Sheppo notify_client = B_FALSE; 18371ae08745Sheppo 18384d39be2bSsg70180 i_ldc_clear_intr(ldcp, CNEX_TX_INTR); 18391ae08745Sheppo 18401ae08745Sheppo if (notify_client) { 18413af08d82Slm66018 ldcp->cb_inprogress = B_TRUE; 18423af08d82Slm66018 mutex_exit(&ldcp->tx_lock); 18433af08d82Slm66018 mutex_exit(&ldcp->lock); 18441ae08745Sheppo rv = ldcp->cb(notify_event, ldcp->cb_arg); 18451ae08745Sheppo if (rv) { 18461ae08745Sheppo DWARN(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) callback " 18471ae08745Sheppo "failure", ldcp->id); 18481ae08745Sheppo } 18491ae08745Sheppo mutex_enter(&ldcp->lock); 18501ae08745Sheppo ldcp->cb_inprogress = B_FALSE; 18511ae08745Sheppo } 18521ae08745Sheppo 18531ae08745Sheppo mutex_exit(&ldcp->lock); 18541ae08745Sheppo 18551ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) exiting handler", ldcp->id); 18561ae08745Sheppo 18571ae08745Sheppo return (DDI_INTR_CLAIMED); 18581ae08745Sheppo } 18591ae08745Sheppo 18601ae08745Sheppo /* 18611ae08745Sheppo * LDC receive interrupt handler 18621ae08745Sheppo * triggered for channel with data pending to read 18631ae08745Sheppo * i.e. Rx queue content changes 18641ae08745Sheppo */ 18651ae08745Sheppo static uint_t 18661ae08745Sheppo i_ldc_rx_hdlr(caddr_t arg1, caddr_t arg2) 18671ae08745Sheppo { 18681ae08745Sheppo _NOTE(ARGUNUSED(arg2)) 18691ae08745Sheppo 18701ae08745Sheppo int rv; 18711ae08745Sheppo uint64_t rx_head, rx_tail; 18721ae08745Sheppo ldc_msg_t *msg; 18731ae08745Sheppo ldc_chan_t *ldcp; 18741ae08745Sheppo boolean_t notify_client = B_FALSE; 18751ae08745Sheppo uint64_t notify_event = 0; 18763af08d82Slm66018 uint64_t link_state, first_fragment = 0; 18773af08d82Slm66018 18781ae08745Sheppo 18791ae08745Sheppo /* Get the channel for which interrupt was received */ 18801ae08745Sheppo if (arg1 == NULL) { 18811ae08745Sheppo cmn_err(CE_WARN, "i_ldc_rx_hdlr: invalid arg\n"); 18821ae08745Sheppo return (DDI_INTR_UNCLAIMED); 18831ae08745Sheppo } 18841ae08745Sheppo 18851ae08745Sheppo ldcp = (ldc_chan_t *)arg1; 18861ae08745Sheppo 18871ae08745Sheppo D1(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n", 18881ae08745Sheppo ldcp->id, ldcp); 18893af08d82Slm66018 D1(ldcp->id, "i_ldc_rx_hdlr: (%llx) USR%lx/TS%lx/HS%lx, LSTATE=%lx\n", 18903af08d82Slm66018 ldcp->id, ldcp->status, ldcp->tstate, ldcp->hstate, 18913af08d82Slm66018 ldcp->link_state); 18921ae08745Sheppo 18931ae08745Sheppo /* Lock channel */ 18941ae08745Sheppo mutex_enter(&ldcp->lock); 18951ae08745Sheppo 18961ae08745Sheppo /* mark interrupt as pending */ 18973af08d82Slm66018 ldcp->rx_intr_state = LDC_INTR_ACTIVE; 18981ae08745Sheppo 18991ae08745Sheppo /* 19001ae08745Sheppo * Read packet(s) from the queue 19011ae08745Sheppo */ 19021ae08745Sheppo for (;;) { 19031ae08745Sheppo 19043af08d82Slm66018 link_state = ldcp->link_state; 19051ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 19061ae08745Sheppo &ldcp->link_state); 19071ae08745Sheppo if (rv) { 19081ae08745Sheppo cmn_err(CE_WARN, 19091ae08745Sheppo "i_ldc_rx_hdlr: (0x%lx) cannot read " 19101ae08745Sheppo "queue ptrs, rv=0x%d\n", ldcp->id, rv); 19111ae08745Sheppo i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 19121ae08745Sheppo mutex_exit(&ldcp->lock); 19131ae08745Sheppo return (DDI_INTR_CLAIMED); 19141ae08745Sheppo } 19151ae08745Sheppo 19161ae08745Sheppo /* 19171ae08745Sheppo * reset the channel state if the channel went down 19181ae08745Sheppo * (other side unconfigured queue) or channel was reset 19193af08d82Slm66018 * (other side reconfigured its queue) 19201ae08745Sheppo */ 19213af08d82Slm66018 19223af08d82Slm66018 if (link_state != ldcp->link_state) { 1923cb112a14Slm66018 19243af08d82Slm66018 switch (ldcp->link_state) { 19253af08d82Slm66018 case LDC_CHANNEL_DOWN: 19263af08d82Slm66018 D1(ldcp->id, "i_ldc_rx_hdlr: channel " 19273af08d82Slm66018 "link down\n", ldcp->id); 1928d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 19293af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 1930d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 19311ae08745Sheppo notify_client = B_TRUE; 19321ae08745Sheppo notify_event = LDC_EVT_DOWN; 19333af08d82Slm66018 goto loop_exit; 19341ae08745Sheppo 19353af08d82Slm66018 case LDC_CHANNEL_UP: 19363af08d82Slm66018 D1(ldcp->id, "i_ldc_rx_hdlr: " 19373af08d82Slm66018 "channel link up\n", ldcp->id); 19383af08d82Slm66018 19393af08d82Slm66018 if ((ldcp->tstate & ~TS_IN_RESET) == TS_OPEN) { 19401ae08745Sheppo notify_client = B_TRUE; 19411ae08745Sheppo notify_event = LDC_EVT_RESET; 19421ae08745Sheppo ldcp->tstate |= TS_LINK_READY; 19431ae08745Sheppo ldcp->status = LDC_READY; 19441ae08745Sheppo } 19453af08d82Slm66018 break; 19463af08d82Slm66018 19473af08d82Slm66018 case LDC_CHANNEL_RESET: 19483af08d82Slm66018 default: 19493af08d82Slm66018 #ifdef DEBUG 19503af08d82Slm66018 force_reset: 19513af08d82Slm66018 #endif 19523af08d82Slm66018 D1(ldcp->id, "i_ldc_rx_hdlr: channel " 19533af08d82Slm66018 "link reset\n", ldcp->id); 19543af08d82Slm66018 mutex_enter(&ldcp->tx_lock); 19553af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 19563af08d82Slm66018 mutex_exit(&ldcp->tx_lock); 19573af08d82Slm66018 notify_client = B_TRUE; 19583af08d82Slm66018 notify_event = LDC_EVT_RESET; 19593af08d82Slm66018 break; 19603af08d82Slm66018 } 19613af08d82Slm66018 } 19623af08d82Slm66018 19633af08d82Slm66018 #ifdef DEBUG 19643af08d82Slm66018 if (LDC_INJECT_RESET(ldcp)) 19653af08d82Slm66018 goto force_reset; 19663af08d82Slm66018 #endif 19671ae08745Sheppo 19681ae08745Sheppo if (rx_head == rx_tail) { 19691ae08745Sheppo D2(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) No packets\n", 19701ae08745Sheppo ldcp->id); 19711ae08745Sheppo break; 19721ae08745Sheppo } 19733af08d82Slm66018 19741ae08745Sheppo D2(ldcp->id, "i_ldc_rx_hdlr: head=0x%llx, tail=0x%llx\n", 19751ae08745Sheppo rx_head, rx_tail); 19761ae08745Sheppo DUMP_LDC_PKT(ldcp, "i_ldc_rx_hdlr rcd", 19771ae08745Sheppo ldcp->rx_q_va + rx_head); 19781ae08745Sheppo 19791ae08745Sheppo /* get the message */ 19801ae08745Sheppo msg = (ldc_msg_t *)(ldcp->rx_q_va + rx_head); 19811ae08745Sheppo 19821ae08745Sheppo /* if channel is in RAW mode or data pkt, notify and return */ 19831ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW) { 19841ae08745Sheppo notify_client = B_TRUE; 19851ae08745Sheppo notify_event |= LDC_EVT_READ; 19861ae08745Sheppo break; 19871ae08745Sheppo } 19881ae08745Sheppo 19891ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_INFO)) { 19901ae08745Sheppo 19911ae08745Sheppo /* discard packet if channel is not up */ 19923af08d82Slm66018 if ((ldcp->tstate & ~TS_IN_RESET) != TS_UP) { 19931ae08745Sheppo 19941ae08745Sheppo /* move the head one position */ 19951ae08745Sheppo rx_head = (rx_head + LDC_PACKET_SIZE) % 19961ae08745Sheppo (ldcp->rx_q_entries << LDC_PACKET_SHIFT); 19971ae08745Sheppo 19981ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, rx_head)) 19991ae08745Sheppo break; 20001ae08745Sheppo 20011ae08745Sheppo continue; 20021ae08745Sheppo } else { 20033af08d82Slm66018 if ((ldcp->tstate & TS_IN_RESET) == 0) 20041ae08745Sheppo notify_client = B_TRUE; 20051ae08745Sheppo notify_event |= LDC_EVT_READ; 20061ae08745Sheppo break; 20071ae08745Sheppo } 20081ae08745Sheppo } 20091ae08745Sheppo 20101ae08745Sheppo /* Check the sequence ID for the message received */ 20113af08d82Slm66018 rv = i_ldc_check_seqid(ldcp, msg); 20123af08d82Slm66018 if (rv != 0) { 20131ae08745Sheppo 20141ae08745Sheppo DWARN(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) seqid error, " 20151ae08745Sheppo "q_ptrs=0x%lx,0x%lx", ldcp->id, rx_head, rx_tail); 20161ae08745Sheppo 20171ae08745Sheppo /* Reset last_msg_rcd to start of message */ 2018d10e4ef2Snarayan if (first_fragment != 0) { 2019d10e4ef2Snarayan ldcp->last_msg_rcd = first_fragment - 1; 2020d10e4ef2Snarayan first_fragment = 0; 20211ae08745Sheppo } 2022d10e4ef2Snarayan 20231ae08745Sheppo /* 20241ae08745Sheppo * Send a NACK due to seqid mismatch 20251ae08745Sheppo */ 20261ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, 20271ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK)); 20281ae08745Sheppo 20291ae08745Sheppo if (rv) { 20301ae08745Sheppo cmn_err(CE_NOTE, 20311ae08745Sheppo "i_ldc_rx_hdlr: (0x%lx) err sending " 20321ae08745Sheppo "CTRL/NACK msg\n", ldcp->id); 2033d10e4ef2Snarayan 2034d10e4ef2Snarayan /* if cannot send NACK - reset channel */ 2035d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 20363af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 2037d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 203883d3bc6fSnarayan 203983d3bc6fSnarayan notify_client = B_TRUE; 204083d3bc6fSnarayan notify_event = LDC_EVT_RESET; 2041d10e4ef2Snarayan break; 20421ae08745Sheppo } 20431ae08745Sheppo 20441ae08745Sheppo /* purge receive queue */ 20451ae08745Sheppo (void) i_ldc_set_rx_head(ldcp, rx_tail); 20461ae08745Sheppo break; 20471ae08745Sheppo } 20481ae08745Sheppo 20491ae08745Sheppo /* record the message ID */ 20501ae08745Sheppo ldcp->last_msg_rcd = msg->seqid; 20511ae08745Sheppo 20521ae08745Sheppo /* process control messages */ 20531ae08745Sheppo if (msg->type & LDC_CTRL) { 20541ae08745Sheppo /* save current internal state */ 20551ae08745Sheppo uint64_t tstate = ldcp->tstate; 20561ae08745Sheppo 20571ae08745Sheppo rv = i_ldc_ctrlmsg(ldcp, msg); 20581ae08745Sheppo if (rv == EAGAIN) { 20591ae08745Sheppo /* re-process pkt - state was adjusted */ 20601ae08745Sheppo continue; 20611ae08745Sheppo } 20621ae08745Sheppo if (rv == ECONNRESET) { 20631ae08745Sheppo notify_client = B_TRUE; 20641ae08745Sheppo notify_event = LDC_EVT_RESET; 20651ae08745Sheppo break; 20661ae08745Sheppo } 20671ae08745Sheppo 20681ae08745Sheppo /* 20691ae08745Sheppo * control message processing was successful 20701ae08745Sheppo * channel transitioned to ready for communication 20711ae08745Sheppo */ 20721ae08745Sheppo if (rv == 0 && ldcp->tstate == TS_UP && 20733af08d82Slm66018 (tstate & ~TS_IN_RESET) != 20743af08d82Slm66018 (ldcp->tstate & ~TS_IN_RESET)) { 20751ae08745Sheppo notify_client = B_TRUE; 20761ae08745Sheppo notify_event = LDC_EVT_UP; 20771ae08745Sheppo } 20781ae08745Sheppo } 20791ae08745Sheppo 208083d3bc6fSnarayan /* process data NACKs */ 208183d3bc6fSnarayan if ((msg->type & LDC_DATA) && (msg->stype & LDC_NACK)) { 208283d3bc6fSnarayan DWARN(ldcp->id, 208383d3bc6fSnarayan "i_ldc_rx_hdlr: (0x%llx) received DATA/NACK", 208483d3bc6fSnarayan ldcp->id); 208583d3bc6fSnarayan mutex_enter(&ldcp->tx_lock); 208683d3bc6fSnarayan i_ldc_reset(ldcp, B_TRUE); 208783d3bc6fSnarayan mutex_exit(&ldcp->tx_lock); 208883d3bc6fSnarayan notify_client = B_TRUE; 208983d3bc6fSnarayan notify_event = LDC_EVT_RESET; 209083d3bc6fSnarayan break; 209183d3bc6fSnarayan } 209283d3bc6fSnarayan 20931ae08745Sheppo /* process data ACKs */ 20941ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 2095d10e4ef2Snarayan if (rv = i_ldc_process_data_ACK(ldcp, msg)) { 2096d10e4ef2Snarayan notify_client = B_TRUE; 2097d10e4ef2Snarayan notify_event = LDC_EVT_RESET; 2098d10e4ef2Snarayan break; 2099d10e4ef2Snarayan } 21001ae08745Sheppo } 21011ae08745Sheppo 21021ae08745Sheppo /* move the head one position */ 21031ae08745Sheppo rx_head = (rx_head + LDC_PACKET_SIZE) % 21041ae08745Sheppo (ldcp->rx_q_entries << LDC_PACKET_SHIFT); 21050a55fbb7Slm66018 if (rv = i_ldc_set_rx_head(ldcp, rx_head)) { 21060a55fbb7Slm66018 notify_client = B_TRUE; 21070a55fbb7Slm66018 notify_event = LDC_EVT_RESET; 21081ae08745Sheppo break; 21090a55fbb7Slm66018 } 21101ae08745Sheppo 21111ae08745Sheppo } /* for */ 21121ae08745Sheppo 21133af08d82Slm66018 loop_exit: 21143af08d82Slm66018 21151ae08745Sheppo /* if callbacks are disabled, do not notify */ 21161ae08745Sheppo if (!ldcp->cb_enabled) 21171ae08745Sheppo notify_client = B_FALSE; 21181ae08745Sheppo 21193af08d82Slm66018 /* 21203af08d82Slm66018 * If there are data packets in the queue, the ldc_read will 21213af08d82Slm66018 * clear interrupts after draining the queue, else clear interrupts 21223af08d82Slm66018 */ 21233af08d82Slm66018 if ((notify_event & LDC_EVT_READ) == 0) { 21243af08d82Slm66018 i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 21253af08d82Slm66018 } else 21263af08d82Slm66018 ldcp->rx_intr_state = LDC_INTR_PEND; 21271ae08745Sheppo 21281ae08745Sheppo 21291ae08745Sheppo if (notify_client) { 21304d39be2bSsg70180 ldcp->cb_inprogress = B_TRUE; 21314d39be2bSsg70180 mutex_exit(&ldcp->lock); 21321ae08745Sheppo rv = ldcp->cb(notify_event, ldcp->cb_arg); 21331ae08745Sheppo if (rv) { 21341ae08745Sheppo DWARN(ldcp->id, 21351ae08745Sheppo "i_ldc_rx_hdlr: (0x%llx) callback failure", 21361ae08745Sheppo ldcp->id); 21371ae08745Sheppo } 21384d39be2bSsg70180 mutex_enter(&ldcp->lock); 21394d39be2bSsg70180 ldcp->cb_inprogress = B_FALSE; 21401ae08745Sheppo } 21411ae08745Sheppo 21424d39be2bSsg70180 mutex_exit(&ldcp->lock); 21434d39be2bSsg70180 21441ae08745Sheppo D1(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) exiting handler", ldcp->id); 21451ae08745Sheppo return (DDI_INTR_CLAIMED); 21461ae08745Sheppo } 21471ae08745Sheppo 21481ae08745Sheppo 21491ae08745Sheppo /* -------------------------------------------------------------------------- */ 21501ae08745Sheppo 21511ae08745Sheppo /* 21521ae08745Sheppo * LDC API functions 21531ae08745Sheppo */ 21541ae08745Sheppo 21551ae08745Sheppo /* 21561ae08745Sheppo * Initialize the channel. Allocate internal structure and memory for 21571ae08745Sheppo * TX/RX queues, and initialize locks. 21581ae08745Sheppo */ 21591ae08745Sheppo int 21601ae08745Sheppo ldc_init(uint64_t id, ldc_attr_t *attr, ldc_handle_t *handle) 21611ae08745Sheppo { 21621ae08745Sheppo ldc_chan_t *ldcp; 21631ae08745Sheppo int rv, exit_val; 21641ae08745Sheppo uint64_t ra_base, nentries; 2165e1ebb9ecSlm66018 uint64_t qlen; 21661ae08745Sheppo 21671ae08745Sheppo exit_val = EINVAL; /* guarantee an error if exit on failure */ 21681ae08745Sheppo 21691ae08745Sheppo if (attr == NULL) { 21701ae08745Sheppo DWARN(id, "ldc_init: (0x%llx) invalid attr\n", id); 21711ae08745Sheppo return (EINVAL); 21721ae08745Sheppo } 21731ae08745Sheppo if (handle == NULL) { 21741ae08745Sheppo DWARN(id, "ldc_init: (0x%llx) invalid handle\n", id); 21751ae08745Sheppo return (EINVAL); 21761ae08745Sheppo } 21771ae08745Sheppo 21781ae08745Sheppo /* check if channel is valid */ 21791ae08745Sheppo rv = hv_ldc_tx_qinfo(id, &ra_base, &nentries); 21801ae08745Sheppo if (rv == H_ECHANNEL) { 21811ae08745Sheppo DWARN(id, "ldc_init: (0x%llx) invalid channel id\n", id); 21821ae08745Sheppo return (EINVAL); 21831ae08745Sheppo } 21841ae08745Sheppo 21851ae08745Sheppo /* check if the channel has already been initialized */ 21861ae08745Sheppo mutex_enter(&ldcssp->lock); 21871ae08745Sheppo ldcp = ldcssp->chan_list; 21881ae08745Sheppo while (ldcp != NULL) { 21891ae08745Sheppo if (ldcp->id == id) { 21901ae08745Sheppo DWARN(id, "ldc_init: (0x%llx) already initialized\n", 21911ae08745Sheppo id); 21921ae08745Sheppo mutex_exit(&ldcssp->lock); 21931ae08745Sheppo return (EADDRINUSE); 21941ae08745Sheppo } 21951ae08745Sheppo ldcp = ldcp->next; 21961ae08745Sheppo } 21971ae08745Sheppo mutex_exit(&ldcssp->lock); 21981ae08745Sheppo 21991ae08745Sheppo ASSERT(ldcp == NULL); 22001ae08745Sheppo 22011ae08745Sheppo *handle = 0; 22021ae08745Sheppo 22031ae08745Sheppo /* Allocate an ldcp structure */ 22041ae08745Sheppo ldcp = kmem_zalloc(sizeof (ldc_chan_t), KM_SLEEP); 22051ae08745Sheppo 2206d10e4ef2Snarayan /* 2207d10e4ef2Snarayan * Initialize the channel and Tx lock 2208d10e4ef2Snarayan * 2209d10e4ef2Snarayan * The channel 'lock' protects the entire channel and 2210d10e4ef2Snarayan * should be acquired before initializing, resetting, 2211d10e4ef2Snarayan * destroying or reading from a channel. 2212d10e4ef2Snarayan * 2213d10e4ef2Snarayan * The 'tx_lock' should be acquired prior to transmitting 2214d10e4ef2Snarayan * data over the channel. The lock should also be acquired 2215d10e4ef2Snarayan * prior to channel reconfiguration (in order to prevent 2216d10e4ef2Snarayan * concurrent writes). 2217d10e4ef2Snarayan * 2218d10e4ef2Snarayan * ORDERING: When both locks are being acquired, to prevent 2219d10e4ef2Snarayan * deadlocks, the channel lock should be always acquired prior 2220d10e4ef2Snarayan * to the tx_lock. 2221d10e4ef2Snarayan */ 22221ae08745Sheppo mutex_init(&ldcp->lock, NULL, MUTEX_DRIVER, NULL); 2223d10e4ef2Snarayan mutex_init(&ldcp->tx_lock, NULL, MUTEX_DRIVER, NULL); 22241ae08745Sheppo 22251ae08745Sheppo /* Initialize the channel */ 22261ae08745Sheppo ldcp->id = id; 22271ae08745Sheppo ldcp->cb = NULL; 22281ae08745Sheppo ldcp->cb_arg = NULL; 22291ae08745Sheppo ldcp->cb_inprogress = B_FALSE; 22301ae08745Sheppo ldcp->cb_enabled = B_FALSE; 22311ae08745Sheppo ldcp->next = NULL; 22321ae08745Sheppo 22331ae08745Sheppo /* Read attributes */ 22341ae08745Sheppo ldcp->mode = attr->mode; 22351ae08745Sheppo ldcp->devclass = attr->devclass; 22361ae08745Sheppo ldcp->devinst = attr->instance; 2237e1ebb9ecSlm66018 ldcp->mtu = (attr->mtu > 0) ? attr->mtu : LDC_DEFAULT_MTU; 22381ae08745Sheppo 22391ae08745Sheppo D1(ldcp->id, 22401ae08745Sheppo "ldc_init: (0x%llx) channel attributes, class=0x%x, " 2241e1ebb9ecSlm66018 "instance=0x%llx, mode=%d, mtu=%d\n", 2242e1ebb9ecSlm66018 ldcp->id, ldcp->devclass, ldcp->devinst, ldcp->mode, ldcp->mtu); 22431ae08745Sheppo 22441ae08745Sheppo ldcp->next_vidx = 0; 22453af08d82Slm66018 ldcp->tstate = TS_IN_RESET; 22461ae08745Sheppo ldcp->hstate = 0; 22471ae08745Sheppo ldcp->last_msg_snt = LDC_INIT_SEQID; 22481ae08745Sheppo ldcp->last_ack_rcd = 0; 22491ae08745Sheppo ldcp->last_msg_rcd = 0; 22501ae08745Sheppo 22511ae08745Sheppo ldcp->stream_bufferp = NULL; 22521ae08745Sheppo ldcp->exp_dring_list = NULL; 22531ae08745Sheppo ldcp->imp_dring_list = NULL; 22541ae08745Sheppo ldcp->mhdl_list = NULL; 22551ae08745Sheppo 22563af08d82Slm66018 ldcp->tx_intr_state = LDC_INTR_NONE; 22573af08d82Slm66018 ldcp->rx_intr_state = LDC_INTR_NONE; 22583af08d82Slm66018 22591ae08745Sheppo /* Initialize payload size depending on whether channel is reliable */ 22601ae08745Sheppo switch (ldcp->mode) { 22611ae08745Sheppo case LDC_MODE_RAW: 22621ae08745Sheppo ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RAW; 22631ae08745Sheppo ldcp->read_p = i_ldc_read_raw; 22641ae08745Sheppo ldcp->write_p = i_ldc_write_raw; 22651ae08745Sheppo break; 22661ae08745Sheppo case LDC_MODE_UNRELIABLE: 22671ae08745Sheppo ldcp->pkt_payload = LDC_PAYLOAD_SIZE_UNRELIABLE; 22681ae08745Sheppo ldcp->read_p = i_ldc_read_packet; 22691ae08745Sheppo ldcp->write_p = i_ldc_write_packet; 22701ae08745Sheppo break; 22711ae08745Sheppo case LDC_MODE_RELIABLE: 22721ae08745Sheppo ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RELIABLE; 22731ae08745Sheppo ldcp->read_p = i_ldc_read_packet; 22741ae08745Sheppo ldcp->write_p = i_ldc_write_packet; 22751ae08745Sheppo break; 22761ae08745Sheppo case LDC_MODE_STREAM: 22771ae08745Sheppo ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RELIABLE; 22781ae08745Sheppo 22791ae08745Sheppo ldcp->stream_remains = 0; 22801ae08745Sheppo ldcp->stream_offset = 0; 22811ae08745Sheppo ldcp->stream_bufferp = kmem_alloc(ldcp->mtu, KM_SLEEP); 22821ae08745Sheppo ldcp->read_p = i_ldc_read_stream; 22831ae08745Sheppo ldcp->write_p = i_ldc_write_stream; 22841ae08745Sheppo break; 22851ae08745Sheppo default: 22861ae08745Sheppo exit_val = EINVAL; 22871ae08745Sheppo goto cleanup_on_exit; 22881ae08745Sheppo } 22891ae08745Sheppo 2290e1ebb9ecSlm66018 /* 2291e1ebb9ecSlm66018 * qlen is (mtu * ldc_mtu_msgs) / pkt_payload. If this 2292e1ebb9ecSlm66018 * value is smaller than default length of ldc_queue_entries, 2293e1ebb9ecSlm66018 * qlen is set to ldc_queue_entries.. 2294e1ebb9ecSlm66018 */ 2295e1ebb9ecSlm66018 qlen = (ldcp->mtu * ldc_mtu_msgs) / ldcp->pkt_payload; 2296e1ebb9ecSlm66018 ldcp->rx_q_entries = 2297e1ebb9ecSlm66018 (qlen < ldc_queue_entries) ? ldc_queue_entries : qlen; 2298e1ebb9ecSlm66018 ldcp->tx_q_entries = ldcp->rx_q_entries; 2299e1ebb9ecSlm66018 2300e1ebb9ecSlm66018 D1(ldcp->id, "ldc_init: queue length = 0x%llx\n", qlen); 2301e1ebb9ecSlm66018 23021ae08745Sheppo /* Create a transmit queue */ 23031ae08745Sheppo ldcp->tx_q_va = (uint64_t) 23041ae08745Sheppo contig_mem_alloc(ldcp->tx_q_entries << LDC_PACKET_SHIFT); 23051ae08745Sheppo if (ldcp->tx_q_va == NULL) { 23061ae08745Sheppo cmn_err(CE_WARN, 23071ae08745Sheppo "ldc_init: (0x%lx) TX queue allocation failed\n", 23081ae08745Sheppo ldcp->id); 23091ae08745Sheppo exit_val = ENOMEM; 23101ae08745Sheppo goto cleanup_on_exit; 23111ae08745Sheppo } 23121ae08745Sheppo ldcp->tx_q_ra = va_to_pa((caddr_t)ldcp->tx_q_va); 23131ae08745Sheppo 23141ae08745Sheppo D2(ldcp->id, "ldc_init: txq_va=0x%llx, txq_ra=0x%llx, entries=0x%llx\n", 23151ae08745Sheppo ldcp->tx_q_va, ldcp->tx_q_ra, ldcp->tx_q_entries); 23161ae08745Sheppo 23171ae08745Sheppo ldcp->tstate |= TS_TXQ_RDY; 23181ae08745Sheppo 23191ae08745Sheppo /* Create a receive queue */ 23201ae08745Sheppo ldcp->rx_q_va = (uint64_t) 23211ae08745Sheppo contig_mem_alloc(ldcp->rx_q_entries << LDC_PACKET_SHIFT); 23221ae08745Sheppo if (ldcp->rx_q_va == NULL) { 23231ae08745Sheppo cmn_err(CE_WARN, 23241ae08745Sheppo "ldc_init: (0x%lx) RX queue allocation failed\n", 23251ae08745Sheppo ldcp->id); 23261ae08745Sheppo exit_val = ENOMEM; 23271ae08745Sheppo goto cleanup_on_exit; 23281ae08745Sheppo } 23291ae08745Sheppo ldcp->rx_q_ra = va_to_pa((caddr_t)ldcp->rx_q_va); 23301ae08745Sheppo 23311ae08745Sheppo D2(ldcp->id, "ldc_init: rxq_va=0x%llx, rxq_ra=0x%llx, entries=0x%llx\n", 23321ae08745Sheppo ldcp->rx_q_va, ldcp->rx_q_ra, ldcp->rx_q_entries); 23331ae08745Sheppo 23341ae08745Sheppo ldcp->tstate |= TS_RXQ_RDY; 23351ae08745Sheppo 23361ae08745Sheppo /* Init descriptor ring and memory handle list lock */ 23371ae08745Sheppo mutex_init(&ldcp->exp_dlist_lock, NULL, MUTEX_DRIVER, NULL); 23381ae08745Sheppo mutex_init(&ldcp->imp_dlist_lock, NULL, MUTEX_DRIVER, NULL); 23391ae08745Sheppo mutex_init(&ldcp->mlist_lock, NULL, MUTEX_DRIVER, NULL); 23401ae08745Sheppo 23411ae08745Sheppo /* mark status as INITialized */ 23421ae08745Sheppo ldcp->status = LDC_INIT; 23431ae08745Sheppo 23441ae08745Sheppo /* Add to channel list */ 23451ae08745Sheppo mutex_enter(&ldcssp->lock); 23461ae08745Sheppo ldcp->next = ldcssp->chan_list; 23471ae08745Sheppo ldcssp->chan_list = ldcp; 23481ae08745Sheppo ldcssp->channel_count++; 23491ae08745Sheppo mutex_exit(&ldcssp->lock); 23501ae08745Sheppo 23511ae08745Sheppo /* set the handle */ 23521ae08745Sheppo *handle = (ldc_handle_t)ldcp; 23531ae08745Sheppo 23541ae08745Sheppo D1(ldcp->id, "ldc_init: (0x%llx) channel initialized\n", ldcp->id); 23551ae08745Sheppo 23561ae08745Sheppo return (0); 23571ae08745Sheppo 23581ae08745Sheppo cleanup_on_exit: 23591ae08745Sheppo 23601ae08745Sheppo if (ldcp->mode == LDC_MODE_STREAM && ldcp->stream_bufferp) 23611ae08745Sheppo kmem_free(ldcp->stream_bufferp, ldcp->mtu); 23621ae08745Sheppo 23631ae08745Sheppo if (ldcp->tstate & TS_TXQ_RDY) 23641ae08745Sheppo contig_mem_free((caddr_t)ldcp->tx_q_va, 23651ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 23661ae08745Sheppo 23671ae08745Sheppo if (ldcp->tstate & TS_RXQ_RDY) 23681ae08745Sheppo contig_mem_free((caddr_t)ldcp->rx_q_va, 23691ae08745Sheppo (ldcp->rx_q_entries << LDC_PACKET_SHIFT)); 23701ae08745Sheppo 2371d10e4ef2Snarayan mutex_destroy(&ldcp->tx_lock); 23721ae08745Sheppo mutex_destroy(&ldcp->lock); 23731ae08745Sheppo 23741ae08745Sheppo if (ldcp) 23751ae08745Sheppo kmem_free(ldcp, sizeof (ldc_chan_t)); 23761ae08745Sheppo 23771ae08745Sheppo return (exit_val); 23781ae08745Sheppo } 23791ae08745Sheppo 23801ae08745Sheppo /* 23811ae08745Sheppo * Finalizes the LDC connection. It will return EBUSY if the 23821ae08745Sheppo * channel is open. A ldc_close() has to be done prior to 23831ae08745Sheppo * a ldc_fini operation. It frees TX/RX queues, associated 23841ae08745Sheppo * with the channel 23851ae08745Sheppo */ 23861ae08745Sheppo int 23871ae08745Sheppo ldc_fini(ldc_handle_t handle) 23881ae08745Sheppo { 23891ae08745Sheppo ldc_chan_t *ldcp; 23901ae08745Sheppo ldc_chan_t *tmp_ldcp; 23911ae08745Sheppo uint64_t id; 23921ae08745Sheppo 23931ae08745Sheppo if (handle == NULL) { 23941ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_fini: invalid channel handle\n"); 23951ae08745Sheppo return (EINVAL); 23961ae08745Sheppo } 23971ae08745Sheppo ldcp = (ldc_chan_t *)handle; 23981ae08745Sheppo id = ldcp->id; 23991ae08745Sheppo 24001ae08745Sheppo mutex_enter(&ldcp->lock); 24011ae08745Sheppo 24023af08d82Slm66018 if ((ldcp->tstate & ~TS_IN_RESET) > TS_INIT) { 24031ae08745Sheppo DWARN(ldcp->id, "ldc_fini: (0x%llx) channel is open\n", 24041ae08745Sheppo ldcp->id); 24051ae08745Sheppo mutex_exit(&ldcp->lock); 24061ae08745Sheppo return (EBUSY); 24071ae08745Sheppo } 24081ae08745Sheppo 24091ae08745Sheppo /* Remove from the channel list */ 24101ae08745Sheppo mutex_enter(&ldcssp->lock); 24111ae08745Sheppo tmp_ldcp = ldcssp->chan_list; 24121ae08745Sheppo if (tmp_ldcp == ldcp) { 24131ae08745Sheppo ldcssp->chan_list = ldcp->next; 24141ae08745Sheppo ldcp->next = NULL; 24151ae08745Sheppo } else { 24161ae08745Sheppo while (tmp_ldcp != NULL) { 24171ae08745Sheppo if (tmp_ldcp->next == ldcp) { 24181ae08745Sheppo tmp_ldcp->next = ldcp->next; 24191ae08745Sheppo ldcp->next = NULL; 24201ae08745Sheppo break; 24211ae08745Sheppo } 24221ae08745Sheppo tmp_ldcp = tmp_ldcp->next; 24231ae08745Sheppo } 24241ae08745Sheppo if (tmp_ldcp == NULL) { 24251ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_fini: invalid channel hdl\n"); 24261ae08745Sheppo mutex_exit(&ldcssp->lock); 24271ae08745Sheppo mutex_exit(&ldcp->lock); 24281ae08745Sheppo return (EINVAL); 24291ae08745Sheppo } 24301ae08745Sheppo } 24311ae08745Sheppo 24321ae08745Sheppo ldcssp->channel_count--; 24331ae08745Sheppo 24341ae08745Sheppo mutex_exit(&ldcssp->lock); 24351ae08745Sheppo 24361ae08745Sheppo /* Free the map table for this channel */ 24371ae08745Sheppo if (ldcp->mtbl) { 24381ae08745Sheppo (void) hv_ldc_set_map_table(ldcp->id, NULL, NULL); 24393af08d82Slm66018 if (ldcp->mtbl->contigmem) 24401ae08745Sheppo contig_mem_free(ldcp->mtbl->table, ldcp->mtbl->size); 24413af08d82Slm66018 else 24423af08d82Slm66018 kmem_free(ldcp->mtbl->table, ldcp->mtbl->size); 24431ae08745Sheppo mutex_destroy(&ldcp->mtbl->lock); 24441ae08745Sheppo kmem_free(ldcp->mtbl, sizeof (ldc_mtbl_t)); 24451ae08745Sheppo } 24461ae08745Sheppo 24471ae08745Sheppo /* Destroy descriptor ring and memory handle list lock */ 24481ae08745Sheppo mutex_destroy(&ldcp->exp_dlist_lock); 24491ae08745Sheppo mutex_destroy(&ldcp->imp_dlist_lock); 24501ae08745Sheppo mutex_destroy(&ldcp->mlist_lock); 24511ae08745Sheppo 24521ae08745Sheppo /* Free the stream buffer for STREAM_MODE */ 24531ae08745Sheppo if (ldcp->mode == LDC_MODE_STREAM && ldcp->stream_bufferp) 24541ae08745Sheppo kmem_free(ldcp->stream_bufferp, ldcp->mtu); 24551ae08745Sheppo 24561ae08745Sheppo /* Free the RX queue */ 24571ae08745Sheppo contig_mem_free((caddr_t)ldcp->rx_q_va, 24581ae08745Sheppo (ldcp->rx_q_entries << LDC_PACKET_SHIFT)); 24591ae08745Sheppo ldcp->tstate &= ~TS_RXQ_RDY; 24601ae08745Sheppo 24611ae08745Sheppo /* Free the TX queue */ 24621ae08745Sheppo contig_mem_free((caddr_t)ldcp->tx_q_va, 24631ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 24641ae08745Sheppo ldcp->tstate &= ~TS_TXQ_RDY; 24651ae08745Sheppo 24661ae08745Sheppo mutex_exit(&ldcp->lock); 24671ae08745Sheppo 24681ae08745Sheppo /* Destroy mutex */ 2469d10e4ef2Snarayan mutex_destroy(&ldcp->tx_lock); 24701ae08745Sheppo mutex_destroy(&ldcp->lock); 24711ae08745Sheppo 24721ae08745Sheppo /* free channel structure */ 24731ae08745Sheppo kmem_free(ldcp, sizeof (ldc_chan_t)); 24741ae08745Sheppo 24751ae08745Sheppo D1(id, "ldc_fini: (0x%llx) channel finalized\n", id); 24761ae08745Sheppo 24771ae08745Sheppo return (0); 24781ae08745Sheppo } 24791ae08745Sheppo 24801ae08745Sheppo /* 24811ae08745Sheppo * Open the LDC channel for use. It registers the TX/RX queues 24821ae08745Sheppo * with the Hypervisor. It also specifies the interrupt number 24831ae08745Sheppo * and target CPU for this channel 24841ae08745Sheppo */ 24851ae08745Sheppo int 24861ae08745Sheppo ldc_open(ldc_handle_t handle) 24871ae08745Sheppo { 24881ae08745Sheppo ldc_chan_t *ldcp; 24891ae08745Sheppo int rv; 24901ae08745Sheppo 24911ae08745Sheppo if (handle == NULL) { 24921ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_open: invalid channel handle\n"); 24931ae08745Sheppo return (EINVAL); 24941ae08745Sheppo } 24951ae08745Sheppo 24961ae08745Sheppo ldcp = (ldc_chan_t *)handle; 24971ae08745Sheppo 24981ae08745Sheppo mutex_enter(&ldcp->lock); 24991ae08745Sheppo 25001ae08745Sheppo if (ldcp->tstate < TS_INIT) { 25011ae08745Sheppo DWARN(ldcp->id, 25021ae08745Sheppo "ldc_open: (0x%llx) channel not initialized\n", ldcp->id); 25031ae08745Sheppo mutex_exit(&ldcp->lock); 25041ae08745Sheppo return (EFAULT); 25051ae08745Sheppo } 25063af08d82Slm66018 if ((ldcp->tstate & ~TS_IN_RESET) >= TS_OPEN) { 25071ae08745Sheppo DWARN(ldcp->id, 25081ae08745Sheppo "ldc_open: (0x%llx) channel is already open\n", ldcp->id); 25091ae08745Sheppo mutex_exit(&ldcp->lock); 25101ae08745Sheppo return (EFAULT); 25111ae08745Sheppo } 25121ae08745Sheppo 25131ae08745Sheppo /* 25141ae08745Sheppo * Unregister/Register the tx queue with the hypervisor 25151ae08745Sheppo */ 25161ae08745Sheppo rv = hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 25171ae08745Sheppo if (rv) { 25181ae08745Sheppo cmn_err(CE_WARN, 25191ae08745Sheppo "ldc_open: (0x%lx) channel tx queue unconf failed\n", 25201ae08745Sheppo ldcp->id); 25211ae08745Sheppo mutex_exit(&ldcp->lock); 25221ae08745Sheppo return (EIO); 25231ae08745Sheppo } 25241ae08745Sheppo 25251ae08745Sheppo rv = hv_ldc_tx_qconf(ldcp->id, ldcp->tx_q_ra, ldcp->tx_q_entries); 25261ae08745Sheppo if (rv) { 25271ae08745Sheppo cmn_err(CE_WARN, 25281ae08745Sheppo "ldc_open: (0x%lx) channel tx queue conf failed\n", 25291ae08745Sheppo ldcp->id); 25301ae08745Sheppo mutex_exit(&ldcp->lock); 25311ae08745Sheppo return (EIO); 25321ae08745Sheppo } 25331ae08745Sheppo 25341ae08745Sheppo D2(ldcp->id, "ldc_open: (0x%llx) registered tx queue with LDC\n", 25351ae08745Sheppo ldcp->id); 25361ae08745Sheppo 25371ae08745Sheppo /* 25381ae08745Sheppo * Unregister/Register the rx queue with the hypervisor 25391ae08745Sheppo */ 25401ae08745Sheppo rv = hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 25411ae08745Sheppo if (rv) { 25421ae08745Sheppo cmn_err(CE_WARN, 25431ae08745Sheppo "ldc_open: (0x%lx) channel rx queue unconf failed\n", 25441ae08745Sheppo ldcp->id); 25451ae08745Sheppo mutex_exit(&ldcp->lock); 25461ae08745Sheppo return (EIO); 25471ae08745Sheppo } 25481ae08745Sheppo 25491ae08745Sheppo rv = hv_ldc_rx_qconf(ldcp->id, ldcp->rx_q_ra, ldcp->rx_q_entries); 25501ae08745Sheppo if (rv) { 25511ae08745Sheppo cmn_err(CE_WARN, 25521ae08745Sheppo "ldc_open: (0x%lx) channel rx queue conf failed\n", 25531ae08745Sheppo ldcp->id); 25541ae08745Sheppo mutex_exit(&ldcp->lock); 25551ae08745Sheppo return (EIO); 25561ae08745Sheppo } 25571ae08745Sheppo 25581ae08745Sheppo D2(ldcp->id, "ldc_open: (0x%llx) registered rx queue with LDC\n", 25591ae08745Sheppo ldcp->id); 25601ae08745Sheppo 25611ae08745Sheppo ldcp->tstate |= TS_QCONF_RDY; 25621ae08745Sheppo 25631ae08745Sheppo /* Register the channel with the channel nexus */ 25641ae08745Sheppo rv = i_ldc_register_channel(ldcp); 25651ae08745Sheppo if (rv && rv != EAGAIN) { 25661ae08745Sheppo cmn_err(CE_WARN, 25671ae08745Sheppo "ldc_open: (0x%lx) channel register failed\n", ldcp->id); 25681ae08745Sheppo (void) hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 25691ae08745Sheppo (void) hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 25701ae08745Sheppo mutex_exit(&ldcp->lock); 25711ae08745Sheppo return (EIO); 25721ae08745Sheppo } 25731ae08745Sheppo 25741ae08745Sheppo /* mark channel in OPEN state */ 25751ae08745Sheppo ldcp->status = LDC_OPEN; 25761ae08745Sheppo 25771ae08745Sheppo /* Read channel state */ 25781ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 25791ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 25801ae08745Sheppo if (rv) { 25811ae08745Sheppo cmn_err(CE_WARN, 25821ae08745Sheppo "ldc_open: (0x%lx) cannot read channel state\n", 25831ae08745Sheppo ldcp->id); 25841ae08745Sheppo (void) i_ldc_unregister_channel(ldcp); 25851ae08745Sheppo (void) hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 25861ae08745Sheppo (void) hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 25871ae08745Sheppo mutex_exit(&ldcp->lock); 25881ae08745Sheppo return (EIO); 25891ae08745Sheppo } 25901ae08745Sheppo 25911ae08745Sheppo /* 25921ae08745Sheppo * set the ACKd head to current head location for reliable & 25931ae08745Sheppo * streaming mode 25941ae08745Sheppo */ 25951ae08745Sheppo ldcp->tx_ackd_head = ldcp->tx_head; 25961ae08745Sheppo 25971ae08745Sheppo /* mark channel ready if HV report link is UP (peer alloc'd Rx queue) */ 25981ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_UP || 25991ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 26001ae08745Sheppo ldcp->tstate |= TS_LINK_READY; 26011ae08745Sheppo ldcp->status = LDC_READY; 26021ae08745Sheppo } 26031ae08745Sheppo 26041ae08745Sheppo /* 26051ae08745Sheppo * if channel is being opened in RAW mode - no handshake is needed 26061ae08745Sheppo * switch the channel READY and UP state 26071ae08745Sheppo */ 26081ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW) { 26091ae08745Sheppo ldcp->tstate = TS_UP; /* set bits associated with LDC UP */ 26101ae08745Sheppo ldcp->status = LDC_UP; 26111ae08745Sheppo } 26121ae08745Sheppo 26131ae08745Sheppo mutex_exit(&ldcp->lock); 26141ae08745Sheppo 26151ae08745Sheppo /* 26161ae08745Sheppo * Increment number of open channels 26171ae08745Sheppo */ 26181ae08745Sheppo mutex_enter(&ldcssp->lock); 26191ae08745Sheppo ldcssp->channels_open++; 26201ae08745Sheppo mutex_exit(&ldcssp->lock); 26211ae08745Sheppo 2622cb112a14Slm66018 D1(ldcp->id, 26233af08d82Slm66018 "ldc_open: (0x%llx) channel (0x%p) open for use " 26243af08d82Slm66018 "(tstate=0x%x, status=0x%x)\n", 26253af08d82Slm66018 ldcp->id, ldcp, ldcp->tstate, ldcp->status); 26261ae08745Sheppo 26271ae08745Sheppo return (0); 26281ae08745Sheppo } 26291ae08745Sheppo 26301ae08745Sheppo /* 26311ae08745Sheppo * Close the LDC connection. It will return EBUSY if there 26321ae08745Sheppo * are memory segments or descriptor rings either bound to or 26331ae08745Sheppo * mapped over the channel 26341ae08745Sheppo */ 26351ae08745Sheppo int 26361ae08745Sheppo ldc_close(ldc_handle_t handle) 26371ae08745Sheppo { 26381ae08745Sheppo ldc_chan_t *ldcp; 2639d10e4ef2Snarayan int rv = 0, retries = 0; 26401ae08745Sheppo boolean_t chk_done = B_FALSE; 26411ae08745Sheppo 26421ae08745Sheppo if (handle == NULL) { 26431ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_close: invalid channel handle\n"); 26441ae08745Sheppo return (EINVAL); 26451ae08745Sheppo } 26461ae08745Sheppo ldcp = (ldc_chan_t *)handle; 26471ae08745Sheppo 26481ae08745Sheppo mutex_enter(&ldcp->lock); 26491ae08745Sheppo 26501ae08745Sheppo /* return error if channel is not open */ 26513af08d82Slm66018 if ((ldcp->tstate & ~TS_IN_RESET) < TS_OPEN) { 26521ae08745Sheppo DWARN(ldcp->id, 26531ae08745Sheppo "ldc_close: (0x%llx) channel is not open\n", ldcp->id); 26541ae08745Sheppo mutex_exit(&ldcp->lock); 26551ae08745Sheppo return (EFAULT); 26561ae08745Sheppo } 26571ae08745Sheppo 26581ae08745Sheppo /* if any memory handles, drings, are bound or mapped cannot close */ 26591ae08745Sheppo if (ldcp->mhdl_list != NULL) { 26601ae08745Sheppo DWARN(ldcp->id, 26611ae08745Sheppo "ldc_close: (0x%llx) channel has bound memory handles\n", 26621ae08745Sheppo ldcp->id); 26631ae08745Sheppo mutex_exit(&ldcp->lock); 26641ae08745Sheppo return (EBUSY); 26651ae08745Sheppo } 26661ae08745Sheppo if (ldcp->exp_dring_list != NULL) { 26671ae08745Sheppo DWARN(ldcp->id, 26681ae08745Sheppo "ldc_close: (0x%llx) channel has bound descriptor rings\n", 26691ae08745Sheppo ldcp->id); 26701ae08745Sheppo mutex_exit(&ldcp->lock); 26711ae08745Sheppo return (EBUSY); 26721ae08745Sheppo } 26731ae08745Sheppo if (ldcp->imp_dring_list != NULL) { 26741ae08745Sheppo DWARN(ldcp->id, 26751ae08745Sheppo "ldc_close: (0x%llx) channel has mapped descriptor rings\n", 26761ae08745Sheppo ldcp->id); 26771ae08745Sheppo mutex_exit(&ldcp->lock); 26781ae08745Sheppo return (EBUSY); 26791ae08745Sheppo } 26801ae08745Sheppo 26814d39be2bSsg70180 if (ldcp->cb_inprogress) { 26824d39be2bSsg70180 DWARN(ldcp->id, "ldc_close: (0x%llx) callback active\n", 26834d39be2bSsg70180 ldcp->id); 26844d39be2bSsg70180 mutex_exit(&ldcp->lock); 26854d39be2bSsg70180 return (EWOULDBLOCK); 26864d39be2bSsg70180 } 26874d39be2bSsg70180 2688d10e4ef2Snarayan /* Obtain Tx lock */ 2689d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 2690d10e4ef2Snarayan 26911ae08745Sheppo /* 26921ae08745Sheppo * Wait for pending transmits to complete i.e Tx queue to drain 26931ae08745Sheppo * if there are pending pkts - wait 1 ms and retry again 26941ae08745Sheppo */ 26951ae08745Sheppo for (;;) { 26961ae08745Sheppo 26971ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 26981ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 26991ae08745Sheppo if (rv) { 27001ae08745Sheppo cmn_err(CE_WARN, 27011ae08745Sheppo "ldc_close: (0x%lx) cannot read qptrs\n", ldcp->id); 2702d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 27031ae08745Sheppo mutex_exit(&ldcp->lock); 27041ae08745Sheppo return (EIO); 27051ae08745Sheppo } 27061ae08745Sheppo 27071ae08745Sheppo if (ldcp->tx_head == ldcp->tx_tail || 27081ae08745Sheppo ldcp->link_state != LDC_CHANNEL_UP) { 27091ae08745Sheppo break; 27101ae08745Sheppo } 27111ae08745Sheppo 27121ae08745Sheppo if (chk_done) { 27131ae08745Sheppo DWARN(ldcp->id, 27141ae08745Sheppo "ldc_close: (0x%llx) Tx queue drain timeout\n", 27151ae08745Sheppo ldcp->id); 27161ae08745Sheppo break; 27171ae08745Sheppo } 27181ae08745Sheppo 27191ae08745Sheppo /* wait for one ms and try again */ 27201ae08745Sheppo delay(drv_usectohz(1000)); 27211ae08745Sheppo chk_done = B_TRUE; 27221ae08745Sheppo } 27231ae08745Sheppo 27241ae08745Sheppo /* 2725a8ea4edeSnarayan * Drain the Tx and Rx queues as we are closing the 2726a8ea4edeSnarayan * channel. We dont care about any pending packets. 2727a8ea4edeSnarayan * We have to also drain the queue prior to clearing 2728a8ea4edeSnarayan * pending interrupts, otherwise the HV will trigger 2729a8ea4edeSnarayan * an interrupt the moment the interrupt state is 2730a8ea4edeSnarayan * cleared. 27313af08d82Slm66018 */ 27323af08d82Slm66018 (void) i_ldc_txq_reconf(ldcp); 2733a8ea4edeSnarayan (void) i_ldc_rxq_drain(ldcp); 27343af08d82Slm66018 27353af08d82Slm66018 /* 27361ae08745Sheppo * Unregister the channel with the nexus 27371ae08745Sheppo */ 2738d10e4ef2Snarayan while ((rv = i_ldc_unregister_channel(ldcp)) != 0) { 2739d10e4ef2Snarayan 2740d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 27411ae08745Sheppo mutex_exit(&ldcp->lock); 2742d10e4ef2Snarayan 2743d10e4ef2Snarayan /* if any error other than EAGAIN return back */ 2744a8ea4edeSnarayan if (rv != EAGAIN || retries >= ldc_max_retries) { 2745d10e4ef2Snarayan cmn_err(CE_WARN, 2746d10e4ef2Snarayan "ldc_close: (0x%lx) unregister failed, %d\n", 2747d10e4ef2Snarayan ldcp->id, rv); 27481ae08745Sheppo return (rv); 27491ae08745Sheppo } 27501ae08745Sheppo 27511ae08745Sheppo /* 2752d10e4ef2Snarayan * As there could be pending interrupts we need 2753d10e4ef2Snarayan * to wait and try again 2754d10e4ef2Snarayan */ 27554d39be2bSsg70180 drv_usecwait(ldc_close_delay); 2756d10e4ef2Snarayan mutex_enter(&ldcp->lock); 2757d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 2758d10e4ef2Snarayan retries++; 2759d10e4ef2Snarayan } 2760d10e4ef2Snarayan 2761d10e4ef2Snarayan /* 27621ae08745Sheppo * Unregister queues 27631ae08745Sheppo */ 27641ae08745Sheppo rv = hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 27651ae08745Sheppo if (rv) { 27661ae08745Sheppo cmn_err(CE_WARN, 27671ae08745Sheppo "ldc_close: (0x%lx) channel TX queue unconf failed\n", 27681ae08745Sheppo ldcp->id); 2769d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 27701ae08745Sheppo mutex_exit(&ldcp->lock); 27711ae08745Sheppo return (EIO); 27721ae08745Sheppo } 27731ae08745Sheppo rv = hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 27741ae08745Sheppo if (rv) { 27751ae08745Sheppo cmn_err(CE_WARN, 27761ae08745Sheppo "ldc_close: (0x%lx) channel RX queue unconf failed\n", 27771ae08745Sheppo ldcp->id); 2778d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 27791ae08745Sheppo mutex_exit(&ldcp->lock); 27801ae08745Sheppo return (EIO); 27811ae08745Sheppo } 27821ae08745Sheppo 27831ae08745Sheppo ldcp->tstate &= ~TS_QCONF_RDY; 27841ae08745Sheppo 27851ae08745Sheppo /* Reset channel state information */ 27861ae08745Sheppo i_ldc_reset_state(ldcp); 27871ae08745Sheppo 27881ae08745Sheppo /* Mark channel as down and in initialized state */ 27891ae08745Sheppo ldcp->tx_ackd_head = 0; 27901ae08745Sheppo ldcp->tx_head = 0; 27913af08d82Slm66018 ldcp->tstate = TS_IN_RESET|TS_INIT; 27921ae08745Sheppo ldcp->status = LDC_INIT; 27931ae08745Sheppo 2794d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 27951ae08745Sheppo mutex_exit(&ldcp->lock); 27961ae08745Sheppo 27971ae08745Sheppo /* Decrement number of open channels */ 27981ae08745Sheppo mutex_enter(&ldcssp->lock); 27991ae08745Sheppo ldcssp->channels_open--; 28001ae08745Sheppo mutex_exit(&ldcssp->lock); 28011ae08745Sheppo 28021ae08745Sheppo D1(ldcp->id, "ldc_close: (0x%llx) channel closed\n", ldcp->id); 28031ae08745Sheppo 28041ae08745Sheppo return (0); 28051ae08745Sheppo } 28061ae08745Sheppo 28071ae08745Sheppo /* 28081ae08745Sheppo * Register channel callback 28091ae08745Sheppo */ 28101ae08745Sheppo int 28111ae08745Sheppo ldc_reg_callback(ldc_handle_t handle, 28121ae08745Sheppo uint_t(*cb)(uint64_t event, caddr_t arg), caddr_t arg) 28131ae08745Sheppo { 28141ae08745Sheppo ldc_chan_t *ldcp; 28151ae08745Sheppo 28161ae08745Sheppo if (handle == NULL) { 28171ae08745Sheppo DWARN(DBG_ALL_LDCS, 28181ae08745Sheppo "ldc_reg_callback: invalid channel handle\n"); 28191ae08745Sheppo return (EINVAL); 28201ae08745Sheppo } 28211ae08745Sheppo if (((uint64_t)cb) < KERNELBASE) { 28221ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_reg_callback: invalid callback\n"); 28231ae08745Sheppo return (EINVAL); 28241ae08745Sheppo } 28251ae08745Sheppo ldcp = (ldc_chan_t *)handle; 28261ae08745Sheppo 28271ae08745Sheppo mutex_enter(&ldcp->lock); 28281ae08745Sheppo 28291ae08745Sheppo if (ldcp->cb) { 28301ae08745Sheppo DWARN(ldcp->id, "ldc_reg_callback: (0x%llx) callback exists\n", 28311ae08745Sheppo ldcp->id); 28321ae08745Sheppo mutex_exit(&ldcp->lock); 28331ae08745Sheppo return (EIO); 28341ae08745Sheppo } 28351ae08745Sheppo if (ldcp->cb_inprogress) { 28361ae08745Sheppo DWARN(ldcp->id, "ldc_reg_callback: (0x%llx) callback active\n", 28371ae08745Sheppo ldcp->id); 28381ae08745Sheppo mutex_exit(&ldcp->lock); 28391ae08745Sheppo return (EWOULDBLOCK); 28401ae08745Sheppo } 28411ae08745Sheppo 28421ae08745Sheppo ldcp->cb = cb; 28431ae08745Sheppo ldcp->cb_arg = arg; 28441ae08745Sheppo ldcp->cb_enabled = B_TRUE; 28451ae08745Sheppo 28461ae08745Sheppo D1(ldcp->id, 28471ae08745Sheppo "ldc_reg_callback: (0x%llx) registered callback for channel\n", 28481ae08745Sheppo ldcp->id); 28491ae08745Sheppo 28501ae08745Sheppo mutex_exit(&ldcp->lock); 28511ae08745Sheppo 28521ae08745Sheppo return (0); 28531ae08745Sheppo } 28541ae08745Sheppo 28551ae08745Sheppo /* 28561ae08745Sheppo * Unregister channel callback 28571ae08745Sheppo */ 28581ae08745Sheppo int 28591ae08745Sheppo ldc_unreg_callback(ldc_handle_t handle) 28601ae08745Sheppo { 28611ae08745Sheppo ldc_chan_t *ldcp; 28621ae08745Sheppo 28631ae08745Sheppo if (handle == NULL) { 28641ae08745Sheppo DWARN(DBG_ALL_LDCS, 28651ae08745Sheppo "ldc_unreg_callback: invalid channel handle\n"); 28661ae08745Sheppo return (EINVAL); 28671ae08745Sheppo } 28681ae08745Sheppo ldcp = (ldc_chan_t *)handle; 28691ae08745Sheppo 28701ae08745Sheppo mutex_enter(&ldcp->lock); 28711ae08745Sheppo 28721ae08745Sheppo if (ldcp->cb == NULL) { 28731ae08745Sheppo DWARN(ldcp->id, 28741ae08745Sheppo "ldc_unreg_callback: (0x%llx) no callback exists\n", 28751ae08745Sheppo ldcp->id); 28761ae08745Sheppo mutex_exit(&ldcp->lock); 28771ae08745Sheppo return (EIO); 28781ae08745Sheppo } 28791ae08745Sheppo if (ldcp->cb_inprogress) { 28801ae08745Sheppo DWARN(ldcp->id, 28811ae08745Sheppo "ldc_unreg_callback: (0x%llx) callback active\n", 28821ae08745Sheppo ldcp->id); 28831ae08745Sheppo mutex_exit(&ldcp->lock); 28841ae08745Sheppo return (EWOULDBLOCK); 28851ae08745Sheppo } 28861ae08745Sheppo 28871ae08745Sheppo ldcp->cb = NULL; 28881ae08745Sheppo ldcp->cb_arg = NULL; 28891ae08745Sheppo ldcp->cb_enabled = B_FALSE; 28901ae08745Sheppo 28911ae08745Sheppo D1(ldcp->id, 28921ae08745Sheppo "ldc_unreg_callback: (0x%llx) unregistered callback for channel\n", 28931ae08745Sheppo ldcp->id); 28941ae08745Sheppo 28951ae08745Sheppo mutex_exit(&ldcp->lock); 28961ae08745Sheppo 28971ae08745Sheppo return (0); 28981ae08745Sheppo } 28991ae08745Sheppo 29001ae08745Sheppo 29011ae08745Sheppo /* 29021ae08745Sheppo * Bring a channel up by initiating a handshake with the peer 29031ae08745Sheppo * This call is asynchronous. It will complete at a later point 29041ae08745Sheppo * in time when the peer responds back with an RTR. 29051ae08745Sheppo */ 29061ae08745Sheppo int 29071ae08745Sheppo ldc_up(ldc_handle_t handle) 29081ae08745Sheppo { 29091ae08745Sheppo int rv; 29101ae08745Sheppo ldc_chan_t *ldcp; 29111ae08745Sheppo ldc_msg_t *ldcmsg; 291257e6a936Ssb155480 uint64_t tx_tail, tstate, link_state; 29131ae08745Sheppo 29141ae08745Sheppo if (handle == NULL) { 29151ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_up: invalid channel handle\n"); 29161ae08745Sheppo return (EINVAL); 29171ae08745Sheppo } 29181ae08745Sheppo ldcp = (ldc_chan_t *)handle; 29191ae08745Sheppo 29201ae08745Sheppo mutex_enter(&ldcp->lock); 29211ae08745Sheppo 29223af08d82Slm66018 D1(ldcp->id, "ldc_up: (0x%llx) doing channel UP\n", ldcp->id); 29233af08d82Slm66018 29243af08d82Slm66018 /* clear the reset state */ 29253af08d82Slm66018 tstate = ldcp->tstate; 29263af08d82Slm66018 ldcp->tstate &= ~TS_IN_RESET; 29273af08d82Slm66018 29281ae08745Sheppo if (ldcp->tstate == TS_UP) { 29293af08d82Slm66018 DWARN(ldcp->id, 29301ae08745Sheppo "ldc_up: (0x%llx) channel is already in UP state\n", 29311ae08745Sheppo ldcp->id); 29323af08d82Slm66018 29333af08d82Slm66018 /* mark channel as up */ 29343af08d82Slm66018 ldcp->status = LDC_UP; 29353af08d82Slm66018 29363af08d82Slm66018 /* 29373af08d82Slm66018 * if channel was in reset state and there was 29383af08d82Slm66018 * pending data clear interrupt state. this will 29393af08d82Slm66018 * trigger an interrupt, causing the RX handler to 29403af08d82Slm66018 * to invoke the client's callback 29413af08d82Slm66018 */ 29423af08d82Slm66018 if ((tstate & TS_IN_RESET) && 29433af08d82Slm66018 ldcp->rx_intr_state == LDC_INTR_PEND) { 2944cb112a14Slm66018 D1(ldcp->id, 29453af08d82Slm66018 "ldc_up: (0x%llx) channel has pending data, " 29463af08d82Slm66018 "clearing interrupt\n", ldcp->id); 29473af08d82Slm66018 i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 29483af08d82Slm66018 } 29493af08d82Slm66018 29501ae08745Sheppo mutex_exit(&ldcp->lock); 29511ae08745Sheppo return (0); 29521ae08745Sheppo } 29531ae08745Sheppo 29541ae08745Sheppo /* if the channel is in RAW mode - mark it as UP, if READY */ 29551ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW && ldcp->tstate >= TS_READY) { 29561ae08745Sheppo ldcp->tstate = TS_UP; 29571ae08745Sheppo mutex_exit(&ldcp->lock); 29581ae08745Sheppo return (0); 29591ae08745Sheppo } 29601ae08745Sheppo 29611ae08745Sheppo /* Don't start another handshake if there is one in progress */ 29621ae08745Sheppo if (ldcp->hstate) { 29633af08d82Slm66018 D1(ldcp->id, 29641ae08745Sheppo "ldc_up: (0x%llx) channel handshake in progress\n", 29651ae08745Sheppo ldcp->id); 29661ae08745Sheppo mutex_exit(&ldcp->lock); 29671ae08745Sheppo return (0); 29681ae08745Sheppo } 29691ae08745Sheppo 2970d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 2971d10e4ef2Snarayan 297257e6a936Ssb155480 /* save current link state */ 297357e6a936Ssb155480 link_state = ldcp->link_state; 297457e6a936Ssb155480 29751ae08745Sheppo /* get the current tail for the LDC msg */ 29761ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 29771ae08745Sheppo if (rv) { 2978cb112a14Slm66018 D1(ldcp->id, "ldc_up: (0x%llx) cannot initiate handshake\n", 29791ae08745Sheppo ldcp->id); 2980d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 29811ae08745Sheppo mutex_exit(&ldcp->lock); 29821ae08745Sheppo return (ECONNREFUSED); 29831ae08745Sheppo } 29841ae08745Sheppo 298557e6a936Ssb155480 /* 298657e6a936Ssb155480 * If i_ldc_get_tx_tail() changed link_state to either RESET or UP, 298757e6a936Ssb155480 * from a previous state of DOWN, then mark the channel as 298857e6a936Ssb155480 * being ready for handshake. 298957e6a936Ssb155480 */ 299057e6a936Ssb155480 if ((link_state == LDC_CHANNEL_DOWN) && 299157e6a936Ssb155480 (link_state != ldcp->link_state)) { 299257e6a936Ssb155480 299357e6a936Ssb155480 ASSERT((ldcp->link_state == LDC_CHANNEL_RESET) || 299457e6a936Ssb155480 (ldcp->link_state == LDC_CHANNEL_UP)); 299557e6a936Ssb155480 299657e6a936Ssb155480 if (ldcp->mode == LDC_MODE_RAW) { 299757e6a936Ssb155480 ldcp->status = LDC_UP; 299857e6a936Ssb155480 ldcp->tstate = TS_UP; 299957e6a936Ssb155480 mutex_exit(&ldcp->tx_lock); 300057e6a936Ssb155480 mutex_exit(&ldcp->lock); 300157e6a936Ssb155480 return (0); 300257e6a936Ssb155480 } else { 300357e6a936Ssb155480 ldcp->status = LDC_READY; 300457e6a936Ssb155480 ldcp->tstate |= TS_LINK_READY; 300557e6a936Ssb155480 } 300657e6a936Ssb155480 300757e6a936Ssb155480 } 300857e6a936Ssb155480 30091ae08745Sheppo ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 30101ae08745Sheppo ZERO_PKT(ldcmsg); 30111ae08745Sheppo 30121ae08745Sheppo ldcmsg->type = LDC_CTRL; 30131ae08745Sheppo ldcmsg->stype = LDC_INFO; 30141ae08745Sheppo ldcmsg->ctrl = LDC_VER; 30151ae08745Sheppo ldcp->next_vidx = 0; 30161ae08745Sheppo bcopy(&ldc_versions[0], ldcmsg->udata, sizeof (ldc_versions[0])); 30171ae08745Sheppo 30181ae08745Sheppo DUMP_LDC_PKT(ldcp, "ldc_up snd ver", (uint64_t)ldcmsg); 30191ae08745Sheppo 30201ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 30211ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 30221ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 30231ae08745Sheppo 30241ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 30251ae08745Sheppo if (rv) { 30261ae08745Sheppo DWARN(ldcp->id, 30271ae08745Sheppo "ldc_up: (0x%llx) cannot initiate handshake rv=%d\n", 30281ae08745Sheppo ldcp->id, rv); 3029d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 30301ae08745Sheppo mutex_exit(&ldcp->lock); 30311ae08745Sheppo return (rv); 30321ae08745Sheppo } 30331ae08745Sheppo 30340a55fbb7Slm66018 ldcp->hstate |= TS_SENT_VER; 30351ae08745Sheppo ldcp->tx_tail = tx_tail; 30361ae08745Sheppo D1(ldcp->id, "ldc_up: (0x%llx) channel up initiated\n", ldcp->id); 30371ae08745Sheppo 3038d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 30391ae08745Sheppo mutex_exit(&ldcp->lock); 30401ae08745Sheppo 30411ae08745Sheppo return (rv); 30421ae08745Sheppo } 30431ae08745Sheppo 30441ae08745Sheppo 30451ae08745Sheppo /* 3046e1ebb9ecSlm66018 * Bring a channel down by resetting its state and queues 30471ae08745Sheppo */ 30481ae08745Sheppo int 3049e1ebb9ecSlm66018 ldc_down(ldc_handle_t handle) 30501ae08745Sheppo { 30511ae08745Sheppo ldc_chan_t *ldcp; 30521ae08745Sheppo 30531ae08745Sheppo if (handle == NULL) { 3054e1ebb9ecSlm66018 DWARN(DBG_ALL_LDCS, "ldc_down: invalid channel handle\n"); 30551ae08745Sheppo return (EINVAL); 30561ae08745Sheppo } 30571ae08745Sheppo ldcp = (ldc_chan_t *)handle; 30581ae08745Sheppo mutex_enter(&ldcp->lock); 3059d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 30603af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 3061d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 30621ae08745Sheppo mutex_exit(&ldcp->lock); 30631ae08745Sheppo 30641ae08745Sheppo return (0); 30651ae08745Sheppo } 30661ae08745Sheppo 30671ae08745Sheppo /* 30681ae08745Sheppo * Get the current channel status 30691ae08745Sheppo */ 30701ae08745Sheppo int 30711ae08745Sheppo ldc_status(ldc_handle_t handle, ldc_status_t *status) 30721ae08745Sheppo { 30731ae08745Sheppo ldc_chan_t *ldcp; 30741ae08745Sheppo 30751ae08745Sheppo if (handle == NULL || status == NULL) { 30761ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_status: invalid argument\n"); 30771ae08745Sheppo return (EINVAL); 30781ae08745Sheppo } 30791ae08745Sheppo ldcp = (ldc_chan_t *)handle; 30801ae08745Sheppo 30811ae08745Sheppo *status = ((ldc_chan_t *)handle)->status; 30821ae08745Sheppo 3083cb112a14Slm66018 D1(ldcp->id, 30841ae08745Sheppo "ldc_status: (0x%llx) returned status %d\n", ldcp->id, *status); 30851ae08745Sheppo return (0); 30861ae08745Sheppo } 30871ae08745Sheppo 30881ae08745Sheppo 30891ae08745Sheppo /* 30901ae08745Sheppo * Set the channel's callback mode - enable/disable callbacks 30911ae08745Sheppo */ 30921ae08745Sheppo int 30931ae08745Sheppo ldc_set_cb_mode(ldc_handle_t handle, ldc_cb_mode_t cmode) 30941ae08745Sheppo { 30951ae08745Sheppo ldc_chan_t *ldcp; 30961ae08745Sheppo 30971ae08745Sheppo if (handle == NULL) { 30981ae08745Sheppo DWARN(DBG_ALL_LDCS, 30991ae08745Sheppo "ldc_set_intr_mode: invalid channel handle\n"); 31001ae08745Sheppo return (EINVAL); 31011ae08745Sheppo } 31021ae08745Sheppo ldcp = (ldc_chan_t *)handle; 31031ae08745Sheppo 31041ae08745Sheppo /* 31051ae08745Sheppo * Record no callbacks should be invoked 31061ae08745Sheppo */ 31071ae08745Sheppo mutex_enter(&ldcp->lock); 31081ae08745Sheppo 31091ae08745Sheppo switch (cmode) { 31101ae08745Sheppo case LDC_CB_DISABLE: 31111ae08745Sheppo if (!ldcp->cb_enabled) { 31121ae08745Sheppo DWARN(ldcp->id, 31131ae08745Sheppo "ldc_set_cb_mode: (0x%llx) callbacks disabled\n", 31141ae08745Sheppo ldcp->id); 31151ae08745Sheppo break; 31161ae08745Sheppo } 31171ae08745Sheppo ldcp->cb_enabled = B_FALSE; 31181ae08745Sheppo 31191ae08745Sheppo D1(ldcp->id, "ldc_set_cb_mode: (0x%llx) disabled callbacks\n", 31201ae08745Sheppo ldcp->id); 31211ae08745Sheppo break; 31221ae08745Sheppo 31231ae08745Sheppo case LDC_CB_ENABLE: 31241ae08745Sheppo if (ldcp->cb_enabled) { 31251ae08745Sheppo DWARN(ldcp->id, 31261ae08745Sheppo "ldc_set_cb_mode: (0x%llx) callbacks enabled\n", 31271ae08745Sheppo ldcp->id); 31281ae08745Sheppo break; 31291ae08745Sheppo } 31301ae08745Sheppo ldcp->cb_enabled = B_TRUE; 31311ae08745Sheppo 31321ae08745Sheppo D1(ldcp->id, "ldc_set_cb_mode: (0x%llx) enabled callbacks\n", 31331ae08745Sheppo ldcp->id); 31341ae08745Sheppo break; 31351ae08745Sheppo } 31361ae08745Sheppo 31371ae08745Sheppo mutex_exit(&ldcp->lock); 31381ae08745Sheppo 31391ae08745Sheppo return (0); 31401ae08745Sheppo } 31411ae08745Sheppo 31421ae08745Sheppo /* 31431ae08745Sheppo * Check to see if there are packets on the incoming queue 3144e1ebb9ecSlm66018 * Will return hasdata = B_FALSE if there are no packets 31451ae08745Sheppo */ 31461ae08745Sheppo int 3147e1ebb9ecSlm66018 ldc_chkq(ldc_handle_t handle, boolean_t *hasdata) 31481ae08745Sheppo { 31491ae08745Sheppo int rv; 31501ae08745Sheppo uint64_t rx_head, rx_tail; 31511ae08745Sheppo ldc_chan_t *ldcp; 31521ae08745Sheppo 31531ae08745Sheppo if (handle == NULL) { 31541ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_chkq: invalid channel handle\n"); 31551ae08745Sheppo return (EINVAL); 31561ae08745Sheppo } 31571ae08745Sheppo ldcp = (ldc_chan_t *)handle; 31581ae08745Sheppo 3159e1ebb9ecSlm66018 *hasdata = B_FALSE; 31601ae08745Sheppo 31611ae08745Sheppo mutex_enter(&ldcp->lock); 31621ae08745Sheppo 31631ae08745Sheppo if (ldcp->tstate != TS_UP) { 31641ae08745Sheppo D1(ldcp->id, 31651ae08745Sheppo "ldc_chkq: (0x%llx) channel is not up\n", ldcp->id); 31661ae08745Sheppo mutex_exit(&ldcp->lock); 31671ae08745Sheppo return (ECONNRESET); 31681ae08745Sheppo } 31691ae08745Sheppo 31701ae08745Sheppo /* Read packet(s) from the queue */ 31711ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 31721ae08745Sheppo &ldcp->link_state); 31731ae08745Sheppo if (rv != 0) { 31741ae08745Sheppo cmn_err(CE_WARN, 31751ae08745Sheppo "ldc_chkq: (0x%lx) unable to read queue ptrs", ldcp->id); 31761ae08745Sheppo mutex_exit(&ldcp->lock); 31771ae08745Sheppo return (EIO); 31781ae08745Sheppo } 31791ae08745Sheppo /* reset the channel state if the channel went down */ 31801ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN || 31811ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 3182d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 31833af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 3184d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 31851ae08745Sheppo mutex_exit(&ldcp->lock); 31861ae08745Sheppo return (ECONNRESET); 31871ae08745Sheppo } 31881ae08745Sheppo 31894bac2208Snarayan if ((rx_head != rx_tail) || 31904bac2208Snarayan (ldcp->mode == LDC_MODE_STREAM && ldcp->stream_remains > 0)) { 31914bac2208Snarayan D1(ldcp->id, 31924bac2208Snarayan "ldc_chkq: (0x%llx) queue has pkt(s) or buffered data\n", 31934bac2208Snarayan ldcp->id); 3194e1ebb9ecSlm66018 *hasdata = B_TRUE; 31951ae08745Sheppo } 31961ae08745Sheppo 31971ae08745Sheppo mutex_exit(&ldcp->lock); 31981ae08745Sheppo 31991ae08745Sheppo return (0); 32001ae08745Sheppo } 32011ae08745Sheppo 32021ae08745Sheppo 32031ae08745Sheppo /* 32041ae08745Sheppo * Read 'size' amount of bytes or less. If incoming buffer 32051ae08745Sheppo * is more than 'size', ENOBUFS is returned. 32061ae08745Sheppo * 32071ae08745Sheppo * On return, size contains the number of bytes read. 32081ae08745Sheppo */ 32091ae08745Sheppo int 32101ae08745Sheppo ldc_read(ldc_handle_t handle, caddr_t bufp, size_t *sizep) 32111ae08745Sheppo { 32121ae08745Sheppo ldc_chan_t *ldcp; 32131ae08745Sheppo uint64_t rx_head = 0, rx_tail = 0; 32141ae08745Sheppo int rv = 0, exit_val; 32151ae08745Sheppo 32161ae08745Sheppo if (handle == NULL) { 32171ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_read: invalid channel handle\n"); 32181ae08745Sheppo return (EINVAL); 32191ae08745Sheppo } 32201ae08745Sheppo 32211ae08745Sheppo ldcp = (ldc_chan_t *)handle; 32221ae08745Sheppo 32231ae08745Sheppo /* channel lock */ 32241ae08745Sheppo mutex_enter(&ldcp->lock); 32251ae08745Sheppo 32261ae08745Sheppo if (ldcp->tstate != TS_UP) { 32271ae08745Sheppo DWARN(ldcp->id, 32281ae08745Sheppo "ldc_read: (0x%llx) channel is not in UP state\n", 32291ae08745Sheppo ldcp->id); 32301ae08745Sheppo exit_val = ECONNRESET; 32311ae08745Sheppo } else { 32321ae08745Sheppo exit_val = ldcp->read_p(ldcp, bufp, sizep); 32331ae08745Sheppo } 32341ae08745Sheppo 32351ae08745Sheppo /* 32361ae08745Sheppo * if queue has been drained - clear interrupt 32371ae08745Sheppo */ 32381ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 32391ae08745Sheppo &ldcp->link_state); 3240cb112a14Slm66018 if (rv != 0) { 3241cb112a14Slm66018 cmn_err(CE_WARN, "ldc_read: (0x%lx) unable to read queue ptrs", 3242cb112a14Slm66018 ldcp->id); 3243cb112a14Slm66018 mutex_enter(&ldcp->tx_lock); 3244cb112a14Slm66018 i_ldc_reset(ldcp, B_TRUE); 3245cb112a14Slm66018 mutex_exit(&ldcp->tx_lock); 3246bd8f0338Snarayan mutex_exit(&ldcp->lock); 3247cb112a14Slm66018 return (ECONNRESET); 3248cb112a14Slm66018 } 32493af08d82Slm66018 32503af08d82Slm66018 if (exit_val == 0) { 32513af08d82Slm66018 if (ldcp->link_state == LDC_CHANNEL_DOWN || 32523af08d82Slm66018 ldcp->link_state == LDC_CHANNEL_RESET) { 32533af08d82Slm66018 mutex_enter(&ldcp->tx_lock); 32543af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 32553af08d82Slm66018 exit_val = ECONNRESET; 32563af08d82Slm66018 mutex_exit(&ldcp->tx_lock); 32573af08d82Slm66018 } 32583af08d82Slm66018 if ((rv == 0) && 32593af08d82Slm66018 (ldcp->rx_intr_state == LDC_INTR_PEND) && 32603af08d82Slm66018 (rx_head == rx_tail)) { 32611ae08745Sheppo i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 32621ae08745Sheppo } 32633af08d82Slm66018 } 32641ae08745Sheppo 32651ae08745Sheppo mutex_exit(&ldcp->lock); 32661ae08745Sheppo return (exit_val); 32671ae08745Sheppo } 32681ae08745Sheppo 32691ae08745Sheppo /* 32701ae08745Sheppo * Basic raw mondo read - 32711ae08745Sheppo * no interpretation of mondo contents at all. 32721ae08745Sheppo * 32731ae08745Sheppo * Enter and exit with ldcp->lock held by caller 32741ae08745Sheppo */ 32751ae08745Sheppo static int 32761ae08745Sheppo i_ldc_read_raw(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 32771ae08745Sheppo { 32781ae08745Sheppo uint64_t q_size_mask; 32791ae08745Sheppo ldc_msg_t *msgp; 32801ae08745Sheppo uint8_t *msgbufp; 32811ae08745Sheppo int rv = 0, space; 32821ae08745Sheppo uint64_t rx_head, rx_tail; 32831ae08745Sheppo 32841ae08745Sheppo space = *sizep; 32851ae08745Sheppo 32861ae08745Sheppo if (space < LDC_PAYLOAD_SIZE_RAW) 32871ae08745Sheppo return (ENOBUFS); 32881ae08745Sheppo 32891ae08745Sheppo ASSERT(mutex_owned(&ldcp->lock)); 32901ae08745Sheppo 32911ae08745Sheppo /* compute mask for increment */ 32921ae08745Sheppo q_size_mask = (ldcp->rx_q_entries-1)<<LDC_PACKET_SHIFT; 32931ae08745Sheppo 32941ae08745Sheppo /* 32951ae08745Sheppo * Read packet(s) from the queue 32961ae08745Sheppo */ 32971ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 32981ae08745Sheppo &ldcp->link_state); 32991ae08745Sheppo if (rv != 0) { 33001ae08745Sheppo cmn_err(CE_WARN, 33011ae08745Sheppo "ldc_read_raw: (0x%lx) unable to read queue ptrs", 33021ae08745Sheppo ldcp->id); 33031ae08745Sheppo return (EIO); 33041ae08745Sheppo } 33051ae08745Sheppo D1(ldcp->id, "ldc_read_raw: (0x%llx) rxh=0x%llx," 33061ae08745Sheppo " rxt=0x%llx, st=0x%llx\n", 33071ae08745Sheppo ldcp->id, rx_head, rx_tail, ldcp->link_state); 33081ae08745Sheppo 33091ae08745Sheppo /* reset the channel state if the channel went down */ 33103af08d82Slm66018 if (ldcp->link_state == LDC_CHANNEL_DOWN || 33113af08d82Slm66018 ldcp->link_state == LDC_CHANNEL_RESET) { 3312d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 33133af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 3314d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 33151ae08745Sheppo return (ECONNRESET); 33161ae08745Sheppo } 33171ae08745Sheppo 33181ae08745Sheppo /* 33191ae08745Sheppo * Check for empty queue 33201ae08745Sheppo */ 33211ae08745Sheppo if (rx_head == rx_tail) { 33221ae08745Sheppo *sizep = 0; 33231ae08745Sheppo return (0); 33241ae08745Sheppo } 33251ae08745Sheppo 33261ae08745Sheppo /* get the message */ 33271ae08745Sheppo msgp = (ldc_msg_t *)(ldcp->rx_q_va + rx_head); 33281ae08745Sheppo 33291ae08745Sheppo /* if channel is in RAW mode, copy data and return */ 33301ae08745Sheppo msgbufp = (uint8_t *)&(msgp->raw[0]); 33311ae08745Sheppo 33321ae08745Sheppo bcopy(msgbufp, target_bufp, LDC_PAYLOAD_SIZE_RAW); 33331ae08745Sheppo 33341ae08745Sheppo DUMP_PAYLOAD(ldcp->id, msgbufp); 33351ae08745Sheppo 33361ae08745Sheppo *sizep = LDC_PAYLOAD_SIZE_RAW; 33371ae08745Sheppo 33381ae08745Sheppo rx_head = (rx_head + LDC_PACKET_SIZE) & q_size_mask; 33390a55fbb7Slm66018 rv = i_ldc_set_rx_head(ldcp, rx_head); 33401ae08745Sheppo 33411ae08745Sheppo return (rv); 33421ae08745Sheppo } 33431ae08745Sheppo 33441ae08745Sheppo /* 33451ae08745Sheppo * Process LDC mondos to build larger packets 33461ae08745Sheppo * with either un-reliable or reliable delivery. 33471ae08745Sheppo * 33481ae08745Sheppo * Enter and exit with ldcp->lock held by caller 33491ae08745Sheppo */ 33501ae08745Sheppo static int 33511ae08745Sheppo i_ldc_read_packet(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 33521ae08745Sheppo { 33531ae08745Sheppo int rv = 0; 33541ae08745Sheppo uint64_t rx_head = 0, rx_tail = 0; 33551ae08745Sheppo uint64_t curr_head = 0; 33561ae08745Sheppo ldc_msg_t *msg; 33571ae08745Sheppo caddr_t target; 33581ae08745Sheppo size_t len = 0, bytes_read = 0; 33590a55fbb7Slm66018 int retries = 0; 33601ae08745Sheppo uint64_t q_size_mask; 3361d10e4ef2Snarayan uint64_t first_fragment = 0; 33621ae08745Sheppo 33631ae08745Sheppo target = target_bufp; 33641ae08745Sheppo 33651ae08745Sheppo ASSERT(mutex_owned(&ldcp->lock)); 33661ae08745Sheppo 33673af08d82Slm66018 /* check if the buffer and size are valid */ 33683af08d82Slm66018 if (target_bufp == NULL || *sizep == 0) { 33693af08d82Slm66018 DWARN(ldcp->id, "ldc_read: (0x%llx) invalid buffer/size\n", 33703af08d82Slm66018 ldcp->id); 33713af08d82Slm66018 return (EINVAL); 33723af08d82Slm66018 } 33733af08d82Slm66018 33741ae08745Sheppo /* compute mask for increment */ 33751ae08745Sheppo q_size_mask = (ldcp->rx_q_entries-1)<<LDC_PACKET_SHIFT; 33761ae08745Sheppo 33771ae08745Sheppo /* 33781ae08745Sheppo * Read packet(s) from the queue 33791ae08745Sheppo */ 33801ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &curr_head, &rx_tail, 33811ae08745Sheppo &ldcp->link_state); 33821ae08745Sheppo if (rv != 0) { 33833af08d82Slm66018 cmn_err(CE_WARN, "ldc_read: (0x%lx) unable to read queue ptrs", 33841ae08745Sheppo ldcp->id); 33853af08d82Slm66018 mutex_enter(&ldcp->tx_lock); 33863af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 33873af08d82Slm66018 mutex_exit(&ldcp->tx_lock); 33883af08d82Slm66018 return (ECONNRESET); 33891ae08745Sheppo } 33901ae08745Sheppo D1(ldcp->id, "ldc_read: (0x%llx) chd=0x%llx, tl=0x%llx, st=0x%llx\n", 33911ae08745Sheppo ldcp->id, curr_head, rx_tail, ldcp->link_state); 33921ae08745Sheppo 33931ae08745Sheppo /* reset the channel state if the channel went down */ 33943af08d82Slm66018 if (ldcp->link_state != LDC_CHANNEL_UP) 33953af08d82Slm66018 goto channel_is_reset; 33961ae08745Sheppo 33971ae08745Sheppo for (;;) { 33981ae08745Sheppo 33991ae08745Sheppo if (curr_head == rx_tail) { 34001ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, 34011ae08745Sheppo &rx_head, &rx_tail, &ldcp->link_state); 34021ae08745Sheppo if (rv != 0) { 34031ae08745Sheppo cmn_err(CE_WARN, 34041ae08745Sheppo "ldc_read: (0x%lx) cannot read queue ptrs", 34051ae08745Sheppo ldcp->id); 3406d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 34073af08d82Slm66018 i_ldc_reset(ldcp, B_TRUE); 3408d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 34091ae08745Sheppo return (ECONNRESET); 34101ae08745Sheppo } 34113af08d82Slm66018 if (ldcp->link_state != LDC_CHANNEL_UP) 34123af08d82Slm66018 goto channel_is_reset; 34131ae08745Sheppo 34141ae08745Sheppo if (curr_head == rx_tail) { 34151ae08745Sheppo 34161ae08745Sheppo /* If in the middle of a fragmented xfer */ 3417d10e4ef2Snarayan if (first_fragment != 0) { 34180a55fbb7Slm66018 34190a55fbb7Slm66018 /* wait for ldc_delay usecs */ 34200a55fbb7Slm66018 drv_usecwait(ldc_delay); 34210a55fbb7Slm66018 34220a55fbb7Slm66018 if (++retries < ldc_max_retries) 34231ae08745Sheppo continue; 34240a55fbb7Slm66018 34251ae08745Sheppo *sizep = 0; 3426d10e4ef2Snarayan ldcp->last_msg_rcd = first_fragment - 1; 34273af08d82Slm66018 DWARN(DBG_ALL_LDCS, "ldc_read: " 34283af08d82Slm66018 "(0x%llx) read timeout", 34291ae08745Sheppo ldcp->id); 34303af08d82Slm66018 return (EAGAIN); 34311ae08745Sheppo } 34321ae08745Sheppo *sizep = 0; 34331ae08745Sheppo break; 34341ae08745Sheppo } 34353af08d82Slm66018 } 34360a55fbb7Slm66018 retries = 0; 34371ae08745Sheppo 34381ae08745Sheppo D2(ldcp->id, 34391ae08745Sheppo "ldc_read: (0x%llx) chd=0x%llx, rxhd=0x%llx, rxtl=0x%llx\n", 34401ae08745Sheppo ldcp->id, curr_head, rx_head, rx_tail); 34411ae08745Sheppo 34421ae08745Sheppo /* get the message */ 34431ae08745Sheppo msg = (ldc_msg_t *)(ldcp->rx_q_va + curr_head); 34441ae08745Sheppo 34451ae08745Sheppo DUMP_LDC_PKT(ldcp, "ldc_read received pkt", 34461ae08745Sheppo ldcp->rx_q_va + curr_head); 34471ae08745Sheppo 34481ae08745Sheppo /* Check the message ID for the message received */ 34491ae08745Sheppo if ((rv = i_ldc_check_seqid(ldcp, msg)) != 0) { 34501ae08745Sheppo 34511ae08745Sheppo DWARN(ldcp->id, "ldc_read: (0x%llx) seqid error, " 34521ae08745Sheppo "q_ptrs=0x%lx,0x%lx", ldcp->id, rx_head, rx_tail); 34531ae08745Sheppo 34540a55fbb7Slm66018 /* throw away data */ 34550a55fbb7Slm66018 bytes_read = 0; 34560a55fbb7Slm66018 34571ae08745Sheppo /* Reset last_msg_rcd to start of message */ 3458d10e4ef2Snarayan if (first_fragment != 0) { 3459d10e4ef2Snarayan ldcp->last_msg_rcd = first_fragment - 1; 3460d10e4ef2Snarayan first_fragment = 0; 34611ae08745Sheppo } 34621ae08745Sheppo /* 34631ae08745Sheppo * Send a NACK -- invalid seqid 34641ae08745Sheppo * get the current tail for the response 34651ae08745Sheppo */ 34661ae08745Sheppo rv = i_ldc_send_pkt(ldcp, msg->type, LDC_NACK, 34671ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK)); 34681ae08745Sheppo if (rv) { 34691ae08745Sheppo cmn_err(CE_NOTE, 34701ae08745Sheppo "ldc_read: (0x%lx) err sending " 34711ae08745Sheppo "NACK msg\n", ldcp->id); 3472d10e4ef2Snarayan 3473d10e4ef2Snarayan /* if cannot send NACK - reset channel */ 3474d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 34753af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 3476d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3477d10e4ef2Snarayan rv = ECONNRESET; 3478d10e4ef2Snarayan break; 34791ae08745Sheppo } 34801ae08745Sheppo 34811ae08745Sheppo /* purge receive queue */ 34820a55fbb7Slm66018 rv = i_ldc_set_rx_head(ldcp, rx_tail); 34831ae08745Sheppo 34841ae08745Sheppo break; 34851ae08745Sheppo } 34861ae08745Sheppo 34871ae08745Sheppo /* 34881ae08745Sheppo * Process any messages of type CTRL messages 3489e1ebb9ecSlm66018 * Future implementations should try to pass these 3490e1ebb9ecSlm66018 * to LDC link by resetting the intr state. 34911ae08745Sheppo * 34921ae08745Sheppo * NOTE: not done as a switch() as type can be both ctrl+data 34931ae08745Sheppo */ 34941ae08745Sheppo if (msg->type & LDC_CTRL) { 34951ae08745Sheppo if (rv = i_ldc_ctrlmsg(ldcp, msg)) { 34961ae08745Sheppo if (rv == EAGAIN) 34971ae08745Sheppo continue; 34980a55fbb7Slm66018 rv = i_ldc_set_rx_head(ldcp, rx_tail); 34991ae08745Sheppo *sizep = 0; 35001ae08745Sheppo bytes_read = 0; 35011ae08745Sheppo break; 35021ae08745Sheppo } 35031ae08745Sheppo } 35041ae08745Sheppo 35051ae08745Sheppo /* process data ACKs */ 35061ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 3507d10e4ef2Snarayan if (rv = i_ldc_process_data_ACK(ldcp, msg)) { 3508d10e4ef2Snarayan *sizep = 0; 3509d10e4ef2Snarayan bytes_read = 0; 3510d10e4ef2Snarayan break; 3511d10e4ef2Snarayan } 35121ae08745Sheppo } 35131ae08745Sheppo 351483d3bc6fSnarayan /* process data NACKs */ 351583d3bc6fSnarayan if ((msg->type & LDC_DATA) && (msg->stype & LDC_NACK)) { 351683d3bc6fSnarayan DWARN(ldcp->id, 351783d3bc6fSnarayan "ldc_read: (0x%llx) received DATA/NACK", ldcp->id); 351883d3bc6fSnarayan mutex_enter(&ldcp->tx_lock); 351983d3bc6fSnarayan i_ldc_reset(ldcp, B_TRUE); 352083d3bc6fSnarayan mutex_exit(&ldcp->tx_lock); 352183d3bc6fSnarayan return (ECONNRESET); 352283d3bc6fSnarayan } 352383d3bc6fSnarayan 35241ae08745Sheppo /* process data messages */ 35251ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_INFO)) { 35261ae08745Sheppo 35271ae08745Sheppo uint8_t *msgbuf = (uint8_t *)( 35281ae08745Sheppo (ldcp->mode == LDC_MODE_RELIABLE || 35291ae08745Sheppo ldcp->mode == LDC_MODE_STREAM) 35301ae08745Sheppo ? msg->rdata : msg->udata); 35311ae08745Sheppo 35321ae08745Sheppo D2(ldcp->id, 35331ae08745Sheppo "ldc_read: (0x%llx) received data msg\n", ldcp->id); 35341ae08745Sheppo 35351ae08745Sheppo /* get the packet length */ 35361ae08745Sheppo len = (msg->env & LDC_LEN_MASK); 35371ae08745Sheppo 35381ae08745Sheppo /* 35391ae08745Sheppo * FUTURE OPTIMIZATION: 35401ae08745Sheppo * dont need to set q head for every 35411ae08745Sheppo * packet we read just need to do this when 35421ae08745Sheppo * we are done or need to wait for more 35431ae08745Sheppo * mondos to make a full packet - this is 35441ae08745Sheppo * currently expensive. 35451ae08745Sheppo */ 35461ae08745Sheppo 3547d10e4ef2Snarayan if (first_fragment == 0) { 35481ae08745Sheppo 35491ae08745Sheppo /* 35501ae08745Sheppo * first packets should always have the start 35511ae08745Sheppo * bit set (even for a single packet). If not 35521ae08745Sheppo * throw away the packet 35531ae08745Sheppo */ 35541ae08745Sheppo if (!(msg->env & LDC_FRAG_START)) { 35551ae08745Sheppo 35561ae08745Sheppo DWARN(DBG_ALL_LDCS, 35571ae08745Sheppo "ldc_read: (0x%llx) not start - " 35581ae08745Sheppo "frag=%x\n", ldcp->id, 35591ae08745Sheppo (msg->env) & LDC_FRAG_MASK); 35601ae08745Sheppo 35611ae08745Sheppo /* toss pkt, inc head, cont reading */ 35621ae08745Sheppo bytes_read = 0; 35631ae08745Sheppo target = target_bufp; 35641ae08745Sheppo curr_head = 35651ae08745Sheppo (curr_head + LDC_PACKET_SIZE) 35661ae08745Sheppo & q_size_mask; 35671ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, 35681ae08745Sheppo curr_head)) 35691ae08745Sheppo break; 35701ae08745Sheppo 35711ae08745Sheppo continue; 35721ae08745Sheppo } 35731ae08745Sheppo 3574d10e4ef2Snarayan first_fragment = msg->seqid; 35751ae08745Sheppo } else { 35761ae08745Sheppo /* check to see if this is a pkt w/ START bit */ 35771ae08745Sheppo if (msg->env & LDC_FRAG_START) { 35781ae08745Sheppo DWARN(DBG_ALL_LDCS, 35791ae08745Sheppo "ldc_read:(0x%llx) unexpected pkt" 35801ae08745Sheppo " env=0x%x discarding %d bytes," 35811ae08745Sheppo " lastmsg=%d, currentmsg=%d\n", 35821ae08745Sheppo ldcp->id, msg->env&LDC_FRAG_MASK, 35831ae08745Sheppo bytes_read, ldcp->last_msg_rcd, 35841ae08745Sheppo msg->seqid); 35851ae08745Sheppo 35861ae08745Sheppo /* throw data we have read so far */ 35871ae08745Sheppo bytes_read = 0; 35881ae08745Sheppo target = target_bufp; 3589d10e4ef2Snarayan first_fragment = msg->seqid; 35901ae08745Sheppo 35911ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, 35921ae08745Sheppo curr_head)) 35931ae08745Sheppo break; 35941ae08745Sheppo } 35951ae08745Sheppo } 35961ae08745Sheppo 35971ae08745Sheppo /* copy (next) pkt into buffer */ 35981ae08745Sheppo if (len <= (*sizep - bytes_read)) { 35991ae08745Sheppo bcopy(msgbuf, target, len); 36001ae08745Sheppo target += len; 36011ae08745Sheppo bytes_read += len; 36021ae08745Sheppo } else { 36031ae08745Sheppo /* 36041ae08745Sheppo * there is not enough space in the buffer to 36051ae08745Sheppo * read this pkt. throw message away & continue 36061ae08745Sheppo * reading data from queue 36071ae08745Sheppo */ 36081ae08745Sheppo DWARN(DBG_ALL_LDCS, 36091ae08745Sheppo "ldc_read: (0x%llx) buffer too small, " 36101ae08745Sheppo "head=0x%lx, expect=%d, got=%d\n", ldcp->id, 36111ae08745Sheppo curr_head, *sizep, bytes_read+len); 36121ae08745Sheppo 3613d10e4ef2Snarayan first_fragment = 0; 36141ae08745Sheppo target = target_bufp; 36151ae08745Sheppo bytes_read = 0; 36161ae08745Sheppo 36171ae08745Sheppo /* throw away everything received so far */ 36181ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, curr_head)) 36191ae08745Sheppo break; 36201ae08745Sheppo 36211ae08745Sheppo /* continue reading remaining pkts */ 36221ae08745Sheppo continue; 36231ae08745Sheppo } 36241ae08745Sheppo } 36251ae08745Sheppo 36261ae08745Sheppo /* set the message id */ 36271ae08745Sheppo ldcp->last_msg_rcd = msg->seqid; 36281ae08745Sheppo 36291ae08745Sheppo /* move the head one position */ 36301ae08745Sheppo curr_head = (curr_head + LDC_PACKET_SIZE) & q_size_mask; 36311ae08745Sheppo 36321ae08745Sheppo if (msg->env & LDC_FRAG_STOP) { 36331ae08745Sheppo 36341ae08745Sheppo /* 36351ae08745Sheppo * All pkts that are part of this fragmented transfer 36361ae08745Sheppo * have been read or this was a single pkt read 36371ae08745Sheppo * or there was an error 36381ae08745Sheppo */ 36391ae08745Sheppo 36401ae08745Sheppo /* set the queue head */ 36411ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, curr_head)) 36421ae08745Sheppo bytes_read = 0; 36431ae08745Sheppo 36441ae08745Sheppo *sizep = bytes_read; 36451ae08745Sheppo 36461ae08745Sheppo break; 36471ae08745Sheppo } 36481ae08745Sheppo 36491ae08745Sheppo /* advance head if it is a DATA ACK */ 36501ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 36511ae08745Sheppo 36521ae08745Sheppo /* set the queue head */ 36531ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, curr_head)) { 36541ae08745Sheppo bytes_read = 0; 36551ae08745Sheppo break; 36561ae08745Sheppo } 36571ae08745Sheppo 36581ae08745Sheppo D2(ldcp->id, "ldc_read: (0x%llx) set ACK qhead 0x%llx", 36591ae08745Sheppo ldcp->id, curr_head); 36601ae08745Sheppo } 36611ae08745Sheppo 36621ae08745Sheppo } /* for (;;) */ 36631ae08745Sheppo 36641ae08745Sheppo 36651ae08745Sheppo /* 36661ae08745Sheppo * If useful data was read - Send msg ACK 36671ae08745Sheppo * OPTIMIZE: do not send ACK for all msgs - use some frequency 36681ae08745Sheppo */ 36691ae08745Sheppo if ((bytes_read > 0) && (ldcp->mode == LDC_MODE_RELIABLE || 36701ae08745Sheppo ldcp->mode == LDC_MODE_STREAM)) { 36711ae08745Sheppo 36721ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_DATA, LDC_ACK, 0); 36733af08d82Slm66018 if (rv && rv != EWOULDBLOCK) { 36741ae08745Sheppo cmn_err(CE_NOTE, 36751ae08745Sheppo "ldc_read: (0x%lx) cannot send ACK\n", ldcp->id); 3676d10e4ef2Snarayan 3677d10e4ef2Snarayan /* if cannot send ACK - reset channel */ 36783af08d82Slm66018 goto channel_is_reset; 36791ae08745Sheppo } 36801ae08745Sheppo } 36811ae08745Sheppo 36821ae08745Sheppo D2(ldcp->id, "ldc_read: (0x%llx) end size=%d", ldcp->id, *sizep); 36831ae08745Sheppo 36841ae08745Sheppo return (rv); 36853af08d82Slm66018 36863af08d82Slm66018 channel_is_reset: 36873af08d82Slm66018 mutex_enter(&ldcp->tx_lock); 36883af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 36893af08d82Slm66018 mutex_exit(&ldcp->tx_lock); 36903af08d82Slm66018 return (ECONNRESET); 36911ae08745Sheppo } 36921ae08745Sheppo 36931ae08745Sheppo /* 36941ae08745Sheppo * Use underlying reliable packet mechanism to fetch 36951ae08745Sheppo * and buffer incoming packets so we can hand them back as 36961ae08745Sheppo * a basic byte stream. 36971ae08745Sheppo * 36981ae08745Sheppo * Enter and exit with ldcp->lock held by caller 36991ae08745Sheppo */ 37001ae08745Sheppo static int 37011ae08745Sheppo i_ldc_read_stream(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 37021ae08745Sheppo { 37031ae08745Sheppo int rv; 37041ae08745Sheppo size_t size; 37051ae08745Sheppo 37061ae08745Sheppo ASSERT(mutex_owned(&ldcp->lock)); 37071ae08745Sheppo 37081ae08745Sheppo D2(ldcp->id, "i_ldc_read_stream: (0x%llx) buffer size=%d", 37091ae08745Sheppo ldcp->id, *sizep); 37101ae08745Sheppo 37111ae08745Sheppo if (ldcp->stream_remains == 0) { 37121ae08745Sheppo size = ldcp->mtu; 37131ae08745Sheppo rv = i_ldc_read_packet(ldcp, 37141ae08745Sheppo (caddr_t)ldcp->stream_bufferp, &size); 37151ae08745Sheppo D2(ldcp->id, "i_ldc_read_stream: read packet (0x%llx) size=%d", 37161ae08745Sheppo ldcp->id, size); 37171ae08745Sheppo 37181ae08745Sheppo if (rv != 0) 37191ae08745Sheppo return (rv); 37201ae08745Sheppo 37211ae08745Sheppo ldcp->stream_remains = size; 37221ae08745Sheppo ldcp->stream_offset = 0; 37231ae08745Sheppo } 37241ae08745Sheppo 37251ae08745Sheppo size = MIN(ldcp->stream_remains, *sizep); 37261ae08745Sheppo 37271ae08745Sheppo bcopy(ldcp->stream_bufferp + ldcp->stream_offset, target_bufp, size); 37281ae08745Sheppo ldcp->stream_offset += size; 37291ae08745Sheppo ldcp->stream_remains -= size; 37301ae08745Sheppo 37311ae08745Sheppo D2(ldcp->id, "i_ldc_read_stream: (0x%llx) fill from buffer size=%d", 37321ae08745Sheppo ldcp->id, size); 37331ae08745Sheppo 37341ae08745Sheppo *sizep = size; 37351ae08745Sheppo return (0); 37361ae08745Sheppo } 37371ae08745Sheppo 37381ae08745Sheppo /* 37391ae08745Sheppo * Write specified amount of bytes to the channel 37401ae08745Sheppo * in multiple pkts of pkt_payload size. Each 37411ae08745Sheppo * packet is tagged with an unique packet ID in 3742e1ebb9ecSlm66018 * the case of a reliable link. 37431ae08745Sheppo * 37441ae08745Sheppo * On return, size contains the number of bytes written. 37451ae08745Sheppo */ 37461ae08745Sheppo int 37471ae08745Sheppo ldc_write(ldc_handle_t handle, caddr_t buf, size_t *sizep) 37481ae08745Sheppo { 37491ae08745Sheppo ldc_chan_t *ldcp; 37501ae08745Sheppo int rv = 0; 37511ae08745Sheppo 37521ae08745Sheppo if (handle == NULL) { 37531ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_write: invalid channel handle\n"); 37541ae08745Sheppo return (EINVAL); 37551ae08745Sheppo } 37561ae08745Sheppo ldcp = (ldc_chan_t *)handle; 37571ae08745Sheppo 3758d10e4ef2Snarayan /* check if writes can occur */ 3759d10e4ef2Snarayan if (!mutex_tryenter(&ldcp->tx_lock)) { 3760d10e4ef2Snarayan /* 3761d10e4ef2Snarayan * Could not get the lock - channel could 3762d10e4ef2Snarayan * be in the process of being unconfigured 3763d10e4ef2Snarayan * or reader has encountered an error 3764d10e4ef2Snarayan */ 3765d10e4ef2Snarayan return (EAGAIN); 3766d10e4ef2Snarayan } 37671ae08745Sheppo 37681ae08745Sheppo /* check if non-zero data to write */ 37691ae08745Sheppo if (buf == NULL || sizep == NULL) { 37701ae08745Sheppo DWARN(ldcp->id, "ldc_write: (0x%llx) invalid data write\n", 37711ae08745Sheppo ldcp->id); 3772d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 37731ae08745Sheppo return (EINVAL); 37741ae08745Sheppo } 37751ae08745Sheppo 37761ae08745Sheppo if (*sizep == 0) { 37771ae08745Sheppo DWARN(ldcp->id, "ldc_write: (0x%llx) write size of zero\n", 37781ae08745Sheppo ldcp->id); 3779d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 37801ae08745Sheppo return (0); 37811ae08745Sheppo } 37821ae08745Sheppo 37831ae08745Sheppo /* Check if channel is UP for data exchange */ 37841ae08745Sheppo if (ldcp->tstate != TS_UP) { 37851ae08745Sheppo DWARN(ldcp->id, 37861ae08745Sheppo "ldc_write: (0x%llx) channel is not in UP state\n", 37871ae08745Sheppo ldcp->id); 37881ae08745Sheppo *sizep = 0; 37891ae08745Sheppo rv = ECONNRESET; 37901ae08745Sheppo } else { 37911ae08745Sheppo rv = ldcp->write_p(ldcp, buf, sizep); 37921ae08745Sheppo } 37931ae08745Sheppo 3794d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 37951ae08745Sheppo 37961ae08745Sheppo return (rv); 37971ae08745Sheppo } 37981ae08745Sheppo 37991ae08745Sheppo /* 38001ae08745Sheppo * Write a raw packet to the channel 38011ae08745Sheppo * On return, size contains the number of bytes written. 38021ae08745Sheppo */ 38031ae08745Sheppo static int 38041ae08745Sheppo i_ldc_write_raw(ldc_chan_t *ldcp, caddr_t buf, size_t *sizep) 38051ae08745Sheppo { 38061ae08745Sheppo ldc_msg_t *ldcmsg; 38071ae08745Sheppo uint64_t tx_head, tx_tail, new_tail; 38081ae08745Sheppo int rv = 0; 38091ae08745Sheppo size_t size; 38101ae08745Sheppo 3811d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 38121ae08745Sheppo ASSERT(ldcp->mode == LDC_MODE_RAW); 38131ae08745Sheppo 38141ae08745Sheppo size = *sizep; 38151ae08745Sheppo 38161ae08745Sheppo /* 38171ae08745Sheppo * Check to see if the packet size is less than or 38181ae08745Sheppo * equal to packet size support in raw mode 38191ae08745Sheppo */ 38201ae08745Sheppo if (size > ldcp->pkt_payload) { 38211ae08745Sheppo DWARN(ldcp->id, 38221ae08745Sheppo "ldc_write: (0x%llx) invalid size (0x%llx) for RAW mode\n", 38231ae08745Sheppo ldcp->id, *sizep); 38241ae08745Sheppo *sizep = 0; 38251ae08745Sheppo return (EMSGSIZE); 38261ae08745Sheppo } 38271ae08745Sheppo 38281ae08745Sheppo /* get the qptrs for the tx queue */ 38291ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 38301ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 38311ae08745Sheppo if (rv != 0) { 38321ae08745Sheppo cmn_err(CE_WARN, 38331ae08745Sheppo "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp->id); 38341ae08745Sheppo *sizep = 0; 38351ae08745Sheppo return (EIO); 38361ae08745Sheppo } 38371ae08745Sheppo 38381ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN || 38391ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 38401ae08745Sheppo DWARN(ldcp->id, 38411ae08745Sheppo "ldc_write: (0x%llx) channel down/reset\n", ldcp->id); 3842d10e4ef2Snarayan 38431ae08745Sheppo *sizep = 0; 3844d10e4ef2Snarayan if (mutex_tryenter(&ldcp->lock)) { 38453af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 3846d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3847d10e4ef2Snarayan } else { 3848d10e4ef2Snarayan /* 3849d10e4ef2Snarayan * Release Tx lock, and then reacquire channel 3850d10e4ef2Snarayan * and Tx lock in correct order 3851d10e4ef2Snarayan */ 3852d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3853d10e4ef2Snarayan mutex_enter(&ldcp->lock); 3854d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 38553af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 3856d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3857d10e4ef2Snarayan } 38581ae08745Sheppo return (ECONNRESET); 38591ae08745Sheppo } 38601ae08745Sheppo 38611ae08745Sheppo tx_tail = ldcp->tx_tail; 38621ae08745Sheppo tx_head = ldcp->tx_head; 38631ae08745Sheppo new_tail = (tx_tail + LDC_PACKET_SIZE) & 38641ae08745Sheppo ((ldcp->tx_q_entries-1) << LDC_PACKET_SHIFT); 38651ae08745Sheppo 38661ae08745Sheppo if (new_tail == tx_head) { 38671ae08745Sheppo DWARN(DBG_ALL_LDCS, 38681ae08745Sheppo "ldc_write: (0x%llx) TX queue is full\n", ldcp->id); 38691ae08745Sheppo *sizep = 0; 38701ae08745Sheppo return (EWOULDBLOCK); 38711ae08745Sheppo } 38721ae08745Sheppo 38731ae08745Sheppo D2(ldcp->id, "ldc_write: (0x%llx) start xfer size=%d", 38741ae08745Sheppo ldcp->id, size); 38751ae08745Sheppo 38761ae08745Sheppo /* Send the data now */ 38771ae08745Sheppo ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 38781ae08745Sheppo 38791ae08745Sheppo /* copy the data into pkt */ 38801ae08745Sheppo bcopy((uint8_t *)buf, ldcmsg, size); 38811ae08745Sheppo 38821ae08745Sheppo /* increment tail */ 38831ae08745Sheppo tx_tail = new_tail; 38841ae08745Sheppo 38851ae08745Sheppo /* 38861ae08745Sheppo * All packets have been copied into the TX queue 38871ae08745Sheppo * update the tail ptr in the HV 38881ae08745Sheppo */ 38891ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 38901ae08745Sheppo if (rv) { 38911ae08745Sheppo if (rv == EWOULDBLOCK) { 38921ae08745Sheppo DWARN(ldcp->id, "ldc_write: (0x%llx) write timed out\n", 38931ae08745Sheppo ldcp->id); 38941ae08745Sheppo *sizep = 0; 38951ae08745Sheppo return (EWOULDBLOCK); 38961ae08745Sheppo } 38971ae08745Sheppo 38981ae08745Sheppo *sizep = 0; 3899d10e4ef2Snarayan if (mutex_tryenter(&ldcp->lock)) { 39003af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 3901d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3902d10e4ef2Snarayan } else { 3903d10e4ef2Snarayan /* 3904d10e4ef2Snarayan * Release Tx lock, and then reacquire channel 3905d10e4ef2Snarayan * and Tx lock in correct order 3906d10e4ef2Snarayan */ 3907d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3908d10e4ef2Snarayan mutex_enter(&ldcp->lock); 3909d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 39103af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 3911d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3912d10e4ef2Snarayan } 39131ae08745Sheppo return (ECONNRESET); 39141ae08745Sheppo } 39151ae08745Sheppo 39161ae08745Sheppo ldcp->tx_tail = tx_tail; 39171ae08745Sheppo *sizep = size; 39181ae08745Sheppo 39191ae08745Sheppo D2(ldcp->id, "ldc_write: (0x%llx) end xfer size=%d", ldcp->id, size); 39201ae08745Sheppo 39211ae08745Sheppo return (rv); 39221ae08745Sheppo } 39231ae08745Sheppo 39241ae08745Sheppo 39251ae08745Sheppo /* 39261ae08745Sheppo * Write specified amount of bytes to the channel 39271ae08745Sheppo * in multiple pkts of pkt_payload size. Each 39281ae08745Sheppo * packet is tagged with an unique packet ID in 3929e1ebb9ecSlm66018 * the case of a reliable link. 39301ae08745Sheppo * 39311ae08745Sheppo * On return, size contains the number of bytes written. 39321ae08745Sheppo * This function needs to ensure that the write size is < MTU size 39331ae08745Sheppo */ 39341ae08745Sheppo static int 39351ae08745Sheppo i_ldc_write_packet(ldc_chan_t *ldcp, caddr_t buf, size_t *size) 39361ae08745Sheppo { 39371ae08745Sheppo ldc_msg_t *ldcmsg; 39381ae08745Sheppo uint64_t tx_head, tx_tail, new_tail, start; 39391ae08745Sheppo uint64_t txq_size_mask, numavail; 39401ae08745Sheppo uint8_t *msgbuf, *source = (uint8_t *)buf; 39411ae08745Sheppo size_t len, bytes_written = 0, remaining; 39421ae08745Sheppo int rv; 39431ae08745Sheppo uint32_t curr_seqid; 39441ae08745Sheppo 3945d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 39461ae08745Sheppo 39471ae08745Sheppo ASSERT(ldcp->mode == LDC_MODE_RELIABLE || 39481ae08745Sheppo ldcp->mode == LDC_MODE_UNRELIABLE || 39491ae08745Sheppo ldcp->mode == LDC_MODE_STREAM); 39501ae08745Sheppo 39511ae08745Sheppo /* compute mask for increment */ 39521ae08745Sheppo txq_size_mask = (ldcp->tx_q_entries - 1) << LDC_PACKET_SHIFT; 39531ae08745Sheppo 39541ae08745Sheppo /* get the qptrs for the tx queue */ 39551ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 39561ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 39571ae08745Sheppo if (rv != 0) { 39581ae08745Sheppo cmn_err(CE_WARN, 39591ae08745Sheppo "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp->id); 39601ae08745Sheppo *size = 0; 39611ae08745Sheppo return (EIO); 39621ae08745Sheppo } 39631ae08745Sheppo 39641ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN || 39651ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 39661ae08745Sheppo DWARN(ldcp->id, 39671ae08745Sheppo "ldc_write: (0x%llx) channel down/reset\n", ldcp->id); 39681ae08745Sheppo *size = 0; 3969d10e4ef2Snarayan if (mutex_tryenter(&ldcp->lock)) { 39703af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 3971d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3972d10e4ef2Snarayan } else { 3973d10e4ef2Snarayan /* 3974d10e4ef2Snarayan * Release Tx lock, and then reacquire channel 3975d10e4ef2Snarayan * and Tx lock in correct order 3976d10e4ef2Snarayan */ 3977d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3978d10e4ef2Snarayan mutex_enter(&ldcp->lock); 3979d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 39803af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 3981d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3982d10e4ef2Snarayan } 39831ae08745Sheppo return (ECONNRESET); 39841ae08745Sheppo } 39851ae08745Sheppo 39861ae08745Sheppo tx_tail = ldcp->tx_tail; 39871ae08745Sheppo new_tail = (tx_tail + LDC_PACKET_SIZE) % 39881ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 39891ae08745Sheppo 39901ae08745Sheppo /* 3991e1ebb9ecSlm66018 * Link mode determines whether we use HV Tx head or the 39921ae08745Sheppo * private protocol head (corresponding to last ACKd pkt) for 39931ae08745Sheppo * determining how much we can write 39941ae08745Sheppo */ 39951ae08745Sheppo tx_head = (ldcp->mode == LDC_MODE_RELIABLE || 39961ae08745Sheppo ldcp->mode == LDC_MODE_STREAM) 39971ae08745Sheppo ? ldcp->tx_ackd_head : ldcp->tx_head; 39981ae08745Sheppo if (new_tail == tx_head) { 39991ae08745Sheppo DWARN(DBG_ALL_LDCS, 40001ae08745Sheppo "ldc_write: (0x%llx) TX queue is full\n", ldcp->id); 40011ae08745Sheppo *size = 0; 40021ae08745Sheppo return (EWOULDBLOCK); 40031ae08745Sheppo } 40041ae08745Sheppo 40051ae08745Sheppo /* 40061ae08745Sheppo * Make sure that the LDC Tx queue has enough space 40071ae08745Sheppo */ 40081ae08745Sheppo numavail = (tx_head >> LDC_PACKET_SHIFT) - (tx_tail >> LDC_PACKET_SHIFT) 40091ae08745Sheppo + ldcp->tx_q_entries - 1; 40101ae08745Sheppo numavail %= ldcp->tx_q_entries; 40111ae08745Sheppo 40121ae08745Sheppo if (*size > (numavail * ldcp->pkt_payload)) { 40131ae08745Sheppo DWARN(DBG_ALL_LDCS, 40141ae08745Sheppo "ldc_write: (0x%llx) TX queue has no space\n", ldcp->id); 40151ae08745Sheppo return (EWOULDBLOCK); 40161ae08745Sheppo } 40171ae08745Sheppo 40181ae08745Sheppo D2(ldcp->id, "ldc_write: (0x%llx) start xfer size=%d", 40191ae08745Sheppo ldcp->id, *size); 40201ae08745Sheppo 40211ae08745Sheppo /* Send the data now */ 40221ae08745Sheppo bytes_written = 0; 40231ae08745Sheppo curr_seqid = ldcp->last_msg_snt; 40241ae08745Sheppo start = tx_tail; 40251ae08745Sheppo 40261ae08745Sheppo while (*size > bytes_written) { 40271ae08745Sheppo 40281ae08745Sheppo ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 40291ae08745Sheppo 40301ae08745Sheppo msgbuf = (uint8_t *)((ldcp->mode == LDC_MODE_RELIABLE || 40311ae08745Sheppo ldcp->mode == LDC_MODE_STREAM) 40321ae08745Sheppo ? ldcmsg->rdata : ldcmsg->udata); 40331ae08745Sheppo 40341ae08745Sheppo ldcmsg->type = LDC_DATA; 40351ae08745Sheppo ldcmsg->stype = LDC_INFO; 40361ae08745Sheppo ldcmsg->ctrl = 0; 40371ae08745Sheppo 40381ae08745Sheppo remaining = *size - bytes_written; 40391ae08745Sheppo len = min(ldcp->pkt_payload, remaining); 40401ae08745Sheppo ldcmsg->env = (uint8_t)len; 40411ae08745Sheppo 40421ae08745Sheppo curr_seqid++; 40431ae08745Sheppo ldcmsg->seqid = curr_seqid; 40441ae08745Sheppo 40451ae08745Sheppo /* copy the data into pkt */ 40461ae08745Sheppo bcopy(source, msgbuf, len); 40471ae08745Sheppo 40481ae08745Sheppo source += len; 40491ae08745Sheppo bytes_written += len; 40501ae08745Sheppo 40511ae08745Sheppo /* increment tail */ 40521ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) & txq_size_mask; 40531ae08745Sheppo 40541ae08745Sheppo ASSERT(tx_tail != tx_head); 40551ae08745Sheppo } 40561ae08745Sheppo 40571ae08745Sheppo /* Set the start and stop bits */ 40581ae08745Sheppo ldcmsg->env |= LDC_FRAG_STOP; 40591ae08745Sheppo ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + start); 40601ae08745Sheppo ldcmsg->env |= LDC_FRAG_START; 40611ae08745Sheppo 40621ae08745Sheppo /* 40631ae08745Sheppo * All packets have been copied into the TX queue 40641ae08745Sheppo * update the tail ptr in the HV 40651ae08745Sheppo */ 40661ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 40671ae08745Sheppo if (rv == 0) { 40681ae08745Sheppo ldcp->tx_tail = tx_tail; 40691ae08745Sheppo ldcp->last_msg_snt = curr_seqid; 40701ae08745Sheppo *size = bytes_written; 40711ae08745Sheppo } else { 40721ae08745Sheppo int rv2; 40731ae08745Sheppo 40741ae08745Sheppo if (rv != EWOULDBLOCK) { 40751ae08745Sheppo *size = 0; 4076d10e4ef2Snarayan if (mutex_tryenter(&ldcp->lock)) { 40773af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 4078d10e4ef2Snarayan mutex_exit(&ldcp->lock); 4079d10e4ef2Snarayan } else { 4080d10e4ef2Snarayan /* 4081d10e4ef2Snarayan * Release Tx lock, and then reacquire channel 4082d10e4ef2Snarayan * and Tx lock in correct order 4083d10e4ef2Snarayan */ 4084d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 4085d10e4ef2Snarayan mutex_enter(&ldcp->lock); 4086d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 40873af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 4088d10e4ef2Snarayan mutex_exit(&ldcp->lock); 4089d10e4ef2Snarayan } 40901ae08745Sheppo return (ECONNRESET); 40911ae08745Sheppo } 40921ae08745Sheppo 4093cb112a14Slm66018 D1(ldcp->id, "hv_tx_set_tail returns 0x%x (head 0x%x, " 40941ae08745Sheppo "old tail 0x%x, new tail 0x%x, qsize=0x%x)\n", 40951ae08745Sheppo rv, ldcp->tx_head, ldcp->tx_tail, tx_tail, 40961ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 40971ae08745Sheppo 40981ae08745Sheppo rv2 = hv_ldc_tx_get_state(ldcp->id, 40991ae08745Sheppo &tx_head, &tx_tail, &ldcp->link_state); 41001ae08745Sheppo 4101cb112a14Slm66018 D1(ldcp->id, "hv_ldc_tx_get_state returns 0x%x " 41021ae08745Sheppo "(head 0x%x, tail 0x%x state 0x%x)\n", 41031ae08745Sheppo rv2, tx_head, tx_tail, ldcp->link_state); 41041ae08745Sheppo 41051ae08745Sheppo *size = 0; 41061ae08745Sheppo } 41071ae08745Sheppo 41081ae08745Sheppo D2(ldcp->id, "ldc_write: (0x%llx) end xfer size=%d", ldcp->id, *size); 41091ae08745Sheppo 41101ae08745Sheppo return (rv); 41111ae08745Sheppo } 41121ae08745Sheppo 41131ae08745Sheppo /* 41141ae08745Sheppo * Write specified amount of bytes to the channel 41151ae08745Sheppo * in multiple pkts of pkt_payload size. Each 41161ae08745Sheppo * packet is tagged with an unique packet ID in 4117e1ebb9ecSlm66018 * the case of a reliable link. 41181ae08745Sheppo * 41191ae08745Sheppo * On return, size contains the number of bytes written. 41201ae08745Sheppo * This function needs to ensure that the write size is < MTU size 41211ae08745Sheppo */ 41221ae08745Sheppo static int 41231ae08745Sheppo i_ldc_write_stream(ldc_chan_t *ldcp, caddr_t buf, size_t *sizep) 41241ae08745Sheppo { 4125d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 41261ae08745Sheppo ASSERT(ldcp->mode == LDC_MODE_STREAM); 41271ae08745Sheppo 41281ae08745Sheppo /* Truncate packet to max of MTU size */ 41291ae08745Sheppo if (*sizep > ldcp->mtu) *sizep = ldcp->mtu; 41301ae08745Sheppo return (i_ldc_write_packet(ldcp, buf, sizep)); 41311ae08745Sheppo } 41321ae08745Sheppo 41331ae08745Sheppo 41341ae08745Sheppo /* 41351ae08745Sheppo * Interfaces for channel nexus to register/unregister with LDC module 41361ae08745Sheppo * The nexus will register functions to be used to register individual 41371ae08745Sheppo * channels with the nexus and enable interrupts for the channels 41381ae08745Sheppo */ 41391ae08745Sheppo int 41401ae08745Sheppo ldc_register(ldc_cnex_t *cinfo) 41411ae08745Sheppo { 41421ae08745Sheppo ldc_chan_t *ldcp; 41431ae08745Sheppo 41441ae08745Sheppo if (cinfo == NULL || cinfo->dip == NULL || 41451ae08745Sheppo cinfo->reg_chan == NULL || cinfo->unreg_chan == NULL || 41461ae08745Sheppo cinfo->add_intr == NULL || cinfo->rem_intr == NULL || 41471ae08745Sheppo cinfo->clr_intr == NULL) { 41481ae08745Sheppo 41491ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_register: invalid nexus info\n"); 41501ae08745Sheppo return (EINVAL); 41511ae08745Sheppo } 41521ae08745Sheppo 41531ae08745Sheppo mutex_enter(&ldcssp->lock); 41541ae08745Sheppo 41551ae08745Sheppo /* nexus registration */ 41561ae08745Sheppo ldcssp->cinfo.dip = cinfo->dip; 41571ae08745Sheppo ldcssp->cinfo.reg_chan = cinfo->reg_chan; 41581ae08745Sheppo ldcssp->cinfo.unreg_chan = cinfo->unreg_chan; 41591ae08745Sheppo ldcssp->cinfo.add_intr = cinfo->add_intr; 41601ae08745Sheppo ldcssp->cinfo.rem_intr = cinfo->rem_intr; 41611ae08745Sheppo ldcssp->cinfo.clr_intr = cinfo->clr_intr; 41621ae08745Sheppo 41631ae08745Sheppo /* register any channels that might have been previously initialized */ 41641ae08745Sheppo ldcp = ldcssp->chan_list; 41651ae08745Sheppo while (ldcp) { 41661ae08745Sheppo if ((ldcp->tstate & TS_QCONF_RDY) && 41671ae08745Sheppo (ldcp->tstate & TS_CNEX_RDY) == 0) 41681ae08745Sheppo (void) i_ldc_register_channel(ldcp); 41691ae08745Sheppo 41701ae08745Sheppo ldcp = ldcp->next; 41711ae08745Sheppo } 41721ae08745Sheppo 41731ae08745Sheppo mutex_exit(&ldcssp->lock); 41741ae08745Sheppo 41751ae08745Sheppo return (0); 41761ae08745Sheppo } 41771ae08745Sheppo 41781ae08745Sheppo int 41791ae08745Sheppo ldc_unregister(ldc_cnex_t *cinfo) 41801ae08745Sheppo { 41811ae08745Sheppo if (cinfo == NULL || cinfo->dip == NULL) { 41821ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_unregister: invalid nexus info\n"); 41831ae08745Sheppo return (EINVAL); 41841ae08745Sheppo } 41851ae08745Sheppo 41861ae08745Sheppo mutex_enter(&ldcssp->lock); 41871ae08745Sheppo 41881ae08745Sheppo if (cinfo->dip != ldcssp->cinfo.dip) { 41891ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_unregister: invalid dip\n"); 41901ae08745Sheppo mutex_exit(&ldcssp->lock); 41911ae08745Sheppo return (EINVAL); 41921ae08745Sheppo } 41931ae08745Sheppo 41941ae08745Sheppo /* nexus unregister */ 41951ae08745Sheppo ldcssp->cinfo.dip = NULL; 41961ae08745Sheppo ldcssp->cinfo.reg_chan = NULL; 41971ae08745Sheppo ldcssp->cinfo.unreg_chan = NULL; 41981ae08745Sheppo ldcssp->cinfo.add_intr = NULL; 41991ae08745Sheppo ldcssp->cinfo.rem_intr = NULL; 42001ae08745Sheppo ldcssp->cinfo.clr_intr = NULL; 42011ae08745Sheppo 42021ae08745Sheppo mutex_exit(&ldcssp->lock); 42031ae08745Sheppo 42041ae08745Sheppo return (0); 42051ae08745Sheppo } 42061ae08745Sheppo 42071ae08745Sheppo 42081ae08745Sheppo /* ------------------------------------------------------------------------- */ 42091ae08745Sheppo 42101ae08745Sheppo /* 42111ae08745Sheppo * Allocate a memory handle for the channel and link it into the list 42121ae08745Sheppo * Also choose which memory table to use if this is the first handle 42131ae08745Sheppo * being assigned to this channel 42141ae08745Sheppo */ 42151ae08745Sheppo int 42161ae08745Sheppo ldc_mem_alloc_handle(ldc_handle_t handle, ldc_mem_handle_t *mhandle) 42171ae08745Sheppo { 42181ae08745Sheppo ldc_chan_t *ldcp; 42191ae08745Sheppo ldc_mhdl_t *mhdl; 42201ae08745Sheppo 42211ae08745Sheppo if (handle == NULL) { 42221ae08745Sheppo DWARN(DBG_ALL_LDCS, 42231ae08745Sheppo "ldc_mem_alloc_handle: invalid channel handle\n"); 42241ae08745Sheppo return (EINVAL); 42251ae08745Sheppo } 42261ae08745Sheppo ldcp = (ldc_chan_t *)handle; 42271ae08745Sheppo 42281ae08745Sheppo mutex_enter(&ldcp->lock); 42291ae08745Sheppo 42301ae08745Sheppo /* check to see if channel is initalized */ 42313af08d82Slm66018 if ((ldcp->tstate & ~TS_IN_RESET) < TS_INIT) { 42321ae08745Sheppo DWARN(ldcp->id, 42331ae08745Sheppo "ldc_mem_alloc_handle: (0x%llx) channel not initialized\n", 42341ae08745Sheppo ldcp->id); 42351ae08745Sheppo mutex_exit(&ldcp->lock); 42361ae08745Sheppo return (EINVAL); 42371ae08745Sheppo } 42381ae08745Sheppo 42391ae08745Sheppo /* allocate handle for channel */ 42404bac2208Snarayan mhdl = kmem_cache_alloc(ldcssp->memhdl_cache, KM_SLEEP); 42411ae08745Sheppo 42421ae08745Sheppo /* initialize the lock */ 42431ae08745Sheppo mutex_init(&mhdl->lock, NULL, MUTEX_DRIVER, NULL); 42441ae08745Sheppo 42454bac2208Snarayan mhdl->myshadow = B_FALSE; 42464bac2208Snarayan mhdl->memseg = NULL; 42471ae08745Sheppo mhdl->ldcp = ldcp; 42484bac2208Snarayan mhdl->status = LDC_UNBOUND; 42491ae08745Sheppo 42501ae08745Sheppo /* insert memory handle (@ head) into list */ 42511ae08745Sheppo if (ldcp->mhdl_list == NULL) { 42521ae08745Sheppo ldcp->mhdl_list = mhdl; 42531ae08745Sheppo mhdl->next = NULL; 42541ae08745Sheppo } else { 42551ae08745Sheppo /* insert @ head */ 42561ae08745Sheppo mhdl->next = ldcp->mhdl_list; 42571ae08745Sheppo ldcp->mhdl_list = mhdl; 42581ae08745Sheppo } 42591ae08745Sheppo 42601ae08745Sheppo /* return the handle */ 42611ae08745Sheppo *mhandle = (ldc_mem_handle_t)mhdl; 42621ae08745Sheppo 42631ae08745Sheppo mutex_exit(&ldcp->lock); 42641ae08745Sheppo 42651ae08745Sheppo D1(ldcp->id, "ldc_mem_alloc_handle: (0x%llx) allocated handle 0x%llx\n", 42661ae08745Sheppo ldcp->id, mhdl); 42671ae08745Sheppo 42681ae08745Sheppo return (0); 42691ae08745Sheppo } 42701ae08745Sheppo 42711ae08745Sheppo /* 42721ae08745Sheppo * Free memory handle for the channel and unlink it from the list 42731ae08745Sheppo */ 42741ae08745Sheppo int 42751ae08745Sheppo ldc_mem_free_handle(ldc_mem_handle_t mhandle) 42761ae08745Sheppo { 42771ae08745Sheppo ldc_mhdl_t *mhdl, *phdl; 42781ae08745Sheppo ldc_chan_t *ldcp; 42791ae08745Sheppo 42801ae08745Sheppo if (mhandle == NULL) { 42811ae08745Sheppo DWARN(DBG_ALL_LDCS, 42821ae08745Sheppo "ldc_mem_free_handle: invalid memory handle\n"); 42831ae08745Sheppo return (EINVAL); 42841ae08745Sheppo } 42851ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 42861ae08745Sheppo 42871ae08745Sheppo mutex_enter(&mhdl->lock); 42881ae08745Sheppo 42891ae08745Sheppo ldcp = mhdl->ldcp; 42901ae08745Sheppo 42911ae08745Sheppo if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) { 42921ae08745Sheppo DWARN(ldcp->id, 42931ae08745Sheppo "ldc_mem_free_handle: cannot free, 0x%llx hdl bound\n", 42941ae08745Sheppo mhdl); 42951ae08745Sheppo mutex_exit(&mhdl->lock); 42961ae08745Sheppo return (EINVAL); 42971ae08745Sheppo } 42981ae08745Sheppo mutex_exit(&mhdl->lock); 42991ae08745Sheppo 43001ae08745Sheppo mutex_enter(&ldcp->mlist_lock); 43011ae08745Sheppo 43021ae08745Sheppo phdl = ldcp->mhdl_list; 43031ae08745Sheppo 43041ae08745Sheppo /* first handle */ 43051ae08745Sheppo if (phdl == mhdl) { 43061ae08745Sheppo ldcp->mhdl_list = mhdl->next; 43071ae08745Sheppo mutex_destroy(&mhdl->lock); 43084bac2208Snarayan kmem_cache_free(ldcssp->memhdl_cache, mhdl); 43094bac2208Snarayan 43101ae08745Sheppo D1(ldcp->id, 43111ae08745Sheppo "ldc_mem_free_handle: (0x%llx) freed handle 0x%llx\n", 43121ae08745Sheppo ldcp->id, mhdl); 43131ae08745Sheppo } else { 43141ae08745Sheppo /* walk the list - unlink and free */ 43151ae08745Sheppo while (phdl != NULL) { 43161ae08745Sheppo if (phdl->next == mhdl) { 43171ae08745Sheppo phdl->next = mhdl->next; 43181ae08745Sheppo mutex_destroy(&mhdl->lock); 43194bac2208Snarayan kmem_cache_free(ldcssp->memhdl_cache, mhdl); 43201ae08745Sheppo D1(ldcp->id, 43211ae08745Sheppo "ldc_mem_free_handle: (0x%llx) freed " 43221ae08745Sheppo "handle 0x%llx\n", ldcp->id, mhdl); 43231ae08745Sheppo break; 43241ae08745Sheppo } 43251ae08745Sheppo phdl = phdl->next; 43261ae08745Sheppo } 43271ae08745Sheppo } 43281ae08745Sheppo 43291ae08745Sheppo if (phdl == NULL) { 43301ae08745Sheppo DWARN(ldcp->id, 43311ae08745Sheppo "ldc_mem_free_handle: invalid handle 0x%llx\n", mhdl); 43321ae08745Sheppo mutex_exit(&ldcp->mlist_lock); 43331ae08745Sheppo return (EINVAL); 43341ae08745Sheppo } 43351ae08745Sheppo 43361ae08745Sheppo mutex_exit(&ldcp->mlist_lock); 43371ae08745Sheppo 43381ae08745Sheppo return (0); 43391ae08745Sheppo } 43401ae08745Sheppo 43411ae08745Sheppo /* 43421ae08745Sheppo * Bind a memory handle to a virtual address. 43431ae08745Sheppo * The virtual address is converted to the corresponding real addresses. 43441ae08745Sheppo * Returns pointer to the first ldc_mem_cookie and the total number 43451ae08745Sheppo * of cookies for this virtual address. Other cookies can be obtained 43461ae08745Sheppo * using the ldc_mem_nextcookie() call. If the pages are stored in 43471ae08745Sheppo * consecutive locations in the table, a single cookie corresponding to 43481ae08745Sheppo * the first location is returned. The cookie size spans all the entries. 43491ae08745Sheppo * 43501ae08745Sheppo * If the VA corresponds to a page that is already being exported, reuse 43511ae08745Sheppo * the page and do not export it again. Bump the page's use count. 43521ae08745Sheppo */ 43531ae08745Sheppo int 43541ae08745Sheppo ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr, size_t len, 43551ae08745Sheppo uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount) 43561ae08745Sheppo { 43571ae08745Sheppo ldc_mhdl_t *mhdl; 43581ae08745Sheppo ldc_chan_t *ldcp; 43591ae08745Sheppo ldc_mtbl_t *mtbl; 43601ae08745Sheppo ldc_memseg_t *memseg; 43611ae08745Sheppo ldc_mte_t tmp_mte; 43621ae08745Sheppo uint64_t index, prev_index = 0; 43631ae08745Sheppo int64_t cookie_idx; 43641ae08745Sheppo uintptr_t raddr, ra_aligned; 43651ae08745Sheppo uint64_t psize, poffset, v_offset; 43661ae08745Sheppo uint64_t pg_shift, pg_size, pg_size_code, pg_mask; 43671ae08745Sheppo pgcnt_t npages; 43681ae08745Sheppo caddr_t v_align, addr; 43694bac2208Snarayan int i, rv; 43701ae08745Sheppo 43711ae08745Sheppo if (mhandle == NULL) { 43721ae08745Sheppo DWARN(DBG_ALL_LDCS, 43731ae08745Sheppo "ldc_mem_bind_handle: invalid memory handle\n"); 43741ae08745Sheppo return (EINVAL); 43751ae08745Sheppo } 43761ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 43771ae08745Sheppo ldcp = mhdl->ldcp; 43781ae08745Sheppo 43791ae08745Sheppo /* clear count */ 43801ae08745Sheppo *ccount = 0; 43811ae08745Sheppo 43821ae08745Sheppo mutex_enter(&mhdl->lock); 43831ae08745Sheppo 43841ae08745Sheppo if (mhdl->status == LDC_BOUND || mhdl->memseg != NULL) { 43851ae08745Sheppo DWARN(ldcp->id, 43861ae08745Sheppo "ldc_mem_bind_handle: (0x%x) handle already bound\n", 43871ae08745Sheppo mhandle); 43881ae08745Sheppo mutex_exit(&mhdl->lock); 43891ae08745Sheppo return (EINVAL); 43901ae08745Sheppo } 43911ae08745Sheppo 43921ae08745Sheppo /* Force address and size to be 8-byte aligned */ 43931ae08745Sheppo if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 43941ae08745Sheppo DWARN(ldcp->id, 43951ae08745Sheppo "ldc_mem_bind_handle: addr/size is not 8-byte aligned\n"); 43961ae08745Sheppo mutex_exit(&mhdl->lock); 43971ae08745Sheppo return (EINVAL); 43981ae08745Sheppo } 43991ae08745Sheppo 44004bac2208Snarayan /* 44014bac2208Snarayan * If this channel is binding a memory handle for the 44024bac2208Snarayan * first time allocate it a memory map table and initialize it 44034bac2208Snarayan */ 44044bac2208Snarayan if ((mtbl = ldcp->mtbl) == NULL) { 44054bac2208Snarayan 44064bac2208Snarayan mutex_enter(&ldcp->lock); 44074bac2208Snarayan 44084bac2208Snarayan /* Allocate and initialize the map table structure */ 44094bac2208Snarayan mtbl = kmem_zalloc(sizeof (ldc_mtbl_t), KM_SLEEP); 44104bac2208Snarayan mtbl->num_entries = mtbl->num_avail = ldc_maptable_entries; 44114bac2208Snarayan mtbl->size = ldc_maptable_entries * sizeof (ldc_mte_slot_t); 44124bac2208Snarayan mtbl->next_entry = NULL; 44133af08d82Slm66018 mtbl->contigmem = B_TRUE; 44144bac2208Snarayan 44154bac2208Snarayan /* Allocate the table itself */ 44164bac2208Snarayan mtbl->table = (ldc_mte_slot_t *) 44174bac2208Snarayan contig_mem_alloc_align(mtbl->size, MMU_PAGESIZE); 44184bac2208Snarayan if (mtbl->table == NULL) { 44193af08d82Slm66018 44203af08d82Slm66018 /* allocate a page of memory using kmem_alloc */ 44213af08d82Slm66018 mtbl->table = kmem_alloc(MMU_PAGESIZE, KM_SLEEP); 44223af08d82Slm66018 mtbl->size = MMU_PAGESIZE; 44233af08d82Slm66018 mtbl->contigmem = B_FALSE; 44243af08d82Slm66018 mtbl->num_entries = mtbl->num_avail = 44253af08d82Slm66018 mtbl->size / sizeof (ldc_mte_slot_t); 44263af08d82Slm66018 DWARN(ldcp->id, 44273af08d82Slm66018 "ldc_mem_bind_handle: (0x%llx) reduced tbl size " 44283af08d82Slm66018 "to %lx entries\n", ldcp->id, mtbl->num_entries); 44294bac2208Snarayan } 44304bac2208Snarayan 44314bac2208Snarayan /* zero out the memory */ 44324bac2208Snarayan bzero(mtbl->table, mtbl->size); 44334bac2208Snarayan 44344bac2208Snarayan /* initialize the lock */ 44354bac2208Snarayan mutex_init(&mtbl->lock, NULL, MUTEX_DRIVER, NULL); 44364bac2208Snarayan 44374bac2208Snarayan /* register table for this channel */ 44384bac2208Snarayan rv = hv_ldc_set_map_table(ldcp->id, 44394bac2208Snarayan va_to_pa(mtbl->table), mtbl->num_entries); 44404bac2208Snarayan if (rv != 0) { 44414bac2208Snarayan cmn_err(CE_WARN, 44424bac2208Snarayan "ldc_mem_bind_handle: (0x%lx) err %d mapping tbl", 44434bac2208Snarayan ldcp->id, rv); 44443af08d82Slm66018 if (mtbl->contigmem) 44454bac2208Snarayan contig_mem_free(mtbl->table, mtbl->size); 44463af08d82Slm66018 else 44473af08d82Slm66018 kmem_free(mtbl->table, mtbl->size); 44484bac2208Snarayan mutex_destroy(&mtbl->lock); 44494bac2208Snarayan kmem_free(mtbl, sizeof (ldc_mtbl_t)); 44504bac2208Snarayan mutex_exit(&ldcp->lock); 44514bac2208Snarayan mutex_exit(&mhdl->lock); 44524bac2208Snarayan return (EIO); 44534bac2208Snarayan } 44544bac2208Snarayan 44554bac2208Snarayan ldcp->mtbl = mtbl; 44564bac2208Snarayan mutex_exit(&ldcp->lock); 44574bac2208Snarayan 44584bac2208Snarayan D1(ldcp->id, 44594bac2208Snarayan "ldc_mem_bind_handle: (0x%llx) alloc'd map table 0x%llx\n", 44604bac2208Snarayan ldcp->id, ldcp->mtbl->table); 44614bac2208Snarayan } 44624bac2208Snarayan 44631ae08745Sheppo /* FUTURE: get the page size, pgsz code, and shift */ 44641ae08745Sheppo pg_size = MMU_PAGESIZE; 44651ae08745Sheppo pg_size_code = page_szc(pg_size); 44661ae08745Sheppo pg_shift = page_get_shift(pg_size_code); 44671ae08745Sheppo pg_mask = ~(pg_size - 1); 44681ae08745Sheppo 44691ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) binding " 44701ae08745Sheppo "va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 44711ae08745Sheppo ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 44721ae08745Sheppo 44731ae08745Sheppo /* aligned VA and its offset */ 44741ae08745Sheppo v_align = (caddr_t)(((uintptr_t)vaddr) & ~(pg_size - 1)); 44751ae08745Sheppo v_offset = ((uintptr_t)vaddr) & (pg_size - 1); 44761ae08745Sheppo 44771ae08745Sheppo npages = (len+v_offset)/pg_size; 44781ae08745Sheppo npages = ((len+v_offset)%pg_size == 0) ? npages : npages+1; 44791ae08745Sheppo 44801ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle: binding " 44811ae08745Sheppo "(0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 44821ae08745Sheppo ldcp->id, vaddr, v_align, v_offset, npages); 44831ae08745Sheppo 44841ae08745Sheppo /* lock the memory table - exclusive access to channel */ 44851ae08745Sheppo mutex_enter(&mtbl->lock); 44861ae08745Sheppo 44871ae08745Sheppo if (npages > mtbl->num_avail) { 44883af08d82Slm66018 D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) no table entries\n", 44891ae08745Sheppo ldcp->id); 44901ae08745Sheppo mutex_exit(&mtbl->lock); 44911ae08745Sheppo mutex_exit(&mhdl->lock); 44921ae08745Sheppo return (ENOMEM); 44931ae08745Sheppo } 44941ae08745Sheppo 44951ae08745Sheppo /* Allocate a memseg structure */ 44964bac2208Snarayan memseg = mhdl->memseg = 44974bac2208Snarayan kmem_cache_alloc(ldcssp->memseg_cache, KM_SLEEP); 44981ae08745Sheppo 44991ae08745Sheppo /* Allocate memory to store all pages and cookies */ 45001ae08745Sheppo memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP); 45011ae08745Sheppo memseg->cookies = 45021ae08745Sheppo kmem_zalloc((sizeof (ldc_mem_cookie_t) * npages), KM_SLEEP); 45031ae08745Sheppo 45041ae08745Sheppo D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) processing 0x%llx pages\n", 45051ae08745Sheppo ldcp->id, npages); 45061ae08745Sheppo 45071ae08745Sheppo addr = v_align; 45081ae08745Sheppo 45091ae08745Sheppo /* 45104bac2208Snarayan * Check if direct shared memory map is enabled, if not change 45114bac2208Snarayan * the mapping type to include SHADOW_MAP. 45124bac2208Snarayan */ 45134bac2208Snarayan if (ldc_shmem_enabled == 0) 45144bac2208Snarayan mtype = LDC_SHADOW_MAP; 45154bac2208Snarayan 45164bac2208Snarayan /* 45171ae08745Sheppo * Table slots are used in a round-robin manner. The algorithm permits 45181ae08745Sheppo * inserting duplicate entries. Slots allocated earlier will typically 45191ae08745Sheppo * get freed before we get back to reusing the slot.Inserting duplicate 45201ae08745Sheppo * entries should be OK as we only lookup entries using the cookie addr 45211ae08745Sheppo * i.e. tbl index, during export, unexport and copy operation. 45221ae08745Sheppo * 45231ae08745Sheppo * One implementation what was tried was to search for a duplicate 45241ae08745Sheppo * page entry first and reuse it. The search overhead is very high and 45251ae08745Sheppo * in the vnet case dropped the perf by almost half, 50 to 24 mbps. 45261ae08745Sheppo * So it does make sense to avoid searching for duplicates. 45271ae08745Sheppo * 45281ae08745Sheppo * But during the process of searching for a free slot, if we find a 45291ae08745Sheppo * duplicate entry we will go ahead and use it, and bump its use count. 45301ae08745Sheppo */ 45311ae08745Sheppo 45321ae08745Sheppo /* index to start searching from */ 45331ae08745Sheppo index = mtbl->next_entry; 45341ae08745Sheppo cookie_idx = -1; 45351ae08745Sheppo 45361ae08745Sheppo tmp_mte.ll = 0; /* initialise fields to 0 */ 45371ae08745Sheppo 45381ae08745Sheppo if (mtype & LDC_DIRECT_MAP) { 45391ae08745Sheppo tmp_mte.mte_r = (perm & LDC_MEM_R) ? 1 : 0; 45401ae08745Sheppo tmp_mte.mte_w = (perm & LDC_MEM_W) ? 1 : 0; 45411ae08745Sheppo tmp_mte.mte_x = (perm & LDC_MEM_X) ? 1 : 0; 45421ae08745Sheppo } 45431ae08745Sheppo 45441ae08745Sheppo if (mtype & LDC_SHADOW_MAP) { 45451ae08745Sheppo tmp_mte.mte_cr = (perm & LDC_MEM_R) ? 1 : 0; 45461ae08745Sheppo tmp_mte.mte_cw = (perm & LDC_MEM_W) ? 1 : 0; 45471ae08745Sheppo } 45481ae08745Sheppo 45491ae08745Sheppo if (mtype & LDC_IO_MAP) { 45501ae08745Sheppo tmp_mte.mte_ir = (perm & LDC_MEM_R) ? 1 : 0; 45511ae08745Sheppo tmp_mte.mte_iw = (perm & LDC_MEM_W) ? 1 : 0; 45521ae08745Sheppo } 45531ae08745Sheppo 45541ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll); 45551ae08745Sheppo 45561ae08745Sheppo tmp_mte.mte_pgszc = pg_size_code; 45571ae08745Sheppo 45581ae08745Sheppo /* initialize each mem table entry */ 45591ae08745Sheppo for (i = 0; i < npages; i++) { 45601ae08745Sheppo 45611ae08745Sheppo /* check if slot is available in the table */ 45621ae08745Sheppo while (mtbl->table[index].entry.ll != 0) { 45631ae08745Sheppo 45641ae08745Sheppo index = (index + 1) % mtbl->num_entries; 45651ae08745Sheppo 45661ae08745Sheppo if (index == mtbl->next_entry) { 45671ae08745Sheppo /* we have looped around */ 45681ae08745Sheppo DWARN(DBG_ALL_LDCS, 45691ae08745Sheppo "ldc_mem_bind_handle: (0x%llx) cannot find " 45701ae08745Sheppo "entry\n", ldcp->id); 45711ae08745Sheppo *ccount = 0; 45721ae08745Sheppo 45731ae08745Sheppo /* NOTE: free memory, remove previous entries */ 45741ae08745Sheppo /* this shouldnt happen as num_avail was ok */ 45751ae08745Sheppo 45761ae08745Sheppo mutex_exit(&mtbl->lock); 45771ae08745Sheppo mutex_exit(&mhdl->lock); 45781ae08745Sheppo return (ENOMEM); 45791ae08745Sheppo } 45801ae08745Sheppo } 45811ae08745Sheppo 45821ae08745Sheppo /* get the real address */ 45831ae08745Sheppo raddr = va_to_pa((void *)addr); 45841ae08745Sheppo ra_aligned = ((uintptr_t)raddr & pg_mask); 45851ae08745Sheppo 45861ae08745Sheppo /* build the mte */ 45871ae08745Sheppo tmp_mte.mte_rpfn = ra_aligned >> pg_shift; 45881ae08745Sheppo 45891ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll); 45901ae08745Sheppo 45911ae08745Sheppo /* update entry in table */ 45921ae08745Sheppo mtbl->table[index].entry = tmp_mte; 45931ae08745Sheppo 45941ae08745Sheppo D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) stored MTE 0x%llx" 45951ae08745Sheppo " into loc 0x%llx\n", ldcp->id, tmp_mte.ll, index); 45961ae08745Sheppo 45971ae08745Sheppo /* calculate the size and offset for this export range */ 45981ae08745Sheppo if (i == 0) { 45991ae08745Sheppo /* first page */ 46001ae08745Sheppo psize = min((pg_size - v_offset), len); 46011ae08745Sheppo poffset = v_offset; 46021ae08745Sheppo 46031ae08745Sheppo } else if (i == (npages - 1)) { 46041ae08745Sheppo /* last page */ 46051ae08745Sheppo psize = (((uintptr_t)(vaddr + len)) & 46061ae08745Sheppo ((uint64_t)(pg_size-1))); 46071ae08745Sheppo if (psize == 0) 46081ae08745Sheppo psize = pg_size; 46091ae08745Sheppo poffset = 0; 46101ae08745Sheppo 46111ae08745Sheppo } else { 46121ae08745Sheppo /* middle pages */ 46131ae08745Sheppo psize = pg_size; 46141ae08745Sheppo poffset = 0; 46151ae08745Sheppo } 46161ae08745Sheppo 46171ae08745Sheppo /* store entry for this page */ 46181ae08745Sheppo memseg->pages[i].index = index; 46191ae08745Sheppo memseg->pages[i].raddr = raddr; 46201ae08745Sheppo memseg->pages[i].offset = poffset; 46211ae08745Sheppo memseg->pages[i].size = psize; 46221ae08745Sheppo memseg->pages[i].mte = &(mtbl->table[index]); 46231ae08745Sheppo 46241ae08745Sheppo /* create the cookie */ 46251ae08745Sheppo if (i == 0 || (index != prev_index + 1)) { 46261ae08745Sheppo cookie_idx++; 46271ae08745Sheppo memseg->cookies[cookie_idx].addr = 46281ae08745Sheppo IDX2COOKIE(index, pg_size_code, pg_shift); 46291ae08745Sheppo memseg->cookies[cookie_idx].addr |= poffset; 46301ae08745Sheppo memseg->cookies[cookie_idx].size = psize; 46311ae08745Sheppo 46321ae08745Sheppo } else { 46331ae08745Sheppo memseg->cookies[cookie_idx].size += psize; 46341ae08745Sheppo } 46351ae08745Sheppo 46361ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle: bound " 46371ae08745Sheppo "(0x%llx) va=0x%llx, idx=0x%llx, " 46381ae08745Sheppo "ra=0x%llx(sz=0x%x,off=0x%x)\n", 46391ae08745Sheppo ldcp->id, addr, index, raddr, psize, poffset); 46401ae08745Sheppo 46411ae08745Sheppo /* decrement number of available entries */ 46421ae08745Sheppo mtbl->num_avail--; 46431ae08745Sheppo 46441ae08745Sheppo /* increment va by page size */ 46451ae08745Sheppo addr += pg_size; 46461ae08745Sheppo 46471ae08745Sheppo /* increment index */ 46481ae08745Sheppo prev_index = index; 46491ae08745Sheppo index = (index + 1) % mtbl->num_entries; 46501ae08745Sheppo 46511ae08745Sheppo /* save the next slot */ 46521ae08745Sheppo mtbl->next_entry = index; 46531ae08745Sheppo } 46541ae08745Sheppo 46551ae08745Sheppo mutex_exit(&mtbl->lock); 46561ae08745Sheppo 46571ae08745Sheppo /* memory handle = bound */ 46581ae08745Sheppo mhdl->mtype = mtype; 46591ae08745Sheppo mhdl->perm = perm; 46601ae08745Sheppo mhdl->status = LDC_BOUND; 46611ae08745Sheppo 46621ae08745Sheppo /* update memseg_t */ 46631ae08745Sheppo memseg->vaddr = vaddr; 46641ae08745Sheppo memseg->raddr = memseg->pages[0].raddr; 46651ae08745Sheppo memseg->size = len; 46661ae08745Sheppo memseg->npages = npages; 46671ae08745Sheppo memseg->ncookies = cookie_idx + 1; 46681ae08745Sheppo memseg->next_cookie = (memseg->ncookies > 1) ? 1 : 0; 46691ae08745Sheppo 46701ae08745Sheppo /* return count and first cookie */ 46711ae08745Sheppo *ccount = memseg->ncookies; 46721ae08745Sheppo cookie->addr = memseg->cookies[0].addr; 46731ae08745Sheppo cookie->size = memseg->cookies[0].size; 46741ae08745Sheppo 46751ae08745Sheppo D1(ldcp->id, 46761ae08745Sheppo "ldc_mem_bind_handle: (0x%llx) bound 0x%llx, va=0x%llx, " 46771ae08745Sheppo "pgs=0x%llx cookies=0x%llx\n", 46781ae08745Sheppo ldcp->id, mhdl, vaddr, npages, memseg->ncookies); 46791ae08745Sheppo 46801ae08745Sheppo mutex_exit(&mhdl->lock); 46811ae08745Sheppo return (0); 46821ae08745Sheppo } 46831ae08745Sheppo 46841ae08745Sheppo /* 46851ae08745Sheppo * Return the next cookie associated with the specified memory handle 46861ae08745Sheppo */ 46871ae08745Sheppo int 46881ae08745Sheppo ldc_mem_nextcookie(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie) 46891ae08745Sheppo { 46901ae08745Sheppo ldc_mhdl_t *mhdl; 46911ae08745Sheppo ldc_chan_t *ldcp; 46921ae08745Sheppo ldc_memseg_t *memseg; 46931ae08745Sheppo 46941ae08745Sheppo if (mhandle == NULL) { 46951ae08745Sheppo DWARN(DBG_ALL_LDCS, 46961ae08745Sheppo "ldc_mem_nextcookie: invalid memory handle\n"); 46971ae08745Sheppo return (EINVAL); 46981ae08745Sheppo } 46991ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 47001ae08745Sheppo 47011ae08745Sheppo mutex_enter(&mhdl->lock); 47021ae08745Sheppo 47031ae08745Sheppo ldcp = mhdl->ldcp; 47041ae08745Sheppo memseg = mhdl->memseg; 47051ae08745Sheppo 47061ae08745Sheppo if (cookie == 0) { 47071ae08745Sheppo DWARN(ldcp->id, 47081ae08745Sheppo "ldc_mem_nextcookie:(0x%llx) invalid cookie arg\n", 47091ae08745Sheppo ldcp->id); 47101ae08745Sheppo mutex_exit(&mhdl->lock); 47111ae08745Sheppo return (EINVAL); 47121ae08745Sheppo } 47131ae08745Sheppo 47141ae08745Sheppo if (memseg->next_cookie != 0) { 47151ae08745Sheppo cookie->addr = memseg->cookies[memseg->next_cookie].addr; 47161ae08745Sheppo cookie->size = memseg->cookies[memseg->next_cookie].size; 47171ae08745Sheppo memseg->next_cookie++; 47181ae08745Sheppo if (memseg->next_cookie == memseg->ncookies) 47191ae08745Sheppo memseg->next_cookie = 0; 47201ae08745Sheppo 47211ae08745Sheppo } else { 47221ae08745Sheppo DWARN(ldcp->id, 47231ae08745Sheppo "ldc_mem_nextcookie:(0x%llx) no more cookies\n", ldcp->id); 47241ae08745Sheppo cookie->addr = 0; 47251ae08745Sheppo cookie->size = 0; 47261ae08745Sheppo mutex_exit(&mhdl->lock); 47271ae08745Sheppo return (EINVAL); 47281ae08745Sheppo } 47291ae08745Sheppo 47301ae08745Sheppo D1(ldcp->id, 47311ae08745Sheppo "ldc_mem_nextcookie: (0x%llx) cookie addr=0x%llx,sz=0x%llx\n", 47321ae08745Sheppo ldcp->id, cookie->addr, cookie->size); 47331ae08745Sheppo 47341ae08745Sheppo mutex_exit(&mhdl->lock); 47351ae08745Sheppo return (0); 47361ae08745Sheppo } 47371ae08745Sheppo 47381ae08745Sheppo /* 47391ae08745Sheppo * Unbind the virtual memory region associated with the specified 47401ae08745Sheppo * memory handle. Allassociated cookies are freed and the corresponding 47411ae08745Sheppo * RA space is no longer exported. 47421ae08745Sheppo */ 47431ae08745Sheppo int 47441ae08745Sheppo ldc_mem_unbind_handle(ldc_mem_handle_t mhandle) 47451ae08745Sheppo { 47461ae08745Sheppo ldc_mhdl_t *mhdl; 47471ae08745Sheppo ldc_chan_t *ldcp; 47481ae08745Sheppo ldc_mtbl_t *mtbl; 47491ae08745Sheppo ldc_memseg_t *memseg; 47504bac2208Snarayan uint64_t cookie_addr; 47514bac2208Snarayan uint64_t pg_shift, pg_size_code; 47524bac2208Snarayan int i, rv; 47531ae08745Sheppo 47541ae08745Sheppo if (mhandle == NULL) { 47551ae08745Sheppo DWARN(DBG_ALL_LDCS, 47561ae08745Sheppo "ldc_mem_unbind_handle: invalid memory handle\n"); 47571ae08745Sheppo return (EINVAL); 47581ae08745Sheppo } 47591ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 47601ae08745Sheppo 47611ae08745Sheppo mutex_enter(&mhdl->lock); 47621ae08745Sheppo 47631ae08745Sheppo if (mhdl->status == LDC_UNBOUND) { 47641ae08745Sheppo DWARN(DBG_ALL_LDCS, 47651ae08745Sheppo "ldc_mem_unbind_handle: (0x%x) handle is not bound\n", 47661ae08745Sheppo mhandle); 47671ae08745Sheppo mutex_exit(&mhdl->lock); 47681ae08745Sheppo return (EINVAL); 47691ae08745Sheppo } 47701ae08745Sheppo 47711ae08745Sheppo ldcp = mhdl->ldcp; 47721ae08745Sheppo mtbl = ldcp->mtbl; 47731ae08745Sheppo 47741ae08745Sheppo memseg = mhdl->memseg; 47751ae08745Sheppo 47761ae08745Sheppo /* lock the memory table - exclusive access to channel */ 47771ae08745Sheppo mutex_enter(&mtbl->lock); 47781ae08745Sheppo 47791ae08745Sheppo /* undo the pages exported */ 47801ae08745Sheppo for (i = 0; i < memseg->npages; i++) { 47811ae08745Sheppo 47824bac2208Snarayan /* check for mapped pages, revocation cookie != 0 */ 47831ae08745Sheppo if (memseg->pages[i].mte->cookie) { 47844bac2208Snarayan 47854bac2208Snarayan pg_size_code = page_szc(memseg->pages[i].size); 47864bac2208Snarayan pg_shift = page_get_shift(memseg->pages[i].size); 47874bac2208Snarayan cookie_addr = IDX2COOKIE(memseg->pages[i].index, 47884bac2208Snarayan pg_size_code, pg_shift); 47894bac2208Snarayan 47904bac2208Snarayan D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) revoke " 47914bac2208Snarayan "cookie 0x%llx, rcookie 0x%llx\n", ldcp->id, 47924bac2208Snarayan cookie_addr, memseg->pages[i].mte->cookie); 47934bac2208Snarayan rv = hv_ldc_revoke(ldcp->id, cookie_addr, 47944bac2208Snarayan memseg->pages[i].mte->cookie); 47954bac2208Snarayan if (rv) { 47964bac2208Snarayan DWARN(ldcp->id, 47974bac2208Snarayan "ldc_mem_unbind_handle: (0x%llx) cannot " 47984bac2208Snarayan "revoke mapping, cookie %llx\n", ldcp->id, 47994bac2208Snarayan cookie_addr); 48004bac2208Snarayan } 48011ae08745Sheppo } 48021ae08745Sheppo 48031ae08745Sheppo /* clear the entry from the table */ 48041ae08745Sheppo memseg->pages[i].mte->entry.ll = 0; 48051ae08745Sheppo mtbl->num_avail++; 48061ae08745Sheppo } 48071ae08745Sheppo mutex_exit(&mtbl->lock); 48081ae08745Sheppo 48091ae08745Sheppo /* free the allocated memseg and page structures */ 48101ae08745Sheppo kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages)); 48111ae08745Sheppo kmem_free(memseg->cookies, 48121ae08745Sheppo (sizeof (ldc_mem_cookie_t) * memseg->npages)); 48134bac2208Snarayan kmem_cache_free(ldcssp->memseg_cache, memseg); 48141ae08745Sheppo 48151ae08745Sheppo /* uninitialize the memory handle */ 48161ae08745Sheppo mhdl->memseg = NULL; 48171ae08745Sheppo mhdl->status = LDC_UNBOUND; 48181ae08745Sheppo 48191ae08745Sheppo D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) unbound handle 0x%llx\n", 48201ae08745Sheppo ldcp->id, mhdl); 48211ae08745Sheppo 48221ae08745Sheppo mutex_exit(&mhdl->lock); 48231ae08745Sheppo return (0); 48241ae08745Sheppo } 48251ae08745Sheppo 48261ae08745Sheppo /* 48271ae08745Sheppo * Get information about the dring. The base address of the descriptor 48281ae08745Sheppo * ring along with the type and permission are returned back. 48291ae08745Sheppo */ 48301ae08745Sheppo int 48311ae08745Sheppo ldc_mem_info(ldc_mem_handle_t mhandle, ldc_mem_info_t *minfo) 48321ae08745Sheppo { 48331ae08745Sheppo ldc_mhdl_t *mhdl; 48341ae08745Sheppo 48351ae08745Sheppo if (mhandle == NULL) { 48361ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid memory handle\n"); 48371ae08745Sheppo return (EINVAL); 48381ae08745Sheppo } 48391ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 48401ae08745Sheppo 48411ae08745Sheppo if (minfo == NULL) { 48421ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid args\n"); 48431ae08745Sheppo return (EINVAL); 48441ae08745Sheppo } 48451ae08745Sheppo 48461ae08745Sheppo mutex_enter(&mhdl->lock); 48471ae08745Sheppo 48481ae08745Sheppo minfo->status = mhdl->status; 48491ae08745Sheppo if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) { 48501ae08745Sheppo minfo->vaddr = mhdl->memseg->vaddr; 48511ae08745Sheppo minfo->raddr = mhdl->memseg->raddr; 48521ae08745Sheppo minfo->mtype = mhdl->mtype; 48531ae08745Sheppo minfo->perm = mhdl->perm; 48541ae08745Sheppo } 48551ae08745Sheppo mutex_exit(&mhdl->lock); 48561ae08745Sheppo 48571ae08745Sheppo return (0); 48581ae08745Sheppo } 48591ae08745Sheppo 48601ae08745Sheppo /* 48611ae08745Sheppo * Copy data either from or to the client specified virtual address 48621ae08745Sheppo * space to or from the exported memory associated with the cookies. 48631ae08745Sheppo * The direction argument determines whether the data is read from or 48641ae08745Sheppo * written to exported memory. 48651ae08745Sheppo */ 48661ae08745Sheppo int 48671ae08745Sheppo ldc_mem_copy(ldc_handle_t handle, caddr_t vaddr, uint64_t off, size_t *size, 48681ae08745Sheppo ldc_mem_cookie_t *cookies, uint32_t ccount, uint8_t direction) 48691ae08745Sheppo { 48701ae08745Sheppo ldc_chan_t *ldcp; 48711ae08745Sheppo uint64_t local_voff, local_valign; 48721ae08745Sheppo uint64_t cookie_addr, cookie_size; 48731ae08745Sheppo uint64_t pg_shift, pg_size, pg_size_code; 48741ae08745Sheppo uint64_t export_caddr, export_poff, export_psize, export_size; 48751ae08745Sheppo uint64_t local_ra, local_poff, local_psize; 48761ae08745Sheppo uint64_t copy_size, copied_len = 0, total_bal = 0, idx = 0; 48771ae08745Sheppo pgcnt_t npages; 48781ae08745Sheppo size_t len = *size; 48791ae08745Sheppo int i, rv = 0; 48801ae08745Sheppo 48813af08d82Slm66018 uint64_t chid; 48823af08d82Slm66018 48831ae08745Sheppo if (handle == NULL) { 48841ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_copy: invalid channel handle\n"); 48851ae08745Sheppo return (EINVAL); 48861ae08745Sheppo } 48871ae08745Sheppo ldcp = (ldc_chan_t *)handle; 48883af08d82Slm66018 chid = ldcp->id; 48891ae08745Sheppo 48901ae08745Sheppo /* check to see if channel is UP */ 48911ae08745Sheppo if (ldcp->tstate != TS_UP) { 48923af08d82Slm66018 DWARN(chid, "ldc_mem_copy: (0x%llx) channel is not UP\n", 48933af08d82Slm66018 chid); 48943af08d82Slm66018 return (ECONNRESET); 48951ae08745Sheppo } 48961ae08745Sheppo 48971ae08745Sheppo /* Force address and size to be 8-byte aligned */ 48981ae08745Sheppo if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 48993af08d82Slm66018 DWARN(chid, 49001ae08745Sheppo "ldc_mem_copy: addr/sz is not 8-byte aligned\n"); 49011ae08745Sheppo return (EINVAL); 49021ae08745Sheppo } 49031ae08745Sheppo 49041ae08745Sheppo /* Find the size of the exported memory */ 49051ae08745Sheppo export_size = 0; 49061ae08745Sheppo for (i = 0; i < ccount; i++) 49071ae08745Sheppo export_size += cookies[i].size; 49081ae08745Sheppo 49091ae08745Sheppo /* check to see if offset is valid */ 49101ae08745Sheppo if (off > export_size) { 49113af08d82Slm66018 DWARN(chid, 49121ae08745Sheppo "ldc_mem_copy: (0x%llx) start offset > export mem size\n", 49133af08d82Slm66018 chid); 49141ae08745Sheppo return (EINVAL); 49151ae08745Sheppo } 49161ae08745Sheppo 49171ae08745Sheppo /* 49181ae08745Sheppo * Check to see if the export size is smaller than the size we 49191ae08745Sheppo * are requesting to copy - if so flag an error 49201ae08745Sheppo */ 49211ae08745Sheppo if ((export_size - off) < *size) { 49223af08d82Slm66018 DWARN(chid, 49231ae08745Sheppo "ldc_mem_copy: (0x%llx) copy size > export mem size\n", 49243af08d82Slm66018 chid); 49251ae08745Sheppo return (EINVAL); 49261ae08745Sheppo } 49271ae08745Sheppo 49281ae08745Sheppo total_bal = min(export_size, *size); 49291ae08745Sheppo 49301ae08745Sheppo /* FUTURE: get the page size, pgsz code, and shift */ 49311ae08745Sheppo pg_size = MMU_PAGESIZE; 49321ae08745Sheppo pg_size_code = page_szc(pg_size); 49331ae08745Sheppo pg_shift = page_get_shift(pg_size_code); 49341ae08745Sheppo 49353af08d82Slm66018 D1(chid, "ldc_mem_copy: copying data " 49361ae08745Sheppo "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 49373af08d82Slm66018 chid, vaddr, pg_size, pg_size_code, pg_shift); 49381ae08745Sheppo 49391ae08745Sheppo /* aligned VA and its offset */ 49401ae08745Sheppo local_valign = (((uintptr_t)vaddr) & ~(pg_size - 1)); 49411ae08745Sheppo local_voff = ((uintptr_t)vaddr) & (pg_size - 1); 49421ae08745Sheppo 49431ae08745Sheppo npages = (len+local_voff)/pg_size; 49441ae08745Sheppo npages = ((len+local_voff)%pg_size == 0) ? npages : npages+1; 49451ae08745Sheppo 49463af08d82Slm66018 D1(chid, 49471ae08745Sheppo "ldc_mem_copy: (0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 49483af08d82Slm66018 chid, vaddr, local_valign, local_voff, npages); 49491ae08745Sheppo 49501ae08745Sheppo local_ra = va_to_pa((void *)local_valign); 49511ae08745Sheppo local_poff = local_voff; 49521ae08745Sheppo local_psize = min(len, (pg_size - local_voff)); 49531ae08745Sheppo 49541ae08745Sheppo len -= local_psize; 49551ae08745Sheppo 49561ae08745Sheppo /* 49571ae08745Sheppo * find the first cookie in the list of cookies 49581ae08745Sheppo * if the offset passed in is not zero 49591ae08745Sheppo */ 49601ae08745Sheppo for (idx = 0; idx < ccount; idx++) { 49611ae08745Sheppo cookie_size = cookies[idx].size; 49621ae08745Sheppo if (off < cookie_size) 49631ae08745Sheppo break; 49641ae08745Sheppo off -= cookie_size; 49651ae08745Sheppo } 49661ae08745Sheppo 49671ae08745Sheppo cookie_addr = cookies[idx].addr + off; 49681ae08745Sheppo cookie_size = cookies[idx].size - off; 49691ae08745Sheppo 49701ae08745Sheppo export_caddr = cookie_addr & ~(pg_size - 1); 49711ae08745Sheppo export_poff = cookie_addr & (pg_size - 1); 49721ae08745Sheppo export_psize = min(cookie_size, (pg_size - export_poff)); 49731ae08745Sheppo 49741ae08745Sheppo for (;;) { 49751ae08745Sheppo 49761ae08745Sheppo copy_size = min(export_psize, local_psize); 49771ae08745Sheppo 49783af08d82Slm66018 D1(chid, 49791ae08745Sheppo "ldc_mem_copy:(0x%llx) dir=0x%x, caddr=0x%llx," 49801ae08745Sheppo " loc_ra=0x%llx, exp_poff=0x%llx, loc_poff=0x%llx," 49811ae08745Sheppo " exp_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 49821ae08745Sheppo " total_bal=0x%llx\n", 49833af08d82Slm66018 chid, direction, export_caddr, local_ra, export_poff, 49841ae08745Sheppo local_poff, export_psize, local_psize, copy_size, 49851ae08745Sheppo total_bal); 49861ae08745Sheppo 49873af08d82Slm66018 rv = hv_ldc_copy(chid, direction, 49881ae08745Sheppo (export_caddr + export_poff), (local_ra + local_poff), 49891ae08745Sheppo copy_size, &copied_len); 49901ae08745Sheppo 49911ae08745Sheppo if (rv != 0) { 49923af08d82Slm66018 int error = EIO; 49933af08d82Slm66018 uint64_t rx_hd, rx_tl; 49943af08d82Slm66018 49953af08d82Slm66018 DWARN(chid, 49963af08d82Slm66018 "ldc_mem_copy: (0x%llx) err %d during copy\n", 49973af08d82Slm66018 (unsigned long long)chid, rv); 49983af08d82Slm66018 DWARN(chid, 49993af08d82Slm66018 "ldc_mem_copy: (0x%llx) dir=0x%x, caddr=0x%lx, " 50004bac2208Snarayan "loc_ra=0x%lx, exp_poff=0x%lx, loc_poff=0x%lx," 50014bac2208Snarayan " exp_psz=0x%lx, loc_psz=0x%lx, copy_sz=0x%lx," 50024bac2208Snarayan " copied_len=0x%lx, total_bal=0x%lx\n", 50033af08d82Slm66018 chid, direction, export_caddr, local_ra, 50041ae08745Sheppo export_poff, local_poff, export_psize, local_psize, 50051ae08745Sheppo copy_size, copied_len, total_bal); 50061ae08745Sheppo 50071ae08745Sheppo *size = *size - total_bal; 50083af08d82Slm66018 50093af08d82Slm66018 /* 50103af08d82Slm66018 * check if reason for copy error was due to 50113af08d82Slm66018 * a channel reset. we need to grab the lock 50123af08d82Slm66018 * just in case we have to do a reset. 50133af08d82Slm66018 */ 50143af08d82Slm66018 mutex_enter(&ldcp->lock); 50153af08d82Slm66018 mutex_enter(&ldcp->tx_lock); 50163af08d82Slm66018 50173af08d82Slm66018 rv = hv_ldc_rx_get_state(ldcp->id, 50183af08d82Slm66018 &rx_hd, &rx_tl, &(ldcp->link_state)); 50193af08d82Slm66018 if (ldcp->link_state == LDC_CHANNEL_DOWN || 50203af08d82Slm66018 ldcp->link_state == LDC_CHANNEL_RESET) { 50213af08d82Slm66018 i_ldc_reset(ldcp, B_FALSE); 50223af08d82Slm66018 error = ECONNRESET; 50233af08d82Slm66018 } 50243af08d82Slm66018 50253af08d82Slm66018 mutex_exit(&ldcp->tx_lock); 50261ae08745Sheppo mutex_exit(&ldcp->lock); 50273af08d82Slm66018 50283af08d82Slm66018 return (error); 50291ae08745Sheppo } 50301ae08745Sheppo 50311ae08745Sheppo ASSERT(copied_len <= copy_size); 50321ae08745Sheppo 50333af08d82Slm66018 D2(chid, "ldc_mem_copy: copied=0x%llx\n", copied_len); 50341ae08745Sheppo export_poff += copied_len; 50351ae08745Sheppo local_poff += copied_len; 50361ae08745Sheppo export_psize -= copied_len; 50371ae08745Sheppo local_psize -= copied_len; 50381ae08745Sheppo cookie_size -= copied_len; 50391ae08745Sheppo 50401ae08745Sheppo total_bal -= copied_len; 50411ae08745Sheppo 50421ae08745Sheppo if (copy_size != copied_len) 50431ae08745Sheppo continue; 50441ae08745Sheppo 50451ae08745Sheppo if (export_psize == 0 && total_bal != 0) { 50461ae08745Sheppo 50471ae08745Sheppo if (cookie_size == 0) { 50481ae08745Sheppo idx++; 50491ae08745Sheppo cookie_addr = cookies[idx].addr; 50501ae08745Sheppo cookie_size = cookies[idx].size; 50511ae08745Sheppo 50521ae08745Sheppo export_caddr = cookie_addr & ~(pg_size - 1); 50531ae08745Sheppo export_poff = cookie_addr & (pg_size - 1); 50541ae08745Sheppo export_psize = 50551ae08745Sheppo min(cookie_size, (pg_size-export_poff)); 50561ae08745Sheppo } else { 50571ae08745Sheppo export_caddr += pg_size; 50581ae08745Sheppo export_poff = 0; 50591ae08745Sheppo export_psize = min(cookie_size, pg_size); 50601ae08745Sheppo } 50611ae08745Sheppo } 50621ae08745Sheppo 50631ae08745Sheppo if (local_psize == 0 && total_bal != 0) { 50641ae08745Sheppo local_valign += pg_size; 50651ae08745Sheppo local_ra = va_to_pa((void *)local_valign); 50661ae08745Sheppo local_poff = 0; 50671ae08745Sheppo local_psize = min(pg_size, len); 50681ae08745Sheppo len -= local_psize; 50691ae08745Sheppo } 50701ae08745Sheppo 50711ae08745Sheppo /* check if we are all done */ 50721ae08745Sheppo if (total_bal == 0) 50731ae08745Sheppo break; 50741ae08745Sheppo } 50751ae08745Sheppo 50761ae08745Sheppo 50773af08d82Slm66018 D1(chid, 50781ae08745Sheppo "ldc_mem_copy: (0x%llx) done copying sz=0x%llx\n", 50793af08d82Slm66018 chid, *size); 50801ae08745Sheppo 50811ae08745Sheppo return (0); 50821ae08745Sheppo } 50831ae08745Sheppo 50841ae08745Sheppo /* 50851ae08745Sheppo * Copy data either from or to the client specified virtual address 50861ae08745Sheppo * space to or from HV physical memory. 50871ae08745Sheppo * 50881ae08745Sheppo * The direction argument determines whether the data is read from or 50891ae08745Sheppo * written to HV memory. direction values are LDC_COPY_IN/OUT similar 50901ae08745Sheppo * to the ldc_mem_copy interface 50911ae08745Sheppo */ 50921ae08745Sheppo int 50933af08d82Slm66018 ldc_mem_rdwr_cookie(ldc_handle_t handle, caddr_t vaddr, size_t *size, 50941ae08745Sheppo caddr_t paddr, uint8_t direction) 50951ae08745Sheppo { 50961ae08745Sheppo ldc_chan_t *ldcp; 50971ae08745Sheppo uint64_t local_voff, local_valign; 50981ae08745Sheppo uint64_t pg_shift, pg_size, pg_size_code; 50991ae08745Sheppo uint64_t target_pa, target_poff, target_psize, target_size; 51001ae08745Sheppo uint64_t local_ra, local_poff, local_psize; 51011ae08745Sheppo uint64_t copy_size, copied_len = 0; 51021ae08745Sheppo pgcnt_t npages; 51031ae08745Sheppo size_t len = *size; 51041ae08745Sheppo int rv = 0; 51051ae08745Sheppo 51061ae08745Sheppo if (handle == NULL) { 51071ae08745Sheppo DWARN(DBG_ALL_LDCS, 51083af08d82Slm66018 "ldc_mem_rdwr_cookie: invalid channel handle\n"); 51091ae08745Sheppo return (EINVAL); 51101ae08745Sheppo } 51111ae08745Sheppo ldcp = (ldc_chan_t *)handle; 51121ae08745Sheppo 51131ae08745Sheppo mutex_enter(&ldcp->lock); 51141ae08745Sheppo 51151ae08745Sheppo /* check to see if channel is UP */ 51161ae08745Sheppo if (ldcp->tstate != TS_UP) { 51171ae08745Sheppo DWARN(ldcp->id, 51183af08d82Slm66018 "ldc_mem_rdwr_cookie: (0x%llx) channel is not UP\n", 51191ae08745Sheppo ldcp->id); 51201ae08745Sheppo mutex_exit(&ldcp->lock); 51213af08d82Slm66018 return (ECONNRESET); 51221ae08745Sheppo } 51231ae08745Sheppo 51241ae08745Sheppo /* Force address and size to be 8-byte aligned */ 51251ae08745Sheppo if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 51261ae08745Sheppo DWARN(ldcp->id, 51273af08d82Slm66018 "ldc_mem_rdwr_cookie: addr/size is not 8-byte aligned\n"); 51281ae08745Sheppo mutex_exit(&ldcp->lock); 51291ae08745Sheppo return (EINVAL); 51301ae08745Sheppo } 51311ae08745Sheppo 51321ae08745Sheppo target_size = *size; 51331ae08745Sheppo 51341ae08745Sheppo /* FUTURE: get the page size, pgsz code, and shift */ 51351ae08745Sheppo pg_size = MMU_PAGESIZE; 51361ae08745Sheppo pg_size_code = page_szc(pg_size); 51371ae08745Sheppo pg_shift = page_get_shift(pg_size_code); 51381ae08745Sheppo 51393af08d82Slm66018 D1(ldcp->id, "ldc_mem_rdwr_cookie: copying data " 51401ae08745Sheppo "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 51411ae08745Sheppo ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 51421ae08745Sheppo 51431ae08745Sheppo /* aligned VA and its offset */ 51441ae08745Sheppo local_valign = ((uintptr_t)vaddr) & ~(pg_size - 1); 51451ae08745Sheppo local_voff = ((uintptr_t)vaddr) & (pg_size - 1); 51461ae08745Sheppo 51471ae08745Sheppo npages = (len + local_voff) / pg_size; 51481ae08745Sheppo npages = ((len + local_voff) % pg_size == 0) ? npages : npages+1; 51491ae08745Sheppo 51503af08d82Slm66018 D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) v=0x%llx, " 51513af08d82Slm66018 "val=0x%llx,off=0x%x,pgs=0x%x\n", 51521ae08745Sheppo ldcp->id, vaddr, local_valign, local_voff, npages); 51531ae08745Sheppo 51541ae08745Sheppo local_ra = va_to_pa((void *)local_valign); 51551ae08745Sheppo local_poff = local_voff; 51561ae08745Sheppo local_psize = min(len, (pg_size - local_voff)); 51571ae08745Sheppo 51581ae08745Sheppo len -= local_psize; 51591ae08745Sheppo 51601ae08745Sheppo target_pa = ((uintptr_t)paddr) & ~(pg_size - 1); 51611ae08745Sheppo target_poff = ((uintptr_t)paddr) & (pg_size - 1); 51621ae08745Sheppo target_psize = pg_size - target_poff; 51631ae08745Sheppo 51641ae08745Sheppo for (;;) { 51651ae08745Sheppo 51661ae08745Sheppo copy_size = min(target_psize, local_psize); 51671ae08745Sheppo 51681ae08745Sheppo D1(ldcp->id, 51693af08d82Slm66018 "ldc_mem_rdwr_cookie: (0x%llx) dir=0x%x, tar_pa=0x%llx," 51701ae08745Sheppo " loc_ra=0x%llx, tar_poff=0x%llx, loc_poff=0x%llx," 51711ae08745Sheppo " tar_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 51721ae08745Sheppo " total_bal=0x%llx\n", 51731ae08745Sheppo ldcp->id, direction, target_pa, local_ra, target_poff, 51741ae08745Sheppo local_poff, target_psize, local_psize, copy_size, 51751ae08745Sheppo target_size); 51761ae08745Sheppo 51771ae08745Sheppo rv = hv_ldc_copy(ldcp->id, direction, 51781ae08745Sheppo (target_pa + target_poff), (local_ra + local_poff), 51791ae08745Sheppo copy_size, &copied_len); 51801ae08745Sheppo 51811ae08745Sheppo if (rv != 0) { 51823af08d82Slm66018 DWARN(DBG_ALL_LDCS, 51833af08d82Slm66018 "ldc_mem_rdwr_cookie: (0x%lx) err %d during copy\n", 51841ae08745Sheppo ldcp->id, rv); 51851ae08745Sheppo DWARN(DBG_ALL_LDCS, 51863af08d82Slm66018 "ldc_mem_rdwr_cookie: (0x%llx) dir=%lld, " 51873af08d82Slm66018 "tar_pa=0x%llx, loc_ra=0x%llx, tar_poff=0x%llx, " 51883af08d82Slm66018 "loc_poff=0x%llx, tar_psz=0x%llx, loc_psz=0x%llx, " 51893af08d82Slm66018 "copy_sz=0x%llx, total_bal=0x%llx\n", 51901ae08745Sheppo ldcp->id, direction, target_pa, local_ra, 51911ae08745Sheppo target_poff, local_poff, target_psize, local_psize, 51921ae08745Sheppo copy_size, target_size); 51931ae08745Sheppo 51941ae08745Sheppo *size = *size - target_size; 51951ae08745Sheppo mutex_exit(&ldcp->lock); 51961ae08745Sheppo return (i_ldc_h2v_error(rv)); 51971ae08745Sheppo } 51981ae08745Sheppo 51993af08d82Slm66018 D2(ldcp->id, "ldc_mem_rdwr_cookie: copied=0x%llx\n", 52003af08d82Slm66018 copied_len); 52011ae08745Sheppo target_poff += copied_len; 52021ae08745Sheppo local_poff += copied_len; 52031ae08745Sheppo target_psize -= copied_len; 52041ae08745Sheppo local_psize -= copied_len; 52051ae08745Sheppo 52061ae08745Sheppo target_size -= copied_len; 52071ae08745Sheppo 52081ae08745Sheppo if (copy_size != copied_len) 52091ae08745Sheppo continue; 52101ae08745Sheppo 52111ae08745Sheppo if (target_psize == 0 && target_size != 0) { 52121ae08745Sheppo target_pa += pg_size; 52131ae08745Sheppo target_poff = 0; 52141ae08745Sheppo target_psize = min(pg_size, target_size); 52151ae08745Sheppo } 52161ae08745Sheppo 52171ae08745Sheppo if (local_psize == 0 && target_size != 0) { 52181ae08745Sheppo local_valign += pg_size; 52191ae08745Sheppo local_ra = va_to_pa((void *)local_valign); 52201ae08745Sheppo local_poff = 0; 52211ae08745Sheppo local_psize = min(pg_size, len); 52221ae08745Sheppo len -= local_psize; 52231ae08745Sheppo } 52241ae08745Sheppo 52251ae08745Sheppo /* check if we are all done */ 52261ae08745Sheppo if (target_size == 0) 52271ae08745Sheppo break; 52281ae08745Sheppo } 52291ae08745Sheppo 52301ae08745Sheppo mutex_exit(&ldcp->lock); 52311ae08745Sheppo 52323af08d82Slm66018 D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) done copying sz=0x%llx\n", 52331ae08745Sheppo ldcp->id, *size); 52341ae08745Sheppo 52351ae08745Sheppo return (0); 52361ae08745Sheppo } 52371ae08745Sheppo 52381ae08745Sheppo /* 52391ae08745Sheppo * Map an exported memory segment into the local address space. If the 52401ae08745Sheppo * memory range was exported for direct map access, a HV call is made 52411ae08745Sheppo * to allocate a RA range. If the map is done via a shadow copy, local 52421ae08745Sheppo * shadow memory is allocated and the base VA is returned in 'vaddr'. If 52431ae08745Sheppo * the mapping is a direct map then the RA is returned in 'raddr'. 52441ae08745Sheppo */ 52451ae08745Sheppo int 52461ae08745Sheppo ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie, uint32_t ccount, 52474bac2208Snarayan uint8_t mtype, uint8_t perm, caddr_t *vaddr, caddr_t *raddr) 52481ae08745Sheppo { 52494bac2208Snarayan int i, j, idx, rv, retries; 52501ae08745Sheppo ldc_chan_t *ldcp; 52511ae08745Sheppo ldc_mhdl_t *mhdl; 52521ae08745Sheppo ldc_memseg_t *memseg; 52534bac2208Snarayan caddr_t tmpaddr; 52544bac2208Snarayan uint64_t map_perm = perm; 52554bac2208Snarayan uint64_t pg_size, pg_shift, pg_size_code, pg_mask; 52564bac2208Snarayan uint64_t exp_size = 0, base_off, map_size, npages; 52574bac2208Snarayan uint64_t cookie_addr, cookie_off, cookie_size; 52584bac2208Snarayan tte_t ldc_tte; 52591ae08745Sheppo 52601ae08745Sheppo if (mhandle == NULL) { 52611ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_map: invalid memory handle\n"); 52621ae08745Sheppo return (EINVAL); 52631ae08745Sheppo } 52641ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 52651ae08745Sheppo 52661ae08745Sheppo mutex_enter(&mhdl->lock); 52671ae08745Sheppo 52681ae08745Sheppo if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED || 52691ae08745Sheppo mhdl->memseg != NULL) { 52701ae08745Sheppo DWARN(DBG_ALL_LDCS, 52711ae08745Sheppo "ldc_mem_map: (0x%llx) handle bound/mapped\n", mhandle); 52721ae08745Sheppo mutex_exit(&mhdl->lock); 52731ae08745Sheppo return (EINVAL); 52741ae08745Sheppo } 52751ae08745Sheppo 52761ae08745Sheppo ldcp = mhdl->ldcp; 52771ae08745Sheppo 52781ae08745Sheppo mutex_enter(&ldcp->lock); 52791ae08745Sheppo 52801ae08745Sheppo if (ldcp->tstate != TS_UP) { 52811ae08745Sheppo DWARN(ldcp->id, 52821ae08745Sheppo "ldc_mem_dring_map: (0x%llx) channel is not UP\n", 52831ae08745Sheppo ldcp->id); 52841ae08745Sheppo mutex_exit(&ldcp->lock); 52851ae08745Sheppo mutex_exit(&mhdl->lock); 52863af08d82Slm66018 return (ECONNRESET); 52871ae08745Sheppo } 52881ae08745Sheppo 52891ae08745Sheppo if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) { 52901ae08745Sheppo DWARN(ldcp->id, "ldc_mem_map: invalid map type\n"); 52911ae08745Sheppo mutex_exit(&ldcp->lock); 52921ae08745Sheppo mutex_exit(&mhdl->lock); 52931ae08745Sheppo return (EINVAL); 52941ae08745Sheppo } 52951ae08745Sheppo 52961ae08745Sheppo D1(ldcp->id, "ldc_mem_map: (0x%llx) cookie = 0x%llx,0x%llx\n", 5297d10e4ef2Snarayan ldcp->id, cookie->addr, cookie->size); 52981ae08745Sheppo 52991ae08745Sheppo /* FUTURE: get the page size, pgsz code, and shift */ 53001ae08745Sheppo pg_size = MMU_PAGESIZE; 53011ae08745Sheppo pg_size_code = page_szc(pg_size); 53021ae08745Sheppo pg_shift = page_get_shift(pg_size_code); 53034bac2208Snarayan pg_mask = ~(pg_size - 1); 53041ae08745Sheppo 53051ae08745Sheppo /* calculate the number of pages in the exported cookie */ 53064bac2208Snarayan base_off = cookie[0].addr & (pg_size - 1); 53074bac2208Snarayan for (idx = 0; idx < ccount; idx++) 53081ae08745Sheppo exp_size += cookie[idx].size; 53094bac2208Snarayan map_size = P2ROUNDUP((exp_size + base_off), pg_size); 53104bac2208Snarayan npages = (map_size >> pg_shift); 53111ae08745Sheppo 53121ae08745Sheppo /* Allocate memseg structure */ 53134bac2208Snarayan memseg = mhdl->memseg = 53144bac2208Snarayan kmem_cache_alloc(ldcssp->memseg_cache, KM_SLEEP); 53151ae08745Sheppo 53161ae08745Sheppo /* Allocate memory to store all pages and cookies */ 53171ae08745Sheppo memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP); 53181ae08745Sheppo memseg->cookies = 53191ae08745Sheppo kmem_zalloc((sizeof (ldc_mem_cookie_t) * ccount), KM_SLEEP); 53201ae08745Sheppo 53214bac2208Snarayan D2(ldcp->id, "ldc_mem_map: (0x%llx) exp_size=0x%llx, map_size=0x%llx," 53224bac2208Snarayan "pages=0x%llx\n", ldcp->id, exp_size, map_size, npages); 53231ae08745Sheppo 53244bac2208Snarayan /* 53254bac2208Snarayan * Check if direct map over shared memory is enabled, if not change 53264bac2208Snarayan * the mapping type to SHADOW_MAP. 53274bac2208Snarayan */ 53284bac2208Snarayan if (ldc_shmem_enabled == 0) 53294bac2208Snarayan mtype = LDC_SHADOW_MAP; 53304bac2208Snarayan 53314bac2208Snarayan /* 53324bac2208Snarayan * Check to see if the client is requesting direct or shadow map 53334bac2208Snarayan * If direct map is requested, try to map remote memory first, 53344bac2208Snarayan * and if that fails, revert to shadow map 53354bac2208Snarayan */ 53364bac2208Snarayan if (mtype == LDC_DIRECT_MAP) { 53374bac2208Snarayan 53384bac2208Snarayan /* Allocate kernel virtual space for mapping */ 53394bac2208Snarayan memseg->vaddr = vmem_xalloc(heap_arena, map_size, 53404bac2208Snarayan pg_size, 0, 0, NULL, NULL, VM_NOSLEEP); 53414bac2208Snarayan if (memseg->vaddr == NULL) { 53424bac2208Snarayan cmn_err(CE_WARN, 53434bac2208Snarayan "ldc_mem_map: (0x%lx) memory map failed\n", 53444bac2208Snarayan ldcp->id); 53454bac2208Snarayan kmem_free(memseg->cookies, 53464bac2208Snarayan (sizeof (ldc_mem_cookie_t) * ccount)); 53474bac2208Snarayan kmem_free(memseg->pages, 53484bac2208Snarayan (sizeof (ldc_page_t) * npages)); 53494bac2208Snarayan kmem_cache_free(ldcssp->memseg_cache, memseg); 53504bac2208Snarayan 53514bac2208Snarayan mutex_exit(&ldcp->lock); 53524bac2208Snarayan mutex_exit(&mhdl->lock); 53534bac2208Snarayan return (ENOMEM); 53544bac2208Snarayan } 53554bac2208Snarayan 53564bac2208Snarayan /* Unload previous mapping */ 53574bac2208Snarayan hat_unload(kas.a_hat, memseg->vaddr, map_size, 53584bac2208Snarayan HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK); 53594bac2208Snarayan 53604bac2208Snarayan /* for each cookie passed in - map into address space */ 53614bac2208Snarayan idx = 0; 53624bac2208Snarayan cookie_size = 0; 53634bac2208Snarayan tmpaddr = memseg->vaddr; 53644bac2208Snarayan 53654bac2208Snarayan for (i = 0; i < npages; i++) { 53664bac2208Snarayan 53674bac2208Snarayan if (cookie_size == 0) { 53684bac2208Snarayan ASSERT(idx < ccount); 53694bac2208Snarayan cookie_addr = cookie[idx].addr & pg_mask; 53704bac2208Snarayan cookie_off = cookie[idx].addr & (pg_size - 1); 53714bac2208Snarayan cookie_size = 53724bac2208Snarayan P2ROUNDUP((cookie_off + cookie[idx].size), 53734bac2208Snarayan pg_size); 53744bac2208Snarayan idx++; 53754bac2208Snarayan } 53764bac2208Snarayan 53774bac2208Snarayan D1(ldcp->id, "ldc_mem_map: (0x%llx) mapping " 53784bac2208Snarayan "cookie 0x%llx, bal=0x%llx\n", ldcp->id, 53794bac2208Snarayan cookie_addr, cookie_size); 53804bac2208Snarayan 53814bac2208Snarayan /* map the cookie into address space */ 53824bac2208Snarayan for (retries = 0; retries < ldc_max_retries; 53834bac2208Snarayan retries++) { 53844bac2208Snarayan 53854bac2208Snarayan rv = hv_ldc_mapin(ldcp->id, cookie_addr, 53864bac2208Snarayan &memseg->pages[i].raddr, &map_perm); 53874bac2208Snarayan if (rv != H_EWOULDBLOCK && rv != H_ETOOMANY) 53884bac2208Snarayan break; 53894bac2208Snarayan 53904bac2208Snarayan drv_usecwait(ldc_delay); 53914bac2208Snarayan } 53924bac2208Snarayan 53934bac2208Snarayan if (rv || memseg->pages[i].raddr == 0) { 53944bac2208Snarayan DWARN(ldcp->id, 53954bac2208Snarayan "ldc_mem_map: (0x%llx) hv mapin err %d\n", 53964bac2208Snarayan ldcp->id, rv); 53974bac2208Snarayan 53984bac2208Snarayan /* remove previous mapins */ 53994bac2208Snarayan hat_unload(kas.a_hat, memseg->vaddr, map_size, 54004bac2208Snarayan HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK); 54014bac2208Snarayan for (j = 0; j < i; j++) { 54024bac2208Snarayan rv = hv_ldc_unmap( 54034bac2208Snarayan memseg->pages[j].raddr); 54044bac2208Snarayan if (rv) { 54054bac2208Snarayan DWARN(ldcp->id, 54064bac2208Snarayan "ldc_mem_map: (0x%llx) " 54074bac2208Snarayan "cannot unmap ra=0x%llx\n", 54084bac2208Snarayan ldcp->id, 54094bac2208Snarayan memseg->pages[j].raddr); 54104bac2208Snarayan } 54114bac2208Snarayan } 54124bac2208Snarayan 54134bac2208Snarayan /* free kernel virtual space */ 54144bac2208Snarayan vmem_free(heap_arena, (void *)memseg->vaddr, 54157636cb21Slm66018 map_size); 54164bac2208Snarayan 54174bac2208Snarayan /* direct map failed - revert to shadow map */ 54184bac2208Snarayan mtype = LDC_SHADOW_MAP; 54194bac2208Snarayan break; 54204bac2208Snarayan 54214bac2208Snarayan } else { 54224bac2208Snarayan 54234bac2208Snarayan D1(ldcp->id, 54244bac2208Snarayan "ldc_mem_map: (0x%llx) vtop map 0x%llx -> " 54254bac2208Snarayan "0x%llx, cookie=0x%llx, perm=0x%llx\n", 54264bac2208Snarayan ldcp->id, tmpaddr, memseg->pages[i].raddr, 54274bac2208Snarayan cookie_addr, perm); 54284bac2208Snarayan 54294bac2208Snarayan /* 54304bac2208Snarayan * NOTE: Calling hat_devload directly, causes it 54314bac2208Snarayan * to look for page_t using the pfn. Since this 54324bac2208Snarayan * addr is greater than the memlist, it treates 54334bac2208Snarayan * it as non-memory 54344bac2208Snarayan */ 54354bac2208Snarayan sfmmu_memtte(&ldc_tte, 54364bac2208Snarayan (pfn_t)(memseg->pages[i].raddr >> pg_shift), 54374bac2208Snarayan PROT_READ | PROT_WRITE | HAT_NOSYNC, TTE8K); 54384bac2208Snarayan 54394bac2208Snarayan D1(ldcp->id, 54404bac2208Snarayan "ldc_mem_map: (0x%llx) ra 0x%llx -> " 54414bac2208Snarayan "tte 0x%llx\n", ldcp->id, 54424bac2208Snarayan memseg->pages[i].raddr, ldc_tte); 54434bac2208Snarayan 54444bac2208Snarayan sfmmu_tteload(kas.a_hat, &ldc_tte, tmpaddr, 54454bac2208Snarayan NULL, HAT_LOAD_LOCK); 54464bac2208Snarayan 54474bac2208Snarayan cookie_size -= pg_size; 54484bac2208Snarayan cookie_addr += pg_size; 54494bac2208Snarayan tmpaddr += pg_size; 54504bac2208Snarayan } 54514bac2208Snarayan } 54524bac2208Snarayan } 54534bac2208Snarayan 54541ae08745Sheppo if (mtype == LDC_SHADOW_MAP) { 54551ae08745Sheppo if (*vaddr == NULL) { 54563af08d82Slm66018 memseg->vaddr = kmem_zalloc(exp_size, KM_SLEEP); 54571ae08745Sheppo mhdl->myshadow = B_TRUE; 54581ae08745Sheppo 54591ae08745Sheppo D1(ldcp->id, "ldc_mem_map: (0x%llx) allocated " 54604bac2208Snarayan "shadow page va=0x%llx\n", ldcp->id, memseg->vaddr); 54611ae08745Sheppo } else { 54621ae08745Sheppo /* 54634bac2208Snarayan * Use client supplied memory for memseg->vaddr 54641ae08745Sheppo * WARNING: assuming that client mem is >= exp_size 54651ae08745Sheppo */ 54664bac2208Snarayan memseg->vaddr = *vaddr; 54671ae08745Sheppo } 54681ae08745Sheppo 54691ae08745Sheppo /* Save all page and cookie information */ 54704bac2208Snarayan for (i = 0, tmpaddr = memseg->vaddr; i < npages; i++) { 54711ae08745Sheppo memseg->pages[i].raddr = va_to_pa(tmpaddr); 54721ae08745Sheppo memseg->pages[i].size = pg_size; 54731ae08745Sheppo tmpaddr += pg_size; 54741ae08745Sheppo } 54754bac2208Snarayan 54761ae08745Sheppo } 54771ae08745Sheppo 54784bac2208Snarayan /* save all cookies */ 54794bac2208Snarayan bcopy(cookie, memseg->cookies, ccount * sizeof (ldc_mem_cookie_t)); 54804bac2208Snarayan 54811ae08745Sheppo /* update memseg_t */ 54821ae08745Sheppo memseg->raddr = memseg->pages[0].raddr; 54834bac2208Snarayan memseg->size = (mtype == LDC_SHADOW_MAP) ? exp_size : map_size; 54841ae08745Sheppo memseg->npages = npages; 54851ae08745Sheppo memseg->ncookies = ccount; 54861ae08745Sheppo memseg->next_cookie = 0; 54871ae08745Sheppo 54881ae08745Sheppo /* memory handle = mapped */ 54891ae08745Sheppo mhdl->mtype = mtype; 54904bac2208Snarayan mhdl->perm = perm; 54911ae08745Sheppo mhdl->status = LDC_MAPPED; 54921ae08745Sheppo 54931ae08745Sheppo D1(ldcp->id, "ldc_mem_map: (0x%llx) mapped 0x%llx, ra=0x%llx, " 54941ae08745Sheppo "va=0x%llx, pgs=0x%llx cookies=0x%llx\n", 54951ae08745Sheppo ldcp->id, mhdl, memseg->raddr, memseg->vaddr, 54961ae08745Sheppo memseg->npages, memseg->ncookies); 54971ae08745Sheppo 54984bac2208Snarayan if (mtype == LDC_SHADOW_MAP) 54994bac2208Snarayan base_off = 0; 55001ae08745Sheppo if (raddr) 55014bac2208Snarayan *raddr = (caddr_t)(memseg->raddr | base_off); 55021ae08745Sheppo if (vaddr) 55034bac2208Snarayan *vaddr = (caddr_t)((uintptr_t)memseg->vaddr | base_off); 55041ae08745Sheppo 55051ae08745Sheppo mutex_exit(&ldcp->lock); 55061ae08745Sheppo mutex_exit(&mhdl->lock); 55071ae08745Sheppo return (0); 55081ae08745Sheppo } 55091ae08745Sheppo 55101ae08745Sheppo /* 55111ae08745Sheppo * Unmap a memory segment. Free shadow memory (if any). 55121ae08745Sheppo */ 55131ae08745Sheppo int 55141ae08745Sheppo ldc_mem_unmap(ldc_mem_handle_t mhandle) 55151ae08745Sheppo { 55164bac2208Snarayan int i, rv; 55171ae08745Sheppo ldc_mhdl_t *mhdl = (ldc_mhdl_t *)mhandle; 55181ae08745Sheppo ldc_chan_t *ldcp; 55191ae08745Sheppo ldc_memseg_t *memseg; 55201ae08745Sheppo 55211ae08745Sheppo if (mhdl == 0 || mhdl->status != LDC_MAPPED) { 55221ae08745Sheppo DWARN(DBG_ALL_LDCS, 55231ae08745Sheppo "ldc_mem_unmap: (0x%llx) handle is not mapped\n", 55241ae08745Sheppo mhandle); 55251ae08745Sheppo return (EINVAL); 55261ae08745Sheppo } 55271ae08745Sheppo 55281ae08745Sheppo mutex_enter(&mhdl->lock); 55291ae08745Sheppo 55301ae08745Sheppo ldcp = mhdl->ldcp; 55311ae08745Sheppo memseg = mhdl->memseg; 55321ae08745Sheppo 55331ae08745Sheppo D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapping handle 0x%llx\n", 55341ae08745Sheppo ldcp->id, mhdl); 55351ae08745Sheppo 55361ae08745Sheppo /* if we allocated shadow memory - free it */ 55371ae08745Sheppo if (mhdl->mtype == LDC_SHADOW_MAP && mhdl->myshadow) { 55383af08d82Slm66018 kmem_free(memseg->vaddr, memseg->size); 55394bac2208Snarayan } else if (mhdl->mtype == LDC_DIRECT_MAP) { 55404bac2208Snarayan 55414bac2208Snarayan /* unmap in the case of DIRECT_MAP */ 55424bac2208Snarayan hat_unload(kas.a_hat, memseg->vaddr, memseg->size, 55434bac2208Snarayan HAT_UNLOAD_UNLOCK); 55444bac2208Snarayan 55454bac2208Snarayan for (i = 0; i < memseg->npages; i++) { 55464bac2208Snarayan rv = hv_ldc_unmap(memseg->pages[i].raddr); 55474bac2208Snarayan if (rv) { 55484bac2208Snarayan cmn_err(CE_WARN, 55494bac2208Snarayan "ldc_mem_map: (0x%lx) hv unmap err %d\n", 55504bac2208Snarayan ldcp->id, rv); 55514bac2208Snarayan } 55524bac2208Snarayan } 55534bac2208Snarayan 55544bac2208Snarayan vmem_free(heap_arena, (void *)memseg->vaddr, memseg->size); 55551ae08745Sheppo } 55561ae08745Sheppo 55571ae08745Sheppo /* free the allocated memseg and page structures */ 55581ae08745Sheppo kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages)); 55591ae08745Sheppo kmem_free(memseg->cookies, 55601ae08745Sheppo (sizeof (ldc_mem_cookie_t) * memseg->ncookies)); 55614bac2208Snarayan kmem_cache_free(ldcssp->memseg_cache, memseg); 55621ae08745Sheppo 55631ae08745Sheppo /* uninitialize the memory handle */ 55641ae08745Sheppo mhdl->memseg = NULL; 55651ae08745Sheppo mhdl->status = LDC_UNBOUND; 55661ae08745Sheppo 55671ae08745Sheppo D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapped handle 0x%llx\n", 55681ae08745Sheppo ldcp->id, mhdl); 55691ae08745Sheppo 55701ae08745Sheppo mutex_exit(&mhdl->lock); 55711ae08745Sheppo return (0); 55721ae08745Sheppo } 55731ae08745Sheppo 55741ae08745Sheppo /* 55751ae08745Sheppo * Internal entry point for LDC mapped memory entry consistency 55761ae08745Sheppo * semantics. Acquire copies the contents of the remote memory 55771ae08745Sheppo * into the local shadow copy. The release operation copies the local 55781ae08745Sheppo * contents into the remote memory. The offset and size specify the 55791ae08745Sheppo * bounds for the memory range being synchronized. 55801ae08745Sheppo */ 55811ae08745Sheppo static int 55821ae08745Sheppo i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, uint8_t direction, 55831ae08745Sheppo uint64_t offset, size_t size) 55841ae08745Sheppo { 55851ae08745Sheppo int err; 55861ae08745Sheppo ldc_mhdl_t *mhdl; 55871ae08745Sheppo ldc_chan_t *ldcp; 55881ae08745Sheppo ldc_memseg_t *memseg; 55891ae08745Sheppo caddr_t local_vaddr; 55901ae08745Sheppo size_t copy_size; 55911ae08745Sheppo 55921ae08745Sheppo if (mhandle == NULL) { 55931ae08745Sheppo DWARN(DBG_ALL_LDCS, 55941ae08745Sheppo "i_ldc_mem_acquire_release: invalid memory handle\n"); 55951ae08745Sheppo return (EINVAL); 55961ae08745Sheppo } 55971ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 55981ae08745Sheppo 55991ae08745Sheppo mutex_enter(&mhdl->lock); 56001ae08745Sheppo 56011ae08745Sheppo if (mhdl->status != LDC_MAPPED || mhdl->ldcp == NULL) { 56021ae08745Sheppo DWARN(DBG_ALL_LDCS, 56031ae08745Sheppo "i_ldc_mem_acquire_release: not mapped memory\n"); 56041ae08745Sheppo mutex_exit(&mhdl->lock); 56051ae08745Sheppo return (EINVAL); 56061ae08745Sheppo } 56071ae08745Sheppo 56084bac2208Snarayan /* do nothing for direct map */ 56094bac2208Snarayan if (mhdl->mtype == LDC_DIRECT_MAP) { 56104bac2208Snarayan mutex_exit(&mhdl->lock); 56114bac2208Snarayan return (0); 56124bac2208Snarayan } 56134bac2208Snarayan 56144bac2208Snarayan /* do nothing if COPY_IN+MEM_W and COPY_OUT+MEM_R */ 56154bac2208Snarayan if ((direction == LDC_COPY_IN && (mhdl->perm & LDC_MEM_R) == 0) || 56164bac2208Snarayan (direction == LDC_COPY_OUT && (mhdl->perm & LDC_MEM_W) == 0)) { 56174bac2208Snarayan mutex_exit(&mhdl->lock); 56184bac2208Snarayan return (0); 56194bac2208Snarayan } 56204bac2208Snarayan 56211ae08745Sheppo if (offset >= mhdl->memseg->size || 56221ae08745Sheppo (offset + size) > mhdl->memseg->size) { 56231ae08745Sheppo DWARN(DBG_ALL_LDCS, 56241ae08745Sheppo "i_ldc_mem_acquire_release: memory out of range\n"); 56251ae08745Sheppo mutex_exit(&mhdl->lock); 56261ae08745Sheppo return (EINVAL); 56271ae08745Sheppo } 56281ae08745Sheppo 56291ae08745Sheppo /* get the channel handle and memory segment */ 56301ae08745Sheppo ldcp = mhdl->ldcp; 56311ae08745Sheppo memseg = mhdl->memseg; 56321ae08745Sheppo 56331ae08745Sheppo if (mhdl->mtype == LDC_SHADOW_MAP) { 56341ae08745Sheppo 56351ae08745Sheppo local_vaddr = memseg->vaddr + offset; 56361ae08745Sheppo copy_size = size; 56371ae08745Sheppo 56381ae08745Sheppo /* copy to/from remote from/to local memory */ 56391ae08745Sheppo err = ldc_mem_copy((ldc_handle_t)ldcp, local_vaddr, offset, 56401ae08745Sheppo ©_size, memseg->cookies, memseg->ncookies, 56411ae08745Sheppo direction); 56421ae08745Sheppo if (err || copy_size != size) { 5643cb112a14Slm66018 DWARN(ldcp->id, 56441ae08745Sheppo "i_ldc_mem_acquire_release: copy failed\n"); 56451ae08745Sheppo mutex_exit(&mhdl->lock); 56461ae08745Sheppo return (err); 56471ae08745Sheppo } 56481ae08745Sheppo } 56491ae08745Sheppo 56501ae08745Sheppo mutex_exit(&mhdl->lock); 56511ae08745Sheppo 56521ae08745Sheppo return (0); 56531ae08745Sheppo } 56541ae08745Sheppo 56551ae08745Sheppo /* 56561ae08745Sheppo * Ensure that the contents in the remote memory seg are consistent 56571ae08745Sheppo * with the contents if of local segment 56581ae08745Sheppo */ 56591ae08745Sheppo int 56601ae08745Sheppo ldc_mem_acquire(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size) 56611ae08745Sheppo { 56621ae08745Sheppo return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_IN, offset, size)); 56631ae08745Sheppo } 56641ae08745Sheppo 56651ae08745Sheppo 56661ae08745Sheppo /* 56671ae08745Sheppo * Ensure that the contents in the local memory seg are consistent 56681ae08745Sheppo * with the contents if of remote segment 56691ae08745Sheppo */ 56701ae08745Sheppo int 56711ae08745Sheppo ldc_mem_release(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size) 56721ae08745Sheppo { 56731ae08745Sheppo return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_OUT, offset, size)); 56741ae08745Sheppo } 56751ae08745Sheppo 56761ae08745Sheppo /* 56771ae08745Sheppo * Allocate a descriptor ring. The size of each each descriptor 56781ae08745Sheppo * must be 8-byte aligned and the entire ring should be a multiple 56791ae08745Sheppo * of MMU_PAGESIZE. 56801ae08745Sheppo */ 56811ae08745Sheppo int 56821ae08745Sheppo ldc_mem_dring_create(uint32_t len, uint32_t dsize, ldc_dring_handle_t *dhandle) 56831ae08745Sheppo { 56841ae08745Sheppo ldc_dring_t *dringp; 56851ae08745Sheppo size_t size = (dsize * len); 56861ae08745Sheppo 56871ae08745Sheppo D1(DBG_ALL_LDCS, "ldc_mem_dring_create: len=0x%x, size=0x%x\n", 56881ae08745Sheppo len, dsize); 56891ae08745Sheppo 56901ae08745Sheppo if (dhandle == NULL) { 56911ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid dhandle\n"); 56921ae08745Sheppo return (EINVAL); 56931ae08745Sheppo } 56941ae08745Sheppo 56951ae08745Sheppo if (len == 0) { 56961ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid length\n"); 56971ae08745Sheppo return (EINVAL); 56981ae08745Sheppo } 56991ae08745Sheppo 57001ae08745Sheppo /* descriptor size should be 8-byte aligned */ 57011ae08745Sheppo if (dsize == 0 || (dsize & 0x7)) { 57021ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid size\n"); 57031ae08745Sheppo return (EINVAL); 57041ae08745Sheppo } 57051ae08745Sheppo 57061ae08745Sheppo *dhandle = 0; 57071ae08745Sheppo 57081ae08745Sheppo /* Allocate a desc ring structure */ 57091ae08745Sheppo dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP); 57101ae08745Sheppo 57111ae08745Sheppo /* Initialize dring */ 57121ae08745Sheppo dringp->length = len; 57131ae08745Sheppo dringp->dsize = dsize; 57141ae08745Sheppo 57151ae08745Sheppo /* round off to multiple of pagesize */ 57161ae08745Sheppo dringp->size = (size & MMU_PAGEMASK); 57171ae08745Sheppo if (size & MMU_PAGEOFFSET) 57181ae08745Sheppo dringp->size += MMU_PAGESIZE; 57191ae08745Sheppo 57201ae08745Sheppo dringp->status = LDC_UNBOUND; 57211ae08745Sheppo 57221ae08745Sheppo /* allocate descriptor ring memory */ 57233af08d82Slm66018 dringp->base = kmem_zalloc(dringp->size, KM_SLEEP); 57241ae08745Sheppo 57251ae08745Sheppo /* initialize the desc ring lock */ 57261ae08745Sheppo mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL); 57271ae08745Sheppo 57281ae08745Sheppo /* Add descriptor ring to the head of global list */ 57291ae08745Sheppo mutex_enter(&ldcssp->lock); 57301ae08745Sheppo dringp->next = ldcssp->dring_list; 57311ae08745Sheppo ldcssp->dring_list = dringp; 57321ae08745Sheppo mutex_exit(&ldcssp->lock); 57331ae08745Sheppo 57341ae08745Sheppo *dhandle = (ldc_dring_handle_t)dringp; 57351ae08745Sheppo 57361ae08745Sheppo D1(DBG_ALL_LDCS, "ldc_mem_dring_create: dring allocated\n"); 57371ae08745Sheppo 57381ae08745Sheppo return (0); 57391ae08745Sheppo } 57401ae08745Sheppo 57411ae08745Sheppo 57421ae08745Sheppo /* 57431ae08745Sheppo * Destroy a descriptor ring. 57441ae08745Sheppo */ 57451ae08745Sheppo int 57461ae08745Sheppo ldc_mem_dring_destroy(ldc_dring_handle_t dhandle) 57471ae08745Sheppo { 57481ae08745Sheppo ldc_dring_t *dringp; 57491ae08745Sheppo ldc_dring_t *tmp_dringp; 57501ae08745Sheppo 57511ae08745Sheppo D1(DBG_ALL_LDCS, "ldc_mem_dring_destroy: entered\n"); 57521ae08745Sheppo 57531ae08745Sheppo if (dhandle == NULL) { 57541ae08745Sheppo DWARN(DBG_ALL_LDCS, 57551ae08745Sheppo "ldc_mem_dring_destroy: invalid desc ring handle\n"); 57561ae08745Sheppo return (EINVAL); 57571ae08745Sheppo } 57581ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 57591ae08745Sheppo 57601ae08745Sheppo if (dringp->status == LDC_BOUND) { 57611ae08745Sheppo DWARN(DBG_ALL_LDCS, 57621ae08745Sheppo "ldc_mem_dring_destroy: desc ring is bound\n"); 57631ae08745Sheppo return (EACCES); 57641ae08745Sheppo } 57651ae08745Sheppo 57661ae08745Sheppo mutex_enter(&dringp->lock); 57671ae08745Sheppo mutex_enter(&ldcssp->lock); 57681ae08745Sheppo 57691ae08745Sheppo /* remove from linked list - if not bound */ 57701ae08745Sheppo tmp_dringp = ldcssp->dring_list; 57711ae08745Sheppo if (tmp_dringp == dringp) { 57721ae08745Sheppo ldcssp->dring_list = dringp->next; 57731ae08745Sheppo dringp->next = NULL; 57741ae08745Sheppo 57751ae08745Sheppo } else { 57761ae08745Sheppo while (tmp_dringp != NULL) { 57771ae08745Sheppo if (tmp_dringp->next == dringp) { 57781ae08745Sheppo tmp_dringp->next = dringp->next; 57791ae08745Sheppo dringp->next = NULL; 57801ae08745Sheppo break; 57811ae08745Sheppo } 57821ae08745Sheppo tmp_dringp = tmp_dringp->next; 57831ae08745Sheppo } 57841ae08745Sheppo if (tmp_dringp == NULL) { 57851ae08745Sheppo DWARN(DBG_ALL_LDCS, 57861ae08745Sheppo "ldc_mem_dring_destroy: invalid descriptor\n"); 57871ae08745Sheppo mutex_exit(&ldcssp->lock); 57881ae08745Sheppo mutex_exit(&dringp->lock); 57891ae08745Sheppo return (EINVAL); 57901ae08745Sheppo } 57911ae08745Sheppo } 57921ae08745Sheppo 57931ae08745Sheppo mutex_exit(&ldcssp->lock); 57941ae08745Sheppo 57951ae08745Sheppo /* free the descriptor ring */ 57963af08d82Slm66018 kmem_free(dringp->base, dringp->size); 57971ae08745Sheppo 57981ae08745Sheppo mutex_exit(&dringp->lock); 57991ae08745Sheppo 58001ae08745Sheppo /* destroy dring lock */ 58011ae08745Sheppo mutex_destroy(&dringp->lock); 58021ae08745Sheppo 58031ae08745Sheppo /* free desc ring object */ 58041ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 58051ae08745Sheppo 58061ae08745Sheppo return (0); 58071ae08745Sheppo } 58081ae08745Sheppo 58091ae08745Sheppo /* 58101ae08745Sheppo * Bind a previously allocated dring to a channel. The channel should 58111ae08745Sheppo * be OPEN in order to bind the ring to the channel. Returns back a 58121ae08745Sheppo * descriptor ring cookie. The descriptor ring is exported for remote 58131ae08745Sheppo * access by the client at the other end of the channel. An entry for 58141ae08745Sheppo * dring pages is stored in map table (via call to ldc_mem_bind_handle). 58151ae08745Sheppo */ 58161ae08745Sheppo int 58171ae08745Sheppo ldc_mem_dring_bind(ldc_handle_t handle, ldc_dring_handle_t dhandle, 58181ae08745Sheppo uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount) 58191ae08745Sheppo { 58201ae08745Sheppo int err; 58211ae08745Sheppo ldc_chan_t *ldcp; 58221ae08745Sheppo ldc_dring_t *dringp; 58231ae08745Sheppo ldc_mem_handle_t mhandle; 58241ae08745Sheppo 58251ae08745Sheppo /* check to see if channel is initalized */ 58261ae08745Sheppo if (handle == NULL) { 58271ae08745Sheppo DWARN(DBG_ALL_LDCS, 58281ae08745Sheppo "ldc_mem_dring_bind: invalid channel handle\n"); 58291ae08745Sheppo return (EINVAL); 58301ae08745Sheppo } 58311ae08745Sheppo ldcp = (ldc_chan_t *)handle; 58321ae08745Sheppo 58331ae08745Sheppo if (dhandle == NULL) { 58341ae08745Sheppo DWARN(DBG_ALL_LDCS, 58351ae08745Sheppo "ldc_mem_dring_bind: invalid desc ring handle\n"); 58361ae08745Sheppo return (EINVAL); 58371ae08745Sheppo } 58381ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 58391ae08745Sheppo 58401ae08745Sheppo if (cookie == NULL) { 58411ae08745Sheppo DWARN(ldcp->id, 58421ae08745Sheppo "ldc_mem_dring_bind: invalid cookie arg\n"); 58431ae08745Sheppo return (EINVAL); 58441ae08745Sheppo } 58451ae08745Sheppo 58461ae08745Sheppo mutex_enter(&dringp->lock); 58471ae08745Sheppo 58481ae08745Sheppo if (dringp->status == LDC_BOUND) { 58491ae08745Sheppo DWARN(DBG_ALL_LDCS, 58501ae08745Sheppo "ldc_mem_dring_bind: (0x%llx) descriptor ring is bound\n", 58511ae08745Sheppo ldcp->id); 58521ae08745Sheppo mutex_exit(&dringp->lock); 58531ae08745Sheppo return (EINVAL); 58541ae08745Sheppo } 58551ae08745Sheppo 58561ae08745Sheppo if ((perm & LDC_MEM_RW) == 0) { 58571ae08745Sheppo DWARN(DBG_ALL_LDCS, 58581ae08745Sheppo "ldc_mem_dring_bind: invalid permissions\n"); 58591ae08745Sheppo mutex_exit(&dringp->lock); 58601ae08745Sheppo return (EINVAL); 58611ae08745Sheppo } 58621ae08745Sheppo 58631ae08745Sheppo if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) { 58641ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_dring_bind: invalid type\n"); 58651ae08745Sheppo mutex_exit(&dringp->lock); 58661ae08745Sheppo return (EINVAL); 58671ae08745Sheppo } 58681ae08745Sheppo 58691ae08745Sheppo dringp->ldcp = ldcp; 58701ae08745Sheppo 58711ae08745Sheppo /* create an memory handle */ 58721ae08745Sheppo err = ldc_mem_alloc_handle(handle, &mhandle); 58731ae08745Sheppo if (err || mhandle == NULL) { 58741ae08745Sheppo DWARN(DBG_ALL_LDCS, 58751ae08745Sheppo "ldc_mem_dring_bind: (0x%llx) error allocating mhandle\n", 58761ae08745Sheppo ldcp->id); 58771ae08745Sheppo mutex_exit(&dringp->lock); 58781ae08745Sheppo return (err); 58791ae08745Sheppo } 58801ae08745Sheppo dringp->mhdl = mhandle; 58811ae08745Sheppo 58821ae08745Sheppo /* bind the descriptor ring to channel */ 58831ae08745Sheppo err = ldc_mem_bind_handle(mhandle, dringp->base, dringp->size, 58841ae08745Sheppo mtype, perm, cookie, ccount); 58851ae08745Sheppo if (err) { 58861ae08745Sheppo DWARN(ldcp->id, 58871ae08745Sheppo "ldc_mem_dring_bind: (0x%llx) error binding mhandle\n", 58881ae08745Sheppo ldcp->id); 58891ae08745Sheppo mutex_exit(&dringp->lock); 58901ae08745Sheppo return (err); 58911ae08745Sheppo } 58921ae08745Sheppo 58931ae08745Sheppo /* 58941ae08745Sheppo * For now return error if we get more than one cookie 58951ae08745Sheppo * FUTURE: Return multiple cookies .. 58961ae08745Sheppo */ 58971ae08745Sheppo if (*ccount > 1) { 58981ae08745Sheppo (void) ldc_mem_unbind_handle(mhandle); 58991ae08745Sheppo (void) ldc_mem_free_handle(mhandle); 59001ae08745Sheppo 59011ae08745Sheppo dringp->ldcp = NULL; 59021ae08745Sheppo dringp->mhdl = NULL; 59031ae08745Sheppo *ccount = 0; 59041ae08745Sheppo 59051ae08745Sheppo mutex_exit(&dringp->lock); 59061ae08745Sheppo return (EAGAIN); 59071ae08745Sheppo } 59081ae08745Sheppo 59091ae08745Sheppo /* Add descriptor ring to channel's exported dring list */ 59101ae08745Sheppo mutex_enter(&ldcp->exp_dlist_lock); 59111ae08745Sheppo dringp->ch_next = ldcp->exp_dring_list; 59121ae08745Sheppo ldcp->exp_dring_list = dringp; 59131ae08745Sheppo mutex_exit(&ldcp->exp_dlist_lock); 59141ae08745Sheppo 59151ae08745Sheppo dringp->status = LDC_BOUND; 59161ae08745Sheppo 59171ae08745Sheppo mutex_exit(&dringp->lock); 59181ae08745Sheppo 59191ae08745Sheppo return (0); 59201ae08745Sheppo } 59211ae08745Sheppo 59221ae08745Sheppo /* 59231ae08745Sheppo * Return the next cookie associated with the specified dring handle 59241ae08745Sheppo */ 59251ae08745Sheppo int 59261ae08745Sheppo ldc_mem_dring_nextcookie(ldc_dring_handle_t dhandle, ldc_mem_cookie_t *cookie) 59271ae08745Sheppo { 59281ae08745Sheppo int rv = 0; 59291ae08745Sheppo ldc_dring_t *dringp; 59301ae08745Sheppo ldc_chan_t *ldcp; 59311ae08745Sheppo 59321ae08745Sheppo if (dhandle == NULL) { 59331ae08745Sheppo DWARN(DBG_ALL_LDCS, 59341ae08745Sheppo "ldc_mem_dring_nextcookie: invalid desc ring handle\n"); 59351ae08745Sheppo return (EINVAL); 59361ae08745Sheppo } 59371ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 59381ae08745Sheppo mutex_enter(&dringp->lock); 59391ae08745Sheppo 59401ae08745Sheppo if (dringp->status != LDC_BOUND) { 59411ae08745Sheppo DWARN(DBG_ALL_LDCS, 59421ae08745Sheppo "ldc_mem_dring_nextcookie: descriptor ring 0x%llx " 59431ae08745Sheppo "is not bound\n", dringp); 59441ae08745Sheppo mutex_exit(&dringp->lock); 59451ae08745Sheppo return (EINVAL); 59461ae08745Sheppo } 59471ae08745Sheppo 59481ae08745Sheppo ldcp = dringp->ldcp; 59491ae08745Sheppo 59501ae08745Sheppo if (cookie == NULL) { 59511ae08745Sheppo DWARN(ldcp->id, 59521ae08745Sheppo "ldc_mem_dring_nextcookie:(0x%llx) invalid cookie arg\n", 59531ae08745Sheppo ldcp->id); 59541ae08745Sheppo mutex_exit(&dringp->lock); 59551ae08745Sheppo return (EINVAL); 59561ae08745Sheppo } 59571ae08745Sheppo 59581ae08745Sheppo rv = ldc_mem_nextcookie((ldc_mem_handle_t)dringp->mhdl, cookie); 59591ae08745Sheppo mutex_exit(&dringp->lock); 59601ae08745Sheppo 59611ae08745Sheppo return (rv); 59621ae08745Sheppo } 59631ae08745Sheppo /* 59641ae08745Sheppo * Unbind a previously bound dring from a channel. 59651ae08745Sheppo */ 59661ae08745Sheppo int 59671ae08745Sheppo ldc_mem_dring_unbind(ldc_dring_handle_t dhandle) 59681ae08745Sheppo { 59691ae08745Sheppo ldc_dring_t *dringp; 59701ae08745Sheppo ldc_dring_t *tmp_dringp; 59711ae08745Sheppo ldc_chan_t *ldcp; 59721ae08745Sheppo 59731ae08745Sheppo if (dhandle == NULL) { 59741ae08745Sheppo DWARN(DBG_ALL_LDCS, 59751ae08745Sheppo "ldc_mem_dring_unbind: invalid desc ring handle\n"); 59761ae08745Sheppo return (EINVAL); 59771ae08745Sheppo } 59781ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 59791ae08745Sheppo 59801ae08745Sheppo mutex_enter(&dringp->lock); 59811ae08745Sheppo 59821ae08745Sheppo if (dringp->status == LDC_UNBOUND) { 59831ae08745Sheppo DWARN(DBG_ALL_LDCS, 59841ae08745Sheppo "ldc_mem_dring_bind: descriptor ring 0x%llx is unbound\n", 59851ae08745Sheppo dringp); 59861ae08745Sheppo mutex_exit(&dringp->lock); 59871ae08745Sheppo return (EINVAL); 59881ae08745Sheppo } 59891ae08745Sheppo ldcp = dringp->ldcp; 59901ae08745Sheppo 59911ae08745Sheppo mutex_enter(&ldcp->exp_dlist_lock); 59921ae08745Sheppo 59931ae08745Sheppo tmp_dringp = ldcp->exp_dring_list; 59941ae08745Sheppo if (tmp_dringp == dringp) { 59951ae08745Sheppo ldcp->exp_dring_list = dringp->ch_next; 59961ae08745Sheppo dringp->ch_next = NULL; 59971ae08745Sheppo 59981ae08745Sheppo } else { 59991ae08745Sheppo while (tmp_dringp != NULL) { 60001ae08745Sheppo if (tmp_dringp->ch_next == dringp) { 60011ae08745Sheppo tmp_dringp->ch_next = dringp->ch_next; 60021ae08745Sheppo dringp->ch_next = NULL; 60031ae08745Sheppo break; 60041ae08745Sheppo } 60051ae08745Sheppo tmp_dringp = tmp_dringp->ch_next; 60061ae08745Sheppo } 60071ae08745Sheppo if (tmp_dringp == NULL) { 60081ae08745Sheppo DWARN(DBG_ALL_LDCS, 60091ae08745Sheppo "ldc_mem_dring_unbind: invalid descriptor\n"); 60101ae08745Sheppo mutex_exit(&ldcp->exp_dlist_lock); 60111ae08745Sheppo mutex_exit(&dringp->lock); 60121ae08745Sheppo return (EINVAL); 60131ae08745Sheppo } 60141ae08745Sheppo } 60151ae08745Sheppo 60161ae08745Sheppo mutex_exit(&ldcp->exp_dlist_lock); 60171ae08745Sheppo 60181ae08745Sheppo (void) ldc_mem_unbind_handle((ldc_mem_handle_t)dringp->mhdl); 60191ae08745Sheppo (void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl); 60201ae08745Sheppo 60211ae08745Sheppo dringp->ldcp = NULL; 60221ae08745Sheppo dringp->mhdl = NULL; 60231ae08745Sheppo dringp->status = LDC_UNBOUND; 60241ae08745Sheppo 60251ae08745Sheppo mutex_exit(&dringp->lock); 60261ae08745Sheppo 60271ae08745Sheppo return (0); 60281ae08745Sheppo } 60291ae08745Sheppo 60301ae08745Sheppo /* 60311ae08745Sheppo * Get information about the dring. The base address of the descriptor 60321ae08745Sheppo * ring along with the type and permission are returned back. 60331ae08745Sheppo */ 60341ae08745Sheppo int 60351ae08745Sheppo ldc_mem_dring_info(ldc_dring_handle_t dhandle, ldc_mem_info_t *minfo) 60361ae08745Sheppo { 60371ae08745Sheppo ldc_dring_t *dringp; 60381ae08745Sheppo int rv; 60391ae08745Sheppo 60401ae08745Sheppo if (dhandle == NULL) { 60411ae08745Sheppo DWARN(DBG_ALL_LDCS, 60421ae08745Sheppo "ldc_mem_dring_info: invalid desc ring handle\n"); 60431ae08745Sheppo return (EINVAL); 60441ae08745Sheppo } 60451ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 60461ae08745Sheppo 60471ae08745Sheppo mutex_enter(&dringp->lock); 60481ae08745Sheppo 60491ae08745Sheppo if (dringp->mhdl) { 60501ae08745Sheppo rv = ldc_mem_info(dringp->mhdl, minfo); 60511ae08745Sheppo if (rv) { 60521ae08745Sheppo DWARN(DBG_ALL_LDCS, 60531ae08745Sheppo "ldc_mem_dring_info: error reading mem info\n"); 60541ae08745Sheppo mutex_exit(&dringp->lock); 60551ae08745Sheppo return (rv); 60561ae08745Sheppo } 60571ae08745Sheppo } else { 60581ae08745Sheppo minfo->vaddr = dringp->base; 60591ae08745Sheppo minfo->raddr = NULL; 60601ae08745Sheppo minfo->status = dringp->status; 60611ae08745Sheppo } 60621ae08745Sheppo 60631ae08745Sheppo mutex_exit(&dringp->lock); 60641ae08745Sheppo 60651ae08745Sheppo return (0); 60661ae08745Sheppo } 60671ae08745Sheppo 60681ae08745Sheppo /* 60691ae08745Sheppo * Map an exported descriptor ring into the local address space. If the 60701ae08745Sheppo * descriptor ring was exported for direct map access, a HV call is made 60711ae08745Sheppo * to allocate a RA range. If the map is done via a shadow copy, local 60721ae08745Sheppo * shadow memory is allocated. 60731ae08745Sheppo */ 60741ae08745Sheppo int 60751ae08745Sheppo ldc_mem_dring_map(ldc_handle_t handle, ldc_mem_cookie_t *cookie, 60761ae08745Sheppo uint32_t ccount, uint32_t len, uint32_t dsize, uint8_t mtype, 60771ae08745Sheppo ldc_dring_handle_t *dhandle) 60781ae08745Sheppo { 60791ae08745Sheppo int err; 60801ae08745Sheppo ldc_chan_t *ldcp = (ldc_chan_t *)handle; 60811ae08745Sheppo ldc_mem_handle_t mhandle; 60821ae08745Sheppo ldc_dring_t *dringp; 60831ae08745Sheppo size_t dring_size; 60841ae08745Sheppo 60851ae08745Sheppo if (dhandle == NULL) { 60861ae08745Sheppo DWARN(DBG_ALL_LDCS, 60871ae08745Sheppo "ldc_mem_dring_map: invalid dhandle\n"); 60881ae08745Sheppo return (EINVAL); 60891ae08745Sheppo } 60901ae08745Sheppo 60911ae08745Sheppo /* check to see if channel is initalized */ 60921ae08745Sheppo if (handle == NULL) { 60931ae08745Sheppo DWARN(DBG_ALL_LDCS, 60941ae08745Sheppo "ldc_mem_dring_map: invalid channel handle\n"); 60951ae08745Sheppo return (EINVAL); 60961ae08745Sheppo } 60971ae08745Sheppo ldcp = (ldc_chan_t *)handle; 60981ae08745Sheppo 60991ae08745Sheppo if (cookie == NULL) { 61001ae08745Sheppo DWARN(ldcp->id, 61011ae08745Sheppo "ldc_mem_dring_map: (0x%llx) invalid cookie\n", 61021ae08745Sheppo ldcp->id); 61031ae08745Sheppo return (EINVAL); 61041ae08745Sheppo } 61051ae08745Sheppo 61061ae08745Sheppo /* FUTURE: For now we support only one cookie per dring */ 61071ae08745Sheppo ASSERT(ccount == 1); 61081ae08745Sheppo 61091ae08745Sheppo if (cookie->size < (dsize * len)) { 61101ae08745Sheppo DWARN(ldcp->id, 61111ae08745Sheppo "ldc_mem_dring_map: (0x%llx) invalid dsize/len\n", 61121ae08745Sheppo ldcp->id); 61131ae08745Sheppo return (EINVAL); 61141ae08745Sheppo } 61151ae08745Sheppo 61161ae08745Sheppo *dhandle = 0; 61171ae08745Sheppo 61181ae08745Sheppo /* Allocate an dring structure */ 61191ae08745Sheppo dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP); 61201ae08745Sheppo 61211ae08745Sheppo D1(ldcp->id, 61221ae08745Sheppo "ldc_mem_dring_map: 0x%x,0x%x,0x%x,0x%llx,0x%llx\n", 61231ae08745Sheppo mtype, len, dsize, cookie->addr, cookie->size); 61241ae08745Sheppo 61251ae08745Sheppo /* Initialize dring */ 61261ae08745Sheppo dringp->length = len; 61271ae08745Sheppo dringp->dsize = dsize; 61281ae08745Sheppo 61291ae08745Sheppo /* round of to multiple of page size */ 61301ae08745Sheppo dring_size = len * dsize; 61311ae08745Sheppo dringp->size = (dring_size & MMU_PAGEMASK); 61321ae08745Sheppo if (dring_size & MMU_PAGEOFFSET) 61331ae08745Sheppo dringp->size += MMU_PAGESIZE; 61341ae08745Sheppo 61351ae08745Sheppo dringp->ldcp = ldcp; 61361ae08745Sheppo 61371ae08745Sheppo /* create an memory handle */ 61381ae08745Sheppo err = ldc_mem_alloc_handle(handle, &mhandle); 61391ae08745Sheppo if (err || mhandle == NULL) { 61401ae08745Sheppo DWARN(DBG_ALL_LDCS, 61411ae08745Sheppo "ldc_mem_dring_map: cannot alloc hdl err=%d\n", 61421ae08745Sheppo err); 61431ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 61441ae08745Sheppo return (ENOMEM); 61451ae08745Sheppo } 61461ae08745Sheppo 61471ae08745Sheppo dringp->mhdl = mhandle; 61481ae08745Sheppo dringp->base = NULL; 61491ae08745Sheppo 61501ae08745Sheppo /* map the dring into local memory */ 61514bac2208Snarayan err = ldc_mem_map(mhandle, cookie, ccount, mtype, LDC_MEM_RW, 61521ae08745Sheppo &(dringp->base), NULL); 61531ae08745Sheppo if (err || dringp->base == NULL) { 61541ae08745Sheppo cmn_err(CE_WARN, 61551ae08745Sheppo "ldc_mem_dring_map: cannot map desc ring err=%d\n", err); 61561ae08745Sheppo (void) ldc_mem_free_handle(mhandle); 61571ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 61581ae08745Sheppo return (ENOMEM); 61591ae08745Sheppo } 61601ae08745Sheppo 61611ae08745Sheppo /* initialize the desc ring lock */ 61621ae08745Sheppo mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL); 61631ae08745Sheppo 61641ae08745Sheppo /* Add descriptor ring to channel's imported dring list */ 61651ae08745Sheppo mutex_enter(&ldcp->imp_dlist_lock); 61661ae08745Sheppo dringp->ch_next = ldcp->imp_dring_list; 61671ae08745Sheppo ldcp->imp_dring_list = dringp; 61681ae08745Sheppo mutex_exit(&ldcp->imp_dlist_lock); 61691ae08745Sheppo 61701ae08745Sheppo dringp->status = LDC_MAPPED; 61711ae08745Sheppo 61721ae08745Sheppo *dhandle = (ldc_dring_handle_t)dringp; 61731ae08745Sheppo 61741ae08745Sheppo return (0); 61751ae08745Sheppo } 61761ae08745Sheppo 61771ae08745Sheppo /* 61781ae08745Sheppo * Unmap a descriptor ring. Free shadow memory (if any). 61791ae08745Sheppo */ 61801ae08745Sheppo int 61811ae08745Sheppo ldc_mem_dring_unmap(ldc_dring_handle_t dhandle) 61821ae08745Sheppo { 61831ae08745Sheppo ldc_dring_t *dringp; 61841ae08745Sheppo ldc_dring_t *tmp_dringp; 61851ae08745Sheppo ldc_chan_t *ldcp; 61861ae08745Sheppo 61871ae08745Sheppo if (dhandle == NULL) { 61881ae08745Sheppo DWARN(DBG_ALL_LDCS, 61891ae08745Sheppo "ldc_mem_dring_unmap: invalid desc ring handle\n"); 61901ae08745Sheppo return (EINVAL); 61911ae08745Sheppo } 61921ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 61931ae08745Sheppo 61941ae08745Sheppo if (dringp->status != LDC_MAPPED) { 61951ae08745Sheppo DWARN(DBG_ALL_LDCS, 61961ae08745Sheppo "ldc_mem_dring_unmap: not a mapped desc ring\n"); 61971ae08745Sheppo return (EINVAL); 61981ae08745Sheppo } 61991ae08745Sheppo 62001ae08745Sheppo mutex_enter(&dringp->lock); 62011ae08745Sheppo 62021ae08745Sheppo ldcp = dringp->ldcp; 62031ae08745Sheppo 62041ae08745Sheppo mutex_enter(&ldcp->imp_dlist_lock); 62051ae08745Sheppo 62061ae08745Sheppo /* find and unlink the desc ring from channel import list */ 62071ae08745Sheppo tmp_dringp = ldcp->imp_dring_list; 62081ae08745Sheppo if (tmp_dringp == dringp) { 62091ae08745Sheppo ldcp->imp_dring_list = dringp->ch_next; 62101ae08745Sheppo dringp->ch_next = NULL; 62111ae08745Sheppo 62121ae08745Sheppo } else { 62131ae08745Sheppo while (tmp_dringp != NULL) { 62141ae08745Sheppo if (tmp_dringp->ch_next == dringp) { 62151ae08745Sheppo tmp_dringp->ch_next = dringp->ch_next; 62161ae08745Sheppo dringp->ch_next = NULL; 62171ae08745Sheppo break; 62181ae08745Sheppo } 62191ae08745Sheppo tmp_dringp = tmp_dringp->ch_next; 62201ae08745Sheppo } 62211ae08745Sheppo if (tmp_dringp == NULL) { 62221ae08745Sheppo DWARN(DBG_ALL_LDCS, 62231ae08745Sheppo "ldc_mem_dring_unmap: invalid descriptor\n"); 62241ae08745Sheppo mutex_exit(&ldcp->imp_dlist_lock); 62251ae08745Sheppo mutex_exit(&dringp->lock); 62261ae08745Sheppo return (EINVAL); 62271ae08745Sheppo } 62281ae08745Sheppo } 62291ae08745Sheppo 62301ae08745Sheppo mutex_exit(&ldcp->imp_dlist_lock); 62311ae08745Sheppo 62321ae08745Sheppo /* do a LDC memory handle unmap and free */ 62331ae08745Sheppo (void) ldc_mem_unmap(dringp->mhdl); 62341ae08745Sheppo (void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl); 62351ae08745Sheppo 62361ae08745Sheppo dringp->status = 0; 62371ae08745Sheppo dringp->ldcp = NULL; 62381ae08745Sheppo 62391ae08745Sheppo mutex_exit(&dringp->lock); 62401ae08745Sheppo 62411ae08745Sheppo /* destroy dring lock */ 62421ae08745Sheppo mutex_destroy(&dringp->lock); 62431ae08745Sheppo 62441ae08745Sheppo /* free desc ring object */ 62451ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 62461ae08745Sheppo 62471ae08745Sheppo return (0); 62481ae08745Sheppo } 62491ae08745Sheppo 62501ae08745Sheppo /* 62511ae08745Sheppo * Internal entry point for descriptor ring access entry consistency 62521ae08745Sheppo * semantics. Acquire copies the contents of the remote descriptor ring 62531ae08745Sheppo * into the local shadow copy. The release operation copies the local 62541ae08745Sheppo * contents into the remote dring. The start and end locations specify 62551ae08745Sheppo * bounds for the entries being synchronized. 62561ae08745Sheppo */ 62571ae08745Sheppo static int 62581ae08745Sheppo i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle, 62591ae08745Sheppo uint8_t direction, uint64_t start, uint64_t end) 62601ae08745Sheppo { 62611ae08745Sheppo int err; 62621ae08745Sheppo ldc_dring_t *dringp; 62631ae08745Sheppo ldc_chan_t *ldcp; 62641ae08745Sheppo uint64_t soff; 62651ae08745Sheppo size_t copy_size; 62661ae08745Sheppo 62671ae08745Sheppo if (dhandle == NULL) { 62681ae08745Sheppo DWARN(DBG_ALL_LDCS, 62691ae08745Sheppo "i_ldc_dring_acquire_release: invalid desc ring handle\n"); 62701ae08745Sheppo return (EINVAL); 62711ae08745Sheppo } 62721ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 62731ae08745Sheppo mutex_enter(&dringp->lock); 62741ae08745Sheppo 62751ae08745Sheppo if (dringp->status != LDC_MAPPED || dringp->ldcp == NULL) { 62761ae08745Sheppo DWARN(DBG_ALL_LDCS, 62771ae08745Sheppo "i_ldc_dring_acquire_release: not a mapped desc ring\n"); 62781ae08745Sheppo mutex_exit(&dringp->lock); 62791ae08745Sheppo return (EINVAL); 62801ae08745Sheppo } 62811ae08745Sheppo 62821ae08745Sheppo if (start >= dringp->length || end >= dringp->length) { 62831ae08745Sheppo DWARN(DBG_ALL_LDCS, 62841ae08745Sheppo "i_ldc_dring_acquire_release: index out of range\n"); 62851ae08745Sheppo mutex_exit(&dringp->lock); 62861ae08745Sheppo return (EINVAL); 62871ae08745Sheppo } 62881ae08745Sheppo 62891ae08745Sheppo /* get the channel handle */ 62901ae08745Sheppo ldcp = dringp->ldcp; 62911ae08745Sheppo 62921ae08745Sheppo copy_size = (start <= end) ? (((end - start) + 1) * dringp->dsize) : 62931ae08745Sheppo ((dringp->length - start) * dringp->dsize); 62941ae08745Sheppo 62951ae08745Sheppo /* Calculate the relative offset for the first desc */ 62961ae08745Sheppo soff = (start * dringp->dsize); 62971ae08745Sheppo 62981ae08745Sheppo /* copy to/from remote from/to local memory */ 62991ae08745Sheppo D1(ldcp->id, "i_ldc_dring_acquire_release: c1 off=0x%llx sz=0x%llx\n", 63001ae08745Sheppo soff, copy_size); 63011ae08745Sheppo err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl, 63021ae08745Sheppo direction, soff, copy_size); 63031ae08745Sheppo if (err) { 63041ae08745Sheppo DWARN(ldcp->id, 63051ae08745Sheppo "i_ldc_dring_acquire_release: copy failed\n"); 63061ae08745Sheppo mutex_exit(&dringp->lock); 63071ae08745Sheppo return (err); 63081ae08745Sheppo } 63091ae08745Sheppo 63101ae08745Sheppo /* do the balance */ 63111ae08745Sheppo if (start > end) { 63121ae08745Sheppo copy_size = ((end + 1) * dringp->dsize); 63131ae08745Sheppo soff = 0; 63141ae08745Sheppo 63151ae08745Sheppo /* copy to/from remote from/to local memory */ 63161ae08745Sheppo D1(ldcp->id, "i_ldc_dring_acquire_release: c2 " 63171ae08745Sheppo "off=0x%llx sz=0x%llx\n", soff, copy_size); 63181ae08745Sheppo err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl, 63191ae08745Sheppo direction, soff, copy_size); 63201ae08745Sheppo if (err) { 63211ae08745Sheppo DWARN(ldcp->id, 63221ae08745Sheppo "i_ldc_dring_acquire_release: copy failed\n"); 63231ae08745Sheppo mutex_exit(&dringp->lock); 63241ae08745Sheppo return (err); 63251ae08745Sheppo } 63261ae08745Sheppo } 63271ae08745Sheppo 63281ae08745Sheppo mutex_exit(&dringp->lock); 63291ae08745Sheppo 63301ae08745Sheppo return (0); 63311ae08745Sheppo } 63321ae08745Sheppo 63331ae08745Sheppo /* 63341ae08745Sheppo * Ensure that the contents in the local dring are consistent 63351ae08745Sheppo * with the contents if of remote dring 63361ae08745Sheppo */ 63371ae08745Sheppo int 63381ae08745Sheppo ldc_mem_dring_acquire(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end) 63391ae08745Sheppo { 63401ae08745Sheppo return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_IN, start, end)); 63411ae08745Sheppo } 63421ae08745Sheppo 63431ae08745Sheppo /* 63441ae08745Sheppo * Ensure that the contents in the remote dring are consistent 63451ae08745Sheppo * with the contents if of local dring 63461ae08745Sheppo */ 63471ae08745Sheppo int 63481ae08745Sheppo ldc_mem_dring_release(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end) 63491ae08745Sheppo { 63501ae08745Sheppo return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_OUT, start, end)); 63511ae08745Sheppo } 63521ae08745Sheppo 63531ae08745Sheppo 63541ae08745Sheppo /* ------------------------------------------------------------------------- */ 6355