11ae08745Sheppo /* 21ae08745Sheppo * CDDL HEADER START 31ae08745Sheppo * 41ae08745Sheppo * The contents of this file are subject to the terms of the 51ae08745Sheppo * Common Development and Distribution License (the "License"). 61ae08745Sheppo * You may not use this file except in compliance with the License. 71ae08745Sheppo * 81ae08745Sheppo * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 91ae08745Sheppo * or http://www.opensolaris.org/os/licensing. 101ae08745Sheppo * See the License for the specific language governing permissions 111ae08745Sheppo * and limitations under the License. 121ae08745Sheppo * 131ae08745Sheppo * When distributing Covered Code, include this CDDL HEADER in each 141ae08745Sheppo * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 151ae08745Sheppo * If applicable, add the following below this CDDL HEADER, with the 161ae08745Sheppo * fields enclosed by brackets "[]" replaced with your own identifying 171ae08745Sheppo * information: Portions Copyright [yyyy] [name of copyright owner] 181ae08745Sheppo * 191ae08745Sheppo * CDDL HEADER END 201ae08745Sheppo */ 211ae08745Sheppo 221ae08745Sheppo /* 231ae08745Sheppo * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 241ae08745Sheppo * Use is subject to license terms. 251ae08745Sheppo */ 261ae08745Sheppo 271ae08745Sheppo #pragma ident "%Z%%M% %I% %E% SMI" 281ae08745Sheppo 291ae08745Sheppo /* 301ae08745Sheppo * sun4v LDC Transport Layer 311ae08745Sheppo */ 321ae08745Sheppo #include <sys/types.h> 331ae08745Sheppo #include <sys/file.h> 341ae08745Sheppo #include <sys/errno.h> 351ae08745Sheppo #include <sys/open.h> 361ae08745Sheppo #include <sys/cred.h> 371ae08745Sheppo #include <sys/kmem.h> 381ae08745Sheppo #include <sys/conf.h> 391ae08745Sheppo #include <sys/cmn_err.h> 401ae08745Sheppo #include <sys/ksynch.h> 411ae08745Sheppo #include <sys/modctl.h> 421ae08745Sheppo #include <sys/stat.h> /* needed for S_IFBLK and S_IFCHR */ 431ae08745Sheppo #include <sys/debug.h> 441ae08745Sheppo #include <sys/types.h> 451ae08745Sheppo #include <sys/cred.h> 461ae08745Sheppo #include <sys/promif.h> 471ae08745Sheppo #include <sys/ddi.h> 481ae08745Sheppo #include <sys/sunddi.h> 491ae08745Sheppo #include <sys/cyclic.h> 501ae08745Sheppo #include <sys/machsystm.h> 511ae08745Sheppo #include <sys/vm.h> 521ae08745Sheppo #include <sys/cpu.h> 531ae08745Sheppo #include <sys/intreg.h> 541ae08745Sheppo #include <sys/machcpuvar.h> 551ae08745Sheppo #include <sys/note.h> 561ae08745Sheppo #include <sys/ivintr.h> 571ae08745Sheppo #include <sys/hypervisor_api.h> 581ae08745Sheppo #include <sys/ldc.h> 591ae08745Sheppo #include <sys/ldc_impl.h> 601ae08745Sheppo #include <sys/cnex.h> 611ae08745Sheppo #include <sys/hsvc.h> 621ae08745Sheppo 631ae08745Sheppo /* Core internal functions */ 641ae08745Sheppo static int i_ldc_h2v_error(int h_error); 651ae08745Sheppo static int i_ldc_txq_reconf(ldc_chan_t *ldcp); 661ae08745Sheppo static int i_ldc_rxq_reconf(ldc_chan_t *ldcp); 671ae08745Sheppo static void i_ldc_reset_state(ldc_chan_t *ldcp); 681ae08745Sheppo static void i_ldc_reset(ldc_chan_t *ldcp); 691ae08745Sheppo 701ae08745Sheppo static int i_ldc_get_tx_tail(ldc_chan_t *ldcp, uint64_t *tail); 711ae08745Sheppo static int i_ldc_set_tx_tail(ldc_chan_t *ldcp, uint64_t tail); 721ae08745Sheppo static int i_ldc_set_rx_head(ldc_chan_t *ldcp, uint64_t head); 731ae08745Sheppo static int i_ldc_send_pkt(ldc_chan_t *ldcp, uint8_t pkttype, uint8_t subtype, 741ae08745Sheppo uint8_t ctrlmsg); 751ae08745Sheppo 761ae08745Sheppo /* Interrupt handling functions */ 771ae08745Sheppo static uint_t i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2); 781ae08745Sheppo static uint_t i_ldc_rx_hdlr(caddr_t arg1, caddr_t arg2); 791ae08745Sheppo static void i_ldc_clear_intr(ldc_chan_t *ldcp, cnex_intrtype_t itype); 801ae08745Sheppo 811ae08745Sheppo /* Read method functions */ 821ae08745Sheppo static int i_ldc_read_raw(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep); 831ae08745Sheppo static int i_ldc_read_packet(ldc_chan_t *ldcp, caddr_t target_bufp, 841ae08745Sheppo size_t *sizep); 851ae08745Sheppo static int i_ldc_read_stream(ldc_chan_t *ldcp, caddr_t target_bufp, 861ae08745Sheppo size_t *sizep); 871ae08745Sheppo 881ae08745Sheppo /* Write method functions */ 891ae08745Sheppo static int i_ldc_write_raw(ldc_chan_t *ldcp, caddr_t target_bufp, 901ae08745Sheppo size_t *sizep); 911ae08745Sheppo static int i_ldc_write_packet(ldc_chan_t *ldcp, caddr_t target_bufp, 921ae08745Sheppo size_t *sizep); 931ae08745Sheppo static int i_ldc_write_stream(ldc_chan_t *ldcp, caddr_t target_bufp, 941ae08745Sheppo size_t *sizep); 951ae08745Sheppo 961ae08745Sheppo /* Pkt processing internal functions */ 971ae08745Sheppo static int i_ldc_check_seqid(ldc_chan_t *ldcp, ldc_msg_t *ldcmsg); 981ae08745Sheppo static int i_ldc_ctrlmsg(ldc_chan_t *ldcp, ldc_msg_t *ldcmsg); 991ae08745Sheppo static int i_ldc_process_VER(ldc_chan_t *ldcp, ldc_msg_t *msg); 1001ae08745Sheppo static int i_ldc_process_RTS(ldc_chan_t *ldcp, ldc_msg_t *msg); 1011ae08745Sheppo static int i_ldc_process_RTR(ldc_chan_t *ldcp, ldc_msg_t *msg); 1021ae08745Sheppo static int i_ldc_process_RDX(ldc_chan_t *ldcp, ldc_msg_t *msg); 1031ae08745Sheppo static int i_ldc_process_data_ACK(ldc_chan_t *ldcp, ldc_msg_t *msg); 1041ae08745Sheppo 1051ae08745Sheppo /* Memory synchronization internal functions */ 1061ae08745Sheppo static int i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, 1071ae08745Sheppo uint8_t direction, uint64_t offset, size_t size); 1081ae08745Sheppo static int i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle, 1091ae08745Sheppo uint8_t direction, uint64_t start, uint64_t end); 1101ae08745Sheppo 1111ae08745Sheppo /* LDC Version */ 1121ae08745Sheppo static ldc_ver_t ldc_versions[] = { {1, 0} }; 1131ae08745Sheppo 1141ae08745Sheppo /* number of supported versions */ 1151ae08745Sheppo #define LDC_NUM_VERS (sizeof (ldc_versions) / sizeof (ldc_versions[0])) 1161ae08745Sheppo 1171ae08745Sheppo /* Module State Pointer */ 1181ae08745Sheppo static ldc_soft_state_t *ldcssp; 1191ae08745Sheppo 1201ae08745Sheppo static struct modldrv md = { 1211ae08745Sheppo &mod_miscops, /* This is a misc module */ 1221ae08745Sheppo "sun4v LDC module v%I%", /* Name of the module */ 1231ae08745Sheppo }; 1241ae08745Sheppo 1251ae08745Sheppo static struct modlinkage ml = { 1261ae08745Sheppo MODREV_1, 1271ae08745Sheppo &md, 1281ae08745Sheppo NULL 1291ae08745Sheppo }; 1301ae08745Sheppo 1311ae08745Sheppo static uint64_t ldc_sup_minor; /* Supported minor number */ 1321ae08745Sheppo static hsvc_info_t ldc_hsvc = { 1331ae08745Sheppo HSVC_REV_1, NULL, HSVC_GROUP_LDC, 1, 0, "ldc" 1341ae08745Sheppo }; 1351ae08745Sheppo 1361ae08745Sheppo static uint64_t intr_sup_minor; /* Supported minor number */ 1371ae08745Sheppo static hsvc_info_t intr_hsvc = { 1381ae08745Sheppo HSVC_REV_1, NULL, HSVC_GROUP_INTR, 1, 0, "ldc" 1391ae08745Sheppo }; 1401ae08745Sheppo 1410a55fbb7Slm66018 /* 1420a55fbb7Slm66018 * LDC retry count and delay 1430a55fbb7Slm66018 */ 1440a55fbb7Slm66018 int ldc_max_retries = LDC_MAX_RETRIES; 1450a55fbb7Slm66018 clock_t ldc_delay = LDC_DELAY; 1460a55fbb7Slm66018 1471ae08745Sheppo #ifdef DEBUG 1481ae08745Sheppo 1491ae08745Sheppo /* 1501ae08745Sheppo * Print debug messages 1511ae08745Sheppo * 1521ae08745Sheppo * set ldcdbg to 0x7 for enabling all msgs 1531ae08745Sheppo * 0x4 - Warnings 1541ae08745Sheppo * 0x2 - All debug messages 1551ae08745Sheppo * 0x1 - Minimal debug messages 1561ae08745Sheppo * 1571ae08745Sheppo * set ldcdbgchan to the channel number you want to debug 1581ae08745Sheppo * setting it to -1 prints debug messages for all channels 1591ae08745Sheppo * NOTE: ldcdbgchan has no effect on error messages 1601ae08745Sheppo */ 1611ae08745Sheppo 1621ae08745Sheppo #define DBG_ALL_LDCS -1 1631ae08745Sheppo 1641ae08745Sheppo int ldcdbg = 0x0; 1651ae08745Sheppo int64_t ldcdbgchan = DBG_ALL_LDCS; 1661ae08745Sheppo 1671ae08745Sheppo static void 1681ae08745Sheppo ldcdebug(int64_t id, const char *fmt, ...) 1691ae08745Sheppo { 1701ae08745Sheppo char buf[512]; 1711ae08745Sheppo va_list ap; 1721ae08745Sheppo 1731ae08745Sheppo /* 1741ae08745Sheppo * Do not return if, 1751ae08745Sheppo * caller wants to print it anyway - (id == DBG_ALL_LDCS) 1761ae08745Sheppo * debug channel is set to all LDCs - (ldcdbgchan == DBG_ALL_LDCS) 1771ae08745Sheppo * debug channel = caller specified channel 1781ae08745Sheppo */ 1791ae08745Sheppo if ((id != DBG_ALL_LDCS) && 1801ae08745Sheppo (ldcdbgchan != DBG_ALL_LDCS) && 1811ae08745Sheppo (ldcdbgchan != id)) { 1821ae08745Sheppo return; 1831ae08745Sheppo } 1841ae08745Sheppo 1851ae08745Sheppo va_start(ap, fmt); 1861ae08745Sheppo (void) vsprintf(buf, fmt, ap); 1871ae08745Sheppo va_end(ap); 1881ae08745Sheppo 1891ae08745Sheppo cmn_err(CE_CONT, "?%s\n", buf); 1901ae08745Sheppo } 1911ae08745Sheppo 1921ae08745Sheppo #define D1 \ 1931ae08745Sheppo if (ldcdbg & 0x01) \ 1941ae08745Sheppo ldcdebug 1951ae08745Sheppo 1961ae08745Sheppo #define D2 \ 1971ae08745Sheppo if (ldcdbg & 0x02) \ 1981ae08745Sheppo ldcdebug 1991ae08745Sheppo 2001ae08745Sheppo #define DWARN \ 2011ae08745Sheppo if (ldcdbg & 0x04) \ 2021ae08745Sheppo ldcdebug 2031ae08745Sheppo 2041ae08745Sheppo #define DUMP_PAYLOAD(id, addr) \ 2051ae08745Sheppo { \ 2061ae08745Sheppo char buf[65*3]; \ 2071ae08745Sheppo int i; \ 2081ae08745Sheppo uint8_t *src = (uint8_t *)addr; \ 2091ae08745Sheppo for (i = 0; i < 64; i++, src++) \ 2101ae08745Sheppo (void) sprintf(&buf[i * 3], "|%02x", *src); \ 2111ae08745Sheppo (void) sprintf(&buf[i * 3], "|\n"); \ 2121ae08745Sheppo D2((id), "payload: %s", buf); \ 2131ae08745Sheppo } 2141ae08745Sheppo 2151ae08745Sheppo #define DUMP_LDC_PKT(c, s, addr) \ 2161ae08745Sheppo { \ 2171ae08745Sheppo ldc_msg_t *msg = (ldc_msg_t *)(addr); \ 2181ae08745Sheppo uint32_t mid = ((c)->mode != LDC_MODE_RAW) ? msg->seqid : 0; \ 2191ae08745Sheppo if (msg->type == LDC_DATA) { \ 2201ae08745Sheppo D2((c)->id, "%s: msg%d (/%x/%x/%x/,env[%c%c,sz=%d])", \ 2211ae08745Sheppo (s), mid, msg->type, msg->stype, msg->ctrl, \ 2221ae08745Sheppo (msg->env & LDC_FRAG_START) ? 'B' : ' ', \ 2231ae08745Sheppo (msg->env & LDC_FRAG_STOP) ? 'E' : ' ', \ 2241ae08745Sheppo (msg->env & LDC_LEN_MASK)); \ 2251ae08745Sheppo } else { \ 2261ae08745Sheppo D2((c)->id, "%s: msg%d (/%x/%x/%x/,env=%x)", (s), \ 2271ae08745Sheppo mid, msg->type, msg->stype, msg->ctrl, msg->env); \ 2281ae08745Sheppo } \ 2291ae08745Sheppo } 2301ae08745Sheppo 2311ae08745Sheppo #else 2321ae08745Sheppo 2331ae08745Sheppo #define DBG_ALL_LDCS -1 2341ae08745Sheppo 2351ae08745Sheppo #define D1 2361ae08745Sheppo #define D2 2371ae08745Sheppo #define DWARN 2381ae08745Sheppo 2391ae08745Sheppo #define DUMP_PAYLOAD(id, addr) 2401ae08745Sheppo #define DUMP_LDC_PKT(c, s, addr) 2411ae08745Sheppo 2421ae08745Sheppo #endif 2431ae08745Sheppo 2441ae08745Sheppo #define ZERO_PKT(p) \ 2451ae08745Sheppo bzero((p), sizeof (ldc_msg_t)); 2461ae08745Sheppo 2471ae08745Sheppo #define IDX2COOKIE(idx, pg_szc, pg_shift) \ 2481ae08745Sheppo (((pg_szc) << LDC_COOKIE_PGSZC_SHIFT) | ((idx) << (pg_shift))) 2491ae08745Sheppo 2501ae08745Sheppo 2511ae08745Sheppo int 2521ae08745Sheppo _init(void) 2531ae08745Sheppo { 2541ae08745Sheppo int status; 2551ae08745Sheppo 2561ae08745Sheppo status = hsvc_register(&ldc_hsvc, &ldc_sup_minor); 2571ae08745Sheppo if (status != 0) { 2581ae08745Sheppo cmn_err(CE_WARN, "%s: cannot negotiate hypervisor LDC services" 2591ae08745Sheppo " group: 0x%lx major: %ld minor: %ld errno: %d", 2601ae08745Sheppo ldc_hsvc.hsvc_modname, ldc_hsvc.hsvc_group, 2611ae08745Sheppo ldc_hsvc.hsvc_major, ldc_hsvc.hsvc_minor, status); 2621ae08745Sheppo return (-1); 2631ae08745Sheppo } 2641ae08745Sheppo 2651ae08745Sheppo status = hsvc_register(&intr_hsvc, &intr_sup_minor); 2661ae08745Sheppo if (status != 0) { 2671ae08745Sheppo cmn_err(CE_WARN, "%s: cannot negotiate hypervisor interrupt " 2681ae08745Sheppo "services group: 0x%lx major: %ld minor: %ld errno: %d", 2691ae08745Sheppo intr_hsvc.hsvc_modname, intr_hsvc.hsvc_group, 2701ae08745Sheppo intr_hsvc.hsvc_major, intr_hsvc.hsvc_minor, status); 2711ae08745Sheppo (void) hsvc_unregister(&ldc_hsvc); 2721ae08745Sheppo return (-1); 2731ae08745Sheppo } 2741ae08745Sheppo 2751ae08745Sheppo /* allocate soft state structure */ 2761ae08745Sheppo ldcssp = kmem_zalloc(sizeof (ldc_soft_state_t), KM_SLEEP); 2771ae08745Sheppo 2781ae08745Sheppo /* Link the module into the system */ 2791ae08745Sheppo status = mod_install(&ml); 2801ae08745Sheppo if (status != 0) { 2811ae08745Sheppo kmem_free(ldcssp, sizeof (ldc_soft_state_t)); 2821ae08745Sheppo return (status); 2831ae08745Sheppo } 2841ae08745Sheppo 2851ae08745Sheppo /* Initialize the LDC state structure */ 2861ae08745Sheppo mutex_init(&ldcssp->lock, NULL, MUTEX_DRIVER, NULL); 2871ae08745Sheppo 2881ae08745Sheppo mutex_enter(&ldcssp->lock); 2891ae08745Sheppo 2901ae08745Sheppo ldcssp->channel_count = 0; 2911ae08745Sheppo ldcssp->channels_open = 0; 2921ae08745Sheppo ldcssp->chan_list = NULL; 2931ae08745Sheppo ldcssp->dring_list = NULL; 2941ae08745Sheppo 2951ae08745Sheppo mutex_exit(&ldcssp->lock); 2961ae08745Sheppo 2971ae08745Sheppo return (0); 2981ae08745Sheppo } 2991ae08745Sheppo 3001ae08745Sheppo int 3011ae08745Sheppo _info(struct modinfo *modinfop) 3021ae08745Sheppo { 3031ae08745Sheppo /* Report status of the dynamically loadable driver module */ 3041ae08745Sheppo return (mod_info(&ml, modinfop)); 3051ae08745Sheppo } 3061ae08745Sheppo 3071ae08745Sheppo int 3081ae08745Sheppo _fini(void) 3091ae08745Sheppo { 3101ae08745Sheppo int rv, status; 3111ae08745Sheppo ldc_chan_t *ldcp; 3121ae08745Sheppo ldc_dring_t *dringp; 3131ae08745Sheppo ldc_mem_info_t minfo; 3141ae08745Sheppo 3151ae08745Sheppo /* Unlink the driver module from the system */ 3161ae08745Sheppo status = mod_remove(&ml); 3171ae08745Sheppo if (status) { 3181ae08745Sheppo DWARN(DBG_ALL_LDCS, "_fini: mod_remove failed\n"); 3191ae08745Sheppo return (EIO); 3201ae08745Sheppo } 3211ae08745Sheppo 3221ae08745Sheppo /* close and finalize channels */ 3231ae08745Sheppo ldcp = ldcssp->chan_list; 3241ae08745Sheppo while (ldcp != NULL) { 3251ae08745Sheppo (void) ldc_close((ldc_handle_t)ldcp); 3261ae08745Sheppo (void) ldc_fini((ldc_handle_t)ldcp); 3271ae08745Sheppo 3281ae08745Sheppo ldcp = ldcp->next; 3291ae08745Sheppo } 3301ae08745Sheppo 3311ae08745Sheppo /* Free descriptor rings */ 3321ae08745Sheppo dringp = ldcssp->dring_list; 3331ae08745Sheppo while (dringp != NULL) { 3341ae08745Sheppo dringp = dringp->next; 3351ae08745Sheppo 3361ae08745Sheppo rv = ldc_mem_dring_info((ldc_dring_handle_t)dringp, &minfo); 3371ae08745Sheppo if (rv == 0 && minfo.status != LDC_UNBOUND) { 3381ae08745Sheppo if (minfo.status == LDC_BOUND) { 3391ae08745Sheppo (void) ldc_mem_dring_unbind( 3401ae08745Sheppo (ldc_dring_handle_t)dringp); 3411ae08745Sheppo } 3421ae08745Sheppo if (minfo.status == LDC_MAPPED) { 3431ae08745Sheppo (void) ldc_mem_dring_unmap( 3441ae08745Sheppo (ldc_dring_handle_t)dringp); 3451ae08745Sheppo } 3461ae08745Sheppo } 3471ae08745Sheppo 3481ae08745Sheppo (void) ldc_mem_dring_destroy((ldc_dring_handle_t)dringp); 3491ae08745Sheppo } 3501ae08745Sheppo ldcssp->dring_list = NULL; 3511ae08745Sheppo 3521ae08745Sheppo /* 3531ae08745Sheppo * We have successfully "removed" the driver. 3541ae08745Sheppo * Destroying soft states 3551ae08745Sheppo */ 3561ae08745Sheppo mutex_destroy(&ldcssp->lock); 3571ae08745Sheppo kmem_free(ldcssp, sizeof (ldc_soft_state_t)); 3581ae08745Sheppo 3591ae08745Sheppo (void) hsvc_unregister(&ldc_hsvc); 3601ae08745Sheppo (void) hsvc_unregister(&intr_hsvc); 3611ae08745Sheppo 3621ae08745Sheppo return (status); 3631ae08745Sheppo } 3641ae08745Sheppo 3651ae08745Sheppo /* -------------------------------------------------------------------------- */ 3661ae08745Sheppo 3671ae08745Sheppo /* 3681ae08745Sheppo * LDC Transport Internal Functions 3691ae08745Sheppo */ 3701ae08745Sheppo 3711ae08745Sheppo /* 3721ae08745Sheppo * Translate HV Errors to sun4v error codes 3731ae08745Sheppo */ 3741ae08745Sheppo static int 3751ae08745Sheppo i_ldc_h2v_error(int h_error) 3761ae08745Sheppo { 3771ae08745Sheppo switch (h_error) { 3781ae08745Sheppo 3791ae08745Sheppo case H_EOK: 3801ae08745Sheppo return (0); 3811ae08745Sheppo 3821ae08745Sheppo case H_ENORADDR: 3831ae08745Sheppo return (EFAULT); 3841ae08745Sheppo 3851ae08745Sheppo case H_EBADPGSZ: 3861ae08745Sheppo case H_EINVAL: 3871ae08745Sheppo return (EINVAL); 3881ae08745Sheppo 3891ae08745Sheppo case H_EWOULDBLOCK: 3901ae08745Sheppo return (EWOULDBLOCK); 3911ae08745Sheppo 3921ae08745Sheppo case H_ENOACCESS: 3931ae08745Sheppo case H_ENOMAP: 3941ae08745Sheppo return (EACCES); 3951ae08745Sheppo 3961ae08745Sheppo case H_EIO: 3971ae08745Sheppo case H_ECPUERROR: 3981ae08745Sheppo return (EIO); 3991ae08745Sheppo 4001ae08745Sheppo case H_ENOTSUPPORTED: 4011ae08745Sheppo return (ENOTSUP); 4021ae08745Sheppo 4031ae08745Sheppo case H_ETOOMANY: 4041ae08745Sheppo return (ENOSPC); 4051ae08745Sheppo 4061ae08745Sheppo case H_ECHANNEL: 4071ae08745Sheppo return (ECHRNG); 4081ae08745Sheppo default: 4091ae08745Sheppo break; 4101ae08745Sheppo } 4111ae08745Sheppo 4121ae08745Sheppo return (EIO); 4131ae08745Sheppo } 4141ae08745Sheppo 4151ae08745Sheppo /* 4161ae08745Sheppo * Reconfigure the transmit queue 4171ae08745Sheppo */ 4181ae08745Sheppo static int 4191ae08745Sheppo i_ldc_txq_reconf(ldc_chan_t *ldcp) 4201ae08745Sheppo { 4211ae08745Sheppo int rv; 4221ae08745Sheppo 4231ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 424*d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 425*d10e4ef2Snarayan 4261ae08745Sheppo rv = hv_ldc_tx_qconf(ldcp->id, ldcp->tx_q_ra, ldcp->tx_q_entries); 4271ae08745Sheppo if (rv) { 4281ae08745Sheppo cmn_err(CE_WARN, 4291ae08745Sheppo "ldc_tx_qconf: (0x%lx) cannot set qconf", ldcp->id); 4301ae08745Sheppo return (EIO); 4311ae08745Sheppo } 4321ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, &(ldcp->tx_head), 4331ae08745Sheppo &(ldcp->tx_tail), &(ldcp->link_state)); 4341ae08745Sheppo if (rv) { 4351ae08745Sheppo cmn_err(CE_WARN, 4361ae08745Sheppo "ldc_tx_get_state: (0x%lx) cannot get qptrs", ldcp->id); 4371ae08745Sheppo return (EIO); 4381ae08745Sheppo } 4391ae08745Sheppo D1(ldcp->id, "ldc_tx_get_state: (0x%llx) h=0x%llx,t=0x%llx," 4401ae08745Sheppo "s=0x%llx\n", ldcp->id, ldcp->tx_head, ldcp->tx_tail, 4411ae08745Sheppo ldcp->link_state); 4421ae08745Sheppo 4431ae08745Sheppo return (0); 4441ae08745Sheppo } 4451ae08745Sheppo 4461ae08745Sheppo /* 4471ae08745Sheppo * Reconfigure the receive queue 4481ae08745Sheppo */ 4491ae08745Sheppo static int 4501ae08745Sheppo i_ldc_rxq_reconf(ldc_chan_t *ldcp) 4511ae08745Sheppo { 4521ae08745Sheppo int rv; 4531ae08745Sheppo uint64_t rx_head, rx_tail; 4541ae08745Sheppo 4551ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 4561ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 4571ae08745Sheppo &(ldcp->link_state)); 4581ae08745Sheppo if (rv) { 4591ae08745Sheppo cmn_err(CE_WARN, 4601ae08745Sheppo "ldc_rx_getstate: (0x%lx) cannot get state", 4611ae08745Sheppo ldcp->id); 4621ae08745Sheppo return (EIO); 4631ae08745Sheppo } 4641ae08745Sheppo 4651ae08745Sheppo if (rx_head != rx_tail || ldcp->tstate > TS_READY) { 4661ae08745Sheppo rv = hv_ldc_rx_qconf(ldcp->id, ldcp->rx_q_ra, 4671ae08745Sheppo ldcp->rx_q_entries); 4681ae08745Sheppo if (rv) { 4691ae08745Sheppo cmn_err(CE_WARN, 4701ae08745Sheppo "ldc_rx_qconf: (0x%lx) cannot set qconf", 4711ae08745Sheppo ldcp->id); 4721ae08745Sheppo return (EIO); 4731ae08745Sheppo } 4741ae08745Sheppo D1(ldcp->id, "ldc_rx_qconf: (0x%llx) completed qconf", 4751ae08745Sheppo ldcp->id); 4761ae08745Sheppo } 4771ae08745Sheppo 4781ae08745Sheppo return (0); 4791ae08745Sheppo } 4801ae08745Sheppo 4811ae08745Sheppo /* 4821ae08745Sheppo * Reset LDC state structure and its contents 4831ae08745Sheppo */ 4841ae08745Sheppo static void 4851ae08745Sheppo i_ldc_reset_state(ldc_chan_t *ldcp) 4861ae08745Sheppo { 4871ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 4881ae08745Sheppo ldcp->last_msg_snt = LDC_INIT_SEQID; 4891ae08745Sheppo ldcp->last_ack_rcd = 0; 4901ae08745Sheppo ldcp->last_msg_rcd = 0; 4911ae08745Sheppo ldcp->tx_ackd_head = ldcp->tx_head; 4921ae08745Sheppo ldcp->next_vidx = 0; 4931ae08745Sheppo ldcp->hstate = 0; 4941ae08745Sheppo ldcp->tstate = TS_OPEN; 4951ae08745Sheppo ldcp->status = LDC_OPEN; 4961ae08745Sheppo 4971ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_UP || 4981ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 4991ae08745Sheppo 5001ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW) { 5011ae08745Sheppo ldcp->status = LDC_UP; 5021ae08745Sheppo ldcp->tstate = TS_UP; 5031ae08745Sheppo } else { 5041ae08745Sheppo ldcp->status = LDC_READY; 5051ae08745Sheppo ldcp->tstate |= TS_LINK_READY; 5061ae08745Sheppo } 5071ae08745Sheppo } 5081ae08745Sheppo } 5091ae08745Sheppo 5101ae08745Sheppo /* 5111ae08745Sheppo * Reset a LDC channel 5121ae08745Sheppo */ 5131ae08745Sheppo static void 5141ae08745Sheppo i_ldc_reset(ldc_chan_t *ldcp) 5151ae08745Sheppo { 5161ae08745Sheppo D2(ldcp->id, "i_ldc_reset: (0x%llx) channel reset\n", ldcp->id); 5171ae08745Sheppo 518*d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->lock)); 519*d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 520*d10e4ef2Snarayan 5211ae08745Sheppo (void) i_ldc_txq_reconf(ldcp); 5221ae08745Sheppo (void) i_ldc_rxq_reconf(ldcp); 5231ae08745Sheppo i_ldc_reset_state(ldcp); 5241ae08745Sheppo } 5251ae08745Sheppo 5261ae08745Sheppo /* 5271ae08745Sheppo * Clear pending interrupts 5281ae08745Sheppo */ 5291ae08745Sheppo static void 5301ae08745Sheppo i_ldc_clear_intr(ldc_chan_t *ldcp, cnex_intrtype_t itype) 5311ae08745Sheppo { 5321ae08745Sheppo ldc_cnex_t *cinfo = &ldcssp->cinfo; 5331ae08745Sheppo 5341ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 5351ae08745Sheppo if (cinfo->dip && ldcp->intr_pending) { 5361ae08745Sheppo ldcp->intr_pending = B_FALSE; 5371ae08745Sheppo (void) cinfo->clr_intr(cinfo->dip, ldcp->id, itype); 5381ae08745Sheppo } 5391ae08745Sheppo } 5401ae08745Sheppo 5411ae08745Sheppo /* 5421ae08745Sheppo * Set the receive queue head 5430a55fbb7Slm66018 * Resets connection and returns an error if it fails. 5441ae08745Sheppo */ 5451ae08745Sheppo static int 5461ae08745Sheppo i_ldc_set_rx_head(ldc_chan_t *ldcp, uint64_t head) 5471ae08745Sheppo { 5481ae08745Sheppo int rv; 5490a55fbb7Slm66018 int retries; 5501ae08745Sheppo 5511ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 5520a55fbb7Slm66018 for (retries = 0; retries < ldc_max_retries; retries++) { 5530a55fbb7Slm66018 5540a55fbb7Slm66018 if ((rv = hv_ldc_rx_set_qhead(ldcp->id, head)) == 0) 5550a55fbb7Slm66018 return (0); 5560a55fbb7Slm66018 5570a55fbb7Slm66018 if (rv != H_EWOULDBLOCK) 5580a55fbb7Slm66018 break; 5590a55fbb7Slm66018 5600a55fbb7Slm66018 /* wait for ldc_delay usecs */ 5610a55fbb7Slm66018 drv_usecwait(ldc_delay); 5621ae08745Sheppo } 5631ae08745Sheppo 5640a55fbb7Slm66018 cmn_err(CE_WARN, "ldc_rx_set_qhead: (0x%lx) cannot set qhead 0x%lx", 5650a55fbb7Slm66018 ldcp->id, head); 566*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 5670a55fbb7Slm66018 i_ldc_reset(ldcp); 568*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 5690a55fbb7Slm66018 5700a55fbb7Slm66018 return (ECONNRESET); 5711ae08745Sheppo } 5721ae08745Sheppo 5731ae08745Sheppo 5741ae08745Sheppo /* 5751ae08745Sheppo * Returns the tx_tail to be used for transfer 5761ae08745Sheppo * Re-reads the TX queue ptrs if and only if the 5771ae08745Sheppo * the cached head and tail are equal (queue is full) 5781ae08745Sheppo */ 5791ae08745Sheppo static int 5801ae08745Sheppo i_ldc_get_tx_tail(ldc_chan_t *ldcp, uint64_t *tail) 5811ae08745Sheppo { 5821ae08745Sheppo int rv; 5831ae08745Sheppo uint64_t current_head, new_tail; 5841ae08745Sheppo 585*d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 5861ae08745Sheppo /* Read the head and tail ptrs from HV */ 5871ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 5881ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 5891ae08745Sheppo if (rv) { 5901ae08745Sheppo cmn_err(CE_WARN, 5911ae08745Sheppo "i_ldc_get_tx_tail: (0x%lx) cannot read qptrs\n", 5921ae08745Sheppo ldcp->id); 5931ae08745Sheppo return (EIO); 5941ae08745Sheppo } 5951ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN) { 5961ae08745Sheppo DWARN(DBG_ALL_LDCS, 5971ae08745Sheppo "i_ldc_get_tx_tail: (0x%llx) channel not ready\n", 5981ae08745Sheppo ldcp->id); 5991ae08745Sheppo return (ECONNRESET); 6001ae08745Sheppo } 6011ae08745Sheppo 6021ae08745Sheppo /* In reliable mode, check against last ACKd msg */ 6031ae08745Sheppo current_head = (ldcp->mode == LDC_MODE_RELIABLE || 6041ae08745Sheppo ldcp->mode == LDC_MODE_STREAM) 6051ae08745Sheppo ? ldcp->tx_ackd_head : ldcp->tx_head; 6061ae08745Sheppo 6071ae08745Sheppo /* increment the tail */ 6081ae08745Sheppo new_tail = (ldcp->tx_tail + LDC_PACKET_SIZE) % 6091ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 6101ae08745Sheppo 6111ae08745Sheppo if (new_tail == current_head) { 6121ae08745Sheppo DWARN(ldcp->id, 6131ae08745Sheppo "i_ldc_get_tx_tail: (0x%llx) TX queue is full\n", 6141ae08745Sheppo ldcp->id); 6151ae08745Sheppo return (EWOULDBLOCK); 6161ae08745Sheppo } 6171ae08745Sheppo 6181ae08745Sheppo D2(ldcp->id, "i_ldc_get_tx_tail: (0x%llx) head=0x%llx, tail=0x%llx\n", 6191ae08745Sheppo ldcp->id, ldcp->tx_head, ldcp->tx_tail); 6201ae08745Sheppo 6211ae08745Sheppo *tail = ldcp->tx_tail; 6221ae08745Sheppo return (0); 6231ae08745Sheppo } 6241ae08745Sheppo 6251ae08745Sheppo /* 6261ae08745Sheppo * Set the tail pointer. If HV returns EWOULDBLOCK, it will back off 6270a55fbb7Slm66018 * and retry ldc_max_retries times before returning an error. 6281ae08745Sheppo * Returns 0, EWOULDBLOCK or EIO 6291ae08745Sheppo */ 6301ae08745Sheppo static int 6311ae08745Sheppo i_ldc_set_tx_tail(ldc_chan_t *ldcp, uint64_t tail) 6321ae08745Sheppo { 6331ae08745Sheppo int rv, retval = EWOULDBLOCK; 6340a55fbb7Slm66018 int retries; 6351ae08745Sheppo 636*d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 6370a55fbb7Slm66018 for (retries = 0; retries < ldc_max_retries; retries++) { 6381ae08745Sheppo 6391ae08745Sheppo if ((rv = hv_ldc_tx_set_qtail(ldcp->id, tail)) == 0) { 6401ae08745Sheppo retval = 0; 6411ae08745Sheppo break; 6421ae08745Sheppo } 6431ae08745Sheppo if (rv != H_EWOULDBLOCK) { 6441ae08745Sheppo DWARN(ldcp->id, "i_ldc_set_tx_tail: (0x%llx) set " 6451ae08745Sheppo "qtail=0x%llx failed, rv=%d\n", ldcp->id, tail, rv); 6461ae08745Sheppo retval = EIO; 6471ae08745Sheppo break; 6481ae08745Sheppo } 6491ae08745Sheppo 6500a55fbb7Slm66018 /* wait for ldc_delay usecs */ 6510a55fbb7Slm66018 drv_usecwait(ldc_delay); 6521ae08745Sheppo } 6531ae08745Sheppo return (retval); 6541ae08745Sheppo } 6551ae08745Sheppo 6561ae08745Sheppo /* 6571ae08745Sheppo * Send a LDC message 6581ae08745Sheppo */ 6591ae08745Sheppo static int 6601ae08745Sheppo i_ldc_send_pkt(ldc_chan_t *ldcp, uint8_t pkttype, uint8_t subtype, 6611ae08745Sheppo uint8_t ctrlmsg) 6621ae08745Sheppo { 6631ae08745Sheppo int rv; 6641ae08745Sheppo ldc_msg_t *pkt; 6651ae08745Sheppo uint64_t tx_tail; 6661ae08745Sheppo uint32_t curr_seqid = ldcp->last_msg_snt; 6671ae08745Sheppo 668*d10e4ef2Snarayan /* Obtain Tx lock */ 669*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 670*d10e4ef2Snarayan 6711ae08745Sheppo /* get the current tail for the message */ 6721ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 6731ae08745Sheppo if (rv) { 6741ae08745Sheppo DWARN(ldcp->id, 6751ae08745Sheppo "i_ldc_send_pkt: (0x%llx) error sending pkt, " 6761ae08745Sheppo "type=0x%x,subtype=0x%x,ctrl=0x%x\n", 6771ae08745Sheppo ldcp->id, pkttype, subtype, ctrlmsg); 678*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 6791ae08745Sheppo return (rv); 6801ae08745Sheppo } 6811ae08745Sheppo 6821ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 6831ae08745Sheppo ZERO_PKT(pkt); 6841ae08745Sheppo 6851ae08745Sheppo /* Initialize the packet */ 6861ae08745Sheppo pkt->type = pkttype; 6871ae08745Sheppo pkt->stype = subtype; 6881ae08745Sheppo pkt->ctrl = ctrlmsg; 6891ae08745Sheppo 6901ae08745Sheppo /* Store ackid/seqid iff it is RELIABLE mode & not a RTS/RTR message */ 6911ae08745Sheppo if (((ctrlmsg & LDC_CTRL_MASK) != LDC_RTS) && 6921ae08745Sheppo ((ctrlmsg & LDC_CTRL_MASK) != LDC_RTR)) { 6931ae08745Sheppo curr_seqid++; 6941ae08745Sheppo if (ldcp->mode != LDC_MODE_RAW) { 6951ae08745Sheppo pkt->seqid = curr_seqid; 6961ae08745Sheppo pkt->ackid = ldcp->last_msg_rcd; 6971ae08745Sheppo } 6981ae08745Sheppo } 6991ae08745Sheppo DUMP_LDC_PKT(ldcp, "i_ldc_send_pkt", (uint64_t)pkt); 7001ae08745Sheppo 7011ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 7021ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 7031ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 7041ae08745Sheppo 7051ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 7061ae08745Sheppo if (rv) { 7071ae08745Sheppo DWARN(ldcp->id, 7081ae08745Sheppo "i_ldc_send_pkt:(0x%llx) error sending pkt, " 7091ae08745Sheppo "type=0x%x,stype=0x%x,ctrl=0x%x\n", 7101ae08745Sheppo ldcp->id, pkttype, subtype, ctrlmsg); 711*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 7121ae08745Sheppo return (EIO); 7131ae08745Sheppo } 7141ae08745Sheppo 7151ae08745Sheppo ldcp->last_msg_snt = curr_seqid; 7161ae08745Sheppo ldcp->tx_tail = tx_tail; 7171ae08745Sheppo 718*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 7191ae08745Sheppo return (0); 7201ae08745Sheppo } 7211ae08745Sheppo 7221ae08745Sheppo /* 7231ae08745Sheppo * Checks if packet was received in right order 7241ae08745Sheppo * in the case of a reliable transport. 7251ae08745Sheppo * Returns 0 if in order, else EIO 7261ae08745Sheppo */ 7271ae08745Sheppo static int 7281ae08745Sheppo i_ldc_check_seqid(ldc_chan_t *ldcp, ldc_msg_t *msg) 7291ae08745Sheppo { 7301ae08745Sheppo /* No seqid checking for RAW mode */ 7311ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW) 7321ae08745Sheppo return (0); 7331ae08745Sheppo 7341ae08745Sheppo /* No seqid checking for version, RTS, RTR message */ 7351ae08745Sheppo if (msg->ctrl == LDC_VER || 7361ae08745Sheppo msg->ctrl == LDC_RTS || 7371ae08745Sheppo msg->ctrl == LDC_RTR) 7381ae08745Sheppo return (0); 7391ae08745Sheppo 7401ae08745Sheppo /* Initial seqid to use is sent in RTS/RTR and saved in last_msg_rcd */ 7411ae08745Sheppo if (msg->seqid != (ldcp->last_msg_rcd + 1)) { 7421ae08745Sheppo DWARN(ldcp->id, 7431ae08745Sheppo "i_ldc_check_seqid: (0x%llx) out-of-order pkt, got 0x%x, " 7441ae08745Sheppo "expecting 0x%x\n", ldcp->id, msg->seqid, 7451ae08745Sheppo (ldcp->last_msg_rcd + 1)); 7461ae08745Sheppo return (EIO); 7471ae08745Sheppo } 7481ae08745Sheppo 7491ae08745Sheppo return (0); 7501ae08745Sheppo } 7511ae08745Sheppo 7521ae08745Sheppo 7531ae08745Sheppo /* 7541ae08745Sheppo * Process an incoming version ctrl message 7551ae08745Sheppo */ 7561ae08745Sheppo static int 7571ae08745Sheppo i_ldc_process_VER(ldc_chan_t *ldcp, ldc_msg_t *msg) 7581ae08745Sheppo { 7591ae08745Sheppo int rv = 0, idx = ldcp->next_vidx; 7601ae08745Sheppo ldc_msg_t *pkt; 7611ae08745Sheppo uint64_t tx_tail; 7621ae08745Sheppo ldc_ver_t *rcvd_ver; 7631ae08745Sheppo 7641ae08745Sheppo /* get the received version */ 7651ae08745Sheppo rcvd_ver = (ldc_ver_t *)((uint64_t)msg + LDC_PAYLOAD_VER_OFF); 7661ae08745Sheppo 7671ae08745Sheppo D2(ldcp->id, "i_ldc_process_VER: (0x%llx) received VER v%u.%u\n", 7681ae08745Sheppo ldcp->id, rcvd_ver->major, rcvd_ver->minor); 7691ae08745Sheppo 770*d10e4ef2Snarayan /* Obtain Tx lock */ 771*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 772*d10e4ef2Snarayan 7731ae08745Sheppo switch (msg->stype) { 7741ae08745Sheppo case LDC_INFO: 7751ae08745Sheppo 7761ae08745Sheppo /* get the current tail and pkt for the response */ 7771ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 7781ae08745Sheppo if (rv != 0) { 7791ae08745Sheppo DWARN(ldcp->id, 7801ae08745Sheppo "i_ldc_process_VER: (0x%llx) err sending " 7811ae08745Sheppo "version ACK/NACK\n", ldcp->id); 7821ae08745Sheppo i_ldc_reset(ldcp); 783*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 7841ae08745Sheppo return (ECONNRESET); 7851ae08745Sheppo } 7861ae08745Sheppo 7871ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 7881ae08745Sheppo ZERO_PKT(pkt); 7891ae08745Sheppo 7901ae08745Sheppo /* initialize the packet */ 7911ae08745Sheppo pkt->type = LDC_CTRL; 7921ae08745Sheppo pkt->ctrl = LDC_VER; 7931ae08745Sheppo 7941ae08745Sheppo for (;;) { 7951ae08745Sheppo 7961ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: got %u.%u chk %u.%u\n", 7971ae08745Sheppo rcvd_ver->major, rcvd_ver->minor, 7981ae08745Sheppo ldc_versions[idx].major, ldc_versions[idx].minor); 7991ae08745Sheppo 8001ae08745Sheppo if (rcvd_ver->major == ldc_versions[idx].major) { 8011ae08745Sheppo /* major version match - ACK version */ 8021ae08745Sheppo pkt->stype = LDC_ACK; 8031ae08745Sheppo 8041ae08745Sheppo /* 8051ae08745Sheppo * lower minor version to the one this endpt 8061ae08745Sheppo * supports, if necessary 8071ae08745Sheppo */ 8081ae08745Sheppo if (rcvd_ver->minor > ldc_versions[idx].minor) 8091ae08745Sheppo rcvd_ver->minor = 8101ae08745Sheppo ldc_versions[idx].minor; 8111ae08745Sheppo bcopy(rcvd_ver, pkt->udata, sizeof (*rcvd_ver)); 8121ae08745Sheppo 8131ae08745Sheppo break; 8141ae08745Sheppo } 8151ae08745Sheppo 8161ae08745Sheppo if (rcvd_ver->major > ldc_versions[idx].major) { 8171ae08745Sheppo 8181ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: using next" 8191ae08745Sheppo " lower idx=%d, v%u.%u\n", idx, 8201ae08745Sheppo ldc_versions[idx].major, 8211ae08745Sheppo ldc_versions[idx].minor); 8221ae08745Sheppo 8231ae08745Sheppo /* nack with next lower version */ 8241ae08745Sheppo pkt->stype = LDC_NACK; 8251ae08745Sheppo bcopy(&ldc_versions[idx], pkt->udata, 8261ae08745Sheppo sizeof (ldc_versions[idx])); 8271ae08745Sheppo ldcp->next_vidx = idx; 8281ae08745Sheppo break; 8291ae08745Sheppo } 8301ae08745Sheppo 8311ae08745Sheppo /* next major version */ 8321ae08745Sheppo idx++; 8331ae08745Sheppo 8341ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: inc idx %x\n", idx); 8351ae08745Sheppo 8361ae08745Sheppo if (idx == LDC_NUM_VERS) { 8371ae08745Sheppo /* no version match - send NACK */ 8381ae08745Sheppo pkt->stype = LDC_NACK; 8391ae08745Sheppo bzero(pkt->udata, sizeof (ldc_ver_t)); 8401ae08745Sheppo ldcp->next_vidx = 0; 8411ae08745Sheppo break; 8421ae08745Sheppo } 8431ae08745Sheppo } 8441ae08745Sheppo 8451ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 8461ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 8471ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 8481ae08745Sheppo 8491ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 8501ae08745Sheppo if (rv == 0) { 8511ae08745Sheppo ldcp->tx_tail = tx_tail; 8521ae08745Sheppo if (pkt->stype == LDC_ACK) { 8531ae08745Sheppo D2(ldcp->id, "i_ldc_process_VER: (0x%llx) sent" 8541ae08745Sheppo " version ACK\n", ldcp->id); 8551ae08745Sheppo /* Save the ACK'd version */ 8561ae08745Sheppo ldcp->version.major = rcvd_ver->major; 8571ae08745Sheppo ldcp->version.minor = rcvd_ver->minor; 8580a55fbb7Slm66018 ldcp->hstate |= TS_RCVD_VER; 8591ae08745Sheppo ldcp->tstate |= TS_VER_DONE; 8601ae08745Sheppo DWARN(DBG_ALL_LDCS, 8611ae08745Sheppo "(0x%llx) Agreed on version v%u.%u\n", 8621ae08745Sheppo ldcp->id, rcvd_ver->major, rcvd_ver->minor); 8631ae08745Sheppo } 8641ae08745Sheppo } else { 8651ae08745Sheppo DWARN(ldcp->id, 8661ae08745Sheppo "i_ldc_process_VER: (0x%llx) error sending " 8671ae08745Sheppo "ACK/NACK\n", ldcp->id); 8681ae08745Sheppo i_ldc_reset(ldcp); 869*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 8701ae08745Sheppo return (ECONNRESET); 8711ae08745Sheppo } 8721ae08745Sheppo 8731ae08745Sheppo break; 8741ae08745Sheppo 8751ae08745Sheppo case LDC_ACK: 8761ae08745Sheppo /* SUCCESS - we have agreed on a version */ 8771ae08745Sheppo ldcp->version.major = rcvd_ver->major; 8781ae08745Sheppo ldcp->version.minor = rcvd_ver->minor; 8791ae08745Sheppo ldcp->tstate |= TS_VER_DONE; 8801ae08745Sheppo 8811ae08745Sheppo D1(DBG_ALL_LDCS, "(0x%llx) Agreed on version v%u.%u\n", 8821ae08745Sheppo ldcp->id, rcvd_ver->major, rcvd_ver->minor); 8831ae08745Sheppo 8841ae08745Sheppo /* initiate RTS-RTR-RDX handshake */ 8851ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 8861ae08745Sheppo if (rv) { 8871ae08745Sheppo DWARN(ldcp->id, 8881ae08745Sheppo "i_ldc_process_VER: (0x%llx) cannot send RTS\n", 8891ae08745Sheppo ldcp->id); 8901ae08745Sheppo i_ldc_reset(ldcp); 891*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 8921ae08745Sheppo return (ECONNRESET); 8931ae08745Sheppo } 8941ae08745Sheppo 8951ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 8961ae08745Sheppo ZERO_PKT(pkt); 8971ae08745Sheppo 8981ae08745Sheppo pkt->type = LDC_CTRL; 8991ae08745Sheppo pkt->stype = LDC_INFO; 9001ae08745Sheppo pkt->ctrl = LDC_RTS; 9011ae08745Sheppo pkt->env = ldcp->mode; 9021ae08745Sheppo if (ldcp->mode != LDC_MODE_RAW) 9031ae08745Sheppo pkt->seqid = LDC_INIT_SEQID; 9041ae08745Sheppo 9051ae08745Sheppo ldcp->last_msg_rcd = LDC_INIT_SEQID; 9061ae08745Sheppo 9071ae08745Sheppo DUMP_LDC_PKT(ldcp, "i_ldc_process_VER snd rts", (uint64_t)pkt); 9081ae08745Sheppo 9091ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 9101ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 9111ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 9121ae08745Sheppo 9131ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 9141ae08745Sheppo if (rv) { 9151ae08745Sheppo D2(ldcp->id, 9161ae08745Sheppo "i_ldc_process_VER: (0x%llx) no listener\n", 9171ae08745Sheppo ldcp->id); 9181ae08745Sheppo i_ldc_reset(ldcp); 919*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 9201ae08745Sheppo return (ECONNRESET); 9211ae08745Sheppo } 9221ae08745Sheppo 9231ae08745Sheppo ldcp->tx_tail = tx_tail; 9241ae08745Sheppo ldcp->hstate |= TS_SENT_RTS; 9251ae08745Sheppo 9261ae08745Sheppo break; 9271ae08745Sheppo 9281ae08745Sheppo case LDC_NACK: 9291ae08745Sheppo /* check if version in NACK is zero */ 9301ae08745Sheppo if (rcvd_ver->major == 0 && rcvd_ver->minor == 0) { 9311ae08745Sheppo /* version handshake failure */ 9321ae08745Sheppo DWARN(DBG_ALL_LDCS, 9331ae08745Sheppo "i_ldc_process_VER: (0x%llx) no version match\n", 9341ae08745Sheppo ldcp->id); 9351ae08745Sheppo i_ldc_reset(ldcp); 936*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 9371ae08745Sheppo return (ECONNRESET); 9381ae08745Sheppo } 9391ae08745Sheppo 9401ae08745Sheppo /* get the current tail and pkt for the response */ 9411ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 9421ae08745Sheppo if (rv != 0) { 9431ae08745Sheppo cmn_err(CE_NOTE, 9441ae08745Sheppo "i_ldc_process_VER: (0x%lx) err sending " 9451ae08745Sheppo "version ACK/NACK\n", ldcp->id); 9461ae08745Sheppo i_ldc_reset(ldcp); 947*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 9481ae08745Sheppo return (ECONNRESET); 9491ae08745Sheppo } 9501ae08745Sheppo 9511ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 9521ae08745Sheppo ZERO_PKT(pkt); 9531ae08745Sheppo 9541ae08745Sheppo /* initialize the packet */ 9551ae08745Sheppo pkt->type = LDC_CTRL; 9561ae08745Sheppo pkt->ctrl = LDC_VER; 9571ae08745Sheppo pkt->stype = LDC_INFO; 9581ae08745Sheppo 9591ae08745Sheppo /* check ver in NACK msg has a match */ 9601ae08745Sheppo for (;;) { 9611ae08745Sheppo if (rcvd_ver->major == ldc_versions[idx].major) { 9621ae08745Sheppo /* 9631ae08745Sheppo * major version match - resubmit request 9641ae08745Sheppo * if lower minor version to the one this endpt 9651ae08745Sheppo * supports, if necessary 9661ae08745Sheppo */ 9671ae08745Sheppo if (rcvd_ver->minor > ldc_versions[idx].minor) 9681ae08745Sheppo rcvd_ver->minor = 9691ae08745Sheppo ldc_versions[idx].minor; 9701ae08745Sheppo bcopy(rcvd_ver, pkt->udata, sizeof (*rcvd_ver)); 9711ae08745Sheppo break; 9721ae08745Sheppo 9731ae08745Sheppo } 9741ae08745Sheppo 9751ae08745Sheppo if (rcvd_ver->major > ldc_versions[idx].major) { 9761ae08745Sheppo 9771ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: using next" 9781ae08745Sheppo " lower idx=%d, v%u.%u\n", idx, 9791ae08745Sheppo ldc_versions[idx].major, 9801ae08745Sheppo ldc_versions[idx].minor); 9811ae08745Sheppo 9821ae08745Sheppo /* send next lower version */ 9831ae08745Sheppo bcopy(&ldc_versions[idx], pkt->udata, 9841ae08745Sheppo sizeof (ldc_versions[idx])); 9851ae08745Sheppo ldcp->next_vidx = idx; 9861ae08745Sheppo break; 9871ae08745Sheppo } 9881ae08745Sheppo 9891ae08745Sheppo /* next version */ 9901ae08745Sheppo idx++; 9911ae08745Sheppo 9921ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: inc idx %x\n", idx); 9931ae08745Sheppo 9941ae08745Sheppo if (idx == LDC_NUM_VERS) { 9951ae08745Sheppo /* no version match - terminate */ 9961ae08745Sheppo ldcp->next_vidx = 0; 997*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 9981ae08745Sheppo return (ECONNRESET); 9991ae08745Sheppo } 10001ae08745Sheppo } 10011ae08745Sheppo 10021ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 10031ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 10041ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 10051ae08745Sheppo 10061ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 10071ae08745Sheppo if (rv == 0) { 10081ae08745Sheppo D2(ldcp->id, "i_ldc_process_VER: (0x%llx) sent version" 10091ae08745Sheppo "INFO v%u.%u\n", ldcp->id, ldc_versions[idx].major, 10101ae08745Sheppo ldc_versions[idx].minor); 10111ae08745Sheppo ldcp->tx_tail = tx_tail; 10121ae08745Sheppo } else { 10131ae08745Sheppo cmn_err(CE_NOTE, 10141ae08745Sheppo "i_ldc_process_VER: (0x%lx) error sending version" 10151ae08745Sheppo "INFO\n", ldcp->id); 10161ae08745Sheppo i_ldc_reset(ldcp); 1017*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 10181ae08745Sheppo return (ECONNRESET); 10191ae08745Sheppo } 10201ae08745Sheppo 10211ae08745Sheppo break; 10221ae08745Sheppo } 10231ae08745Sheppo 1024*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 10251ae08745Sheppo return (rv); 10261ae08745Sheppo } 10271ae08745Sheppo 10281ae08745Sheppo 10291ae08745Sheppo /* 10301ae08745Sheppo * Process an incoming RTS ctrl message 10311ae08745Sheppo */ 10321ae08745Sheppo static int 10331ae08745Sheppo i_ldc_process_RTS(ldc_chan_t *ldcp, ldc_msg_t *msg) 10341ae08745Sheppo { 10351ae08745Sheppo int rv = 0; 10361ae08745Sheppo ldc_msg_t *pkt; 10371ae08745Sheppo uint64_t tx_tail; 10381ae08745Sheppo boolean_t sent_NACK = B_FALSE; 10391ae08745Sheppo 10401ae08745Sheppo D2(ldcp->id, "i_ldc_process_RTS: (0x%llx) received RTS\n", ldcp->id); 10411ae08745Sheppo 10421ae08745Sheppo switch (msg->stype) { 10431ae08745Sheppo case LDC_NACK: 10441ae08745Sheppo DWARN(ldcp->id, 10451ae08745Sheppo "i_ldc_process_RTS: (0x%llx) RTS NACK received\n", 10461ae08745Sheppo ldcp->id); 10471ae08745Sheppo 10481ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1049*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 10501ae08745Sheppo i_ldc_reset(ldcp); 1051*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 10521ae08745Sheppo rv = ECONNRESET; 10531ae08745Sheppo break; 10541ae08745Sheppo 10551ae08745Sheppo case LDC_INFO: 10561ae08745Sheppo 10571ae08745Sheppo /* check mode */ 10581ae08745Sheppo if (ldcp->mode != (ldc_mode_t)msg->env) { 10591ae08745Sheppo cmn_err(CE_NOTE, 10601ae08745Sheppo "i_ldc_process_RTS: (0x%lx) mode mismatch\n", 10611ae08745Sheppo ldcp->id); 10621ae08745Sheppo /* 10631ae08745Sheppo * send NACK in response to MODE message 10641ae08745Sheppo * get the current tail for the response 10651ae08745Sheppo */ 10661ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, LDC_RTS); 10671ae08745Sheppo if (rv) { 10681ae08745Sheppo /* if cannot send NACK - reset channel */ 1069*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 10701ae08745Sheppo i_ldc_reset(ldcp); 1071*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 10721ae08745Sheppo rv = ECONNRESET; 10731ae08745Sheppo break; 10741ae08745Sheppo } 10751ae08745Sheppo sent_NACK = B_TRUE; 10761ae08745Sheppo } 10771ae08745Sheppo break; 10781ae08745Sheppo default: 10791ae08745Sheppo DWARN(ldcp->id, "i_ldc_process_RTS: (0x%llx) unexp ACK\n", 10801ae08745Sheppo ldcp->id); 1081*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 10821ae08745Sheppo i_ldc_reset(ldcp); 1083*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 10841ae08745Sheppo rv = ECONNRESET; 10851ae08745Sheppo break; 10861ae08745Sheppo } 10871ae08745Sheppo 10881ae08745Sheppo /* 10891ae08745Sheppo * If either the connection was reset (when rv != 0) or 10901ae08745Sheppo * a NACK was sent, we return. In the case of a NACK 10911ae08745Sheppo * we dont want to consume the packet that came in but 10921ae08745Sheppo * not record that we received the RTS 10931ae08745Sheppo */ 10941ae08745Sheppo if (rv || sent_NACK) 10951ae08745Sheppo return (rv); 10961ae08745Sheppo 10971ae08745Sheppo /* record RTS received */ 10981ae08745Sheppo ldcp->hstate |= TS_RCVD_RTS; 10991ae08745Sheppo 11001ae08745Sheppo /* store initial SEQID info */ 11011ae08745Sheppo ldcp->last_msg_snt = msg->seqid; 11021ae08745Sheppo 1103*d10e4ef2Snarayan /* Obtain Tx lock */ 1104*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 1105*d10e4ef2Snarayan 11061ae08745Sheppo /* get the current tail for the response */ 11071ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 11081ae08745Sheppo if (rv != 0) { 11091ae08745Sheppo cmn_err(CE_NOTE, 11101ae08745Sheppo "i_ldc_process_RTS: (0x%lx) err sending RTR\n", 11111ae08745Sheppo ldcp->id); 11121ae08745Sheppo i_ldc_reset(ldcp); 1113*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11141ae08745Sheppo return (ECONNRESET); 11151ae08745Sheppo } 11161ae08745Sheppo 11171ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 11181ae08745Sheppo ZERO_PKT(pkt); 11191ae08745Sheppo 11201ae08745Sheppo /* initialize the packet */ 11211ae08745Sheppo pkt->type = LDC_CTRL; 11221ae08745Sheppo pkt->stype = LDC_INFO; 11231ae08745Sheppo pkt->ctrl = LDC_RTR; 11241ae08745Sheppo pkt->env = ldcp->mode; 11251ae08745Sheppo if (ldcp->mode != LDC_MODE_RAW) 11261ae08745Sheppo pkt->seqid = LDC_INIT_SEQID; 11271ae08745Sheppo 11281ae08745Sheppo ldcp->last_msg_rcd = msg->seqid; 11291ae08745Sheppo 11301ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 11311ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 11321ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 11331ae08745Sheppo 11341ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 11351ae08745Sheppo if (rv == 0) { 11361ae08745Sheppo D2(ldcp->id, 11371ae08745Sheppo "i_ldc_process_RTS: (0x%llx) sent RTR\n", ldcp->id); 11381ae08745Sheppo DUMP_LDC_PKT(ldcp, "i_ldc_process_RTS sent rtr", (uint64_t)pkt); 11391ae08745Sheppo 11401ae08745Sheppo ldcp->tx_tail = tx_tail; 11411ae08745Sheppo ldcp->hstate |= TS_SENT_RTR; 11421ae08745Sheppo 11431ae08745Sheppo } else { 11441ae08745Sheppo cmn_err(CE_NOTE, 11451ae08745Sheppo "i_ldc_process_RTS: (0x%lx) error sending RTR\n", 11461ae08745Sheppo ldcp->id); 11471ae08745Sheppo i_ldc_reset(ldcp); 1148*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11491ae08745Sheppo return (ECONNRESET); 11501ae08745Sheppo } 11511ae08745Sheppo 1152*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11531ae08745Sheppo return (0); 11541ae08745Sheppo } 11551ae08745Sheppo 11561ae08745Sheppo /* 11571ae08745Sheppo * Process an incoming RTR ctrl message 11581ae08745Sheppo */ 11591ae08745Sheppo static int 11601ae08745Sheppo i_ldc_process_RTR(ldc_chan_t *ldcp, ldc_msg_t *msg) 11611ae08745Sheppo { 11621ae08745Sheppo int rv = 0; 11631ae08745Sheppo boolean_t sent_NACK = B_FALSE; 11641ae08745Sheppo 11651ae08745Sheppo D2(ldcp->id, "i_ldc_process_RTR: (0x%llx) received RTR\n", ldcp->id); 11661ae08745Sheppo 11671ae08745Sheppo switch (msg->stype) { 11681ae08745Sheppo case LDC_NACK: 11691ae08745Sheppo /* RTR NACK received */ 11701ae08745Sheppo DWARN(ldcp->id, 11711ae08745Sheppo "i_ldc_process_RTR: (0x%llx) RTR NACK received\n", 11721ae08745Sheppo ldcp->id); 11731ae08745Sheppo 11741ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1175*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 11761ae08745Sheppo i_ldc_reset(ldcp); 1177*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11781ae08745Sheppo rv = ECONNRESET; 11791ae08745Sheppo 11801ae08745Sheppo break; 11811ae08745Sheppo 11821ae08745Sheppo case LDC_INFO: 11831ae08745Sheppo 11841ae08745Sheppo /* check mode */ 11851ae08745Sheppo if (ldcp->mode != (ldc_mode_t)msg->env) { 11861ae08745Sheppo DWARN(ldcp->id, 11871ae08745Sheppo "i_ldc_process_RTR: (0x%llx) mode mismatch\n", 11881ae08745Sheppo ldcp->id); 11891ae08745Sheppo /* 11901ae08745Sheppo * send NACK in response to MODE message 11911ae08745Sheppo * get the current tail for the response 11921ae08745Sheppo */ 11931ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, LDC_RTR); 11941ae08745Sheppo if (rv) { 11951ae08745Sheppo /* if cannot send NACK - reset channel */ 1196*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 11971ae08745Sheppo i_ldc_reset(ldcp); 1198*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11991ae08745Sheppo rv = ECONNRESET; 12001ae08745Sheppo break; 12011ae08745Sheppo } 12021ae08745Sheppo sent_NACK = B_TRUE; 12031ae08745Sheppo } 12041ae08745Sheppo break; 12051ae08745Sheppo 12061ae08745Sheppo default: 12071ae08745Sheppo DWARN(ldcp->id, "i_ldc_process_RTR: (0x%llx) unexp ACK\n", 12081ae08745Sheppo ldcp->id); 12091ae08745Sheppo 12101ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1211*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 12121ae08745Sheppo i_ldc_reset(ldcp); 1213*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 12141ae08745Sheppo rv = ECONNRESET; 12151ae08745Sheppo break; 12161ae08745Sheppo } 12171ae08745Sheppo 12181ae08745Sheppo /* 12191ae08745Sheppo * If either the connection was reset (when rv != 0) or 12201ae08745Sheppo * a NACK was sent, we return. In the case of a NACK 12211ae08745Sheppo * we dont want to consume the packet that came in but 12221ae08745Sheppo * not record that we received the RTR 12231ae08745Sheppo */ 12241ae08745Sheppo if (rv || sent_NACK) 12251ae08745Sheppo return (rv); 12261ae08745Sheppo 12271ae08745Sheppo ldcp->last_msg_snt = msg->seqid; 12281ae08745Sheppo ldcp->hstate |= TS_RCVD_RTR; 12291ae08745Sheppo 12301ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_INFO, LDC_RDX); 12311ae08745Sheppo if (rv) { 12321ae08745Sheppo cmn_err(CE_NOTE, 12331ae08745Sheppo "i_ldc_process_RTR: (0x%lx) cannot send RDX\n", 12341ae08745Sheppo ldcp->id); 1235*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 12361ae08745Sheppo i_ldc_reset(ldcp); 1237*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 12381ae08745Sheppo return (ECONNRESET); 12391ae08745Sheppo } 12401ae08745Sheppo D2(ldcp->id, 12411ae08745Sheppo "i_ldc_process_RTR: (0x%llx) sent RDX\n", ldcp->id); 12421ae08745Sheppo 12431ae08745Sheppo ldcp->hstate |= TS_SENT_RDX; 12441ae08745Sheppo ldcp->tstate |= TS_HSHAKE_DONE; 12451ae08745Sheppo ldcp->status = LDC_UP; 12461ae08745Sheppo 12471ae08745Sheppo DWARN(DBG_ALL_LDCS, "(0x%llx) Handshake Complete\n", ldcp->id); 12481ae08745Sheppo 12491ae08745Sheppo return (0); 12501ae08745Sheppo } 12511ae08745Sheppo 12521ae08745Sheppo 12531ae08745Sheppo /* 12541ae08745Sheppo * Process an incoming RDX ctrl message 12551ae08745Sheppo */ 12561ae08745Sheppo static int 12571ae08745Sheppo i_ldc_process_RDX(ldc_chan_t *ldcp, ldc_msg_t *msg) 12581ae08745Sheppo { 12591ae08745Sheppo int rv = 0; 12601ae08745Sheppo 12611ae08745Sheppo D2(ldcp->id, "i_ldc_process_RDX: (0x%llx) received RDX\n", ldcp->id); 12621ae08745Sheppo 12631ae08745Sheppo switch (msg->stype) { 12641ae08745Sheppo case LDC_NACK: 12651ae08745Sheppo /* RDX NACK received */ 12661ae08745Sheppo DWARN(ldcp->id, 12671ae08745Sheppo "i_ldc_process_RDX: (0x%llx) RDX NACK received\n", 12681ae08745Sheppo ldcp->id); 12691ae08745Sheppo 12701ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1271*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 12721ae08745Sheppo i_ldc_reset(ldcp); 1273*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 12741ae08745Sheppo rv = ECONNRESET; 12751ae08745Sheppo 12761ae08745Sheppo break; 12771ae08745Sheppo 12781ae08745Sheppo case LDC_INFO: 12791ae08745Sheppo 12801ae08745Sheppo /* 12811ae08745Sheppo * if channel is UP and a RDX received after data transmission 12821ae08745Sheppo * has commenced it is an error 12831ae08745Sheppo */ 12841ae08745Sheppo if ((ldcp->tstate == TS_UP) && (ldcp->hstate & TS_RCVD_RDX)) { 12851ae08745Sheppo DWARN(DBG_ALL_LDCS, 12861ae08745Sheppo "i_ldc_process_RDX: (0x%llx) unexpected RDX" 12871ae08745Sheppo " - LDC reset\n", ldcp->id); 1288*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 12891ae08745Sheppo i_ldc_reset(ldcp); 1290*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 12911ae08745Sheppo return (ECONNRESET); 12921ae08745Sheppo } 12931ae08745Sheppo 12941ae08745Sheppo ldcp->hstate |= TS_RCVD_RDX; 12951ae08745Sheppo ldcp->tstate |= TS_HSHAKE_DONE; 12961ae08745Sheppo ldcp->status = LDC_UP; 12971ae08745Sheppo 12981ae08745Sheppo D1(DBG_ALL_LDCS, "(0x%llx) Handshake Complete\n", ldcp->id); 12991ae08745Sheppo break; 13001ae08745Sheppo 13011ae08745Sheppo default: 13021ae08745Sheppo DWARN(ldcp->id, "i_ldc_process_RDX: (0x%llx) unexp ACK\n", 13031ae08745Sheppo ldcp->id); 13041ae08745Sheppo 13051ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1306*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 13071ae08745Sheppo i_ldc_reset(ldcp); 1308*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 13091ae08745Sheppo rv = ECONNRESET; 13101ae08745Sheppo break; 13111ae08745Sheppo } 13121ae08745Sheppo 13131ae08745Sheppo return (rv); 13141ae08745Sheppo } 13151ae08745Sheppo 13161ae08745Sheppo /* 13171ae08745Sheppo * Process an incoming ACK for a data packet 13181ae08745Sheppo */ 13191ae08745Sheppo static int 13201ae08745Sheppo i_ldc_process_data_ACK(ldc_chan_t *ldcp, ldc_msg_t *msg) 13211ae08745Sheppo { 13221ae08745Sheppo int rv; 13231ae08745Sheppo uint64_t tx_head; 13241ae08745Sheppo ldc_msg_t *pkt; 13251ae08745Sheppo 1326*d10e4ef2Snarayan /* Obtain Tx lock */ 1327*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 1328*d10e4ef2Snarayan 13291ae08745Sheppo /* 1330*d10e4ef2Snarayan * Read the current Tx head and tail 13311ae08745Sheppo */ 13321ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 13331ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 13341ae08745Sheppo if (rv != 0) { 13351ae08745Sheppo cmn_err(CE_WARN, 13361ae08745Sheppo "i_ldc_process_data_ACK: (0x%lx) cannot read qptrs\n", 13371ae08745Sheppo ldcp->id); 1338*d10e4ef2Snarayan 1339*d10e4ef2Snarayan /* Reset the channel -- as we cannot continue */ 1340*d10e4ef2Snarayan i_ldc_reset(ldcp); 1341*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 1342*d10e4ef2Snarayan return (ECONNRESET); 13431ae08745Sheppo } 13441ae08745Sheppo 13451ae08745Sheppo /* 13461ae08745Sheppo * loop from where the previous ACK location was to the 13471ae08745Sheppo * current head location. This is how far the HV has 13481ae08745Sheppo * actually send pkts. Pkts between head and tail are 13491ae08745Sheppo * yet to be sent by HV. 13501ae08745Sheppo */ 13511ae08745Sheppo tx_head = ldcp->tx_ackd_head; 13521ae08745Sheppo for (;;) { 13531ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_head); 13541ae08745Sheppo tx_head = (tx_head + LDC_PACKET_SIZE) % 13551ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 13561ae08745Sheppo 13571ae08745Sheppo if (pkt->seqid == msg->ackid) { 13581ae08745Sheppo D2(ldcp->id, 13591ae08745Sheppo "i_ldc_process_data_ACK: (0x%llx) found packet\n", 13601ae08745Sheppo ldcp->id); 13611ae08745Sheppo ldcp->last_ack_rcd = msg->ackid; 13621ae08745Sheppo ldcp->tx_ackd_head = tx_head; 13631ae08745Sheppo break; 13641ae08745Sheppo } 13651ae08745Sheppo if (tx_head == ldcp->tx_head) { 13661ae08745Sheppo /* could not find packet */ 13671ae08745Sheppo DWARN(ldcp->id, 13681ae08745Sheppo "i_ldc_process_data_ACK: (0x%llx) invalid ACKid\n", 13691ae08745Sheppo ldcp->id); 1370*d10e4ef2Snarayan 1371*d10e4ef2Snarayan /* Reset the channel -- as we cannot continue */ 1372*d10e4ef2Snarayan i_ldc_reset(ldcp); 1373*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 1374*d10e4ef2Snarayan return (ECONNRESET); 13751ae08745Sheppo } 13761ae08745Sheppo } 13771ae08745Sheppo 1378*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 13791ae08745Sheppo return (0); 13801ae08745Sheppo } 13811ae08745Sheppo 13821ae08745Sheppo /* 13831ae08745Sheppo * Process incoming control message 13841ae08745Sheppo * Return 0 - session can continue 13851ae08745Sheppo * EAGAIN - reprocess packet - state was changed 13861ae08745Sheppo * ECONNRESET - channel was reset 13871ae08745Sheppo */ 13881ae08745Sheppo static int 13891ae08745Sheppo i_ldc_ctrlmsg(ldc_chan_t *ldcp, ldc_msg_t *msg) 13901ae08745Sheppo { 13911ae08745Sheppo int rv = 0; 13921ae08745Sheppo 13931ae08745Sheppo switch (ldcp->tstate) { 13941ae08745Sheppo 13951ae08745Sheppo case TS_OPEN: 13961ae08745Sheppo case TS_READY: 13971ae08745Sheppo 13981ae08745Sheppo switch (msg->ctrl & LDC_CTRL_MASK) { 13991ae08745Sheppo case LDC_VER: 14001ae08745Sheppo /* process version message */ 14011ae08745Sheppo rv = i_ldc_process_VER(ldcp, msg); 14021ae08745Sheppo break; 14031ae08745Sheppo default: 14041ae08745Sheppo DWARN(ldcp->id, 14051ae08745Sheppo "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 14061ae08745Sheppo "tstate=0x%x\n", ldcp->id, 14071ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 14081ae08745Sheppo break; 14091ae08745Sheppo } 14101ae08745Sheppo 14111ae08745Sheppo break; 14121ae08745Sheppo 14131ae08745Sheppo case TS_VREADY: 14141ae08745Sheppo 14151ae08745Sheppo switch (msg->ctrl & LDC_CTRL_MASK) { 14161ae08745Sheppo case LDC_VER: 14171ae08745Sheppo /* peer is redoing version negotiation */ 1418*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 14191ae08745Sheppo (void) i_ldc_txq_reconf(ldcp); 14201ae08745Sheppo i_ldc_reset_state(ldcp); 1421*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 14221ae08745Sheppo rv = EAGAIN; 14231ae08745Sheppo break; 14241ae08745Sheppo case LDC_RTS: 14251ae08745Sheppo /* process RTS message */ 14261ae08745Sheppo rv = i_ldc_process_RTS(ldcp, msg); 14271ae08745Sheppo break; 14281ae08745Sheppo case LDC_RTR: 14291ae08745Sheppo /* process RTR message */ 14301ae08745Sheppo rv = i_ldc_process_RTR(ldcp, msg); 14311ae08745Sheppo break; 14321ae08745Sheppo case LDC_RDX: 14331ae08745Sheppo /* process RDX message */ 14341ae08745Sheppo rv = i_ldc_process_RDX(ldcp, msg); 14351ae08745Sheppo break; 14361ae08745Sheppo default: 14371ae08745Sheppo DWARN(ldcp->id, 14381ae08745Sheppo "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 14391ae08745Sheppo "tstate=0x%x\n", ldcp->id, 14401ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 14411ae08745Sheppo break; 14421ae08745Sheppo } 14431ae08745Sheppo 14441ae08745Sheppo break; 14451ae08745Sheppo 14461ae08745Sheppo case TS_UP: 14471ae08745Sheppo 14481ae08745Sheppo switch (msg->ctrl & LDC_CTRL_MASK) { 14491ae08745Sheppo case LDC_VER: 14501ae08745Sheppo DWARN(ldcp->id, 14511ae08745Sheppo "i_ldc_ctrlmsg: (0x%llx) unexpected VER " 14521ae08745Sheppo "- LDC reset\n", ldcp->id); 14531ae08745Sheppo /* peer is redoing version negotiation */ 1454*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 14551ae08745Sheppo (void) i_ldc_txq_reconf(ldcp); 14561ae08745Sheppo i_ldc_reset_state(ldcp); 1457*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 14581ae08745Sheppo rv = EAGAIN; 14591ae08745Sheppo break; 14601ae08745Sheppo 14611ae08745Sheppo case LDC_RDX: 14621ae08745Sheppo /* process RDX message */ 14631ae08745Sheppo rv = i_ldc_process_RDX(ldcp, msg); 14641ae08745Sheppo break; 14651ae08745Sheppo 14661ae08745Sheppo default: 14671ae08745Sheppo DWARN(ldcp->id, 14681ae08745Sheppo "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 14691ae08745Sheppo "tstate=0x%x\n", ldcp->id, 14701ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 14711ae08745Sheppo break; 14721ae08745Sheppo } 14731ae08745Sheppo } 14741ae08745Sheppo 14751ae08745Sheppo return (rv); 14761ae08745Sheppo } 14771ae08745Sheppo 14781ae08745Sheppo /* 14791ae08745Sheppo * Register channel with the channel nexus 14801ae08745Sheppo */ 14811ae08745Sheppo static int 14821ae08745Sheppo i_ldc_register_channel(ldc_chan_t *ldcp) 14831ae08745Sheppo { 14841ae08745Sheppo int rv = 0; 14851ae08745Sheppo ldc_cnex_t *cinfo = &ldcssp->cinfo; 14861ae08745Sheppo 14871ae08745Sheppo if (cinfo->dip == NULL) { 14881ae08745Sheppo DWARN(ldcp->id, 14891ae08745Sheppo "i_ldc_register_channel: cnex has not registered\n"); 14901ae08745Sheppo return (EAGAIN); 14911ae08745Sheppo } 14921ae08745Sheppo 14931ae08745Sheppo rv = cinfo->reg_chan(cinfo->dip, ldcp->id, ldcp->devclass); 14941ae08745Sheppo if (rv) { 14951ae08745Sheppo DWARN(ldcp->id, 14961ae08745Sheppo "i_ldc_register_channel: cannot register channel\n"); 14971ae08745Sheppo return (rv); 14981ae08745Sheppo } 14991ae08745Sheppo 15001ae08745Sheppo rv = cinfo->add_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR, 15011ae08745Sheppo i_ldc_tx_hdlr, ldcp, NULL); 15021ae08745Sheppo if (rv) { 15031ae08745Sheppo DWARN(ldcp->id, 15041ae08745Sheppo "i_ldc_register_channel: cannot add Tx interrupt\n"); 15051ae08745Sheppo (void) cinfo->unreg_chan(cinfo->dip, ldcp->id); 15061ae08745Sheppo return (rv); 15071ae08745Sheppo } 15081ae08745Sheppo 15091ae08745Sheppo rv = cinfo->add_intr(cinfo->dip, ldcp->id, CNEX_RX_INTR, 15101ae08745Sheppo i_ldc_rx_hdlr, ldcp, NULL); 15111ae08745Sheppo if (rv) { 15121ae08745Sheppo DWARN(ldcp->id, 15131ae08745Sheppo "i_ldc_register_channel: cannot add Rx interrupt\n"); 15141ae08745Sheppo (void) cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR); 15151ae08745Sheppo (void) cinfo->unreg_chan(cinfo->dip, ldcp->id); 15161ae08745Sheppo return (rv); 15171ae08745Sheppo } 15181ae08745Sheppo 15191ae08745Sheppo ldcp->tstate |= TS_CNEX_RDY; 15201ae08745Sheppo 15211ae08745Sheppo return (0); 15221ae08745Sheppo } 15231ae08745Sheppo 15241ae08745Sheppo /* 15251ae08745Sheppo * Unregister a channel with the channel nexus 15261ae08745Sheppo */ 15271ae08745Sheppo static int 15281ae08745Sheppo i_ldc_unregister_channel(ldc_chan_t *ldcp) 15291ae08745Sheppo { 15301ae08745Sheppo int rv = 0; 15311ae08745Sheppo ldc_cnex_t *cinfo = &ldcssp->cinfo; 15321ae08745Sheppo 15331ae08745Sheppo if (cinfo->dip == NULL) { 15341ae08745Sheppo DWARN(ldcp->id, 15351ae08745Sheppo "i_ldc_unregister_channel: cnex has not registered\n"); 15361ae08745Sheppo return (EAGAIN); 15371ae08745Sheppo } 15381ae08745Sheppo 15391ae08745Sheppo if (ldcp->tstate & TS_CNEX_RDY) { 15401ae08745Sheppo 1541*d10e4ef2Snarayan /* Remove the Rx interrupt */ 15421ae08745Sheppo rv = cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_RX_INTR); 15431ae08745Sheppo if (rv) { 15441ae08745Sheppo DWARN(ldcp->id, 15451ae08745Sheppo "i_ldc_unregister_channel: err removing Rx intr\n"); 1546*d10e4ef2Snarayan return (rv); 15471ae08745Sheppo } 1548*d10e4ef2Snarayan 1549*d10e4ef2Snarayan /* Remove the Tx interrupt */ 15501ae08745Sheppo rv = cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR); 15511ae08745Sheppo if (rv) { 15521ae08745Sheppo DWARN(ldcp->id, 15531ae08745Sheppo "i_ldc_unregister_channel: err removing Tx intr\n"); 1554*d10e4ef2Snarayan return (rv); 15551ae08745Sheppo } 1556*d10e4ef2Snarayan 1557*d10e4ef2Snarayan /* Unregister the channel */ 15581ae08745Sheppo rv = cinfo->unreg_chan(ldcssp->cinfo.dip, ldcp->id); 15591ae08745Sheppo if (rv) { 15601ae08745Sheppo DWARN(ldcp->id, 15611ae08745Sheppo "i_ldc_unregister_channel: cannot unreg channel\n"); 1562*d10e4ef2Snarayan return (rv); 15631ae08745Sheppo } 15641ae08745Sheppo 15651ae08745Sheppo ldcp->tstate &= ~TS_CNEX_RDY; 15661ae08745Sheppo } 15671ae08745Sheppo 15681ae08745Sheppo return (0); 15691ae08745Sheppo } 15701ae08745Sheppo 15711ae08745Sheppo 15721ae08745Sheppo /* 15731ae08745Sheppo * LDC transmit interrupt handler 15741ae08745Sheppo * triggered for chanel up/down/reset events 15751ae08745Sheppo * and Tx queue content changes 15761ae08745Sheppo */ 15771ae08745Sheppo static uint_t 15781ae08745Sheppo i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2) 15791ae08745Sheppo { 15801ae08745Sheppo _NOTE(ARGUNUSED(arg2)) 15811ae08745Sheppo 15821ae08745Sheppo int rv; 15831ae08745Sheppo ldc_chan_t *ldcp; 15841ae08745Sheppo boolean_t notify_client = B_FALSE; 15851ae08745Sheppo uint64_t notify_event = 0; 15861ae08745Sheppo 15871ae08745Sheppo /* Get the channel for which interrupt was received */ 15881ae08745Sheppo ASSERT(arg1 != NULL); 15891ae08745Sheppo ldcp = (ldc_chan_t *)arg1; 15901ae08745Sheppo 15911ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n", 15921ae08745Sheppo ldcp->id, ldcp); 15931ae08745Sheppo 15941ae08745Sheppo /* Lock channel */ 15951ae08745Sheppo mutex_enter(&ldcp->lock); 15961ae08745Sheppo 1597*d10e4ef2Snarayan /* Obtain Tx lock */ 1598*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 1599*d10e4ef2Snarayan 16001ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, &ldcp->tx_head, &ldcp->tx_tail, 16011ae08745Sheppo &ldcp->link_state); 16021ae08745Sheppo if (rv) { 16031ae08745Sheppo cmn_err(CE_WARN, 16041ae08745Sheppo "i_ldc_tx_hdlr: (0x%lx) cannot read queue ptrs rv=0x%d\n", 16051ae08745Sheppo ldcp->id, rv); 1606*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 16071ae08745Sheppo mutex_exit(&ldcp->lock); 16081ae08745Sheppo return (DDI_INTR_CLAIMED); 16091ae08745Sheppo } 16101ae08745Sheppo 16111ae08745Sheppo /* 16121ae08745Sheppo * reset the channel state if the channel went down 16131ae08745Sheppo * (other side unconfigured queue) or channel was reset 16141ae08745Sheppo * (other side reconfigured its queue) 16151ae08745Sheppo */ 16161ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN) { 16171ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: channel link down\n", ldcp->id); 16181ae08745Sheppo i_ldc_reset(ldcp); 16191ae08745Sheppo notify_client = B_TRUE; 16201ae08745Sheppo notify_event = LDC_EVT_DOWN; 16211ae08745Sheppo } 16221ae08745Sheppo 16231ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_RESET) { 16241ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: channel link reset\n", ldcp->id); 16251ae08745Sheppo i_ldc_reset(ldcp); 16261ae08745Sheppo notify_client = B_TRUE; 16271ae08745Sheppo notify_event = LDC_EVT_RESET; 16281ae08745Sheppo } 16291ae08745Sheppo 16301ae08745Sheppo if (ldcp->tstate == TS_OPEN && ldcp->link_state == LDC_CHANNEL_UP) { 16311ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: channel link up\n", ldcp->id); 16321ae08745Sheppo notify_client = B_TRUE; 16331ae08745Sheppo notify_event = LDC_EVT_RESET; 16341ae08745Sheppo ldcp->tstate |= TS_LINK_READY; 16351ae08745Sheppo ldcp->status = LDC_READY; 16361ae08745Sheppo } 16371ae08745Sheppo 16381ae08745Sheppo /* if callbacks are disabled, do not notify */ 16391ae08745Sheppo if (!ldcp->cb_enabled) 16401ae08745Sheppo notify_client = B_FALSE; 16411ae08745Sheppo 16421ae08745Sheppo if (notify_client) 16431ae08745Sheppo ldcp->cb_inprogress = B_TRUE; 16441ae08745Sheppo 16451ae08745Sheppo /* Unlock channel */ 1646*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 16471ae08745Sheppo mutex_exit(&ldcp->lock); 16481ae08745Sheppo 16491ae08745Sheppo if (notify_client) { 16501ae08745Sheppo rv = ldcp->cb(notify_event, ldcp->cb_arg); 16511ae08745Sheppo if (rv) { 16521ae08745Sheppo DWARN(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) callback " 16531ae08745Sheppo "failure", ldcp->id); 16541ae08745Sheppo } 16551ae08745Sheppo mutex_enter(&ldcp->lock); 16561ae08745Sheppo ldcp->cb_inprogress = B_FALSE; 16571ae08745Sheppo mutex_exit(&ldcp->lock); 16581ae08745Sheppo } 16591ae08745Sheppo 16601ae08745Sheppo mutex_enter(&ldcp->lock); 16611ae08745Sheppo i_ldc_clear_intr(ldcp, CNEX_TX_INTR); 16621ae08745Sheppo mutex_exit(&ldcp->lock); 16631ae08745Sheppo 16641ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) exiting handler", ldcp->id); 16651ae08745Sheppo 16661ae08745Sheppo return (DDI_INTR_CLAIMED); 16671ae08745Sheppo } 16681ae08745Sheppo 16691ae08745Sheppo /* 16701ae08745Sheppo * LDC receive interrupt handler 16711ae08745Sheppo * triggered for channel with data pending to read 16721ae08745Sheppo * i.e. Rx queue content changes 16731ae08745Sheppo */ 16741ae08745Sheppo static uint_t 16751ae08745Sheppo i_ldc_rx_hdlr(caddr_t arg1, caddr_t arg2) 16761ae08745Sheppo { 16771ae08745Sheppo _NOTE(ARGUNUSED(arg2)) 16781ae08745Sheppo 16791ae08745Sheppo int rv; 16801ae08745Sheppo uint64_t rx_head, rx_tail; 16811ae08745Sheppo ldc_msg_t *msg; 16821ae08745Sheppo ldc_chan_t *ldcp; 16831ae08745Sheppo boolean_t notify_client = B_FALSE; 16841ae08745Sheppo uint64_t notify_event = 0; 1685*d10e4ef2Snarayan uint64_t first_fragment = 0; 16861ae08745Sheppo 16871ae08745Sheppo /* Get the channel for which interrupt was received */ 16881ae08745Sheppo if (arg1 == NULL) { 16891ae08745Sheppo cmn_err(CE_WARN, "i_ldc_rx_hdlr: invalid arg\n"); 16901ae08745Sheppo return (DDI_INTR_UNCLAIMED); 16911ae08745Sheppo } 16921ae08745Sheppo 16931ae08745Sheppo ldcp = (ldc_chan_t *)arg1; 16941ae08745Sheppo 16951ae08745Sheppo D1(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n", 16961ae08745Sheppo ldcp->id, ldcp); 16971ae08745Sheppo 16981ae08745Sheppo /* Lock channel */ 16991ae08745Sheppo mutex_enter(&ldcp->lock); 17001ae08745Sheppo 17011ae08745Sheppo /* mark interrupt as pending */ 17021ae08745Sheppo ldcp->intr_pending = B_TRUE; 17031ae08745Sheppo 17041ae08745Sheppo /* 17051ae08745Sheppo * Read packet(s) from the queue 17061ae08745Sheppo */ 17071ae08745Sheppo for (;;) { 17081ae08745Sheppo 17091ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 17101ae08745Sheppo &ldcp->link_state); 17111ae08745Sheppo if (rv) { 17121ae08745Sheppo cmn_err(CE_WARN, 17131ae08745Sheppo "i_ldc_rx_hdlr: (0x%lx) cannot read " 17141ae08745Sheppo "queue ptrs, rv=0x%d\n", ldcp->id, rv); 17151ae08745Sheppo i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 17161ae08745Sheppo mutex_exit(&ldcp->lock); 17171ae08745Sheppo return (DDI_INTR_CLAIMED); 17181ae08745Sheppo } 17191ae08745Sheppo 17201ae08745Sheppo /* 17211ae08745Sheppo * reset the channel state if the channel went down 17221ae08745Sheppo * (other side unconfigured queue) or channel was reset 17231ae08745Sheppo * (other side reconfigured its queue 17241ae08745Sheppo */ 17251ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN) { 17261ae08745Sheppo D1(ldcp->id, "i_ldc_rx_hdlr: channel link down\n", 17271ae08745Sheppo ldcp->id); 1728*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 17291ae08745Sheppo i_ldc_reset(ldcp); 1730*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 17311ae08745Sheppo notify_client = B_TRUE; 17321ae08745Sheppo notify_event = LDC_EVT_DOWN; 17331ae08745Sheppo break; 17341ae08745Sheppo } 17351ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_RESET) { 17361ae08745Sheppo D1(ldcp->id, "i_ldc_rx_hdlr: channel link reset\n", 17371ae08745Sheppo ldcp->id); 1738*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 17391ae08745Sheppo i_ldc_reset(ldcp); 1740*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 17411ae08745Sheppo notify_client = B_TRUE; 17421ae08745Sheppo notify_event = LDC_EVT_RESET; 17431ae08745Sheppo } 17441ae08745Sheppo 17451ae08745Sheppo if (ldcp->tstate == TS_OPEN && 17461ae08745Sheppo ldcp->link_state == LDC_CHANNEL_UP) { 17471ae08745Sheppo D1(ldcp->id, "i_ldc_rx_hdlr: channel link up\n", 17481ae08745Sheppo ldcp->id); 17491ae08745Sheppo notify_client = B_TRUE; 17501ae08745Sheppo notify_event = LDC_EVT_RESET; 17511ae08745Sheppo ldcp->tstate |= TS_LINK_READY; 17521ae08745Sheppo ldcp->status = LDC_READY; 17531ae08745Sheppo } 17541ae08745Sheppo 17551ae08745Sheppo if (rx_head == rx_tail) { 17561ae08745Sheppo D2(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) No packets\n", 17571ae08745Sheppo ldcp->id); 17581ae08745Sheppo break; 17591ae08745Sheppo } 17601ae08745Sheppo D2(ldcp->id, "i_ldc_rx_hdlr: head=0x%llx, tail=0x%llx\n", 17611ae08745Sheppo rx_head, rx_tail); 17621ae08745Sheppo DUMP_LDC_PKT(ldcp, "i_ldc_rx_hdlr rcd", 17631ae08745Sheppo ldcp->rx_q_va + rx_head); 17641ae08745Sheppo 17651ae08745Sheppo /* get the message */ 17661ae08745Sheppo msg = (ldc_msg_t *)(ldcp->rx_q_va + rx_head); 17671ae08745Sheppo 17681ae08745Sheppo /* if channel is in RAW mode or data pkt, notify and return */ 17691ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW) { 17701ae08745Sheppo notify_client = B_TRUE; 17711ae08745Sheppo notify_event |= LDC_EVT_READ; 17721ae08745Sheppo break; 17731ae08745Sheppo } 17741ae08745Sheppo 17751ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_INFO)) { 17761ae08745Sheppo 17771ae08745Sheppo /* discard packet if channel is not up */ 17781ae08745Sheppo if (ldcp->tstate != TS_UP) { 17791ae08745Sheppo 17801ae08745Sheppo /* move the head one position */ 17811ae08745Sheppo rx_head = (rx_head + LDC_PACKET_SIZE) % 17821ae08745Sheppo (ldcp->rx_q_entries << LDC_PACKET_SHIFT); 17831ae08745Sheppo 17841ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, rx_head)) 17851ae08745Sheppo break; 17861ae08745Sheppo 17871ae08745Sheppo continue; 17881ae08745Sheppo } else { 17891ae08745Sheppo notify_client = B_TRUE; 17901ae08745Sheppo notify_event |= LDC_EVT_READ; 17911ae08745Sheppo break; 17921ae08745Sheppo } 17931ae08745Sheppo } 17941ae08745Sheppo 17951ae08745Sheppo /* Check the sequence ID for the message received */ 17961ae08745Sheppo if ((rv = i_ldc_check_seqid(ldcp, msg)) != 0) { 17971ae08745Sheppo 17981ae08745Sheppo DWARN(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) seqid error, " 17991ae08745Sheppo "q_ptrs=0x%lx,0x%lx", ldcp->id, rx_head, rx_tail); 18001ae08745Sheppo 18011ae08745Sheppo /* Reset last_msg_rcd to start of message */ 1802*d10e4ef2Snarayan if (first_fragment != 0) { 1803*d10e4ef2Snarayan ldcp->last_msg_rcd = first_fragment - 1; 1804*d10e4ef2Snarayan first_fragment = 0; 18051ae08745Sheppo } 1806*d10e4ef2Snarayan 18071ae08745Sheppo /* 18081ae08745Sheppo * Send a NACK due to seqid mismatch 18091ae08745Sheppo */ 18101ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, 18111ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK)); 18121ae08745Sheppo 18131ae08745Sheppo if (rv) { 18141ae08745Sheppo cmn_err(CE_NOTE, 18151ae08745Sheppo "i_ldc_rx_hdlr: (0x%lx) err sending " 18161ae08745Sheppo "CTRL/NACK msg\n", ldcp->id); 1817*d10e4ef2Snarayan 1818*d10e4ef2Snarayan /* if cannot send NACK - reset channel */ 1819*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 1820*d10e4ef2Snarayan i_ldc_reset(ldcp); 1821*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 1822*d10e4ef2Snarayan rv = ECONNRESET; 1823*d10e4ef2Snarayan break; 18241ae08745Sheppo } 18251ae08745Sheppo 18261ae08745Sheppo /* purge receive queue */ 18271ae08745Sheppo (void) i_ldc_set_rx_head(ldcp, rx_tail); 18281ae08745Sheppo break; 18291ae08745Sheppo } 18301ae08745Sheppo 18311ae08745Sheppo /* record the message ID */ 18321ae08745Sheppo ldcp->last_msg_rcd = msg->seqid; 18331ae08745Sheppo 18341ae08745Sheppo /* process control messages */ 18351ae08745Sheppo if (msg->type & LDC_CTRL) { 18361ae08745Sheppo /* save current internal state */ 18371ae08745Sheppo uint64_t tstate = ldcp->tstate; 18381ae08745Sheppo 18391ae08745Sheppo rv = i_ldc_ctrlmsg(ldcp, msg); 18401ae08745Sheppo if (rv == EAGAIN) { 18411ae08745Sheppo /* re-process pkt - state was adjusted */ 18421ae08745Sheppo continue; 18431ae08745Sheppo } 18441ae08745Sheppo if (rv == ECONNRESET) { 18451ae08745Sheppo notify_client = B_TRUE; 18461ae08745Sheppo notify_event = LDC_EVT_RESET; 18471ae08745Sheppo break; 18481ae08745Sheppo } 18491ae08745Sheppo 18501ae08745Sheppo /* 18511ae08745Sheppo * control message processing was successful 18521ae08745Sheppo * channel transitioned to ready for communication 18531ae08745Sheppo */ 18541ae08745Sheppo if (rv == 0 && ldcp->tstate == TS_UP && 18551ae08745Sheppo tstate != ldcp->tstate) { 18561ae08745Sheppo notify_client = B_TRUE; 18571ae08745Sheppo notify_event = LDC_EVT_UP; 18581ae08745Sheppo } 18591ae08745Sheppo } 18601ae08745Sheppo 18611ae08745Sheppo /* process data ACKs */ 18621ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 1863*d10e4ef2Snarayan if (rv = i_ldc_process_data_ACK(ldcp, msg)) { 1864*d10e4ef2Snarayan notify_client = B_TRUE; 1865*d10e4ef2Snarayan notify_event = LDC_EVT_RESET; 1866*d10e4ef2Snarayan break; 1867*d10e4ef2Snarayan } 18681ae08745Sheppo } 18691ae08745Sheppo 18701ae08745Sheppo /* move the head one position */ 18711ae08745Sheppo rx_head = (rx_head + LDC_PACKET_SIZE) % 18721ae08745Sheppo (ldcp->rx_q_entries << LDC_PACKET_SHIFT); 18730a55fbb7Slm66018 if (rv = i_ldc_set_rx_head(ldcp, rx_head)) { 18740a55fbb7Slm66018 notify_client = B_TRUE; 18750a55fbb7Slm66018 notify_event = LDC_EVT_RESET; 18761ae08745Sheppo break; 18770a55fbb7Slm66018 } 18781ae08745Sheppo 18791ae08745Sheppo } /* for */ 18801ae08745Sheppo 18811ae08745Sheppo /* if callbacks are disabled, do not notify */ 18821ae08745Sheppo if (!ldcp->cb_enabled) 18831ae08745Sheppo notify_client = B_FALSE; 18841ae08745Sheppo 18851ae08745Sheppo if (notify_client) 18861ae08745Sheppo ldcp->cb_inprogress = B_TRUE; 18871ae08745Sheppo 18881ae08745Sheppo /* Unlock channel */ 18891ae08745Sheppo mutex_exit(&ldcp->lock); 18901ae08745Sheppo 18911ae08745Sheppo if (notify_client) { 18921ae08745Sheppo rv = ldcp->cb(notify_event, ldcp->cb_arg); 18931ae08745Sheppo if (rv) { 18941ae08745Sheppo DWARN(ldcp->id, 18951ae08745Sheppo "i_ldc_rx_hdlr: (0x%llx) callback failure", 18961ae08745Sheppo ldcp->id); 18971ae08745Sheppo } 18981ae08745Sheppo mutex_enter(&ldcp->lock); 18991ae08745Sheppo ldcp->cb_inprogress = B_FALSE; 19001ae08745Sheppo mutex_exit(&ldcp->lock); 19011ae08745Sheppo } 19021ae08745Sheppo 19031ae08745Sheppo mutex_enter(&ldcp->lock); 19041ae08745Sheppo 19051ae08745Sheppo /* 19061ae08745Sheppo * If there are data packets in the queue, the ldc_read will 19071ae08745Sheppo * clear interrupts after draining the queue, else clear interrupts 19081ae08745Sheppo */ 19091ae08745Sheppo if ((notify_event & LDC_EVT_READ) == 0) { 19101ae08745Sheppo i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 19111ae08745Sheppo } 19121ae08745Sheppo 19131ae08745Sheppo mutex_exit(&ldcp->lock); 19141ae08745Sheppo 19151ae08745Sheppo D1(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) exiting handler", ldcp->id); 19161ae08745Sheppo return (DDI_INTR_CLAIMED); 19171ae08745Sheppo } 19181ae08745Sheppo 19191ae08745Sheppo 19201ae08745Sheppo /* -------------------------------------------------------------------------- */ 19211ae08745Sheppo 19221ae08745Sheppo /* 19231ae08745Sheppo * LDC API functions 19241ae08745Sheppo */ 19251ae08745Sheppo 19261ae08745Sheppo /* 19271ae08745Sheppo * Initialize the channel. Allocate internal structure and memory for 19281ae08745Sheppo * TX/RX queues, and initialize locks. 19291ae08745Sheppo */ 19301ae08745Sheppo int 19311ae08745Sheppo ldc_init(uint64_t id, ldc_attr_t *attr, ldc_handle_t *handle) 19321ae08745Sheppo { 19331ae08745Sheppo ldc_chan_t *ldcp; 19341ae08745Sheppo int rv, exit_val; 19351ae08745Sheppo uint64_t ra_base, nentries; 19361ae08745Sheppo 19371ae08745Sheppo exit_val = EINVAL; /* guarantee an error if exit on failure */ 19381ae08745Sheppo 19391ae08745Sheppo if (attr == NULL) { 19401ae08745Sheppo DWARN(id, "ldc_init: (0x%llx) invalid attr\n", id); 19411ae08745Sheppo return (EINVAL); 19421ae08745Sheppo } 19431ae08745Sheppo if (handle == NULL) { 19441ae08745Sheppo DWARN(id, "ldc_init: (0x%llx) invalid handle\n", id); 19451ae08745Sheppo return (EINVAL); 19461ae08745Sheppo } 19471ae08745Sheppo 19481ae08745Sheppo /* check if channel is valid */ 19491ae08745Sheppo rv = hv_ldc_tx_qinfo(id, &ra_base, &nentries); 19501ae08745Sheppo if (rv == H_ECHANNEL) { 19511ae08745Sheppo DWARN(id, "ldc_init: (0x%llx) invalid channel id\n", id); 19521ae08745Sheppo return (EINVAL); 19531ae08745Sheppo } 19541ae08745Sheppo 19551ae08745Sheppo /* check if the channel has already been initialized */ 19561ae08745Sheppo mutex_enter(&ldcssp->lock); 19571ae08745Sheppo ldcp = ldcssp->chan_list; 19581ae08745Sheppo while (ldcp != NULL) { 19591ae08745Sheppo if (ldcp->id == id) { 19601ae08745Sheppo DWARN(id, "ldc_init: (0x%llx) already initialized\n", 19611ae08745Sheppo id); 19621ae08745Sheppo mutex_exit(&ldcssp->lock); 19631ae08745Sheppo return (EADDRINUSE); 19641ae08745Sheppo } 19651ae08745Sheppo ldcp = ldcp->next; 19661ae08745Sheppo } 19671ae08745Sheppo mutex_exit(&ldcssp->lock); 19681ae08745Sheppo 19691ae08745Sheppo ASSERT(ldcp == NULL); 19701ae08745Sheppo 19711ae08745Sheppo *handle = 0; 19721ae08745Sheppo 19731ae08745Sheppo /* Allocate an ldcp structure */ 19741ae08745Sheppo ldcp = kmem_zalloc(sizeof (ldc_chan_t), KM_SLEEP); 19751ae08745Sheppo 1976*d10e4ef2Snarayan /* 1977*d10e4ef2Snarayan * Initialize the channel and Tx lock 1978*d10e4ef2Snarayan * 1979*d10e4ef2Snarayan * The channel 'lock' protects the entire channel and 1980*d10e4ef2Snarayan * should be acquired before initializing, resetting, 1981*d10e4ef2Snarayan * destroying or reading from a channel. 1982*d10e4ef2Snarayan * 1983*d10e4ef2Snarayan * The 'tx_lock' should be acquired prior to transmitting 1984*d10e4ef2Snarayan * data over the channel. The lock should also be acquired 1985*d10e4ef2Snarayan * prior to channel reconfiguration (in order to prevent 1986*d10e4ef2Snarayan * concurrent writes). 1987*d10e4ef2Snarayan * 1988*d10e4ef2Snarayan * ORDERING: When both locks are being acquired, to prevent 1989*d10e4ef2Snarayan * deadlocks, the channel lock should be always acquired prior 1990*d10e4ef2Snarayan * to the tx_lock. 1991*d10e4ef2Snarayan */ 19921ae08745Sheppo mutex_init(&ldcp->lock, NULL, MUTEX_DRIVER, NULL); 1993*d10e4ef2Snarayan mutex_init(&ldcp->tx_lock, NULL, MUTEX_DRIVER, NULL); 19941ae08745Sheppo 19951ae08745Sheppo /* Initialize the channel */ 19961ae08745Sheppo ldcp->id = id; 19971ae08745Sheppo ldcp->cb = NULL; 19981ae08745Sheppo ldcp->cb_arg = NULL; 19991ae08745Sheppo ldcp->cb_inprogress = B_FALSE; 20001ae08745Sheppo ldcp->cb_enabled = B_FALSE; 20011ae08745Sheppo ldcp->next = NULL; 20021ae08745Sheppo 20031ae08745Sheppo /* Read attributes */ 20041ae08745Sheppo ldcp->mode = attr->mode; 20051ae08745Sheppo ldcp->devclass = attr->devclass; 20061ae08745Sheppo ldcp->devinst = attr->instance; 20071ae08745Sheppo 20081ae08745Sheppo ldcp->rx_q_entries = 20091ae08745Sheppo (attr->qlen > 0) ? attr->qlen : LDC_QUEUE_ENTRIES; 20101ae08745Sheppo ldcp->tx_q_entries = ldcp->rx_q_entries; 20111ae08745Sheppo 20121ae08745Sheppo D1(ldcp->id, 20131ae08745Sheppo "ldc_init: (0x%llx) channel attributes, class=0x%x, " 20141ae08745Sheppo "instance=0x%llx,mode=%d, qlen=%d\n", 20151ae08745Sheppo ldcp->id, ldcp->devclass, ldcp->devinst, 20161ae08745Sheppo ldcp->mode, ldcp->rx_q_entries); 20171ae08745Sheppo 20181ae08745Sheppo ldcp->next_vidx = 0; 20191ae08745Sheppo ldcp->tstate = 0; 20201ae08745Sheppo ldcp->hstate = 0; 20211ae08745Sheppo ldcp->last_msg_snt = LDC_INIT_SEQID; 20221ae08745Sheppo ldcp->last_ack_rcd = 0; 20231ae08745Sheppo ldcp->last_msg_rcd = 0; 20241ae08745Sheppo 20251ae08745Sheppo ldcp->stream_bufferp = NULL; 20261ae08745Sheppo ldcp->exp_dring_list = NULL; 20271ae08745Sheppo ldcp->imp_dring_list = NULL; 20281ae08745Sheppo ldcp->mhdl_list = NULL; 20291ae08745Sheppo 20301ae08745Sheppo /* Initialize payload size depending on whether channel is reliable */ 20311ae08745Sheppo switch (ldcp->mode) { 20321ae08745Sheppo case LDC_MODE_RAW: 20331ae08745Sheppo ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RAW; 20341ae08745Sheppo ldcp->read_p = i_ldc_read_raw; 20351ae08745Sheppo ldcp->write_p = i_ldc_write_raw; 20361ae08745Sheppo ldcp->mtu = 0; 20371ae08745Sheppo break; 20381ae08745Sheppo case LDC_MODE_UNRELIABLE: 20391ae08745Sheppo ldcp->pkt_payload = LDC_PAYLOAD_SIZE_UNRELIABLE; 20401ae08745Sheppo ldcp->read_p = i_ldc_read_packet; 20411ae08745Sheppo ldcp->write_p = i_ldc_write_packet; 20421ae08745Sheppo ldcp->mtu = 0; 20431ae08745Sheppo break; 20441ae08745Sheppo case LDC_MODE_RELIABLE: 20451ae08745Sheppo ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RELIABLE; 20461ae08745Sheppo ldcp->read_p = i_ldc_read_packet; 20471ae08745Sheppo ldcp->write_p = i_ldc_write_packet; 20481ae08745Sheppo ldcp->mtu = 0; 20491ae08745Sheppo break; 20501ae08745Sheppo case LDC_MODE_STREAM: 20511ae08745Sheppo ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RELIABLE; 20521ae08745Sheppo 20531ae08745Sheppo ldcp->stream_remains = 0; 20541ae08745Sheppo ldcp->stream_offset = 0; 20551ae08745Sheppo ldcp->mtu = LDC_STREAM_MTU; 20561ae08745Sheppo ldcp->stream_bufferp = kmem_alloc(ldcp->mtu, KM_SLEEP); 20571ae08745Sheppo ldcp->read_p = i_ldc_read_stream; 20581ae08745Sheppo ldcp->write_p = i_ldc_write_stream; 20591ae08745Sheppo break; 20601ae08745Sheppo default: 20611ae08745Sheppo exit_val = EINVAL; 20621ae08745Sheppo goto cleanup_on_exit; 20631ae08745Sheppo } 20641ae08745Sheppo 20651ae08745Sheppo /* Create a transmit queue */ 20661ae08745Sheppo ldcp->tx_q_va = (uint64_t) 20671ae08745Sheppo contig_mem_alloc(ldcp->tx_q_entries << LDC_PACKET_SHIFT); 20681ae08745Sheppo if (ldcp->tx_q_va == NULL) { 20691ae08745Sheppo cmn_err(CE_WARN, 20701ae08745Sheppo "ldc_init: (0x%lx) TX queue allocation failed\n", 20711ae08745Sheppo ldcp->id); 20721ae08745Sheppo exit_val = ENOMEM; 20731ae08745Sheppo goto cleanup_on_exit; 20741ae08745Sheppo } 20751ae08745Sheppo ldcp->tx_q_ra = va_to_pa((caddr_t)ldcp->tx_q_va); 20761ae08745Sheppo 20771ae08745Sheppo D2(ldcp->id, "ldc_init: txq_va=0x%llx, txq_ra=0x%llx, entries=0x%llx\n", 20781ae08745Sheppo ldcp->tx_q_va, ldcp->tx_q_ra, ldcp->tx_q_entries); 20791ae08745Sheppo 20801ae08745Sheppo ldcp->tstate |= TS_TXQ_RDY; 20811ae08745Sheppo 20821ae08745Sheppo /* Create a receive queue */ 20831ae08745Sheppo ldcp->rx_q_va = (uint64_t) 20841ae08745Sheppo contig_mem_alloc(ldcp->rx_q_entries << LDC_PACKET_SHIFT); 20851ae08745Sheppo if (ldcp->rx_q_va == NULL) { 20861ae08745Sheppo cmn_err(CE_WARN, 20871ae08745Sheppo "ldc_init: (0x%lx) RX queue allocation failed\n", 20881ae08745Sheppo ldcp->id); 20891ae08745Sheppo exit_val = ENOMEM; 20901ae08745Sheppo goto cleanup_on_exit; 20911ae08745Sheppo } 20921ae08745Sheppo ldcp->rx_q_ra = va_to_pa((caddr_t)ldcp->rx_q_va); 20931ae08745Sheppo 20941ae08745Sheppo D2(ldcp->id, "ldc_init: rxq_va=0x%llx, rxq_ra=0x%llx, entries=0x%llx\n", 20951ae08745Sheppo ldcp->rx_q_va, ldcp->rx_q_ra, ldcp->rx_q_entries); 20961ae08745Sheppo 20971ae08745Sheppo ldcp->tstate |= TS_RXQ_RDY; 20981ae08745Sheppo 20991ae08745Sheppo /* Init descriptor ring and memory handle list lock */ 21001ae08745Sheppo mutex_init(&ldcp->exp_dlist_lock, NULL, MUTEX_DRIVER, NULL); 21011ae08745Sheppo mutex_init(&ldcp->imp_dlist_lock, NULL, MUTEX_DRIVER, NULL); 21021ae08745Sheppo mutex_init(&ldcp->mlist_lock, NULL, MUTEX_DRIVER, NULL); 21031ae08745Sheppo 21041ae08745Sheppo /* mark status as INITialized */ 21051ae08745Sheppo ldcp->status = LDC_INIT; 21061ae08745Sheppo 21071ae08745Sheppo /* Add to channel list */ 21081ae08745Sheppo mutex_enter(&ldcssp->lock); 21091ae08745Sheppo ldcp->next = ldcssp->chan_list; 21101ae08745Sheppo ldcssp->chan_list = ldcp; 21111ae08745Sheppo ldcssp->channel_count++; 21121ae08745Sheppo mutex_exit(&ldcssp->lock); 21131ae08745Sheppo 21141ae08745Sheppo /* set the handle */ 21151ae08745Sheppo *handle = (ldc_handle_t)ldcp; 21161ae08745Sheppo 21171ae08745Sheppo D1(ldcp->id, "ldc_init: (0x%llx) channel initialized\n", ldcp->id); 21181ae08745Sheppo 21191ae08745Sheppo return (0); 21201ae08745Sheppo 21211ae08745Sheppo cleanup_on_exit: 21221ae08745Sheppo 21231ae08745Sheppo if (ldcp->mode == LDC_MODE_STREAM && ldcp->stream_bufferp) 21241ae08745Sheppo kmem_free(ldcp->stream_bufferp, ldcp->mtu); 21251ae08745Sheppo 21261ae08745Sheppo if (ldcp->tstate & TS_TXQ_RDY) 21271ae08745Sheppo contig_mem_free((caddr_t)ldcp->tx_q_va, 21281ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 21291ae08745Sheppo 21301ae08745Sheppo if (ldcp->tstate & TS_RXQ_RDY) 21311ae08745Sheppo contig_mem_free((caddr_t)ldcp->rx_q_va, 21321ae08745Sheppo (ldcp->rx_q_entries << LDC_PACKET_SHIFT)); 21331ae08745Sheppo 2134*d10e4ef2Snarayan mutex_destroy(&ldcp->tx_lock); 21351ae08745Sheppo mutex_destroy(&ldcp->lock); 21361ae08745Sheppo 21371ae08745Sheppo if (ldcp) 21381ae08745Sheppo kmem_free(ldcp, sizeof (ldc_chan_t)); 21391ae08745Sheppo 21401ae08745Sheppo return (exit_val); 21411ae08745Sheppo } 21421ae08745Sheppo 21431ae08745Sheppo /* 21441ae08745Sheppo * Finalizes the LDC connection. It will return EBUSY if the 21451ae08745Sheppo * channel is open. A ldc_close() has to be done prior to 21461ae08745Sheppo * a ldc_fini operation. It frees TX/RX queues, associated 21471ae08745Sheppo * with the channel 21481ae08745Sheppo */ 21491ae08745Sheppo int 21501ae08745Sheppo ldc_fini(ldc_handle_t handle) 21511ae08745Sheppo { 21521ae08745Sheppo ldc_chan_t *ldcp; 21531ae08745Sheppo ldc_chan_t *tmp_ldcp; 21541ae08745Sheppo uint64_t id; 21551ae08745Sheppo 21561ae08745Sheppo if (handle == NULL) { 21571ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_fini: invalid channel handle\n"); 21581ae08745Sheppo return (EINVAL); 21591ae08745Sheppo } 21601ae08745Sheppo ldcp = (ldc_chan_t *)handle; 21611ae08745Sheppo id = ldcp->id; 21621ae08745Sheppo 21631ae08745Sheppo mutex_enter(&ldcp->lock); 21641ae08745Sheppo 21651ae08745Sheppo if (ldcp->tstate > TS_INIT) { 21661ae08745Sheppo DWARN(ldcp->id, "ldc_fini: (0x%llx) channel is open\n", 21671ae08745Sheppo ldcp->id); 21681ae08745Sheppo mutex_exit(&ldcp->lock); 21691ae08745Sheppo return (EBUSY); 21701ae08745Sheppo } 21711ae08745Sheppo 21721ae08745Sheppo /* Remove from the channel list */ 21731ae08745Sheppo mutex_enter(&ldcssp->lock); 21741ae08745Sheppo tmp_ldcp = ldcssp->chan_list; 21751ae08745Sheppo if (tmp_ldcp == ldcp) { 21761ae08745Sheppo ldcssp->chan_list = ldcp->next; 21771ae08745Sheppo ldcp->next = NULL; 21781ae08745Sheppo } else { 21791ae08745Sheppo while (tmp_ldcp != NULL) { 21801ae08745Sheppo if (tmp_ldcp->next == ldcp) { 21811ae08745Sheppo tmp_ldcp->next = ldcp->next; 21821ae08745Sheppo ldcp->next = NULL; 21831ae08745Sheppo break; 21841ae08745Sheppo } 21851ae08745Sheppo tmp_ldcp = tmp_ldcp->next; 21861ae08745Sheppo } 21871ae08745Sheppo if (tmp_ldcp == NULL) { 21881ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_fini: invalid channel hdl\n"); 21891ae08745Sheppo mutex_exit(&ldcssp->lock); 21901ae08745Sheppo mutex_exit(&ldcp->lock); 21911ae08745Sheppo return (EINVAL); 21921ae08745Sheppo } 21931ae08745Sheppo } 21941ae08745Sheppo 21951ae08745Sheppo ldcssp->channel_count--; 21961ae08745Sheppo 21971ae08745Sheppo mutex_exit(&ldcssp->lock); 21981ae08745Sheppo 21991ae08745Sheppo /* Free the map table for this channel */ 22001ae08745Sheppo if (ldcp->mtbl) { 22011ae08745Sheppo (void) hv_ldc_set_map_table(ldcp->id, NULL, NULL); 22021ae08745Sheppo contig_mem_free(ldcp->mtbl->table, ldcp->mtbl->size); 22031ae08745Sheppo mutex_destroy(&ldcp->mtbl->lock); 22041ae08745Sheppo kmem_free(ldcp->mtbl, sizeof (ldc_mtbl_t)); 22051ae08745Sheppo } 22061ae08745Sheppo 22071ae08745Sheppo /* Destroy descriptor ring and memory handle list lock */ 22081ae08745Sheppo mutex_destroy(&ldcp->exp_dlist_lock); 22091ae08745Sheppo mutex_destroy(&ldcp->imp_dlist_lock); 22101ae08745Sheppo mutex_destroy(&ldcp->mlist_lock); 22111ae08745Sheppo 22121ae08745Sheppo /* Free the stream buffer for STREAM_MODE */ 22131ae08745Sheppo if (ldcp->mode == LDC_MODE_STREAM && ldcp->stream_bufferp) 22141ae08745Sheppo kmem_free(ldcp->stream_bufferp, ldcp->mtu); 22151ae08745Sheppo 22161ae08745Sheppo /* Free the RX queue */ 22171ae08745Sheppo contig_mem_free((caddr_t)ldcp->rx_q_va, 22181ae08745Sheppo (ldcp->rx_q_entries << LDC_PACKET_SHIFT)); 22191ae08745Sheppo ldcp->tstate &= ~TS_RXQ_RDY; 22201ae08745Sheppo 22211ae08745Sheppo /* Free the TX queue */ 22221ae08745Sheppo contig_mem_free((caddr_t)ldcp->tx_q_va, 22231ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 22241ae08745Sheppo ldcp->tstate &= ~TS_TXQ_RDY; 22251ae08745Sheppo 22261ae08745Sheppo 22271ae08745Sheppo mutex_exit(&ldcp->lock); 22281ae08745Sheppo 22291ae08745Sheppo /* Destroy mutex */ 2230*d10e4ef2Snarayan mutex_destroy(&ldcp->tx_lock); 22311ae08745Sheppo mutex_destroy(&ldcp->lock); 22321ae08745Sheppo 22331ae08745Sheppo /* free channel structure */ 22341ae08745Sheppo kmem_free(ldcp, sizeof (ldc_chan_t)); 22351ae08745Sheppo 22361ae08745Sheppo D1(id, "ldc_fini: (0x%llx) channel finalized\n", id); 22371ae08745Sheppo 22381ae08745Sheppo return (0); 22391ae08745Sheppo } 22401ae08745Sheppo 22411ae08745Sheppo /* 22421ae08745Sheppo * Open the LDC channel for use. It registers the TX/RX queues 22431ae08745Sheppo * with the Hypervisor. It also specifies the interrupt number 22441ae08745Sheppo * and target CPU for this channel 22451ae08745Sheppo */ 22461ae08745Sheppo int 22471ae08745Sheppo ldc_open(ldc_handle_t handle) 22481ae08745Sheppo { 22491ae08745Sheppo ldc_chan_t *ldcp; 22501ae08745Sheppo int rv; 22511ae08745Sheppo 22521ae08745Sheppo if (handle == NULL) { 22531ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_open: invalid channel handle\n"); 22541ae08745Sheppo return (EINVAL); 22551ae08745Sheppo } 22561ae08745Sheppo 22571ae08745Sheppo ldcp = (ldc_chan_t *)handle; 22581ae08745Sheppo 22591ae08745Sheppo mutex_enter(&ldcp->lock); 22601ae08745Sheppo 22611ae08745Sheppo if (ldcp->tstate < TS_INIT) { 22621ae08745Sheppo DWARN(ldcp->id, 22631ae08745Sheppo "ldc_open: (0x%llx) channel not initialized\n", ldcp->id); 22641ae08745Sheppo mutex_exit(&ldcp->lock); 22651ae08745Sheppo return (EFAULT); 22661ae08745Sheppo } 22671ae08745Sheppo if (ldcp->tstate >= TS_OPEN) { 22681ae08745Sheppo DWARN(ldcp->id, 22691ae08745Sheppo "ldc_open: (0x%llx) channel is already open\n", ldcp->id); 22701ae08745Sheppo mutex_exit(&ldcp->lock); 22711ae08745Sheppo return (EFAULT); 22721ae08745Sheppo } 22731ae08745Sheppo 22741ae08745Sheppo /* 22751ae08745Sheppo * Unregister/Register the tx queue with the hypervisor 22761ae08745Sheppo */ 22771ae08745Sheppo rv = hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 22781ae08745Sheppo if (rv) { 22791ae08745Sheppo cmn_err(CE_WARN, 22801ae08745Sheppo "ldc_open: (0x%lx) channel tx queue unconf failed\n", 22811ae08745Sheppo ldcp->id); 22821ae08745Sheppo mutex_exit(&ldcp->lock); 22831ae08745Sheppo return (EIO); 22841ae08745Sheppo } 22851ae08745Sheppo 22861ae08745Sheppo rv = hv_ldc_tx_qconf(ldcp->id, ldcp->tx_q_ra, ldcp->tx_q_entries); 22871ae08745Sheppo if (rv) { 22881ae08745Sheppo cmn_err(CE_WARN, 22891ae08745Sheppo "ldc_open: (0x%lx) channel tx queue conf failed\n", 22901ae08745Sheppo ldcp->id); 22911ae08745Sheppo mutex_exit(&ldcp->lock); 22921ae08745Sheppo return (EIO); 22931ae08745Sheppo } 22941ae08745Sheppo 22951ae08745Sheppo D2(ldcp->id, "ldc_open: (0x%llx) registered tx queue with LDC\n", 22961ae08745Sheppo ldcp->id); 22971ae08745Sheppo 22981ae08745Sheppo /* 22991ae08745Sheppo * Unregister/Register the rx queue with the hypervisor 23001ae08745Sheppo */ 23011ae08745Sheppo rv = hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 23021ae08745Sheppo if (rv) { 23031ae08745Sheppo cmn_err(CE_WARN, 23041ae08745Sheppo "ldc_open: (0x%lx) channel rx queue unconf failed\n", 23051ae08745Sheppo ldcp->id); 23061ae08745Sheppo mutex_exit(&ldcp->lock); 23071ae08745Sheppo return (EIO); 23081ae08745Sheppo } 23091ae08745Sheppo 23101ae08745Sheppo rv = hv_ldc_rx_qconf(ldcp->id, ldcp->rx_q_ra, ldcp->rx_q_entries); 23111ae08745Sheppo if (rv) { 23121ae08745Sheppo cmn_err(CE_WARN, 23131ae08745Sheppo "ldc_open: (0x%lx) channel rx queue conf failed\n", 23141ae08745Sheppo ldcp->id); 23151ae08745Sheppo mutex_exit(&ldcp->lock); 23161ae08745Sheppo return (EIO); 23171ae08745Sheppo } 23181ae08745Sheppo 23191ae08745Sheppo D2(ldcp->id, "ldc_open: (0x%llx) registered rx queue with LDC\n", 23201ae08745Sheppo ldcp->id); 23211ae08745Sheppo 23221ae08745Sheppo ldcp->tstate |= TS_QCONF_RDY; 23231ae08745Sheppo 23241ae08745Sheppo /* Register the channel with the channel nexus */ 23251ae08745Sheppo rv = i_ldc_register_channel(ldcp); 23261ae08745Sheppo if (rv && rv != EAGAIN) { 23271ae08745Sheppo cmn_err(CE_WARN, 23281ae08745Sheppo "ldc_open: (0x%lx) channel register failed\n", ldcp->id); 23291ae08745Sheppo (void) hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 23301ae08745Sheppo (void) hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 23311ae08745Sheppo mutex_exit(&ldcp->lock); 23321ae08745Sheppo return (EIO); 23331ae08745Sheppo } 23341ae08745Sheppo 23351ae08745Sheppo /* mark channel in OPEN state */ 23361ae08745Sheppo ldcp->status = LDC_OPEN; 23371ae08745Sheppo 23381ae08745Sheppo /* Read channel state */ 23391ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 23401ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 23411ae08745Sheppo if (rv) { 23421ae08745Sheppo cmn_err(CE_WARN, 23431ae08745Sheppo "ldc_open: (0x%lx) cannot read channel state\n", 23441ae08745Sheppo ldcp->id); 23451ae08745Sheppo (void) i_ldc_unregister_channel(ldcp); 23461ae08745Sheppo (void) hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 23471ae08745Sheppo (void) hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 23481ae08745Sheppo mutex_exit(&ldcp->lock); 23491ae08745Sheppo return (EIO); 23501ae08745Sheppo } 23511ae08745Sheppo 23521ae08745Sheppo /* 23531ae08745Sheppo * set the ACKd head to current head location for reliable & 23541ae08745Sheppo * streaming mode 23551ae08745Sheppo */ 23561ae08745Sheppo ldcp->tx_ackd_head = ldcp->tx_head; 23571ae08745Sheppo 23581ae08745Sheppo /* mark channel ready if HV report link is UP (peer alloc'd Rx queue) */ 23591ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_UP || 23601ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 23611ae08745Sheppo ldcp->tstate |= TS_LINK_READY; 23621ae08745Sheppo ldcp->status = LDC_READY; 23631ae08745Sheppo } 23641ae08745Sheppo 23651ae08745Sheppo /* 23661ae08745Sheppo * if channel is being opened in RAW mode - no handshake is needed 23671ae08745Sheppo * switch the channel READY and UP state 23681ae08745Sheppo */ 23691ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW) { 23701ae08745Sheppo ldcp->tstate = TS_UP; /* set bits associated with LDC UP */ 23711ae08745Sheppo ldcp->status = LDC_UP; 23721ae08745Sheppo } 23731ae08745Sheppo 23741ae08745Sheppo mutex_exit(&ldcp->lock); 23751ae08745Sheppo 23761ae08745Sheppo /* 23771ae08745Sheppo * Increment number of open channels 23781ae08745Sheppo */ 23791ae08745Sheppo mutex_enter(&ldcssp->lock); 23801ae08745Sheppo ldcssp->channels_open++; 23811ae08745Sheppo mutex_exit(&ldcssp->lock); 23821ae08745Sheppo 23831ae08745Sheppo D1(ldcp->id, 23841ae08745Sheppo "ldc_open: (0x%llx) channel (0x%p) open for use (tstate=0x%x)\n", 23851ae08745Sheppo ldcp->id, ldcp, ldcp->tstate); 23861ae08745Sheppo 23871ae08745Sheppo return (0); 23881ae08745Sheppo } 23891ae08745Sheppo 23901ae08745Sheppo /* 23911ae08745Sheppo * Close the LDC connection. It will return EBUSY if there 23921ae08745Sheppo * are memory segments or descriptor rings either bound to or 23931ae08745Sheppo * mapped over the channel 23941ae08745Sheppo */ 23951ae08745Sheppo int 23961ae08745Sheppo ldc_close(ldc_handle_t handle) 23971ae08745Sheppo { 23981ae08745Sheppo ldc_chan_t *ldcp; 2399*d10e4ef2Snarayan int rv = 0, retries = 0; 24001ae08745Sheppo boolean_t chk_done = B_FALSE; 24011ae08745Sheppo 24021ae08745Sheppo if (handle == NULL) { 24031ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_close: invalid channel handle\n"); 24041ae08745Sheppo return (EINVAL); 24051ae08745Sheppo } 24061ae08745Sheppo ldcp = (ldc_chan_t *)handle; 24071ae08745Sheppo 24081ae08745Sheppo mutex_enter(&ldcp->lock); 24091ae08745Sheppo 24101ae08745Sheppo /* return error if channel is not open */ 24111ae08745Sheppo if (ldcp->tstate < TS_OPEN) { 24121ae08745Sheppo DWARN(ldcp->id, 24131ae08745Sheppo "ldc_close: (0x%llx) channel is not open\n", ldcp->id); 24141ae08745Sheppo mutex_exit(&ldcp->lock); 24151ae08745Sheppo return (EFAULT); 24161ae08745Sheppo } 24171ae08745Sheppo 24181ae08745Sheppo /* if any memory handles, drings, are bound or mapped cannot close */ 24191ae08745Sheppo if (ldcp->mhdl_list != NULL) { 24201ae08745Sheppo DWARN(ldcp->id, 24211ae08745Sheppo "ldc_close: (0x%llx) channel has bound memory handles\n", 24221ae08745Sheppo ldcp->id); 24231ae08745Sheppo mutex_exit(&ldcp->lock); 24241ae08745Sheppo return (EBUSY); 24251ae08745Sheppo } 24261ae08745Sheppo if (ldcp->exp_dring_list != NULL) { 24271ae08745Sheppo DWARN(ldcp->id, 24281ae08745Sheppo "ldc_close: (0x%llx) channel has bound descriptor rings\n", 24291ae08745Sheppo ldcp->id); 24301ae08745Sheppo mutex_exit(&ldcp->lock); 24311ae08745Sheppo return (EBUSY); 24321ae08745Sheppo } 24331ae08745Sheppo if (ldcp->imp_dring_list != NULL) { 24341ae08745Sheppo DWARN(ldcp->id, 24351ae08745Sheppo "ldc_close: (0x%llx) channel has mapped descriptor rings\n", 24361ae08745Sheppo ldcp->id); 24371ae08745Sheppo mutex_exit(&ldcp->lock); 24381ae08745Sheppo return (EBUSY); 24391ae08745Sheppo } 24401ae08745Sheppo 2441*d10e4ef2Snarayan /* Obtain Tx lock */ 2442*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 2443*d10e4ef2Snarayan 24441ae08745Sheppo /* 24451ae08745Sheppo * Wait for pending transmits to complete i.e Tx queue to drain 24461ae08745Sheppo * if there are pending pkts - wait 1 ms and retry again 24471ae08745Sheppo */ 24481ae08745Sheppo for (;;) { 24491ae08745Sheppo 24501ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 24511ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 24521ae08745Sheppo if (rv) { 24531ae08745Sheppo cmn_err(CE_WARN, 24541ae08745Sheppo "ldc_close: (0x%lx) cannot read qptrs\n", ldcp->id); 2455*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 24561ae08745Sheppo mutex_exit(&ldcp->lock); 24571ae08745Sheppo return (EIO); 24581ae08745Sheppo } 24591ae08745Sheppo 24601ae08745Sheppo if (ldcp->tx_head == ldcp->tx_tail || 24611ae08745Sheppo ldcp->link_state != LDC_CHANNEL_UP) { 24621ae08745Sheppo break; 24631ae08745Sheppo } 24641ae08745Sheppo 24651ae08745Sheppo if (chk_done) { 24661ae08745Sheppo DWARN(ldcp->id, 24671ae08745Sheppo "ldc_close: (0x%llx) Tx queue drain timeout\n", 24681ae08745Sheppo ldcp->id); 24691ae08745Sheppo break; 24701ae08745Sheppo } 24711ae08745Sheppo 24721ae08745Sheppo /* wait for one ms and try again */ 24731ae08745Sheppo delay(drv_usectohz(1000)); 24741ae08745Sheppo chk_done = B_TRUE; 24751ae08745Sheppo } 24761ae08745Sheppo 24771ae08745Sheppo /* 24781ae08745Sheppo * Unregister the channel with the nexus 24791ae08745Sheppo */ 2480*d10e4ef2Snarayan while ((rv = i_ldc_unregister_channel(ldcp)) != 0) { 2481*d10e4ef2Snarayan 2482*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 24831ae08745Sheppo mutex_exit(&ldcp->lock); 2484*d10e4ef2Snarayan 2485*d10e4ef2Snarayan /* if any error other than EAGAIN return back */ 2486*d10e4ef2Snarayan if (rv != EAGAIN || retries >= LDC_MAX_RETRIES) { 2487*d10e4ef2Snarayan cmn_err(CE_WARN, 2488*d10e4ef2Snarayan "ldc_close: (0x%lx) unregister failed, %d\n", 2489*d10e4ef2Snarayan ldcp->id, rv); 24901ae08745Sheppo return (rv); 24911ae08745Sheppo } 24921ae08745Sheppo 24931ae08745Sheppo /* 2494*d10e4ef2Snarayan * As there could be pending interrupts we need 2495*d10e4ef2Snarayan * to wait and try again 2496*d10e4ef2Snarayan */ 2497*d10e4ef2Snarayan drv_usecwait(LDC_DELAY); 2498*d10e4ef2Snarayan mutex_enter(&ldcp->lock); 2499*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 2500*d10e4ef2Snarayan retries++; 2501*d10e4ef2Snarayan } 2502*d10e4ef2Snarayan 2503*d10e4ef2Snarayan /* 25041ae08745Sheppo * Unregister queues 25051ae08745Sheppo */ 25061ae08745Sheppo rv = hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 25071ae08745Sheppo if (rv) { 25081ae08745Sheppo cmn_err(CE_WARN, 25091ae08745Sheppo "ldc_close: (0x%lx) channel TX queue unconf failed\n", 25101ae08745Sheppo ldcp->id); 2511*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 25121ae08745Sheppo mutex_exit(&ldcp->lock); 25131ae08745Sheppo return (EIO); 25141ae08745Sheppo } 25151ae08745Sheppo rv = hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 25161ae08745Sheppo if (rv) { 25171ae08745Sheppo cmn_err(CE_WARN, 25181ae08745Sheppo "ldc_close: (0x%lx) channel RX queue unconf failed\n", 25191ae08745Sheppo ldcp->id); 2520*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 25211ae08745Sheppo mutex_exit(&ldcp->lock); 25221ae08745Sheppo return (EIO); 25231ae08745Sheppo } 25241ae08745Sheppo 25251ae08745Sheppo ldcp->tstate &= ~TS_QCONF_RDY; 25261ae08745Sheppo 25271ae08745Sheppo /* Reset channel state information */ 25281ae08745Sheppo i_ldc_reset_state(ldcp); 25291ae08745Sheppo 25301ae08745Sheppo /* Mark channel as down and in initialized state */ 25311ae08745Sheppo ldcp->tx_ackd_head = 0; 25321ae08745Sheppo ldcp->tx_head = 0; 25331ae08745Sheppo ldcp->tstate = TS_INIT; 25341ae08745Sheppo ldcp->status = LDC_INIT; 25351ae08745Sheppo 2536*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 25371ae08745Sheppo mutex_exit(&ldcp->lock); 25381ae08745Sheppo 25391ae08745Sheppo /* Decrement number of open channels */ 25401ae08745Sheppo mutex_enter(&ldcssp->lock); 25411ae08745Sheppo ldcssp->channels_open--; 25421ae08745Sheppo mutex_exit(&ldcssp->lock); 25431ae08745Sheppo 25441ae08745Sheppo D1(ldcp->id, "ldc_close: (0x%llx) channel closed\n", ldcp->id); 25451ae08745Sheppo 25461ae08745Sheppo return (0); 25471ae08745Sheppo } 25481ae08745Sheppo 25491ae08745Sheppo /* 25501ae08745Sheppo * Register channel callback 25511ae08745Sheppo */ 25521ae08745Sheppo int 25531ae08745Sheppo ldc_reg_callback(ldc_handle_t handle, 25541ae08745Sheppo uint_t(*cb)(uint64_t event, caddr_t arg), caddr_t arg) 25551ae08745Sheppo { 25561ae08745Sheppo ldc_chan_t *ldcp; 25571ae08745Sheppo 25581ae08745Sheppo if (handle == NULL) { 25591ae08745Sheppo DWARN(DBG_ALL_LDCS, 25601ae08745Sheppo "ldc_reg_callback: invalid channel handle\n"); 25611ae08745Sheppo return (EINVAL); 25621ae08745Sheppo } 25631ae08745Sheppo if (((uint64_t)cb) < KERNELBASE) { 25641ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_reg_callback: invalid callback\n"); 25651ae08745Sheppo return (EINVAL); 25661ae08745Sheppo } 25671ae08745Sheppo ldcp = (ldc_chan_t *)handle; 25681ae08745Sheppo 25691ae08745Sheppo mutex_enter(&ldcp->lock); 25701ae08745Sheppo 25711ae08745Sheppo if (ldcp->cb) { 25721ae08745Sheppo DWARN(ldcp->id, "ldc_reg_callback: (0x%llx) callback exists\n", 25731ae08745Sheppo ldcp->id); 25741ae08745Sheppo mutex_exit(&ldcp->lock); 25751ae08745Sheppo return (EIO); 25761ae08745Sheppo } 25771ae08745Sheppo if (ldcp->cb_inprogress) { 25781ae08745Sheppo DWARN(ldcp->id, "ldc_reg_callback: (0x%llx) callback active\n", 25791ae08745Sheppo ldcp->id); 25801ae08745Sheppo mutex_exit(&ldcp->lock); 25811ae08745Sheppo return (EWOULDBLOCK); 25821ae08745Sheppo } 25831ae08745Sheppo 25841ae08745Sheppo ldcp->cb = cb; 25851ae08745Sheppo ldcp->cb_arg = arg; 25861ae08745Sheppo ldcp->cb_enabled = B_TRUE; 25871ae08745Sheppo 25881ae08745Sheppo D1(ldcp->id, 25891ae08745Sheppo "ldc_reg_callback: (0x%llx) registered callback for channel\n", 25901ae08745Sheppo ldcp->id); 25911ae08745Sheppo 25921ae08745Sheppo mutex_exit(&ldcp->lock); 25931ae08745Sheppo 25941ae08745Sheppo return (0); 25951ae08745Sheppo } 25961ae08745Sheppo 25971ae08745Sheppo /* 25981ae08745Sheppo * Unregister channel callback 25991ae08745Sheppo */ 26001ae08745Sheppo int 26011ae08745Sheppo ldc_unreg_callback(ldc_handle_t handle) 26021ae08745Sheppo { 26031ae08745Sheppo ldc_chan_t *ldcp; 26041ae08745Sheppo 26051ae08745Sheppo if (handle == NULL) { 26061ae08745Sheppo DWARN(DBG_ALL_LDCS, 26071ae08745Sheppo "ldc_unreg_callback: invalid channel handle\n"); 26081ae08745Sheppo return (EINVAL); 26091ae08745Sheppo } 26101ae08745Sheppo ldcp = (ldc_chan_t *)handle; 26111ae08745Sheppo 26121ae08745Sheppo mutex_enter(&ldcp->lock); 26131ae08745Sheppo 26141ae08745Sheppo if (ldcp->cb == NULL) { 26151ae08745Sheppo DWARN(ldcp->id, 26161ae08745Sheppo "ldc_unreg_callback: (0x%llx) no callback exists\n", 26171ae08745Sheppo ldcp->id); 26181ae08745Sheppo mutex_exit(&ldcp->lock); 26191ae08745Sheppo return (EIO); 26201ae08745Sheppo } 26211ae08745Sheppo if (ldcp->cb_inprogress) { 26221ae08745Sheppo DWARN(ldcp->id, 26231ae08745Sheppo "ldc_unreg_callback: (0x%llx) callback active\n", 26241ae08745Sheppo ldcp->id); 26251ae08745Sheppo mutex_exit(&ldcp->lock); 26261ae08745Sheppo return (EWOULDBLOCK); 26271ae08745Sheppo } 26281ae08745Sheppo 26291ae08745Sheppo ldcp->cb = NULL; 26301ae08745Sheppo ldcp->cb_arg = NULL; 26311ae08745Sheppo ldcp->cb_enabled = B_FALSE; 26321ae08745Sheppo 26331ae08745Sheppo D1(ldcp->id, 26341ae08745Sheppo "ldc_unreg_callback: (0x%llx) unregistered callback for channel\n", 26351ae08745Sheppo ldcp->id); 26361ae08745Sheppo 26371ae08745Sheppo mutex_exit(&ldcp->lock); 26381ae08745Sheppo 26391ae08745Sheppo return (0); 26401ae08745Sheppo } 26411ae08745Sheppo 26421ae08745Sheppo 26431ae08745Sheppo /* 26441ae08745Sheppo * Bring a channel up by initiating a handshake with the peer 26451ae08745Sheppo * This call is asynchronous. It will complete at a later point 26461ae08745Sheppo * in time when the peer responds back with an RTR. 26471ae08745Sheppo */ 26481ae08745Sheppo int 26491ae08745Sheppo ldc_up(ldc_handle_t handle) 26501ae08745Sheppo { 26511ae08745Sheppo int rv; 26521ae08745Sheppo ldc_chan_t *ldcp; 26531ae08745Sheppo ldc_msg_t *ldcmsg; 26541ae08745Sheppo uint64_t tx_tail; 26551ae08745Sheppo 26561ae08745Sheppo if (handle == NULL) { 26571ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_up: invalid channel handle\n"); 26581ae08745Sheppo return (EINVAL); 26591ae08745Sheppo } 26601ae08745Sheppo ldcp = (ldc_chan_t *)handle; 26611ae08745Sheppo 26621ae08745Sheppo mutex_enter(&ldcp->lock); 26631ae08745Sheppo 26641ae08745Sheppo if (ldcp->tstate == TS_UP) { 26651ae08745Sheppo D2(ldcp->id, 26661ae08745Sheppo "ldc_up: (0x%llx) channel is already in UP state\n", 26671ae08745Sheppo ldcp->id); 26681ae08745Sheppo mutex_exit(&ldcp->lock); 26691ae08745Sheppo return (0); 26701ae08745Sheppo } 26711ae08745Sheppo 26721ae08745Sheppo /* if the channel is in RAW mode - mark it as UP, if READY */ 26731ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW && ldcp->tstate >= TS_READY) { 26741ae08745Sheppo ldcp->tstate = TS_UP; 26751ae08745Sheppo mutex_exit(&ldcp->lock); 26761ae08745Sheppo return (0); 26771ae08745Sheppo } 26781ae08745Sheppo 26791ae08745Sheppo /* Don't start another handshake if there is one in progress */ 26801ae08745Sheppo if (ldcp->hstate) { 26811ae08745Sheppo D2(ldcp->id, 26821ae08745Sheppo "ldc_up: (0x%llx) channel handshake in progress\n", 26831ae08745Sheppo ldcp->id); 26841ae08745Sheppo mutex_exit(&ldcp->lock); 26851ae08745Sheppo return (0); 26861ae08745Sheppo } 26871ae08745Sheppo 2688*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 2689*d10e4ef2Snarayan 26901ae08745Sheppo /* get the current tail for the LDC msg */ 26911ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 26921ae08745Sheppo if (rv) { 26931ae08745Sheppo DWARN(ldcp->id, "ldc_up: (0x%llx) cannot initiate handshake\n", 26941ae08745Sheppo ldcp->id); 2695*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 26961ae08745Sheppo mutex_exit(&ldcp->lock); 26971ae08745Sheppo return (ECONNREFUSED); 26981ae08745Sheppo } 26991ae08745Sheppo 27001ae08745Sheppo ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 27011ae08745Sheppo ZERO_PKT(ldcmsg); 27021ae08745Sheppo 27031ae08745Sheppo ldcmsg->type = LDC_CTRL; 27041ae08745Sheppo ldcmsg->stype = LDC_INFO; 27051ae08745Sheppo ldcmsg->ctrl = LDC_VER; 27061ae08745Sheppo ldcp->next_vidx = 0; 27071ae08745Sheppo bcopy(&ldc_versions[0], ldcmsg->udata, sizeof (ldc_versions[0])); 27081ae08745Sheppo 27091ae08745Sheppo DUMP_LDC_PKT(ldcp, "ldc_up snd ver", (uint64_t)ldcmsg); 27101ae08745Sheppo 27111ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 27121ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 27131ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 27141ae08745Sheppo 27151ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 27161ae08745Sheppo if (rv) { 27171ae08745Sheppo DWARN(ldcp->id, 27181ae08745Sheppo "ldc_up: (0x%llx) cannot initiate handshake rv=%d\n", 27191ae08745Sheppo ldcp->id, rv); 2720*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 27211ae08745Sheppo mutex_exit(&ldcp->lock); 27221ae08745Sheppo return (rv); 27231ae08745Sheppo } 27241ae08745Sheppo 27250a55fbb7Slm66018 ldcp->hstate |= TS_SENT_VER; 27261ae08745Sheppo ldcp->tx_tail = tx_tail; 27271ae08745Sheppo D1(ldcp->id, "ldc_up: (0x%llx) channel up initiated\n", ldcp->id); 27281ae08745Sheppo 2729*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 27301ae08745Sheppo mutex_exit(&ldcp->lock); 27311ae08745Sheppo 27321ae08745Sheppo return (rv); 27331ae08745Sheppo } 27341ae08745Sheppo 27351ae08745Sheppo 27361ae08745Sheppo /* 27371ae08745Sheppo * Reset a channel by re-registering the Rx queues 27381ae08745Sheppo */ 27391ae08745Sheppo int 27401ae08745Sheppo ldc_reset(ldc_handle_t handle) 27411ae08745Sheppo { 27421ae08745Sheppo ldc_chan_t *ldcp; 27431ae08745Sheppo 27441ae08745Sheppo if (handle == NULL) { 27451ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_reset: invalid channel handle\n"); 27461ae08745Sheppo return (EINVAL); 27471ae08745Sheppo } 27481ae08745Sheppo ldcp = (ldc_chan_t *)handle; 27491ae08745Sheppo 27501ae08745Sheppo mutex_enter(&ldcp->lock); 2751*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 27521ae08745Sheppo i_ldc_reset(ldcp); 2753*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 27541ae08745Sheppo mutex_exit(&ldcp->lock); 27551ae08745Sheppo 27561ae08745Sheppo return (0); 27571ae08745Sheppo } 27581ae08745Sheppo 27591ae08745Sheppo /* 27601ae08745Sheppo * Get the current channel status 27611ae08745Sheppo */ 27621ae08745Sheppo int 27631ae08745Sheppo ldc_status(ldc_handle_t handle, ldc_status_t *status) 27641ae08745Sheppo { 27651ae08745Sheppo ldc_chan_t *ldcp; 27661ae08745Sheppo 27671ae08745Sheppo if (handle == NULL || status == NULL) { 27681ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_status: invalid argument\n"); 27691ae08745Sheppo return (EINVAL); 27701ae08745Sheppo } 27711ae08745Sheppo ldcp = (ldc_chan_t *)handle; 27721ae08745Sheppo 27731ae08745Sheppo *status = ((ldc_chan_t *)handle)->status; 27741ae08745Sheppo 27751ae08745Sheppo D1(ldcp->id, 27761ae08745Sheppo "ldc_status: (0x%llx) returned status %d\n", ldcp->id, *status); 27771ae08745Sheppo return (0); 27781ae08745Sheppo } 27791ae08745Sheppo 27801ae08745Sheppo 27811ae08745Sheppo /* 27821ae08745Sheppo * Set the channel's callback mode - enable/disable callbacks 27831ae08745Sheppo */ 27841ae08745Sheppo int 27851ae08745Sheppo ldc_set_cb_mode(ldc_handle_t handle, ldc_cb_mode_t cmode) 27861ae08745Sheppo { 27871ae08745Sheppo ldc_chan_t *ldcp; 27881ae08745Sheppo 27891ae08745Sheppo if (handle == NULL) { 27901ae08745Sheppo DWARN(DBG_ALL_LDCS, 27911ae08745Sheppo "ldc_set_intr_mode: invalid channel handle\n"); 27921ae08745Sheppo return (EINVAL); 27931ae08745Sheppo } 27941ae08745Sheppo ldcp = (ldc_chan_t *)handle; 27951ae08745Sheppo 27961ae08745Sheppo /* 27971ae08745Sheppo * Record no callbacks should be invoked 27981ae08745Sheppo */ 27991ae08745Sheppo mutex_enter(&ldcp->lock); 28001ae08745Sheppo 28011ae08745Sheppo switch (cmode) { 28021ae08745Sheppo case LDC_CB_DISABLE: 28031ae08745Sheppo if (!ldcp->cb_enabled) { 28041ae08745Sheppo DWARN(ldcp->id, 28051ae08745Sheppo "ldc_set_cb_mode: (0x%llx) callbacks disabled\n", 28061ae08745Sheppo ldcp->id); 28071ae08745Sheppo break; 28081ae08745Sheppo } 28091ae08745Sheppo ldcp->cb_enabled = B_FALSE; 28101ae08745Sheppo 28111ae08745Sheppo D1(ldcp->id, "ldc_set_cb_mode: (0x%llx) disabled callbacks\n", 28121ae08745Sheppo ldcp->id); 28131ae08745Sheppo break; 28141ae08745Sheppo 28151ae08745Sheppo case LDC_CB_ENABLE: 28161ae08745Sheppo if (ldcp->cb_enabled) { 28171ae08745Sheppo DWARN(ldcp->id, 28181ae08745Sheppo "ldc_set_cb_mode: (0x%llx) callbacks enabled\n", 28191ae08745Sheppo ldcp->id); 28201ae08745Sheppo break; 28211ae08745Sheppo } 28221ae08745Sheppo ldcp->cb_enabled = B_TRUE; 28231ae08745Sheppo 28241ae08745Sheppo D1(ldcp->id, "ldc_set_cb_mode: (0x%llx) enabled callbacks\n", 28251ae08745Sheppo ldcp->id); 28261ae08745Sheppo break; 28271ae08745Sheppo } 28281ae08745Sheppo 28291ae08745Sheppo mutex_exit(&ldcp->lock); 28301ae08745Sheppo 28311ae08745Sheppo return (0); 28321ae08745Sheppo } 28331ae08745Sheppo 28341ae08745Sheppo /* 28351ae08745Sheppo * Check to see if there are packets on the incoming queue 28361ae08745Sheppo * Will return isempty = B_FALSE if there are packets 28371ae08745Sheppo */ 28381ae08745Sheppo int 28391ae08745Sheppo ldc_chkq(ldc_handle_t handle, boolean_t *isempty) 28401ae08745Sheppo { 28411ae08745Sheppo int rv; 28421ae08745Sheppo uint64_t rx_head, rx_tail; 28431ae08745Sheppo ldc_chan_t *ldcp; 28441ae08745Sheppo 28451ae08745Sheppo if (handle == NULL) { 28461ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_chkq: invalid channel handle\n"); 28471ae08745Sheppo return (EINVAL); 28481ae08745Sheppo } 28491ae08745Sheppo ldcp = (ldc_chan_t *)handle; 28501ae08745Sheppo 28511ae08745Sheppo *isempty = B_TRUE; 28521ae08745Sheppo 28531ae08745Sheppo mutex_enter(&ldcp->lock); 28541ae08745Sheppo 28551ae08745Sheppo if (ldcp->tstate != TS_UP) { 28561ae08745Sheppo D1(ldcp->id, 28571ae08745Sheppo "ldc_chkq: (0x%llx) channel is not up\n", ldcp->id); 28581ae08745Sheppo mutex_exit(&ldcp->lock); 28591ae08745Sheppo return (ECONNRESET); 28601ae08745Sheppo } 28611ae08745Sheppo 28621ae08745Sheppo /* Read packet(s) from the queue */ 28631ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 28641ae08745Sheppo &ldcp->link_state); 28651ae08745Sheppo if (rv != 0) { 28661ae08745Sheppo cmn_err(CE_WARN, 28671ae08745Sheppo "ldc_chkq: (0x%lx) unable to read queue ptrs", ldcp->id); 28681ae08745Sheppo mutex_exit(&ldcp->lock); 28691ae08745Sheppo return (EIO); 28701ae08745Sheppo } 28711ae08745Sheppo /* reset the channel state if the channel went down */ 28721ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN || 28731ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 2874*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 28751ae08745Sheppo i_ldc_reset(ldcp); 2876*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 28771ae08745Sheppo mutex_exit(&ldcp->lock); 28781ae08745Sheppo return (ECONNRESET); 28791ae08745Sheppo } 28801ae08745Sheppo 28811ae08745Sheppo if (rx_head != rx_tail) { 28821ae08745Sheppo D1(ldcp->id, "ldc_chkq: (0x%llx) queue has pkt(s)\n", ldcp->id); 28831ae08745Sheppo *isempty = B_FALSE; 28841ae08745Sheppo } 28851ae08745Sheppo 28861ae08745Sheppo mutex_exit(&ldcp->lock); 28871ae08745Sheppo 28881ae08745Sheppo return (0); 28891ae08745Sheppo } 28901ae08745Sheppo 28911ae08745Sheppo 28921ae08745Sheppo /* 28931ae08745Sheppo * Read 'size' amount of bytes or less. If incoming buffer 28941ae08745Sheppo * is more than 'size', ENOBUFS is returned. 28951ae08745Sheppo * 28961ae08745Sheppo * On return, size contains the number of bytes read. 28971ae08745Sheppo */ 28981ae08745Sheppo int 28991ae08745Sheppo ldc_read(ldc_handle_t handle, caddr_t bufp, size_t *sizep) 29001ae08745Sheppo { 29011ae08745Sheppo ldc_chan_t *ldcp; 29021ae08745Sheppo uint64_t rx_head = 0, rx_tail = 0; 29031ae08745Sheppo int rv = 0, exit_val; 29041ae08745Sheppo 29051ae08745Sheppo if (handle == NULL) { 29061ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_read: invalid channel handle\n"); 29071ae08745Sheppo return (EINVAL); 29081ae08745Sheppo } 29091ae08745Sheppo 29101ae08745Sheppo ldcp = (ldc_chan_t *)handle; 29111ae08745Sheppo 29121ae08745Sheppo /* channel lock */ 29131ae08745Sheppo mutex_enter(&ldcp->lock); 29141ae08745Sheppo 29151ae08745Sheppo if (ldcp->tstate != TS_UP) { 29161ae08745Sheppo DWARN(ldcp->id, 29171ae08745Sheppo "ldc_read: (0x%llx) channel is not in UP state\n", 29181ae08745Sheppo ldcp->id); 29191ae08745Sheppo exit_val = ECONNRESET; 29201ae08745Sheppo } else { 29211ae08745Sheppo exit_val = ldcp->read_p(ldcp, bufp, sizep); 29221ae08745Sheppo } 29231ae08745Sheppo 29241ae08745Sheppo /* 29251ae08745Sheppo * if queue has been drained - clear interrupt 29261ae08745Sheppo */ 29271ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 29281ae08745Sheppo &ldcp->link_state); 29291ae08745Sheppo if (exit_val == 0 && rv == 0 && rx_head == rx_tail) { 29301ae08745Sheppo i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 29311ae08745Sheppo } 29321ae08745Sheppo 29331ae08745Sheppo mutex_exit(&ldcp->lock); 29341ae08745Sheppo return (exit_val); 29351ae08745Sheppo } 29361ae08745Sheppo 29371ae08745Sheppo /* 29381ae08745Sheppo * Basic raw mondo read - 29391ae08745Sheppo * no interpretation of mondo contents at all. 29401ae08745Sheppo * 29411ae08745Sheppo * Enter and exit with ldcp->lock held by caller 29421ae08745Sheppo */ 29431ae08745Sheppo static int 29441ae08745Sheppo i_ldc_read_raw(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 29451ae08745Sheppo { 29461ae08745Sheppo uint64_t q_size_mask; 29471ae08745Sheppo ldc_msg_t *msgp; 29481ae08745Sheppo uint8_t *msgbufp; 29491ae08745Sheppo int rv = 0, space; 29501ae08745Sheppo uint64_t rx_head, rx_tail; 29511ae08745Sheppo 29521ae08745Sheppo space = *sizep; 29531ae08745Sheppo 29541ae08745Sheppo if (space < LDC_PAYLOAD_SIZE_RAW) 29551ae08745Sheppo return (ENOBUFS); 29561ae08745Sheppo 29571ae08745Sheppo ASSERT(mutex_owned(&ldcp->lock)); 29581ae08745Sheppo 29591ae08745Sheppo /* compute mask for increment */ 29601ae08745Sheppo q_size_mask = (ldcp->rx_q_entries-1)<<LDC_PACKET_SHIFT; 29611ae08745Sheppo 29621ae08745Sheppo /* 29631ae08745Sheppo * Read packet(s) from the queue 29641ae08745Sheppo */ 29651ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 29661ae08745Sheppo &ldcp->link_state); 29671ae08745Sheppo if (rv != 0) { 29681ae08745Sheppo cmn_err(CE_WARN, 29691ae08745Sheppo "ldc_read_raw: (0x%lx) unable to read queue ptrs", 29701ae08745Sheppo ldcp->id); 29711ae08745Sheppo return (EIO); 29721ae08745Sheppo } 29731ae08745Sheppo D1(ldcp->id, "ldc_read_raw: (0x%llx) rxh=0x%llx," 29741ae08745Sheppo " rxt=0x%llx, st=0x%llx\n", 29751ae08745Sheppo ldcp->id, rx_head, rx_tail, ldcp->link_state); 29761ae08745Sheppo 29771ae08745Sheppo /* reset the channel state if the channel went down */ 29781ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN) { 2979*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 29801ae08745Sheppo i_ldc_reset(ldcp); 2981*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 29821ae08745Sheppo return (ECONNRESET); 29831ae08745Sheppo } 29841ae08745Sheppo 29851ae08745Sheppo /* 29861ae08745Sheppo * Check for empty queue 29871ae08745Sheppo */ 29881ae08745Sheppo if (rx_head == rx_tail) { 29891ae08745Sheppo *sizep = 0; 29901ae08745Sheppo return (0); 29911ae08745Sheppo } 29921ae08745Sheppo 29931ae08745Sheppo /* get the message */ 29941ae08745Sheppo msgp = (ldc_msg_t *)(ldcp->rx_q_va + rx_head); 29951ae08745Sheppo 29961ae08745Sheppo /* if channel is in RAW mode, copy data and return */ 29971ae08745Sheppo msgbufp = (uint8_t *)&(msgp->raw[0]); 29981ae08745Sheppo 29991ae08745Sheppo bcopy(msgbufp, target_bufp, LDC_PAYLOAD_SIZE_RAW); 30001ae08745Sheppo 30011ae08745Sheppo DUMP_PAYLOAD(ldcp->id, msgbufp); 30021ae08745Sheppo 30031ae08745Sheppo *sizep = LDC_PAYLOAD_SIZE_RAW; 30041ae08745Sheppo 30051ae08745Sheppo rx_head = (rx_head + LDC_PACKET_SIZE) & q_size_mask; 30060a55fbb7Slm66018 rv = i_ldc_set_rx_head(ldcp, rx_head); 30071ae08745Sheppo 30081ae08745Sheppo return (rv); 30091ae08745Sheppo } 30101ae08745Sheppo 30111ae08745Sheppo /* 30121ae08745Sheppo * Process LDC mondos to build larger packets 30131ae08745Sheppo * with either un-reliable or reliable delivery. 30141ae08745Sheppo * 30151ae08745Sheppo * Enter and exit with ldcp->lock held by caller 30161ae08745Sheppo */ 30171ae08745Sheppo static int 30181ae08745Sheppo i_ldc_read_packet(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 30191ae08745Sheppo { 30201ae08745Sheppo int rv = 0; 30211ae08745Sheppo uint64_t rx_head = 0, rx_tail = 0; 30221ae08745Sheppo uint64_t curr_head = 0; 30231ae08745Sheppo ldc_msg_t *msg; 30241ae08745Sheppo caddr_t target; 30251ae08745Sheppo size_t len = 0, bytes_read = 0; 30260a55fbb7Slm66018 int retries = 0; 30271ae08745Sheppo uint64_t q_size_mask; 3028*d10e4ef2Snarayan uint64_t first_fragment = 0; 30291ae08745Sheppo 30301ae08745Sheppo target = target_bufp; 30311ae08745Sheppo 30321ae08745Sheppo ASSERT(mutex_owned(&ldcp->lock)); 30331ae08745Sheppo 30341ae08745Sheppo /* compute mask for increment */ 30351ae08745Sheppo q_size_mask = (ldcp->rx_q_entries-1)<<LDC_PACKET_SHIFT; 30361ae08745Sheppo 30371ae08745Sheppo /* 30381ae08745Sheppo * Read packet(s) from the queue 30391ae08745Sheppo */ 30401ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &curr_head, &rx_tail, 30411ae08745Sheppo &ldcp->link_state); 30421ae08745Sheppo if (rv != 0) { 30431ae08745Sheppo cmn_err(CE_WARN, 30441ae08745Sheppo "ldc_read: (0x%lx) unable to read queue ptrs", 30451ae08745Sheppo ldcp->id); 30461ae08745Sheppo return (EIO); 30471ae08745Sheppo } 30481ae08745Sheppo D1(ldcp->id, "ldc_read: (0x%llx) chd=0x%llx, tl=0x%llx, st=0x%llx\n", 30491ae08745Sheppo ldcp->id, curr_head, rx_tail, ldcp->link_state); 30501ae08745Sheppo 30511ae08745Sheppo /* reset the channel state if the channel went down */ 30521ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN) { 3053*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 30541ae08745Sheppo i_ldc_reset(ldcp); 3055*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 30561ae08745Sheppo return (ECONNRESET); 30571ae08745Sheppo } 30581ae08745Sheppo 30591ae08745Sheppo for (;;) { 30601ae08745Sheppo 30611ae08745Sheppo if (curr_head == rx_tail) { 30621ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, 30631ae08745Sheppo &rx_head, &rx_tail, &ldcp->link_state); 30641ae08745Sheppo if (rv != 0) { 30651ae08745Sheppo cmn_err(CE_WARN, 30661ae08745Sheppo "ldc_read: (0x%lx) cannot read queue ptrs", 30671ae08745Sheppo ldcp->id); 30681ae08745Sheppo return (EIO); 30691ae08745Sheppo } 30701ae08745Sheppo /* reset the channel state if the channel went down */ 30711ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN) { 3072*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 30731ae08745Sheppo i_ldc_reset(ldcp); 3074*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 30751ae08745Sheppo return (ECONNRESET); 30761ae08745Sheppo } 30771ae08745Sheppo } 30781ae08745Sheppo 30791ae08745Sheppo if (curr_head == rx_tail) { 30801ae08745Sheppo 30811ae08745Sheppo /* If in the middle of a fragmented xfer */ 3082*d10e4ef2Snarayan if (first_fragment != 0) { 30830a55fbb7Slm66018 30840a55fbb7Slm66018 /* wait for ldc_delay usecs */ 30850a55fbb7Slm66018 drv_usecwait(ldc_delay); 30860a55fbb7Slm66018 30870a55fbb7Slm66018 if (++retries < ldc_max_retries) 30881ae08745Sheppo continue; 30890a55fbb7Slm66018 30901ae08745Sheppo *sizep = 0; 3091*d10e4ef2Snarayan ldcp->last_msg_rcd = first_fragment - 1; 30921ae08745Sheppo DWARN(DBG_ALL_LDCS, 30931ae08745Sheppo "ldc_read: (0x%llx) read timeout", 30941ae08745Sheppo ldcp->id); 30951ae08745Sheppo return (ETIMEDOUT); 30961ae08745Sheppo } 30971ae08745Sheppo *sizep = 0; 30981ae08745Sheppo break; 30991ae08745Sheppo } 31000a55fbb7Slm66018 retries = 0; 31011ae08745Sheppo 31021ae08745Sheppo D2(ldcp->id, 31031ae08745Sheppo "ldc_read: (0x%llx) chd=0x%llx, rxhd=0x%llx, rxtl=0x%llx\n", 31041ae08745Sheppo ldcp->id, curr_head, rx_head, rx_tail); 31051ae08745Sheppo 31061ae08745Sheppo /* get the message */ 31071ae08745Sheppo msg = (ldc_msg_t *)(ldcp->rx_q_va + curr_head); 31081ae08745Sheppo 31091ae08745Sheppo DUMP_LDC_PKT(ldcp, "ldc_read received pkt", 31101ae08745Sheppo ldcp->rx_q_va + curr_head); 31111ae08745Sheppo 31121ae08745Sheppo /* Check the message ID for the message received */ 31131ae08745Sheppo if ((rv = i_ldc_check_seqid(ldcp, msg)) != 0) { 31141ae08745Sheppo 31151ae08745Sheppo DWARN(ldcp->id, "ldc_read: (0x%llx) seqid error, " 31161ae08745Sheppo "q_ptrs=0x%lx,0x%lx", ldcp->id, rx_head, rx_tail); 31171ae08745Sheppo 31180a55fbb7Slm66018 /* throw away data */ 31190a55fbb7Slm66018 bytes_read = 0; 31200a55fbb7Slm66018 31211ae08745Sheppo /* Reset last_msg_rcd to start of message */ 3122*d10e4ef2Snarayan if (first_fragment != 0) { 3123*d10e4ef2Snarayan ldcp->last_msg_rcd = first_fragment - 1; 3124*d10e4ef2Snarayan first_fragment = 0; 31251ae08745Sheppo } 31261ae08745Sheppo /* 31271ae08745Sheppo * Send a NACK -- invalid seqid 31281ae08745Sheppo * get the current tail for the response 31291ae08745Sheppo */ 31301ae08745Sheppo rv = i_ldc_send_pkt(ldcp, msg->type, LDC_NACK, 31311ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK)); 31321ae08745Sheppo if (rv) { 31331ae08745Sheppo cmn_err(CE_NOTE, 31341ae08745Sheppo "ldc_read: (0x%lx) err sending " 31351ae08745Sheppo "NACK msg\n", ldcp->id); 3136*d10e4ef2Snarayan 3137*d10e4ef2Snarayan /* if cannot send NACK - reset channel */ 3138*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 3139*d10e4ef2Snarayan i_ldc_reset(ldcp); 3140*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3141*d10e4ef2Snarayan rv = ECONNRESET; 3142*d10e4ef2Snarayan break; 31431ae08745Sheppo } 31441ae08745Sheppo 31451ae08745Sheppo /* purge receive queue */ 31460a55fbb7Slm66018 rv = i_ldc_set_rx_head(ldcp, rx_tail); 31471ae08745Sheppo 31481ae08745Sheppo break; 31491ae08745Sheppo } 31501ae08745Sheppo 31511ae08745Sheppo /* 31521ae08745Sheppo * Process any messages of type CTRL messages 31531ae08745Sheppo * Future implementations should try to pass these to 31541ae08745Sheppo * LDC transport by resetting the intr state. 31551ae08745Sheppo * 31561ae08745Sheppo * NOTE: not done as a switch() as type can be both ctrl+data 31571ae08745Sheppo */ 31581ae08745Sheppo if (msg->type & LDC_CTRL) { 31591ae08745Sheppo if (rv = i_ldc_ctrlmsg(ldcp, msg)) { 31601ae08745Sheppo if (rv == EAGAIN) 31611ae08745Sheppo continue; 31620a55fbb7Slm66018 rv = i_ldc_set_rx_head(ldcp, rx_tail); 31631ae08745Sheppo *sizep = 0; 31641ae08745Sheppo bytes_read = 0; 31651ae08745Sheppo break; 31661ae08745Sheppo } 31671ae08745Sheppo } 31681ae08745Sheppo 31691ae08745Sheppo /* process data ACKs */ 31701ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 3171*d10e4ef2Snarayan if (rv = i_ldc_process_data_ACK(ldcp, msg)) { 3172*d10e4ef2Snarayan *sizep = 0; 3173*d10e4ef2Snarayan bytes_read = 0; 3174*d10e4ef2Snarayan break; 3175*d10e4ef2Snarayan } 31761ae08745Sheppo } 31771ae08745Sheppo 31781ae08745Sheppo /* process data messages */ 31791ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_INFO)) { 31801ae08745Sheppo 31811ae08745Sheppo uint8_t *msgbuf = (uint8_t *)( 31821ae08745Sheppo (ldcp->mode == LDC_MODE_RELIABLE || 31831ae08745Sheppo ldcp->mode == LDC_MODE_STREAM) 31841ae08745Sheppo ? msg->rdata : msg->udata); 31851ae08745Sheppo 31861ae08745Sheppo D2(ldcp->id, 31871ae08745Sheppo "ldc_read: (0x%llx) received data msg\n", ldcp->id); 31881ae08745Sheppo 31891ae08745Sheppo /* get the packet length */ 31901ae08745Sheppo len = (msg->env & LDC_LEN_MASK); 31911ae08745Sheppo 31921ae08745Sheppo /* 31931ae08745Sheppo * FUTURE OPTIMIZATION: 31941ae08745Sheppo * dont need to set q head for every 31951ae08745Sheppo * packet we read just need to do this when 31961ae08745Sheppo * we are done or need to wait for more 31971ae08745Sheppo * mondos to make a full packet - this is 31981ae08745Sheppo * currently expensive. 31991ae08745Sheppo */ 32001ae08745Sheppo 3201*d10e4ef2Snarayan if (first_fragment == 0) { 32021ae08745Sheppo 32031ae08745Sheppo /* 32041ae08745Sheppo * first packets should always have the start 32051ae08745Sheppo * bit set (even for a single packet). If not 32061ae08745Sheppo * throw away the packet 32071ae08745Sheppo */ 32081ae08745Sheppo if (!(msg->env & LDC_FRAG_START)) { 32091ae08745Sheppo 32101ae08745Sheppo DWARN(DBG_ALL_LDCS, 32111ae08745Sheppo "ldc_read: (0x%llx) not start - " 32121ae08745Sheppo "frag=%x\n", ldcp->id, 32131ae08745Sheppo (msg->env) & LDC_FRAG_MASK); 32141ae08745Sheppo 32151ae08745Sheppo /* toss pkt, inc head, cont reading */ 32161ae08745Sheppo bytes_read = 0; 32171ae08745Sheppo target = target_bufp; 32181ae08745Sheppo curr_head = 32191ae08745Sheppo (curr_head + LDC_PACKET_SIZE) 32201ae08745Sheppo & q_size_mask; 32211ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, 32221ae08745Sheppo curr_head)) 32231ae08745Sheppo break; 32241ae08745Sheppo 32251ae08745Sheppo continue; 32261ae08745Sheppo } 32271ae08745Sheppo 3228*d10e4ef2Snarayan first_fragment = msg->seqid; 32291ae08745Sheppo } else { 32301ae08745Sheppo /* check to see if this is a pkt w/ START bit */ 32311ae08745Sheppo if (msg->env & LDC_FRAG_START) { 32321ae08745Sheppo DWARN(DBG_ALL_LDCS, 32331ae08745Sheppo "ldc_read:(0x%llx) unexpected pkt" 32341ae08745Sheppo " env=0x%x discarding %d bytes," 32351ae08745Sheppo " lastmsg=%d, currentmsg=%d\n", 32361ae08745Sheppo ldcp->id, msg->env&LDC_FRAG_MASK, 32371ae08745Sheppo bytes_read, ldcp->last_msg_rcd, 32381ae08745Sheppo msg->seqid); 32391ae08745Sheppo 32401ae08745Sheppo /* throw data we have read so far */ 32411ae08745Sheppo bytes_read = 0; 32421ae08745Sheppo target = target_bufp; 3243*d10e4ef2Snarayan first_fragment = msg->seqid; 32441ae08745Sheppo 32451ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, 32461ae08745Sheppo curr_head)) 32471ae08745Sheppo break; 32481ae08745Sheppo } 32491ae08745Sheppo } 32501ae08745Sheppo 32511ae08745Sheppo /* copy (next) pkt into buffer */ 32521ae08745Sheppo if (len <= (*sizep - bytes_read)) { 32531ae08745Sheppo bcopy(msgbuf, target, len); 32541ae08745Sheppo target += len; 32551ae08745Sheppo bytes_read += len; 32561ae08745Sheppo } else { 32571ae08745Sheppo /* 32581ae08745Sheppo * there is not enough space in the buffer to 32591ae08745Sheppo * read this pkt. throw message away & continue 32601ae08745Sheppo * reading data from queue 32611ae08745Sheppo */ 32621ae08745Sheppo DWARN(DBG_ALL_LDCS, 32631ae08745Sheppo "ldc_read: (0x%llx) buffer too small, " 32641ae08745Sheppo "head=0x%lx, expect=%d, got=%d\n", ldcp->id, 32651ae08745Sheppo curr_head, *sizep, bytes_read+len); 32661ae08745Sheppo 3267*d10e4ef2Snarayan first_fragment = 0; 32681ae08745Sheppo target = target_bufp; 32691ae08745Sheppo bytes_read = 0; 32701ae08745Sheppo 32711ae08745Sheppo /* throw away everything received so far */ 32721ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, curr_head)) 32731ae08745Sheppo break; 32741ae08745Sheppo 32751ae08745Sheppo /* continue reading remaining pkts */ 32761ae08745Sheppo continue; 32771ae08745Sheppo } 32781ae08745Sheppo } 32791ae08745Sheppo 32801ae08745Sheppo /* set the message id */ 32811ae08745Sheppo ldcp->last_msg_rcd = msg->seqid; 32821ae08745Sheppo 32831ae08745Sheppo /* move the head one position */ 32841ae08745Sheppo curr_head = (curr_head + LDC_PACKET_SIZE) & q_size_mask; 32851ae08745Sheppo 32861ae08745Sheppo if (msg->env & LDC_FRAG_STOP) { 32871ae08745Sheppo 32881ae08745Sheppo /* 32891ae08745Sheppo * All pkts that are part of this fragmented transfer 32901ae08745Sheppo * have been read or this was a single pkt read 32911ae08745Sheppo * or there was an error 32921ae08745Sheppo */ 32931ae08745Sheppo 32941ae08745Sheppo /* set the queue head */ 32951ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, curr_head)) 32961ae08745Sheppo bytes_read = 0; 32971ae08745Sheppo 32981ae08745Sheppo *sizep = bytes_read; 32991ae08745Sheppo 33001ae08745Sheppo break; 33011ae08745Sheppo } 33021ae08745Sheppo 33031ae08745Sheppo /* advance head if it is a DATA ACK */ 33041ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 33051ae08745Sheppo 33061ae08745Sheppo /* set the queue head */ 33071ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, curr_head)) { 33081ae08745Sheppo bytes_read = 0; 33091ae08745Sheppo break; 33101ae08745Sheppo } 33111ae08745Sheppo 33121ae08745Sheppo D2(ldcp->id, "ldc_read: (0x%llx) set ACK qhead 0x%llx", 33131ae08745Sheppo ldcp->id, curr_head); 33141ae08745Sheppo } 33151ae08745Sheppo 33161ae08745Sheppo } /* for (;;) */ 33171ae08745Sheppo 33181ae08745Sheppo 33191ae08745Sheppo /* 33201ae08745Sheppo * If useful data was read - Send msg ACK 33211ae08745Sheppo * OPTIMIZE: do not send ACK for all msgs - use some frequency 33221ae08745Sheppo */ 33231ae08745Sheppo if ((bytes_read > 0) && (ldcp->mode == LDC_MODE_RELIABLE || 33241ae08745Sheppo ldcp->mode == LDC_MODE_STREAM)) { 33251ae08745Sheppo 33261ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_DATA, LDC_ACK, 0); 3327*d10e4ef2Snarayan if (rv) { 33281ae08745Sheppo cmn_err(CE_NOTE, 33291ae08745Sheppo "ldc_read: (0x%lx) cannot send ACK\n", ldcp->id); 3330*d10e4ef2Snarayan 3331*d10e4ef2Snarayan /* if cannot send ACK - reset channel */ 3332*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 3333*d10e4ef2Snarayan i_ldc_reset(ldcp); 3334*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3335*d10e4ef2Snarayan rv = ECONNRESET; 33361ae08745Sheppo } 33371ae08745Sheppo } 33381ae08745Sheppo 33391ae08745Sheppo D2(ldcp->id, "ldc_read: (0x%llx) end size=%d", ldcp->id, *sizep); 33401ae08745Sheppo 33411ae08745Sheppo return (rv); 33421ae08745Sheppo } 33431ae08745Sheppo 33441ae08745Sheppo /* 33451ae08745Sheppo * Use underlying reliable packet mechanism to fetch 33461ae08745Sheppo * and buffer incoming packets so we can hand them back as 33471ae08745Sheppo * a basic byte stream. 33481ae08745Sheppo * 33491ae08745Sheppo * Enter and exit with ldcp->lock held by caller 33501ae08745Sheppo */ 33511ae08745Sheppo static int 33521ae08745Sheppo i_ldc_read_stream(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 33531ae08745Sheppo { 33541ae08745Sheppo int rv; 33551ae08745Sheppo size_t size; 33561ae08745Sheppo 33571ae08745Sheppo ASSERT(mutex_owned(&ldcp->lock)); 33581ae08745Sheppo 33591ae08745Sheppo D2(ldcp->id, "i_ldc_read_stream: (0x%llx) buffer size=%d", 33601ae08745Sheppo ldcp->id, *sizep); 33611ae08745Sheppo 33621ae08745Sheppo if (ldcp->stream_remains == 0) { 33631ae08745Sheppo size = ldcp->mtu; 33641ae08745Sheppo rv = i_ldc_read_packet(ldcp, 33651ae08745Sheppo (caddr_t)ldcp->stream_bufferp, &size); 33661ae08745Sheppo D2(ldcp->id, "i_ldc_read_stream: read packet (0x%llx) size=%d", 33671ae08745Sheppo ldcp->id, size); 33681ae08745Sheppo 33691ae08745Sheppo if (rv != 0) 33701ae08745Sheppo return (rv); 33711ae08745Sheppo 33721ae08745Sheppo ldcp->stream_remains = size; 33731ae08745Sheppo ldcp->stream_offset = 0; 33741ae08745Sheppo } 33751ae08745Sheppo 33761ae08745Sheppo size = MIN(ldcp->stream_remains, *sizep); 33771ae08745Sheppo 33781ae08745Sheppo bcopy(ldcp->stream_bufferp + ldcp->stream_offset, target_bufp, size); 33791ae08745Sheppo ldcp->stream_offset += size; 33801ae08745Sheppo ldcp->stream_remains -= size; 33811ae08745Sheppo 33821ae08745Sheppo D2(ldcp->id, "i_ldc_read_stream: (0x%llx) fill from buffer size=%d", 33831ae08745Sheppo ldcp->id, size); 33841ae08745Sheppo 33851ae08745Sheppo *sizep = size; 33861ae08745Sheppo return (0); 33871ae08745Sheppo } 33881ae08745Sheppo 33891ae08745Sheppo /* 33901ae08745Sheppo * Write specified amount of bytes to the channel 33911ae08745Sheppo * in multiple pkts of pkt_payload size. Each 33921ae08745Sheppo * packet is tagged with an unique packet ID in 33931ae08745Sheppo * the case of a reliable transport. 33941ae08745Sheppo * 33951ae08745Sheppo * On return, size contains the number of bytes written. 33961ae08745Sheppo */ 33971ae08745Sheppo int 33981ae08745Sheppo ldc_write(ldc_handle_t handle, caddr_t buf, size_t *sizep) 33991ae08745Sheppo { 34001ae08745Sheppo ldc_chan_t *ldcp; 34011ae08745Sheppo int rv = 0; 34021ae08745Sheppo 34031ae08745Sheppo if (handle == NULL) { 34041ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_write: invalid channel handle\n"); 34051ae08745Sheppo return (EINVAL); 34061ae08745Sheppo } 34071ae08745Sheppo ldcp = (ldc_chan_t *)handle; 34081ae08745Sheppo 3409*d10e4ef2Snarayan /* check if writes can occur */ 3410*d10e4ef2Snarayan if (!mutex_tryenter(&ldcp->tx_lock)) { 3411*d10e4ef2Snarayan /* 3412*d10e4ef2Snarayan * Could not get the lock - channel could 3413*d10e4ef2Snarayan * be in the process of being unconfigured 3414*d10e4ef2Snarayan * or reader has encountered an error 3415*d10e4ef2Snarayan */ 3416*d10e4ef2Snarayan return (EAGAIN); 3417*d10e4ef2Snarayan } 34181ae08745Sheppo 34191ae08745Sheppo /* check if non-zero data to write */ 34201ae08745Sheppo if (buf == NULL || sizep == NULL) { 34211ae08745Sheppo DWARN(ldcp->id, "ldc_write: (0x%llx) invalid data write\n", 34221ae08745Sheppo ldcp->id); 3423*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 34241ae08745Sheppo return (EINVAL); 34251ae08745Sheppo } 34261ae08745Sheppo 34271ae08745Sheppo if (*sizep == 0) { 34281ae08745Sheppo DWARN(ldcp->id, "ldc_write: (0x%llx) write size of zero\n", 34291ae08745Sheppo ldcp->id); 3430*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 34311ae08745Sheppo return (0); 34321ae08745Sheppo } 34331ae08745Sheppo 34341ae08745Sheppo /* Check if channel is UP for data exchange */ 34351ae08745Sheppo if (ldcp->tstate != TS_UP) { 34361ae08745Sheppo DWARN(ldcp->id, 34371ae08745Sheppo "ldc_write: (0x%llx) channel is not in UP state\n", 34381ae08745Sheppo ldcp->id); 34391ae08745Sheppo *sizep = 0; 34401ae08745Sheppo rv = ECONNRESET; 34411ae08745Sheppo } else { 34421ae08745Sheppo rv = ldcp->write_p(ldcp, buf, sizep); 34431ae08745Sheppo } 34441ae08745Sheppo 3445*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 34461ae08745Sheppo 34471ae08745Sheppo return (rv); 34481ae08745Sheppo } 34491ae08745Sheppo 34501ae08745Sheppo /* 34511ae08745Sheppo * Write a raw packet to the channel 34521ae08745Sheppo * On return, size contains the number of bytes written. 34531ae08745Sheppo */ 34541ae08745Sheppo static int 34551ae08745Sheppo i_ldc_write_raw(ldc_chan_t *ldcp, caddr_t buf, size_t *sizep) 34561ae08745Sheppo { 34571ae08745Sheppo ldc_msg_t *ldcmsg; 34581ae08745Sheppo uint64_t tx_head, tx_tail, new_tail; 34591ae08745Sheppo int rv = 0; 34601ae08745Sheppo size_t size; 34611ae08745Sheppo 3462*d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 34631ae08745Sheppo ASSERT(ldcp->mode == LDC_MODE_RAW); 34641ae08745Sheppo 34651ae08745Sheppo size = *sizep; 34661ae08745Sheppo 34671ae08745Sheppo /* 34681ae08745Sheppo * Check to see if the packet size is less than or 34691ae08745Sheppo * equal to packet size support in raw mode 34701ae08745Sheppo */ 34711ae08745Sheppo if (size > ldcp->pkt_payload) { 34721ae08745Sheppo DWARN(ldcp->id, 34731ae08745Sheppo "ldc_write: (0x%llx) invalid size (0x%llx) for RAW mode\n", 34741ae08745Sheppo ldcp->id, *sizep); 34751ae08745Sheppo *sizep = 0; 34761ae08745Sheppo return (EMSGSIZE); 34771ae08745Sheppo } 34781ae08745Sheppo 34791ae08745Sheppo /* get the qptrs for the tx queue */ 34801ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 34811ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 34821ae08745Sheppo if (rv != 0) { 34831ae08745Sheppo cmn_err(CE_WARN, 34841ae08745Sheppo "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp->id); 34851ae08745Sheppo *sizep = 0; 34861ae08745Sheppo return (EIO); 34871ae08745Sheppo } 34881ae08745Sheppo 34891ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN || 34901ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 34911ae08745Sheppo DWARN(ldcp->id, 34921ae08745Sheppo "ldc_write: (0x%llx) channel down/reset\n", ldcp->id); 3493*d10e4ef2Snarayan 34941ae08745Sheppo *sizep = 0; 3495*d10e4ef2Snarayan if (mutex_tryenter(&ldcp->lock)) { 3496*d10e4ef2Snarayan i_ldc_reset(ldcp); 3497*d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3498*d10e4ef2Snarayan } else { 3499*d10e4ef2Snarayan /* 3500*d10e4ef2Snarayan * Release Tx lock, and then reacquire channel 3501*d10e4ef2Snarayan * and Tx lock in correct order 3502*d10e4ef2Snarayan */ 3503*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3504*d10e4ef2Snarayan mutex_enter(&ldcp->lock); 3505*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 3506*d10e4ef2Snarayan i_ldc_reset(ldcp); 3507*d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3508*d10e4ef2Snarayan } 35091ae08745Sheppo return (ECONNRESET); 35101ae08745Sheppo } 35111ae08745Sheppo 35121ae08745Sheppo tx_tail = ldcp->tx_tail; 35131ae08745Sheppo tx_head = ldcp->tx_head; 35141ae08745Sheppo new_tail = (tx_tail + LDC_PACKET_SIZE) & 35151ae08745Sheppo ((ldcp->tx_q_entries-1) << LDC_PACKET_SHIFT); 35161ae08745Sheppo 35171ae08745Sheppo if (new_tail == tx_head) { 35181ae08745Sheppo DWARN(DBG_ALL_LDCS, 35191ae08745Sheppo "ldc_write: (0x%llx) TX queue is full\n", ldcp->id); 35201ae08745Sheppo *sizep = 0; 35211ae08745Sheppo return (EWOULDBLOCK); 35221ae08745Sheppo } 35231ae08745Sheppo 35241ae08745Sheppo D2(ldcp->id, "ldc_write: (0x%llx) start xfer size=%d", 35251ae08745Sheppo ldcp->id, size); 35261ae08745Sheppo 35271ae08745Sheppo /* Send the data now */ 35281ae08745Sheppo ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 35291ae08745Sheppo 35301ae08745Sheppo /* copy the data into pkt */ 35311ae08745Sheppo bcopy((uint8_t *)buf, ldcmsg, size); 35321ae08745Sheppo 35331ae08745Sheppo /* increment tail */ 35341ae08745Sheppo tx_tail = new_tail; 35351ae08745Sheppo 35361ae08745Sheppo /* 35371ae08745Sheppo * All packets have been copied into the TX queue 35381ae08745Sheppo * update the tail ptr in the HV 35391ae08745Sheppo */ 35401ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 35411ae08745Sheppo if (rv) { 35421ae08745Sheppo if (rv == EWOULDBLOCK) { 35431ae08745Sheppo DWARN(ldcp->id, "ldc_write: (0x%llx) write timed out\n", 35441ae08745Sheppo ldcp->id); 35451ae08745Sheppo *sizep = 0; 35461ae08745Sheppo return (EWOULDBLOCK); 35471ae08745Sheppo } 35481ae08745Sheppo 35491ae08745Sheppo *sizep = 0; 3550*d10e4ef2Snarayan if (mutex_tryenter(&ldcp->lock)) { 3551*d10e4ef2Snarayan i_ldc_reset(ldcp); 3552*d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3553*d10e4ef2Snarayan } else { 3554*d10e4ef2Snarayan /* 3555*d10e4ef2Snarayan * Release Tx lock, and then reacquire channel 3556*d10e4ef2Snarayan * and Tx lock in correct order 3557*d10e4ef2Snarayan */ 3558*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3559*d10e4ef2Snarayan mutex_enter(&ldcp->lock); 3560*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 3561*d10e4ef2Snarayan i_ldc_reset(ldcp); 3562*d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3563*d10e4ef2Snarayan } 35641ae08745Sheppo return (ECONNRESET); 35651ae08745Sheppo } 35661ae08745Sheppo 35671ae08745Sheppo ldcp->tx_tail = tx_tail; 35681ae08745Sheppo *sizep = size; 35691ae08745Sheppo 35701ae08745Sheppo D2(ldcp->id, "ldc_write: (0x%llx) end xfer size=%d", ldcp->id, size); 35711ae08745Sheppo 35721ae08745Sheppo return (rv); 35731ae08745Sheppo } 35741ae08745Sheppo 35751ae08745Sheppo 35761ae08745Sheppo /* 35771ae08745Sheppo * Write specified amount of bytes to the channel 35781ae08745Sheppo * in multiple pkts of pkt_payload size. Each 35791ae08745Sheppo * packet is tagged with an unique packet ID in 35801ae08745Sheppo * the case of a reliable transport. 35811ae08745Sheppo * 35821ae08745Sheppo * On return, size contains the number of bytes written. 35831ae08745Sheppo * This function needs to ensure that the write size is < MTU size 35841ae08745Sheppo */ 35851ae08745Sheppo static int 35861ae08745Sheppo i_ldc_write_packet(ldc_chan_t *ldcp, caddr_t buf, size_t *size) 35871ae08745Sheppo { 35881ae08745Sheppo ldc_msg_t *ldcmsg; 35891ae08745Sheppo uint64_t tx_head, tx_tail, new_tail, start; 35901ae08745Sheppo uint64_t txq_size_mask, numavail; 35911ae08745Sheppo uint8_t *msgbuf, *source = (uint8_t *)buf; 35921ae08745Sheppo size_t len, bytes_written = 0, remaining; 35931ae08745Sheppo int rv; 35941ae08745Sheppo uint32_t curr_seqid; 35951ae08745Sheppo 3596*d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 35971ae08745Sheppo 35981ae08745Sheppo ASSERT(ldcp->mode == LDC_MODE_RELIABLE || 35991ae08745Sheppo ldcp->mode == LDC_MODE_UNRELIABLE || 36001ae08745Sheppo ldcp->mode == LDC_MODE_STREAM); 36011ae08745Sheppo 36021ae08745Sheppo /* compute mask for increment */ 36031ae08745Sheppo txq_size_mask = (ldcp->tx_q_entries - 1) << LDC_PACKET_SHIFT; 36041ae08745Sheppo 36051ae08745Sheppo /* get the qptrs for the tx queue */ 36061ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 36071ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 36081ae08745Sheppo if (rv != 0) { 36091ae08745Sheppo cmn_err(CE_WARN, 36101ae08745Sheppo "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp->id); 36111ae08745Sheppo *size = 0; 36121ae08745Sheppo return (EIO); 36131ae08745Sheppo } 36141ae08745Sheppo 36151ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN || 36161ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 36171ae08745Sheppo DWARN(ldcp->id, 36181ae08745Sheppo "ldc_write: (0x%llx) channel down/reset\n", ldcp->id); 36191ae08745Sheppo *size = 0; 3620*d10e4ef2Snarayan if (mutex_tryenter(&ldcp->lock)) { 36211ae08745Sheppo i_ldc_reset(ldcp); 3622*d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3623*d10e4ef2Snarayan } else { 3624*d10e4ef2Snarayan /* 3625*d10e4ef2Snarayan * Release Tx lock, and then reacquire channel 3626*d10e4ef2Snarayan * and Tx lock in correct order 3627*d10e4ef2Snarayan */ 3628*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3629*d10e4ef2Snarayan mutex_enter(&ldcp->lock); 3630*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 3631*d10e4ef2Snarayan i_ldc_reset(ldcp); 3632*d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3633*d10e4ef2Snarayan } 36341ae08745Sheppo return (ECONNRESET); 36351ae08745Sheppo } 36361ae08745Sheppo 36371ae08745Sheppo tx_tail = ldcp->tx_tail; 36381ae08745Sheppo new_tail = (tx_tail + LDC_PACKET_SIZE) % 36391ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 36401ae08745Sheppo 36411ae08745Sheppo /* 36421ae08745Sheppo * Transport mode determines whether we use HV Tx head or the 36431ae08745Sheppo * private protocol head (corresponding to last ACKd pkt) for 36441ae08745Sheppo * determining how much we can write 36451ae08745Sheppo */ 36461ae08745Sheppo tx_head = (ldcp->mode == LDC_MODE_RELIABLE || 36471ae08745Sheppo ldcp->mode == LDC_MODE_STREAM) 36481ae08745Sheppo ? ldcp->tx_ackd_head : ldcp->tx_head; 36491ae08745Sheppo if (new_tail == tx_head) { 36501ae08745Sheppo DWARN(DBG_ALL_LDCS, 36511ae08745Sheppo "ldc_write: (0x%llx) TX queue is full\n", ldcp->id); 36521ae08745Sheppo *size = 0; 36531ae08745Sheppo return (EWOULDBLOCK); 36541ae08745Sheppo } 36551ae08745Sheppo 36561ae08745Sheppo /* 36571ae08745Sheppo * Make sure that the LDC Tx queue has enough space 36581ae08745Sheppo */ 36591ae08745Sheppo numavail = (tx_head >> LDC_PACKET_SHIFT) - (tx_tail >> LDC_PACKET_SHIFT) 36601ae08745Sheppo + ldcp->tx_q_entries - 1; 36611ae08745Sheppo numavail %= ldcp->tx_q_entries; 36621ae08745Sheppo 36631ae08745Sheppo if (*size > (numavail * ldcp->pkt_payload)) { 36641ae08745Sheppo DWARN(DBG_ALL_LDCS, 36651ae08745Sheppo "ldc_write: (0x%llx) TX queue has no space\n", ldcp->id); 36661ae08745Sheppo return (EWOULDBLOCK); 36671ae08745Sheppo } 36681ae08745Sheppo 36691ae08745Sheppo D2(ldcp->id, "ldc_write: (0x%llx) start xfer size=%d", 36701ae08745Sheppo ldcp->id, *size); 36711ae08745Sheppo 36721ae08745Sheppo /* Send the data now */ 36731ae08745Sheppo bytes_written = 0; 36741ae08745Sheppo curr_seqid = ldcp->last_msg_snt; 36751ae08745Sheppo start = tx_tail; 36761ae08745Sheppo 36771ae08745Sheppo while (*size > bytes_written) { 36781ae08745Sheppo 36791ae08745Sheppo ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 36801ae08745Sheppo 36811ae08745Sheppo msgbuf = (uint8_t *)((ldcp->mode == LDC_MODE_RELIABLE || 36821ae08745Sheppo ldcp->mode == LDC_MODE_STREAM) 36831ae08745Sheppo ? ldcmsg->rdata : ldcmsg->udata); 36841ae08745Sheppo 36851ae08745Sheppo ldcmsg->type = LDC_DATA; 36861ae08745Sheppo ldcmsg->stype = LDC_INFO; 36871ae08745Sheppo ldcmsg->ctrl = 0; 36881ae08745Sheppo 36891ae08745Sheppo remaining = *size - bytes_written; 36901ae08745Sheppo len = min(ldcp->pkt_payload, remaining); 36911ae08745Sheppo ldcmsg->env = (uint8_t)len; 36921ae08745Sheppo 36931ae08745Sheppo curr_seqid++; 36941ae08745Sheppo ldcmsg->seqid = curr_seqid; 36951ae08745Sheppo 36961ae08745Sheppo DUMP_LDC_PKT(ldcp, "ldc_write snd data", (uint64_t)ldcmsg); 36971ae08745Sheppo 36981ae08745Sheppo /* copy the data into pkt */ 36991ae08745Sheppo bcopy(source, msgbuf, len); 37001ae08745Sheppo 37011ae08745Sheppo source += len; 37021ae08745Sheppo bytes_written += len; 37031ae08745Sheppo 37041ae08745Sheppo /* increment tail */ 37051ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) & txq_size_mask; 37061ae08745Sheppo 37071ae08745Sheppo ASSERT(tx_tail != tx_head); 37081ae08745Sheppo } 37091ae08745Sheppo 37101ae08745Sheppo /* Set the start and stop bits */ 37111ae08745Sheppo ldcmsg->env |= LDC_FRAG_STOP; 37121ae08745Sheppo ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + start); 37131ae08745Sheppo ldcmsg->env |= LDC_FRAG_START; 37141ae08745Sheppo 37151ae08745Sheppo /* 37161ae08745Sheppo * All packets have been copied into the TX queue 37171ae08745Sheppo * update the tail ptr in the HV 37181ae08745Sheppo */ 37191ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 37201ae08745Sheppo if (rv == 0) { 37211ae08745Sheppo ldcp->tx_tail = tx_tail; 37221ae08745Sheppo ldcp->last_msg_snt = curr_seqid; 37231ae08745Sheppo *size = bytes_written; 37241ae08745Sheppo } else { 37251ae08745Sheppo int rv2; 37261ae08745Sheppo 37271ae08745Sheppo if (rv != EWOULDBLOCK) { 37281ae08745Sheppo *size = 0; 3729*d10e4ef2Snarayan if (mutex_tryenter(&ldcp->lock)) { 3730*d10e4ef2Snarayan i_ldc_reset(ldcp); 3731*d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3732*d10e4ef2Snarayan } else { 3733*d10e4ef2Snarayan /* 3734*d10e4ef2Snarayan * Release Tx lock, and then reacquire channel 3735*d10e4ef2Snarayan * and Tx lock in correct order 3736*d10e4ef2Snarayan */ 3737*d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3738*d10e4ef2Snarayan mutex_enter(&ldcp->lock); 3739*d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 3740*d10e4ef2Snarayan i_ldc_reset(ldcp); 3741*d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3742*d10e4ef2Snarayan } 37431ae08745Sheppo return (ECONNRESET); 37441ae08745Sheppo } 37451ae08745Sheppo 37461ae08745Sheppo DWARN(ldcp->id, "hv_tx_set_tail returns 0x%x (head 0x%x, " 37471ae08745Sheppo "old tail 0x%x, new tail 0x%x, qsize=0x%x)\n", 37481ae08745Sheppo rv, ldcp->tx_head, ldcp->tx_tail, tx_tail, 37491ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 37501ae08745Sheppo 37511ae08745Sheppo rv2 = hv_ldc_tx_get_state(ldcp->id, 37521ae08745Sheppo &tx_head, &tx_tail, &ldcp->link_state); 37531ae08745Sheppo 37541ae08745Sheppo DWARN(ldcp->id, "hv_ldc_tx_get_state returns 0x%x " 37551ae08745Sheppo "(head 0x%x, tail 0x%x state 0x%x)\n", 37561ae08745Sheppo rv2, tx_head, tx_tail, ldcp->link_state); 37571ae08745Sheppo 37581ae08745Sheppo *size = 0; 37591ae08745Sheppo } 37601ae08745Sheppo 37611ae08745Sheppo D2(ldcp->id, "ldc_write: (0x%llx) end xfer size=%d", ldcp->id, *size); 37621ae08745Sheppo 37631ae08745Sheppo return (rv); 37641ae08745Sheppo } 37651ae08745Sheppo 37661ae08745Sheppo /* 37671ae08745Sheppo * Write specified amount of bytes to the channel 37681ae08745Sheppo * in multiple pkts of pkt_payload size. Each 37691ae08745Sheppo * packet is tagged with an unique packet ID in 37701ae08745Sheppo * the case of a reliable transport. 37711ae08745Sheppo * 37721ae08745Sheppo * On return, size contains the number of bytes written. 37731ae08745Sheppo * This function needs to ensure that the write size is < MTU size 37741ae08745Sheppo */ 37751ae08745Sheppo static int 37761ae08745Sheppo i_ldc_write_stream(ldc_chan_t *ldcp, caddr_t buf, size_t *sizep) 37771ae08745Sheppo { 3778*d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 37791ae08745Sheppo ASSERT(ldcp->mode == LDC_MODE_STREAM); 37801ae08745Sheppo 37811ae08745Sheppo /* Truncate packet to max of MTU size */ 37821ae08745Sheppo if (*sizep > ldcp->mtu) *sizep = ldcp->mtu; 37831ae08745Sheppo return (i_ldc_write_packet(ldcp, buf, sizep)); 37841ae08745Sheppo } 37851ae08745Sheppo 37861ae08745Sheppo 37871ae08745Sheppo /* 37881ae08745Sheppo * Interfaces for channel nexus to register/unregister with LDC module 37891ae08745Sheppo * The nexus will register functions to be used to register individual 37901ae08745Sheppo * channels with the nexus and enable interrupts for the channels 37911ae08745Sheppo */ 37921ae08745Sheppo int 37931ae08745Sheppo ldc_register(ldc_cnex_t *cinfo) 37941ae08745Sheppo { 37951ae08745Sheppo ldc_chan_t *ldcp; 37961ae08745Sheppo 37971ae08745Sheppo if (cinfo == NULL || cinfo->dip == NULL || 37981ae08745Sheppo cinfo->reg_chan == NULL || cinfo->unreg_chan == NULL || 37991ae08745Sheppo cinfo->add_intr == NULL || cinfo->rem_intr == NULL || 38001ae08745Sheppo cinfo->clr_intr == NULL) { 38011ae08745Sheppo 38021ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_register: invalid nexus info\n"); 38031ae08745Sheppo return (EINVAL); 38041ae08745Sheppo } 38051ae08745Sheppo 38061ae08745Sheppo mutex_enter(&ldcssp->lock); 38071ae08745Sheppo 38081ae08745Sheppo /* nexus registration */ 38091ae08745Sheppo ldcssp->cinfo.dip = cinfo->dip; 38101ae08745Sheppo ldcssp->cinfo.reg_chan = cinfo->reg_chan; 38111ae08745Sheppo ldcssp->cinfo.unreg_chan = cinfo->unreg_chan; 38121ae08745Sheppo ldcssp->cinfo.add_intr = cinfo->add_intr; 38131ae08745Sheppo ldcssp->cinfo.rem_intr = cinfo->rem_intr; 38141ae08745Sheppo ldcssp->cinfo.clr_intr = cinfo->clr_intr; 38151ae08745Sheppo 38161ae08745Sheppo /* register any channels that might have been previously initialized */ 38171ae08745Sheppo ldcp = ldcssp->chan_list; 38181ae08745Sheppo while (ldcp) { 38191ae08745Sheppo if ((ldcp->tstate & TS_QCONF_RDY) && 38201ae08745Sheppo (ldcp->tstate & TS_CNEX_RDY) == 0) 38211ae08745Sheppo (void) i_ldc_register_channel(ldcp); 38221ae08745Sheppo 38231ae08745Sheppo ldcp = ldcp->next; 38241ae08745Sheppo } 38251ae08745Sheppo 38261ae08745Sheppo mutex_exit(&ldcssp->lock); 38271ae08745Sheppo 38281ae08745Sheppo return (0); 38291ae08745Sheppo } 38301ae08745Sheppo 38311ae08745Sheppo int 38321ae08745Sheppo ldc_unregister(ldc_cnex_t *cinfo) 38331ae08745Sheppo { 38341ae08745Sheppo if (cinfo == NULL || cinfo->dip == NULL) { 38351ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_unregister: invalid nexus info\n"); 38361ae08745Sheppo return (EINVAL); 38371ae08745Sheppo } 38381ae08745Sheppo 38391ae08745Sheppo mutex_enter(&ldcssp->lock); 38401ae08745Sheppo 38411ae08745Sheppo if (cinfo->dip != ldcssp->cinfo.dip) { 38421ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_unregister: invalid dip\n"); 38431ae08745Sheppo mutex_exit(&ldcssp->lock); 38441ae08745Sheppo return (EINVAL); 38451ae08745Sheppo } 38461ae08745Sheppo 38471ae08745Sheppo /* nexus unregister */ 38481ae08745Sheppo ldcssp->cinfo.dip = NULL; 38491ae08745Sheppo ldcssp->cinfo.reg_chan = NULL; 38501ae08745Sheppo ldcssp->cinfo.unreg_chan = NULL; 38511ae08745Sheppo ldcssp->cinfo.add_intr = NULL; 38521ae08745Sheppo ldcssp->cinfo.rem_intr = NULL; 38531ae08745Sheppo ldcssp->cinfo.clr_intr = NULL; 38541ae08745Sheppo 38551ae08745Sheppo mutex_exit(&ldcssp->lock); 38561ae08745Sheppo 38571ae08745Sheppo return (0); 38581ae08745Sheppo } 38591ae08745Sheppo 38601ae08745Sheppo 38611ae08745Sheppo /* ------------------------------------------------------------------------- */ 38621ae08745Sheppo 38631ae08745Sheppo /* 38641ae08745Sheppo * Allocate a memory handle for the channel and link it into the list 38651ae08745Sheppo * Also choose which memory table to use if this is the first handle 38661ae08745Sheppo * being assigned to this channel 38671ae08745Sheppo */ 38681ae08745Sheppo int 38691ae08745Sheppo ldc_mem_alloc_handle(ldc_handle_t handle, ldc_mem_handle_t *mhandle) 38701ae08745Sheppo { 38711ae08745Sheppo ldc_chan_t *ldcp; 38721ae08745Sheppo ldc_mhdl_t *mhdl; 38731ae08745Sheppo int rv; 38741ae08745Sheppo 38751ae08745Sheppo if (handle == NULL) { 38761ae08745Sheppo DWARN(DBG_ALL_LDCS, 38771ae08745Sheppo "ldc_mem_alloc_handle: invalid channel handle\n"); 38781ae08745Sheppo return (EINVAL); 38791ae08745Sheppo } 38801ae08745Sheppo ldcp = (ldc_chan_t *)handle; 38811ae08745Sheppo 38821ae08745Sheppo mutex_enter(&ldcp->lock); 38831ae08745Sheppo 38841ae08745Sheppo /* check to see if channel is initalized */ 38851ae08745Sheppo if (ldcp->tstate < TS_INIT) { 38861ae08745Sheppo DWARN(ldcp->id, 38871ae08745Sheppo "ldc_mem_alloc_handle: (0x%llx) channel not initialized\n", 38881ae08745Sheppo ldcp->id); 38891ae08745Sheppo mutex_exit(&ldcp->lock); 38901ae08745Sheppo return (EINVAL); 38911ae08745Sheppo } 38921ae08745Sheppo 38931ae08745Sheppo /* 38941ae08745Sheppo * If this channel is allocating a mem handle for the 38951ae08745Sheppo * first time allocate it a memory map table and initialize it 38961ae08745Sheppo */ 38971ae08745Sheppo if (ldcp->mtbl == NULL) { 38981ae08745Sheppo 38991ae08745Sheppo ldc_mtbl_t *mtbl; 39001ae08745Sheppo 39011ae08745Sheppo /* Allocate and initialize the map table structure */ 39021ae08745Sheppo mtbl = kmem_zalloc(sizeof (ldc_mtbl_t), KM_SLEEP); 39031ae08745Sheppo mtbl->size = MTBL_MAX_SIZE; 39041ae08745Sheppo mtbl->num_entries = mtbl->num_avail = 39051ae08745Sheppo (MTBL_MAX_SIZE/sizeof (ldc_mte_slot_t)); 39061ae08745Sheppo mtbl->next_entry = NULL; 39071ae08745Sheppo 39081ae08745Sheppo /* Allocate the table itself */ 39091ae08745Sheppo mtbl->table = (ldc_mte_slot_t *) 39101ae08745Sheppo contig_mem_alloc_align(mtbl->size, MMU_PAGESIZE); 39111ae08745Sheppo if (mtbl->table == NULL) { 39121ae08745Sheppo cmn_err(CE_WARN, 39131ae08745Sheppo "ldc_mem_alloc_handle: (0x%lx) error allocating " 39141ae08745Sheppo "table memory", ldcp->id); 39151ae08745Sheppo kmem_free(mtbl, sizeof (ldc_mtbl_t)); 39161ae08745Sheppo mutex_exit(&ldcp->lock); 39171ae08745Sheppo return (ENOMEM); 39181ae08745Sheppo } 39191ae08745Sheppo 39201ae08745Sheppo /* zero out the memory */ 39211ae08745Sheppo bzero(mtbl->table, mtbl->size); 39221ae08745Sheppo 39231ae08745Sheppo /* initialize the lock */ 39241ae08745Sheppo mutex_init(&mtbl->lock, NULL, MUTEX_DRIVER, NULL); 39251ae08745Sheppo 39261ae08745Sheppo /* register table for this channel */ 39271ae08745Sheppo rv = hv_ldc_set_map_table(ldcp->id, 39281ae08745Sheppo va_to_pa(mtbl->table), mtbl->num_entries); 39291ae08745Sheppo if (rv != 0) { 39301ae08745Sheppo cmn_err(CE_WARN, 39311ae08745Sheppo "ldc_mem_alloc_handle: (0x%lx) err %d mapping tbl", 39321ae08745Sheppo ldcp->id, rv); 39331ae08745Sheppo contig_mem_free(mtbl->table, mtbl->size); 39341ae08745Sheppo mutex_destroy(&mtbl->lock); 39351ae08745Sheppo kmem_free(mtbl, sizeof (ldc_mtbl_t)); 39361ae08745Sheppo mutex_exit(&ldcp->lock); 39371ae08745Sheppo return (EIO); 39381ae08745Sheppo } 39391ae08745Sheppo 39401ae08745Sheppo ldcp->mtbl = mtbl; 39411ae08745Sheppo 39421ae08745Sheppo D1(ldcp->id, 39431ae08745Sheppo "ldc_mem_alloc_handle: (0x%llx) alloc'd map table 0x%llx\n", 39441ae08745Sheppo ldcp->id, ldcp->mtbl->table); 39451ae08745Sheppo } 39461ae08745Sheppo 39471ae08745Sheppo /* allocate handle for channel */ 39481ae08745Sheppo mhdl = kmem_zalloc(sizeof (ldc_mhdl_t), KM_SLEEP); 39491ae08745Sheppo 39501ae08745Sheppo /* initialize the lock */ 39511ae08745Sheppo mutex_init(&mhdl->lock, NULL, MUTEX_DRIVER, NULL); 39521ae08745Sheppo 39531ae08745Sheppo mhdl->status = LDC_UNBOUND; 39541ae08745Sheppo mhdl->ldcp = ldcp; 39551ae08745Sheppo 39561ae08745Sheppo /* insert memory handle (@ head) into list */ 39571ae08745Sheppo if (ldcp->mhdl_list == NULL) { 39581ae08745Sheppo ldcp->mhdl_list = mhdl; 39591ae08745Sheppo mhdl->next = NULL; 39601ae08745Sheppo } else { 39611ae08745Sheppo /* insert @ head */ 39621ae08745Sheppo mhdl->next = ldcp->mhdl_list; 39631ae08745Sheppo ldcp->mhdl_list = mhdl; 39641ae08745Sheppo } 39651ae08745Sheppo 39661ae08745Sheppo /* return the handle */ 39671ae08745Sheppo *mhandle = (ldc_mem_handle_t)mhdl; 39681ae08745Sheppo 39691ae08745Sheppo mutex_exit(&ldcp->lock); 39701ae08745Sheppo 39711ae08745Sheppo D1(ldcp->id, "ldc_mem_alloc_handle: (0x%llx) allocated handle 0x%llx\n", 39721ae08745Sheppo ldcp->id, mhdl); 39731ae08745Sheppo 39741ae08745Sheppo return (0); 39751ae08745Sheppo } 39761ae08745Sheppo 39771ae08745Sheppo /* 39781ae08745Sheppo * Free memory handle for the channel and unlink it from the list 39791ae08745Sheppo */ 39801ae08745Sheppo int 39811ae08745Sheppo ldc_mem_free_handle(ldc_mem_handle_t mhandle) 39821ae08745Sheppo { 39831ae08745Sheppo ldc_mhdl_t *mhdl, *phdl; 39841ae08745Sheppo ldc_chan_t *ldcp; 39851ae08745Sheppo 39861ae08745Sheppo if (mhandle == NULL) { 39871ae08745Sheppo DWARN(DBG_ALL_LDCS, 39881ae08745Sheppo "ldc_mem_free_handle: invalid memory handle\n"); 39891ae08745Sheppo return (EINVAL); 39901ae08745Sheppo } 39911ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 39921ae08745Sheppo 39931ae08745Sheppo mutex_enter(&mhdl->lock); 39941ae08745Sheppo 39951ae08745Sheppo ldcp = mhdl->ldcp; 39961ae08745Sheppo 39971ae08745Sheppo if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) { 39981ae08745Sheppo DWARN(ldcp->id, 39991ae08745Sheppo "ldc_mem_free_handle: cannot free, 0x%llx hdl bound\n", 40001ae08745Sheppo mhdl); 40011ae08745Sheppo mutex_exit(&mhdl->lock); 40021ae08745Sheppo return (EINVAL); 40031ae08745Sheppo } 40041ae08745Sheppo mutex_exit(&mhdl->lock); 40051ae08745Sheppo 40061ae08745Sheppo mutex_enter(&ldcp->mlist_lock); 40071ae08745Sheppo 40081ae08745Sheppo phdl = ldcp->mhdl_list; 40091ae08745Sheppo 40101ae08745Sheppo /* first handle */ 40111ae08745Sheppo if (phdl == mhdl) { 40121ae08745Sheppo ldcp->mhdl_list = mhdl->next; 40131ae08745Sheppo mutex_destroy(&mhdl->lock); 40141ae08745Sheppo kmem_free(mhdl, sizeof (ldc_mhdl_t)); 40151ae08745Sheppo D1(ldcp->id, 40161ae08745Sheppo "ldc_mem_free_handle: (0x%llx) freed handle 0x%llx\n", 40171ae08745Sheppo ldcp->id, mhdl); 40181ae08745Sheppo } else { 40191ae08745Sheppo /* walk the list - unlink and free */ 40201ae08745Sheppo while (phdl != NULL) { 40211ae08745Sheppo if (phdl->next == mhdl) { 40221ae08745Sheppo phdl->next = mhdl->next; 40231ae08745Sheppo mutex_destroy(&mhdl->lock); 40241ae08745Sheppo kmem_free(mhdl, sizeof (ldc_mhdl_t)); 40251ae08745Sheppo D1(ldcp->id, 40261ae08745Sheppo "ldc_mem_free_handle: (0x%llx) freed " 40271ae08745Sheppo "handle 0x%llx\n", ldcp->id, mhdl); 40281ae08745Sheppo break; 40291ae08745Sheppo } 40301ae08745Sheppo phdl = phdl->next; 40311ae08745Sheppo } 40321ae08745Sheppo } 40331ae08745Sheppo 40341ae08745Sheppo if (phdl == NULL) { 40351ae08745Sheppo DWARN(ldcp->id, 40361ae08745Sheppo "ldc_mem_free_handle: invalid handle 0x%llx\n", mhdl); 40371ae08745Sheppo mutex_exit(&ldcp->mlist_lock); 40381ae08745Sheppo return (EINVAL); 40391ae08745Sheppo } 40401ae08745Sheppo 40411ae08745Sheppo mutex_exit(&ldcp->mlist_lock); 40421ae08745Sheppo 40431ae08745Sheppo return (0); 40441ae08745Sheppo } 40451ae08745Sheppo 40461ae08745Sheppo /* 40471ae08745Sheppo * Bind a memory handle to a virtual address. 40481ae08745Sheppo * The virtual address is converted to the corresponding real addresses. 40491ae08745Sheppo * Returns pointer to the first ldc_mem_cookie and the total number 40501ae08745Sheppo * of cookies for this virtual address. Other cookies can be obtained 40511ae08745Sheppo * using the ldc_mem_nextcookie() call. If the pages are stored in 40521ae08745Sheppo * consecutive locations in the table, a single cookie corresponding to 40531ae08745Sheppo * the first location is returned. The cookie size spans all the entries. 40541ae08745Sheppo * 40551ae08745Sheppo * If the VA corresponds to a page that is already being exported, reuse 40561ae08745Sheppo * the page and do not export it again. Bump the page's use count. 40571ae08745Sheppo */ 40581ae08745Sheppo int 40591ae08745Sheppo ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr, size_t len, 40601ae08745Sheppo uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount) 40611ae08745Sheppo { 40621ae08745Sheppo ldc_mhdl_t *mhdl; 40631ae08745Sheppo ldc_chan_t *ldcp; 40641ae08745Sheppo ldc_mtbl_t *mtbl; 40651ae08745Sheppo ldc_memseg_t *memseg; 40661ae08745Sheppo ldc_mte_t tmp_mte; 40671ae08745Sheppo uint64_t index, prev_index = 0; 40681ae08745Sheppo int64_t cookie_idx; 40691ae08745Sheppo uintptr_t raddr, ra_aligned; 40701ae08745Sheppo uint64_t psize, poffset, v_offset; 40711ae08745Sheppo uint64_t pg_shift, pg_size, pg_size_code, pg_mask; 40721ae08745Sheppo pgcnt_t npages; 40731ae08745Sheppo caddr_t v_align, addr; 40741ae08745Sheppo int i; 40751ae08745Sheppo 40761ae08745Sheppo if (mhandle == NULL) { 40771ae08745Sheppo DWARN(DBG_ALL_LDCS, 40781ae08745Sheppo "ldc_mem_bind_handle: invalid memory handle\n"); 40791ae08745Sheppo return (EINVAL); 40801ae08745Sheppo } 40811ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 40821ae08745Sheppo ldcp = mhdl->ldcp; 40831ae08745Sheppo mtbl = ldcp->mtbl; 40841ae08745Sheppo 40851ae08745Sheppo /* clear count */ 40861ae08745Sheppo *ccount = 0; 40871ae08745Sheppo 40881ae08745Sheppo mutex_enter(&mhdl->lock); 40891ae08745Sheppo 40901ae08745Sheppo if (mhdl->status == LDC_BOUND || mhdl->memseg != NULL) { 40911ae08745Sheppo DWARN(ldcp->id, 40921ae08745Sheppo "ldc_mem_bind_handle: (0x%x) handle already bound\n", 40931ae08745Sheppo mhandle); 40941ae08745Sheppo mutex_exit(&mhdl->lock); 40951ae08745Sheppo return (EINVAL); 40961ae08745Sheppo } 40971ae08745Sheppo 40981ae08745Sheppo /* Force address and size to be 8-byte aligned */ 40991ae08745Sheppo if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 41001ae08745Sheppo DWARN(ldcp->id, 41011ae08745Sheppo "ldc_mem_bind_handle: addr/size is not 8-byte aligned\n"); 41021ae08745Sheppo mutex_exit(&mhdl->lock); 41031ae08745Sheppo return (EINVAL); 41041ae08745Sheppo } 41051ae08745Sheppo 41061ae08745Sheppo /* FUTURE: get the page size, pgsz code, and shift */ 41071ae08745Sheppo pg_size = MMU_PAGESIZE; 41081ae08745Sheppo pg_size_code = page_szc(pg_size); 41091ae08745Sheppo pg_shift = page_get_shift(pg_size_code); 41101ae08745Sheppo pg_mask = ~(pg_size - 1); 41111ae08745Sheppo 41121ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) binding " 41131ae08745Sheppo "va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 41141ae08745Sheppo ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 41151ae08745Sheppo 41161ae08745Sheppo /* aligned VA and its offset */ 41171ae08745Sheppo v_align = (caddr_t)(((uintptr_t)vaddr) & ~(pg_size - 1)); 41181ae08745Sheppo v_offset = ((uintptr_t)vaddr) & (pg_size - 1); 41191ae08745Sheppo 41201ae08745Sheppo npages = (len+v_offset)/pg_size; 41211ae08745Sheppo npages = ((len+v_offset)%pg_size == 0) ? npages : npages+1; 41221ae08745Sheppo 41231ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle: binding " 41241ae08745Sheppo "(0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 41251ae08745Sheppo ldcp->id, vaddr, v_align, v_offset, npages); 41261ae08745Sheppo 41271ae08745Sheppo /* lock the memory table - exclusive access to channel */ 41281ae08745Sheppo mutex_enter(&mtbl->lock); 41291ae08745Sheppo 41301ae08745Sheppo if (npages > mtbl->num_avail) { 41311ae08745Sheppo DWARN(ldcp->id, 41321ae08745Sheppo "ldc_mem_bind_handle: (0x%llx) no table entries\n", 41331ae08745Sheppo ldcp->id); 41341ae08745Sheppo mutex_exit(&mtbl->lock); 41351ae08745Sheppo mutex_exit(&mhdl->lock); 41361ae08745Sheppo return (ENOMEM); 41371ae08745Sheppo } 41381ae08745Sheppo 41391ae08745Sheppo /* Allocate a memseg structure */ 41401ae08745Sheppo memseg = mhdl->memseg = kmem_zalloc(sizeof (ldc_memseg_t), KM_SLEEP); 41411ae08745Sheppo 41421ae08745Sheppo /* Allocate memory to store all pages and cookies */ 41431ae08745Sheppo memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP); 41441ae08745Sheppo memseg->cookies = 41451ae08745Sheppo kmem_zalloc((sizeof (ldc_mem_cookie_t) * npages), KM_SLEEP); 41461ae08745Sheppo 41471ae08745Sheppo D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) processing 0x%llx pages\n", 41481ae08745Sheppo ldcp->id, npages); 41491ae08745Sheppo 41501ae08745Sheppo addr = v_align; 41511ae08745Sheppo 41521ae08745Sheppo /* 41531ae08745Sheppo * Table slots are used in a round-robin manner. The algorithm permits 41541ae08745Sheppo * inserting duplicate entries. Slots allocated earlier will typically 41551ae08745Sheppo * get freed before we get back to reusing the slot.Inserting duplicate 41561ae08745Sheppo * entries should be OK as we only lookup entries using the cookie addr 41571ae08745Sheppo * i.e. tbl index, during export, unexport and copy operation. 41581ae08745Sheppo * 41591ae08745Sheppo * One implementation what was tried was to search for a duplicate 41601ae08745Sheppo * page entry first and reuse it. The search overhead is very high and 41611ae08745Sheppo * in the vnet case dropped the perf by almost half, 50 to 24 mbps. 41621ae08745Sheppo * So it does make sense to avoid searching for duplicates. 41631ae08745Sheppo * 41641ae08745Sheppo * But during the process of searching for a free slot, if we find a 41651ae08745Sheppo * duplicate entry we will go ahead and use it, and bump its use count. 41661ae08745Sheppo */ 41671ae08745Sheppo 41681ae08745Sheppo /* index to start searching from */ 41691ae08745Sheppo index = mtbl->next_entry; 41701ae08745Sheppo cookie_idx = -1; 41711ae08745Sheppo 41721ae08745Sheppo tmp_mte.ll = 0; /* initialise fields to 0 */ 41731ae08745Sheppo 41741ae08745Sheppo if (mtype & LDC_DIRECT_MAP) { 41751ae08745Sheppo tmp_mte.mte_r = (perm & LDC_MEM_R) ? 1 : 0; 41761ae08745Sheppo tmp_mte.mte_w = (perm & LDC_MEM_W) ? 1 : 0; 41771ae08745Sheppo tmp_mte.mte_x = (perm & LDC_MEM_X) ? 1 : 0; 41781ae08745Sheppo } 41791ae08745Sheppo 41801ae08745Sheppo if (mtype & LDC_SHADOW_MAP) { 41811ae08745Sheppo tmp_mte.mte_cr = (perm & LDC_MEM_R) ? 1 : 0; 41821ae08745Sheppo tmp_mte.mte_cw = (perm & LDC_MEM_W) ? 1 : 0; 41831ae08745Sheppo } 41841ae08745Sheppo 41851ae08745Sheppo if (mtype & LDC_IO_MAP) { 41861ae08745Sheppo tmp_mte.mte_ir = (perm & LDC_MEM_R) ? 1 : 0; 41871ae08745Sheppo tmp_mte.mte_iw = (perm & LDC_MEM_W) ? 1 : 0; 41881ae08745Sheppo } 41891ae08745Sheppo 41901ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll); 41911ae08745Sheppo 41921ae08745Sheppo tmp_mte.mte_pgszc = pg_size_code; 41931ae08745Sheppo 41941ae08745Sheppo /* initialize each mem table entry */ 41951ae08745Sheppo for (i = 0; i < npages; i++) { 41961ae08745Sheppo 41971ae08745Sheppo /* check if slot is available in the table */ 41981ae08745Sheppo while (mtbl->table[index].entry.ll != 0) { 41991ae08745Sheppo 42001ae08745Sheppo index = (index + 1) % mtbl->num_entries; 42011ae08745Sheppo 42021ae08745Sheppo if (index == mtbl->next_entry) { 42031ae08745Sheppo /* we have looped around */ 42041ae08745Sheppo DWARN(DBG_ALL_LDCS, 42051ae08745Sheppo "ldc_mem_bind_handle: (0x%llx) cannot find " 42061ae08745Sheppo "entry\n", ldcp->id); 42071ae08745Sheppo *ccount = 0; 42081ae08745Sheppo 42091ae08745Sheppo /* NOTE: free memory, remove previous entries */ 42101ae08745Sheppo /* this shouldnt happen as num_avail was ok */ 42111ae08745Sheppo 42121ae08745Sheppo mutex_exit(&mtbl->lock); 42131ae08745Sheppo mutex_exit(&mhdl->lock); 42141ae08745Sheppo return (ENOMEM); 42151ae08745Sheppo } 42161ae08745Sheppo } 42171ae08745Sheppo 42181ae08745Sheppo /* get the real address */ 42191ae08745Sheppo raddr = va_to_pa((void *)addr); 42201ae08745Sheppo ra_aligned = ((uintptr_t)raddr & pg_mask); 42211ae08745Sheppo 42221ae08745Sheppo /* build the mte */ 42231ae08745Sheppo tmp_mte.mte_rpfn = ra_aligned >> pg_shift; 42241ae08745Sheppo 42251ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll); 42261ae08745Sheppo 42271ae08745Sheppo /* update entry in table */ 42281ae08745Sheppo mtbl->table[index].entry = tmp_mte; 42291ae08745Sheppo 42301ae08745Sheppo D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) stored MTE 0x%llx" 42311ae08745Sheppo " into loc 0x%llx\n", ldcp->id, tmp_mte.ll, index); 42321ae08745Sheppo 42331ae08745Sheppo /* calculate the size and offset for this export range */ 42341ae08745Sheppo if (i == 0) { 42351ae08745Sheppo /* first page */ 42361ae08745Sheppo psize = min((pg_size - v_offset), len); 42371ae08745Sheppo poffset = v_offset; 42381ae08745Sheppo 42391ae08745Sheppo } else if (i == (npages - 1)) { 42401ae08745Sheppo /* last page */ 42411ae08745Sheppo psize = (((uintptr_t)(vaddr + len)) & 42421ae08745Sheppo ((uint64_t)(pg_size-1))); 42431ae08745Sheppo if (psize == 0) 42441ae08745Sheppo psize = pg_size; 42451ae08745Sheppo poffset = 0; 42461ae08745Sheppo 42471ae08745Sheppo } else { 42481ae08745Sheppo /* middle pages */ 42491ae08745Sheppo psize = pg_size; 42501ae08745Sheppo poffset = 0; 42511ae08745Sheppo } 42521ae08745Sheppo 42531ae08745Sheppo /* store entry for this page */ 42541ae08745Sheppo memseg->pages[i].index = index; 42551ae08745Sheppo memseg->pages[i].raddr = raddr; 42561ae08745Sheppo memseg->pages[i].offset = poffset; 42571ae08745Sheppo memseg->pages[i].size = psize; 42581ae08745Sheppo memseg->pages[i].mte = &(mtbl->table[index]); 42591ae08745Sheppo 42601ae08745Sheppo /* create the cookie */ 42611ae08745Sheppo if (i == 0 || (index != prev_index + 1)) { 42621ae08745Sheppo cookie_idx++; 42631ae08745Sheppo memseg->cookies[cookie_idx].addr = 42641ae08745Sheppo IDX2COOKIE(index, pg_size_code, pg_shift); 42651ae08745Sheppo memseg->cookies[cookie_idx].addr |= poffset; 42661ae08745Sheppo memseg->cookies[cookie_idx].size = psize; 42671ae08745Sheppo 42681ae08745Sheppo } else { 42691ae08745Sheppo memseg->cookies[cookie_idx].size += psize; 42701ae08745Sheppo } 42711ae08745Sheppo 42721ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle: bound " 42731ae08745Sheppo "(0x%llx) va=0x%llx, idx=0x%llx, " 42741ae08745Sheppo "ra=0x%llx(sz=0x%x,off=0x%x)\n", 42751ae08745Sheppo ldcp->id, addr, index, raddr, psize, poffset); 42761ae08745Sheppo 42771ae08745Sheppo /* decrement number of available entries */ 42781ae08745Sheppo mtbl->num_avail--; 42791ae08745Sheppo 42801ae08745Sheppo /* increment va by page size */ 42811ae08745Sheppo addr += pg_size; 42821ae08745Sheppo 42831ae08745Sheppo /* increment index */ 42841ae08745Sheppo prev_index = index; 42851ae08745Sheppo index = (index + 1) % mtbl->num_entries; 42861ae08745Sheppo 42871ae08745Sheppo /* save the next slot */ 42881ae08745Sheppo mtbl->next_entry = index; 42891ae08745Sheppo } 42901ae08745Sheppo 42911ae08745Sheppo mutex_exit(&mtbl->lock); 42921ae08745Sheppo 42931ae08745Sheppo /* memory handle = bound */ 42941ae08745Sheppo mhdl->mtype = mtype; 42951ae08745Sheppo mhdl->perm = perm; 42961ae08745Sheppo mhdl->status = LDC_BOUND; 42971ae08745Sheppo 42981ae08745Sheppo /* update memseg_t */ 42991ae08745Sheppo memseg->vaddr = vaddr; 43001ae08745Sheppo memseg->raddr = memseg->pages[0].raddr; 43011ae08745Sheppo memseg->size = len; 43021ae08745Sheppo memseg->npages = npages; 43031ae08745Sheppo memseg->ncookies = cookie_idx + 1; 43041ae08745Sheppo memseg->next_cookie = (memseg->ncookies > 1) ? 1 : 0; 43051ae08745Sheppo 43061ae08745Sheppo /* return count and first cookie */ 43071ae08745Sheppo *ccount = memseg->ncookies; 43081ae08745Sheppo cookie->addr = memseg->cookies[0].addr; 43091ae08745Sheppo cookie->size = memseg->cookies[0].size; 43101ae08745Sheppo 43111ae08745Sheppo D1(ldcp->id, 43121ae08745Sheppo "ldc_mem_bind_handle: (0x%llx) bound 0x%llx, va=0x%llx, " 43131ae08745Sheppo "pgs=0x%llx cookies=0x%llx\n", 43141ae08745Sheppo ldcp->id, mhdl, vaddr, npages, memseg->ncookies); 43151ae08745Sheppo 43161ae08745Sheppo mutex_exit(&mhdl->lock); 43171ae08745Sheppo return (0); 43181ae08745Sheppo } 43191ae08745Sheppo 43201ae08745Sheppo /* 43211ae08745Sheppo * Return the next cookie associated with the specified memory handle 43221ae08745Sheppo */ 43231ae08745Sheppo int 43241ae08745Sheppo ldc_mem_nextcookie(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie) 43251ae08745Sheppo { 43261ae08745Sheppo ldc_mhdl_t *mhdl; 43271ae08745Sheppo ldc_chan_t *ldcp; 43281ae08745Sheppo ldc_memseg_t *memseg; 43291ae08745Sheppo 43301ae08745Sheppo if (mhandle == NULL) { 43311ae08745Sheppo DWARN(DBG_ALL_LDCS, 43321ae08745Sheppo "ldc_mem_nextcookie: invalid memory handle\n"); 43331ae08745Sheppo return (EINVAL); 43341ae08745Sheppo } 43351ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 43361ae08745Sheppo 43371ae08745Sheppo mutex_enter(&mhdl->lock); 43381ae08745Sheppo 43391ae08745Sheppo ldcp = mhdl->ldcp; 43401ae08745Sheppo memseg = mhdl->memseg; 43411ae08745Sheppo 43421ae08745Sheppo if (cookie == 0) { 43431ae08745Sheppo DWARN(ldcp->id, 43441ae08745Sheppo "ldc_mem_nextcookie:(0x%llx) invalid cookie arg\n", 43451ae08745Sheppo ldcp->id); 43461ae08745Sheppo mutex_exit(&mhdl->lock); 43471ae08745Sheppo return (EINVAL); 43481ae08745Sheppo } 43491ae08745Sheppo 43501ae08745Sheppo if (memseg->next_cookie != 0) { 43511ae08745Sheppo cookie->addr = memseg->cookies[memseg->next_cookie].addr; 43521ae08745Sheppo cookie->size = memseg->cookies[memseg->next_cookie].size; 43531ae08745Sheppo memseg->next_cookie++; 43541ae08745Sheppo if (memseg->next_cookie == memseg->ncookies) 43551ae08745Sheppo memseg->next_cookie = 0; 43561ae08745Sheppo 43571ae08745Sheppo } else { 43581ae08745Sheppo DWARN(ldcp->id, 43591ae08745Sheppo "ldc_mem_nextcookie:(0x%llx) no more cookies\n", ldcp->id); 43601ae08745Sheppo cookie->addr = 0; 43611ae08745Sheppo cookie->size = 0; 43621ae08745Sheppo mutex_exit(&mhdl->lock); 43631ae08745Sheppo return (EINVAL); 43641ae08745Sheppo } 43651ae08745Sheppo 43661ae08745Sheppo D1(ldcp->id, 43671ae08745Sheppo "ldc_mem_nextcookie: (0x%llx) cookie addr=0x%llx,sz=0x%llx\n", 43681ae08745Sheppo ldcp->id, cookie->addr, cookie->size); 43691ae08745Sheppo 43701ae08745Sheppo mutex_exit(&mhdl->lock); 43711ae08745Sheppo return (0); 43721ae08745Sheppo } 43731ae08745Sheppo 43741ae08745Sheppo /* 43751ae08745Sheppo * Unbind the virtual memory region associated with the specified 43761ae08745Sheppo * memory handle. Allassociated cookies are freed and the corresponding 43771ae08745Sheppo * RA space is no longer exported. 43781ae08745Sheppo */ 43791ae08745Sheppo int 43801ae08745Sheppo ldc_mem_unbind_handle(ldc_mem_handle_t mhandle) 43811ae08745Sheppo { 43821ae08745Sheppo ldc_mhdl_t *mhdl; 43831ae08745Sheppo ldc_chan_t *ldcp; 43841ae08745Sheppo ldc_mtbl_t *mtbl; 43851ae08745Sheppo ldc_memseg_t *memseg; 43861ae08745Sheppo int i; 43871ae08745Sheppo 43881ae08745Sheppo if (mhandle == NULL) { 43891ae08745Sheppo DWARN(DBG_ALL_LDCS, 43901ae08745Sheppo "ldc_mem_unbind_handle: invalid memory handle\n"); 43911ae08745Sheppo return (EINVAL); 43921ae08745Sheppo } 43931ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 43941ae08745Sheppo 43951ae08745Sheppo mutex_enter(&mhdl->lock); 43961ae08745Sheppo 43971ae08745Sheppo if (mhdl->status == LDC_UNBOUND) { 43981ae08745Sheppo DWARN(DBG_ALL_LDCS, 43991ae08745Sheppo "ldc_mem_unbind_handle: (0x%x) handle is not bound\n", 44001ae08745Sheppo mhandle); 44011ae08745Sheppo mutex_exit(&mhdl->lock); 44021ae08745Sheppo return (EINVAL); 44031ae08745Sheppo } 44041ae08745Sheppo 44051ae08745Sheppo ldcp = mhdl->ldcp; 44061ae08745Sheppo mtbl = ldcp->mtbl; 44071ae08745Sheppo 44081ae08745Sheppo memseg = mhdl->memseg; 44091ae08745Sheppo 44101ae08745Sheppo /* lock the memory table - exclusive access to channel */ 44111ae08745Sheppo mutex_enter(&mtbl->lock); 44121ae08745Sheppo 44131ae08745Sheppo /* undo the pages exported */ 44141ae08745Sheppo for (i = 0; i < memseg->npages; i++) { 44151ae08745Sheppo 44161ae08745Sheppo /* FUTURE: check for mapped pages */ 44171ae08745Sheppo if (memseg->pages[i].mte->cookie) { 44181ae08745Sheppo _NOTE(EMPTY) 44191ae08745Sheppo } 44201ae08745Sheppo 44211ae08745Sheppo /* clear the entry from the table */ 44221ae08745Sheppo memseg->pages[i].mte->entry.ll = 0; 44231ae08745Sheppo mtbl->num_avail++; 44241ae08745Sheppo } 44251ae08745Sheppo mutex_exit(&mtbl->lock); 44261ae08745Sheppo 44271ae08745Sheppo /* free the allocated memseg and page structures */ 44281ae08745Sheppo kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages)); 44291ae08745Sheppo kmem_free(memseg->cookies, 44301ae08745Sheppo (sizeof (ldc_mem_cookie_t) * memseg->npages)); 44311ae08745Sheppo kmem_free(memseg, sizeof (ldc_memseg_t)); 44321ae08745Sheppo 44331ae08745Sheppo /* uninitialize the memory handle */ 44341ae08745Sheppo mhdl->memseg = NULL; 44351ae08745Sheppo mhdl->status = LDC_UNBOUND; 44361ae08745Sheppo 44371ae08745Sheppo D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) unbound handle 0x%llx\n", 44381ae08745Sheppo ldcp->id, mhdl); 44391ae08745Sheppo 44401ae08745Sheppo mutex_exit(&mhdl->lock); 44411ae08745Sheppo return (0); 44421ae08745Sheppo } 44431ae08745Sheppo 44441ae08745Sheppo /* 44451ae08745Sheppo * Get information about the dring. The base address of the descriptor 44461ae08745Sheppo * ring along with the type and permission are returned back. 44471ae08745Sheppo */ 44481ae08745Sheppo int 44491ae08745Sheppo ldc_mem_info(ldc_mem_handle_t mhandle, ldc_mem_info_t *minfo) 44501ae08745Sheppo { 44511ae08745Sheppo ldc_mhdl_t *mhdl; 44521ae08745Sheppo 44531ae08745Sheppo if (mhandle == NULL) { 44541ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid memory handle\n"); 44551ae08745Sheppo return (EINVAL); 44561ae08745Sheppo } 44571ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 44581ae08745Sheppo 44591ae08745Sheppo if (minfo == NULL) { 44601ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid args\n"); 44611ae08745Sheppo return (EINVAL); 44621ae08745Sheppo } 44631ae08745Sheppo 44641ae08745Sheppo mutex_enter(&mhdl->lock); 44651ae08745Sheppo 44661ae08745Sheppo minfo->status = mhdl->status; 44671ae08745Sheppo if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) { 44681ae08745Sheppo minfo->vaddr = mhdl->memseg->vaddr; 44691ae08745Sheppo minfo->raddr = mhdl->memseg->raddr; 44701ae08745Sheppo minfo->mtype = mhdl->mtype; 44711ae08745Sheppo minfo->perm = mhdl->perm; 44721ae08745Sheppo } 44731ae08745Sheppo mutex_exit(&mhdl->lock); 44741ae08745Sheppo 44751ae08745Sheppo return (0); 44761ae08745Sheppo } 44771ae08745Sheppo 44781ae08745Sheppo /* 44791ae08745Sheppo * Copy data either from or to the client specified virtual address 44801ae08745Sheppo * space to or from the exported memory associated with the cookies. 44811ae08745Sheppo * The direction argument determines whether the data is read from or 44821ae08745Sheppo * written to exported memory. 44831ae08745Sheppo */ 44841ae08745Sheppo int 44851ae08745Sheppo ldc_mem_copy(ldc_handle_t handle, caddr_t vaddr, uint64_t off, size_t *size, 44861ae08745Sheppo ldc_mem_cookie_t *cookies, uint32_t ccount, uint8_t direction) 44871ae08745Sheppo { 44881ae08745Sheppo ldc_chan_t *ldcp; 44891ae08745Sheppo uint64_t local_voff, local_valign; 44901ae08745Sheppo uint64_t cookie_addr, cookie_size; 44911ae08745Sheppo uint64_t pg_shift, pg_size, pg_size_code; 44921ae08745Sheppo uint64_t export_caddr, export_poff, export_psize, export_size; 44931ae08745Sheppo uint64_t local_ra, local_poff, local_psize; 44941ae08745Sheppo uint64_t copy_size, copied_len = 0, total_bal = 0, idx = 0; 44951ae08745Sheppo pgcnt_t npages; 44961ae08745Sheppo size_t len = *size; 44971ae08745Sheppo int i, rv = 0; 44981ae08745Sheppo 44991ae08745Sheppo if (handle == NULL) { 45001ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_copy: invalid channel handle\n"); 45011ae08745Sheppo return (EINVAL); 45021ae08745Sheppo } 45031ae08745Sheppo ldcp = (ldc_chan_t *)handle; 45041ae08745Sheppo 45051ae08745Sheppo mutex_enter(&ldcp->lock); 45061ae08745Sheppo 45071ae08745Sheppo /* check to see if channel is UP */ 45081ae08745Sheppo if (ldcp->tstate != TS_UP) { 45091ae08745Sheppo DWARN(ldcp->id, "ldc_mem_copy: (0x%llx) channel is not UP\n", 45101ae08745Sheppo ldcp->id); 45111ae08745Sheppo mutex_exit(&ldcp->lock); 45121ae08745Sheppo return (EINVAL); 45131ae08745Sheppo } 45141ae08745Sheppo 45151ae08745Sheppo /* Force address and size to be 8-byte aligned */ 45161ae08745Sheppo if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 45171ae08745Sheppo DWARN(ldcp->id, 45181ae08745Sheppo "ldc_mem_copy: addr/sz is not 8-byte aligned\n"); 45191ae08745Sheppo mutex_exit(&ldcp->lock); 45201ae08745Sheppo return (EINVAL); 45211ae08745Sheppo } 45221ae08745Sheppo 45231ae08745Sheppo /* Find the size of the exported memory */ 45241ae08745Sheppo export_size = 0; 45251ae08745Sheppo for (i = 0; i < ccount; i++) 45261ae08745Sheppo export_size += cookies[i].size; 45271ae08745Sheppo 45281ae08745Sheppo /* check to see if offset is valid */ 45291ae08745Sheppo if (off > export_size) { 45301ae08745Sheppo DWARN(ldcp->id, 45311ae08745Sheppo "ldc_mem_copy: (0x%llx) start offset > export mem size\n", 45321ae08745Sheppo ldcp->id); 45331ae08745Sheppo mutex_exit(&ldcp->lock); 45341ae08745Sheppo return (EINVAL); 45351ae08745Sheppo } 45361ae08745Sheppo 45371ae08745Sheppo /* 45381ae08745Sheppo * Check to see if the export size is smaller than the size we 45391ae08745Sheppo * are requesting to copy - if so flag an error 45401ae08745Sheppo */ 45411ae08745Sheppo if ((export_size - off) < *size) { 45421ae08745Sheppo DWARN(ldcp->id, 45431ae08745Sheppo "ldc_mem_copy: (0x%llx) copy size > export mem size\n", 45441ae08745Sheppo ldcp->id); 45451ae08745Sheppo mutex_exit(&ldcp->lock); 45461ae08745Sheppo return (EINVAL); 45471ae08745Sheppo } 45481ae08745Sheppo 45491ae08745Sheppo total_bal = min(export_size, *size); 45501ae08745Sheppo 45511ae08745Sheppo /* FUTURE: get the page size, pgsz code, and shift */ 45521ae08745Sheppo pg_size = MMU_PAGESIZE; 45531ae08745Sheppo pg_size_code = page_szc(pg_size); 45541ae08745Sheppo pg_shift = page_get_shift(pg_size_code); 45551ae08745Sheppo 45561ae08745Sheppo D1(ldcp->id, "ldc_mem_copy: copying data " 45571ae08745Sheppo "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 45581ae08745Sheppo ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 45591ae08745Sheppo 45601ae08745Sheppo /* aligned VA and its offset */ 45611ae08745Sheppo local_valign = (((uintptr_t)vaddr) & ~(pg_size - 1)); 45621ae08745Sheppo local_voff = ((uintptr_t)vaddr) & (pg_size - 1); 45631ae08745Sheppo 45641ae08745Sheppo npages = (len+local_voff)/pg_size; 45651ae08745Sheppo npages = ((len+local_voff)%pg_size == 0) ? npages : npages+1; 45661ae08745Sheppo 45671ae08745Sheppo D1(ldcp->id, 45681ae08745Sheppo "ldc_mem_copy: (0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 45691ae08745Sheppo ldcp->id, vaddr, local_valign, local_voff, npages); 45701ae08745Sheppo 45711ae08745Sheppo local_ra = va_to_pa((void *)local_valign); 45721ae08745Sheppo local_poff = local_voff; 45731ae08745Sheppo local_psize = min(len, (pg_size - local_voff)); 45741ae08745Sheppo 45751ae08745Sheppo len -= local_psize; 45761ae08745Sheppo 45771ae08745Sheppo /* 45781ae08745Sheppo * find the first cookie in the list of cookies 45791ae08745Sheppo * if the offset passed in is not zero 45801ae08745Sheppo */ 45811ae08745Sheppo for (idx = 0; idx < ccount; idx++) { 45821ae08745Sheppo cookie_size = cookies[idx].size; 45831ae08745Sheppo if (off < cookie_size) 45841ae08745Sheppo break; 45851ae08745Sheppo off -= cookie_size; 45861ae08745Sheppo } 45871ae08745Sheppo 45881ae08745Sheppo cookie_addr = cookies[idx].addr + off; 45891ae08745Sheppo cookie_size = cookies[idx].size - off; 45901ae08745Sheppo 45911ae08745Sheppo export_caddr = cookie_addr & ~(pg_size - 1); 45921ae08745Sheppo export_poff = cookie_addr & (pg_size - 1); 45931ae08745Sheppo export_psize = min(cookie_size, (pg_size - export_poff)); 45941ae08745Sheppo 45951ae08745Sheppo for (;;) { 45961ae08745Sheppo 45971ae08745Sheppo copy_size = min(export_psize, local_psize); 45981ae08745Sheppo 45991ae08745Sheppo D1(ldcp->id, 46001ae08745Sheppo "ldc_mem_copy:(0x%llx) dir=0x%x, caddr=0x%llx," 46011ae08745Sheppo " loc_ra=0x%llx, exp_poff=0x%llx, loc_poff=0x%llx," 46021ae08745Sheppo " exp_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 46031ae08745Sheppo " total_bal=0x%llx\n", 46041ae08745Sheppo ldcp->id, direction, export_caddr, local_ra, export_poff, 46051ae08745Sheppo local_poff, export_psize, local_psize, copy_size, 46061ae08745Sheppo total_bal); 46071ae08745Sheppo 46081ae08745Sheppo rv = hv_ldc_copy(ldcp->id, direction, 46091ae08745Sheppo (export_caddr + export_poff), (local_ra + local_poff), 46101ae08745Sheppo copy_size, &copied_len); 46111ae08745Sheppo 46121ae08745Sheppo if (rv != 0) { 46131ae08745Sheppo cmn_err(CE_WARN, 46141ae08745Sheppo "ldc_mem_copy: (0x%lx) err %d during copy\n", 46151ae08745Sheppo ldcp->id, rv); 46161ae08745Sheppo DWARN(DBG_ALL_LDCS, 46171ae08745Sheppo "ldc_mem_copy: (0x%llx) dir=0x%x, caddr=0x%llx, " 46181ae08745Sheppo "loc_ra=0x%llx, exp_poff=0x%llx, loc_poff=0x%llx," 46191ae08745Sheppo " exp_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 46201ae08745Sheppo " copied_len=0x%llx, total_bal=0x%llx\n", 46211ae08745Sheppo ldcp->id, direction, export_caddr, local_ra, 46221ae08745Sheppo export_poff, local_poff, export_psize, local_psize, 46231ae08745Sheppo copy_size, copied_len, total_bal); 46241ae08745Sheppo 46251ae08745Sheppo *size = *size - total_bal; 46261ae08745Sheppo mutex_exit(&ldcp->lock); 46271ae08745Sheppo return (EIO); 46281ae08745Sheppo } 46291ae08745Sheppo 46301ae08745Sheppo ASSERT(copied_len <= copy_size); 46311ae08745Sheppo 46321ae08745Sheppo D2(ldcp->id, "ldc_mem_copy: copied=0x%llx\n", copied_len); 46331ae08745Sheppo export_poff += copied_len; 46341ae08745Sheppo local_poff += copied_len; 46351ae08745Sheppo export_psize -= copied_len; 46361ae08745Sheppo local_psize -= copied_len; 46371ae08745Sheppo cookie_size -= copied_len; 46381ae08745Sheppo 46391ae08745Sheppo total_bal -= copied_len; 46401ae08745Sheppo 46411ae08745Sheppo if (copy_size != copied_len) 46421ae08745Sheppo continue; 46431ae08745Sheppo 46441ae08745Sheppo if (export_psize == 0 && total_bal != 0) { 46451ae08745Sheppo 46461ae08745Sheppo if (cookie_size == 0) { 46471ae08745Sheppo idx++; 46481ae08745Sheppo cookie_addr = cookies[idx].addr; 46491ae08745Sheppo cookie_size = cookies[idx].size; 46501ae08745Sheppo 46511ae08745Sheppo export_caddr = cookie_addr & ~(pg_size - 1); 46521ae08745Sheppo export_poff = cookie_addr & (pg_size - 1); 46531ae08745Sheppo export_psize = 46541ae08745Sheppo min(cookie_size, (pg_size-export_poff)); 46551ae08745Sheppo } else { 46561ae08745Sheppo export_caddr += pg_size; 46571ae08745Sheppo export_poff = 0; 46581ae08745Sheppo export_psize = min(cookie_size, pg_size); 46591ae08745Sheppo } 46601ae08745Sheppo } 46611ae08745Sheppo 46621ae08745Sheppo if (local_psize == 0 && total_bal != 0) { 46631ae08745Sheppo local_valign += pg_size; 46641ae08745Sheppo local_ra = va_to_pa((void *)local_valign); 46651ae08745Sheppo local_poff = 0; 46661ae08745Sheppo local_psize = min(pg_size, len); 46671ae08745Sheppo len -= local_psize; 46681ae08745Sheppo } 46691ae08745Sheppo 46701ae08745Sheppo /* check if we are all done */ 46711ae08745Sheppo if (total_bal == 0) 46721ae08745Sheppo break; 46731ae08745Sheppo } 46741ae08745Sheppo 46751ae08745Sheppo mutex_exit(&ldcp->lock); 46761ae08745Sheppo 46771ae08745Sheppo D1(ldcp->id, 46781ae08745Sheppo "ldc_mem_copy: (0x%llx) done copying sz=0x%llx\n", 46791ae08745Sheppo ldcp->id, *size); 46801ae08745Sheppo 46811ae08745Sheppo return (0); 46821ae08745Sheppo } 46831ae08745Sheppo 46841ae08745Sheppo /* 46851ae08745Sheppo * Copy data either from or to the client specified virtual address 46861ae08745Sheppo * space to or from HV physical memory. 46871ae08745Sheppo * 46881ae08745Sheppo * The direction argument determines whether the data is read from or 46891ae08745Sheppo * written to HV memory. direction values are LDC_COPY_IN/OUT similar 46901ae08745Sheppo * to the ldc_mem_copy interface 46911ae08745Sheppo */ 46921ae08745Sheppo int 46931ae08745Sheppo ldc_mem_rdwr_pa(ldc_handle_t handle, caddr_t vaddr, size_t *size, 46941ae08745Sheppo caddr_t paddr, uint8_t direction) 46951ae08745Sheppo { 46961ae08745Sheppo ldc_chan_t *ldcp; 46971ae08745Sheppo uint64_t local_voff, local_valign; 46981ae08745Sheppo uint64_t pg_shift, pg_size, pg_size_code; 46991ae08745Sheppo uint64_t target_pa, target_poff, target_psize, target_size; 47001ae08745Sheppo uint64_t local_ra, local_poff, local_psize; 47011ae08745Sheppo uint64_t copy_size, copied_len = 0; 47021ae08745Sheppo pgcnt_t npages; 47031ae08745Sheppo size_t len = *size; 47041ae08745Sheppo int rv = 0; 47051ae08745Sheppo 47061ae08745Sheppo if (handle == NULL) { 47071ae08745Sheppo DWARN(DBG_ALL_LDCS, 47081ae08745Sheppo "ldc_mem_rdwr_pa: invalid channel handle\n"); 47091ae08745Sheppo return (EINVAL); 47101ae08745Sheppo } 47111ae08745Sheppo ldcp = (ldc_chan_t *)handle; 47121ae08745Sheppo 47131ae08745Sheppo mutex_enter(&ldcp->lock); 47141ae08745Sheppo 47151ae08745Sheppo /* check to see if channel is UP */ 47161ae08745Sheppo if (ldcp->tstate != TS_UP) { 47171ae08745Sheppo DWARN(ldcp->id, 47181ae08745Sheppo "ldc_mem_rdwr_pa: (0x%llx) channel is not UP\n", 47191ae08745Sheppo ldcp->id); 47201ae08745Sheppo mutex_exit(&ldcp->lock); 47211ae08745Sheppo return (EINVAL); 47221ae08745Sheppo } 47231ae08745Sheppo 47241ae08745Sheppo /* Force address and size to be 8-byte aligned */ 47251ae08745Sheppo if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 47261ae08745Sheppo DWARN(ldcp->id, 47271ae08745Sheppo "ldc_mem_rdwr_pa: addr/size is not 8-byte aligned\n"); 47281ae08745Sheppo mutex_exit(&ldcp->lock); 47291ae08745Sheppo return (EINVAL); 47301ae08745Sheppo } 47311ae08745Sheppo 47321ae08745Sheppo target_size = *size; 47331ae08745Sheppo 47341ae08745Sheppo /* FUTURE: get the page size, pgsz code, and shift */ 47351ae08745Sheppo pg_size = MMU_PAGESIZE; 47361ae08745Sheppo pg_size_code = page_szc(pg_size); 47371ae08745Sheppo pg_shift = page_get_shift(pg_size_code); 47381ae08745Sheppo 47391ae08745Sheppo D1(ldcp->id, "ldc_mem_rdwr_pa: copying data " 47401ae08745Sheppo "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 47411ae08745Sheppo ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 47421ae08745Sheppo 47431ae08745Sheppo /* aligned VA and its offset */ 47441ae08745Sheppo local_valign = ((uintptr_t)vaddr) & ~(pg_size - 1); 47451ae08745Sheppo local_voff = ((uintptr_t)vaddr) & (pg_size - 1); 47461ae08745Sheppo 47471ae08745Sheppo npages = (len + local_voff) / pg_size; 47481ae08745Sheppo npages = ((len + local_voff) % pg_size == 0) ? npages : npages+1; 47491ae08745Sheppo 47501ae08745Sheppo D1(ldcp->id, 47511ae08745Sheppo "ldc_mem_rdwr_pa: (0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 47521ae08745Sheppo ldcp->id, vaddr, local_valign, local_voff, npages); 47531ae08745Sheppo 47541ae08745Sheppo local_ra = va_to_pa((void *)local_valign); 47551ae08745Sheppo local_poff = local_voff; 47561ae08745Sheppo local_psize = min(len, (pg_size - local_voff)); 47571ae08745Sheppo 47581ae08745Sheppo len -= local_psize; 47591ae08745Sheppo 47601ae08745Sheppo target_pa = ((uintptr_t)paddr) & ~(pg_size - 1); 47611ae08745Sheppo target_poff = ((uintptr_t)paddr) & (pg_size - 1); 47621ae08745Sheppo target_psize = pg_size - target_poff; 47631ae08745Sheppo 47641ae08745Sheppo for (;;) { 47651ae08745Sheppo 47661ae08745Sheppo copy_size = min(target_psize, local_psize); 47671ae08745Sheppo 47681ae08745Sheppo D1(ldcp->id, 47691ae08745Sheppo "ldc_mem_rdwr_pa: (0x%llx) dir=0x%x, tar_pa=0x%llx," 47701ae08745Sheppo " loc_ra=0x%llx, tar_poff=0x%llx, loc_poff=0x%llx," 47711ae08745Sheppo " tar_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 47721ae08745Sheppo " total_bal=0x%llx\n", 47731ae08745Sheppo ldcp->id, direction, target_pa, local_ra, target_poff, 47741ae08745Sheppo local_poff, target_psize, local_psize, copy_size, 47751ae08745Sheppo target_size); 47761ae08745Sheppo 47771ae08745Sheppo rv = hv_ldc_copy(ldcp->id, direction, 47781ae08745Sheppo (target_pa + target_poff), (local_ra + local_poff), 47791ae08745Sheppo copy_size, &copied_len); 47801ae08745Sheppo 47811ae08745Sheppo if (rv != 0) { 47821ae08745Sheppo cmn_err(CE_WARN, 47831ae08745Sheppo "ldc_mem_rdwr_pa: (0x%lx) err %d during copy\n", 47841ae08745Sheppo ldcp->id, rv); 47851ae08745Sheppo DWARN(DBG_ALL_LDCS, 47861ae08745Sheppo "ldc_mem_rdwr_pa: (0x%llx) dir=%lld,tar_pa=0x%llx, " 47871ae08745Sheppo "loc_ra=0x%llx, tar_poff=0x%llx, loc_poff=0x%llx," 47881ae08745Sheppo " tar_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 47891ae08745Sheppo " total_bal=0x%llx\n", 47901ae08745Sheppo ldcp->id, direction, target_pa, local_ra, 47911ae08745Sheppo target_poff, local_poff, target_psize, local_psize, 47921ae08745Sheppo copy_size, target_size); 47931ae08745Sheppo 47941ae08745Sheppo *size = *size - target_size; 47951ae08745Sheppo mutex_exit(&ldcp->lock); 47961ae08745Sheppo return (i_ldc_h2v_error(rv)); 47971ae08745Sheppo } 47981ae08745Sheppo 47991ae08745Sheppo D2(ldcp->id, "ldc_mem_rdwr_pa: copied=0x%llx\n", copied_len); 48001ae08745Sheppo target_poff += copied_len; 48011ae08745Sheppo local_poff += copied_len; 48021ae08745Sheppo target_psize -= copied_len; 48031ae08745Sheppo local_psize -= copied_len; 48041ae08745Sheppo 48051ae08745Sheppo target_size -= copied_len; 48061ae08745Sheppo 48071ae08745Sheppo if (copy_size != copied_len) 48081ae08745Sheppo continue; 48091ae08745Sheppo 48101ae08745Sheppo if (target_psize == 0 && target_size != 0) { 48111ae08745Sheppo target_pa += pg_size; 48121ae08745Sheppo target_poff = 0; 48131ae08745Sheppo target_psize = min(pg_size, target_size); 48141ae08745Sheppo } 48151ae08745Sheppo 48161ae08745Sheppo if (local_psize == 0 && target_size != 0) { 48171ae08745Sheppo local_valign += pg_size; 48181ae08745Sheppo local_ra = va_to_pa((void *)local_valign); 48191ae08745Sheppo local_poff = 0; 48201ae08745Sheppo local_psize = min(pg_size, len); 48211ae08745Sheppo len -= local_psize; 48221ae08745Sheppo } 48231ae08745Sheppo 48241ae08745Sheppo /* check if we are all done */ 48251ae08745Sheppo if (target_size == 0) 48261ae08745Sheppo break; 48271ae08745Sheppo } 48281ae08745Sheppo 48291ae08745Sheppo mutex_exit(&ldcp->lock); 48301ae08745Sheppo 48311ae08745Sheppo D1(ldcp->id, "ldc_mem_rdwr_pa: (0x%llx) done copying sz=0x%llx\n", 48321ae08745Sheppo ldcp->id, *size); 48331ae08745Sheppo 48341ae08745Sheppo return (0); 48351ae08745Sheppo } 48361ae08745Sheppo 48371ae08745Sheppo /* 48381ae08745Sheppo * Map an exported memory segment into the local address space. If the 48391ae08745Sheppo * memory range was exported for direct map access, a HV call is made 48401ae08745Sheppo * to allocate a RA range. If the map is done via a shadow copy, local 48411ae08745Sheppo * shadow memory is allocated and the base VA is returned in 'vaddr'. If 48421ae08745Sheppo * the mapping is a direct map then the RA is returned in 'raddr'. 48431ae08745Sheppo */ 48441ae08745Sheppo int 48451ae08745Sheppo ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie, uint32_t ccount, 48461ae08745Sheppo uint8_t mtype, caddr_t *vaddr, caddr_t *raddr) 48471ae08745Sheppo { 48481ae08745Sheppo int i, idx; 48491ae08745Sheppo ldc_chan_t *ldcp; 48501ae08745Sheppo ldc_mhdl_t *mhdl; 48511ae08745Sheppo ldc_memseg_t *memseg; 48521ae08745Sheppo caddr_t shadow_base = NULL, tmpaddr; 48531ae08745Sheppo uint64_t pg_size, pg_shift, pg_size_code; 48541ae08745Sheppo uint64_t exp_size = 0, npages; 48551ae08745Sheppo 48561ae08745Sheppo if (mhandle == NULL) { 48571ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_map: invalid memory handle\n"); 48581ae08745Sheppo return (EINVAL); 48591ae08745Sheppo } 48601ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 48611ae08745Sheppo 48621ae08745Sheppo mutex_enter(&mhdl->lock); 48631ae08745Sheppo 48641ae08745Sheppo if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED || 48651ae08745Sheppo mhdl->memseg != NULL) { 48661ae08745Sheppo DWARN(DBG_ALL_LDCS, 48671ae08745Sheppo "ldc_mem_map: (0x%llx) handle bound/mapped\n", mhandle); 48681ae08745Sheppo mutex_exit(&mhdl->lock); 48691ae08745Sheppo return (EINVAL); 48701ae08745Sheppo } 48711ae08745Sheppo 48721ae08745Sheppo ldcp = mhdl->ldcp; 48731ae08745Sheppo 48741ae08745Sheppo mutex_enter(&ldcp->lock); 48751ae08745Sheppo 48761ae08745Sheppo if (ldcp->tstate != TS_UP) { 48771ae08745Sheppo DWARN(ldcp->id, 48781ae08745Sheppo "ldc_mem_dring_map: (0x%llx) channel is not UP\n", 48791ae08745Sheppo ldcp->id); 48801ae08745Sheppo mutex_exit(&ldcp->lock); 48811ae08745Sheppo mutex_exit(&mhdl->lock); 48821ae08745Sheppo return (EINVAL); 48831ae08745Sheppo } 48841ae08745Sheppo 48851ae08745Sheppo if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) { 48861ae08745Sheppo DWARN(ldcp->id, "ldc_mem_map: invalid map type\n"); 48871ae08745Sheppo mutex_exit(&ldcp->lock); 48881ae08745Sheppo mutex_exit(&mhdl->lock); 48891ae08745Sheppo return (EINVAL); 48901ae08745Sheppo } 48911ae08745Sheppo 48921ae08745Sheppo if (mtype == LDC_SHADOW_MAP && vaddr == NULL) { 48931ae08745Sheppo DWARN(ldcp->id, 48941ae08745Sheppo "ldc_mem_map: invalid vaddr arg0x%llx\n", vaddr); 48951ae08745Sheppo mutex_exit(&ldcp->lock); 48961ae08745Sheppo mutex_exit(&mhdl->lock); 48971ae08745Sheppo return (EINVAL); 48981ae08745Sheppo } 48991ae08745Sheppo 49001ae08745Sheppo if (mtype == LDC_SHADOW_MAP && 49011ae08745Sheppo (vaddr) && ((uintptr_t)(*vaddr) & MMU_PAGEOFFSET)) { 49021ae08745Sheppo DWARN(ldcp->id, 49031ae08745Sheppo "ldc_mem_map: vaddr not page aligned, 0x%llx\n", *vaddr); 49041ae08745Sheppo mutex_exit(&ldcp->lock); 49051ae08745Sheppo mutex_exit(&mhdl->lock); 49061ae08745Sheppo return (EINVAL); 49071ae08745Sheppo } 49081ae08745Sheppo 49091ae08745Sheppo D1(ldcp->id, "ldc_mem_map: (0x%llx) cookie = 0x%llx,0x%llx\n", 4910*d10e4ef2Snarayan ldcp->id, cookie->addr, cookie->size); 49111ae08745Sheppo 49121ae08745Sheppo /* FUTURE: get the page size, pgsz code, and shift */ 49131ae08745Sheppo pg_size = MMU_PAGESIZE; 49141ae08745Sheppo pg_size_code = page_szc(pg_size); 49151ae08745Sheppo pg_shift = page_get_shift(pg_size_code); 49161ae08745Sheppo 49171ae08745Sheppo /* calculate the number of pages in the exported cookie */ 49181ae08745Sheppo for (idx = 0; idx < ccount; idx++) { 49191ae08745Sheppo if (cookie[idx].addr & MMU_PAGEOFFSET || 49201ae08745Sheppo cookie[idx].size & MMU_PAGEOFFSET) { 49211ae08745Sheppo DWARN(ldcp->id, 49221ae08745Sheppo "ldc_mem_map: cookie addr/size not page aligned, " 49231ae08745Sheppo "0x%llx\n", cookie[idx].addr); 49241ae08745Sheppo mutex_exit(&ldcp->lock); 49251ae08745Sheppo mutex_exit(&mhdl->lock); 49261ae08745Sheppo return (EINVAL); 49271ae08745Sheppo } 49281ae08745Sheppo exp_size += cookie[idx].size; 49291ae08745Sheppo } 49301ae08745Sheppo npages = (exp_size >> pg_shift); 49311ae08745Sheppo 49321ae08745Sheppo /* Allocate memseg structure */ 49331ae08745Sheppo memseg = mhdl->memseg = kmem_zalloc(sizeof (ldc_memseg_t), KM_SLEEP); 49341ae08745Sheppo 49351ae08745Sheppo /* Allocate memory to store all pages and cookies */ 49361ae08745Sheppo memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP); 49371ae08745Sheppo memseg->cookies = 49381ae08745Sheppo kmem_zalloc((sizeof (ldc_mem_cookie_t) * ccount), KM_SLEEP); 49391ae08745Sheppo 49401ae08745Sheppo D2(ldcp->id, "ldc_mem_map: (0x%llx) processing 0x%llx pages\n", 49411ae08745Sheppo ldcp->id, npages); 49421ae08745Sheppo 49431ae08745Sheppo /* Check to see if the client is requesting direct or shadow map */ 49441ae08745Sheppo if (mtype == LDC_SHADOW_MAP) { 49451ae08745Sheppo if (*vaddr == NULL) { 49461ae08745Sheppo shadow_base = 49471ae08745Sheppo contig_mem_alloc_align(exp_size, PAGESIZE); 49481ae08745Sheppo if (shadow_base == NULL) { 49491ae08745Sheppo cmn_err(CE_WARN, "ldc_mem_map: shadow memory " 49501ae08745Sheppo "allocation failed\n"); 49511ae08745Sheppo kmem_free(memseg->cookies, 49521ae08745Sheppo (sizeof (ldc_mem_cookie_t) * ccount)); 49531ae08745Sheppo kmem_free(memseg->pages, 49541ae08745Sheppo (sizeof (ldc_page_t) * npages)); 49551ae08745Sheppo kmem_free(memseg, sizeof (ldc_memseg_t)); 49561ae08745Sheppo mutex_exit(&ldcp->lock); 49571ae08745Sheppo mutex_exit(&mhdl->lock); 49581ae08745Sheppo return (ENOMEM); 49591ae08745Sheppo } 49601ae08745Sheppo 49611ae08745Sheppo bzero(shadow_base, exp_size); 49621ae08745Sheppo mhdl->myshadow = B_TRUE; 49631ae08745Sheppo 49641ae08745Sheppo D1(ldcp->id, "ldc_mem_map: (0x%llx) allocated " 49651ae08745Sheppo "shadow page va=0x%llx\n", ldcp->id, shadow_base); 49661ae08745Sheppo } else { 49671ae08745Sheppo /* 49681ae08745Sheppo * Use client supplied memory for shadow_base 49691ae08745Sheppo * WARNING: assuming that client mem is >= exp_size 49701ae08745Sheppo */ 49711ae08745Sheppo shadow_base = *vaddr; 49721ae08745Sheppo } 49731ae08745Sheppo } else if (mtype == LDC_DIRECT_MAP) { 49741ae08745Sheppo /* FUTURE: Do a direct map by calling into HV */ 49751ae08745Sheppo _NOTE(EMPTY) 49761ae08745Sheppo } 49771ae08745Sheppo 49781ae08745Sheppo /* Save all page and cookie information */ 49791ae08745Sheppo for (i = 0, tmpaddr = shadow_base; i < npages; i++) { 49801ae08745Sheppo memseg->pages[i].raddr = va_to_pa(tmpaddr); 49811ae08745Sheppo memseg->pages[i].size = pg_size; 49821ae08745Sheppo memseg->pages[i].index = 0; 49831ae08745Sheppo memseg->pages[i].offset = 0; 49841ae08745Sheppo memseg->pages[i].mte = NULL; 49851ae08745Sheppo tmpaddr += pg_size; 49861ae08745Sheppo } 49871ae08745Sheppo for (i = 0; i < ccount; i++) { 49881ae08745Sheppo memseg->cookies[i].addr = cookie[i].addr; 49891ae08745Sheppo memseg->cookies[i].size = cookie[i].size; 49901ae08745Sheppo } 49911ae08745Sheppo 49921ae08745Sheppo /* update memseg_t */ 49931ae08745Sheppo memseg->vaddr = shadow_base; 49941ae08745Sheppo memseg->raddr = memseg->pages[0].raddr; 49951ae08745Sheppo memseg->size = exp_size; 49961ae08745Sheppo memseg->npages = npages; 49971ae08745Sheppo memseg->ncookies = ccount; 49981ae08745Sheppo memseg->next_cookie = 0; 49991ae08745Sheppo 50001ae08745Sheppo /* memory handle = mapped */ 50011ae08745Sheppo mhdl->mtype = mtype; 50021ae08745Sheppo mhdl->perm = 0; 50031ae08745Sheppo mhdl->status = LDC_MAPPED; 50041ae08745Sheppo 50051ae08745Sheppo D1(ldcp->id, "ldc_mem_map: (0x%llx) mapped 0x%llx, ra=0x%llx, " 50061ae08745Sheppo "va=0x%llx, pgs=0x%llx cookies=0x%llx\n", 50071ae08745Sheppo ldcp->id, mhdl, memseg->raddr, memseg->vaddr, 50081ae08745Sheppo memseg->npages, memseg->ncookies); 50091ae08745Sheppo 50101ae08745Sheppo if (raddr) 50111ae08745Sheppo *raddr = (caddr_t)memseg->raddr; 50121ae08745Sheppo if (vaddr) 50131ae08745Sheppo *vaddr = memseg->vaddr; 50141ae08745Sheppo 50151ae08745Sheppo mutex_exit(&ldcp->lock); 50161ae08745Sheppo mutex_exit(&mhdl->lock); 50171ae08745Sheppo return (0); 50181ae08745Sheppo } 50191ae08745Sheppo 50201ae08745Sheppo /* 50211ae08745Sheppo * Unmap a memory segment. Free shadow memory (if any). 50221ae08745Sheppo */ 50231ae08745Sheppo int 50241ae08745Sheppo ldc_mem_unmap(ldc_mem_handle_t mhandle) 50251ae08745Sheppo { 50261ae08745Sheppo ldc_mhdl_t *mhdl = (ldc_mhdl_t *)mhandle; 50271ae08745Sheppo ldc_chan_t *ldcp; 50281ae08745Sheppo ldc_memseg_t *memseg; 50291ae08745Sheppo 50301ae08745Sheppo if (mhdl == 0 || mhdl->status != LDC_MAPPED) { 50311ae08745Sheppo DWARN(DBG_ALL_LDCS, 50321ae08745Sheppo "ldc_mem_unmap: (0x%llx) handle is not mapped\n", 50331ae08745Sheppo mhandle); 50341ae08745Sheppo return (EINVAL); 50351ae08745Sheppo } 50361ae08745Sheppo 50371ae08745Sheppo mutex_enter(&mhdl->lock); 50381ae08745Sheppo 50391ae08745Sheppo ldcp = mhdl->ldcp; 50401ae08745Sheppo memseg = mhdl->memseg; 50411ae08745Sheppo 50421ae08745Sheppo D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapping handle 0x%llx\n", 50431ae08745Sheppo ldcp->id, mhdl); 50441ae08745Sheppo 50451ae08745Sheppo /* if we allocated shadow memory - free it */ 50461ae08745Sheppo if (mhdl->mtype == LDC_SHADOW_MAP && mhdl->myshadow) { 50471ae08745Sheppo contig_mem_free(memseg->vaddr, memseg->size); 50481ae08745Sheppo } 50491ae08745Sheppo 50501ae08745Sheppo /* free the allocated memseg and page structures */ 50511ae08745Sheppo kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages)); 50521ae08745Sheppo kmem_free(memseg->cookies, 50531ae08745Sheppo (sizeof (ldc_mem_cookie_t) * memseg->ncookies)); 50541ae08745Sheppo kmem_free(memseg, sizeof (ldc_memseg_t)); 50551ae08745Sheppo 50561ae08745Sheppo /* uninitialize the memory handle */ 50571ae08745Sheppo mhdl->memseg = NULL; 50581ae08745Sheppo mhdl->status = LDC_UNBOUND; 50591ae08745Sheppo 50601ae08745Sheppo D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapped handle 0x%llx\n", 50611ae08745Sheppo ldcp->id, mhdl); 50621ae08745Sheppo 50631ae08745Sheppo mutex_exit(&mhdl->lock); 50641ae08745Sheppo return (0); 50651ae08745Sheppo } 50661ae08745Sheppo 50671ae08745Sheppo /* 50681ae08745Sheppo * Internal entry point for LDC mapped memory entry consistency 50691ae08745Sheppo * semantics. Acquire copies the contents of the remote memory 50701ae08745Sheppo * into the local shadow copy. The release operation copies the local 50711ae08745Sheppo * contents into the remote memory. The offset and size specify the 50721ae08745Sheppo * bounds for the memory range being synchronized. 50731ae08745Sheppo */ 50741ae08745Sheppo static int 50751ae08745Sheppo i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, uint8_t direction, 50761ae08745Sheppo uint64_t offset, size_t size) 50771ae08745Sheppo { 50781ae08745Sheppo int err; 50791ae08745Sheppo ldc_mhdl_t *mhdl; 50801ae08745Sheppo ldc_chan_t *ldcp; 50811ae08745Sheppo ldc_memseg_t *memseg; 50821ae08745Sheppo caddr_t local_vaddr; 50831ae08745Sheppo size_t copy_size; 50841ae08745Sheppo 50851ae08745Sheppo if (mhandle == NULL) { 50861ae08745Sheppo DWARN(DBG_ALL_LDCS, 50871ae08745Sheppo "i_ldc_mem_acquire_release: invalid memory handle\n"); 50881ae08745Sheppo return (EINVAL); 50891ae08745Sheppo } 50901ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 50911ae08745Sheppo 50921ae08745Sheppo mutex_enter(&mhdl->lock); 50931ae08745Sheppo 50941ae08745Sheppo if (mhdl->status != LDC_MAPPED || mhdl->ldcp == NULL) { 50951ae08745Sheppo DWARN(DBG_ALL_LDCS, 50961ae08745Sheppo "i_ldc_mem_acquire_release: not mapped memory\n"); 50971ae08745Sheppo mutex_exit(&mhdl->lock); 50981ae08745Sheppo return (EINVAL); 50991ae08745Sheppo } 51001ae08745Sheppo 51011ae08745Sheppo if (offset >= mhdl->memseg->size || 51021ae08745Sheppo (offset + size) > mhdl->memseg->size) { 51031ae08745Sheppo DWARN(DBG_ALL_LDCS, 51041ae08745Sheppo "i_ldc_mem_acquire_release: memory out of range\n"); 51051ae08745Sheppo mutex_exit(&mhdl->lock); 51061ae08745Sheppo return (EINVAL); 51071ae08745Sheppo } 51081ae08745Sheppo 51091ae08745Sheppo /* get the channel handle and memory segment */ 51101ae08745Sheppo ldcp = mhdl->ldcp; 51111ae08745Sheppo memseg = mhdl->memseg; 51121ae08745Sheppo 51131ae08745Sheppo if (mhdl->mtype == LDC_SHADOW_MAP) { 51141ae08745Sheppo 51151ae08745Sheppo local_vaddr = memseg->vaddr + offset; 51161ae08745Sheppo copy_size = size; 51171ae08745Sheppo 51181ae08745Sheppo /* copy to/from remote from/to local memory */ 51191ae08745Sheppo err = ldc_mem_copy((ldc_handle_t)ldcp, local_vaddr, offset, 51201ae08745Sheppo ©_size, memseg->cookies, memseg->ncookies, 51211ae08745Sheppo direction); 51221ae08745Sheppo if (err || copy_size != size) { 51231ae08745Sheppo cmn_err(CE_WARN, 51241ae08745Sheppo "i_ldc_mem_acquire_release: copy failed\n"); 51251ae08745Sheppo mutex_exit(&mhdl->lock); 51261ae08745Sheppo return (err); 51271ae08745Sheppo } 51281ae08745Sheppo } 51291ae08745Sheppo 51301ae08745Sheppo mutex_exit(&mhdl->lock); 51311ae08745Sheppo 51321ae08745Sheppo return (0); 51331ae08745Sheppo } 51341ae08745Sheppo 51351ae08745Sheppo /* 51361ae08745Sheppo * Ensure that the contents in the remote memory seg are consistent 51371ae08745Sheppo * with the contents if of local segment 51381ae08745Sheppo */ 51391ae08745Sheppo int 51401ae08745Sheppo ldc_mem_acquire(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size) 51411ae08745Sheppo { 51421ae08745Sheppo return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_IN, offset, size)); 51431ae08745Sheppo } 51441ae08745Sheppo 51451ae08745Sheppo 51461ae08745Sheppo /* 51471ae08745Sheppo * Ensure that the contents in the local memory seg are consistent 51481ae08745Sheppo * with the contents if of remote segment 51491ae08745Sheppo */ 51501ae08745Sheppo int 51511ae08745Sheppo ldc_mem_release(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size) 51521ae08745Sheppo { 51531ae08745Sheppo return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_OUT, offset, size)); 51541ae08745Sheppo } 51551ae08745Sheppo 51561ae08745Sheppo /* 51571ae08745Sheppo * Allocate a descriptor ring. The size of each each descriptor 51581ae08745Sheppo * must be 8-byte aligned and the entire ring should be a multiple 51591ae08745Sheppo * of MMU_PAGESIZE. 51601ae08745Sheppo */ 51611ae08745Sheppo int 51621ae08745Sheppo ldc_mem_dring_create(uint32_t len, uint32_t dsize, ldc_dring_handle_t *dhandle) 51631ae08745Sheppo { 51641ae08745Sheppo ldc_dring_t *dringp; 51651ae08745Sheppo size_t size = (dsize * len); 51661ae08745Sheppo 51671ae08745Sheppo D1(DBG_ALL_LDCS, "ldc_mem_dring_create: len=0x%x, size=0x%x\n", 51681ae08745Sheppo len, dsize); 51691ae08745Sheppo 51701ae08745Sheppo if (dhandle == NULL) { 51711ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid dhandle\n"); 51721ae08745Sheppo return (EINVAL); 51731ae08745Sheppo } 51741ae08745Sheppo 51751ae08745Sheppo if (len == 0) { 51761ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid length\n"); 51771ae08745Sheppo return (EINVAL); 51781ae08745Sheppo } 51791ae08745Sheppo 51801ae08745Sheppo /* descriptor size should be 8-byte aligned */ 51811ae08745Sheppo if (dsize == 0 || (dsize & 0x7)) { 51821ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid size\n"); 51831ae08745Sheppo return (EINVAL); 51841ae08745Sheppo } 51851ae08745Sheppo 51861ae08745Sheppo *dhandle = 0; 51871ae08745Sheppo 51881ae08745Sheppo /* Allocate a desc ring structure */ 51891ae08745Sheppo dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP); 51901ae08745Sheppo 51911ae08745Sheppo /* Initialize dring */ 51921ae08745Sheppo dringp->length = len; 51931ae08745Sheppo dringp->dsize = dsize; 51941ae08745Sheppo 51951ae08745Sheppo /* round off to multiple of pagesize */ 51961ae08745Sheppo dringp->size = (size & MMU_PAGEMASK); 51971ae08745Sheppo if (size & MMU_PAGEOFFSET) 51981ae08745Sheppo dringp->size += MMU_PAGESIZE; 51991ae08745Sheppo 52001ae08745Sheppo dringp->status = LDC_UNBOUND; 52011ae08745Sheppo 52021ae08745Sheppo /* allocate descriptor ring memory */ 52031ae08745Sheppo dringp->base = contig_mem_alloc_align(dringp->size, PAGESIZE); 52041ae08745Sheppo if (dringp->base == NULL) { 52051ae08745Sheppo cmn_err(CE_WARN, 52061ae08745Sheppo "ldc_mem_dring_create: unable to alloc desc\n"); 52071ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 52081ae08745Sheppo return (ENOMEM); 52091ae08745Sheppo } 52101ae08745Sheppo 52111ae08745Sheppo bzero(dringp->base, dringp->size); 52121ae08745Sheppo 52131ae08745Sheppo /* initialize the desc ring lock */ 52141ae08745Sheppo mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL); 52151ae08745Sheppo 52161ae08745Sheppo /* Add descriptor ring to the head of global list */ 52171ae08745Sheppo mutex_enter(&ldcssp->lock); 52181ae08745Sheppo dringp->next = ldcssp->dring_list; 52191ae08745Sheppo ldcssp->dring_list = dringp; 52201ae08745Sheppo mutex_exit(&ldcssp->lock); 52211ae08745Sheppo 52221ae08745Sheppo *dhandle = (ldc_dring_handle_t)dringp; 52231ae08745Sheppo 52241ae08745Sheppo D1(DBG_ALL_LDCS, "ldc_mem_dring_create: dring allocated\n"); 52251ae08745Sheppo 52261ae08745Sheppo return (0); 52271ae08745Sheppo } 52281ae08745Sheppo 52291ae08745Sheppo 52301ae08745Sheppo /* 52311ae08745Sheppo * Destroy a descriptor ring. 52321ae08745Sheppo */ 52331ae08745Sheppo int 52341ae08745Sheppo ldc_mem_dring_destroy(ldc_dring_handle_t dhandle) 52351ae08745Sheppo { 52361ae08745Sheppo ldc_dring_t *dringp; 52371ae08745Sheppo ldc_dring_t *tmp_dringp; 52381ae08745Sheppo 52391ae08745Sheppo D1(DBG_ALL_LDCS, "ldc_mem_dring_destroy: entered\n"); 52401ae08745Sheppo 52411ae08745Sheppo if (dhandle == NULL) { 52421ae08745Sheppo DWARN(DBG_ALL_LDCS, 52431ae08745Sheppo "ldc_mem_dring_destroy: invalid desc ring handle\n"); 52441ae08745Sheppo return (EINVAL); 52451ae08745Sheppo } 52461ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 52471ae08745Sheppo 52481ae08745Sheppo if (dringp->status == LDC_BOUND) { 52491ae08745Sheppo DWARN(DBG_ALL_LDCS, 52501ae08745Sheppo "ldc_mem_dring_destroy: desc ring is bound\n"); 52511ae08745Sheppo return (EACCES); 52521ae08745Sheppo } 52531ae08745Sheppo 52541ae08745Sheppo mutex_enter(&dringp->lock); 52551ae08745Sheppo mutex_enter(&ldcssp->lock); 52561ae08745Sheppo 52571ae08745Sheppo /* remove from linked list - if not bound */ 52581ae08745Sheppo tmp_dringp = ldcssp->dring_list; 52591ae08745Sheppo if (tmp_dringp == dringp) { 52601ae08745Sheppo ldcssp->dring_list = dringp->next; 52611ae08745Sheppo dringp->next = NULL; 52621ae08745Sheppo 52631ae08745Sheppo } else { 52641ae08745Sheppo while (tmp_dringp != NULL) { 52651ae08745Sheppo if (tmp_dringp->next == dringp) { 52661ae08745Sheppo tmp_dringp->next = dringp->next; 52671ae08745Sheppo dringp->next = NULL; 52681ae08745Sheppo break; 52691ae08745Sheppo } 52701ae08745Sheppo tmp_dringp = tmp_dringp->next; 52711ae08745Sheppo } 52721ae08745Sheppo if (tmp_dringp == NULL) { 52731ae08745Sheppo DWARN(DBG_ALL_LDCS, 52741ae08745Sheppo "ldc_mem_dring_destroy: invalid descriptor\n"); 52751ae08745Sheppo mutex_exit(&ldcssp->lock); 52761ae08745Sheppo mutex_exit(&dringp->lock); 52771ae08745Sheppo return (EINVAL); 52781ae08745Sheppo } 52791ae08745Sheppo } 52801ae08745Sheppo 52811ae08745Sheppo mutex_exit(&ldcssp->lock); 52821ae08745Sheppo 52831ae08745Sheppo /* free the descriptor ring */ 52841ae08745Sheppo contig_mem_free((caddr_t)dringp->base, dringp->size); 52851ae08745Sheppo 52861ae08745Sheppo mutex_exit(&dringp->lock); 52871ae08745Sheppo 52881ae08745Sheppo /* destroy dring lock */ 52891ae08745Sheppo mutex_destroy(&dringp->lock); 52901ae08745Sheppo 52911ae08745Sheppo /* free desc ring object */ 52921ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 52931ae08745Sheppo 52941ae08745Sheppo return (0); 52951ae08745Sheppo } 52961ae08745Sheppo 52971ae08745Sheppo /* 52981ae08745Sheppo * Bind a previously allocated dring to a channel. The channel should 52991ae08745Sheppo * be OPEN in order to bind the ring to the channel. Returns back a 53001ae08745Sheppo * descriptor ring cookie. The descriptor ring is exported for remote 53011ae08745Sheppo * access by the client at the other end of the channel. An entry for 53021ae08745Sheppo * dring pages is stored in map table (via call to ldc_mem_bind_handle). 53031ae08745Sheppo */ 53041ae08745Sheppo int 53051ae08745Sheppo ldc_mem_dring_bind(ldc_handle_t handle, ldc_dring_handle_t dhandle, 53061ae08745Sheppo uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount) 53071ae08745Sheppo { 53081ae08745Sheppo int err; 53091ae08745Sheppo ldc_chan_t *ldcp; 53101ae08745Sheppo ldc_dring_t *dringp; 53111ae08745Sheppo ldc_mem_handle_t mhandle; 53121ae08745Sheppo 53131ae08745Sheppo /* check to see if channel is initalized */ 53141ae08745Sheppo if (handle == NULL) { 53151ae08745Sheppo DWARN(DBG_ALL_LDCS, 53161ae08745Sheppo "ldc_mem_dring_bind: invalid channel handle\n"); 53171ae08745Sheppo return (EINVAL); 53181ae08745Sheppo } 53191ae08745Sheppo ldcp = (ldc_chan_t *)handle; 53201ae08745Sheppo 53211ae08745Sheppo if (dhandle == NULL) { 53221ae08745Sheppo DWARN(DBG_ALL_LDCS, 53231ae08745Sheppo "ldc_mem_dring_bind: invalid desc ring handle\n"); 53241ae08745Sheppo return (EINVAL); 53251ae08745Sheppo } 53261ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 53271ae08745Sheppo 53281ae08745Sheppo if (cookie == NULL) { 53291ae08745Sheppo DWARN(ldcp->id, 53301ae08745Sheppo "ldc_mem_dring_bind: invalid cookie arg\n"); 53311ae08745Sheppo return (EINVAL); 53321ae08745Sheppo } 53331ae08745Sheppo 53341ae08745Sheppo mutex_enter(&dringp->lock); 53351ae08745Sheppo 53361ae08745Sheppo if (dringp->status == LDC_BOUND) { 53371ae08745Sheppo DWARN(DBG_ALL_LDCS, 53381ae08745Sheppo "ldc_mem_dring_bind: (0x%llx) descriptor ring is bound\n", 53391ae08745Sheppo ldcp->id); 53401ae08745Sheppo mutex_exit(&dringp->lock); 53411ae08745Sheppo return (EINVAL); 53421ae08745Sheppo } 53431ae08745Sheppo 53441ae08745Sheppo if ((perm & LDC_MEM_RW) == 0) { 53451ae08745Sheppo DWARN(DBG_ALL_LDCS, 53461ae08745Sheppo "ldc_mem_dring_bind: invalid permissions\n"); 53471ae08745Sheppo mutex_exit(&dringp->lock); 53481ae08745Sheppo return (EINVAL); 53491ae08745Sheppo } 53501ae08745Sheppo 53511ae08745Sheppo if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) { 53521ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_dring_bind: invalid type\n"); 53531ae08745Sheppo mutex_exit(&dringp->lock); 53541ae08745Sheppo return (EINVAL); 53551ae08745Sheppo } 53561ae08745Sheppo 53571ae08745Sheppo dringp->ldcp = ldcp; 53581ae08745Sheppo 53591ae08745Sheppo /* create an memory handle */ 53601ae08745Sheppo err = ldc_mem_alloc_handle(handle, &mhandle); 53611ae08745Sheppo if (err || mhandle == NULL) { 53621ae08745Sheppo DWARN(DBG_ALL_LDCS, 53631ae08745Sheppo "ldc_mem_dring_bind: (0x%llx) error allocating mhandle\n", 53641ae08745Sheppo ldcp->id); 53651ae08745Sheppo mutex_exit(&dringp->lock); 53661ae08745Sheppo return (err); 53671ae08745Sheppo } 53681ae08745Sheppo dringp->mhdl = mhandle; 53691ae08745Sheppo 53701ae08745Sheppo /* bind the descriptor ring to channel */ 53711ae08745Sheppo err = ldc_mem_bind_handle(mhandle, dringp->base, dringp->size, 53721ae08745Sheppo mtype, perm, cookie, ccount); 53731ae08745Sheppo if (err) { 53741ae08745Sheppo DWARN(ldcp->id, 53751ae08745Sheppo "ldc_mem_dring_bind: (0x%llx) error binding mhandle\n", 53761ae08745Sheppo ldcp->id); 53771ae08745Sheppo mutex_exit(&dringp->lock); 53781ae08745Sheppo return (err); 53791ae08745Sheppo } 53801ae08745Sheppo 53811ae08745Sheppo /* 53821ae08745Sheppo * For now return error if we get more than one cookie 53831ae08745Sheppo * FUTURE: Return multiple cookies .. 53841ae08745Sheppo */ 53851ae08745Sheppo if (*ccount > 1) { 53861ae08745Sheppo (void) ldc_mem_unbind_handle(mhandle); 53871ae08745Sheppo (void) ldc_mem_free_handle(mhandle); 53881ae08745Sheppo 53891ae08745Sheppo dringp->ldcp = NULL; 53901ae08745Sheppo dringp->mhdl = NULL; 53911ae08745Sheppo *ccount = 0; 53921ae08745Sheppo 53931ae08745Sheppo mutex_exit(&dringp->lock); 53941ae08745Sheppo return (EAGAIN); 53951ae08745Sheppo } 53961ae08745Sheppo 53971ae08745Sheppo /* Add descriptor ring to channel's exported dring list */ 53981ae08745Sheppo mutex_enter(&ldcp->exp_dlist_lock); 53991ae08745Sheppo dringp->ch_next = ldcp->exp_dring_list; 54001ae08745Sheppo ldcp->exp_dring_list = dringp; 54011ae08745Sheppo mutex_exit(&ldcp->exp_dlist_lock); 54021ae08745Sheppo 54031ae08745Sheppo dringp->status = LDC_BOUND; 54041ae08745Sheppo 54051ae08745Sheppo mutex_exit(&dringp->lock); 54061ae08745Sheppo 54071ae08745Sheppo return (0); 54081ae08745Sheppo } 54091ae08745Sheppo 54101ae08745Sheppo /* 54111ae08745Sheppo * Return the next cookie associated with the specified dring handle 54121ae08745Sheppo */ 54131ae08745Sheppo int 54141ae08745Sheppo ldc_mem_dring_nextcookie(ldc_dring_handle_t dhandle, ldc_mem_cookie_t *cookie) 54151ae08745Sheppo { 54161ae08745Sheppo int rv = 0; 54171ae08745Sheppo ldc_dring_t *dringp; 54181ae08745Sheppo ldc_chan_t *ldcp; 54191ae08745Sheppo 54201ae08745Sheppo if (dhandle == NULL) { 54211ae08745Sheppo DWARN(DBG_ALL_LDCS, 54221ae08745Sheppo "ldc_mem_dring_nextcookie: invalid desc ring handle\n"); 54231ae08745Sheppo return (EINVAL); 54241ae08745Sheppo } 54251ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 54261ae08745Sheppo mutex_enter(&dringp->lock); 54271ae08745Sheppo 54281ae08745Sheppo if (dringp->status != LDC_BOUND) { 54291ae08745Sheppo DWARN(DBG_ALL_LDCS, 54301ae08745Sheppo "ldc_mem_dring_nextcookie: descriptor ring 0x%llx " 54311ae08745Sheppo "is not bound\n", dringp); 54321ae08745Sheppo mutex_exit(&dringp->lock); 54331ae08745Sheppo return (EINVAL); 54341ae08745Sheppo } 54351ae08745Sheppo 54361ae08745Sheppo ldcp = dringp->ldcp; 54371ae08745Sheppo 54381ae08745Sheppo if (cookie == NULL) { 54391ae08745Sheppo DWARN(ldcp->id, 54401ae08745Sheppo "ldc_mem_dring_nextcookie:(0x%llx) invalid cookie arg\n", 54411ae08745Sheppo ldcp->id); 54421ae08745Sheppo mutex_exit(&dringp->lock); 54431ae08745Sheppo return (EINVAL); 54441ae08745Sheppo } 54451ae08745Sheppo 54461ae08745Sheppo rv = ldc_mem_nextcookie((ldc_mem_handle_t)dringp->mhdl, cookie); 54471ae08745Sheppo mutex_exit(&dringp->lock); 54481ae08745Sheppo 54491ae08745Sheppo return (rv); 54501ae08745Sheppo } 54511ae08745Sheppo /* 54521ae08745Sheppo * Unbind a previously bound dring from a channel. 54531ae08745Sheppo */ 54541ae08745Sheppo int 54551ae08745Sheppo ldc_mem_dring_unbind(ldc_dring_handle_t dhandle) 54561ae08745Sheppo { 54571ae08745Sheppo ldc_dring_t *dringp; 54581ae08745Sheppo ldc_dring_t *tmp_dringp; 54591ae08745Sheppo ldc_chan_t *ldcp; 54601ae08745Sheppo 54611ae08745Sheppo if (dhandle == NULL) { 54621ae08745Sheppo DWARN(DBG_ALL_LDCS, 54631ae08745Sheppo "ldc_mem_dring_unbind: invalid desc ring handle\n"); 54641ae08745Sheppo return (EINVAL); 54651ae08745Sheppo } 54661ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 54671ae08745Sheppo 54681ae08745Sheppo mutex_enter(&dringp->lock); 54691ae08745Sheppo 54701ae08745Sheppo if (dringp->status == LDC_UNBOUND) { 54711ae08745Sheppo DWARN(DBG_ALL_LDCS, 54721ae08745Sheppo "ldc_mem_dring_bind: descriptor ring 0x%llx is unbound\n", 54731ae08745Sheppo dringp); 54741ae08745Sheppo mutex_exit(&dringp->lock); 54751ae08745Sheppo return (EINVAL); 54761ae08745Sheppo } 54771ae08745Sheppo ldcp = dringp->ldcp; 54781ae08745Sheppo 54791ae08745Sheppo mutex_enter(&ldcp->exp_dlist_lock); 54801ae08745Sheppo 54811ae08745Sheppo tmp_dringp = ldcp->exp_dring_list; 54821ae08745Sheppo if (tmp_dringp == dringp) { 54831ae08745Sheppo ldcp->exp_dring_list = dringp->ch_next; 54841ae08745Sheppo dringp->ch_next = NULL; 54851ae08745Sheppo 54861ae08745Sheppo } else { 54871ae08745Sheppo while (tmp_dringp != NULL) { 54881ae08745Sheppo if (tmp_dringp->ch_next == dringp) { 54891ae08745Sheppo tmp_dringp->ch_next = dringp->ch_next; 54901ae08745Sheppo dringp->ch_next = NULL; 54911ae08745Sheppo break; 54921ae08745Sheppo } 54931ae08745Sheppo tmp_dringp = tmp_dringp->ch_next; 54941ae08745Sheppo } 54951ae08745Sheppo if (tmp_dringp == NULL) { 54961ae08745Sheppo DWARN(DBG_ALL_LDCS, 54971ae08745Sheppo "ldc_mem_dring_unbind: invalid descriptor\n"); 54981ae08745Sheppo mutex_exit(&ldcp->exp_dlist_lock); 54991ae08745Sheppo mutex_exit(&dringp->lock); 55001ae08745Sheppo return (EINVAL); 55011ae08745Sheppo } 55021ae08745Sheppo } 55031ae08745Sheppo 55041ae08745Sheppo mutex_exit(&ldcp->exp_dlist_lock); 55051ae08745Sheppo 55061ae08745Sheppo (void) ldc_mem_unbind_handle((ldc_mem_handle_t)dringp->mhdl); 55071ae08745Sheppo (void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl); 55081ae08745Sheppo 55091ae08745Sheppo dringp->ldcp = NULL; 55101ae08745Sheppo dringp->mhdl = NULL; 55111ae08745Sheppo dringp->status = LDC_UNBOUND; 55121ae08745Sheppo 55131ae08745Sheppo mutex_exit(&dringp->lock); 55141ae08745Sheppo 55151ae08745Sheppo return (0); 55161ae08745Sheppo } 55171ae08745Sheppo 55181ae08745Sheppo /* 55191ae08745Sheppo * Get information about the dring. The base address of the descriptor 55201ae08745Sheppo * ring along with the type and permission are returned back. 55211ae08745Sheppo */ 55221ae08745Sheppo int 55231ae08745Sheppo ldc_mem_dring_info(ldc_dring_handle_t dhandle, ldc_mem_info_t *minfo) 55241ae08745Sheppo { 55251ae08745Sheppo ldc_dring_t *dringp; 55261ae08745Sheppo int rv; 55271ae08745Sheppo 55281ae08745Sheppo if (dhandle == NULL) { 55291ae08745Sheppo DWARN(DBG_ALL_LDCS, 55301ae08745Sheppo "ldc_mem_dring_info: invalid desc ring handle\n"); 55311ae08745Sheppo return (EINVAL); 55321ae08745Sheppo } 55331ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 55341ae08745Sheppo 55351ae08745Sheppo mutex_enter(&dringp->lock); 55361ae08745Sheppo 55371ae08745Sheppo if (dringp->mhdl) { 55381ae08745Sheppo rv = ldc_mem_info(dringp->mhdl, minfo); 55391ae08745Sheppo if (rv) { 55401ae08745Sheppo DWARN(DBG_ALL_LDCS, 55411ae08745Sheppo "ldc_mem_dring_info: error reading mem info\n"); 55421ae08745Sheppo mutex_exit(&dringp->lock); 55431ae08745Sheppo return (rv); 55441ae08745Sheppo } 55451ae08745Sheppo } else { 55461ae08745Sheppo minfo->vaddr = dringp->base; 55471ae08745Sheppo minfo->raddr = NULL; 55481ae08745Sheppo minfo->status = dringp->status; 55491ae08745Sheppo } 55501ae08745Sheppo 55511ae08745Sheppo mutex_exit(&dringp->lock); 55521ae08745Sheppo 55531ae08745Sheppo return (0); 55541ae08745Sheppo } 55551ae08745Sheppo 55561ae08745Sheppo /* 55571ae08745Sheppo * Map an exported descriptor ring into the local address space. If the 55581ae08745Sheppo * descriptor ring was exported for direct map access, a HV call is made 55591ae08745Sheppo * to allocate a RA range. If the map is done via a shadow copy, local 55601ae08745Sheppo * shadow memory is allocated. 55611ae08745Sheppo */ 55621ae08745Sheppo int 55631ae08745Sheppo ldc_mem_dring_map(ldc_handle_t handle, ldc_mem_cookie_t *cookie, 55641ae08745Sheppo uint32_t ccount, uint32_t len, uint32_t dsize, uint8_t mtype, 55651ae08745Sheppo ldc_dring_handle_t *dhandle) 55661ae08745Sheppo { 55671ae08745Sheppo int err; 55681ae08745Sheppo ldc_chan_t *ldcp = (ldc_chan_t *)handle; 55691ae08745Sheppo ldc_mem_handle_t mhandle; 55701ae08745Sheppo ldc_dring_t *dringp; 55711ae08745Sheppo size_t dring_size; 55721ae08745Sheppo 55731ae08745Sheppo if (dhandle == NULL) { 55741ae08745Sheppo DWARN(DBG_ALL_LDCS, 55751ae08745Sheppo "ldc_mem_dring_map: invalid dhandle\n"); 55761ae08745Sheppo return (EINVAL); 55771ae08745Sheppo } 55781ae08745Sheppo 55791ae08745Sheppo /* check to see if channel is initalized */ 55801ae08745Sheppo if (handle == NULL) { 55811ae08745Sheppo DWARN(DBG_ALL_LDCS, 55821ae08745Sheppo "ldc_mem_dring_map: invalid channel handle\n"); 55831ae08745Sheppo return (EINVAL); 55841ae08745Sheppo } 55851ae08745Sheppo ldcp = (ldc_chan_t *)handle; 55861ae08745Sheppo 55871ae08745Sheppo if (cookie == NULL) { 55881ae08745Sheppo DWARN(ldcp->id, 55891ae08745Sheppo "ldc_mem_dring_map: (0x%llx) invalid cookie\n", 55901ae08745Sheppo ldcp->id); 55911ae08745Sheppo return (EINVAL); 55921ae08745Sheppo } 55931ae08745Sheppo 55941ae08745Sheppo /* FUTURE: For now we support only one cookie per dring */ 55951ae08745Sheppo ASSERT(ccount == 1); 55961ae08745Sheppo 55971ae08745Sheppo if (cookie->size < (dsize * len)) { 55981ae08745Sheppo DWARN(ldcp->id, 55991ae08745Sheppo "ldc_mem_dring_map: (0x%llx) invalid dsize/len\n", 56001ae08745Sheppo ldcp->id); 56011ae08745Sheppo return (EINVAL); 56021ae08745Sheppo } 56031ae08745Sheppo 56041ae08745Sheppo *dhandle = 0; 56051ae08745Sheppo 56061ae08745Sheppo /* Allocate an dring structure */ 56071ae08745Sheppo dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP); 56081ae08745Sheppo 56091ae08745Sheppo D1(ldcp->id, 56101ae08745Sheppo "ldc_mem_dring_map: 0x%x,0x%x,0x%x,0x%llx,0x%llx\n", 56111ae08745Sheppo mtype, len, dsize, cookie->addr, cookie->size); 56121ae08745Sheppo 56131ae08745Sheppo /* Initialize dring */ 56141ae08745Sheppo dringp->length = len; 56151ae08745Sheppo dringp->dsize = dsize; 56161ae08745Sheppo 56171ae08745Sheppo /* round of to multiple of page size */ 56181ae08745Sheppo dring_size = len * dsize; 56191ae08745Sheppo dringp->size = (dring_size & MMU_PAGEMASK); 56201ae08745Sheppo if (dring_size & MMU_PAGEOFFSET) 56211ae08745Sheppo dringp->size += MMU_PAGESIZE; 56221ae08745Sheppo 56231ae08745Sheppo dringp->ldcp = ldcp; 56241ae08745Sheppo 56251ae08745Sheppo /* create an memory handle */ 56261ae08745Sheppo err = ldc_mem_alloc_handle(handle, &mhandle); 56271ae08745Sheppo if (err || mhandle == NULL) { 56281ae08745Sheppo DWARN(DBG_ALL_LDCS, 56291ae08745Sheppo "ldc_mem_dring_map: cannot alloc hdl err=%d\n", 56301ae08745Sheppo err); 56311ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 56321ae08745Sheppo return (ENOMEM); 56331ae08745Sheppo } 56341ae08745Sheppo 56351ae08745Sheppo dringp->mhdl = mhandle; 56361ae08745Sheppo dringp->base = NULL; 56371ae08745Sheppo 56381ae08745Sheppo /* map the dring into local memory */ 56391ae08745Sheppo err = ldc_mem_map(mhandle, cookie, ccount, mtype, 56401ae08745Sheppo &(dringp->base), NULL); 56411ae08745Sheppo if (err || dringp->base == NULL) { 56421ae08745Sheppo cmn_err(CE_WARN, 56431ae08745Sheppo "ldc_mem_dring_map: cannot map desc ring err=%d\n", err); 56441ae08745Sheppo (void) ldc_mem_free_handle(mhandle); 56451ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 56461ae08745Sheppo return (ENOMEM); 56471ae08745Sheppo } 56481ae08745Sheppo 56491ae08745Sheppo /* initialize the desc ring lock */ 56501ae08745Sheppo mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL); 56511ae08745Sheppo 56521ae08745Sheppo /* Add descriptor ring to channel's imported dring list */ 56531ae08745Sheppo mutex_enter(&ldcp->imp_dlist_lock); 56541ae08745Sheppo dringp->ch_next = ldcp->imp_dring_list; 56551ae08745Sheppo ldcp->imp_dring_list = dringp; 56561ae08745Sheppo mutex_exit(&ldcp->imp_dlist_lock); 56571ae08745Sheppo 56581ae08745Sheppo dringp->status = LDC_MAPPED; 56591ae08745Sheppo 56601ae08745Sheppo *dhandle = (ldc_dring_handle_t)dringp; 56611ae08745Sheppo 56621ae08745Sheppo return (0); 56631ae08745Sheppo } 56641ae08745Sheppo 56651ae08745Sheppo /* 56661ae08745Sheppo * Unmap a descriptor ring. Free shadow memory (if any). 56671ae08745Sheppo */ 56681ae08745Sheppo int 56691ae08745Sheppo ldc_mem_dring_unmap(ldc_dring_handle_t dhandle) 56701ae08745Sheppo { 56711ae08745Sheppo ldc_dring_t *dringp; 56721ae08745Sheppo ldc_dring_t *tmp_dringp; 56731ae08745Sheppo ldc_chan_t *ldcp; 56741ae08745Sheppo 56751ae08745Sheppo if (dhandle == NULL) { 56761ae08745Sheppo DWARN(DBG_ALL_LDCS, 56771ae08745Sheppo "ldc_mem_dring_unmap: invalid desc ring handle\n"); 56781ae08745Sheppo return (EINVAL); 56791ae08745Sheppo } 56801ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 56811ae08745Sheppo 56821ae08745Sheppo if (dringp->status != LDC_MAPPED) { 56831ae08745Sheppo DWARN(DBG_ALL_LDCS, 56841ae08745Sheppo "ldc_mem_dring_unmap: not a mapped desc ring\n"); 56851ae08745Sheppo return (EINVAL); 56861ae08745Sheppo } 56871ae08745Sheppo 56881ae08745Sheppo mutex_enter(&dringp->lock); 56891ae08745Sheppo 56901ae08745Sheppo ldcp = dringp->ldcp; 56911ae08745Sheppo 56921ae08745Sheppo mutex_enter(&ldcp->imp_dlist_lock); 56931ae08745Sheppo 56941ae08745Sheppo /* find and unlink the desc ring from channel import list */ 56951ae08745Sheppo tmp_dringp = ldcp->imp_dring_list; 56961ae08745Sheppo if (tmp_dringp == dringp) { 56971ae08745Sheppo ldcp->imp_dring_list = dringp->ch_next; 56981ae08745Sheppo dringp->ch_next = NULL; 56991ae08745Sheppo 57001ae08745Sheppo } else { 57011ae08745Sheppo while (tmp_dringp != NULL) { 57021ae08745Sheppo if (tmp_dringp->ch_next == dringp) { 57031ae08745Sheppo tmp_dringp->ch_next = dringp->ch_next; 57041ae08745Sheppo dringp->ch_next = NULL; 57051ae08745Sheppo break; 57061ae08745Sheppo } 57071ae08745Sheppo tmp_dringp = tmp_dringp->ch_next; 57081ae08745Sheppo } 57091ae08745Sheppo if (tmp_dringp == NULL) { 57101ae08745Sheppo DWARN(DBG_ALL_LDCS, 57111ae08745Sheppo "ldc_mem_dring_unmap: invalid descriptor\n"); 57121ae08745Sheppo mutex_exit(&ldcp->imp_dlist_lock); 57131ae08745Sheppo mutex_exit(&dringp->lock); 57141ae08745Sheppo return (EINVAL); 57151ae08745Sheppo } 57161ae08745Sheppo } 57171ae08745Sheppo 57181ae08745Sheppo mutex_exit(&ldcp->imp_dlist_lock); 57191ae08745Sheppo 57201ae08745Sheppo /* do a LDC memory handle unmap and free */ 57211ae08745Sheppo (void) ldc_mem_unmap(dringp->mhdl); 57221ae08745Sheppo (void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl); 57231ae08745Sheppo 57241ae08745Sheppo dringp->status = 0; 57251ae08745Sheppo dringp->ldcp = NULL; 57261ae08745Sheppo 57271ae08745Sheppo mutex_exit(&dringp->lock); 57281ae08745Sheppo 57291ae08745Sheppo /* destroy dring lock */ 57301ae08745Sheppo mutex_destroy(&dringp->lock); 57311ae08745Sheppo 57321ae08745Sheppo /* free desc ring object */ 57331ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 57341ae08745Sheppo 57351ae08745Sheppo return (0); 57361ae08745Sheppo } 57371ae08745Sheppo 57381ae08745Sheppo /* 57391ae08745Sheppo * Internal entry point for descriptor ring access entry consistency 57401ae08745Sheppo * semantics. Acquire copies the contents of the remote descriptor ring 57411ae08745Sheppo * into the local shadow copy. The release operation copies the local 57421ae08745Sheppo * contents into the remote dring. The start and end locations specify 57431ae08745Sheppo * bounds for the entries being synchronized. 57441ae08745Sheppo */ 57451ae08745Sheppo static int 57461ae08745Sheppo i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle, 57471ae08745Sheppo uint8_t direction, uint64_t start, uint64_t end) 57481ae08745Sheppo { 57491ae08745Sheppo int err; 57501ae08745Sheppo ldc_dring_t *dringp; 57511ae08745Sheppo ldc_chan_t *ldcp; 57521ae08745Sheppo uint64_t soff; 57531ae08745Sheppo size_t copy_size; 57541ae08745Sheppo 57551ae08745Sheppo if (dhandle == NULL) { 57561ae08745Sheppo DWARN(DBG_ALL_LDCS, 57571ae08745Sheppo "i_ldc_dring_acquire_release: invalid desc ring handle\n"); 57581ae08745Sheppo return (EINVAL); 57591ae08745Sheppo } 57601ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 57611ae08745Sheppo mutex_enter(&dringp->lock); 57621ae08745Sheppo 57631ae08745Sheppo if (dringp->status != LDC_MAPPED || dringp->ldcp == NULL) { 57641ae08745Sheppo DWARN(DBG_ALL_LDCS, 57651ae08745Sheppo "i_ldc_dring_acquire_release: not a mapped desc ring\n"); 57661ae08745Sheppo mutex_exit(&dringp->lock); 57671ae08745Sheppo return (EINVAL); 57681ae08745Sheppo } 57691ae08745Sheppo 57701ae08745Sheppo if (start >= dringp->length || end >= dringp->length) { 57711ae08745Sheppo DWARN(DBG_ALL_LDCS, 57721ae08745Sheppo "i_ldc_dring_acquire_release: index out of range\n"); 57731ae08745Sheppo mutex_exit(&dringp->lock); 57741ae08745Sheppo return (EINVAL); 57751ae08745Sheppo } 57761ae08745Sheppo 57771ae08745Sheppo /* get the channel handle */ 57781ae08745Sheppo ldcp = dringp->ldcp; 57791ae08745Sheppo 57801ae08745Sheppo copy_size = (start <= end) ? (((end - start) + 1) * dringp->dsize) : 57811ae08745Sheppo ((dringp->length - start) * dringp->dsize); 57821ae08745Sheppo 57831ae08745Sheppo /* Calculate the relative offset for the first desc */ 57841ae08745Sheppo soff = (start * dringp->dsize); 57851ae08745Sheppo 57861ae08745Sheppo /* copy to/from remote from/to local memory */ 57871ae08745Sheppo D1(ldcp->id, "i_ldc_dring_acquire_release: c1 off=0x%llx sz=0x%llx\n", 57881ae08745Sheppo soff, copy_size); 57891ae08745Sheppo err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl, 57901ae08745Sheppo direction, soff, copy_size); 57911ae08745Sheppo if (err) { 57921ae08745Sheppo DWARN(ldcp->id, 57931ae08745Sheppo "i_ldc_dring_acquire_release: copy failed\n"); 57941ae08745Sheppo mutex_exit(&dringp->lock); 57951ae08745Sheppo return (err); 57961ae08745Sheppo } 57971ae08745Sheppo 57981ae08745Sheppo /* do the balance */ 57991ae08745Sheppo if (start > end) { 58001ae08745Sheppo copy_size = ((end + 1) * dringp->dsize); 58011ae08745Sheppo soff = 0; 58021ae08745Sheppo 58031ae08745Sheppo /* copy to/from remote from/to local memory */ 58041ae08745Sheppo D1(ldcp->id, "i_ldc_dring_acquire_release: c2 " 58051ae08745Sheppo "off=0x%llx sz=0x%llx\n", soff, copy_size); 58061ae08745Sheppo err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl, 58071ae08745Sheppo direction, soff, copy_size); 58081ae08745Sheppo if (err) { 58091ae08745Sheppo DWARN(ldcp->id, 58101ae08745Sheppo "i_ldc_dring_acquire_release: copy failed\n"); 58111ae08745Sheppo mutex_exit(&dringp->lock); 58121ae08745Sheppo return (err); 58131ae08745Sheppo } 58141ae08745Sheppo } 58151ae08745Sheppo 58161ae08745Sheppo mutex_exit(&dringp->lock); 58171ae08745Sheppo 58181ae08745Sheppo return (0); 58191ae08745Sheppo } 58201ae08745Sheppo 58211ae08745Sheppo /* 58221ae08745Sheppo * Ensure that the contents in the local dring are consistent 58231ae08745Sheppo * with the contents if of remote dring 58241ae08745Sheppo */ 58251ae08745Sheppo int 58261ae08745Sheppo ldc_mem_dring_acquire(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end) 58271ae08745Sheppo { 58281ae08745Sheppo return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_IN, start, end)); 58291ae08745Sheppo } 58301ae08745Sheppo 58311ae08745Sheppo /* 58321ae08745Sheppo * Ensure that the contents in the remote dring are consistent 58331ae08745Sheppo * with the contents if of local dring 58341ae08745Sheppo */ 58351ae08745Sheppo int 58361ae08745Sheppo ldc_mem_dring_release(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end) 58371ae08745Sheppo { 58381ae08745Sheppo return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_OUT, start, end)); 58391ae08745Sheppo } 58401ae08745Sheppo 58411ae08745Sheppo 58421ae08745Sheppo /* ------------------------------------------------------------------------- */ 5843