11ae08745Sheppo /* 21ae08745Sheppo * CDDL HEADER START 31ae08745Sheppo * 41ae08745Sheppo * The contents of this file are subject to the terms of the 51ae08745Sheppo * Common Development and Distribution License (the "License"). 61ae08745Sheppo * You may not use this file except in compliance with the License. 71ae08745Sheppo * 81ae08745Sheppo * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 91ae08745Sheppo * or http://www.opensolaris.org/os/licensing. 101ae08745Sheppo * See the License for the specific language governing permissions 111ae08745Sheppo * and limitations under the License. 121ae08745Sheppo * 131ae08745Sheppo * When distributing Covered Code, include this CDDL HEADER in each 141ae08745Sheppo * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 151ae08745Sheppo * If applicable, add the following below this CDDL HEADER, with the 161ae08745Sheppo * fields enclosed by brackets "[]" replaced with your own identifying 171ae08745Sheppo * information: Portions Copyright [yyyy] [name of copyright owner] 181ae08745Sheppo * 191ae08745Sheppo * CDDL HEADER END 201ae08745Sheppo */ 211ae08745Sheppo 221ae08745Sheppo /* 231ae08745Sheppo * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 241ae08745Sheppo * Use is subject to license terms. 251ae08745Sheppo */ 261ae08745Sheppo 271ae08745Sheppo #pragma ident "%Z%%M% %I% %E% SMI" 281ae08745Sheppo 291ae08745Sheppo /* 30*e1ebb9ecSlm66018 * sun4v LDC Link Layer 311ae08745Sheppo */ 321ae08745Sheppo #include <sys/types.h> 331ae08745Sheppo #include <sys/file.h> 341ae08745Sheppo #include <sys/errno.h> 351ae08745Sheppo #include <sys/open.h> 361ae08745Sheppo #include <sys/cred.h> 371ae08745Sheppo #include <sys/kmem.h> 381ae08745Sheppo #include <sys/conf.h> 391ae08745Sheppo #include <sys/cmn_err.h> 401ae08745Sheppo #include <sys/ksynch.h> 411ae08745Sheppo #include <sys/modctl.h> 421ae08745Sheppo #include <sys/stat.h> /* needed for S_IFBLK and S_IFCHR */ 431ae08745Sheppo #include <sys/debug.h> 441ae08745Sheppo #include <sys/types.h> 451ae08745Sheppo #include <sys/cred.h> 461ae08745Sheppo #include <sys/promif.h> 471ae08745Sheppo #include <sys/ddi.h> 481ae08745Sheppo #include <sys/sunddi.h> 491ae08745Sheppo #include <sys/cyclic.h> 501ae08745Sheppo #include <sys/machsystm.h> 511ae08745Sheppo #include <sys/vm.h> 521ae08745Sheppo #include <sys/cpu.h> 531ae08745Sheppo #include <sys/intreg.h> 541ae08745Sheppo #include <sys/machcpuvar.h> 551ae08745Sheppo #include <sys/note.h> 561ae08745Sheppo #include <sys/ivintr.h> 571ae08745Sheppo #include <sys/hypervisor_api.h> 581ae08745Sheppo #include <sys/ldc.h> 591ae08745Sheppo #include <sys/ldc_impl.h> 601ae08745Sheppo #include <sys/cnex.h> 611ae08745Sheppo #include <sys/hsvc.h> 621ae08745Sheppo 631ae08745Sheppo /* Core internal functions */ 641ae08745Sheppo static int i_ldc_h2v_error(int h_error); 651ae08745Sheppo static int i_ldc_txq_reconf(ldc_chan_t *ldcp); 661ae08745Sheppo static int i_ldc_rxq_reconf(ldc_chan_t *ldcp); 671ae08745Sheppo static void i_ldc_reset_state(ldc_chan_t *ldcp); 681ae08745Sheppo static void i_ldc_reset(ldc_chan_t *ldcp); 691ae08745Sheppo 701ae08745Sheppo static int i_ldc_get_tx_tail(ldc_chan_t *ldcp, uint64_t *tail); 711ae08745Sheppo static int i_ldc_set_tx_tail(ldc_chan_t *ldcp, uint64_t tail); 721ae08745Sheppo static int i_ldc_set_rx_head(ldc_chan_t *ldcp, uint64_t head); 731ae08745Sheppo static int i_ldc_send_pkt(ldc_chan_t *ldcp, uint8_t pkttype, uint8_t subtype, 741ae08745Sheppo uint8_t ctrlmsg); 751ae08745Sheppo 761ae08745Sheppo /* Interrupt handling functions */ 771ae08745Sheppo static uint_t i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2); 781ae08745Sheppo static uint_t i_ldc_rx_hdlr(caddr_t arg1, caddr_t arg2); 791ae08745Sheppo static void i_ldc_clear_intr(ldc_chan_t *ldcp, cnex_intrtype_t itype); 801ae08745Sheppo 811ae08745Sheppo /* Read method functions */ 821ae08745Sheppo static int i_ldc_read_raw(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep); 831ae08745Sheppo static int i_ldc_read_packet(ldc_chan_t *ldcp, caddr_t target_bufp, 841ae08745Sheppo size_t *sizep); 851ae08745Sheppo static int i_ldc_read_stream(ldc_chan_t *ldcp, caddr_t target_bufp, 861ae08745Sheppo size_t *sizep); 871ae08745Sheppo 881ae08745Sheppo /* Write method functions */ 891ae08745Sheppo static int i_ldc_write_raw(ldc_chan_t *ldcp, caddr_t target_bufp, 901ae08745Sheppo size_t *sizep); 911ae08745Sheppo static int i_ldc_write_packet(ldc_chan_t *ldcp, caddr_t target_bufp, 921ae08745Sheppo size_t *sizep); 931ae08745Sheppo static int i_ldc_write_stream(ldc_chan_t *ldcp, caddr_t target_bufp, 941ae08745Sheppo size_t *sizep); 951ae08745Sheppo 961ae08745Sheppo /* Pkt processing internal functions */ 971ae08745Sheppo static int i_ldc_check_seqid(ldc_chan_t *ldcp, ldc_msg_t *ldcmsg); 981ae08745Sheppo static int i_ldc_ctrlmsg(ldc_chan_t *ldcp, ldc_msg_t *ldcmsg); 991ae08745Sheppo static int i_ldc_process_VER(ldc_chan_t *ldcp, ldc_msg_t *msg); 1001ae08745Sheppo static int i_ldc_process_RTS(ldc_chan_t *ldcp, ldc_msg_t *msg); 1011ae08745Sheppo static int i_ldc_process_RTR(ldc_chan_t *ldcp, ldc_msg_t *msg); 1021ae08745Sheppo static int i_ldc_process_RDX(ldc_chan_t *ldcp, ldc_msg_t *msg); 1031ae08745Sheppo static int i_ldc_process_data_ACK(ldc_chan_t *ldcp, ldc_msg_t *msg); 1041ae08745Sheppo 1051ae08745Sheppo /* Memory synchronization internal functions */ 1061ae08745Sheppo static int i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, 1071ae08745Sheppo uint8_t direction, uint64_t offset, size_t size); 1081ae08745Sheppo static int i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle, 1091ae08745Sheppo uint8_t direction, uint64_t start, uint64_t end); 1101ae08745Sheppo 1111ae08745Sheppo /* LDC Version */ 1121ae08745Sheppo static ldc_ver_t ldc_versions[] = { {1, 0} }; 1131ae08745Sheppo 1141ae08745Sheppo /* number of supported versions */ 1151ae08745Sheppo #define LDC_NUM_VERS (sizeof (ldc_versions) / sizeof (ldc_versions[0])) 1161ae08745Sheppo 1171ae08745Sheppo /* Module State Pointer */ 1181ae08745Sheppo static ldc_soft_state_t *ldcssp; 1191ae08745Sheppo 1201ae08745Sheppo static struct modldrv md = { 1211ae08745Sheppo &mod_miscops, /* This is a misc module */ 1221ae08745Sheppo "sun4v LDC module v%I%", /* Name of the module */ 1231ae08745Sheppo }; 1241ae08745Sheppo 1251ae08745Sheppo static struct modlinkage ml = { 1261ae08745Sheppo MODREV_1, 1271ae08745Sheppo &md, 1281ae08745Sheppo NULL 1291ae08745Sheppo }; 1301ae08745Sheppo 1311ae08745Sheppo static uint64_t ldc_sup_minor; /* Supported minor number */ 1321ae08745Sheppo static hsvc_info_t ldc_hsvc = { 1331ae08745Sheppo HSVC_REV_1, NULL, HSVC_GROUP_LDC, 1, 0, "ldc" 1341ae08745Sheppo }; 1351ae08745Sheppo 1361ae08745Sheppo static uint64_t intr_sup_minor; /* Supported minor number */ 1371ae08745Sheppo static hsvc_info_t intr_hsvc = { 1381ae08745Sheppo HSVC_REV_1, NULL, HSVC_GROUP_INTR, 1, 0, "ldc" 1391ae08745Sheppo }; 1401ae08745Sheppo 141*e1ebb9ecSlm66018 1420a55fbb7Slm66018 /* 143*e1ebb9ecSlm66018 * The no. of MTU size messages that can be stored in 144*e1ebb9ecSlm66018 * the LDC Tx queue. The number of Tx queue entries is 145*e1ebb9ecSlm66018 * then computed as (mtu * mtu_msgs)/sizeof(queue_entry) 146*e1ebb9ecSlm66018 */ 147*e1ebb9ecSlm66018 uint64_t ldc_mtu_msgs = LDC_MTU_MSGS; 148*e1ebb9ecSlm66018 149*e1ebb9ecSlm66018 /* 150*e1ebb9ecSlm66018 * The minimum queue length. This is the size of the smallest 151*e1ebb9ecSlm66018 * LDC queue. If the computed value is less than this default, 152*e1ebb9ecSlm66018 * the queue length is rounded up to 'ldc_queue_entries'. 153*e1ebb9ecSlm66018 */ 154*e1ebb9ecSlm66018 uint64_t ldc_queue_entries = LDC_QUEUE_ENTRIES; 155*e1ebb9ecSlm66018 156*e1ebb9ecSlm66018 /* 157*e1ebb9ecSlm66018 * Pages exported for remote access over each channel is 158*e1ebb9ecSlm66018 * maintained in a table registered with the Hypervisor. 159*e1ebb9ecSlm66018 * The default number of entries in the table is set to 160*e1ebb9ecSlm66018 * 'ldc_mtbl_entries'. 161*e1ebb9ecSlm66018 */ 162*e1ebb9ecSlm66018 uint64_t ldc_maptable_entries = LDC_MTBL_ENTRIES; 163*e1ebb9ecSlm66018 164*e1ebb9ecSlm66018 /* 165*e1ebb9ecSlm66018 * LDC retry count and delay - when the HV returns EWOULDBLOCK 166*e1ebb9ecSlm66018 * the operation is retried 'ldc_max_retries' times with a 167*e1ebb9ecSlm66018 * wait of 'ldc_delay' usecs between each retry. 1680a55fbb7Slm66018 */ 1690a55fbb7Slm66018 int ldc_max_retries = LDC_MAX_RETRIES; 1700a55fbb7Slm66018 clock_t ldc_delay = LDC_DELAY; 1710a55fbb7Slm66018 1721ae08745Sheppo #ifdef DEBUG 1731ae08745Sheppo 1741ae08745Sheppo /* 1751ae08745Sheppo * Print debug messages 1761ae08745Sheppo * 1771ae08745Sheppo * set ldcdbg to 0x7 for enabling all msgs 1781ae08745Sheppo * 0x4 - Warnings 1791ae08745Sheppo * 0x2 - All debug messages 1801ae08745Sheppo * 0x1 - Minimal debug messages 1811ae08745Sheppo * 1821ae08745Sheppo * set ldcdbgchan to the channel number you want to debug 1831ae08745Sheppo * setting it to -1 prints debug messages for all channels 1841ae08745Sheppo * NOTE: ldcdbgchan has no effect on error messages 1851ae08745Sheppo */ 1861ae08745Sheppo 1871ae08745Sheppo #define DBG_ALL_LDCS -1 1881ae08745Sheppo 1891ae08745Sheppo int ldcdbg = 0x0; 1901ae08745Sheppo int64_t ldcdbgchan = DBG_ALL_LDCS; 1911ae08745Sheppo 1921ae08745Sheppo static void 1931ae08745Sheppo ldcdebug(int64_t id, const char *fmt, ...) 1941ae08745Sheppo { 1951ae08745Sheppo char buf[512]; 1961ae08745Sheppo va_list ap; 1971ae08745Sheppo 1981ae08745Sheppo /* 1991ae08745Sheppo * Do not return if, 2001ae08745Sheppo * caller wants to print it anyway - (id == DBG_ALL_LDCS) 2011ae08745Sheppo * debug channel is set to all LDCs - (ldcdbgchan == DBG_ALL_LDCS) 2021ae08745Sheppo * debug channel = caller specified channel 2031ae08745Sheppo */ 2041ae08745Sheppo if ((id != DBG_ALL_LDCS) && 2051ae08745Sheppo (ldcdbgchan != DBG_ALL_LDCS) && 2061ae08745Sheppo (ldcdbgchan != id)) { 2071ae08745Sheppo return; 2081ae08745Sheppo } 2091ae08745Sheppo 2101ae08745Sheppo va_start(ap, fmt); 2111ae08745Sheppo (void) vsprintf(buf, fmt, ap); 2121ae08745Sheppo va_end(ap); 2131ae08745Sheppo 2141ae08745Sheppo cmn_err(CE_CONT, "?%s\n", buf); 2151ae08745Sheppo } 2161ae08745Sheppo 2171ae08745Sheppo #define D1 \ 2181ae08745Sheppo if (ldcdbg & 0x01) \ 2191ae08745Sheppo ldcdebug 2201ae08745Sheppo 2211ae08745Sheppo #define D2 \ 2221ae08745Sheppo if (ldcdbg & 0x02) \ 2231ae08745Sheppo ldcdebug 2241ae08745Sheppo 2251ae08745Sheppo #define DWARN \ 2261ae08745Sheppo if (ldcdbg & 0x04) \ 2271ae08745Sheppo ldcdebug 2281ae08745Sheppo 2291ae08745Sheppo #define DUMP_PAYLOAD(id, addr) \ 2301ae08745Sheppo { \ 2311ae08745Sheppo char buf[65*3]; \ 2321ae08745Sheppo int i; \ 2331ae08745Sheppo uint8_t *src = (uint8_t *)addr; \ 2341ae08745Sheppo for (i = 0; i < 64; i++, src++) \ 2351ae08745Sheppo (void) sprintf(&buf[i * 3], "|%02x", *src); \ 2361ae08745Sheppo (void) sprintf(&buf[i * 3], "|\n"); \ 2371ae08745Sheppo D2((id), "payload: %s", buf); \ 2381ae08745Sheppo } 2391ae08745Sheppo 2401ae08745Sheppo #define DUMP_LDC_PKT(c, s, addr) \ 2411ae08745Sheppo { \ 2421ae08745Sheppo ldc_msg_t *msg = (ldc_msg_t *)(addr); \ 2431ae08745Sheppo uint32_t mid = ((c)->mode != LDC_MODE_RAW) ? msg->seqid : 0; \ 2441ae08745Sheppo if (msg->type == LDC_DATA) { \ 2451ae08745Sheppo D2((c)->id, "%s: msg%d (/%x/%x/%x/,env[%c%c,sz=%d])", \ 2461ae08745Sheppo (s), mid, msg->type, msg->stype, msg->ctrl, \ 2471ae08745Sheppo (msg->env & LDC_FRAG_START) ? 'B' : ' ', \ 2481ae08745Sheppo (msg->env & LDC_FRAG_STOP) ? 'E' : ' ', \ 2491ae08745Sheppo (msg->env & LDC_LEN_MASK)); \ 2501ae08745Sheppo } else { \ 2511ae08745Sheppo D2((c)->id, "%s: msg%d (/%x/%x/%x/,env=%x)", (s), \ 2521ae08745Sheppo mid, msg->type, msg->stype, msg->ctrl, msg->env); \ 2531ae08745Sheppo } \ 2541ae08745Sheppo } 2551ae08745Sheppo 2561ae08745Sheppo #else 2571ae08745Sheppo 2581ae08745Sheppo #define DBG_ALL_LDCS -1 2591ae08745Sheppo 2601ae08745Sheppo #define D1 2611ae08745Sheppo #define D2 2621ae08745Sheppo #define DWARN 2631ae08745Sheppo 2641ae08745Sheppo #define DUMP_PAYLOAD(id, addr) 2651ae08745Sheppo #define DUMP_LDC_PKT(c, s, addr) 2661ae08745Sheppo 2671ae08745Sheppo #endif 2681ae08745Sheppo 2691ae08745Sheppo #define ZERO_PKT(p) \ 2701ae08745Sheppo bzero((p), sizeof (ldc_msg_t)); 2711ae08745Sheppo 2721ae08745Sheppo #define IDX2COOKIE(idx, pg_szc, pg_shift) \ 2731ae08745Sheppo (((pg_szc) << LDC_COOKIE_PGSZC_SHIFT) | ((idx) << (pg_shift))) 2741ae08745Sheppo 2751ae08745Sheppo 2761ae08745Sheppo int 2771ae08745Sheppo _init(void) 2781ae08745Sheppo { 2791ae08745Sheppo int status; 2801ae08745Sheppo 2811ae08745Sheppo status = hsvc_register(&ldc_hsvc, &ldc_sup_minor); 2821ae08745Sheppo if (status != 0) { 2831ae08745Sheppo cmn_err(CE_WARN, "%s: cannot negotiate hypervisor LDC services" 2841ae08745Sheppo " group: 0x%lx major: %ld minor: %ld errno: %d", 2851ae08745Sheppo ldc_hsvc.hsvc_modname, ldc_hsvc.hsvc_group, 2861ae08745Sheppo ldc_hsvc.hsvc_major, ldc_hsvc.hsvc_minor, status); 2871ae08745Sheppo return (-1); 2881ae08745Sheppo } 2891ae08745Sheppo 2901ae08745Sheppo status = hsvc_register(&intr_hsvc, &intr_sup_minor); 2911ae08745Sheppo if (status != 0) { 2921ae08745Sheppo cmn_err(CE_WARN, "%s: cannot negotiate hypervisor interrupt " 2931ae08745Sheppo "services group: 0x%lx major: %ld minor: %ld errno: %d", 2941ae08745Sheppo intr_hsvc.hsvc_modname, intr_hsvc.hsvc_group, 2951ae08745Sheppo intr_hsvc.hsvc_major, intr_hsvc.hsvc_minor, status); 2961ae08745Sheppo (void) hsvc_unregister(&ldc_hsvc); 2971ae08745Sheppo return (-1); 2981ae08745Sheppo } 2991ae08745Sheppo 3001ae08745Sheppo /* allocate soft state structure */ 3011ae08745Sheppo ldcssp = kmem_zalloc(sizeof (ldc_soft_state_t), KM_SLEEP); 3021ae08745Sheppo 3031ae08745Sheppo /* Link the module into the system */ 3041ae08745Sheppo status = mod_install(&ml); 3051ae08745Sheppo if (status != 0) { 3061ae08745Sheppo kmem_free(ldcssp, sizeof (ldc_soft_state_t)); 3071ae08745Sheppo return (status); 3081ae08745Sheppo } 3091ae08745Sheppo 3101ae08745Sheppo /* Initialize the LDC state structure */ 3111ae08745Sheppo mutex_init(&ldcssp->lock, NULL, MUTEX_DRIVER, NULL); 3121ae08745Sheppo 3131ae08745Sheppo mutex_enter(&ldcssp->lock); 3141ae08745Sheppo 3151ae08745Sheppo ldcssp->channel_count = 0; 3161ae08745Sheppo ldcssp->channels_open = 0; 3171ae08745Sheppo ldcssp->chan_list = NULL; 3181ae08745Sheppo ldcssp->dring_list = NULL; 3191ae08745Sheppo 3201ae08745Sheppo mutex_exit(&ldcssp->lock); 3211ae08745Sheppo 3221ae08745Sheppo return (0); 3231ae08745Sheppo } 3241ae08745Sheppo 3251ae08745Sheppo int 3261ae08745Sheppo _info(struct modinfo *modinfop) 3271ae08745Sheppo { 3281ae08745Sheppo /* Report status of the dynamically loadable driver module */ 3291ae08745Sheppo return (mod_info(&ml, modinfop)); 3301ae08745Sheppo } 3311ae08745Sheppo 3321ae08745Sheppo int 3331ae08745Sheppo _fini(void) 3341ae08745Sheppo { 3351ae08745Sheppo int rv, status; 3361ae08745Sheppo ldc_chan_t *ldcp; 3371ae08745Sheppo ldc_dring_t *dringp; 3381ae08745Sheppo ldc_mem_info_t minfo; 3391ae08745Sheppo 3401ae08745Sheppo /* Unlink the driver module from the system */ 3411ae08745Sheppo status = mod_remove(&ml); 3421ae08745Sheppo if (status) { 3431ae08745Sheppo DWARN(DBG_ALL_LDCS, "_fini: mod_remove failed\n"); 3441ae08745Sheppo return (EIO); 3451ae08745Sheppo } 3461ae08745Sheppo 3471ae08745Sheppo /* close and finalize channels */ 3481ae08745Sheppo ldcp = ldcssp->chan_list; 3491ae08745Sheppo while (ldcp != NULL) { 3501ae08745Sheppo (void) ldc_close((ldc_handle_t)ldcp); 3511ae08745Sheppo (void) ldc_fini((ldc_handle_t)ldcp); 3521ae08745Sheppo 3531ae08745Sheppo ldcp = ldcp->next; 3541ae08745Sheppo } 3551ae08745Sheppo 3561ae08745Sheppo /* Free descriptor rings */ 3571ae08745Sheppo dringp = ldcssp->dring_list; 3581ae08745Sheppo while (dringp != NULL) { 3591ae08745Sheppo dringp = dringp->next; 3601ae08745Sheppo 3611ae08745Sheppo rv = ldc_mem_dring_info((ldc_dring_handle_t)dringp, &minfo); 3621ae08745Sheppo if (rv == 0 && minfo.status != LDC_UNBOUND) { 3631ae08745Sheppo if (minfo.status == LDC_BOUND) { 3641ae08745Sheppo (void) ldc_mem_dring_unbind( 3651ae08745Sheppo (ldc_dring_handle_t)dringp); 3661ae08745Sheppo } 3671ae08745Sheppo if (minfo.status == LDC_MAPPED) { 3681ae08745Sheppo (void) ldc_mem_dring_unmap( 3691ae08745Sheppo (ldc_dring_handle_t)dringp); 3701ae08745Sheppo } 3711ae08745Sheppo } 3721ae08745Sheppo 3731ae08745Sheppo (void) ldc_mem_dring_destroy((ldc_dring_handle_t)dringp); 3741ae08745Sheppo } 3751ae08745Sheppo ldcssp->dring_list = NULL; 3761ae08745Sheppo 3771ae08745Sheppo /* 3781ae08745Sheppo * We have successfully "removed" the driver. 3791ae08745Sheppo * Destroying soft states 3801ae08745Sheppo */ 3811ae08745Sheppo mutex_destroy(&ldcssp->lock); 3821ae08745Sheppo kmem_free(ldcssp, sizeof (ldc_soft_state_t)); 3831ae08745Sheppo 3841ae08745Sheppo (void) hsvc_unregister(&ldc_hsvc); 3851ae08745Sheppo (void) hsvc_unregister(&intr_hsvc); 3861ae08745Sheppo 3871ae08745Sheppo return (status); 3881ae08745Sheppo } 3891ae08745Sheppo 3901ae08745Sheppo /* -------------------------------------------------------------------------- */ 3911ae08745Sheppo 3921ae08745Sheppo /* 393*e1ebb9ecSlm66018 * LDC Link Layer Internal Functions 3941ae08745Sheppo */ 3951ae08745Sheppo 3961ae08745Sheppo /* 3971ae08745Sheppo * Translate HV Errors to sun4v error codes 3981ae08745Sheppo */ 3991ae08745Sheppo static int 4001ae08745Sheppo i_ldc_h2v_error(int h_error) 4011ae08745Sheppo { 4021ae08745Sheppo switch (h_error) { 4031ae08745Sheppo 4041ae08745Sheppo case H_EOK: 4051ae08745Sheppo return (0); 4061ae08745Sheppo 4071ae08745Sheppo case H_ENORADDR: 4081ae08745Sheppo return (EFAULT); 4091ae08745Sheppo 4101ae08745Sheppo case H_EBADPGSZ: 4111ae08745Sheppo case H_EINVAL: 4121ae08745Sheppo return (EINVAL); 4131ae08745Sheppo 4141ae08745Sheppo case H_EWOULDBLOCK: 4151ae08745Sheppo return (EWOULDBLOCK); 4161ae08745Sheppo 4171ae08745Sheppo case H_ENOACCESS: 4181ae08745Sheppo case H_ENOMAP: 4191ae08745Sheppo return (EACCES); 4201ae08745Sheppo 4211ae08745Sheppo case H_EIO: 4221ae08745Sheppo case H_ECPUERROR: 4231ae08745Sheppo return (EIO); 4241ae08745Sheppo 4251ae08745Sheppo case H_ENOTSUPPORTED: 4261ae08745Sheppo return (ENOTSUP); 4271ae08745Sheppo 4281ae08745Sheppo case H_ETOOMANY: 4291ae08745Sheppo return (ENOSPC); 4301ae08745Sheppo 4311ae08745Sheppo case H_ECHANNEL: 4321ae08745Sheppo return (ECHRNG); 4331ae08745Sheppo default: 4341ae08745Sheppo break; 4351ae08745Sheppo } 4361ae08745Sheppo 4371ae08745Sheppo return (EIO); 4381ae08745Sheppo } 4391ae08745Sheppo 4401ae08745Sheppo /* 4411ae08745Sheppo * Reconfigure the transmit queue 4421ae08745Sheppo */ 4431ae08745Sheppo static int 4441ae08745Sheppo i_ldc_txq_reconf(ldc_chan_t *ldcp) 4451ae08745Sheppo { 4461ae08745Sheppo int rv; 4471ae08745Sheppo 4481ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 449d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 450d10e4ef2Snarayan 4511ae08745Sheppo rv = hv_ldc_tx_qconf(ldcp->id, ldcp->tx_q_ra, ldcp->tx_q_entries); 4521ae08745Sheppo if (rv) { 4531ae08745Sheppo cmn_err(CE_WARN, 4541ae08745Sheppo "ldc_tx_qconf: (0x%lx) cannot set qconf", ldcp->id); 4551ae08745Sheppo return (EIO); 4561ae08745Sheppo } 4571ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, &(ldcp->tx_head), 4581ae08745Sheppo &(ldcp->tx_tail), &(ldcp->link_state)); 4591ae08745Sheppo if (rv) { 4601ae08745Sheppo cmn_err(CE_WARN, 4611ae08745Sheppo "ldc_tx_get_state: (0x%lx) cannot get qptrs", ldcp->id); 4621ae08745Sheppo return (EIO); 4631ae08745Sheppo } 4641ae08745Sheppo D1(ldcp->id, "ldc_tx_get_state: (0x%llx) h=0x%llx,t=0x%llx," 4651ae08745Sheppo "s=0x%llx\n", ldcp->id, ldcp->tx_head, ldcp->tx_tail, 4661ae08745Sheppo ldcp->link_state); 4671ae08745Sheppo 4681ae08745Sheppo return (0); 4691ae08745Sheppo } 4701ae08745Sheppo 4711ae08745Sheppo /* 4721ae08745Sheppo * Reconfigure the receive queue 4731ae08745Sheppo */ 4741ae08745Sheppo static int 4751ae08745Sheppo i_ldc_rxq_reconf(ldc_chan_t *ldcp) 4761ae08745Sheppo { 4771ae08745Sheppo int rv; 4781ae08745Sheppo uint64_t rx_head, rx_tail; 4791ae08745Sheppo 4801ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 4811ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 4821ae08745Sheppo &(ldcp->link_state)); 4831ae08745Sheppo if (rv) { 4841ae08745Sheppo cmn_err(CE_WARN, 4851ae08745Sheppo "ldc_rx_getstate: (0x%lx) cannot get state", 4861ae08745Sheppo ldcp->id); 4871ae08745Sheppo return (EIO); 4881ae08745Sheppo } 4891ae08745Sheppo 4901ae08745Sheppo if (rx_head != rx_tail || ldcp->tstate > TS_READY) { 4911ae08745Sheppo rv = hv_ldc_rx_qconf(ldcp->id, ldcp->rx_q_ra, 4921ae08745Sheppo ldcp->rx_q_entries); 4931ae08745Sheppo if (rv) { 4941ae08745Sheppo cmn_err(CE_WARN, 4951ae08745Sheppo "ldc_rx_qconf: (0x%lx) cannot set qconf", 4961ae08745Sheppo ldcp->id); 4971ae08745Sheppo return (EIO); 4981ae08745Sheppo } 4991ae08745Sheppo D1(ldcp->id, "ldc_rx_qconf: (0x%llx) completed qconf", 5001ae08745Sheppo ldcp->id); 5011ae08745Sheppo } 5021ae08745Sheppo 5031ae08745Sheppo return (0); 5041ae08745Sheppo } 5051ae08745Sheppo 5061ae08745Sheppo /* 5071ae08745Sheppo * Reset LDC state structure and its contents 5081ae08745Sheppo */ 5091ae08745Sheppo static void 5101ae08745Sheppo i_ldc_reset_state(ldc_chan_t *ldcp) 5111ae08745Sheppo { 5121ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 5131ae08745Sheppo ldcp->last_msg_snt = LDC_INIT_SEQID; 5141ae08745Sheppo ldcp->last_ack_rcd = 0; 5151ae08745Sheppo ldcp->last_msg_rcd = 0; 5161ae08745Sheppo ldcp->tx_ackd_head = ldcp->tx_head; 5171ae08745Sheppo ldcp->next_vidx = 0; 5181ae08745Sheppo ldcp->hstate = 0; 5191ae08745Sheppo ldcp->tstate = TS_OPEN; 5201ae08745Sheppo ldcp->status = LDC_OPEN; 5211ae08745Sheppo 5221ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_UP || 5231ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 5241ae08745Sheppo 5251ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW) { 5261ae08745Sheppo ldcp->status = LDC_UP; 5271ae08745Sheppo ldcp->tstate = TS_UP; 5281ae08745Sheppo } else { 5291ae08745Sheppo ldcp->status = LDC_READY; 5301ae08745Sheppo ldcp->tstate |= TS_LINK_READY; 5311ae08745Sheppo } 5321ae08745Sheppo } 5331ae08745Sheppo } 5341ae08745Sheppo 5351ae08745Sheppo /* 5361ae08745Sheppo * Reset a LDC channel 5371ae08745Sheppo */ 5381ae08745Sheppo static void 5391ae08745Sheppo i_ldc_reset(ldc_chan_t *ldcp) 5401ae08745Sheppo { 5411ae08745Sheppo D2(ldcp->id, "i_ldc_reset: (0x%llx) channel reset\n", ldcp->id); 5421ae08745Sheppo 543d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->lock)); 544d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 545d10e4ef2Snarayan 5461ae08745Sheppo (void) i_ldc_txq_reconf(ldcp); 5471ae08745Sheppo (void) i_ldc_rxq_reconf(ldcp); 5481ae08745Sheppo i_ldc_reset_state(ldcp); 5491ae08745Sheppo } 5501ae08745Sheppo 5511ae08745Sheppo /* 5521ae08745Sheppo * Clear pending interrupts 5531ae08745Sheppo */ 5541ae08745Sheppo static void 5551ae08745Sheppo i_ldc_clear_intr(ldc_chan_t *ldcp, cnex_intrtype_t itype) 5561ae08745Sheppo { 5571ae08745Sheppo ldc_cnex_t *cinfo = &ldcssp->cinfo; 5581ae08745Sheppo 5591ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 5601ae08745Sheppo if (cinfo->dip && ldcp->intr_pending) { 5611ae08745Sheppo ldcp->intr_pending = B_FALSE; 5621ae08745Sheppo (void) cinfo->clr_intr(cinfo->dip, ldcp->id, itype); 5631ae08745Sheppo } 5641ae08745Sheppo } 5651ae08745Sheppo 5661ae08745Sheppo /* 5671ae08745Sheppo * Set the receive queue head 5680a55fbb7Slm66018 * Resets connection and returns an error if it fails. 5691ae08745Sheppo */ 5701ae08745Sheppo static int 5711ae08745Sheppo i_ldc_set_rx_head(ldc_chan_t *ldcp, uint64_t head) 5721ae08745Sheppo { 5731ae08745Sheppo int rv; 5740a55fbb7Slm66018 int retries; 5751ae08745Sheppo 5761ae08745Sheppo ASSERT(MUTEX_HELD(&ldcp->lock)); 5770a55fbb7Slm66018 for (retries = 0; retries < ldc_max_retries; retries++) { 5780a55fbb7Slm66018 5790a55fbb7Slm66018 if ((rv = hv_ldc_rx_set_qhead(ldcp->id, head)) == 0) 5800a55fbb7Slm66018 return (0); 5810a55fbb7Slm66018 5820a55fbb7Slm66018 if (rv != H_EWOULDBLOCK) 5830a55fbb7Slm66018 break; 5840a55fbb7Slm66018 5850a55fbb7Slm66018 /* wait for ldc_delay usecs */ 5860a55fbb7Slm66018 drv_usecwait(ldc_delay); 5871ae08745Sheppo } 5881ae08745Sheppo 5890a55fbb7Slm66018 cmn_err(CE_WARN, "ldc_rx_set_qhead: (0x%lx) cannot set qhead 0x%lx", 5900a55fbb7Slm66018 ldcp->id, head); 591d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 5920a55fbb7Slm66018 i_ldc_reset(ldcp); 593d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 5940a55fbb7Slm66018 5950a55fbb7Slm66018 return (ECONNRESET); 5961ae08745Sheppo } 5971ae08745Sheppo 5981ae08745Sheppo 5991ae08745Sheppo /* 6001ae08745Sheppo * Returns the tx_tail to be used for transfer 6011ae08745Sheppo * Re-reads the TX queue ptrs if and only if the 6021ae08745Sheppo * the cached head and tail are equal (queue is full) 6031ae08745Sheppo */ 6041ae08745Sheppo static int 6051ae08745Sheppo i_ldc_get_tx_tail(ldc_chan_t *ldcp, uint64_t *tail) 6061ae08745Sheppo { 6071ae08745Sheppo int rv; 6081ae08745Sheppo uint64_t current_head, new_tail; 6091ae08745Sheppo 610d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 6111ae08745Sheppo /* Read the head and tail ptrs from HV */ 6121ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 6131ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 6141ae08745Sheppo if (rv) { 6151ae08745Sheppo cmn_err(CE_WARN, 6161ae08745Sheppo "i_ldc_get_tx_tail: (0x%lx) cannot read qptrs\n", 6171ae08745Sheppo ldcp->id); 6181ae08745Sheppo return (EIO); 6191ae08745Sheppo } 6201ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN) { 6211ae08745Sheppo DWARN(DBG_ALL_LDCS, 6221ae08745Sheppo "i_ldc_get_tx_tail: (0x%llx) channel not ready\n", 6231ae08745Sheppo ldcp->id); 6241ae08745Sheppo return (ECONNRESET); 6251ae08745Sheppo } 6261ae08745Sheppo 6271ae08745Sheppo /* In reliable mode, check against last ACKd msg */ 6281ae08745Sheppo current_head = (ldcp->mode == LDC_MODE_RELIABLE || 6291ae08745Sheppo ldcp->mode == LDC_MODE_STREAM) 6301ae08745Sheppo ? ldcp->tx_ackd_head : ldcp->tx_head; 6311ae08745Sheppo 6321ae08745Sheppo /* increment the tail */ 6331ae08745Sheppo new_tail = (ldcp->tx_tail + LDC_PACKET_SIZE) % 6341ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 6351ae08745Sheppo 6361ae08745Sheppo if (new_tail == current_head) { 6371ae08745Sheppo DWARN(ldcp->id, 6381ae08745Sheppo "i_ldc_get_tx_tail: (0x%llx) TX queue is full\n", 6391ae08745Sheppo ldcp->id); 6401ae08745Sheppo return (EWOULDBLOCK); 6411ae08745Sheppo } 6421ae08745Sheppo 6431ae08745Sheppo D2(ldcp->id, "i_ldc_get_tx_tail: (0x%llx) head=0x%llx, tail=0x%llx\n", 6441ae08745Sheppo ldcp->id, ldcp->tx_head, ldcp->tx_tail); 6451ae08745Sheppo 6461ae08745Sheppo *tail = ldcp->tx_tail; 6471ae08745Sheppo return (0); 6481ae08745Sheppo } 6491ae08745Sheppo 6501ae08745Sheppo /* 6511ae08745Sheppo * Set the tail pointer. If HV returns EWOULDBLOCK, it will back off 6520a55fbb7Slm66018 * and retry ldc_max_retries times before returning an error. 6531ae08745Sheppo * Returns 0, EWOULDBLOCK or EIO 6541ae08745Sheppo */ 6551ae08745Sheppo static int 6561ae08745Sheppo i_ldc_set_tx_tail(ldc_chan_t *ldcp, uint64_t tail) 6571ae08745Sheppo { 6581ae08745Sheppo int rv, retval = EWOULDBLOCK; 6590a55fbb7Slm66018 int retries; 6601ae08745Sheppo 661d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 6620a55fbb7Slm66018 for (retries = 0; retries < ldc_max_retries; retries++) { 6631ae08745Sheppo 6641ae08745Sheppo if ((rv = hv_ldc_tx_set_qtail(ldcp->id, tail)) == 0) { 6651ae08745Sheppo retval = 0; 6661ae08745Sheppo break; 6671ae08745Sheppo } 6681ae08745Sheppo if (rv != H_EWOULDBLOCK) { 6691ae08745Sheppo DWARN(ldcp->id, "i_ldc_set_tx_tail: (0x%llx) set " 6701ae08745Sheppo "qtail=0x%llx failed, rv=%d\n", ldcp->id, tail, rv); 6711ae08745Sheppo retval = EIO; 6721ae08745Sheppo break; 6731ae08745Sheppo } 6741ae08745Sheppo 6750a55fbb7Slm66018 /* wait for ldc_delay usecs */ 6760a55fbb7Slm66018 drv_usecwait(ldc_delay); 6771ae08745Sheppo } 6781ae08745Sheppo return (retval); 6791ae08745Sheppo } 6801ae08745Sheppo 6811ae08745Sheppo /* 6821ae08745Sheppo * Send a LDC message 6831ae08745Sheppo */ 6841ae08745Sheppo static int 6851ae08745Sheppo i_ldc_send_pkt(ldc_chan_t *ldcp, uint8_t pkttype, uint8_t subtype, 6861ae08745Sheppo uint8_t ctrlmsg) 6871ae08745Sheppo { 6881ae08745Sheppo int rv; 6891ae08745Sheppo ldc_msg_t *pkt; 6901ae08745Sheppo uint64_t tx_tail; 6911ae08745Sheppo uint32_t curr_seqid = ldcp->last_msg_snt; 6921ae08745Sheppo 693d10e4ef2Snarayan /* Obtain Tx lock */ 694d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 695d10e4ef2Snarayan 6961ae08745Sheppo /* get the current tail for the message */ 6971ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 6981ae08745Sheppo if (rv) { 6991ae08745Sheppo DWARN(ldcp->id, 7001ae08745Sheppo "i_ldc_send_pkt: (0x%llx) error sending pkt, " 7011ae08745Sheppo "type=0x%x,subtype=0x%x,ctrl=0x%x\n", 7021ae08745Sheppo ldcp->id, pkttype, subtype, ctrlmsg); 703d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 7041ae08745Sheppo return (rv); 7051ae08745Sheppo } 7061ae08745Sheppo 7071ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 7081ae08745Sheppo ZERO_PKT(pkt); 7091ae08745Sheppo 7101ae08745Sheppo /* Initialize the packet */ 7111ae08745Sheppo pkt->type = pkttype; 7121ae08745Sheppo pkt->stype = subtype; 7131ae08745Sheppo pkt->ctrl = ctrlmsg; 7141ae08745Sheppo 7151ae08745Sheppo /* Store ackid/seqid iff it is RELIABLE mode & not a RTS/RTR message */ 7161ae08745Sheppo if (((ctrlmsg & LDC_CTRL_MASK) != LDC_RTS) && 7171ae08745Sheppo ((ctrlmsg & LDC_CTRL_MASK) != LDC_RTR)) { 7181ae08745Sheppo curr_seqid++; 7191ae08745Sheppo if (ldcp->mode != LDC_MODE_RAW) { 7201ae08745Sheppo pkt->seqid = curr_seqid; 7211ae08745Sheppo pkt->ackid = ldcp->last_msg_rcd; 7221ae08745Sheppo } 7231ae08745Sheppo } 7241ae08745Sheppo DUMP_LDC_PKT(ldcp, "i_ldc_send_pkt", (uint64_t)pkt); 7251ae08745Sheppo 7261ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 7271ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 7281ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 7291ae08745Sheppo 7301ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 7311ae08745Sheppo if (rv) { 7321ae08745Sheppo DWARN(ldcp->id, 7331ae08745Sheppo "i_ldc_send_pkt:(0x%llx) error sending pkt, " 7341ae08745Sheppo "type=0x%x,stype=0x%x,ctrl=0x%x\n", 7351ae08745Sheppo ldcp->id, pkttype, subtype, ctrlmsg); 736d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 7371ae08745Sheppo return (EIO); 7381ae08745Sheppo } 7391ae08745Sheppo 7401ae08745Sheppo ldcp->last_msg_snt = curr_seqid; 7411ae08745Sheppo ldcp->tx_tail = tx_tail; 7421ae08745Sheppo 743d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 7441ae08745Sheppo return (0); 7451ae08745Sheppo } 7461ae08745Sheppo 7471ae08745Sheppo /* 7481ae08745Sheppo * Checks if packet was received in right order 749*e1ebb9ecSlm66018 * in the case of a reliable link. 7501ae08745Sheppo * Returns 0 if in order, else EIO 7511ae08745Sheppo */ 7521ae08745Sheppo static int 7531ae08745Sheppo i_ldc_check_seqid(ldc_chan_t *ldcp, ldc_msg_t *msg) 7541ae08745Sheppo { 7551ae08745Sheppo /* No seqid checking for RAW mode */ 7561ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW) 7571ae08745Sheppo return (0); 7581ae08745Sheppo 7591ae08745Sheppo /* No seqid checking for version, RTS, RTR message */ 7601ae08745Sheppo if (msg->ctrl == LDC_VER || 7611ae08745Sheppo msg->ctrl == LDC_RTS || 7621ae08745Sheppo msg->ctrl == LDC_RTR) 7631ae08745Sheppo return (0); 7641ae08745Sheppo 7651ae08745Sheppo /* Initial seqid to use is sent in RTS/RTR and saved in last_msg_rcd */ 7661ae08745Sheppo if (msg->seqid != (ldcp->last_msg_rcd + 1)) { 7671ae08745Sheppo DWARN(ldcp->id, 7681ae08745Sheppo "i_ldc_check_seqid: (0x%llx) out-of-order pkt, got 0x%x, " 7691ae08745Sheppo "expecting 0x%x\n", ldcp->id, msg->seqid, 7701ae08745Sheppo (ldcp->last_msg_rcd + 1)); 7711ae08745Sheppo return (EIO); 7721ae08745Sheppo } 7731ae08745Sheppo 7741ae08745Sheppo return (0); 7751ae08745Sheppo } 7761ae08745Sheppo 7771ae08745Sheppo 7781ae08745Sheppo /* 7791ae08745Sheppo * Process an incoming version ctrl message 7801ae08745Sheppo */ 7811ae08745Sheppo static int 7821ae08745Sheppo i_ldc_process_VER(ldc_chan_t *ldcp, ldc_msg_t *msg) 7831ae08745Sheppo { 7841ae08745Sheppo int rv = 0, idx = ldcp->next_vidx; 7851ae08745Sheppo ldc_msg_t *pkt; 7861ae08745Sheppo uint64_t tx_tail; 7871ae08745Sheppo ldc_ver_t *rcvd_ver; 7881ae08745Sheppo 7891ae08745Sheppo /* get the received version */ 7901ae08745Sheppo rcvd_ver = (ldc_ver_t *)((uint64_t)msg + LDC_PAYLOAD_VER_OFF); 7911ae08745Sheppo 7921ae08745Sheppo D2(ldcp->id, "i_ldc_process_VER: (0x%llx) received VER v%u.%u\n", 7931ae08745Sheppo ldcp->id, rcvd_ver->major, rcvd_ver->minor); 7941ae08745Sheppo 795d10e4ef2Snarayan /* Obtain Tx lock */ 796d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 797d10e4ef2Snarayan 7981ae08745Sheppo switch (msg->stype) { 7991ae08745Sheppo case LDC_INFO: 8001ae08745Sheppo 8011ae08745Sheppo /* get the current tail and pkt for the response */ 8021ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 8031ae08745Sheppo if (rv != 0) { 8041ae08745Sheppo DWARN(ldcp->id, 8051ae08745Sheppo "i_ldc_process_VER: (0x%llx) err sending " 8061ae08745Sheppo "version ACK/NACK\n", ldcp->id); 8071ae08745Sheppo i_ldc_reset(ldcp); 808d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 8091ae08745Sheppo return (ECONNRESET); 8101ae08745Sheppo } 8111ae08745Sheppo 8121ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 8131ae08745Sheppo ZERO_PKT(pkt); 8141ae08745Sheppo 8151ae08745Sheppo /* initialize the packet */ 8161ae08745Sheppo pkt->type = LDC_CTRL; 8171ae08745Sheppo pkt->ctrl = LDC_VER; 8181ae08745Sheppo 8191ae08745Sheppo for (;;) { 8201ae08745Sheppo 8211ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: got %u.%u chk %u.%u\n", 8221ae08745Sheppo rcvd_ver->major, rcvd_ver->minor, 8231ae08745Sheppo ldc_versions[idx].major, ldc_versions[idx].minor); 8241ae08745Sheppo 8251ae08745Sheppo if (rcvd_ver->major == ldc_versions[idx].major) { 8261ae08745Sheppo /* major version match - ACK version */ 8271ae08745Sheppo pkt->stype = LDC_ACK; 8281ae08745Sheppo 8291ae08745Sheppo /* 8301ae08745Sheppo * lower minor version to the one this endpt 8311ae08745Sheppo * supports, if necessary 8321ae08745Sheppo */ 8331ae08745Sheppo if (rcvd_ver->minor > ldc_versions[idx].minor) 8341ae08745Sheppo rcvd_ver->minor = 8351ae08745Sheppo ldc_versions[idx].minor; 8361ae08745Sheppo bcopy(rcvd_ver, pkt->udata, sizeof (*rcvd_ver)); 8371ae08745Sheppo 8381ae08745Sheppo break; 8391ae08745Sheppo } 8401ae08745Sheppo 8411ae08745Sheppo if (rcvd_ver->major > ldc_versions[idx].major) { 8421ae08745Sheppo 8431ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: using next" 8441ae08745Sheppo " lower idx=%d, v%u.%u\n", idx, 8451ae08745Sheppo ldc_versions[idx].major, 8461ae08745Sheppo ldc_versions[idx].minor); 8471ae08745Sheppo 8481ae08745Sheppo /* nack with next lower version */ 8491ae08745Sheppo pkt->stype = LDC_NACK; 8501ae08745Sheppo bcopy(&ldc_versions[idx], pkt->udata, 8511ae08745Sheppo sizeof (ldc_versions[idx])); 8521ae08745Sheppo ldcp->next_vidx = idx; 8531ae08745Sheppo break; 8541ae08745Sheppo } 8551ae08745Sheppo 8561ae08745Sheppo /* next major version */ 8571ae08745Sheppo idx++; 8581ae08745Sheppo 8591ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: inc idx %x\n", idx); 8601ae08745Sheppo 8611ae08745Sheppo if (idx == LDC_NUM_VERS) { 8621ae08745Sheppo /* no version match - send NACK */ 8631ae08745Sheppo pkt->stype = LDC_NACK; 8641ae08745Sheppo bzero(pkt->udata, sizeof (ldc_ver_t)); 8651ae08745Sheppo ldcp->next_vidx = 0; 8661ae08745Sheppo break; 8671ae08745Sheppo } 8681ae08745Sheppo } 8691ae08745Sheppo 8701ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 8711ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 8721ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 8731ae08745Sheppo 8741ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 8751ae08745Sheppo if (rv == 0) { 8761ae08745Sheppo ldcp->tx_tail = tx_tail; 8771ae08745Sheppo if (pkt->stype == LDC_ACK) { 8781ae08745Sheppo D2(ldcp->id, "i_ldc_process_VER: (0x%llx) sent" 8791ae08745Sheppo " version ACK\n", ldcp->id); 8801ae08745Sheppo /* Save the ACK'd version */ 8811ae08745Sheppo ldcp->version.major = rcvd_ver->major; 8821ae08745Sheppo ldcp->version.minor = rcvd_ver->minor; 8830a55fbb7Slm66018 ldcp->hstate |= TS_RCVD_VER; 8841ae08745Sheppo ldcp->tstate |= TS_VER_DONE; 8851ae08745Sheppo DWARN(DBG_ALL_LDCS, 8861ae08745Sheppo "(0x%llx) Agreed on version v%u.%u\n", 8871ae08745Sheppo ldcp->id, rcvd_ver->major, rcvd_ver->minor); 8881ae08745Sheppo } 8891ae08745Sheppo } else { 8901ae08745Sheppo DWARN(ldcp->id, 8911ae08745Sheppo "i_ldc_process_VER: (0x%llx) error sending " 8921ae08745Sheppo "ACK/NACK\n", ldcp->id); 8931ae08745Sheppo i_ldc_reset(ldcp); 894d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 8951ae08745Sheppo return (ECONNRESET); 8961ae08745Sheppo } 8971ae08745Sheppo 8981ae08745Sheppo break; 8991ae08745Sheppo 9001ae08745Sheppo case LDC_ACK: 9011ae08745Sheppo /* SUCCESS - we have agreed on a version */ 9021ae08745Sheppo ldcp->version.major = rcvd_ver->major; 9031ae08745Sheppo ldcp->version.minor = rcvd_ver->minor; 9041ae08745Sheppo ldcp->tstate |= TS_VER_DONE; 9051ae08745Sheppo 9061ae08745Sheppo D1(DBG_ALL_LDCS, "(0x%llx) Agreed on version v%u.%u\n", 9071ae08745Sheppo ldcp->id, rcvd_ver->major, rcvd_ver->minor); 9081ae08745Sheppo 9091ae08745Sheppo /* initiate RTS-RTR-RDX handshake */ 9101ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 9111ae08745Sheppo if (rv) { 9121ae08745Sheppo DWARN(ldcp->id, 9131ae08745Sheppo "i_ldc_process_VER: (0x%llx) cannot send RTS\n", 9141ae08745Sheppo ldcp->id); 9151ae08745Sheppo i_ldc_reset(ldcp); 916d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 9171ae08745Sheppo return (ECONNRESET); 9181ae08745Sheppo } 9191ae08745Sheppo 9201ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 9211ae08745Sheppo ZERO_PKT(pkt); 9221ae08745Sheppo 9231ae08745Sheppo pkt->type = LDC_CTRL; 9241ae08745Sheppo pkt->stype = LDC_INFO; 9251ae08745Sheppo pkt->ctrl = LDC_RTS; 9261ae08745Sheppo pkt->env = ldcp->mode; 9271ae08745Sheppo if (ldcp->mode != LDC_MODE_RAW) 9281ae08745Sheppo pkt->seqid = LDC_INIT_SEQID; 9291ae08745Sheppo 9301ae08745Sheppo ldcp->last_msg_rcd = LDC_INIT_SEQID; 9311ae08745Sheppo 9321ae08745Sheppo DUMP_LDC_PKT(ldcp, "i_ldc_process_VER snd rts", (uint64_t)pkt); 9331ae08745Sheppo 9341ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 9351ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 9361ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 9371ae08745Sheppo 9381ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 9391ae08745Sheppo if (rv) { 9401ae08745Sheppo D2(ldcp->id, 9411ae08745Sheppo "i_ldc_process_VER: (0x%llx) no listener\n", 9421ae08745Sheppo ldcp->id); 9431ae08745Sheppo i_ldc_reset(ldcp); 944d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 9451ae08745Sheppo return (ECONNRESET); 9461ae08745Sheppo } 9471ae08745Sheppo 9481ae08745Sheppo ldcp->tx_tail = tx_tail; 9491ae08745Sheppo ldcp->hstate |= TS_SENT_RTS; 9501ae08745Sheppo 9511ae08745Sheppo break; 9521ae08745Sheppo 9531ae08745Sheppo case LDC_NACK: 9541ae08745Sheppo /* check if version in NACK is zero */ 9551ae08745Sheppo if (rcvd_ver->major == 0 && rcvd_ver->minor == 0) { 9561ae08745Sheppo /* version handshake failure */ 9571ae08745Sheppo DWARN(DBG_ALL_LDCS, 9581ae08745Sheppo "i_ldc_process_VER: (0x%llx) no version match\n", 9591ae08745Sheppo ldcp->id); 9601ae08745Sheppo i_ldc_reset(ldcp); 961d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 9621ae08745Sheppo return (ECONNRESET); 9631ae08745Sheppo } 9641ae08745Sheppo 9651ae08745Sheppo /* get the current tail and pkt for the response */ 9661ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 9671ae08745Sheppo if (rv != 0) { 9681ae08745Sheppo cmn_err(CE_NOTE, 9691ae08745Sheppo "i_ldc_process_VER: (0x%lx) err sending " 9701ae08745Sheppo "version ACK/NACK\n", ldcp->id); 9711ae08745Sheppo i_ldc_reset(ldcp); 972d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 9731ae08745Sheppo return (ECONNRESET); 9741ae08745Sheppo } 9751ae08745Sheppo 9761ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 9771ae08745Sheppo ZERO_PKT(pkt); 9781ae08745Sheppo 9791ae08745Sheppo /* initialize the packet */ 9801ae08745Sheppo pkt->type = LDC_CTRL; 9811ae08745Sheppo pkt->ctrl = LDC_VER; 9821ae08745Sheppo pkt->stype = LDC_INFO; 9831ae08745Sheppo 9841ae08745Sheppo /* check ver in NACK msg has a match */ 9851ae08745Sheppo for (;;) { 9861ae08745Sheppo if (rcvd_ver->major == ldc_versions[idx].major) { 9871ae08745Sheppo /* 9881ae08745Sheppo * major version match - resubmit request 9891ae08745Sheppo * if lower minor version to the one this endpt 9901ae08745Sheppo * supports, if necessary 9911ae08745Sheppo */ 9921ae08745Sheppo if (rcvd_ver->minor > ldc_versions[idx].minor) 9931ae08745Sheppo rcvd_ver->minor = 9941ae08745Sheppo ldc_versions[idx].minor; 9951ae08745Sheppo bcopy(rcvd_ver, pkt->udata, sizeof (*rcvd_ver)); 9961ae08745Sheppo break; 9971ae08745Sheppo 9981ae08745Sheppo } 9991ae08745Sheppo 10001ae08745Sheppo if (rcvd_ver->major > ldc_versions[idx].major) { 10011ae08745Sheppo 10021ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: using next" 10031ae08745Sheppo " lower idx=%d, v%u.%u\n", idx, 10041ae08745Sheppo ldc_versions[idx].major, 10051ae08745Sheppo ldc_versions[idx].minor); 10061ae08745Sheppo 10071ae08745Sheppo /* send next lower version */ 10081ae08745Sheppo bcopy(&ldc_versions[idx], pkt->udata, 10091ae08745Sheppo sizeof (ldc_versions[idx])); 10101ae08745Sheppo ldcp->next_vidx = idx; 10111ae08745Sheppo break; 10121ae08745Sheppo } 10131ae08745Sheppo 10141ae08745Sheppo /* next version */ 10151ae08745Sheppo idx++; 10161ae08745Sheppo 10171ae08745Sheppo D1(ldcp->id, "i_ldc_process_VER: inc idx %x\n", idx); 10181ae08745Sheppo 10191ae08745Sheppo if (idx == LDC_NUM_VERS) { 10201ae08745Sheppo /* no version match - terminate */ 10211ae08745Sheppo ldcp->next_vidx = 0; 1022d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 10231ae08745Sheppo return (ECONNRESET); 10241ae08745Sheppo } 10251ae08745Sheppo } 10261ae08745Sheppo 10271ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 10281ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 10291ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 10301ae08745Sheppo 10311ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 10321ae08745Sheppo if (rv == 0) { 10331ae08745Sheppo D2(ldcp->id, "i_ldc_process_VER: (0x%llx) sent version" 10341ae08745Sheppo "INFO v%u.%u\n", ldcp->id, ldc_versions[idx].major, 10351ae08745Sheppo ldc_versions[idx].minor); 10361ae08745Sheppo ldcp->tx_tail = tx_tail; 10371ae08745Sheppo } else { 10381ae08745Sheppo cmn_err(CE_NOTE, 10391ae08745Sheppo "i_ldc_process_VER: (0x%lx) error sending version" 10401ae08745Sheppo "INFO\n", ldcp->id); 10411ae08745Sheppo i_ldc_reset(ldcp); 1042d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 10431ae08745Sheppo return (ECONNRESET); 10441ae08745Sheppo } 10451ae08745Sheppo 10461ae08745Sheppo break; 10471ae08745Sheppo } 10481ae08745Sheppo 1049d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 10501ae08745Sheppo return (rv); 10511ae08745Sheppo } 10521ae08745Sheppo 10531ae08745Sheppo 10541ae08745Sheppo /* 10551ae08745Sheppo * Process an incoming RTS ctrl message 10561ae08745Sheppo */ 10571ae08745Sheppo static int 10581ae08745Sheppo i_ldc_process_RTS(ldc_chan_t *ldcp, ldc_msg_t *msg) 10591ae08745Sheppo { 10601ae08745Sheppo int rv = 0; 10611ae08745Sheppo ldc_msg_t *pkt; 10621ae08745Sheppo uint64_t tx_tail; 10631ae08745Sheppo boolean_t sent_NACK = B_FALSE; 10641ae08745Sheppo 10651ae08745Sheppo D2(ldcp->id, "i_ldc_process_RTS: (0x%llx) received RTS\n", ldcp->id); 10661ae08745Sheppo 10671ae08745Sheppo switch (msg->stype) { 10681ae08745Sheppo case LDC_NACK: 10691ae08745Sheppo DWARN(ldcp->id, 10701ae08745Sheppo "i_ldc_process_RTS: (0x%llx) RTS NACK received\n", 10711ae08745Sheppo ldcp->id); 10721ae08745Sheppo 10731ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1074d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 10751ae08745Sheppo i_ldc_reset(ldcp); 1076d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 10771ae08745Sheppo rv = ECONNRESET; 10781ae08745Sheppo break; 10791ae08745Sheppo 10801ae08745Sheppo case LDC_INFO: 10811ae08745Sheppo 10821ae08745Sheppo /* check mode */ 10831ae08745Sheppo if (ldcp->mode != (ldc_mode_t)msg->env) { 10841ae08745Sheppo cmn_err(CE_NOTE, 10851ae08745Sheppo "i_ldc_process_RTS: (0x%lx) mode mismatch\n", 10861ae08745Sheppo ldcp->id); 10871ae08745Sheppo /* 10881ae08745Sheppo * send NACK in response to MODE message 10891ae08745Sheppo * get the current tail for the response 10901ae08745Sheppo */ 10911ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, LDC_RTS); 10921ae08745Sheppo if (rv) { 10931ae08745Sheppo /* if cannot send NACK - reset channel */ 1094d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 10951ae08745Sheppo i_ldc_reset(ldcp); 1096d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 10971ae08745Sheppo rv = ECONNRESET; 10981ae08745Sheppo break; 10991ae08745Sheppo } 11001ae08745Sheppo sent_NACK = B_TRUE; 11011ae08745Sheppo } 11021ae08745Sheppo break; 11031ae08745Sheppo default: 11041ae08745Sheppo DWARN(ldcp->id, "i_ldc_process_RTS: (0x%llx) unexp ACK\n", 11051ae08745Sheppo ldcp->id); 1106d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 11071ae08745Sheppo i_ldc_reset(ldcp); 1108d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11091ae08745Sheppo rv = ECONNRESET; 11101ae08745Sheppo break; 11111ae08745Sheppo } 11121ae08745Sheppo 11131ae08745Sheppo /* 11141ae08745Sheppo * If either the connection was reset (when rv != 0) or 11151ae08745Sheppo * a NACK was sent, we return. In the case of a NACK 11161ae08745Sheppo * we dont want to consume the packet that came in but 11171ae08745Sheppo * not record that we received the RTS 11181ae08745Sheppo */ 11191ae08745Sheppo if (rv || sent_NACK) 11201ae08745Sheppo return (rv); 11211ae08745Sheppo 11221ae08745Sheppo /* record RTS received */ 11231ae08745Sheppo ldcp->hstate |= TS_RCVD_RTS; 11241ae08745Sheppo 11251ae08745Sheppo /* store initial SEQID info */ 11261ae08745Sheppo ldcp->last_msg_snt = msg->seqid; 11271ae08745Sheppo 1128d10e4ef2Snarayan /* Obtain Tx lock */ 1129d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 1130d10e4ef2Snarayan 11311ae08745Sheppo /* get the current tail for the response */ 11321ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 11331ae08745Sheppo if (rv != 0) { 11341ae08745Sheppo cmn_err(CE_NOTE, 11351ae08745Sheppo "i_ldc_process_RTS: (0x%lx) err sending RTR\n", 11361ae08745Sheppo ldcp->id); 11371ae08745Sheppo i_ldc_reset(ldcp); 1138d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11391ae08745Sheppo return (ECONNRESET); 11401ae08745Sheppo } 11411ae08745Sheppo 11421ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 11431ae08745Sheppo ZERO_PKT(pkt); 11441ae08745Sheppo 11451ae08745Sheppo /* initialize the packet */ 11461ae08745Sheppo pkt->type = LDC_CTRL; 11471ae08745Sheppo pkt->stype = LDC_INFO; 11481ae08745Sheppo pkt->ctrl = LDC_RTR; 11491ae08745Sheppo pkt->env = ldcp->mode; 11501ae08745Sheppo if (ldcp->mode != LDC_MODE_RAW) 11511ae08745Sheppo pkt->seqid = LDC_INIT_SEQID; 11521ae08745Sheppo 11531ae08745Sheppo ldcp->last_msg_rcd = msg->seqid; 11541ae08745Sheppo 11551ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 11561ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 11571ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 11581ae08745Sheppo 11591ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 11601ae08745Sheppo if (rv == 0) { 11611ae08745Sheppo D2(ldcp->id, 11621ae08745Sheppo "i_ldc_process_RTS: (0x%llx) sent RTR\n", ldcp->id); 11631ae08745Sheppo DUMP_LDC_PKT(ldcp, "i_ldc_process_RTS sent rtr", (uint64_t)pkt); 11641ae08745Sheppo 11651ae08745Sheppo ldcp->tx_tail = tx_tail; 11661ae08745Sheppo ldcp->hstate |= TS_SENT_RTR; 11671ae08745Sheppo 11681ae08745Sheppo } else { 11691ae08745Sheppo cmn_err(CE_NOTE, 11701ae08745Sheppo "i_ldc_process_RTS: (0x%lx) error sending RTR\n", 11711ae08745Sheppo ldcp->id); 11721ae08745Sheppo i_ldc_reset(ldcp); 1173d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11741ae08745Sheppo return (ECONNRESET); 11751ae08745Sheppo } 11761ae08745Sheppo 1177d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 11781ae08745Sheppo return (0); 11791ae08745Sheppo } 11801ae08745Sheppo 11811ae08745Sheppo /* 11821ae08745Sheppo * Process an incoming RTR ctrl message 11831ae08745Sheppo */ 11841ae08745Sheppo static int 11851ae08745Sheppo i_ldc_process_RTR(ldc_chan_t *ldcp, ldc_msg_t *msg) 11861ae08745Sheppo { 11871ae08745Sheppo int rv = 0; 11881ae08745Sheppo boolean_t sent_NACK = B_FALSE; 11891ae08745Sheppo 11901ae08745Sheppo D2(ldcp->id, "i_ldc_process_RTR: (0x%llx) received RTR\n", ldcp->id); 11911ae08745Sheppo 11921ae08745Sheppo switch (msg->stype) { 11931ae08745Sheppo case LDC_NACK: 11941ae08745Sheppo /* RTR NACK received */ 11951ae08745Sheppo DWARN(ldcp->id, 11961ae08745Sheppo "i_ldc_process_RTR: (0x%llx) RTR NACK received\n", 11971ae08745Sheppo ldcp->id); 11981ae08745Sheppo 11991ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1200d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 12011ae08745Sheppo i_ldc_reset(ldcp); 1202d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 12031ae08745Sheppo rv = ECONNRESET; 12041ae08745Sheppo 12051ae08745Sheppo break; 12061ae08745Sheppo 12071ae08745Sheppo case LDC_INFO: 12081ae08745Sheppo 12091ae08745Sheppo /* check mode */ 12101ae08745Sheppo if (ldcp->mode != (ldc_mode_t)msg->env) { 12111ae08745Sheppo DWARN(ldcp->id, 12121ae08745Sheppo "i_ldc_process_RTR: (0x%llx) mode mismatch\n", 12131ae08745Sheppo ldcp->id); 12141ae08745Sheppo /* 12151ae08745Sheppo * send NACK in response to MODE message 12161ae08745Sheppo * get the current tail for the response 12171ae08745Sheppo */ 12181ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, LDC_RTR); 12191ae08745Sheppo if (rv) { 12201ae08745Sheppo /* if cannot send NACK - reset channel */ 1221d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 12221ae08745Sheppo i_ldc_reset(ldcp); 1223d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 12241ae08745Sheppo rv = ECONNRESET; 12251ae08745Sheppo break; 12261ae08745Sheppo } 12271ae08745Sheppo sent_NACK = B_TRUE; 12281ae08745Sheppo } 12291ae08745Sheppo break; 12301ae08745Sheppo 12311ae08745Sheppo default: 12321ae08745Sheppo DWARN(ldcp->id, "i_ldc_process_RTR: (0x%llx) unexp ACK\n", 12331ae08745Sheppo ldcp->id); 12341ae08745Sheppo 12351ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1236d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 12371ae08745Sheppo i_ldc_reset(ldcp); 1238d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 12391ae08745Sheppo rv = ECONNRESET; 12401ae08745Sheppo break; 12411ae08745Sheppo } 12421ae08745Sheppo 12431ae08745Sheppo /* 12441ae08745Sheppo * If either the connection was reset (when rv != 0) or 12451ae08745Sheppo * a NACK was sent, we return. In the case of a NACK 12461ae08745Sheppo * we dont want to consume the packet that came in but 12471ae08745Sheppo * not record that we received the RTR 12481ae08745Sheppo */ 12491ae08745Sheppo if (rv || sent_NACK) 12501ae08745Sheppo return (rv); 12511ae08745Sheppo 12521ae08745Sheppo ldcp->last_msg_snt = msg->seqid; 12531ae08745Sheppo ldcp->hstate |= TS_RCVD_RTR; 12541ae08745Sheppo 12551ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_INFO, LDC_RDX); 12561ae08745Sheppo if (rv) { 12571ae08745Sheppo cmn_err(CE_NOTE, 12581ae08745Sheppo "i_ldc_process_RTR: (0x%lx) cannot send RDX\n", 12591ae08745Sheppo ldcp->id); 1260d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 12611ae08745Sheppo i_ldc_reset(ldcp); 1262d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 12631ae08745Sheppo return (ECONNRESET); 12641ae08745Sheppo } 12651ae08745Sheppo D2(ldcp->id, 12661ae08745Sheppo "i_ldc_process_RTR: (0x%llx) sent RDX\n", ldcp->id); 12671ae08745Sheppo 12681ae08745Sheppo ldcp->hstate |= TS_SENT_RDX; 12691ae08745Sheppo ldcp->tstate |= TS_HSHAKE_DONE; 12701ae08745Sheppo ldcp->status = LDC_UP; 12711ae08745Sheppo 12721ae08745Sheppo DWARN(DBG_ALL_LDCS, "(0x%llx) Handshake Complete\n", ldcp->id); 12731ae08745Sheppo 12741ae08745Sheppo return (0); 12751ae08745Sheppo } 12761ae08745Sheppo 12771ae08745Sheppo 12781ae08745Sheppo /* 12791ae08745Sheppo * Process an incoming RDX ctrl message 12801ae08745Sheppo */ 12811ae08745Sheppo static int 12821ae08745Sheppo i_ldc_process_RDX(ldc_chan_t *ldcp, ldc_msg_t *msg) 12831ae08745Sheppo { 12841ae08745Sheppo int rv = 0; 12851ae08745Sheppo 12861ae08745Sheppo D2(ldcp->id, "i_ldc_process_RDX: (0x%llx) received RDX\n", ldcp->id); 12871ae08745Sheppo 12881ae08745Sheppo switch (msg->stype) { 12891ae08745Sheppo case LDC_NACK: 12901ae08745Sheppo /* RDX NACK received */ 12911ae08745Sheppo DWARN(ldcp->id, 12921ae08745Sheppo "i_ldc_process_RDX: (0x%llx) RDX NACK received\n", 12931ae08745Sheppo ldcp->id); 12941ae08745Sheppo 12951ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1296d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 12971ae08745Sheppo i_ldc_reset(ldcp); 1298d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 12991ae08745Sheppo rv = ECONNRESET; 13001ae08745Sheppo 13011ae08745Sheppo break; 13021ae08745Sheppo 13031ae08745Sheppo case LDC_INFO: 13041ae08745Sheppo 13051ae08745Sheppo /* 13061ae08745Sheppo * if channel is UP and a RDX received after data transmission 13071ae08745Sheppo * has commenced it is an error 13081ae08745Sheppo */ 13091ae08745Sheppo if ((ldcp->tstate == TS_UP) && (ldcp->hstate & TS_RCVD_RDX)) { 13101ae08745Sheppo DWARN(DBG_ALL_LDCS, 13111ae08745Sheppo "i_ldc_process_RDX: (0x%llx) unexpected RDX" 13121ae08745Sheppo " - LDC reset\n", ldcp->id); 1313d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 13141ae08745Sheppo i_ldc_reset(ldcp); 1315d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 13161ae08745Sheppo return (ECONNRESET); 13171ae08745Sheppo } 13181ae08745Sheppo 13191ae08745Sheppo ldcp->hstate |= TS_RCVD_RDX; 13201ae08745Sheppo ldcp->tstate |= TS_HSHAKE_DONE; 13211ae08745Sheppo ldcp->status = LDC_UP; 13221ae08745Sheppo 13231ae08745Sheppo D1(DBG_ALL_LDCS, "(0x%llx) Handshake Complete\n", ldcp->id); 13241ae08745Sheppo break; 13251ae08745Sheppo 13261ae08745Sheppo default: 13271ae08745Sheppo DWARN(ldcp->id, "i_ldc_process_RDX: (0x%llx) unexp ACK\n", 13281ae08745Sheppo ldcp->id); 13291ae08745Sheppo 13301ae08745Sheppo /* Reset the channel -- as we cannot continue */ 1331d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 13321ae08745Sheppo i_ldc_reset(ldcp); 1333d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 13341ae08745Sheppo rv = ECONNRESET; 13351ae08745Sheppo break; 13361ae08745Sheppo } 13371ae08745Sheppo 13381ae08745Sheppo return (rv); 13391ae08745Sheppo } 13401ae08745Sheppo 13411ae08745Sheppo /* 13421ae08745Sheppo * Process an incoming ACK for a data packet 13431ae08745Sheppo */ 13441ae08745Sheppo static int 13451ae08745Sheppo i_ldc_process_data_ACK(ldc_chan_t *ldcp, ldc_msg_t *msg) 13461ae08745Sheppo { 13471ae08745Sheppo int rv; 13481ae08745Sheppo uint64_t tx_head; 13491ae08745Sheppo ldc_msg_t *pkt; 13501ae08745Sheppo 1351d10e4ef2Snarayan /* Obtain Tx lock */ 1352d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 1353d10e4ef2Snarayan 13541ae08745Sheppo /* 1355d10e4ef2Snarayan * Read the current Tx head and tail 13561ae08745Sheppo */ 13571ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 13581ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 13591ae08745Sheppo if (rv != 0) { 13601ae08745Sheppo cmn_err(CE_WARN, 13611ae08745Sheppo "i_ldc_process_data_ACK: (0x%lx) cannot read qptrs\n", 13621ae08745Sheppo ldcp->id); 1363d10e4ef2Snarayan 1364d10e4ef2Snarayan /* Reset the channel -- as we cannot continue */ 1365d10e4ef2Snarayan i_ldc_reset(ldcp); 1366d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 1367d10e4ef2Snarayan return (ECONNRESET); 13681ae08745Sheppo } 13691ae08745Sheppo 13701ae08745Sheppo /* 13711ae08745Sheppo * loop from where the previous ACK location was to the 13721ae08745Sheppo * current head location. This is how far the HV has 13731ae08745Sheppo * actually send pkts. Pkts between head and tail are 13741ae08745Sheppo * yet to be sent by HV. 13751ae08745Sheppo */ 13761ae08745Sheppo tx_head = ldcp->tx_ackd_head; 13771ae08745Sheppo for (;;) { 13781ae08745Sheppo pkt = (ldc_msg_t *)(ldcp->tx_q_va + tx_head); 13791ae08745Sheppo tx_head = (tx_head + LDC_PACKET_SIZE) % 13801ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 13811ae08745Sheppo 13821ae08745Sheppo if (pkt->seqid == msg->ackid) { 13831ae08745Sheppo D2(ldcp->id, 13841ae08745Sheppo "i_ldc_process_data_ACK: (0x%llx) found packet\n", 13851ae08745Sheppo ldcp->id); 13861ae08745Sheppo ldcp->last_ack_rcd = msg->ackid; 13871ae08745Sheppo ldcp->tx_ackd_head = tx_head; 13881ae08745Sheppo break; 13891ae08745Sheppo } 13901ae08745Sheppo if (tx_head == ldcp->tx_head) { 13911ae08745Sheppo /* could not find packet */ 13921ae08745Sheppo DWARN(ldcp->id, 13931ae08745Sheppo "i_ldc_process_data_ACK: (0x%llx) invalid ACKid\n", 13941ae08745Sheppo ldcp->id); 1395d10e4ef2Snarayan 1396d10e4ef2Snarayan /* Reset the channel -- as we cannot continue */ 1397d10e4ef2Snarayan i_ldc_reset(ldcp); 1398d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 1399d10e4ef2Snarayan return (ECONNRESET); 14001ae08745Sheppo } 14011ae08745Sheppo } 14021ae08745Sheppo 1403d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 14041ae08745Sheppo return (0); 14051ae08745Sheppo } 14061ae08745Sheppo 14071ae08745Sheppo /* 14081ae08745Sheppo * Process incoming control message 14091ae08745Sheppo * Return 0 - session can continue 14101ae08745Sheppo * EAGAIN - reprocess packet - state was changed 14111ae08745Sheppo * ECONNRESET - channel was reset 14121ae08745Sheppo */ 14131ae08745Sheppo static int 14141ae08745Sheppo i_ldc_ctrlmsg(ldc_chan_t *ldcp, ldc_msg_t *msg) 14151ae08745Sheppo { 14161ae08745Sheppo int rv = 0; 14171ae08745Sheppo 14181ae08745Sheppo switch (ldcp->tstate) { 14191ae08745Sheppo 14201ae08745Sheppo case TS_OPEN: 14211ae08745Sheppo case TS_READY: 14221ae08745Sheppo 14231ae08745Sheppo switch (msg->ctrl & LDC_CTRL_MASK) { 14241ae08745Sheppo case LDC_VER: 14251ae08745Sheppo /* process version message */ 14261ae08745Sheppo rv = i_ldc_process_VER(ldcp, msg); 14271ae08745Sheppo break; 14281ae08745Sheppo default: 14291ae08745Sheppo DWARN(ldcp->id, 14301ae08745Sheppo "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 14311ae08745Sheppo "tstate=0x%x\n", ldcp->id, 14321ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 14331ae08745Sheppo break; 14341ae08745Sheppo } 14351ae08745Sheppo 14361ae08745Sheppo break; 14371ae08745Sheppo 14381ae08745Sheppo case TS_VREADY: 14391ae08745Sheppo 14401ae08745Sheppo switch (msg->ctrl & LDC_CTRL_MASK) { 14411ae08745Sheppo case LDC_VER: 14421ae08745Sheppo /* peer is redoing version negotiation */ 1443d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 14441ae08745Sheppo (void) i_ldc_txq_reconf(ldcp); 14451ae08745Sheppo i_ldc_reset_state(ldcp); 1446d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 14471ae08745Sheppo rv = EAGAIN; 14481ae08745Sheppo break; 14491ae08745Sheppo case LDC_RTS: 14501ae08745Sheppo /* process RTS message */ 14511ae08745Sheppo rv = i_ldc_process_RTS(ldcp, msg); 14521ae08745Sheppo break; 14531ae08745Sheppo case LDC_RTR: 14541ae08745Sheppo /* process RTR message */ 14551ae08745Sheppo rv = i_ldc_process_RTR(ldcp, msg); 14561ae08745Sheppo break; 14571ae08745Sheppo case LDC_RDX: 14581ae08745Sheppo /* process RDX message */ 14591ae08745Sheppo rv = i_ldc_process_RDX(ldcp, msg); 14601ae08745Sheppo break; 14611ae08745Sheppo default: 14621ae08745Sheppo DWARN(ldcp->id, 14631ae08745Sheppo "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 14641ae08745Sheppo "tstate=0x%x\n", ldcp->id, 14651ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 14661ae08745Sheppo break; 14671ae08745Sheppo } 14681ae08745Sheppo 14691ae08745Sheppo break; 14701ae08745Sheppo 14711ae08745Sheppo case TS_UP: 14721ae08745Sheppo 14731ae08745Sheppo switch (msg->ctrl & LDC_CTRL_MASK) { 14741ae08745Sheppo case LDC_VER: 14751ae08745Sheppo DWARN(ldcp->id, 14761ae08745Sheppo "i_ldc_ctrlmsg: (0x%llx) unexpected VER " 14771ae08745Sheppo "- LDC reset\n", ldcp->id); 14781ae08745Sheppo /* peer is redoing version negotiation */ 1479d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 14801ae08745Sheppo (void) i_ldc_txq_reconf(ldcp); 14811ae08745Sheppo i_ldc_reset_state(ldcp); 1482d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 14831ae08745Sheppo rv = EAGAIN; 14841ae08745Sheppo break; 14851ae08745Sheppo 14861ae08745Sheppo case LDC_RDX: 14871ae08745Sheppo /* process RDX message */ 14881ae08745Sheppo rv = i_ldc_process_RDX(ldcp, msg); 14891ae08745Sheppo break; 14901ae08745Sheppo 14911ae08745Sheppo default: 14921ae08745Sheppo DWARN(ldcp->id, 14931ae08745Sheppo "i_ldc_ctrlmsg: (0x%llx) unexp ctrl 0x%x " 14941ae08745Sheppo "tstate=0x%x\n", ldcp->id, 14951ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK), ldcp->tstate); 14961ae08745Sheppo break; 14971ae08745Sheppo } 14981ae08745Sheppo } 14991ae08745Sheppo 15001ae08745Sheppo return (rv); 15011ae08745Sheppo } 15021ae08745Sheppo 15031ae08745Sheppo /* 15041ae08745Sheppo * Register channel with the channel nexus 15051ae08745Sheppo */ 15061ae08745Sheppo static int 15071ae08745Sheppo i_ldc_register_channel(ldc_chan_t *ldcp) 15081ae08745Sheppo { 15091ae08745Sheppo int rv = 0; 15101ae08745Sheppo ldc_cnex_t *cinfo = &ldcssp->cinfo; 15111ae08745Sheppo 15121ae08745Sheppo if (cinfo->dip == NULL) { 15131ae08745Sheppo DWARN(ldcp->id, 15141ae08745Sheppo "i_ldc_register_channel: cnex has not registered\n"); 15151ae08745Sheppo return (EAGAIN); 15161ae08745Sheppo } 15171ae08745Sheppo 15181ae08745Sheppo rv = cinfo->reg_chan(cinfo->dip, ldcp->id, ldcp->devclass); 15191ae08745Sheppo if (rv) { 15201ae08745Sheppo DWARN(ldcp->id, 15211ae08745Sheppo "i_ldc_register_channel: cannot register channel\n"); 15221ae08745Sheppo return (rv); 15231ae08745Sheppo } 15241ae08745Sheppo 15251ae08745Sheppo rv = cinfo->add_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR, 15261ae08745Sheppo i_ldc_tx_hdlr, ldcp, NULL); 15271ae08745Sheppo if (rv) { 15281ae08745Sheppo DWARN(ldcp->id, 15291ae08745Sheppo "i_ldc_register_channel: cannot add Tx interrupt\n"); 15301ae08745Sheppo (void) cinfo->unreg_chan(cinfo->dip, ldcp->id); 15311ae08745Sheppo return (rv); 15321ae08745Sheppo } 15331ae08745Sheppo 15341ae08745Sheppo rv = cinfo->add_intr(cinfo->dip, ldcp->id, CNEX_RX_INTR, 15351ae08745Sheppo i_ldc_rx_hdlr, ldcp, NULL); 15361ae08745Sheppo if (rv) { 15371ae08745Sheppo DWARN(ldcp->id, 15381ae08745Sheppo "i_ldc_register_channel: cannot add Rx interrupt\n"); 15391ae08745Sheppo (void) cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR); 15401ae08745Sheppo (void) cinfo->unreg_chan(cinfo->dip, ldcp->id); 15411ae08745Sheppo return (rv); 15421ae08745Sheppo } 15431ae08745Sheppo 15441ae08745Sheppo ldcp->tstate |= TS_CNEX_RDY; 15451ae08745Sheppo 15461ae08745Sheppo return (0); 15471ae08745Sheppo } 15481ae08745Sheppo 15491ae08745Sheppo /* 15501ae08745Sheppo * Unregister a channel with the channel nexus 15511ae08745Sheppo */ 15521ae08745Sheppo static int 15531ae08745Sheppo i_ldc_unregister_channel(ldc_chan_t *ldcp) 15541ae08745Sheppo { 15551ae08745Sheppo int rv = 0; 15561ae08745Sheppo ldc_cnex_t *cinfo = &ldcssp->cinfo; 15571ae08745Sheppo 15581ae08745Sheppo if (cinfo->dip == NULL) { 15591ae08745Sheppo DWARN(ldcp->id, 15601ae08745Sheppo "i_ldc_unregister_channel: cnex has not registered\n"); 15611ae08745Sheppo return (EAGAIN); 15621ae08745Sheppo } 15631ae08745Sheppo 15641ae08745Sheppo if (ldcp->tstate & TS_CNEX_RDY) { 15651ae08745Sheppo 1566d10e4ef2Snarayan /* Remove the Rx interrupt */ 15671ae08745Sheppo rv = cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_RX_INTR); 15681ae08745Sheppo if (rv) { 15691ae08745Sheppo DWARN(ldcp->id, 15701ae08745Sheppo "i_ldc_unregister_channel: err removing Rx intr\n"); 1571d10e4ef2Snarayan return (rv); 15721ae08745Sheppo } 1573d10e4ef2Snarayan 1574d10e4ef2Snarayan /* Remove the Tx interrupt */ 15751ae08745Sheppo rv = cinfo->rem_intr(cinfo->dip, ldcp->id, CNEX_TX_INTR); 15761ae08745Sheppo if (rv) { 15771ae08745Sheppo DWARN(ldcp->id, 15781ae08745Sheppo "i_ldc_unregister_channel: err removing Tx intr\n"); 1579d10e4ef2Snarayan return (rv); 15801ae08745Sheppo } 1581d10e4ef2Snarayan 1582d10e4ef2Snarayan /* Unregister the channel */ 15831ae08745Sheppo rv = cinfo->unreg_chan(ldcssp->cinfo.dip, ldcp->id); 15841ae08745Sheppo if (rv) { 15851ae08745Sheppo DWARN(ldcp->id, 15861ae08745Sheppo "i_ldc_unregister_channel: cannot unreg channel\n"); 1587d10e4ef2Snarayan return (rv); 15881ae08745Sheppo } 15891ae08745Sheppo 15901ae08745Sheppo ldcp->tstate &= ~TS_CNEX_RDY; 15911ae08745Sheppo } 15921ae08745Sheppo 15931ae08745Sheppo return (0); 15941ae08745Sheppo } 15951ae08745Sheppo 15961ae08745Sheppo 15971ae08745Sheppo /* 15981ae08745Sheppo * LDC transmit interrupt handler 15991ae08745Sheppo * triggered for chanel up/down/reset events 16001ae08745Sheppo * and Tx queue content changes 16011ae08745Sheppo */ 16021ae08745Sheppo static uint_t 16031ae08745Sheppo i_ldc_tx_hdlr(caddr_t arg1, caddr_t arg2) 16041ae08745Sheppo { 16051ae08745Sheppo _NOTE(ARGUNUSED(arg2)) 16061ae08745Sheppo 16071ae08745Sheppo int rv; 16081ae08745Sheppo ldc_chan_t *ldcp; 16091ae08745Sheppo boolean_t notify_client = B_FALSE; 16101ae08745Sheppo uint64_t notify_event = 0; 16111ae08745Sheppo 16121ae08745Sheppo /* Get the channel for which interrupt was received */ 16131ae08745Sheppo ASSERT(arg1 != NULL); 16141ae08745Sheppo ldcp = (ldc_chan_t *)arg1; 16151ae08745Sheppo 16161ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n", 16171ae08745Sheppo ldcp->id, ldcp); 16181ae08745Sheppo 16191ae08745Sheppo /* Lock channel */ 16201ae08745Sheppo mutex_enter(&ldcp->lock); 16211ae08745Sheppo 1622d10e4ef2Snarayan /* Obtain Tx lock */ 1623d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 1624d10e4ef2Snarayan 16251ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, &ldcp->tx_head, &ldcp->tx_tail, 16261ae08745Sheppo &ldcp->link_state); 16271ae08745Sheppo if (rv) { 16281ae08745Sheppo cmn_err(CE_WARN, 16291ae08745Sheppo "i_ldc_tx_hdlr: (0x%lx) cannot read queue ptrs rv=0x%d\n", 16301ae08745Sheppo ldcp->id, rv); 1631d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 16321ae08745Sheppo mutex_exit(&ldcp->lock); 16331ae08745Sheppo return (DDI_INTR_CLAIMED); 16341ae08745Sheppo } 16351ae08745Sheppo 16361ae08745Sheppo /* 16371ae08745Sheppo * reset the channel state if the channel went down 16381ae08745Sheppo * (other side unconfigured queue) or channel was reset 16391ae08745Sheppo * (other side reconfigured its queue) 16401ae08745Sheppo */ 16411ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN) { 16421ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: channel link down\n", ldcp->id); 16431ae08745Sheppo i_ldc_reset(ldcp); 16441ae08745Sheppo notify_client = B_TRUE; 16451ae08745Sheppo notify_event = LDC_EVT_DOWN; 16461ae08745Sheppo } 16471ae08745Sheppo 16481ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_RESET) { 16491ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: channel link reset\n", ldcp->id); 16501ae08745Sheppo i_ldc_reset(ldcp); 16511ae08745Sheppo notify_client = B_TRUE; 16521ae08745Sheppo notify_event = LDC_EVT_RESET; 16531ae08745Sheppo } 16541ae08745Sheppo 16551ae08745Sheppo if (ldcp->tstate == TS_OPEN && ldcp->link_state == LDC_CHANNEL_UP) { 16561ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: channel link up\n", ldcp->id); 16571ae08745Sheppo notify_client = B_TRUE; 16581ae08745Sheppo notify_event = LDC_EVT_RESET; 16591ae08745Sheppo ldcp->tstate |= TS_LINK_READY; 16601ae08745Sheppo ldcp->status = LDC_READY; 16611ae08745Sheppo } 16621ae08745Sheppo 16631ae08745Sheppo /* if callbacks are disabled, do not notify */ 16641ae08745Sheppo if (!ldcp->cb_enabled) 16651ae08745Sheppo notify_client = B_FALSE; 16661ae08745Sheppo 16671ae08745Sheppo if (notify_client) 16681ae08745Sheppo ldcp->cb_inprogress = B_TRUE; 16691ae08745Sheppo 16701ae08745Sheppo /* Unlock channel */ 1671d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 16721ae08745Sheppo mutex_exit(&ldcp->lock); 16731ae08745Sheppo 16741ae08745Sheppo if (notify_client) { 16751ae08745Sheppo rv = ldcp->cb(notify_event, ldcp->cb_arg); 16761ae08745Sheppo if (rv) { 16771ae08745Sheppo DWARN(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) callback " 16781ae08745Sheppo "failure", ldcp->id); 16791ae08745Sheppo } 16801ae08745Sheppo mutex_enter(&ldcp->lock); 16811ae08745Sheppo ldcp->cb_inprogress = B_FALSE; 16821ae08745Sheppo mutex_exit(&ldcp->lock); 16831ae08745Sheppo } 16841ae08745Sheppo 16851ae08745Sheppo mutex_enter(&ldcp->lock); 16861ae08745Sheppo i_ldc_clear_intr(ldcp, CNEX_TX_INTR); 16871ae08745Sheppo mutex_exit(&ldcp->lock); 16881ae08745Sheppo 16891ae08745Sheppo D1(ldcp->id, "i_ldc_tx_hdlr: (0x%llx) exiting handler", ldcp->id); 16901ae08745Sheppo 16911ae08745Sheppo return (DDI_INTR_CLAIMED); 16921ae08745Sheppo } 16931ae08745Sheppo 16941ae08745Sheppo /* 16951ae08745Sheppo * LDC receive interrupt handler 16961ae08745Sheppo * triggered for channel with data pending to read 16971ae08745Sheppo * i.e. Rx queue content changes 16981ae08745Sheppo */ 16991ae08745Sheppo static uint_t 17001ae08745Sheppo i_ldc_rx_hdlr(caddr_t arg1, caddr_t arg2) 17011ae08745Sheppo { 17021ae08745Sheppo _NOTE(ARGUNUSED(arg2)) 17031ae08745Sheppo 17041ae08745Sheppo int rv; 17051ae08745Sheppo uint64_t rx_head, rx_tail; 17061ae08745Sheppo ldc_msg_t *msg; 17071ae08745Sheppo ldc_chan_t *ldcp; 17081ae08745Sheppo boolean_t notify_client = B_FALSE; 17091ae08745Sheppo uint64_t notify_event = 0; 1710d10e4ef2Snarayan uint64_t first_fragment = 0; 17111ae08745Sheppo 17121ae08745Sheppo /* Get the channel for which interrupt was received */ 17131ae08745Sheppo if (arg1 == NULL) { 17141ae08745Sheppo cmn_err(CE_WARN, "i_ldc_rx_hdlr: invalid arg\n"); 17151ae08745Sheppo return (DDI_INTR_UNCLAIMED); 17161ae08745Sheppo } 17171ae08745Sheppo 17181ae08745Sheppo ldcp = (ldc_chan_t *)arg1; 17191ae08745Sheppo 17201ae08745Sheppo D1(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) Received intr, ldcp=0x%p\n", 17211ae08745Sheppo ldcp->id, ldcp); 17221ae08745Sheppo 17231ae08745Sheppo /* Lock channel */ 17241ae08745Sheppo mutex_enter(&ldcp->lock); 17251ae08745Sheppo 17261ae08745Sheppo /* mark interrupt as pending */ 17271ae08745Sheppo ldcp->intr_pending = B_TRUE; 17281ae08745Sheppo 17291ae08745Sheppo /* 17301ae08745Sheppo * Read packet(s) from the queue 17311ae08745Sheppo */ 17321ae08745Sheppo for (;;) { 17331ae08745Sheppo 17341ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 17351ae08745Sheppo &ldcp->link_state); 17361ae08745Sheppo if (rv) { 17371ae08745Sheppo cmn_err(CE_WARN, 17381ae08745Sheppo "i_ldc_rx_hdlr: (0x%lx) cannot read " 17391ae08745Sheppo "queue ptrs, rv=0x%d\n", ldcp->id, rv); 17401ae08745Sheppo i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 17411ae08745Sheppo mutex_exit(&ldcp->lock); 17421ae08745Sheppo return (DDI_INTR_CLAIMED); 17431ae08745Sheppo } 17441ae08745Sheppo 17451ae08745Sheppo /* 17461ae08745Sheppo * reset the channel state if the channel went down 17471ae08745Sheppo * (other side unconfigured queue) or channel was reset 17481ae08745Sheppo * (other side reconfigured its queue 17491ae08745Sheppo */ 17501ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN) { 17511ae08745Sheppo D1(ldcp->id, "i_ldc_rx_hdlr: channel link down\n", 17521ae08745Sheppo ldcp->id); 1753d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 17541ae08745Sheppo i_ldc_reset(ldcp); 1755d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 17561ae08745Sheppo notify_client = B_TRUE; 17571ae08745Sheppo notify_event = LDC_EVT_DOWN; 17581ae08745Sheppo break; 17591ae08745Sheppo } 17601ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_RESET) { 17611ae08745Sheppo D1(ldcp->id, "i_ldc_rx_hdlr: channel link reset\n", 17621ae08745Sheppo ldcp->id); 1763d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 17641ae08745Sheppo i_ldc_reset(ldcp); 1765d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 17661ae08745Sheppo notify_client = B_TRUE; 17671ae08745Sheppo notify_event = LDC_EVT_RESET; 17681ae08745Sheppo } 17691ae08745Sheppo 17701ae08745Sheppo if (ldcp->tstate == TS_OPEN && 17711ae08745Sheppo ldcp->link_state == LDC_CHANNEL_UP) { 17721ae08745Sheppo D1(ldcp->id, "i_ldc_rx_hdlr: channel link up\n", 17731ae08745Sheppo ldcp->id); 17741ae08745Sheppo notify_client = B_TRUE; 17751ae08745Sheppo notify_event = LDC_EVT_RESET; 17761ae08745Sheppo ldcp->tstate |= TS_LINK_READY; 17771ae08745Sheppo ldcp->status = LDC_READY; 17781ae08745Sheppo } 17791ae08745Sheppo 17801ae08745Sheppo if (rx_head == rx_tail) { 17811ae08745Sheppo D2(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) No packets\n", 17821ae08745Sheppo ldcp->id); 17831ae08745Sheppo break; 17841ae08745Sheppo } 17851ae08745Sheppo D2(ldcp->id, "i_ldc_rx_hdlr: head=0x%llx, tail=0x%llx\n", 17861ae08745Sheppo rx_head, rx_tail); 17871ae08745Sheppo DUMP_LDC_PKT(ldcp, "i_ldc_rx_hdlr rcd", 17881ae08745Sheppo ldcp->rx_q_va + rx_head); 17891ae08745Sheppo 17901ae08745Sheppo /* get the message */ 17911ae08745Sheppo msg = (ldc_msg_t *)(ldcp->rx_q_va + rx_head); 17921ae08745Sheppo 17931ae08745Sheppo /* if channel is in RAW mode or data pkt, notify and return */ 17941ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW) { 17951ae08745Sheppo notify_client = B_TRUE; 17961ae08745Sheppo notify_event |= LDC_EVT_READ; 17971ae08745Sheppo break; 17981ae08745Sheppo } 17991ae08745Sheppo 18001ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_INFO)) { 18011ae08745Sheppo 18021ae08745Sheppo /* discard packet if channel is not up */ 18031ae08745Sheppo if (ldcp->tstate != TS_UP) { 18041ae08745Sheppo 18051ae08745Sheppo /* move the head one position */ 18061ae08745Sheppo rx_head = (rx_head + LDC_PACKET_SIZE) % 18071ae08745Sheppo (ldcp->rx_q_entries << LDC_PACKET_SHIFT); 18081ae08745Sheppo 18091ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, rx_head)) 18101ae08745Sheppo break; 18111ae08745Sheppo 18121ae08745Sheppo continue; 18131ae08745Sheppo } else { 18141ae08745Sheppo notify_client = B_TRUE; 18151ae08745Sheppo notify_event |= LDC_EVT_READ; 18161ae08745Sheppo break; 18171ae08745Sheppo } 18181ae08745Sheppo } 18191ae08745Sheppo 18201ae08745Sheppo /* Check the sequence ID for the message received */ 18211ae08745Sheppo if ((rv = i_ldc_check_seqid(ldcp, msg)) != 0) { 18221ae08745Sheppo 18231ae08745Sheppo DWARN(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) seqid error, " 18241ae08745Sheppo "q_ptrs=0x%lx,0x%lx", ldcp->id, rx_head, rx_tail); 18251ae08745Sheppo 18261ae08745Sheppo /* Reset last_msg_rcd to start of message */ 1827d10e4ef2Snarayan if (first_fragment != 0) { 1828d10e4ef2Snarayan ldcp->last_msg_rcd = first_fragment - 1; 1829d10e4ef2Snarayan first_fragment = 0; 18301ae08745Sheppo } 1831d10e4ef2Snarayan 18321ae08745Sheppo /* 18331ae08745Sheppo * Send a NACK due to seqid mismatch 18341ae08745Sheppo */ 18351ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_CTRL, LDC_NACK, 18361ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK)); 18371ae08745Sheppo 18381ae08745Sheppo if (rv) { 18391ae08745Sheppo cmn_err(CE_NOTE, 18401ae08745Sheppo "i_ldc_rx_hdlr: (0x%lx) err sending " 18411ae08745Sheppo "CTRL/NACK msg\n", ldcp->id); 1842d10e4ef2Snarayan 1843d10e4ef2Snarayan /* if cannot send NACK - reset channel */ 1844d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 1845d10e4ef2Snarayan i_ldc_reset(ldcp); 1846d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 1847d10e4ef2Snarayan rv = ECONNRESET; 1848d10e4ef2Snarayan break; 18491ae08745Sheppo } 18501ae08745Sheppo 18511ae08745Sheppo /* purge receive queue */ 18521ae08745Sheppo (void) i_ldc_set_rx_head(ldcp, rx_tail); 18531ae08745Sheppo break; 18541ae08745Sheppo } 18551ae08745Sheppo 18561ae08745Sheppo /* record the message ID */ 18571ae08745Sheppo ldcp->last_msg_rcd = msg->seqid; 18581ae08745Sheppo 18591ae08745Sheppo /* process control messages */ 18601ae08745Sheppo if (msg->type & LDC_CTRL) { 18611ae08745Sheppo /* save current internal state */ 18621ae08745Sheppo uint64_t tstate = ldcp->tstate; 18631ae08745Sheppo 18641ae08745Sheppo rv = i_ldc_ctrlmsg(ldcp, msg); 18651ae08745Sheppo if (rv == EAGAIN) { 18661ae08745Sheppo /* re-process pkt - state was adjusted */ 18671ae08745Sheppo continue; 18681ae08745Sheppo } 18691ae08745Sheppo if (rv == ECONNRESET) { 18701ae08745Sheppo notify_client = B_TRUE; 18711ae08745Sheppo notify_event = LDC_EVT_RESET; 18721ae08745Sheppo break; 18731ae08745Sheppo } 18741ae08745Sheppo 18751ae08745Sheppo /* 18761ae08745Sheppo * control message processing was successful 18771ae08745Sheppo * channel transitioned to ready for communication 18781ae08745Sheppo */ 18791ae08745Sheppo if (rv == 0 && ldcp->tstate == TS_UP && 18801ae08745Sheppo tstate != ldcp->tstate) { 18811ae08745Sheppo notify_client = B_TRUE; 18821ae08745Sheppo notify_event = LDC_EVT_UP; 18831ae08745Sheppo } 18841ae08745Sheppo } 18851ae08745Sheppo 18861ae08745Sheppo /* process data ACKs */ 18871ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 1888d10e4ef2Snarayan if (rv = i_ldc_process_data_ACK(ldcp, msg)) { 1889d10e4ef2Snarayan notify_client = B_TRUE; 1890d10e4ef2Snarayan notify_event = LDC_EVT_RESET; 1891d10e4ef2Snarayan break; 1892d10e4ef2Snarayan } 18931ae08745Sheppo } 18941ae08745Sheppo 18951ae08745Sheppo /* move the head one position */ 18961ae08745Sheppo rx_head = (rx_head + LDC_PACKET_SIZE) % 18971ae08745Sheppo (ldcp->rx_q_entries << LDC_PACKET_SHIFT); 18980a55fbb7Slm66018 if (rv = i_ldc_set_rx_head(ldcp, rx_head)) { 18990a55fbb7Slm66018 notify_client = B_TRUE; 19000a55fbb7Slm66018 notify_event = LDC_EVT_RESET; 19011ae08745Sheppo break; 19020a55fbb7Slm66018 } 19031ae08745Sheppo 19041ae08745Sheppo } /* for */ 19051ae08745Sheppo 19061ae08745Sheppo /* if callbacks are disabled, do not notify */ 19071ae08745Sheppo if (!ldcp->cb_enabled) 19081ae08745Sheppo notify_client = B_FALSE; 19091ae08745Sheppo 19101ae08745Sheppo if (notify_client) 19111ae08745Sheppo ldcp->cb_inprogress = B_TRUE; 19121ae08745Sheppo 19131ae08745Sheppo /* Unlock channel */ 19141ae08745Sheppo mutex_exit(&ldcp->lock); 19151ae08745Sheppo 19161ae08745Sheppo if (notify_client) { 19171ae08745Sheppo rv = ldcp->cb(notify_event, ldcp->cb_arg); 19181ae08745Sheppo if (rv) { 19191ae08745Sheppo DWARN(ldcp->id, 19201ae08745Sheppo "i_ldc_rx_hdlr: (0x%llx) callback failure", 19211ae08745Sheppo ldcp->id); 19221ae08745Sheppo } 19231ae08745Sheppo mutex_enter(&ldcp->lock); 19241ae08745Sheppo ldcp->cb_inprogress = B_FALSE; 19251ae08745Sheppo mutex_exit(&ldcp->lock); 19261ae08745Sheppo } 19271ae08745Sheppo 19281ae08745Sheppo mutex_enter(&ldcp->lock); 19291ae08745Sheppo 19301ae08745Sheppo /* 19311ae08745Sheppo * If there are data packets in the queue, the ldc_read will 19321ae08745Sheppo * clear interrupts after draining the queue, else clear interrupts 19331ae08745Sheppo */ 19341ae08745Sheppo if ((notify_event & LDC_EVT_READ) == 0) { 19351ae08745Sheppo i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 19361ae08745Sheppo } 19371ae08745Sheppo 19381ae08745Sheppo mutex_exit(&ldcp->lock); 19391ae08745Sheppo 19401ae08745Sheppo D1(ldcp->id, "i_ldc_rx_hdlr: (0x%llx) exiting handler", ldcp->id); 19411ae08745Sheppo return (DDI_INTR_CLAIMED); 19421ae08745Sheppo } 19431ae08745Sheppo 19441ae08745Sheppo 19451ae08745Sheppo /* -------------------------------------------------------------------------- */ 19461ae08745Sheppo 19471ae08745Sheppo /* 19481ae08745Sheppo * LDC API functions 19491ae08745Sheppo */ 19501ae08745Sheppo 19511ae08745Sheppo /* 19521ae08745Sheppo * Initialize the channel. Allocate internal structure and memory for 19531ae08745Sheppo * TX/RX queues, and initialize locks. 19541ae08745Sheppo */ 19551ae08745Sheppo int 19561ae08745Sheppo ldc_init(uint64_t id, ldc_attr_t *attr, ldc_handle_t *handle) 19571ae08745Sheppo { 19581ae08745Sheppo ldc_chan_t *ldcp; 19591ae08745Sheppo int rv, exit_val; 19601ae08745Sheppo uint64_t ra_base, nentries; 1961*e1ebb9ecSlm66018 uint64_t qlen; 19621ae08745Sheppo 19631ae08745Sheppo exit_val = EINVAL; /* guarantee an error if exit on failure */ 19641ae08745Sheppo 19651ae08745Sheppo if (attr == NULL) { 19661ae08745Sheppo DWARN(id, "ldc_init: (0x%llx) invalid attr\n", id); 19671ae08745Sheppo return (EINVAL); 19681ae08745Sheppo } 19691ae08745Sheppo if (handle == NULL) { 19701ae08745Sheppo DWARN(id, "ldc_init: (0x%llx) invalid handle\n", id); 19711ae08745Sheppo return (EINVAL); 19721ae08745Sheppo } 19731ae08745Sheppo 19741ae08745Sheppo /* check if channel is valid */ 19751ae08745Sheppo rv = hv_ldc_tx_qinfo(id, &ra_base, &nentries); 19761ae08745Sheppo if (rv == H_ECHANNEL) { 19771ae08745Sheppo DWARN(id, "ldc_init: (0x%llx) invalid channel id\n", id); 19781ae08745Sheppo return (EINVAL); 19791ae08745Sheppo } 19801ae08745Sheppo 19811ae08745Sheppo /* check if the channel has already been initialized */ 19821ae08745Sheppo mutex_enter(&ldcssp->lock); 19831ae08745Sheppo ldcp = ldcssp->chan_list; 19841ae08745Sheppo while (ldcp != NULL) { 19851ae08745Sheppo if (ldcp->id == id) { 19861ae08745Sheppo DWARN(id, "ldc_init: (0x%llx) already initialized\n", 19871ae08745Sheppo id); 19881ae08745Sheppo mutex_exit(&ldcssp->lock); 19891ae08745Sheppo return (EADDRINUSE); 19901ae08745Sheppo } 19911ae08745Sheppo ldcp = ldcp->next; 19921ae08745Sheppo } 19931ae08745Sheppo mutex_exit(&ldcssp->lock); 19941ae08745Sheppo 19951ae08745Sheppo ASSERT(ldcp == NULL); 19961ae08745Sheppo 19971ae08745Sheppo *handle = 0; 19981ae08745Sheppo 19991ae08745Sheppo /* Allocate an ldcp structure */ 20001ae08745Sheppo ldcp = kmem_zalloc(sizeof (ldc_chan_t), KM_SLEEP); 20011ae08745Sheppo 2002d10e4ef2Snarayan /* 2003d10e4ef2Snarayan * Initialize the channel and Tx lock 2004d10e4ef2Snarayan * 2005d10e4ef2Snarayan * The channel 'lock' protects the entire channel and 2006d10e4ef2Snarayan * should be acquired before initializing, resetting, 2007d10e4ef2Snarayan * destroying or reading from a channel. 2008d10e4ef2Snarayan * 2009d10e4ef2Snarayan * The 'tx_lock' should be acquired prior to transmitting 2010d10e4ef2Snarayan * data over the channel. The lock should also be acquired 2011d10e4ef2Snarayan * prior to channel reconfiguration (in order to prevent 2012d10e4ef2Snarayan * concurrent writes). 2013d10e4ef2Snarayan * 2014d10e4ef2Snarayan * ORDERING: When both locks are being acquired, to prevent 2015d10e4ef2Snarayan * deadlocks, the channel lock should be always acquired prior 2016d10e4ef2Snarayan * to the tx_lock. 2017d10e4ef2Snarayan */ 20181ae08745Sheppo mutex_init(&ldcp->lock, NULL, MUTEX_DRIVER, NULL); 2019d10e4ef2Snarayan mutex_init(&ldcp->tx_lock, NULL, MUTEX_DRIVER, NULL); 20201ae08745Sheppo 20211ae08745Sheppo /* Initialize the channel */ 20221ae08745Sheppo ldcp->id = id; 20231ae08745Sheppo ldcp->cb = NULL; 20241ae08745Sheppo ldcp->cb_arg = NULL; 20251ae08745Sheppo ldcp->cb_inprogress = B_FALSE; 20261ae08745Sheppo ldcp->cb_enabled = B_FALSE; 20271ae08745Sheppo ldcp->next = NULL; 20281ae08745Sheppo 20291ae08745Sheppo /* Read attributes */ 20301ae08745Sheppo ldcp->mode = attr->mode; 20311ae08745Sheppo ldcp->devclass = attr->devclass; 20321ae08745Sheppo ldcp->devinst = attr->instance; 2033*e1ebb9ecSlm66018 ldcp->mtu = (attr->mtu > 0) ? attr->mtu : LDC_DEFAULT_MTU; 20341ae08745Sheppo 20351ae08745Sheppo D1(ldcp->id, 20361ae08745Sheppo "ldc_init: (0x%llx) channel attributes, class=0x%x, " 2037*e1ebb9ecSlm66018 "instance=0x%llx, mode=%d, mtu=%d\n", 2038*e1ebb9ecSlm66018 ldcp->id, ldcp->devclass, ldcp->devinst, ldcp->mode, ldcp->mtu); 20391ae08745Sheppo 20401ae08745Sheppo ldcp->next_vidx = 0; 20411ae08745Sheppo ldcp->tstate = 0; 20421ae08745Sheppo ldcp->hstate = 0; 20431ae08745Sheppo ldcp->last_msg_snt = LDC_INIT_SEQID; 20441ae08745Sheppo ldcp->last_ack_rcd = 0; 20451ae08745Sheppo ldcp->last_msg_rcd = 0; 20461ae08745Sheppo 20471ae08745Sheppo ldcp->stream_bufferp = NULL; 20481ae08745Sheppo ldcp->exp_dring_list = NULL; 20491ae08745Sheppo ldcp->imp_dring_list = NULL; 20501ae08745Sheppo ldcp->mhdl_list = NULL; 20511ae08745Sheppo 20521ae08745Sheppo /* Initialize payload size depending on whether channel is reliable */ 20531ae08745Sheppo switch (ldcp->mode) { 20541ae08745Sheppo case LDC_MODE_RAW: 20551ae08745Sheppo ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RAW; 20561ae08745Sheppo ldcp->read_p = i_ldc_read_raw; 20571ae08745Sheppo ldcp->write_p = i_ldc_write_raw; 20581ae08745Sheppo break; 20591ae08745Sheppo case LDC_MODE_UNRELIABLE: 20601ae08745Sheppo ldcp->pkt_payload = LDC_PAYLOAD_SIZE_UNRELIABLE; 20611ae08745Sheppo ldcp->read_p = i_ldc_read_packet; 20621ae08745Sheppo ldcp->write_p = i_ldc_write_packet; 20631ae08745Sheppo break; 20641ae08745Sheppo case LDC_MODE_RELIABLE: 20651ae08745Sheppo ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RELIABLE; 20661ae08745Sheppo ldcp->read_p = i_ldc_read_packet; 20671ae08745Sheppo ldcp->write_p = i_ldc_write_packet; 20681ae08745Sheppo break; 20691ae08745Sheppo case LDC_MODE_STREAM: 20701ae08745Sheppo ldcp->pkt_payload = LDC_PAYLOAD_SIZE_RELIABLE; 20711ae08745Sheppo 20721ae08745Sheppo ldcp->stream_remains = 0; 20731ae08745Sheppo ldcp->stream_offset = 0; 20741ae08745Sheppo ldcp->stream_bufferp = kmem_alloc(ldcp->mtu, KM_SLEEP); 20751ae08745Sheppo ldcp->read_p = i_ldc_read_stream; 20761ae08745Sheppo ldcp->write_p = i_ldc_write_stream; 20771ae08745Sheppo break; 20781ae08745Sheppo default: 20791ae08745Sheppo exit_val = EINVAL; 20801ae08745Sheppo goto cleanup_on_exit; 20811ae08745Sheppo } 20821ae08745Sheppo 2083*e1ebb9ecSlm66018 /* 2084*e1ebb9ecSlm66018 * qlen is (mtu * ldc_mtu_msgs) / pkt_payload. If this 2085*e1ebb9ecSlm66018 * value is smaller than default length of ldc_queue_entries, 2086*e1ebb9ecSlm66018 * qlen is set to ldc_queue_entries.. 2087*e1ebb9ecSlm66018 */ 2088*e1ebb9ecSlm66018 qlen = (ldcp->mtu * ldc_mtu_msgs) / ldcp->pkt_payload; 2089*e1ebb9ecSlm66018 ldcp->rx_q_entries = 2090*e1ebb9ecSlm66018 (qlen < ldc_queue_entries) ? ldc_queue_entries : qlen; 2091*e1ebb9ecSlm66018 ldcp->tx_q_entries = ldcp->rx_q_entries; 2092*e1ebb9ecSlm66018 2093*e1ebb9ecSlm66018 D1(ldcp->id, "ldc_init: queue length = 0x%llx\n", qlen); 2094*e1ebb9ecSlm66018 20951ae08745Sheppo /* Create a transmit queue */ 20961ae08745Sheppo ldcp->tx_q_va = (uint64_t) 20971ae08745Sheppo contig_mem_alloc(ldcp->tx_q_entries << LDC_PACKET_SHIFT); 20981ae08745Sheppo if (ldcp->tx_q_va == NULL) { 20991ae08745Sheppo cmn_err(CE_WARN, 21001ae08745Sheppo "ldc_init: (0x%lx) TX queue allocation failed\n", 21011ae08745Sheppo ldcp->id); 21021ae08745Sheppo exit_val = ENOMEM; 21031ae08745Sheppo goto cleanup_on_exit; 21041ae08745Sheppo } 21051ae08745Sheppo ldcp->tx_q_ra = va_to_pa((caddr_t)ldcp->tx_q_va); 21061ae08745Sheppo 21071ae08745Sheppo D2(ldcp->id, "ldc_init: txq_va=0x%llx, txq_ra=0x%llx, entries=0x%llx\n", 21081ae08745Sheppo ldcp->tx_q_va, ldcp->tx_q_ra, ldcp->tx_q_entries); 21091ae08745Sheppo 21101ae08745Sheppo ldcp->tstate |= TS_TXQ_RDY; 21111ae08745Sheppo 21121ae08745Sheppo /* Create a receive queue */ 21131ae08745Sheppo ldcp->rx_q_va = (uint64_t) 21141ae08745Sheppo contig_mem_alloc(ldcp->rx_q_entries << LDC_PACKET_SHIFT); 21151ae08745Sheppo if (ldcp->rx_q_va == NULL) { 21161ae08745Sheppo cmn_err(CE_WARN, 21171ae08745Sheppo "ldc_init: (0x%lx) RX queue allocation failed\n", 21181ae08745Sheppo ldcp->id); 21191ae08745Sheppo exit_val = ENOMEM; 21201ae08745Sheppo goto cleanup_on_exit; 21211ae08745Sheppo } 21221ae08745Sheppo ldcp->rx_q_ra = va_to_pa((caddr_t)ldcp->rx_q_va); 21231ae08745Sheppo 21241ae08745Sheppo D2(ldcp->id, "ldc_init: rxq_va=0x%llx, rxq_ra=0x%llx, entries=0x%llx\n", 21251ae08745Sheppo ldcp->rx_q_va, ldcp->rx_q_ra, ldcp->rx_q_entries); 21261ae08745Sheppo 21271ae08745Sheppo ldcp->tstate |= TS_RXQ_RDY; 21281ae08745Sheppo 21291ae08745Sheppo /* Init descriptor ring and memory handle list lock */ 21301ae08745Sheppo mutex_init(&ldcp->exp_dlist_lock, NULL, MUTEX_DRIVER, NULL); 21311ae08745Sheppo mutex_init(&ldcp->imp_dlist_lock, NULL, MUTEX_DRIVER, NULL); 21321ae08745Sheppo mutex_init(&ldcp->mlist_lock, NULL, MUTEX_DRIVER, NULL); 21331ae08745Sheppo 21341ae08745Sheppo /* mark status as INITialized */ 21351ae08745Sheppo ldcp->status = LDC_INIT; 21361ae08745Sheppo 21371ae08745Sheppo /* Add to channel list */ 21381ae08745Sheppo mutex_enter(&ldcssp->lock); 21391ae08745Sheppo ldcp->next = ldcssp->chan_list; 21401ae08745Sheppo ldcssp->chan_list = ldcp; 21411ae08745Sheppo ldcssp->channel_count++; 21421ae08745Sheppo mutex_exit(&ldcssp->lock); 21431ae08745Sheppo 21441ae08745Sheppo /* set the handle */ 21451ae08745Sheppo *handle = (ldc_handle_t)ldcp; 21461ae08745Sheppo 21471ae08745Sheppo D1(ldcp->id, "ldc_init: (0x%llx) channel initialized\n", ldcp->id); 21481ae08745Sheppo 21491ae08745Sheppo return (0); 21501ae08745Sheppo 21511ae08745Sheppo cleanup_on_exit: 21521ae08745Sheppo 21531ae08745Sheppo if (ldcp->mode == LDC_MODE_STREAM && ldcp->stream_bufferp) 21541ae08745Sheppo kmem_free(ldcp->stream_bufferp, ldcp->mtu); 21551ae08745Sheppo 21561ae08745Sheppo if (ldcp->tstate & TS_TXQ_RDY) 21571ae08745Sheppo contig_mem_free((caddr_t)ldcp->tx_q_va, 21581ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 21591ae08745Sheppo 21601ae08745Sheppo if (ldcp->tstate & TS_RXQ_RDY) 21611ae08745Sheppo contig_mem_free((caddr_t)ldcp->rx_q_va, 21621ae08745Sheppo (ldcp->rx_q_entries << LDC_PACKET_SHIFT)); 21631ae08745Sheppo 2164d10e4ef2Snarayan mutex_destroy(&ldcp->tx_lock); 21651ae08745Sheppo mutex_destroy(&ldcp->lock); 21661ae08745Sheppo 21671ae08745Sheppo if (ldcp) 21681ae08745Sheppo kmem_free(ldcp, sizeof (ldc_chan_t)); 21691ae08745Sheppo 21701ae08745Sheppo return (exit_val); 21711ae08745Sheppo } 21721ae08745Sheppo 21731ae08745Sheppo /* 21741ae08745Sheppo * Finalizes the LDC connection. It will return EBUSY if the 21751ae08745Sheppo * channel is open. A ldc_close() has to be done prior to 21761ae08745Sheppo * a ldc_fini operation. It frees TX/RX queues, associated 21771ae08745Sheppo * with the channel 21781ae08745Sheppo */ 21791ae08745Sheppo int 21801ae08745Sheppo ldc_fini(ldc_handle_t handle) 21811ae08745Sheppo { 21821ae08745Sheppo ldc_chan_t *ldcp; 21831ae08745Sheppo ldc_chan_t *tmp_ldcp; 21841ae08745Sheppo uint64_t id; 21851ae08745Sheppo 21861ae08745Sheppo if (handle == NULL) { 21871ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_fini: invalid channel handle\n"); 21881ae08745Sheppo return (EINVAL); 21891ae08745Sheppo } 21901ae08745Sheppo ldcp = (ldc_chan_t *)handle; 21911ae08745Sheppo id = ldcp->id; 21921ae08745Sheppo 21931ae08745Sheppo mutex_enter(&ldcp->lock); 21941ae08745Sheppo 21951ae08745Sheppo if (ldcp->tstate > TS_INIT) { 21961ae08745Sheppo DWARN(ldcp->id, "ldc_fini: (0x%llx) channel is open\n", 21971ae08745Sheppo ldcp->id); 21981ae08745Sheppo mutex_exit(&ldcp->lock); 21991ae08745Sheppo return (EBUSY); 22001ae08745Sheppo } 22011ae08745Sheppo 22021ae08745Sheppo /* Remove from the channel list */ 22031ae08745Sheppo mutex_enter(&ldcssp->lock); 22041ae08745Sheppo tmp_ldcp = ldcssp->chan_list; 22051ae08745Sheppo if (tmp_ldcp == ldcp) { 22061ae08745Sheppo ldcssp->chan_list = ldcp->next; 22071ae08745Sheppo ldcp->next = NULL; 22081ae08745Sheppo } else { 22091ae08745Sheppo while (tmp_ldcp != NULL) { 22101ae08745Sheppo if (tmp_ldcp->next == ldcp) { 22111ae08745Sheppo tmp_ldcp->next = ldcp->next; 22121ae08745Sheppo ldcp->next = NULL; 22131ae08745Sheppo break; 22141ae08745Sheppo } 22151ae08745Sheppo tmp_ldcp = tmp_ldcp->next; 22161ae08745Sheppo } 22171ae08745Sheppo if (tmp_ldcp == NULL) { 22181ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_fini: invalid channel hdl\n"); 22191ae08745Sheppo mutex_exit(&ldcssp->lock); 22201ae08745Sheppo mutex_exit(&ldcp->lock); 22211ae08745Sheppo return (EINVAL); 22221ae08745Sheppo } 22231ae08745Sheppo } 22241ae08745Sheppo 22251ae08745Sheppo ldcssp->channel_count--; 22261ae08745Sheppo 22271ae08745Sheppo mutex_exit(&ldcssp->lock); 22281ae08745Sheppo 22291ae08745Sheppo /* Free the map table for this channel */ 22301ae08745Sheppo if (ldcp->mtbl) { 22311ae08745Sheppo (void) hv_ldc_set_map_table(ldcp->id, NULL, NULL); 22321ae08745Sheppo contig_mem_free(ldcp->mtbl->table, ldcp->mtbl->size); 22331ae08745Sheppo mutex_destroy(&ldcp->mtbl->lock); 22341ae08745Sheppo kmem_free(ldcp->mtbl, sizeof (ldc_mtbl_t)); 22351ae08745Sheppo } 22361ae08745Sheppo 22371ae08745Sheppo /* Destroy descriptor ring and memory handle list lock */ 22381ae08745Sheppo mutex_destroy(&ldcp->exp_dlist_lock); 22391ae08745Sheppo mutex_destroy(&ldcp->imp_dlist_lock); 22401ae08745Sheppo mutex_destroy(&ldcp->mlist_lock); 22411ae08745Sheppo 22421ae08745Sheppo /* Free the stream buffer for STREAM_MODE */ 22431ae08745Sheppo if (ldcp->mode == LDC_MODE_STREAM && ldcp->stream_bufferp) 22441ae08745Sheppo kmem_free(ldcp->stream_bufferp, ldcp->mtu); 22451ae08745Sheppo 22461ae08745Sheppo /* Free the RX queue */ 22471ae08745Sheppo contig_mem_free((caddr_t)ldcp->rx_q_va, 22481ae08745Sheppo (ldcp->rx_q_entries << LDC_PACKET_SHIFT)); 22491ae08745Sheppo ldcp->tstate &= ~TS_RXQ_RDY; 22501ae08745Sheppo 22511ae08745Sheppo /* Free the TX queue */ 22521ae08745Sheppo contig_mem_free((caddr_t)ldcp->tx_q_va, 22531ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 22541ae08745Sheppo ldcp->tstate &= ~TS_TXQ_RDY; 22551ae08745Sheppo 22561ae08745Sheppo 22571ae08745Sheppo mutex_exit(&ldcp->lock); 22581ae08745Sheppo 22591ae08745Sheppo /* Destroy mutex */ 2260d10e4ef2Snarayan mutex_destroy(&ldcp->tx_lock); 22611ae08745Sheppo mutex_destroy(&ldcp->lock); 22621ae08745Sheppo 22631ae08745Sheppo /* free channel structure */ 22641ae08745Sheppo kmem_free(ldcp, sizeof (ldc_chan_t)); 22651ae08745Sheppo 22661ae08745Sheppo D1(id, "ldc_fini: (0x%llx) channel finalized\n", id); 22671ae08745Sheppo 22681ae08745Sheppo return (0); 22691ae08745Sheppo } 22701ae08745Sheppo 22711ae08745Sheppo /* 22721ae08745Sheppo * Open the LDC channel for use. It registers the TX/RX queues 22731ae08745Sheppo * with the Hypervisor. It also specifies the interrupt number 22741ae08745Sheppo * and target CPU for this channel 22751ae08745Sheppo */ 22761ae08745Sheppo int 22771ae08745Sheppo ldc_open(ldc_handle_t handle) 22781ae08745Sheppo { 22791ae08745Sheppo ldc_chan_t *ldcp; 22801ae08745Sheppo int rv; 22811ae08745Sheppo 22821ae08745Sheppo if (handle == NULL) { 22831ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_open: invalid channel handle\n"); 22841ae08745Sheppo return (EINVAL); 22851ae08745Sheppo } 22861ae08745Sheppo 22871ae08745Sheppo ldcp = (ldc_chan_t *)handle; 22881ae08745Sheppo 22891ae08745Sheppo mutex_enter(&ldcp->lock); 22901ae08745Sheppo 22911ae08745Sheppo if (ldcp->tstate < TS_INIT) { 22921ae08745Sheppo DWARN(ldcp->id, 22931ae08745Sheppo "ldc_open: (0x%llx) channel not initialized\n", ldcp->id); 22941ae08745Sheppo mutex_exit(&ldcp->lock); 22951ae08745Sheppo return (EFAULT); 22961ae08745Sheppo } 22971ae08745Sheppo if (ldcp->tstate >= TS_OPEN) { 22981ae08745Sheppo DWARN(ldcp->id, 22991ae08745Sheppo "ldc_open: (0x%llx) channel is already open\n", ldcp->id); 23001ae08745Sheppo mutex_exit(&ldcp->lock); 23011ae08745Sheppo return (EFAULT); 23021ae08745Sheppo } 23031ae08745Sheppo 23041ae08745Sheppo /* 23051ae08745Sheppo * Unregister/Register the tx queue with the hypervisor 23061ae08745Sheppo */ 23071ae08745Sheppo rv = hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 23081ae08745Sheppo if (rv) { 23091ae08745Sheppo cmn_err(CE_WARN, 23101ae08745Sheppo "ldc_open: (0x%lx) channel tx queue unconf failed\n", 23111ae08745Sheppo ldcp->id); 23121ae08745Sheppo mutex_exit(&ldcp->lock); 23131ae08745Sheppo return (EIO); 23141ae08745Sheppo } 23151ae08745Sheppo 23161ae08745Sheppo rv = hv_ldc_tx_qconf(ldcp->id, ldcp->tx_q_ra, ldcp->tx_q_entries); 23171ae08745Sheppo if (rv) { 23181ae08745Sheppo cmn_err(CE_WARN, 23191ae08745Sheppo "ldc_open: (0x%lx) channel tx queue conf failed\n", 23201ae08745Sheppo ldcp->id); 23211ae08745Sheppo mutex_exit(&ldcp->lock); 23221ae08745Sheppo return (EIO); 23231ae08745Sheppo } 23241ae08745Sheppo 23251ae08745Sheppo D2(ldcp->id, "ldc_open: (0x%llx) registered tx queue with LDC\n", 23261ae08745Sheppo ldcp->id); 23271ae08745Sheppo 23281ae08745Sheppo /* 23291ae08745Sheppo * Unregister/Register the rx queue with the hypervisor 23301ae08745Sheppo */ 23311ae08745Sheppo rv = hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 23321ae08745Sheppo if (rv) { 23331ae08745Sheppo cmn_err(CE_WARN, 23341ae08745Sheppo "ldc_open: (0x%lx) channel rx queue unconf failed\n", 23351ae08745Sheppo ldcp->id); 23361ae08745Sheppo mutex_exit(&ldcp->lock); 23371ae08745Sheppo return (EIO); 23381ae08745Sheppo } 23391ae08745Sheppo 23401ae08745Sheppo rv = hv_ldc_rx_qconf(ldcp->id, ldcp->rx_q_ra, ldcp->rx_q_entries); 23411ae08745Sheppo if (rv) { 23421ae08745Sheppo cmn_err(CE_WARN, 23431ae08745Sheppo "ldc_open: (0x%lx) channel rx queue conf failed\n", 23441ae08745Sheppo ldcp->id); 23451ae08745Sheppo mutex_exit(&ldcp->lock); 23461ae08745Sheppo return (EIO); 23471ae08745Sheppo } 23481ae08745Sheppo 23491ae08745Sheppo D2(ldcp->id, "ldc_open: (0x%llx) registered rx queue with LDC\n", 23501ae08745Sheppo ldcp->id); 23511ae08745Sheppo 23521ae08745Sheppo ldcp->tstate |= TS_QCONF_RDY; 23531ae08745Sheppo 23541ae08745Sheppo /* Register the channel with the channel nexus */ 23551ae08745Sheppo rv = i_ldc_register_channel(ldcp); 23561ae08745Sheppo if (rv && rv != EAGAIN) { 23571ae08745Sheppo cmn_err(CE_WARN, 23581ae08745Sheppo "ldc_open: (0x%lx) channel register failed\n", ldcp->id); 23591ae08745Sheppo (void) hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 23601ae08745Sheppo (void) hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 23611ae08745Sheppo mutex_exit(&ldcp->lock); 23621ae08745Sheppo return (EIO); 23631ae08745Sheppo } 23641ae08745Sheppo 23651ae08745Sheppo /* mark channel in OPEN state */ 23661ae08745Sheppo ldcp->status = LDC_OPEN; 23671ae08745Sheppo 23681ae08745Sheppo /* Read channel state */ 23691ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 23701ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 23711ae08745Sheppo if (rv) { 23721ae08745Sheppo cmn_err(CE_WARN, 23731ae08745Sheppo "ldc_open: (0x%lx) cannot read channel state\n", 23741ae08745Sheppo ldcp->id); 23751ae08745Sheppo (void) i_ldc_unregister_channel(ldcp); 23761ae08745Sheppo (void) hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 23771ae08745Sheppo (void) hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 23781ae08745Sheppo mutex_exit(&ldcp->lock); 23791ae08745Sheppo return (EIO); 23801ae08745Sheppo } 23811ae08745Sheppo 23821ae08745Sheppo /* 23831ae08745Sheppo * set the ACKd head to current head location for reliable & 23841ae08745Sheppo * streaming mode 23851ae08745Sheppo */ 23861ae08745Sheppo ldcp->tx_ackd_head = ldcp->tx_head; 23871ae08745Sheppo 23881ae08745Sheppo /* mark channel ready if HV report link is UP (peer alloc'd Rx queue) */ 23891ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_UP || 23901ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 23911ae08745Sheppo ldcp->tstate |= TS_LINK_READY; 23921ae08745Sheppo ldcp->status = LDC_READY; 23931ae08745Sheppo } 23941ae08745Sheppo 23951ae08745Sheppo /* 23961ae08745Sheppo * if channel is being opened in RAW mode - no handshake is needed 23971ae08745Sheppo * switch the channel READY and UP state 23981ae08745Sheppo */ 23991ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW) { 24001ae08745Sheppo ldcp->tstate = TS_UP; /* set bits associated with LDC UP */ 24011ae08745Sheppo ldcp->status = LDC_UP; 24021ae08745Sheppo } 24031ae08745Sheppo 24041ae08745Sheppo mutex_exit(&ldcp->lock); 24051ae08745Sheppo 24061ae08745Sheppo /* 24071ae08745Sheppo * Increment number of open channels 24081ae08745Sheppo */ 24091ae08745Sheppo mutex_enter(&ldcssp->lock); 24101ae08745Sheppo ldcssp->channels_open++; 24111ae08745Sheppo mutex_exit(&ldcssp->lock); 24121ae08745Sheppo 24131ae08745Sheppo D1(ldcp->id, 24141ae08745Sheppo "ldc_open: (0x%llx) channel (0x%p) open for use (tstate=0x%x)\n", 24151ae08745Sheppo ldcp->id, ldcp, ldcp->tstate); 24161ae08745Sheppo 24171ae08745Sheppo return (0); 24181ae08745Sheppo } 24191ae08745Sheppo 24201ae08745Sheppo /* 24211ae08745Sheppo * Close the LDC connection. It will return EBUSY if there 24221ae08745Sheppo * are memory segments or descriptor rings either bound to or 24231ae08745Sheppo * mapped over the channel 24241ae08745Sheppo */ 24251ae08745Sheppo int 24261ae08745Sheppo ldc_close(ldc_handle_t handle) 24271ae08745Sheppo { 24281ae08745Sheppo ldc_chan_t *ldcp; 2429d10e4ef2Snarayan int rv = 0, retries = 0; 24301ae08745Sheppo boolean_t chk_done = B_FALSE; 24311ae08745Sheppo 24321ae08745Sheppo if (handle == NULL) { 24331ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_close: invalid channel handle\n"); 24341ae08745Sheppo return (EINVAL); 24351ae08745Sheppo } 24361ae08745Sheppo ldcp = (ldc_chan_t *)handle; 24371ae08745Sheppo 24381ae08745Sheppo mutex_enter(&ldcp->lock); 24391ae08745Sheppo 24401ae08745Sheppo /* return error if channel is not open */ 24411ae08745Sheppo if (ldcp->tstate < TS_OPEN) { 24421ae08745Sheppo DWARN(ldcp->id, 24431ae08745Sheppo "ldc_close: (0x%llx) channel is not open\n", ldcp->id); 24441ae08745Sheppo mutex_exit(&ldcp->lock); 24451ae08745Sheppo return (EFAULT); 24461ae08745Sheppo } 24471ae08745Sheppo 24481ae08745Sheppo /* if any memory handles, drings, are bound or mapped cannot close */ 24491ae08745Sheppo if (ldcp->mhdl_list != NULL) { 24501ae08745Sheppo DWARN(ldcp->id, 24511ae08745Sheppo "ldc_close: (0x%llx) channel has bound memory handles\n", 24521ae08745Sheppo ldcp->id); 24531ae08745Sheppo mutex_exit(&ldcp->lock); 24541ae08745Sheppo return (EBUSY); 24551ae08745Sheppo } 24561ae08745Sheppo if (ldcp->exp_dring_list != NULL) { 24571ae08745Sheppo DWARN(ldcp->id, 24581ae08745Sheppo "ldc_close: (0x%llx) channel has bound descriptor rings\n", 24591ae08745Sheppo ldcp->id); 24601ae08745Sheppo mutex_exit(&ldcp->lock); 24611ae08745Sheppo return (EBUSY); 24621ae08745Sheppo } 24631ae08745Sheppo if (ldcp->imp_dring_list != NULL) { 24641ae08745Sheppo DWARN(ldcp->id, 24651ae08745Sheppo "ldc_close: (0x%llx) channel has mapped descriptor rings\n", 24661ae08745Sheppo ldcp->id); 24671ae08745Sheppo mutex_exit(&ldcp->lock); 24681ae08745Sheppo return (EBUSY); 24691ae08745Sheppo } 24701ae08745Sheppo 2471d10e4ef2Snarayan /* Obtain Tx lock */ 2472d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 2473d10e4ef2Snarayan 24741ae08745Sheppo /* 24751ae08745Sheppo * Wait for pending transmits to complete i.e Tx queue to drain 24761ae08745Sheppo * if there are pending pkts - wait 1 ms and retry again 24771ae08745Sheppo */ 24781ae08745Sheppo for (;;) { 24791ae08745Sheppo 24801ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 24811ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 24821ae08745Sheppo if (rv) { 24831ae08745Sheppo cmn_err(CE_WARN, 24841ae08745Sheppo "ldc_close: (0x%lx) cannot read qptrs\n", ldcp->id); 2485d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 24861ae08745Sheppo mutex_exit(&ldcp->lock); 24871ae08745Sheppo return (EIO); 24881ae08745Sheppo } 24891ae08745Sheppo 24901ae08745Sheppo if (ldcp->tx_head == ldcp->tx_tail || 24911ae08745Sheppo ldcp->link_state != LDC_CHANNEL_UP) { 24921ae08745Sheppo break; 24931ae08745Sheppo } 24941ae08745Sheppo 24951ae08745Sheppo if (chk_done) { 24961ae08745Sheppo DWARN(ldcp->id, 24971ae08745Sheppo "ldc_close: (0x%llx) Tx queue drain timeout\n", 24981ae08745Sheppo ldcp->id); 24991ae08745Sheppo break; 25001ae08745Sheppo } 25011ae08745Sheppo 25021ae08745Sheppo /* wait for one ms and try again */ 25031ae08745Sheppo delay(drv_usectohz(1000)); 25041ae08745Sheppo chk_done = B_TRUE; 25051ae08745Sheppo } 25061ae08745Sheppo 25071ae08745Sheppo /* 25081ae08745Sheppo * Unregister the channel with the nexus 25091ae08745Sheppo */ 2510d10e4ef2Snarayan while ((rv = i_ldc_unregister_channel(ldcp)) != 0) { 2511d10e4ef2Snarayan 2512d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 25131ae08745Sheppo mutex_exit(&ldcp->lock); 2514d10e4ef2Snarayan 2515d10e4ef2Snarayan /* if any error other than EAGAIN return back */ 2516d10e4ef2Snarayan if (rv != EAGAIN || retries >= LDC_MAX_RETRIES) { 2517d10e4ef2Snarayan cmn_err(CE_WARN, 2518d10e4ef2Snarayan "ldc_close: (0x%lx) unregister failed, %d\n", 2519d10e4ef2Snarayan ldcp->id, rv); 25201ae08745Sheppo return (rv); 25211ae08745Sheppo } 25221ae08745Sheppo 25231ae08745Sheppo /* 2524d10e4ef2Snarayan * As there could be pending interrupts we need 2525d10e4ef2Snarayan * to wait and try again 2526d10e4ef2Snarayan */ 2527d10e4ef2Snarayan drv_usecwait(LDC_DELAY); 2528d10e4ef2Snarayan mutex_enter(&ldcp->lock); 2529d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 2530d10e4ef2Snarayan retries++; 2531d10e4ef2Snarayan } 2532d10e4ef2Snarayan 2533d10e4ef2Snarayan /* 25341ae08745Sheppo * Unregister queues 25351ae08745Sheppo */ 25361ae08745Sheppo rv = hv_ldc_tx_qconf(ldcp->id, NULL, NULL); 25371ae08745Sheppo if (rv) { 25381ae08745Sheppo cmn_err(CE_WARN, 25391ae08745Sheppo "ldc_close: (0x%lx) channel TX queue unconf failed\n", 25401ae08745Sheppo ldcp->id); 2541d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 25421ae08745Sheppo mutex_exit(&ldcp->lock); 25431ae08745Sheppo return (EIO); 25441ae08745Sheppo } 25451ae08745Sheppo rv = hv_ldc_rx_qconf(ldcp->id, NULL, NULL); 25461ae08745Sheppo if (rv) { 25471ae08745Sheppo cmn_err(CE_WARN, 25481ae08745Sheppo "ldc_close: (0x%lx) channel RX queue unconf failed\n", 25491ae08745Sheppo ldcp->id); 2550d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 25511ae08745Sheppo mutex_exit(&ldcp->lock); 25521ae08745Sheppo return (EIO); 25531ae08745Sheppo } 25541ae08745Sheppo 25551ae08745Sheppo ldcp->tstate &= ~TS_QCONF_RDY; 25561ae08745Sheppo 25571ae08745Sheppo /* Reset channel state information */ 25581ae08745Sheppo i_ldc_reset_state(ldcp); 25591ae08745Sheppo 25601ae08745Sheppo /* Mark channel as down and in initialized state */ 25611ae08745Sheppo ldcp->tx_ackd_head = 0; 25621ae08745Sheppo ldcp->tx_head = 0; 25631ae08745Sheppo ldcp->tstate = TS_INIT; 25641ae08745Sheppo ldcp->status = LDC_INIT; 25651ae08745Sheppo 2566d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 25671ae08745Sheppo mutex_exit(&ldcp->lock); 25681ae08745Sheppo 25691ae08745Sheppo /* Decrement number of open channels */ 25701ae08745Sheppo mutex_enter(&ldcssp->lock); 25711ae08745Sheppo ldcssp->channels_open--; 25721ae08745Sheppo mutex_exit(&ldcssp->lock); 25731ae08745Sheppo 25741ae08745Sheppo D1(ldcp->id, "ldc_close: (0x%llx) channel closed\n", ldcp->id); 25751ae08745Sheppo 25761ae08745Sheppo return (0); 25771ae08745Sheppo } 25781ae08745Sheppo 25791ae08745Sheppo /* 25801ae08745Sheppo * Register channel callback 25811ae08745Sheppo */ 25821ae08745Sheppo int 25831ae08745Sheppo ldc_reg_callback(ldc_handle_t handle, 25841ae08745Sheppo uint_t(*cb)(uint64_t event, caddr_t arg), caddr_t arg) 25851ae08745Sheppo { 25861ae08745Sheppo ldc_chan_t *ldcp; 25871ae08745Sheppo 25881ae08745Sheppo if (handle == NULL) { 25891ae08745Sheppo DWARN(DBG_ALL_LDCS, 25901ae08745Sheppo "ldc_reg_callback: invalid channel handle\n"); 25911ae08745Sheppo return (EINVAL); 25921ae08745Sheppo } 25931ae08745Sheppo if (((uint64_t)cb) < KERNELBASE) { 25941ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_reg_callback: invalid callback\n"); 25951ae08745Sheppo return (EINVAL); 25961ae08745Sheppo } 25971ae08745Sheppo ldcp = (ldc_chan_t *)handle; 25981ae08745Sheppo 25991ae08745Sheppo mutex_enter(&ldcp->lock); 26001ae08745Sheppo 26011ae08745Sheppo if (ldcp->cb) { 26021ae08745Sheppo DWARN(ldcp->id, "ldc_reg_callback: (0x%llx) callback exists\n", 26031ae08745Sheppo ldcp->id); 26041ae08745Sheppo mutex_exit(&ldcp->lock); 26051ae08745Sheppo return (EIO); 26061ae08745Sheppo } 26071ae08745Sheppo if (ldcp->cb_inprogress) { 26081ae08745Sheppo DWARN(ldcp->id, "ldc_reg_callback: (0x%llx) callback active\n", 26091ae08745Sheppo ldcp->id); 26101ae08745Sheppo mutex_exit(&ldcp->lock); 26111ae08745Sheppo return (EWOULDBLOCK); 26121ae08745Sheppo } 26131ae08745Sheppo 26141ae08745Sheppo ldcp->cb = cb; 26151ae08745Sheppo ldcp->cb_arg = arg; 26161ae08745Sheppo ldcp->cb_enabled = B_TRUE; 26171ae08745Sheppo 26181ae08745Sheppo D1(ldcp->id, 26191ae08745Sheppo "ldc_reg_callback: (0x%llx) registered callback for channel\n", 26201ae08745Sheppo ldcp->id); 26211ae08745Sheppo 26221ae08745Sheppo mutex_exit(&ldcp->lock); 26231ae08745Sheppo 26241ae08745Sheppo return (0); 26251ae08745Sheppo } 26261ae08745Sheppo 26271ae08745Sheppo /* 26281ae08745Sheppo * Unregister channel callback 26291ae08745Sheppo */ 26301ae08745Sheppo int 26311ae08745Sheppo ldc_unreg_callback(ldc_handle_t handle) 26321ae08745Sheppo { 26331ae08745Sheppo ldc_chan_t *ldcp; 26341ae08745Sheppo 26351ae08745Sheppo if (handle == NULL) { 26361ae08745Sheppo DWARN(DBG_ALL_LDCS, 26371ae08745Sheppo "ldc_unreg_callback: invalid channel handle\n"); 26381ae08745Sheppo return (EINVAL); 26391ae08745Sheppo } 26401ae08745Sheppo ldcp = (ldc_chan_t *)handle; 26411ae08745Sheppo 26421ae08745Sheppo mutex_enter(&ldcp->lock); 26431ae08745Sheppo 26441ae08745Sheppo if (ldcp->cb == NULL) { 26451ae08745Sheppo DWARN(ldcp->id, 26461ae08745Sheppo "ldc_unreg_callback: (0x%llx) no callback exists\n", 26471ae08745Sheppo ldcp->id); 26481ae08745Sheppo mutex_exit(&ldcp->lock); 26491ae08745Sheppo return (EIO); 26501ae08745Sheppo } 26511ae08745Sheppo if (ldcp->cb_inprogress) { 26521ae08745Sheppo DWARN(ldcp->id, 26531ae08745Sheppo "ldc_unreg_callback: (0x%llx) callback active\n", 26541ae08745Sheppo ldcp->id); 26551ae08745Sheppo mutex_exit(&ldcp->lock); 26561ae08745Sheppo return (EWOULDBLOCK); 26571ae08745Sheppo } 26581ae08745Sheppo 26591ae08745Sheppo ldcp->cb = NULL; 26601ae08745Sheppo ldcp->cb_arg = NULL; 26611ae08745Sheppo ldcp->cb_enabled = B_FALSE; 26621ae08745Sheppo 26631ae08745Sheppo D1(ldcp->id, 26641ae08745Sheppo "ldc_unreg_callback: (0x%llx) unregistered callback for channel\n", 26651ae08745Sheppo ldcp->id); 26661ae08745Sheppo 26671ae08745Sheppo mutex_exit(&ldcp->lock); 26681ae08745Sheppo 26691ae08745Sheppo return (0); 26701ae08745Sheppo } 26711ae08745Sheppo 26721ae08745Sheppo 26731ae08745Sheppo /* 26741ae08745Sheppo * Bring a channel up by initiating a handshake with the peer 26751ae08745Sheppo * This call is asynchronous. It will complete at a later point 26761ae08745Sheppo * in time when the peer responds back with an RTR. 26771ae08745Sheppo */ 26781ae08745Sheppo int 26791ae08745Sheppo ldc_up(ldc_handle_t handle) 26801ae08745Sheppo { 26811ae08745Sheppo int rv; 26821ae08745Sheppo ldc_chan_t *ldcp; 26831ae08745Sheppo ldc_msg_t *ldcmsg; 26841ae08745Sheppo uint64_t tx_tail; 26851ae08745Sheppo 26861ae08745Sheppo if (handle == NULL) { 26871ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_up: invalid channel handle\n"); 26881ae08745Sheppo return (EINVAL); 26891ae08745Sheppo } 26901ae08745Sheppo ldcp = (ldc_chan_t *)handle; 26911ae08745Sheppo 26921ae08745Sheppo mutex_enter(&ldcp->lock); 26931ae08745Sheppo 26941ae08745Sheppo if (ldcp->tstate == TS_UP) { 26951ae08745Sheppo D2(ldcp->id, 26961ae08745Sheppo "ldc_up: (0x%llx) channel is already in UP state\n", 26971ae08745Sheppo ldcp->id); 26981ae08745Sheppo mutex_exit(&ldcp->lock); 26991ae08745Sheppo return (0); 27001ae08745Sheppo } 27011ae08745Sheppo 27021ae08745Sheppo /* if the channel is in RAW mode - mark it as UP, if READY */ 27031ae08745Sheppo if (ldcp->mode == LDC_MODE_RAW && ldcp->tstate >= TS_READY) { 27041ae08745Sheppo ldcp->tstate = TS_UP; 27051ae08745Sheppo mutex_exit(&ldcp->lock); 27061ae08745Sheppo return (0); 27071ae08745Sheppo } 27081ae08745Sheppo 27091ae08745Sheppo /* Don't start another handshake if there is one in progress */ 27101ae08745Sheppo if (ldcp->hstate) { 27111ae08745Sheppo D2(ldcp->id, 27121ae08745Sheppo "ldc_up: (0x%llx) channel handshake in progress\n", 27131ae08745Sheppo ldcp->id); 27141ae08745Sheppo mutex_exit(&ldcp->lock); 27151ae08745Sheppo return (0); 27161ae08745Sheppo } 27171ae08745Sheppo 2718d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 2719d10e4ef2Snarayan 27201ae08745Sheppo /* get the current tail for the LDC msg */ 27211ae08745Sheppo rv = i_ldc_get_tx_tail(ldcp, &tx_tail); 27221ae08745Sheppo if (rv) { 27231ae08745Sheppo DWARN(ldcp->id, "ldc_up: (0x%llx) cannot initiate handshake\n", 27241ae08745Sheppo ldcp->id); 2725d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 27261ae08745Sheppo mutex_exit(&ldcp->lock); 27271ae08745Sheppo return (ECONNREFUSED); 27281ae08745Sheppo } 27291ae08745Sheppo 27301ae08745Sheppo ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 27311ae08745Sheppo ZERO_PKT(ldcmsg); 27321ae08745Sheppo 27331ae08745Sheppo ldcmsg->type = LDC_CTRL; 27341ae08745Sheppo ldcmsg->stype = LDC_INFO; 27351ae08745Sheppo ldcmsg->ctrl = LDC_VER; 27361ae08745Sheppo ldcp->next_vidx = 0; 27371ae08745Sheppo bcopy(&ldc_versions[0], ldcmsg->udata, sizeof (ldc_versions[0])); 27381ae08745Sheppo 27391ae08745Sheppo DUMP_LDC_PKT(ldcp, "ldc_up snd ver", (uint64_t)ldcmsg); 27401ae08745Sheppo 27411ae08745Sheppo /* initiate the send by calling into HV and set the new tail */ 27421ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) % 27431ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 27441ae08745Sheppo 27451ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 27461ae08745Sheppo if (rv) { 27471ae08745Sheppo DWARN(ldcp->id, 27481ae08745Sheppo "ldc_up: (0x%llx) cannot initiate handshake rv=%d\n", 27491ae08745Sheppo ldcp->id, rv); 2750d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 27511ae08745Sheppo mutex_exit(&ldcp->lock); 27521ae08745Sheppo return (rv); 27531ae08745Sheppo } 27541ae08745Sheppo 27550a55fbb7Slm66018 ldcp->hstate |= TS_SENT_VER; 27561ae08745Sheppo ldcp->tx_tail = tx_tail; 27571ae08745Sheppo D1(ldcp->id, "ldc_up: (0x%llx) channel up initiated\n", ldcp->id); 27581ae08745Sheppo 2759d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 27601ae08745Sheppo mutex_exit(&ldcp->lock); 27611ae08745Sheppo 27621ae08745Sheppo return (rv); 27631ae08745Sheppo } 27641ae08745Sheppo 27651ae08745Sheppo 27661ae08745Sheppo /* 2767*e1ebb9ecSlm66018 * Bring a channel down by resetting its state and queues 27681ae08745Sheppo */ 27691ae08745Sheppo int 2770*e1ebb9ecSlm66018 ldc_down(ldc_handle_t handle) 27711ae08745Sheppo { 27721ae08745Sheppo ldc_chan_t *ldcp; 27731ae08745Sheppo 27741ae08745Sheppo if (handle == NULL) { 2775*e1ebb9ecSlm66018 DWARN(DBG_ALL_LDCS, "ldc_down: invalid channel handle\n"); 27761ae08745Sheppo return (EINVAL); 27771ae08745Sheppo } 27781ae08745Sheppo ldcp = (ldc_chan_t *)handle; 27791ae08745Sheppo 27801ae08745Sheppo mutex_enter(&ldcp->lock); 2781d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 27821ae08745Sheppo i_ldc_reset(ldcp); 2783d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 27841ae08745Sheppo mutex_exit(&ldcp->lock); 27851ae08745Sheppo 27861ae08745Sheppo return (0); 27871ae08745Sheppo } 27881ae08745Sheppo 27891ae08745Sheppo /* 27901ae08745Sheppo * Get the current channel status 27911ae08745Sheppo */ 27921ae08745Sheppo int 27931ae08745Sheppo ldc_status(ldc_handle_t handle, ldc_status_t *status) 27941ae08745Sheppo { 27951ae08745Sheppo ldc_chan_t *ldcp; 27961ae08745Sheppo 27971ae08745Sheppo if (handle == NULL || status == NULL) { 27981ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_status: invalid argument\n"); 27991ae08745Sheppo return (EINVAL); 28001ae08745Sheppo } 28011ae08745Sheppo ldcp = (ldc_chan_t *)handle; 28021ae08745Sheppo 28031ae08745Sheppo *status = ((ldc_chan_t *)handle)->status; 28041ae08745Sheppo 28051ae08745Sheppo D1(ldcp->id, 28061ae08745Sheppo "ldc_status: (0x%llx) returned status %d\n", ldcp->id, *status); 28071ae08745Sheppo return (0); 28081ae08745Sheppo } 28091ae08745Sheppo 28101ae08745Sheppo 28111ae08745Sheppo /* 28121ae08745Sheppo * Set the channel's callback mode - enable/disable callbacks 28131ae08745Sheppo */ 28141ae08745Sheppo int 28151ae08745Sheppo ldc_set_cb_mode(ldc_handle_t handle, ldc_cb_mode_t cmode) 28161ae08745Sheppo { 28171ae08745Sheppo ldc_chan_t *ldcp; 28181ae08745Sheppo 28191ae08745Sheppo if (handle == NULL) { 28201ae08745Sheppo DWARN(DBG_ALL_LDCS, 28211ae08745Sheppo "ldc_set_intr_mode: invalid channel handle\n"); 28221ae08745Sheppo return (EINVAL); 28231ae08745Sheppo } 28241ae08745Sheppo ldcp = (ldc_chan_t *)handle; 28251ae08745Sheppo 28261ae08745Sheppo /* 28271ae08745Sheppo * Record no callbacks should be invoked 28281ae08745Sheppo */ 28291ae08745Sheppo mutex_enter(&ldcp->lock); 28301ae08745Sheppo 28311ae08745Sheppo switch (cmode) { 28321ae08745Sheppo case LDC_CB_DISABLE: 28331ae08745Sheppo if (!ldcp->cb_enabled) { 28341ae08745Sheppo DWARN(ldcp->id, 28351ae08745Sheppo "ldc_set_cb_mode: (0x%llx) callbacks disabled\n", 28361ae08745Sheppo ldcp->id); 28371ae08745Sheppo break; 28381ae08745Sheppo } 28391ae08745Sheppo ldcp->cb_enabled = B_FALSE; 28401ae08745Sheppo 28411ae08745Sheppo D1(ldcp->id, "ldc_set_cb_mode: (0x%llx) disabled callbacks\n", 28421ae08745Sheppo ldcp->id); 28431ae08745Sheppo break; 28441ae08745Sheppo 28451ae08745Sheppo case LDC_CB_ENABLE: 28461ae08745Sheppo if (ldcp->cb_enabled) { 28471ae08745Sheppo DWARN(ldcp->id, 28481ae08745Sheppo "ldc_set_cb_mode: (0x%llx) callbacks enabled\n", 28491ae08745Sheppo ldcp->id); 28501ae08745Sheppo break; 28511ae08745Sheppo } 28521ae08745Sheppo ldcp->cb_enabled = B_TRUE; 28531ae08745Sheppo 28541ae08745Sheppo D1(ldcp->id, "ldc_set_cb_mode: (0x%llx) enabled callbacks\n", 28551ae08745Sheppo ldcp->id); 28561ae08745Sheppo break; 28571ae08745Sheppo } 28581ae08745Sheppo 28591ae08745Sheppo mutex_exit(&ldcp->lock); 28601ae08745Sheppo 28611ae08745Sheppo return (0); 28621ae08745Sheppo } 28631ae08745Sheppo 28641ae08745Sheppo /* 28651ae08745Sheppo * Check to see if there are packets on the incoming queue 2866*e1ebb9ecSlm66018 * Will return hasdata = B_FALSE if there are no packets 28671ae08745Sheppo */ 28681ae08745Sheppo int 2869*e1ebb9ecSlm66018 ldc_chkq(ldc_handle_t handle, boolean_t *hasdata) 28701ae08745Sheppo { 28711ae08745Sheppo int rv; 28721ae08745Sheppo uint64_t rx_head, rx_tail; 28731ae08745Sheppo ldc_chan_t *ldcp; 28741ae08745Sheppo 28751ae08745Sheppo if (handle == NULL) { 28761ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_chkq: invalid channel handle\n"); 28771ae08745Sheppo return (EINVAL); 28781ae08745Sheppo } 28791ae08745Sheppo ldcp = (ldc_chan_t *)handle; 28801ae08745Sheppo 2881*e1ebb9ecSlm66018 *hasdata = B_FALSE; 28821ae08745Sheppo 28831ae08745Sheppo mutex_enter(&ldcp->lock); 28841ae08745Sheppo 28851ae08745Sheppo if (ldcp->tstate != TS_UP) { 28861ae08745Sheppo D1(ldcp->id, 28871ae08745Sheppo "ldc_chkq: (0x%llx) channel is not up\n", ldcp->id); 28881ae08745Sheppo mutex_exit(&ldcp->lock); 28891ae08745Sheppo return (ECONNRESET); 28901ae08745Sheppo } 28911ae08745Sheppo 28921ae08745Sheppo /* Read packet(s) from the queue */ 28931ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 28941ae08745Sheppo &ldcp->link_state); 28951ae08745Sheppo if (rv != 0) { 28961ae08745Sheppo cmn_err(CE_WARN, 28971ae08745Sheppo "ldc_chkq: (0x%lx) unable to read queue ptrs", ldcp->id); 28981ae08745Sheppo mutex_exit(&ldcp->lock); 28991ae08745Sheppo return (EIO); 29001ae08745Sheppo } 29011ae08745Sheppo /* reset the channel state if the channel went down */ 29021ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN || 29031ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 2904d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 29051ae08745Sheppo i_ldc_reset(ldcp); 2906d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 29071ae08745Sheppo mutex_exit(&ldcp->lock); 29081ae08745Sheppo return (ECONNRESET); 29091ae08745Sheppo } 29101ae08745Sheppo 29111ae08745Sheppo if (rx_head != rx_tail) { 29121ae08745Sheppo D1(ldcp->id, "ldc_chkq: (0x%llx) queue has pkt(s)\n", ldcp->id); 2913*e1ebb9ecSlm66018 *hasdata = B_TRUE; 29141ae08745Sheppo } 29151ae08745Sheppo 29161ae08745Sheppo mutex_exit(&ldcp->lock); 29171ae08745Sheppo 29181ae08745Sheppo return (0); 29191ae08745Sheppo } 29201ae08745Sheppo 29211ae08745Sheppo 29221ae08745Sheppo /* 29231ae08745Sheppo * Read 'size' amount of bytes or less. If incoming buffer 29241ae08745Sheppo * is more than 'size', ENOBUFS is returned. 29251ae08745Sheppo * 29261ae08745Sheppo * On return, size contains the number of bytes read. 29271ae08745Sheppo */ 29281ae08745Sheppo int 29291ae08745Sheppo ldc_read(ldc_handle_t handle, caddr_t bufp, size_t *sizep) 29301ae08745Sheppo { 29311ae08745Sheppo ldc_chan_t *ldcp; 29321ae08745Sheppo uint64_t rx_head = 0, rx_tail = 0; 29331ae08745Sheppo int rv = 0, exit_val; 29341ae08745Sheppo 29351ae08745Sheppo if (handle == NULL) { 29361ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_read: invalid channel handle\n"); 29371ae08745Sheppo return (EINVAL); 29381ae08745Sheppo } 29391ae08745Sheppo 29401ae08745Sheppo ldcp = (ldc_chan_t *)handle; 29411ae08745Sheppo 29421ae08745Sheppo /* channel lock */ 29431ae08745Sheppo mutex_enter(&ldcp->lock); 29441ae08745Sheppo 29451ae08745Sheppo if (ldcp->tstate != TS_UP) { 29461ae08745Sheppo DWARN(ldcp->id, 29471ae08745Sheppo "ldc_read: (0x%llx) channel is not in UP state\n", 29481ae08745Sheppo ldcp->id); 29491ae08745Sheppo exit_val = ECONNRESET; 29501ae08745Sheppo } else { 29511ae08745Sheppo exit_val = ldcp->read_p(ldcp, bufp, sizep); 29521ae08745Sheppo } 29531ae08745Sheppo 29541ae08745Sheppo /* 29551ae08745Sheppo * if queue has been drained - clear interrupt 29561ae08745Sheppo */ 29571ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 29581ae08745Sheppo &ldcp->link_state); 29591ae08745Sheppo if (exit_val == 0 && rv == 0 && rx_head == rx_tail) { 29601ae08745Sheppo i_ldc_clear_intr(ldcp, CNEX_RX_INTR); 29611ae08745Sheppo } 29621ae08745Sheppo 29631ae08745Sheppo mutex_exit(&ldcp->lock); 29641ae08745Sheppo return (exit_val); 29651ae08745Sheppo } 29661ae08745Sheppo 29671ae08745Sheppo /* 29681ae08745Sheppo * Basic raw mondo read - 29691ae08745Sheppo * no interpretation of mondo contents at all. 29701ae08745Sheppo * 29711ae08745Sheppo * Enter and exit with ldcp->lock held by caller 29721ae08745Sheppo */ 29731ae08745Sheppo static int 29741ae08745Sheppo i_ldc_read_raw(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 29751ae08745Sheppo { 29761ae08745Sheppo uint64_t q_size_mask; 29771ae08745Sheppo ldc_msg_t *msgp; 29781ae08745Sheppo uint8_t *msgbufp; 29791ae08745Sheppo int rv = 0, space; 29801ae08745Sheppo uint64_t rx_head, rx_tail; 29811ae08745Sheppo 29821ae08745Sheppo space = *sizep; 29831ae08745Sheppo 29841ae08745Sheppo if (space < LDC_PAYLOAD_SIZE_RAW) 29851ae08745Sheppo return (ENOBUFS); 29861ae08745Sheppo 29871ae08745Sheppo ASSERT(mutex_owned(&ldcp->lock)); 29881ae08745Sheppo 29891ae08745Sheppo /* compute mask for increment */ 29901ae08745Sheppo q_size_mask = (ldcp->rx_q_entries-1)<<LDC_PACKET_SHIFT; 29911ae08745Sheppo 29921ae08745Sheppo /* 29931ae08745Sheppo * Read packet(s) from the queue 29941ae08745Sheppo */ 29951ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &rx_head, &rx_tail, 29961ae08745Sheppo &ldcp->link_state); 29971ae08745Sheppo if (rv != 0) { 29981ae08745Sheppo cmn_err(CE_WARN, 29991ae08745Sheppo "ldc_read_raw: (0x%lx) unable to read queue ptrs", 30001ae08745Sheppo ldcp->id); 30011ae08745Sheppo return (EIO); 30021ae08745Sheppo } 30031ae08745Sheppo D1(ldcp->id, "ldc_read_raw: (0x%llx) rxh=0x%llx," 30041ae08745Sheppo " rxt=0x%llx, st=0x%llx\n", 30051ae08745Sheppo ldcp->id, rx_head, rx_tail, ldcp->link_state); 30061ae08745Sheppo 30071ae08745Sheppo /* reset the channel state if the channel went down */ 30081ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN) { 3009d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 30101ae08745Sheppo i_ldc_reset(ldcp); 3011d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 30121ae08745Sheppo return (ECONNRESET); 30131ae08745Sheppo } 30141ae08745Sheppo 30151ae08745Sheppo /* 30161ae08745Sheppo * Check for empty queue 30171ae08745Sheppo */ 30181ae08745Sheppo if (rx_head == rx_tail) { 30191ae08745Sheppo *sizep = 0; 30201ae08745Sheppo return (0); 30211ae08745Sheppo } 30221ae08745Sheppo 30231ae08745Sheppo /* get the message */ 30241ae08745Sheppo msgp = (ldc_msg_t *)(ldcp->rx_q_va + rx_head); 30251ae08745Sheppo 30261ae08745Sheppo /* if channel is in RAW mode, copy data and return */ 30271ae08745Sheppo msgbufp = (uint8_t *)&(msgp->raw[0]); 30281ae08745Sheppo 30291ae08745Sheppo bcopy(msgbufp, target_bufp, LDC_PAYLOAD_SIZE_RAW); 30301ae08745Sheppo 30311ae08745Sheppo DUMP_PAYLOAD(ldcp->id, msgbufp); 30321ae08745Sheppo 30331ae08745Sheppo *sizep = LDC_PAYLOAD_SIZE_RAW; 30341ae08745Sheppo 30351ae08745Sheppo rx_head = (rx_head + LDC_PACKET_SIZE) & q_size_mask; 30360a55fbb7Slm66018 rv = i_ldc_set_rx_head(ldcp, rx_head); 30371ae08745Sheppo 30381ae08745Sheppo return (rv); 30391ae08745Sheppo } 30401ae08745Sheppo 30411ae08745Sheppo /* 30421ae08745Sheppo * Process LDC mondos to build larger packets 30431ae08745Sheppo * with either un-reliable or reliable delivery. 30441ae08745Sheppo * 30451ae08745Sheppo * Enter and exit with ldcp->lock held by caller 30461ae08745Sheppo */ 30471ae08745Sheppo static int 30481ae08745Sheppo i_ldc_read_packet(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 30491ae08745Sheppo { 30501ae08745Sheppo int rv = 0; 30511ae08745Sheppo uint64_t rx_head = 0, rx_tail = 0; 30521ae08745Sheppo uint64_t curr_head = 0; 30531ae08745Sheppo ldc_msg_t *msg; 30541ae08745Sheppo caddr_t target; 30551ae08745Sheppo size_t len = 0, bytes_read = 0; 30560a55fbb7Slm66018 int retries = 0; 30571ae08745Sheppo uint64_t q_size_mask; 3058d10e4ef2Snarayan uint64_t first_fragment = 0; 30591ae08745Sheppo 30601ae08745Sheppo target = target_bufp; 30611ae08745Sheppo 30621ae08745Sheppo ASSERT(mutex_owned(&ldcp->lock)); 30631ae08745Sheppo 30641ae08745Sheppo /* compute mask for increment */ 30651ae08745Sheppo q_size_mask = (ldcp->rx_q_entries-1)<<LDC_PACKET_SHIFT; 30661ae08745Sheppo 30671ae08745Sheppo /* 30681ae08745Sheppo * Read packet(s) from the queue 30691ae08745Sheppo */ 30701ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, &curr_head, &rx_tail, 30711ae08745Sheppo &ldcp->link_state); 30721ae08745Sheppo if (rv != 0) { 30731ae08745Sheppo cmn_err(CE_WARN, 30741ae08745Sheppo "ldc_read: (0x%lx) unable to read queue ptrs", 30751ae08745Sheppo ldcp->id); 30761ae08745Sheppo return (EIO); 30771ae08745Sheppo } 30781ae08745Sheppo D1(ldcp->id, "ldc_read: (0x%llx) chd=0x%llx, tl=0x%llx, st=0x%llx\n", 30791ae08745Sheppo ldcp->id, curr_head, rx_tail, ldcp->link_state); 30801ae08745Sheppo 30811ae08745Sheppo /* reset the channel state if the channel went down */ 30821ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN) { 3083d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 30841ae08745Sheppo i_ldc_reset(ldcp); 3085d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 30861ae08745Sheppo return (ECONNRESET); 30871ae08745Sheppo } 30881ae08745Sheppo 30891ae08745Sheppo for (;;) { 30901ae08745Sheppo 30911ae08745Sheppo if (curr_head == rx_tail) { 30921ae08745Sheppo rv = hv_ldc_rx_get_state(ldcp->id, 30931ae08745Sheppo &rx_head, &rx_tail, &ldcp->link_state); 30941ae08745Sheppo if (rv != 0) { 30951ae08745Sheppo cmn_err(CE_WARN, 30961ae08745Sheppo "ldc_read: (0x%lx) cannot read queue ptrs", 30971ae08745Sheppo ldcp->id); 30981ae08745Sheppo return (EIO); 30991ae08745Sheppo } 31001ae08745Sheppo /* reset the channel state if the channel went down */ 31011ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN) { 3102d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 31031ae08745Sheppo i_ldc_reset(ldcp); 3104d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 31051ae08745Sheppo return (ECONNRESET); 31061ae08745Sheppo } 31071ae08745Sheppo } 31081ae08745Sheppo 31091ae08745Sheppo if (curr_head == rx_tail) { 31101ae08745Sheppo 31111ae08745Sheppo /* If in the middle of a fragmented xfer */ 3112d10e4ef2Snarayan if (first_fragment != 0) { 31130a55fbb7Slm66018 31140a55fbb7Slm66018 /* wait for ldc_delay usecs */ 31150a55fbb7Slm66018 drv_usecwait(ldc_delay); 31160a55fbb7Slm66018 31170a55fbb7Slm66018 if (++retries < ldc_max_retries) 31181ae08745Sheppo continue; 31190a55fbb7Slm66018 31201ae08745Sheppo *sizep = 0; 3121d10e4ef2Snarayan ldcp->last_msg_rcd = first_fragment - 1; 31221ae08745Sheppo DWARN(DBG_ALL_LDCS, 31231ae08745Sheppo "ldc_read: (0x%llx) read timeout", 31241ae08745Sheppo ldcp->id); 31251ae08745Sheppo return (ETIMEDOUT); 31261ae08745Sheppo } 31271ae08745Sheppo *sizep = 0; 31281ae08745Sheppo break; 31291ae08745Sheppo } 31300a55fbb7Slm66018 retries = 0; 31311ae08745Sheppo 31321ae08745Sheppo D2(ldcp->id, 31331ae08745Sheppo "ldc_read: (0x%llx) chd=0x%llx, rxhd=0x%llx, rxtl=0x%llx\n", 31341ae08745Sheppo ldcp->id, curr_head, rx_head, rx_tail); 31351ae08745Sheppo 31361ae08745Sheppo /* get the message */ 31371ae08745Sheppo msg = (ldc_msg_t *)(ldcp->rx_q_va + curr_head); 31381ae08745Sheppo 31391ae08745Sheppo DUMP_LDC_PKT(ldcp, "ldc_read received pkt", 31401ae08745Sheppo ldcp->rx_q_va + curr_head); 31411ae08745Sheppo 31421ae08745Sheppo /* Check the message ID for the message received */ 31431ae08745Sheppo if ((rv = i_ldc_check_seqid(ldcp, msg)) != 0) { 31441ae08745Sheppo 31451ae08745Sheppo DWARN(ldcp->id, "ldc_read: (0x%llx) seqid error, " 31461ae08745Sheppo "q_ptrs=0x%lx,0x%lx", ldcp->id, rx_head, rx_tail); 31471ae08745Sheppo 31480a55fbb7Slm66018 /* throw away data */ 31490a55fbb7Slm66018 bytes_read = 0; 31500a55fbb7Slm66018 31511ae08745Sheppo /* Reset last_msg_rcd to start of message */ 3152d10e4ef2Snarayan if (first_fragment != 0) { 3153d10e4ef2Snarayan ldcp->last_msg_rcd = first_fragment - 1; 3154d10e4ef2Snarayan first_fragment = 0; 31551ae08745Sheppo } 31561ae08745Sheppo /* 31571ae08745Sheppo * Send a NACK -- invalid seqid 31581ae08745Sheppo * get the current tail for the response 31591ae08745Sheppo */ 31601ae08745Sheppo rv = i_ldc_send_pkt(ldcp, msg->type, LDC_NACK, 31611ae08745Sheppo (msg->ctrl & LDC_CTRL_MASK)); 31621ae08745Sheppo if (rv) { 31631ae08745Sheppo cmn_err(CE_NOTE, 31641ae08745Sheppo "ldc_read: (0x%lx) err sending " 31651ae08745Sheppo "NACK msg\n", ldcp->id); 3166d10e4ef2Snarayan 3167d10e4ef2Snarayan /* if cannot send NACK - reset channel */ 3168d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 3169d10e4ef2Snarayan i_ldc_reset(ldcp); 3170d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3171d10e4ef2Snarayan rv = ECONNRESET; 3172d10e4ef2Snarayan break; 31731ae08745Sheppo } 31741ae08745Sheppo 31751ae08745Sheppo /* purge receive queue */ 31760a55fbb7Slm66018 rv = i_ldc_set_rx_head(ldcp, rx_tail); 31771ae08745Sheppo 31781ae08745Sheppo break; 31791ae08745Sheppo } 31801ae08745Sheppo 31811ae08745Sheppo /* 31821ae08745Sheppo * Process any messages of type CTRL messages 3183*e1ebb9ecSlm66018 * Future implementations should try to pass these 3184*e1ebb9ecSlm66018 * to LDC link by resetting the intr state. 31851ae08745Sheppo * 31861ae08745Sheppo * NOTE: not done as a switch() as type can be both ctrl+data 31871ae08745Sheppo */ 31881ae08745Sheppo if (msg->type & LDC_CTRL) { 31891ae08745Sheppo if (rv = i_ldc_ctrlmsg(ldcp, msg)) { 31901ae08745Sheppo if (rv == EAGAIN) 31911ae08745Sheppo continue; 31920a55fbb7Slm66018 rv = i_ldc_set_rx_head(ldcp, rx_tail); 31931ae08745Sheppo *sizep = 0; 31941ae08745Sheppo bytes_read = 0; 31951ae08745Sheppo break; 31961ae08745Sheppo } 31971ae08745Sheppo } 31981ae08745Sheppo 31991ae08745Sheppo /* process data ACKs */ 32001ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 3201d10e4ef2Snarayan if (rv = i_ldc_process_data_ACK(ldcp, msg)) { 3202d10e4ef2Snarayan *sizep = 0; 3203d10e4ef2Snarayan bytes_read = 0; 3204d10e4ef2Snarayan break; 3205d10e4ef2Snarayan } 32061ae08745Sheppo } 32071ae08745Sheppo 32081ae08745Sheppo /* process data messages */ 32091ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_INFO)) { 32101ae08745Sheppo 32111ae08745Sheppo uint8_t *msgbuf = (uint8_t *)( 32121ae08745Sheppo (ldcp->mode == LDC_MODE_RELIABLE || 32131ae08745Sheppo ldcp->mode == LDC_MODE_STREAM) 32141ae08745Sheppo ? msg->rdata : msg->udata); 32151ae08745Sheppo 32161ae08745Sheppo D2(ldcp->id, 32171ae08745Sheppo "ldc_read: (0x%llx) received data msg\n", ldcp->id); 32181ae08745Sheppo 32191ae08745Sheppo /* get the packet length */ 32201ae08745Sheppo len = (msg->env & LDC_LEN_MASK); 32211ae08745Sheppo 32221ae08745Sheppo /* 32231ae08745Sheppo * FUTURE OPTIMIZATION: 32241ae08745Sheppo * dont need to set q head for every 32251ae08745Sheppo * packet we read just need to do this when 32261ae08745Sheppo * we are done or need to wait for more 32271ae08745Sheppo * mondos to make a full packet - this is 32281ae08745Sheppo * currently expensive. 32291ae08745Sheppo */ 32301ae08745Sheppo 3231d10e4ef2Snarayan if (first_fragment == 0) { 32321ae08745Sheppo 32331ae08745Sheppo /* 32341ae08745Sheppo * first packets should always have the start 32351ae08745Sheppo * bit set (even for a single packet). If not 32361ae08745Sheppo * throw away the packet 32371ae08745Sheppo */ 32381ae08745Sheppo if (!(msg->env & LDC_FRAG_START)) { 32391ae08745Sheppo 32401ae08745Sheppo DWARN(DBG_ALL_LDCS, 32411ae08745Sheppo "ldc_read: (0x%llx) not start - " 32421ae08745Sheppo "frag=%x\n", ldcp->id, 32431ae08745Sheppo (msg->env) & LDC_FRAG_MASK); 32441ae08745Sheppo 32451ae08745Sheppo /* toss pkt, inc head, cont reading */ 32461ae08745Sheppo bytes_read = 0; 32471ae08745Sheppo target = target_bufp; 32481ae08745Sheppo curr_head = 32491ae08745Sheppo (curr_head + LDC_PACKET_SIZE) 32501ae08745Sheppo & q_size_mask; 32511ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, 32521ae08745Sheppo curr_head)) 32531ae08745Sheppo break; 32541ae08745Sheppo 32551ae08745Sheppo continue; 32561ae08745Sheppo } 32571ae08745Sheppo 3258d10e4ef2Snarayan first_fragment = msg->seqid; 32591ae08745Sheppo } else { 32601ae08745Sheppo /* check to see if this is a pkt w/ START bit */ 32611ae08745Sheppo if (msg->env & LDC_FRAG_START) { 32621ae08745Sheppo DWARN(DBG_ALL_LDCS, 32631ae08745Sheppo "ldc_read:(0x%llx) unexpected pkt" 32641ae08745Sheppo " env=0x%x discarding %d bytes," 32651ae08745Sheppo " lastmsg=%d, currentmsg=%d\n", 32661ae08745Sheppo ldcp->id, msg->env&LDC_FRAG_MASK, 32671ae08745Sheppo bytes_read, ldcp->last_msg_rcd, 32681ae08745Sheppo msg->seqid); 32691ae08745Sheppo 32701ae08745Sheppo /* throw data we have read so far */ 32711ae08745Sheppo bytes_read = 0; 32721ae08745Sheppo target = target_bufp; 3273d10e4ef2Snarayan first_fragment = msg->seqid; 32741ae08745Sheppo 32751ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, 32761ae08745Sheppo curr_head)) 32771ae08745Sheppo break; 32781ae08745Sheppo } 32791ae08745Sheppo } 32801ae08745Sheppo 32811ae08745Sheppo /* copy (next) pkt into buffer */ 32821ae08745Sheppo if (len <= (*sizep - bytes_read)) { 32831ae08745Sheppo bcopy(msgbuf, target, len); 32841ae08745Sheppo target += len; 32851ae08745Sheppo bytes_read += len; 32861ae08745Sheppo } else { 32871ae08745Sheppo /* 32881ae08745Sheppo * there is not enough space in the buffer to 32891ae08745Sheppo * read this pkt. throw message away & continue 32901ae08745Sheppo * reading data from queue 32911ae08745Sheppo */ 32921ae08745Sheppo DWARN(DBG_ALL_LDCS, 32931ae08745Sheppo "ldc_read: (0x%llx) buffer too small, " 32941ae08745Sheppo "head=0x%lx, expect=%d, got=%d\n", ldcp->id, 32951ae08745Sheppo curr_head, *sizep, bytes_read+len); 32961ae08745Sheppo 3297d10e4ef2Snarayan first_fragment = 0; 32981ae08745Sheppo target = target_bufp; 32991ae08745Sheppo bytes_read = 0; 33001ae08745Sheppo 33011ae08745Sheppo /* throw away everything received so far */ 33021ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, curr_head)) 33031ae08745Sheppo break; 33041ae08745Sheppo 33051ae08745Sheppo /* continue reading remaining pkts */ 33061ae08745Sheppo continue; 33071ae08745Sheppo } 33081ae08745Sheppo } 33091ae08745Sheppo 33101ae08745Sheppo /* set the message id */ 33111ae08745Sheppo ldcp->last_msg_rcd = msg->seqid; 33121ae08745Sheppo 33131ae08745Sheppo /* move the head one position */ 33141ae08745Sheppo curr_head = (curr_head + LDC_PACKET_SIZE) & q_size_mask; 33151ae08745Sheppo 33161ae08745Sheppo if (msg->env & LDC_FRAG_STOP) { 33171ae08745Sheppo 33181ae08745Sheppo /* 33191ae08745Sheppo * All pkts that are part of this fragmented transfer 33201ae08745Sheppo * have been read or this was a single pkt read 33211ae08745Sheppo * or there was an error 33221ae08745Sheppo */ 33231ae08745Sheppo 33241ae08745Sheppo /* set the queue head */ 33251ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, curr_head)) 33261ae08745Sheppo bytes_read = 0; 33271ae08745Sheppo 33281ae08745Sheppo *sizep = bytes_read; 33291ae08745Sheppo 33301ae08745Sheppo break; 33311ae08745Sheppo } 33321ae08745Sheppo 33331ae08745Sheppo /* advance head if it is a DATA ACK */ 33341ae08745Sheppo if ((msg->type & LDC_DATA) && (msg->stype & LDC_ACK)) { 33351ae08745Sheppo 33361ae08745Sheppo /* set the queue head */ 33371ae08745Sheppo if (rv = i_ldc_set_rx_head(ldcp, curr_head)) { 33381ae08745Sheppo bytes_read = 0; 33391ae08745Sheppo break; 33401ae08745Sheppo } 33411ae08745Sheppo 33421ae08745Sheppo D2(ldcp->id, "ldc_read: (0x%llx) set ACK qhead 0x%llx", 33431ae08745Sheppo ldcp->id, curr_head); 33441ae08745Sheppo } 33451ae08745Sheppo 33461ae08745Sheppo } /* for (;;) */ 33471ae08745Sheppo 33481ae08745Sheppo 33491ae08745Sheppo /* 33501ae08745Sheppo * If useful data was read - Send msg ACK 33511ae08745Sheppo * OPTIMIZE: do not send ACK for all msgs - use some frequency 33521ae08745Sheppo */ 33531ae08745Sheppo if ((bytes_read > 0) && (ldcp->mode == LDC_MODE_RELIABLE || 33541ae08745Sheppo ldcp->mode == LDC_MODE_STREAM)) { 33551ae08745Sheppo 33561ae08745Sheppo rv = i_ldc_send_pkt(ldcp, LDC_DATA, LDC_ACK, 0); 3357d10e4ef2Snarayan if (rv) { 33581ae08745Sheppo cmn_err(CE_NOTE, 33591ae08745Sheppo "ldc_read: (0x%lx) cannot send ACK\n", ldcp->id); 3360d10e4ef2Snarayan 3361d10e4ef2Snarayan /* if cannot send ACK - reset channel */ 3362d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 3363d10e4ef2Snarayan i_ldc_reset(ldcp); 3364d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3365d10e4ef2Snarayan rv = ECONNRESET; 33661ae08745Sheppo } 33671ae08745Sheppo } 33681ae08745Sheppo 33691ae08745Sheppo D2(ldcp->id, "ldc_read: (0x%llx) end size=%d", ldcp->id, *sizep); 33701ae08745Sheppo 33711ae08745Sheppo return (rv); 33721ae08745Sheppo } 33731ae08745Sheppo 33741ae08745Sheppo /* 33751ae08745Sheppo * Use underlying reliable packet mechanism to fetch 33761ae08745Sheppo * and buffer incoming packets so we can hand them back as 33771ae08745Sheppo * a basic byte stream. 33781ae08745Sheppo * 33791ae08745Sheppo * Enter and exit with ldcp->lock held by caller 33801ae08745Sheppo */ 33811ae08745Sheppo static int 33821ae08745Sheppo i_ldc_read_stream(ldc_chan_t *ldcp, caddr_t target_bufp, size_t *sizep) 33831ae08745Sheppo { 33841ae08745Sheppo int rv; 33851ae08745Sheppo size_t size; 33861ae08745Sheppo 33871ae08745Sheppo ASSERT(mutex_owned(&ldcp->lock)); 33881ae08745Sheppo 33891ae08745Sheppo D2(ldcp->id, "i_ldc_read_stream: (0x%llx) buffer size=%d", 33901ae08745Sheppo ldcp->id, *sizep); 33911ae08745Sheppo 33921ae08745Sheppo if (ldcp->stream_remains == 0) { 33931ae08745Sheppo size = ldcp->mtu; 33941ae08745Sheppo rv = i_ldc_read_packet(ldcp, 33951ae08745Sheppo (caddr_t)ldcp->stream_bufferp, &size); 33961ae08745Sheppo D2(ldcp->id, "i_ldc_read_stream: read packet (0x%llx) size=%d", 33971ae08745Sheppo ldcp->id, size); 33981ae08745Sheppo 33991ae08745Sheppo if (rv != 0) 34001ae08745Sheppo return (rv); 34011ae08745Sheppo 34021ae08745Sheppo ldcp->stream_remains = size; 34031ae08745Sheppo ldcp->stream_offset = 0; 34041ae08745Sheppo } 34051ae08745Sheppo 34061ae08745Sheppo size = MIN(ldcp->stream_remains, *sizep); 34071ae08745Sheppo 34081ae08745Sheppo bcopy(ldcp->stream_bufferp + ldcp->stream_offset, target_bufp, size); 34091ae08745Sheppo ldcp->stream_offset += size; 34101ae08745Sheppo ldcp->stream_remains -= size; 34111ae08745Sheppo 34121ae08745Sheppo D2(ldcp->id, "i_ldc_read_stream: (0x%llx) fill from buffer size=%d", 34131ae08745Sheppo ldcp->id, size); 34141ae08745Sheppo 34151ae08745Sheppo *sizep = size; 34161ae08745Sheppo return (0); 34171ae08745Sheppo } 34181ae08745Sheppo 34191ae08745Sheppo /* 34201ae08745Sheppo * Write specified amount of bytes to the channel 34211ae08745Sheppo * in multiple pkts of pkt_payload size. Each 34221ae08745Sheppo * packet is tagged with an unique packet ID in 3423*e1ebb9ecSlm66018 * the case of a reliable link. 34241ae08745Sheppo * 34251ae08745Sheppo * On return, size contains the number of bytes written. 34261ae08745Sheppo */ 34271ae08745Sheppo int 34281ae08745Sheppo ldc_write(ldc_handle_t handle, caddr_t buf, size_t *sizep) 34291ae08745Sheppo { 34301ae08745Sheppo ldc_chan_t *ldcp; 34311ae08745Sheppo int rv = 0; 34321ae08745Sheppo 34331ae08745Sheppo if (handle == NULL) { 34341ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_write: invalid channel handle\n"); 34351ae08745Sheppo return (EINVAL); 34361ae08745Sheppo } 34371ae08745Sheppo ldcp = (ldc_chan_t *)handle; 34381ae08745Sheppo 3439d10e4ef2Snarayan /* check if writes can occur */ 3440d10e4ef2Snarayan if (!mutex_tryenter(&ldcp->tx_lock)) { 3441d10e4ef2Snarayan /* 3442d10e4ef2Snarayan * Could not get the lock - channel could 3443d10e4ef2Snarayan * be in the process of being unconfigured 3444d10e4ef2Snarayan * or reader has encountered an error 3445d10e4ef2Snarayan */ 3446d10e4ef2Snarayan return (EAGAIN); 3447d10e4ef2Snarayan } 34481ae08745Sheppo 34491ae08745Sheppo /* check if non-zero data to write */ 34501ae08745Sheppo if (buf == NULL || sizep == NULL) { 34511ae08745Sheppo DWARN(ldcp->id, "ldc_write: (0x%llx) invalid data write\n", 34521ae08745Sheppo ldcp->id); 3453d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 34541ae08745Sheppo return (EINVAL); 34551ae08745Sheppo } 34561ae08745Sheppo 34571ae08745Sheppo if (*sizep == 0) { 34581ae08745Sheppo DWARN(ldcp->id, "ldc_write: (0x%llx) write size of zero\n", 34591ae08745Sheppo ldcp->id); 3460d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 34611ae08745Sheppo return (0); 34621ae08745Sheppo } 34631ae08745Sheppo 34641ae08745Sheppo /* Check if channel is UP for data exchange */ 34651ae08745Sheppo if (ldcp->tstate != TS_UP) { 34661ae08745Sheppo DWARN(ldcp->id, 34671ae08745Sheppo "ldc_write: (0x%llx) channel is not in UP state\n", 34681ae08745Sheppo ldcp->id); 34691ae08745Sheppo *sizep = 0; 34701ae08745Sheppo rv = ECONNRESET; 34711ae08745Sheppo } else { 34721ae08745Sheppo rv = ldcp->write_p(ldcp, buf, sizep); 34731ae08745Sheppo } 34741ae08745Sheppo 3475d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 34761ae08745Sheppo 34771ae08745Sheppo return (rv); 34781ae08745Sheppo } 34791ae08745Sheppo 34801ae08745Sheppo /* 34811ae08745Sheppo * Write a raw packet to the channel 34821ae08745Sheppo * On return, size contains the number of bytes written. 34831ae08745Sheppo */ 34841ae08745Sheppo static int 34851ae08745Sheppo i_ldc_write_raw(ldc_chan_t *ldcp, caddr_t buf, size_t *sizep) 34861ae08745Sheppo { 34871ae08745Sheppo ldc_msg_t *ldcmsg; 34881ae08745Sheppo uint64_t tx_head, tx_tail, new_tail; 34891ae08745Sheppo int rv = 0; 34901ae08745Sheppo size_t size; 34911ae08745Sheppo 3492d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 34931ae08745Sheppo ASSERT(ldcp->mode == LDC_MODE_RAW); 34941ae08745Sheppo 34951ae08745Sheppo size = *sizep; 34961ae08745Sheppo 34971ae08745Sheppo /* 34981ae08745Sheppo * Check to see if the packet size is less than or 34991ae08745Sheppo * equal to packet size support in raw mode 35001ae08745Sheppo */ 35011ae08745Sheppo if (size > ldcp->pkt_payload) { 35021ae08745Sheppo DWARN(ldcp->id, 35031ae08745Sheppo "ldc_write: (0x%llx) invalid size (0x%llx) for RAW mode\n", 35041ae08745Sheppo ldcp->id, *sizep); 35051ae08745Sheppo *sizep = 0; 35061ae08745Sheppo return (EMSGSIZE); 35071ae08745Sheppo } 35081ae08745Sheppo 35091ae08745Sheppo /* get the qptrs for the tx queue */ 35101ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 35111ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 35121ae08745Sheppo if (rv != 0) { 35131ae08745Sheppo cmn_err(CE_WARN, 35141ae08745Sheppo "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp->id); 35151ae08745Sheppo *sizep = 0; 35161ae08745Sheppo return (EIO); 35171ae08745Sheppo } 35181ae08745Sheppo 35191ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN || 35201ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 35211ae08745Sheppo DWARN(ldcp->id, 35221ae08745Sheppo "ldc_write: (0x%llx) channel down/reset\n", ldcp->id); 3523d10e4ef2Snarayan 35241ae08745Sheppo *sizep = 0; 3525d10e4ef2Snarayan if (mutex_tryenter(&ldcp->lock)) { 3526d10e4ef2Snarayan i_ldc_reset(ldcp); 3527d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3528d10e4ef2Snarayan } else { 3529d10e4ef2Snarayan /* 3530d10e4ef2Snarayan * Release Tx lock, and then reacquire channel 3531d10e4ef2Snarayan * and Tx lock in correct order 3532d10e4ef2Snarayan */ 3533d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3534d10e4ef2Snarayan mutex_enter(&ldcp->lock); 3535d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 3536d10e4ef2Snarayan i_ldc_reset(ldcp); 3537d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3538d10e4ef2Snarayan } 35391ae08745Sheppo return (ECONNRESET); 35401ae08745Sheppo } 35411ae08745Sheppo 35421ae08745Sheppo tx_tail = ldcp->tx_tail; 35431ae08745Sheppo tx_head = ldcp->tx_head; 35441ae08745Sheppo new_tail = (tx_tail + LDC_PACKET_SIZE) & 35451ae08745Sheppo ((ldcp->tx_q_entries-1) << LDC_PACKET_SHIFT); 35461ae08745Sheppo 35471ae08745Sheppo if (new_tail == tx_head) { 35481ae08745Sheppo DWARN(DBG_ALL_LDCS, 35491ae08745Sheppo "ldc_write: (0x%llx) TX queue is full\n", ldcp->id); 35501ae08745Sheppo *sizep = 0; 35511ae08745Sheppo return (EWOULDBLOCK); 35521ae08745Sheppo } 35531ae08745Sheppo 35541ae08745Sheppo D2(ldcp->id, "ldc_write: (0x%llx) start xfer size=%d", 35551ae08745Sheppo ldcp->id, size); 35561ae08745Sheppo 35571ae08745Sheppo /* Send the data now */ 35581ae08745Sheppo ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 35591ae08745Sheppo 35601ae08745Sheppo /* copy the data into pkt */ 35611ae08745Sheppo bcopy((uint8_t *)buf, ldcmsg, size); 35621ae08745Sheppo 35631ae08745Sheppo /* increment tail */ 35641ae08745Sheppo tx_tail = new_tail; 35651ae08745Sheppo 35661ae08745Sheppo /* 35671ae08745Sheppo * All packets have been copied into the TX queue 35681ae08745Sheppo * update the tail ptr in the HV 35691ae08745Sheppo */ 35701ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 35711ae08745Sheppo if (rv) { 35721ae08745Sheppo if (rv == EWOULDBLOCK) { 35731ae08745Sheppo DWARN(ldcp->id, "ldc_write: (0x%llx) write timed out\n", 35741ae08745Sheppo ldcp->id); 35751ae08745Sheppo *sizep = 0; 35761ae08745Sheppo return (EWOULDBLOCK); 35771ae08745Sheppo } 35781ae08745Sheppo 35791ae08745Sheppo *sizep = 0; 3580d10e4ef2Snarayan if (mutex_tryenter(&ldcp->lock)) { 3581d10e4ef2Snarayan i_ldc_reset(ldcp); 3582d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3583d10e4ef2Snarayan } else { 3584d10e4ef2Snarayan /* 3585d10e4ef2Snarayan * Release Tx lock, and then reacquire channel 3586d10e4ef2Snarayan * and Tx lock in correct order 3587d10e4ef2Snarayan */ 3588d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3589d10e4ef2Snarayan mutex_enter(&ldcp->lock); 3590d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 3591d10e4ef2Snarayan i_ldc_reset(ldcp); 3592d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3593d10e4ef2Snarayan } 35941ae08745Sheppo return (ECONNRESET); 35951ae08745Sheppo } 35961ae08745Sheppo 35971ae08745Sheppo ldcp->tx_tail = tx_tail; 35981ae08745Sheppo *sizep = size; 35991ae08745Sheppo 36001ae08745Sheppo D2(ldcp->id, "ldc_write: (0x%llx) end xfer size=%d", ldcp->id, size); 36011ae08745Sheppo 36021ae08745Sheppo return (rv); 36031ae08745Sheppo } 36041ae08745Sheppo 36051ae08745Sheppo 36061ae08745Sheppo /* 36071ae08745Sheppo * Write specified amount of bytes to the channel 36081ae08745Sheppo * in multiple pkts of pkt_payload size. Each 36091ae08745Sheppo * packet is tagged with an unique packet ID in 3610*e1ebb9ecSlm66018 * the case of a reliable link. 36111ae08745Sheppo * 36121ae08745Sheppo * On return, size contains the number of bytes written. 36131ae08745Sheppo * This function needs to ensure that the write size is < MTU size 36141ae08745Sheppo */ 36151ae08745Sheppo static int 36161ae08745Sheppo i_ldc_write_packet(ldc_chan_t *ldcp, caddr_t buf, size_t *size) 36171ae08745Sheppo { 36181ae08745Sheppo ldc_msg_t *ldcmsg; 36191ae08745Sheppo uint64_t tx_head, tx_tail, new_tail, start; 36201ae08745Sheppo uint64_t txq_size_mask, numavail; 36211ae08745Sheppo uint8_t *msgbuf, *source = (uint8_t *)buf; 36221ae08745Sheppo size_t len, bytes_written = 0, remaining; 36231ae08745Sheppo int rv; 36241ae08745Sheppo uint32_t curr_seqid; 36251ae08745Sheppo 3626d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 36271ae08745Sheppo 36281ae08745Sheppo ASSERT(ldcp->mode == LDC_MODE_RELIABLE || 36291ae08745Sheppo ldcp->mode == LDC_MODE_UNRELIABLE || 36301ae08745Sheppo ldcp->mode == LDC_MODE_STREAM); 36311ae08745Sheppo 36321ae08745Sheppo /* compute mask for increment */ 36331ae08745Sheppo txq_size_mask = (ldcp->tx_q_entries - 1) << LDC_PACKET_SHIFT; 36341ae08745Sheppo 36351ae08745Sheppo /* get the qptrs for the tx queue */ 36361ae08745Sheppo rv = hv_ldc_tx_get_state(ldcp->id, 36371ae08745Sheppo &ldcp->tx_head, &ldcp->tx_tail, &ldcp->link_state); 36381ae08745Sheppo if (rv != 0) { 36391ae08745Sheppo cmn_err(CE_WARN, 36401ae08745Sheppo "ldc_write: (0x%lx) cannot read queue ptrs\n", ldcp->id); 36411ae08745Sheppo *size = 0; 36421ae08745Sheppo return (EIO); 36431ae08745Sheppo } 36441ae08745Sheppo 36451ae08745Sheppo if (ldcp->link_state == LDC_CHANNEL_DOWN || 36461ae08745Sheppo ldcp->link_state == LDC_CHANNEL_RESET) { 36471ae08745Sheppo DWARN(ldcp->id, 36481ae08745Sheppo "ldc_write: (0x%llx) channel down/reset\n", ldcp->id); 36491ae08745Sheppo *size = 0; 3650d10e4ef2Snarayan if (mutex_tryenter(&ldcp->lock)) { 36511ae08745Sheppo i_ldc_reset(ldcp); 3652d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3653d10e4ef2Snarayan } else { 3654d10e4ef2Snarayan /* 3655d10e4ef2Snarayan * Release Tx lock, and then reacquire channel 3656d10e4ef2Snarayan * and Tx lock in correct order 3657d10e4ef2Snarayan */ 3658d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3659d10e4ef2Snarayan mutex_enter(&ldcp->lock); 3660d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 3661d10e4ef2Snarayan i_ldc_reset(ldcp); 3662d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3663d10e4ef2Snarayan } 36641ae08745Sheppo return (ECONNRESET); 36651ae08745Sheppo } 36661ae08745Sheppo 36671ae08745Sheppo tx_tail = ldcp->tx_tail; 36681ae08745Sheppo new_tail = (tx_tail + LDC_PACKET_SIZE) % 36691ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT); 36701ae08745Sheppo 36711ae08745Sheppo /* 3672*e1ebb9ecSlm66018 * Link mode determines whether we use HV Tx head or the 36731ae08745Sheppo * private protocol head (corresponding to last ACKd pkt) for 36741ae08745Sheppo * determining how much we can write 36751ae08745Sheppo */ 36761ae08745Sheppo tx_head = (ldcp->mode == LDC_MODE_RELIABLE || 36771ae08745Sheppo ldcp->mode == LDC_MODE_STREAM) 36781ae08745Sheppo ? ldcp->tx_ackd_head : ldcp->tx_head; 36791ae08745Sheppo if (new_tail == tx_head) { 36801ae08745Sheppo DWARN(DBG_ALL_LDCS, 36811ae08745Sheppo "ldc_write: (0x%llx) TX queue is full\n", ldcp->id); 36821ae08745Sheppo *size = 0; 36831ae08745Sheppo return (EWOULDBLOCK); 36841ae08745Sheppo } 36851ae08745Sheppo 36861ae08745Sheppo /* 36871ae08745Sheppo * Make sure that the LDC Tx queue has enough space 36881ae08745Sheppo */ 36891ae08745Sheppo numavail = (tx_head >> LDC_PACKET_SHIFT) - (tx_tail >> LDC_PACKET_SHIFT) 36901ae08745Sheppo + ldcp->tx_q_entries - 1; 36911ae08745Sheppo numavail %= ldcp->tx_q_entries; 36921ae08745Sheppo 36931ae08745Sheppo if (*size > (numavail * ldcp->pkt_payload)) { 36941ae08745Sheppo DWARN(DBG_ALL_LDCS, 36951ae08745Sheppo "ldc_write: (0x%llx) TX queue has no space\n", ldcp->id); 36961ae08745Sheppo return (EWOULDBLOCK); 36971ae08745Sheppo } 36981ae08745Sheppo 36991ae08745Sheppo D2(ldcp->id, "ldc_write: (0x%llx) start xfer size=%d", 37001ae08745Sheppo ldcp->id, *size); 37011ae08745Sheppo 37021ae08745Sheppo /* Send the data now */ 37031ae08745Sheppo bytes_written = 0; 37041ae08745Sheppo curr_seqid = ldcp->last_msg_snt; 37051ae08745Sheppo start = tx_tail; 37061ae08745Sheppo 37071ae08745Sheppo while (*size > bytes_written) { 37081ae08745Sheppo 37091ae08745Sheppo ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + tx_tail); 37101ae08745Sheppo 37111ae08745Sheppo msgbuf = (uint8_t *)((ldcp->mode == LDC_MODE_RELIABLE || 37121ae08745Sheppo ldcp->mode == LDC_MODE_STREAM) 37131ae08745Sheppo ? ldcmsg->rdata : ldcmsg->udata); 37141ae08745Sheppo 37151ae08745Sheppo ldcmsg->type = LDC_DATA; 37161ae08745Sheppo ldcmsg->stype = LDC_INFO; 37171ae08745Sheppo ldcmsg->ctrl = 0; 37181ae08745Sheppo 37191ae08745Sheppo remaining = *size - bytes_written; 37201ae08745Sheppo len = min(ldcp->pkt_payload, remaining); 37211ae08745Sheppo ldcmsg->env = (uint8_t)len; 37221ae08745Sheppo 37231ae08745Sheppo curr_seqid++; 37241ae08745Sheppo ldcmsg->seqid = curr_seqid; 37251ae08745Sheppo 37261ae08745Sheppo DUMP_LDC_PKT(ldcp, "ldc_write snd data", (uint64_t)ldcmsg); 37271ae08745Sheppo 37281ae08745Sheppo /* copy the data into pkt */ 37291ae08745Sheppo bcopy(source, msgbuf, len); 37301ae08745Sheppo 37311ae08745Sheppo source += len; 37321ae08745Sheppo bytes_written += len; 37331ae08745Sheppo 37341ae08745Sheppo /* increment tail */ 37351ae08745Sheppo tx_tail = (tx_tail + LDC_PACKET_SIZE) & txq_size_mask; 37361ae08745Sheppo 37371ae08745Sheppo ASSERT(tx_tail != tx_head); 37381ae08745Sheppo } 37391ae08745Sheppo 37401ae08745Sheppo /* Set the start and stop bits */ 37411ae08745Sheppo ldcmsg->env |= LDC_FRAG_STOP; 37421ae08745Sheppo ldcmsg = (ldc_msg_t *)(ldcp->tx_q_va + start); 37431ae08745Sheppo ldcmsg->env |= LDC_FRAG_START; 37441ae08745Sheppo 37451ae08745Sheppo /* 37461ae08745Sheppo * All packets have been copied into the TX queue 37471ae08745Sheppo * update the tail ptr in the HV 37481ae08745Sheppo */ 37491ae08745Sheppo rv = i_ldc_set_tx_tail(ldcp, tx_tail); 37501ae08745Sheppo if (rv == 0) { 37511ae08745Sheppo ldcp->tx_tail = tx_tail; 37521ae08745Sheppo ldcp->last_msg_snt = curr_seqid; 37531ae08745Sheppo *size = bytes_written; 37541ae08745Sheppo } else { 37551ae08745Sheppo int rv2; 37561ae08745Sheppo 37571ae08745Sheppo if (rv != EWOULDBLOCK) { 37581ae08745Sheppo *size = 0; 3759d10e4ef2Snarayan if (mutex_tryenter(&ldcp->lock)) { 3760d10e4ef2Snarayan i_ldc_reset(ldcp); 3761d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3762d10e4ef2Snarayan } else { 3763d10e4ef2Snarayan /* 3764d10e4ef2Snarayan * Release Tx lock, and then reacquire channel 3765d10e4ef2Snarayan * and Tx lock in correct order 3766d10e4ef2Snarayan */ 3767d10e4ef2Snarayan mutex_exit(&ldcp->tx_lock); 3768d10e4ef2Snarayan mutex_enter(&ldcp->lock); 3769d10e4ef2Snarayan mutex_enter(&ldcp->tx_lock); 3770d10e4ef2Snarayan i_ldc_reset(ldcp); 3771d10e4ef2Snarayan mutex_exit(&ldcp->lock); 3772d10e4ef2Snarayan } 37731ae08745Sheppo return (ECONNRESET); 37741ae08745Sheppo } 37751ae08745Sheppo 37761ae08745Sheppo DWARN(ldcp->id, "hv_tx_set_tail returns 0x%x (head 0x%x, " 37771ae08745Sheppo "old tail 0x%x, new tail 0x%x, qsize=0x%x)\n", 37781ae08745Sheppo rv, ldcp->tx_head, ldcp->tx_tail, tx_tail, 37791ae08745Sheppo (ldcp->tx_q_entries << LDC_PACKET_SHIFT)); 37801ae08745Sheppo 37811ae08745Sheppo rv2 = hv_ldc_tx_get_state(ldcp->id, 37821ae08745Sheppo &tx_head, &tx_tail, &ldcp->link_state); 37831ae08745Sheppo 37841ae08745Sheppo DWARN(ldcp->id, "hv_ldc_tx_get_state returns 0x%x " 37851ae08745Sheppo "(head 0x%x, tail 0x%x state 0x%x)\n", 37861ae08745Sheppo rv2, tx_head, tx_tail, ldcp->link_state); 37871ae08745Sheppo 37881ae08745Sheppo *size = 0; 37891ae08745Sheppo } 37901ae08745Sheppo 37911ae08745Sheppo D2(ldcp->id, "ldc_write: (0x%llx) end xfer size=%d", ldcp->id, *size); 37921ae08745Sheppo 37931ae08745Sheppo return (rv); 37941ae08745Sheppo } 37951ae08745Sheppo 37961ae08745Sheppo /* 37971ae08745Sheppo * Write specified amount of bytes to the channel 37981ae08745Sheppo * in multiple pkts of pkt_payload size. Each 37991ae08745Sheppo * packet is tagged with an unique packet ID in 3800*e1ebb9ecSlm66018 * the case of a reliable link. 38011ae08745Sheppo * 38021ae08745Sheppo * On return, size contains the number of bytes written. 38031ae08745Sheppo * This function needs to ensure that the write size is < MTU size 38041ae08745Sheppo */ 38051ae08745Sheppo static int 38061ae08745Sheppo i_ldc_write_stream(ldc_chan_t *ldcp, caddr_t buf, size_t *sizep) 38071ae08745Sheppo { 3808d10e4ef2Snarayan ASSERT(MUTEX_HELD(&ldcp->tx_lock)); 38091ae08745Sheppo ASSERT(ldcp->mode == LDC_MODE_STREAM); 38101ae08745Sheppo 38111ae08745Sheppo /* Truncate packet to max of MTU size */ 38121ae08745Sheppo if (*sizep > ldcp->mtu) *sizep = ldcp->mtu; 38131ae08745Sheppo return (i_ldc_write_packet(ldcp, buf, sizep)); 38141ae08745Sheppo } 38151ae08745Sheppo 38161ae08745Sheppo 38171ae08745Sheppo /* 38181ae08745Sheppo * Interfaces for channel nexus to register/unregister with LDC module 38191ae08745Sheppo * The nexus will register functions to be used to register individual 38201ae08745Sheppo * channels with the nexus and enable interrupts for the channels 38211ae08745Sheppo */ 38221ae08745Sheppo int 38231ae08745Sheppo ldc_register(ldc_cnex_t *cinfo) 38241ae08745Sheppo { 38251ae08745Sheppo ldc_chan_t *ldcp; 38261ae08745Sheppo 38271ae08745Sheppo if (cinfo == NULL || cinfo->dip == NULL || 38281ae08745Sheppo cinfo->reg_chan == NULL || cinfo->unreg_chan == NULL || 38291ae08745Sheppo cinfo->add_intr == NULL || cinfo->rem_intr == NULL || 38301ae08745Sheppo cinfo->clr_intr == NULL) { 38311ae08745Sheppo 38321ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_register: invalid nexus info\n"); 38331ae08745Sheppo return (EINVAL); 38341ae08745Sheppo } 38351ae08745Sheppo 38361ae08745Sheppo mutex_enter(&ldcssp->lock); 38371ae08745Sheppo 38381ae08745Sheppo /* nexus registration */ 38391ae08745Sheppo ldcssp->cinfo.dip = cinfo->dip; 38401ae08745Sheppo ldcssp->cinfo.reg_chan = cinfo->reg_chan; 38411ae08745Sheppo ldcssp->cinfo.unreg_chan = cinfo->unreg_chan; 38421ae08745Sheppo ldcssp->cinfo.add_intr = cinfo->add_intr; 38431ae08745Sheppo ldcssp->cinfo.rem_intr = cinfo->rem_intr; 38441ae08745Sheppo ldcssp->cinfo.clr_intr = cinfo->clr_intr; 38451ae08745Sheppo 38461ae08745Sheppo /* register any channels that might have been previously initialized */ 38471ae08745Sheppo ldcp = ldcssp->chan_list; 38481ae08745Sheppo while (ldcp) { 38491ae08745Sheppo if ((ldcp->tstate & TS_QCONF_RDY) && 38501ae08745Sheppo (ldcp->tstate & TS_CNEX_RDY) == 0) 38511ae08745Sheppo (void) i_ldc_register_channel(ldcp); 38521ae08745Sheppo 38531ae08745Sheppo ldcp = ldcp->next; 38541ae08745Sheppo } 38551ae08745Sheppo 38561ae08745Sheppo mutex_exit(&ldcssp->lock); 38571ae08745Sheppo 38581ae08745Sheppo return (0); 38591ae08745Sheppo } 38601ae08745Sheppo 38611ae08745Sheppo int 38621ae08745Sheppo ldc_unregister(ldc_cnex_t *cinfo) 38631ae08745Sheppo { 38641ae08745Sheppo if (cinfo == NULL || cinfo->dip == NULL) { 38651ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_unregister: invalid nexus info\n"); 38661ae08745Sheppo return (EINVAL); 38671ae08745Sheppo } 38681ae08745Sheppo 38691ae08745Sheppo mutex_enter(&ldcssp->lock); 38701ae08745Sheppo 38711ae08745Sheppo if (cinfo->dip != ldcssp->cinfo.dip) { 38721ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_unregister: invalid dip\n"); 38731ae08745Sheppo mutex_exit(&ldcssp->lock); 38741ae08745Sheppo return (EINVAL); 38751ae08745Sheppo } 38761ae08745Sheppo 38771ae08745Sheppo /* nexus unregister */ 38781ae08745Sheppo ldcssp->cinfo.dip = NULL; 38791ae08745Sheppo ldcssp->cinfo.reg_chan = NULL; 38801ae08745Sheppo ldcssp->cinfo.unreg_chan = NULL; 38811ae08745Sheppo ldcssp->cinfo.add_intr = NULL; 38821ae08745Sheppo ldcssp->cinfo.rem_intr = NULL; 38831ae08745Sheppo ldcssp->cinfo.clr_intr = NULL; 38841ae08745Sheppo 38851ae08745Sheppo mutex_exit(&ldcssp->lock); 38861ae08745Sheppo 38871ae08745Sheppo return (0); 38881ae08745Sheppo } 38891ae08745Sheppo 38901ae08745Sheppo 38911ae08745Sheppo /* ------------------------------------------------------------------------- */ 38921ae08745Sheppo 38931ae08745Sheppo /* 38941ae08745Sheppo * Allocate a memory handle for the channel and link it into the list 38951ae08745Sheppo * Also choose which memory table to use if this is the first handle 38961ae08745Sheppo * being assigned to this channel 38971ae08745Sheppo */ 38981ae08745Sheppo int 38991ae08745Sheppo ldc_mem_alloc_handle(ldc_handle_t handle, ldc_mem_handle_t *mhandle) 39001ae08745Sheppo { 39011ae08745Sheppo ldc_chan_t *ldcp; 39021ae08745Sheppo ldc_mhdl_t *mhdl; 39031ae08745Sheppo int rv; 39041ae08745Sheppo 39051ae08745Sheppo if (handle == NULL) { 39061ae08745Sheppo DWARN(DBG_ALL_LDCS, 39071ae08745Sheppo "ldc_mem_alloc_handle: invalid channel handle\n"); 39081ae08745Sheppo return (EINVAL); 39091ae08745Sheppo } 39101ae08745Sheppo ldcp = (ldc_chan_t *)handle; 39111ae08745Sheppo 39121ae08745Sheppo mutex_enter(&ldcp->lock); 39131ae08745Sheppo 39141ae08745Sheppo /* check to see if channel is initalized */ 39151ae08745Sheppo if (ldcp->tstate < TS_INIT) { 39161ae08745Sheppo DWARN(ldcp->id, 39171ae08745Sheppo "ldc_mem_alloc_handle: (0x%llx) channel not initialized\n", 39181ae08745Sheppo ldcp->id); 39191ae08745Sheppo mutex_exit(&ldcp->lock); 39201ae08745Sheppo return (EINVAL); 39211ae08745Sheppo } 39221ae08745Sheppo 39231ae08745Sheppo /* 39241ae08745Sheppo * If this channel is allocating a mem handle for the 39251ae08745Sheppo * first time allocate it a memory map table and initialize it 39261ae08745Sheppo */ 39271ae08745Sheppo if (ldcp->mtbl == NULL) { 39281ae08745Sheppo 39291ae08745Sheppo ldc_mtbl_t *mtbl; 39301ae08745Sheppo 39311ae08745Sheppo /* Allocate and initialize the map table structure */ 39321ae08745Sheppo mtbl = kmem_zalloc(sizeof (ldc_mtbl_t), KM_SLEEP); 3933*e1ebb9ecSlm66018 mtbl->num_entries = mtbl->num_avail = ldc_maptable_entries; 3934*e1ebb9ecSlm66018 mtbl->size = ldc_maptable_entries * sizeof (ldc_mte_slot_t); 39351ae08745Sheppo mtbl->next_entry = NULL; 39361ae08745Sheppo 39371ae08745Sheppo /* Allocate the table itself */ 39381ae08745Sheppo mtbl->table = (ldc_mte_slot_t *) 39391ae08745Sheppo contig_mem_alloc_align(mtbl->size, MMU_PAGESIZE); 39401ae08745Sheppo if (mtbl->table == NULL) { 39411ae08745Sheppo cmn_err(CE_WARN, 39421ae08745Sheppo "ldc_mem_alloc_handle: (0x%lx) error allocating " 39431ae08745Sheppo "table memory", ldcp->id); 39441ae08745Sheppo kmem_free(mtbl, sizeof (ldc_mtbl_t)); 39451ae08745Sheppo mutex_exit(&ldcp->lock); 39461ae08745Sheppo return (ENOMEM); 39471ae08745Sheppo } 39481ae08745Sheppo 39491ae08745Sheppo /* zero out the memory */ 39501ae08745Sheppo bzero(mtbl->table, mtbl->size); 39511ae08745Sheppo 39521ae08745Sheppo /* initialize the lock */ 39531ae08745Sheppo mutex_init(&mtbl->lock, NULL, MUTEX_DRIVER, NULL); 39541ae08745Sheppo 39551ae08745Sheppo /* register table for this channel */ 39561ae08745Sheppo rv = hv_ldc_set_map_table(ldcp->id, 39571ae08745Sheppo va_to_pa(mtbl->table), mtbl->num_entries); 39581ae08745Sheppo if (rv != 0) { 39591ae08745Sheppo cmn_err(CE_WARN, 39601ae08745Sheppo "ldc_mem_alloc_handle: (0x%lx) err %d mapping tbl", 39611ae08745Sheppo ldcp->id, rv); 39621ae08745Sheppo contig_mem_free(mtbl->table, mtbl->size); 39631ae08745Sheppo mutex_destroy(&mtbl->lock); 39641ae08745Sheppo kmem_free(mtbl, sizeof (ldc_mtbl_t)); 39651ae08745Sheppo mutex_exit(&ldcp->lock); 39661ae08745Sheppo return (EIO); 39671ae08745Sheppo } 39681ae08745Sheppo 39691ae08745Sheppo ldcp->mtbl = mtbl; 39701ae08745Sheppo 39711ae08745Sheppo D1(ldcp->id, 39721ae08745Sheppo "ldc_mem_alloc_handle: (0x%llx) alloc'd map table 0x%llx\n", 39731ae08745Sheppo ldcp->id, ldcp->mtbl->table); 39741ae08745Sheppo } 39751ae08745Sheppo 39761ae08745Sheppo /* allocate handle for channel */ 39771ae08745Sheppo mhdl = kmem_zalloc(sizeof (ldc_mhdl_t), KM_SLEEP); 39781ae08745Sheppo 39791ae08745Sheppo /* initialize the lock */ 39801ae08745Sheppo mutex_init(&mhdl->lock, NULL, MUTEX_DRIVER, NULL); 39811ae08745Sheppo 39821ae08745Sheppo mhdl->status = LDC_UNBOUND; 39831ae08745Sheppo mhdl->ldcp = ldcp; 39841ae08745Sheppo 39851ae08745Sheppo /* insert memory handle (@ head) into list */ 39861ae08745Sheppo if (ldcp->mhdl_list == NULL) { 39871ae08745Sheppo ldcp->mhdl_list = mhdl; 39881ae08745Sheppo mhdl->next = NULL; 39891ae08745Sheppo } else { 39901ae08745Sheppo /* insert @ head */ 39911ae08745Sheppo mhdl->next = ldcp->mhdl_list; 39921ae08745Sheppo ldcp->mhdl_list = mhdl; 39931ae08745Sheppo } 39941ae08745Sheppo 39951ae08745Sheppo /* return the handle */ 39961ae08745Sheppo *mhandle = (ldc_mem_handle_t)mhdl; 39971ae08745Sheppo 39981ae08745Sheppo mutex_exit(&ldcp->lock); 39991ae08745Sheppo 40001ae08745Sheppo D1(ldcp->id, "ldc_mem_alloc_handle: (0x%llx) allocated handle 0x%llx\n", 40011ae08745Sheppo ldcp->id, mhdl); 40021ae08745Sheppo 40031ae08745Sheppo return (0); 40041ae08745Sheppo } 40051ae08745Sheppo 40061ae08745Sheppo /* 40071ae08745Sheppo * Free memory handle for the channel and unlink it from the list 40081ae08745Sheppo */ 40091ae08745Sheppo int 40101ae08745Sheppo ldc_mem_free_handle(ldc_mem_handle_t mhandle) 40111ae08745Sheppo { 40121ae08745Sheppo ldc_mhdl_t *mhdl, *phdl; 40131ae08745Sheppo ldc_chan_t *ldcp; 40141ae08745Sheppo 40151ae08745Sheppo if (mhandle == NULL) { 40161ae08745Sheppo DWARN(DBG_ALL_LDCS, 40171ae08745Sheppo "ldc_mem_free_handle: invalid memory handle\n"); 40181ae08745Sheppo return (EINVAL); 40191ae08745Sheppo } 40201ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 40211ae08745Sheppo 40221ae08745Sheppo mutex_enter(&mhdl->lock); 40231ae08745Sheppo 40241ae08745Sheppo ldcp = mhdl->ldcp; 40251ae08745Sheppo 40261ae08745Sheppo if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) { 40271ae08745Sheppo DWARN(ldcp->id, 40281ae08745Sheppo "ldc_mem_free_handle: cannot free, 0x%llx hdl bound\n", 40291ae08745Sheppo mhdl); 40301ae08745Sheppo mutex_exit(&mhdl->lock); 40311ae08745Sheppo return (EINVAL); 40321ae08745Sheppo } 40331ae08745Sheppo mutex_exit(&mhdl->lock); 40341ae08745Sheppo 40351ae08745Sheppo mutex_enter(&ldcp->mlist_lock); 40361ae08745Sheppo 40371ae08745Sheppo phdl = ldcp->mhdl_list; 40381ae08745Sheppo 40391ae08745Sheppo /* first handle */ 40401ae08745Sheppo if (phdl == mhdl) { 40411ae08745Sheppo ldcp->mhdl_list = mhdl->next; 40421ae08745Sheppo mutex_destroy(&mhdl->lock); 40431ae08745Sheppo kmem_free(mhdl, sizeof (ldc_mhdl_t)); 40441ae08745Sheppo D1(ldcp->id, 40451ae08745Sheppo "ldc_mem_free_handle: (0x%llx) freed handle 0x%llx\n", 40461ae08745Sheppo ldcp->id, mhdl); 40471ae08745Sheppo } else { 40481ae08745Sheppo /* walk the list - unlink and free */ 40491ae08745Sheppo while (phdl != NULL) { 40501ae08745Sheppo if (phdl->next == mhdl) { 40511ae08745Sheppo phdl->next = mhdl->next; 40521ae08745Sheppo mutex_destroy(&mhdl->lock); 40531ae08745Sheppo kmem_free(mhdl, sizeof (ldc_mhdl_t)); 40541ae08745Sheppo D1(ldcp->id, 40551ae08745Sheppo "ldc_mem_free_handle: (0x%llx) freed " 40561ae08745Sheppo "handle 0x%llx\n", ldcp->id, mhdl); 40571ae08745Sheppo break; 40581ae08745Sheppo } 40591ae08745Sheppo phdl = phdl->next; 40601ae08745Sheppo } 40611ae08745Sheppo } 40621ae08745Sheppo 40631ae08745Sheppo if (phdl == NULL) { 40641ae08745Sheppo DWARN(ldcp->id, 40651ae08745Sheppo "ldc_mem_free_handle: invalid handle 0x%llx\n", mhdl); 40661ae08745Sheppo mutex_exit(&ldcp->mlist_lock); 40671ae08745Sheppo return (EINVAL); 40681ae08745Sheppo } 40691ae08745Sheppo 40701ae08745Sheppo mutex_exit(&ldcp->mlist_lock); 40711ae08745Sheppo 40721ae08745Sheppo return (0); 40731ae08745Sheppo } 40741ae08745Sheppo 40751ae08745Sheppo /* 40761ae08745Sheppo * Bind a memory handle to a virtual address. 40771ae08745Sheppo * The virtual address is converted to the corresponding real addresses. 40781ae08745Sheppo * Returns pointer to the first ldc_mem_cookie and the total number 40791ae08745Sheppo * of cookies for this virtual address. Other cookies can be obtained 40801ae08745Sheppo * using the ldc_mem_nextcookie() call. If the pages are stored in 40811ae08745Sheppo * consecutive locations in the table, a single cookie corresponding to 40821ae08745Sheppo * the first location is returned. The cookie size spans all the entries. 40831ae08745Sheppo * 40841ae08745Sheppo * If the VA corresponds to a page that is already being exported, reuse 40851ae08745Sheppo * the page and do not export it again. Bump the page's use count. 40861ae08745Sheppo */ 40871ae08745Sheppo int 40881ae08745Sheppo ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr, size_t len, 40891ae08745Sheppo uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount) 40901ae08745Sheppo { 40911ae08745Sheppo ldc_mhdl_t *mhdl; 40921ae08745Sheppo ldc_chan_t *ldcp; 40931ae08745Sheppo ldc_mtbl_t *mtbl; 40941ae08745Sheppo ldc_memseg_t *memseg; 40951ae08745Sheppo ldc_mte_t tmp_mte; 40961ae08745Sheppo uint64_t index, prev_index = 0; 40971ae08745Sheppo int64_t cookie_idx; 40981ae08745Sheppo uintptr_t raddr, ra_aligned; 40991ae08745Sheppo uint64_t psize, poffset, v_offset; 41001ae08745Sheppo uint64_t pg_shift, pg_size, pg_size_code, pg_mask; 41011ae08745Sheppo pgcnt_t npages; 41021ae08745Sheppo caddr_t v_align, addr; 41031ae08745Sheppo int i; 41041ae08745Sheppo 41051ae08745Sheppo if (mhandle == NULL) { 41061ae08745Sheppo DWARN(DBG_ALL_LDCS, 41071ae08745Sheppo "ldc_mem_bind_handle: invalid memory handle\n"); 41081ae08745Sheppo return (EINVAL); 41091ae08745Sheppo } 41101ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 41111ae08745Sheppo ldcp = mhdl->ldcp; 41121ae08745Sheppo mtbl = ldcp->mtbl; 41131ae08745Sheppo 41141ae08745Sheppo /* clear count */ 41151ae08745Sheppo *ccount = 0; 41161ae08745Sheppo 41171ae08745Sheppo mutex_enter(&mhdl->lock); 41181ae08745Sheppo 41191ae08745Sheppo if (mhdl->status == LDC_BOUND || mhdl->memseg != NULL) { 41201ae08745Sheppo DWARN(ldcp->id, 41211ae08745Sheppo "ldc_mem_bind_handle: (0x%x) handle already bound\n", 41221ae08745Sheppo mhandle); 41231ae08745Sheppo mutex_exit(&mhdl->lock); 41241ae08745Sheppo return (EINVAL); 41251ae08745Sheppo } 41261ae08745Sheppo 41271ae08745Sheppo /* Force address and size to be 8-byte aligned */ 41281ae08745Sheppo if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 41291ae08745Sheppo DWARN(ldcp->id, 41301ae08745Sheppo "ldc_mem_bind_handle: addr/size is not 8-byte aligned\n"); 41311ae08745Sheppo mutex_exit(&mhdl->lock); 41321ae08745Sheppo return (EINVAL); 41331ae08745Sheppo } 41341ae08745Sheppo 41351ae08745Sheppo /* FUTURE: get the page size, pgsz code, and shift */ 41361ae08745Sheppo pg_size = MMU_PAGESIZE; 41371ae08745Sheppo pg_size_code = page_szc(pg_size); 41381ae08745Sheppo pg_shift = page_get_shift(pg_size_code); 41391ae08745Sheppo pg_mask = ~(pg_size - 1); 41401ae08745Sheppo 41411ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) binding " 41421ae08745Sheppo "va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 41431ae08745Sheppo ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 41441ae08745Sheppo 41451ae08745Sheppo /* aligned VA and its offset */ 41461ae08745Sheppo v_align = (caddr_t)(((uintptr_t)vaddr) & ~(pg_size - 1)); 41471ae08745Sheppo v_offset = ((uintptr_t)vaddr) & (pg_size - 1); 41481ae08745Sheppo 41491ae08745Sheppo npages = (len+v_offset)/pg_size; 41501ae08745Sheppo npages = ((len+v_offset)%pg_size == 0) ? npages : npages+1; 41511ae08745Sheppo 41521ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle: binding " 41531ae08745Sheppo "(0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 41541ae08745Sheppo ldcp->id, vaddr, v_align, v_offset, npages); 41551ae08745Sheppo 41561ae08745Sheppo /* lock the memory table - exclusive access to channel */ 41571ae08745Sheppo mutex_enter(&mtbl->lock); 41581ae08745Sheppo 41591ae08745Sheppo if (npages > mtbl->num_avail) { 41601ae08745Sheppo DWARN(ldcp->id, 41611ae08745Sheppo "ldc_mem_bind_handle: (0x%llx) no table entries\n", 41621ae08745Sheppo ldcp->id); 41631ae08745Sheppo mutex_exit(&mtbl->lock); 41641ae08745Sheppo mutex_exit(&mhdl->lock); 41651ae08745Sheppo return (ENOMEM); 41661ae08745Sheppo } 41671ae08745Sheppo 41681ae08745Sheppo /* Allocate a memseg structure */ 41691ae08745Sheppo memseg = mhdl->memseg = kmem_zalloc(sizeof (ldc_memseg_t), KM_SLEEP); 41701ae08745Sheppo 41711ae08745Sheppo /* Allocate memory to store all pages and cookies */ 41721ae08745Sheppo memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP); 41731ae08745Sheppo memseg->cookies = 41741ae08745Sheppo kmem_zalloc((sizeof (ldc_mem_cookie_t) * npages), KM_SLEEP); 41751ae08745Sheppo 41761ae08745Sheppo D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) processing 0x%llx pages\n", 41771ae08745Sheppo ldcp->id, npages); 41781ae08745Sheppo 41791ae08745Sheppo addr = v_align; 41801ae08745Sheppo 41811ae08745Sheppo /* 41821ae08745Sheppo * Table slots are used in a round-robin manner. The algorithm permits 41831ae08745Sheppo * inserting duplicate entries. Slots allocated earlier will typically 41841ae08745Sheppo * get freed before we get back to reusing the slot.Inserting duplicate 41851ae08745Sheppo * entries should be OK as we only lookup entries using the cookie addr 41861ae08745Sheppo * i.e. tbl index, during export, unexport and copy operation. 41871ae08745Sheppo * 41881ae08745Sheppo * One implementation what was tried was to search for a duplicate 41891ae08745Sheppo * page entry first and reuse it. The search overhead is very high and 41901ae08745Sheppo * in the vnet case dropped the perf by almost half, 50 to 24 mbps. 41911ae08745Sheppo * So it does make sense to avoid searching for duplicates. 41921ae08745Sheppo * 41931ae08745Sheppo * But during the process of searching for a free slot, if we find a 41941ae08745Sheppo * duplicate entry we will go ahead and use it, and bump its use count. 41951ae08745Sheppo */ 41961ae08745Sheppo 41971ae08745Sheppo /* index to start searching from */ 41981ae08745Sheppo index = mtbl->next_entry; 41991ae08745Sheppo cookie_idx = -1; 42001ae08745Sheppo 42011ae08745Sheppo tmp_mte.ll = 0; /* initialise fields to 0 */ 42021ae08745Sheppo 42031ae08745Sheppo if (mtype & LDC_DIRECT_MAP) { 42041ae08745Sheppo tmp_mte.mte_r = (perm & LDC_MEM_R) ? 1 : 0; 42051ae08745Sheppo tmp_mte.mte_w = (perm & LDC_MEM_W) ? 1 : 0; 42061ae08745Sheppo tmp_mte.mte_x = (perm & LDC_MEM_X) ? 1 : 0; 42071ae08745Sheppo } 42081ae08745Sheppo 42091ae08745Sheppo if (mtype & LDC_SHADOW_MAP) { 42101ae08745Sheppo tmp_mte.mte_cr = (perm & LDC_MEM_R) ? 1 : 0; 42111ae08745Sheppo tmp_mte.mte_cw = (perm & LDC_MEM_W) ? 1 : 0; 42121ae08745Sheppo } 42131ae08745Sheppo 42141ae08745Sheppo if (mtype & LDC_IO_MAP) { 42151ae08745Sheppo tmp_mte.mte_ir = (perm & LDC_MEM_R) ? 1 : 0; 42161ae08745Sheppo tmp_mte.mte_iw = (perm & LDC_MEM_W) ? 1 : 0; 42171ae08745Sheppo } 42181ae08745Sheppo 42191ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll); 42201ae08745Sheppo 42211ae08745Sheppo tmp_mte.mte_pgszc = pg_size_code; 42221ae08745Sheppo 42231ae08745Sheppo /* initialize each mem table entry */ 42241ae08745Sheppo for (i = 0; i < npages; i++) { 42251ae08745Sheppo 42261ae08745Sheppo /* check if slot is available in the table */ 42271ae08745Sheppo while (mtbl->table[index].entry.ll != 0) { 42281ae08745Sheppo 42291ae08745Sheppo index = (index + 1) % mtbl->num_entries; 42301ae08745Sheppo 42311ae08745Sheppo if (index == mtbl->next_entry) { 42321ae08745Sheppo /* we have looped around */ 42331ae08745Sheppo DWARN(DBG_ALL_LDCS, 42341ae08745Sheppo "ldc_mem_bind_handle: (0x%llx) cannot find " 42351ae08745Sheppo "entry\n", ldcp->id); 42361ae08745Sheppo *ccount = 0; 42371ae08745Sheppo 42381ae08745Sheppo /* NOTE: free memory, remove previous entries */ 42391ae08745Sheppo /* this shouldnt happen as num_avail was ok */ 42401ae08745Sheppo 42411ae08745Sheppo mutex_exit(&mtbl->lock); 42421ae08745Sheppo mutex_exit(&mhdl->lock); 42431ae08745Sheppo return (ENOMEM); 42441ae08745Sheppo } 42451ae08745Sheppo } 42461ae08745Sheppo 42471ae08745Sheppo /* get the real address */ 42481ae08745Sheppo raddr = va_to_pa((void *)addr); 42491ae08745Sheppo ra_aligned = ((uintptr_t)raddr & pg_mask); 42501ae08745Sheppo 42511ae08745Sheppo /* build the mte */ 42521ae08745Sheppo tmp_mte.mte_rpfn = ra_aligned >> pg_shift; 42531ae08745Sheppo 42541ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll); 42551ae08745Sheppo 42561ae08745Sheppo /* update entry in table */ 42571ae08745Sheppo mtbl->table[index].entry = tmp_mte; 42581ae08745Sheppo 42591ae08745Sheppo D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) stored MTE 0x%llx" 42601ae08745Sheppo " into loc 0x%llx\n", ldcp->id, tmp_mte.ll, index); 42611ae08745Sheppo 42621ae08745Sheppo /* calculate the size and offset for this export range */ 42631ae08745Sheppo if (i == 0) { 42641ae08745Sheppo /* first page */ 42651ae08745Sheppo psize = min((pg_size - v_offset), len); 42661ae08745Sheppo poffset = v_offset; 42671ae08745Sheppo 42681ae08745Sheppo } else if (i == (npages - 1)) { 42691ae08745Sheppo /* last page */ 42701ae08745Sheppo psize = (((uintptr_t)(vaddr + len)) & 42711ae08745Sheppo ((uint64_t)(pg_size-1))); 42721ae08745Sheppo if (psize == 0) 42731ae08745Sheppo psize = pg_size; 42741ae08745Sheppo poffset = 0; 42751ae08745Sheppo 42761ae08745Sheppo } else { 42771ae08745Sheppo /* middle pages */ 42781ae08745Sheppo psize = pg_size; 42791ae08745Sheppo poffset = 0; 42801ae08745Sheppo } 42811ae08745Sheppo 42821ae08745Sheppo /* store entry for this page */ 42831ae08745Sheppo memseg->pages[i].index = index; 42841ae08745Sheppo memseg->pages[i].raddr = raddr; 42851ae08745Sheppo memseg->pages[i].offset = poffset; 42861ae08745Sheppo memseg->pages[i].size = psize; 42871ae08745Sheppo memseg->pages[i].mte = &(mtbl->table[index]); 42881ae08745Sheppo 42891ae08745Sheppo /* create the cookie */ 42901ae08745Sheppo if (i == 0 || (index != prev_index + 1)) { 42911ae08745Sheppo cookie_idx++; 42921ae08745Sheppo memseg->cookies[cookie_idx].addr = 42931ae08745Sheppo IDX2COOKIE(index, pg_size_code, pg_shift); 42941ae08745Sheppo memseg->cookies[cookie_idx].addr |= poffset; 42951ae08745Sheppo memseg->cookies[cookie_idx].size = psize; 42961ae08745Sheppo 42971ae08745Sheppo } else { 42981ae08745Sheppo memseg->cookies[cookie_idx].size += psize; 42991ae08745Sheppo } 43001ae08745Sheppo 43011ae08745Sheppo D1(ldcp->id, "ldc_mem_bind_handle: bound " 43021ae08745Sheppo "(0x%llx) va=0x%llx, idx=0x%llx, " 43031ae08745Sheppo "ra=0x%llx(sz=0x%x,off=0x%x)\n", 43041ae08745Sheppo ldcp->id, addr, index, raddr, psize, poffset); 43051ae08745Sheppo 43061ae08745Sheppo /* decrement number of available entries */ 43071ae08745Sheppo mtbl->num_avail--; 43081ae08745Sheppo 43091ae08745Sheppo /* increment va by page size */ 43101ae08745Sheppo addr += pg_size; 43111ae08745Sheppo 43121ae08745Sheppo /* increment index */ 43131ae08745Sheppo prev_index = index; 43141ae08745Sheppo index = (index + 1) % mtbl->num_entries; 43151ae08745Sheppo 43161ae08745Sheppo /* save the next slot */ 43171ae08745Sheppo mtbl->next_entry = index; 43181ae08745Sheppo } 43191ae08745Sheppo 43201ae08745Sheppo mutex_exit(&mtbl->lock); 43211ae08745Sheppo 43221ae08745Sheppo /* memory handle = bound */ 43231ae08745Sheppo mhdl->mtype = mtype; 43241ae08745Sheppo mhdl->perm = perm; 43251ae08745Sheppo mhdl->status = LDC_BOUND; 43261ae08745Sheppo 43271ae08745Sheppo /* update memseg_t */ 43281ae08745Sheppo memseg->vaddr = vaddr; 43291ae08745Sheppo memseg->raddr = memseg->pages[0].raddr; 43301ae08745Sheppo memseg->size = len; 43311ae08745Sheppo memseg->npages = npages; 43321ae08745Sheppo memseg->ncookies = cookie_idx + 1; 43331ae08745Sheppo memseg->next_cookie = (memseg->ncookies > 1) ? 1 : 0; 43341ae08745Sheppo 43351ae08745Sheppo /* return count and first cookie */ 43361ae08745Sheppo *ccount = memseg->ncookies; 43371ae08745Sheppo cookie->addr = memseg->cookies[0].addr; 43381ae08745Sheppo cookie->size = memseg->cookies[0].size; 43391ae08745Sheppo 43401ae08745Sheppo D1(ldcp->id, 43411ae08745Sheppo "ldc_mem_bind_handle: (0x%llx) bound 0x%llx, va=0x%llx, " 43421ae08745Sheppo "pgs=0x%llx cookies=0x%llx\n", 43431ae08745Sheppo ldcp->id, mhdl, vaddr, npages, memseg->ncookies); 43441ae08745Sheppo 43451ae08745Sheppo mutex_exit(&mhdl->lock); 43461ae08745Sheppo return (0); 43471ae08745Sheppo } 43481ae08745Sheppo 43491ae08745Sheppo /* 43501ae08745Sheppo * Return the next cookie associated with the specified memory handle 43511ae08745Sheppo */ 43521ae08745Sheppo int 43531ae08745Sheppo ldc_mem_nextcookie(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie) 43541ae08745Sheppo { 43551ae08745Sheppo ldc_mhdl_t *mhdl; 43561ae08745Sheppo ldc_chan_t *ldcp; 43571ae08745Sheppo ldc_memseg_t *memseg; 43581ae08745Sheppo 43591ae08745Sheppo if (mhandle == NULL) { 43601ae08745Sheppo DWARN(DBG_ALL_LDCS, 43611ae08745Sheppo "ldc_mem_nextcookie: invalid memory handle\n"); 43621ae08745Sheppo return (EINVAL); 43631ae08745Sheppo } 43641ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 43651ae08745Sheppo 43661ae08745Sheppo mutex_enter(&mhdl->lock); 43671ae08745Sheppo 43681ae08745Sheppo ldcp = mhdl->ldcp; 43691ae08745Sheppo memseg = mhdl->memseg; 43701ae08745Sheppo 43711ae08745Sheppo if (cookie == 0) { 43721ae08745Sheppo DWARN(ldcp->id, 43731ae08745Sheppo "ldc_mem_nextcookie:(0x%llx) invalid cookie arg\n", 43741ae08745Sheppo ldcp->id); 43751ae08745Sheppo mutex_exit(&mhdl->lock); 43761ae08745Sheppo return (EINVAL); 43771ae08745Sheppo } 43781ae08745Sheppo 43791ae08745Sheppo if (memseg->next_cookie != 0) { 43801ae08745Sheppo cookie->addr = memseg->cookies[memseg->next_cookie].addr; 43811ae08745Sheppo cookie->size = memseg->cookies[memseg->next_cookie].size; 43821ae08745Sheppo memseg->next_cookie++; 43831ae08745Sheppo if (memseg->next_cookie == memseg->ncookies) 43841ae08745Sheppo memseg->next_cookie = 0; 43851ae08745Sheppo 43861ae08745Sheppo } else { 43871ae08745Sheppo DWARN(ldcp->id, 43881ae08745Sheppo "ldc_mem_nextcookie:(0x%llx) no more cookies\n", ldcp->id); 43891ae08745Sheppo cookie->addr = 0; 43901ae08745Sheppo cookie->size = 0; 43911ae08745Sheppo mutex_exit(&mhdl->lock); 43921ae08745Sheppo return (EINVAL); 43931ae08745Sheppo } 43941ae08745Sheppo 43951ae08745Sheppo D1(ldcp->id, 43961ae08745Sheppo "ldc_mem_nextcookie: (0x%llx) cookie addr=0x%llx,sz=0x%llx\n", 43971ae08745Sheppo ldcp->id, cookie->addr, cookie->size); 43981ae08745Sheppo 43991ae08745Sheppo mutex_exit(&mhdl->lock); 44001ae08745Sheppo return (0); 44011ae08745Sheppo } 44021ae08745Sheppo 44031ae08745Sheppo /* 44041ae08745Sheppo * Unbind the virtual memory region associated with the specified 44051ae08745Sheppo * memory handle. Allassociated cookies are freed and the corresponding 44061ae08745Sheppo * RA space is no longer exported. 44071ae08745Sheppo */ 44081ae08745Sheppo int 44091ae08745Sheppo ldc_mem_unbind_handle(ldc_mem_handle_t mhandle) 44101ae08745Sheppo { 44111ae08745Sheppo ldc_mhdl_t *mhdl; 44121ae08745Sheppo ldc_chan_t *ldcp; 44131ae08745Sheppo ldc_mtbl_t *mtbl; 44141ae08745Sheppo ldc_memseg_t *memseg; 44151ae08745Sheppo int i; 44161ae08745Sheppo 44171ae08745Sheppo if (mhandle == NULL) { 44181ae08745Sheppo DWARN(DBG_ALL_LDCS, 44191ae08745Sheppo "ldc_mem_unbind_handle: invalid memory handle\n"); 44201ae08745Sheppo return (EINVAL); 44211ae08745Sheppo } 44221ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 44231ae08745Sheppo 44241ae08745Sheppo mutex_enter(&mhdl->lock); 44251ae08745Sheppo 44261ae08745Sheppo if (mhdl->status == LDC_UNBOUND) { 44271ae08745Sheppo DWARN(DBG_ALL_LDCS, 44281ae08745Sheppo "ldc_mem_unbind_handle: (0x%x) handle is not bound\n", 44291ae08745Sheppo mhandle); 44301ae08745Sheppo mutex_exit(&mhdl->lock); 44311ae08745Sheppo return (EINVAL); 44321ae08745Sheppo } 44331ae08745Sheppo 44341ae08745Sheppo ldcp = mhdl->ldcp; 44351ae08745Sheppo mtbl = ldcp->mtbl; 44361ae08745Sheppo 44371ae08745Sheppo memseg = mhdl->memseg; 44381ae08745Sheppo 44391ae08745Sheppo /* lock the memory table - exclusive access to channel */ 44401ae08745Sheppo mutex_enter(&mtbl->lock); 44411ae08745Sheppo 44421ae08745Sheppo /* undo the pages exported */ 44431ae08745Sheppo for (i = 0; i < memseg->npages; i++) { 44441ae08745Sheppo 44451ae08745Sheppo /* FUTURE: check for mapped pages */ 44461ae08745Sheppo if (memseg->pages[i].mte->cookie) { 44471ae08745Sheppo _NOTE(EMPTY) 44481ae08745Sheppo } 44491ae08745Sheppo 44501ae08745Sheppo /* clear the entry from the table */ 44511ae08745Sheppo memseg->pages[i].mte->entry.ll = 0; 44521ae08745Sheppo mtbl->num_avail++; 44531ae08745Sheppo } 44541ae08745Sheppo mutex_exit(&mtbl->lock); 44551ae08745Sheppo 44561ae08745Sheppo /* free the allocated memseg and page structures */ 44571ae08745Sheppo kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages)); 44581ae08745Sheppo kmem_free(memseg->cookies, 44591ae08745Sheppo (sizeof (ldc_mem_cookie_t) * memseg->npages)); 44601ae08745Sheppo kmem_free(memseg, sizeof (ldc_memseg_t)); 44611ae08745Sheppo 44621ae08745Sheppo /* uninitialize the memory handle */ 44631ae08745Sheppo mhdl->memseg = NULL; 44641ae08745Sheppo mhdl->status = LDC_UNBOUND; 44651ae08745Sheppo 44661ae08745Sheppo D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) unbound handle 0x%llx\n", 44671ae08745Sheppo ldcp->id, mhdl); 44681ae08745Sheppo 44691ae08745Sheppo mutex_exit(&mhdl->lock); 44701ae08745Sheppo return (0); 44711ae08745Sheppo } 44721ae08745Sheppo 44731ae08745Sheppo /* 44741ae08745Sheppo * Get information about the dring. The base address of the descriptor 44751ae08745Sheppo * ring along with the type and permission are returned back. 44761ae08745Sheppo */ 44771ae08745Sheppo int 44781ae08745Sheppo ldc_mem_info(ldc_mem_handle_t mhandle, ldc_mem_info_t *minfo) 44791ae08745Sheppo { 44801ae08745Sheppo ldc_mhdl_t *mhdl; 44811ae08745Sheppo 44821ae08745Sheppo if (mhandle == NULL) { 44831ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid memory handle\n"); 44841ae08745Sheppo return (EINVAL); 44851ae08745Sheppo } 44861ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 44871ae08745Sheppo 44881ae08745Sheppo if (minfo == NULL) { 44891ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid args\n"); 44901ae08745Sheppo return (EINVAL); 44911ae08745Sheppo } 44921ae08745Sheppo 44931ae08745Sheppo mutex_enter(&mhdl->lock); 44941ae08745Sheppo 44951ae08745Sheppo minfo->status = mhdl->status; 44961ae08745Sheppo if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) { 44971ae08745Sheppo minfo->vaddr = mhdl->memseg->vaddr; 44981ae08745Sheppo minfo->raddr = mhdl->memseg->raddr; 44991ae08745Sheppo minfo->mtype = mhdl->mtype; 45001ae08745Sheppo minfo->perm = mhdl->perm; 45011ae08745Sheppo } 45021ae08745Sheppo mutex_exit(&mhdl->lock); 45031ae08745Sheppo 45041ae08745Sheppo return (0); 45051ae08745Sheppo } 45061ae08745Sheppo 45071ae08745Sheppo /* 45081ae08745Sheppo * Copy data either from or to the client specified virtual address 45091ae08745Sheppo * space to or from the exported memory associated with the cookies. 45101ae08745Sheppo * The direction argument determines whether the data is read from or 45111ae08745Sheppo * written to exported memory. 45121ae08745Sheppo */ 45131ae08745Sheppo int 45141ae08745Sheppo ldc_mem_copy(ldc_handle_t handle, caddr_t vaddr, uint64_t off, size_t *size, 45151ae08745Sheppo ldc_mem_cookie_t *cookies, uint32_t ccount, uint8_t direction) 45161ae08745Sheppo { 45171ae08745Sheppo ldc_chan_t *ldcp; 45181ae08745Sheppo uint64_t local_voff, local_valign; 45191ae08745Sheppo uint64_t cookie_addr, cookie_size; 45201ae08745Sheppo uint64_t pg_shift, pg_size, pg_size_code; 45211ae08745Sheppo uint64_t export_caddr, export_poff, export_psize, export_size; 45221ae08745Sheppo uint64_t local_ra, local_poff, local_psize; 45231ae08745Sheppo uint64_t copy_size, copied_len = 0, total_bal = 0, idx = 0; 45241ae08745Sheppo pgcnt_t npages; 45251ae08745Sheppo size_t len = *size; 45261ae08745Sheppo int i, rv = 0; 45271ae08745Sheppo 45281ae08745Sheppo if (handle == NULL) { 45291ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_copy: invalid channel handle\n"); 45301ae08745Sheppo return (EINVAL); 45311ae08745Sheppo } 45321ae08745Sheppo ldcp = (ldc_chan_t *)handle; 45331ae08745Sheppo 45341ae08745Sheppo mutex_enter(&ldcp->lock); 45351ae08745Sheppo 45361ae08745Sheppo /* check to see if channel is UP */ 45371ae08745Sheppo if (ldcp->tstate != TS_UP) { 45381ae08745Sheppo DWARN(ldcp->id, "ldc_mem_copy: (0x%llx) channel is not UP\n", 45391ae08745Sheppo ldcp->id); 45401ae08745Sheppo mutex_exit(&ldcp->lock); 45411ae08745Sheppo return (EINVAL); 45421ae08745Sheppo } 45431ae08745Sheppo 45441ae08745Sheppo /* Force address and size to be 8-byte aligned */ 45451ae08745Sheppo if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 45461ae08745Sheppo DWARN(ldcp->id, 45471ae08745Sheppo "ldc_mem_copy: addr/sz is not 8-byte aligned\n"); 45481ae08745Sheppo mutex_exit(&ldcp->lock); 45491ae08745Sheppo return (EINVAL); 45501ae08745Sheppo } 45511ae08745Sheppo 45521ae08745Sheppo /* Find the size of the exported memory */ 45531ae08745Sheppo export_size = 0; 45541ae08745Sheppo for (i = 0; i < ccount; i++) 45551ae08745Sheppo export_size += cookies[i].size; 45561ae08745Sheppo 45571ae08745Sheppo /* check to see if offset is valid */ 45581ae08745Sheppo if (off > export_size) { 45591ae08745Sheppo DWARN(ldcp->id, 45601ae08745Sheppo "ldc_mem_copy: (0x%llx) start offset > export mem size\n", 45611ae08745Sheppo ldcp->id); 45621ae08745Sheppo mutex_exit(&ldcp->lock); 45631ae08745Sheppo return (EINVAL); 45641ae08745Sheppo } 45651ae08745Sheppo 45661ae08745Sheppo /* 45671ae08745Sheppo * Check to see if the export size is smaller than the size we 45681ae08745Sheppo * are requesting to copy - if so flag an error 45691ae08745Sheppo */ 45701ae08745Sheppo if ((export_size - off) < *size) { 45711ae08745Sheppo DWARN(ldcp->id, 45721ae08745Sheppo "ldc_mem_copy: (0x%llx) copy size > export mem size\n", 45731ae08745Sheppo ldcp->id); 45741ae08745Sheppo mutex_exit(&ldcp->lock); 45751ae08745Sheppo return (EINVAL); 45761ae08745Sheppo } 45771ae08745Sheppo 45781ae08745Sheppo total_bal = min(export_size, *size); 45791ae08745Sheppo 45801ae08745Sheppo /* FUTURE: get the page size, pgsz code, and shift */ 45811ae08745Sheppo pg_size = MMU_PAGESIZE; 45821ae08745Sheppo pg_size_code = page_szc(pg_size); 45831ae08745Sheppo pg_shift = page_get_shift(pg_size_code); 45841ae08745Sheppo 45851ae08745Sheppo D1(ldcp->id, "ldc_mem_copy: copying data " 45861ae08745Sheppo "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 45871ae08745Sheppo ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 45881ae08745Sheppo 45891ae08745Sheppo /* aligned VA and its offset */ 45901ae08745Sheppo local_valign = (((uintptr_t)vaddr) & ~(pg_size - 1)); 45911ae08745Sheppo local_voff = ((uintptr_t)vaddr) & (pg_size - 1); 45921ae08745Sheppo 45931ae08745Sheppo npages = (len+local_voff)/pg_size; 45941ae08745Sheppo npages = ((len+local_voff)%pg_size == 0) ? npages : npages+1; 45951ae08745Sheppo 45961ae08745Sheppo D1(ldcp->id, 45971ae08745Sheppo "ldc_mem_copy: (0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 45981ae08745Sheppo ldcp->id, vaddr, local_valign, local_voff, npages); 45991ae08745Sheppo 46001ae08745Sheppo local_ra = va_to_pa((void *)local_valign); 46011ae08745Sheppo local_poff = local_voff; 46021ae08745Sheppo local_psize = min(len, (pg_size - local_voff)); 46031ae08745Sheppo 46041ae08745Sheppo len -= local_psize; 46051ae08745Sheppo 46061ae08745Sheppo /* 46071ae08745Sheppo * find the first cookie in the list of cookies 46081ae08745Sheppo * if the offset passed in is not zero 46091ae08745Sheppo */ 46101ae08745Sheppo for (idx = 0; idx < ccount; idx++) { 46111ae08745Sheppo cookie_size = cookies[idx].size; 46121ae08745Sheppo if (off < cookie_size) 46131ae08745Sheppo break; 46141ae08745Sheppo off -= cookie_size; 46151ae08745Sheppo } 46161ae08745Sheppo 46171ae08745Sheppo cookie_addr = cookies[idx].addr + off; 46181ae08745Sheppo cookie_size = cookies[idx].size - off; 46191ae08745Sheppo 46201ae08745Sheppo export_caddr = cookie_addr & ~(pg_size - 1); 46211ae08745Sheppo export_poff = cookie_addr & (pg_size - 1); 46221ae08745Sheppo export_psize = min(cookie_size, (pg_size - export_poff)); 46231ae08745Sheppo 46241ae08745Sheppo for (;;) { 46251ae08745Sheppo 46261ae08745Sheppo copy_size = min(export_psize, local_psize); 46271ae08745Sheppo 46281ae08745Sheppo D1(ldcp->id, 46291ae08745Sheppo "ldc_mem_copy:(0x%llx) dir=0x%x, caddr=0x%llx," 46301ae08745Sheppo " loc_ra=0x%llx, exp_poff=0x%llx, loc_poff=0x%llx," 46311ae08745Sheppo " exp_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 46321ae08745Sheppo " total_bal=0x%llx\n", 46331ae08745Sheppo ldcp->id, direction, export_caddr, local_ra, export_poff, 46341ae08745Sheppo local_poff, export_psize, local_psize, copy_size, 46351ae08745Sheppo total_bal); 46361ae08745Sheppo 46371ae08745Sheppo rv = hv_ldc_copy(ldcp->id, direction, 46381ae08745Sheppo (export_caddr + export_poff), (local_ra + local_poff), 46391ae08745Sheppo copy_size, &copied_len); 46401ae08745Sheppo 46411ae08745Sheppo if (rv != 0) { 46421ae08745Sheppo cmn_err(CE_WARN, 46431ae08745Sheppo "ldc_mem_copy: (0x%lx) err %d during copy\n", 46441ae08745Sheppo ldcp->id, rv); 46451ae08745Sheppo DWARN(DBG_ALL_LDCS, 46461ae08745Sheppo "ldc_mem_copy: (0x%llx) dir=0x%x, caddr=0x%llx, " 46471ae08745Sheppo "loc_ra=0x%llx, exp_poff=0x%llx, loc_poff=0x%llx," 46481ae08745Sheppo " exp_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 46491ae08745Sheppo " copied_len=0x%llx, total_bal=0x%llx\n", 46501ae08745Sheppo ldcp->id, direction, export_caddr, local_ra, 46511ae08745Sheppo export_poff, local_poff, export_psize, local_psize, 46521ae08745Sheppo copy_size, copied_len, total_bal); 46531ae08745Sheppo 46541ae08745Sheppo *size = *size - total_bal; 46551ae08745Sheppo mutex_exit(&ldcp->lock); 46561ae08745Sheppo return (EIO); 46571ae08745Sheppo } 46581ae08745Sheppo 46591ae08745Sheppo ASSERT(copied_len <= copy_size); 46601ae08745Sheppo 46611ae08745Sheppo D2(ldcp->id, "ldc_mem_copy: copied=0x%llx\n", copied_len); 46621ae08745Sheppo export_poff += copied_len; 46631ae08745Sheppo local_poff += copied_len; 46641ae08745Sheppo export_psize -= copied_len; 46651ae08745Sheppo local_psize -= copied_len; 46661ae08745Sheppo cookie_size -= copied_len; 46671ae08745Sheppo 46681ae08745Sheppo total_bal -= copied_len; 46691ae08745Sheppo 46701ae08745Sheppo if (copy_size != copied_len) 46711ae08745Sheppo continue; 46721ae08745Sheppo 46731ae08745Sheppo if (export_psize == 0 && total_bal != 0) { 46741ae08745Sheppo 46751ae08745Sheppo if (cookie_size == 0) { 46761ae08745Sheppo idx++; 46771ae08745Sheppo cookie_addr = cookies[idx].addr; 46781ae08745Sheppo cookie_size = cookies[idx].size; 46791ae08745Sheppo 46801ae08745Sheppo export_caddr = cookie_addr & ~(pg_size - 1); 46811ae08745Sheppo export_poff = cookie_addr & (pg_size - 1); 46821ae08745Sheppo export_psize = 46831ae08745Sheppo min(cookie_size, (pg_size-export_poff)); 46841ae08745Sheppo } else { 46851ae08745Sheppo export_caddr += pg_size; 46861ae08745Sheppo export_poff = 0; 46871ae08745Sheppo export_psize = min(cookie_size, pg_size); 46881ae08745Sheppo } 46891ae08745Sheppo } 46901ae08745Sheppo 46911ae08745Sheppo if (local_psize == 0 && total_bal != 0) { 46921ae08745Sheppo local_valign += pg_size; 46931ae08745Sheppo local_ra = va_to_pa((void *)local_valign); 46941ae08745Sheppo local_poff = 0; 46951ae08745Sheppo local_psize = min(pg_size, len); 46961ae08745Sheppo len -= local_psize; 46971ae08745Sheppo } 46981ae08745Sheppo 46991ae08745Sheppo /* check if we are all done */ 47001ae08745Sheppo if (total_bal == 0) 47011ae08745Sheppo break; 47021ae08745Sheppo } 47031ae08745Sheppo 47041ae08745Sheppo mutex_exit(&ldcp->lock); 47051ae08745Sheppo 47061ae08745Sheppo D1(ldcp->id, 47071ae08745Sheppo "ldc_mem_copy: (0x%llx) done copying sz=0x%llx\n", 47081ae08745Sheppo ldcp->id, *size); 47091ae08745Sheppo 47101ae08745Sheppo return (0); 47111ae08745Sheppo } 47121ae08745Sheppo 47131ae08745Sheppo /* 47141ae08745Sheppo * Copy data either from or to the client specified virtual address 47151ae08745Sheppo * space to or from HV physical memory. 47161ae08745Sheppo * 47171ae08745Sheppo * The direction argument determines whether the data is read from or 47181ae08745Sheppo * written to HV memory. direction values are LDC_COPY_IN/OUT similar 47191ae08745Sheppo * to the ldc_mem_copy interface 47201ae08745Sheppo */ 47211ae08745Sheppo int 47221ae08745Sheppo ldc_mem_rdwr_pa(ldc_handle_t handle, caddr_t vaddr, size_t *size, 47231ae08745Sheppo caddr_t paddr, uint8_t direction) 47241ae08745Sheppo { 47251ae08745Sheppo ldc_chan_t *ldcp; 47261ae08745Sheppo uint64_t local_voff, local_valign; 47271ae08745Sheppo uint64_t pg_shift, pg_size, pg_size_code; 47281ae08745Sheppo uint64_t target_pa, target_poff, target_psize, target_size; 47291ae08745Sheppo uint64_t local_ra, local_poff, local_psize; 47301ae08745Sheppo uint64_t copy_size, copied_len = 0; 47311ae08745Sheppo pgcnt_t npages; 47321ae08745Sheppo size_t len = *size; 47331ae08745Sheppo int rv = 0; 47341ae08745Sheppo 47351ae08745Sheppo if (handle == NULL) { 47361ae08745Sheppo DWARN(DBG_ALL_LDCS, 47371ae08745Sheppo "ldc_mem_rdwr_pa: invalid channel handle\n"); 47381ae08745Sheppo return (EINVAL); 47391ae08745Sheppo } 47401ae08745Sheppo ldcp = (ldc_chan_t *)handle; 47411ae08745Sheppo 47421ae08745Sheppo mutex_enter(&ldcp->lock); 47431ae08745Sheppo 47441ae08745Sheppo /* check to see if channel is UP */ 47451ae08745Sheppo if (ldcp->tstate != TS_UP) { 47461ae08745Sheppo DWARN(ldcp->id, 47471ae08745Sheppo "ldc_mem_rdwr_pa: (0x%llx) channel is not UP\n", 47481ae08745Sheppo ldcp->id); 47491ae08745Sheppo mutex_exit(&ldcp->lock); 47501ae08745Sheppo return (EINVAL); 47511ae08745Sheppo } 47521ae08745Sheppo 47531ae08745Sheppo /* Force address and size to be 8-byte aligned */ 47541ae08745Sheppo if ((((uintptr_t)vaddr | len) & 0x7) != 0) { 47551ae08745Sheppo DWARN(ldcp->id, 47561ae08745Sheppo "ldc_mem_rdwr_pa: addr/size is not 8-byte aligned\n"); 47571ae08745Sheppo mutex_exit(&ldcp->lock); 47581ae08745Sheppo return (EINVAL); 47591ae08745Sheppo } 47601ae08745Sheppo 47611ae08745Sheppo target_size = *size; 47621ae08745Sheppo 47631ae08745Sheppo /* FUTURE: get the page size, pgsz code, and shift */ 47641ae08745Sheppo pg_size = MMU_PAGESIZE; 47651ae08745Sheppo pg_size_code = page_szc(pg_size); 47661ae08745Sheppo pg_shift = page_get_shift(pg_size_code); 47671ae08745Sheppo 47681ae08745Sheppo D1(ldcp->id, "ldc_mem_rdwr_pa: copying data " 47691ae08745Sheppo "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n", 47701ae08745Sheppo ldcp->id, vaddr, pg_size, pg_size_code, pg_shift); 47711ae08745Sheppo 47721ae08745Sheppo /* aligned VA and its offset */ 47731ae08745Sheppo local_valign = ((uintptr_t)vaddr) & ~(pg_size - 1); 47741ae08745Sheppo local_voff = ((uintptr_t)vaddr) & (pg_size - 1); 47751ae08745Sheppo 47761ae08745Sheppo npages = (len + local_voff) / pg_size; 47771ae08745Sheppo npages = ((len + local_voff) % pg_size == 0) ? npages : npages+1; 47781ae08745Sheppo 47791ae08745Sheppo D1(ldcp->id, 47801ae08745Sheppo "ldc_mem_rdwr_pa: (0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n", 47811ae08745Sheppo ldcp->id, vaddr, local_valign, local_voff, npages); 47821ae08745Sheppo 47831ae08745Sheppo local_ra = va_to_pa((void *)local_valign); 47841ae08745Sheppo local_poff = local_voff; 47851ae08745Sheppo local_psize = min(len, (pg_size - local_voff)); 47861ae08745Sheppo 47871ae08745Sheppo len -= local_psize; 47881ae08745Sheppo 47891ae08745Sheppo target_pa = ((uintptr_t)paddr) & ~(pg_size - 1); 47901ae08745Sheppo target_poff = ((uintptr_t)paddr) & (pg_size - 1); 47911ae08745Sheppo target_psize = pg_size - target_poff; 47921ae08745Sheppo 47931ae08745Sheppo for (;;) { 47941ae08745Sheppo 47951ae08745Sheppo copy_size = min(target_psize, local_psize); 47961ae08745Sheppo 47971ae08745Sheppo D1(ldcp->id, 47981ae08745Sheppo "ldc_mem_rdwr_pa: (0x%llx) dir=0x%x, tar_pa=0x%llx," 47991ae08745Sheppo " loc_ra=0x%llx, tar_poff=0x%llx, loc_poff=0x%llx," 48001ae08745Sheppo " tar_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 48011ae08745Sheppo " total_bal=0x%llx\n", 48021ae08745Sheppo ldcp->id, direction, target_pa, local_ra, target_poff, 48031ae08745Sheppo local_poff, target_psize, local_psize, copy_size, 48041ae08745Sheppo target_size); 48051ae08745Sheppo 48061ae08745Sheppo rv = hv_ldc_copy(ldcp->id, direction, 48071ae08745Sheppo (target_pa + target_poff), (local_ra + local_poff), 48081ae08745Sheppo copy_size, &copied_len); 48091ae08745Sheppo 48101ae08745Sheppo if (rv != 0) { 48111ae08745Sheppo cmn_err(CE_WARN, 48121ae08745Sheppo "ldc_mem_rdwr_pa: (0x%lx) err %d during copy\n", 48131ae08745Sheppo ldcp->id, rv); 48141ae08745Sheppo DWARN(DBG_ALL_LDCS, 48151ae08745Sheppo "ldc_mem_rdwr_pa: (0x%llx) dir=%lld,tar_pa=0x%llx, " 48161ae08745Sheppo "loc_ra=0x%llx, tar_poff=0x%llx, loc_poff=0x%llx," 48171ae08745Sheppo " tar_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx," 48181ae08745Sheppo " total_bal=0x%llx\n", 48191ae08745Sheppo ldcp->id, direction, target_pa, local_ra, 48201ae08745Sheppo target_poff, local_poff, target_psize, local_psize, 48211ae08745Sheppo copy_size, target_size); 48221ae08745Sheppo 48231ae08745Sheppo *size = *size - target_size; 48241ae08745Sheppo mutex_exit(&ldcp->lock); 48251ae08745Sheppo return (i_ldc_h2v_error(rv)); 48261ae08745Sheppo } 48271ae08745Sheppo 48281ae08745Sheppo D2(ldcp->id, "ldc_mem_rdwr_pa: copied=0x%llx\n", copied_len); 48291ae08745Sheppo target_poff += copied_len; 48301ae08745Sheppo local_poff += copied_len; 48311ae08745Sheppo target_psize -= copied_len; 48321ae08745Sheppo local_psize -= copied_len; 48331ae08745Sheppo 48341ae08745Sheppo target_size -= copied_len; 48351ae08745Sheppo 48361ae08745Sheppo if (copy_size != copied_len) 48371ae08745Sheppo continue; 48381ae08745Sheppo 48391ae08745Sheppo if (target_psize == 0 && target_size != 0) { 48401ae08745Sheppo target_pa += pg_size; 48411ae08745Sheppo target_poff = 0; 48421ae08745Sheppo target_psize = min(pg_size, target_size); 48431ae08745Sheppo } 48441ae08745Sheppo 48451ae08745Sheppo if (local_psize == 0 && target_size != 0) { 48461ae08745Sheppo local_valign += pg_size; 48471ae08745Sheppo local_ra = va_to_pa((void *)local_valign); 48481ae08745Sheppo local_poff = 0; 48491ae08745Sheppo local_psize = min(pg_size, len); 48501ae08745Sheppo len -= local_psize; 48511ae08745Sheppo } 48521ae08745Sheppo 48531ae08745Sheppo /* check if we are all done */ 48541ae08745Sheppo if (target_size == 0) 48551ae08745Sheppo break; 48561ae08745Sheppo } 48571ae08745Sheppo 48581ae08745Sheppo mutex_exit(&ldcp->lock); 48591ae08745Sheppo 48601ae08745Sheppo D1(ldcp->id, "ldc_mem_rdwr_pa: (0x%llx) done copying sz=0x%llx\n", 48611ae08745Sheppo ldcp->id, *size); 48621ae08745Sheppo 48631ae08745Sheppo return (0); 48641ae08745Sheppo } 48651ae08745Sheppo 48661ae08745Sheppo /* 48671ae08745Sheppo * Map an exported memory segment into the local address space. If the 48681ae08745Sheppo * memory range was exported for direct map access, a HV call is made 48691ae08745Sheppo * to allocate a RA range. If the map is done via a shadow copy, local 48701ae08745Sheppo * shadow memory is allocated and the base VA is returned in 'vaddr'. If 48711ae08745Sheppo * the mapping is a direct map then the RA is returned in 'raddr'. 48721ae08745Sheppo */ 48731ae08745Sheppo int 48741ae08745Sheppo ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie, uint32_t ccount, 48751ae08745Sheppo uint8_t mtype, caddr_t *vaddr, caddr_t *raddr) 48761ae08745Sheppo { 48771ae08745Sheppo int i, idx; 48781ae08745Sheppo ldc_chan_t *ldcp; 48791ae08745Sheppo ldc_mhdl_t *mhdl; 48801ae08745Sheppo ldc_memseg_t *memseg; 48811ae08745Sheppo caddr_t shadow_base = NULL, tmpaddr; 48821ae08745Sheppo uint64_t pg_size, pg_shift, pg_size_code; 48831ae08745Sheppo uint64_t exp_size = 0, npages; 48841ae08745Sheppo 48851ae08745Sheppo if (mhandle == NULL) { 48861ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_map: invalid memory handle\n"); 48871ae08745Sheppo return (EINVAL); 48881ae08745Sheppo } 48891ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 48901ae08745Sheppo 48911ae08745Sheppo mutex_enter(&mhdl->lock); 48921ae08745Sheppo 48931ae08745Sheppo if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED || 48941ae08745Sheppo mhdl->memseg != NULL) { 48951ae08745Sheppo DWARN(DBG_ALL_LDCS, 48961ae08745Sheppo "ldc_mem_map: (0x%llx) handle bound/mapped\n", mhandle); 48971ae08745Sheppo mutex_exit(&mhdl->lock); 48981ae08745Sheppo return (EINVAL); 48991ae08745Sheppo } 49001ae08745Sheppo 49011ae08745Sheppo ldcp = mhdl->ldcp; 49021ae08745Sheppo 49031ae08745Sheppo mutex_enter(&ldcp->lock); 49041ae08745Sheppo 49051ae08745Sheppo if (ldcp->tstate != TS_UP) { 49061ae08745Sheppo DWARN(ldcp->id, 49071ae08745Sheppo "ldc_mem_dring_map: (0x%llx) channel is not UP\n", 49081ae08745Sheppo ldcp->id); 49091ae08745Sheppo mutex_exit(&ldcp->lock); 49101ae08745Sheppo mutex_exit(&mhdl->lock); 49111ae08745Sheppo return (EINVAL); 49121ae08745Sheppo } 49131ae08745Sheppo 49141ae08745Sheppo if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) { 49151ae08745Sheppo DWARN(ldcp->id, "ldc_mem_map: invalid map type\n"); 49161ae08745Sheppo mutex_exit(&ldcp->lock); 49171ae08745Sheppo mutex_exit(&mhdl->lock); 49181ae08745Sheppo return (EINVAL); 49191ae08745Sheppo } 49201ae08745Sheppo 49211ae08745Sheppo if (mtype == LDC_SHADOW_MAP && vaddr == NULL) { 49221ae08745Sheppo DWARN(ldcp->id, 49231ae08745Sheppo "ldc_mem_map: invalid vaddr arg0x%llx\n", vaddr); 49241ae08745Sheppo mutex_exit(&ldcp->lock); 49251ae08745Sheppo mutex_exit(&mhdl->lock); 49261ae08745Sheppo return (EINVAL); 49271ae08745Sheppo } 49281ae08745Sheppo 49291ae08745Sheppo if (mtype == LDC_SHADOW_MAP && 49301ae08745Sheppo (vaddr) && ((uintptr_t)(*vaddr) & MMU_PAGEOFFSET)) { 49311ae08745Sheppo DWARN(ldcp->id, 49321ae08745Sheppo "ldc_mem_map: vaddr not page aligned, 0x%llx\n", *vaddr); 49331ae08745Sheppo mutex_exit(&ldcp->lock); 49341ae08745Sheppo mutex_exit(&mhdl->lock); 49351ae08745Sheppo return (EINVAL); 49361ae08745Sheppo } 49371ae08745Sheppo 49381ae08745Sheppo D1(ldcp->id, "ldc_mem_map: (0x%llx) cookie = 0x%llx,0x%llx\n", 4939d10e4ef2Snarayan ldcp->id, cookie->addr, cookie->size); 49401ae08745Sheppo 49411ae08745Sheppo /* FUTURE: get the page size, pgsz code, and shift */ 49421ae08745Sheppo pg_size = MMU_PAGESIZE; 49431ae08745Sheppo pg_size_code = page_szc(pg_size); 49441ae08745Sheppo pg_shift = page_get_shift(pg_size_code); 49451ae08745Sheppo 49461ae08745Sheppo /* calculate the number of pages in the exported cookie */ 49471ae08745Sheppo for (idx = 0; idx < ccount; idx++) { 49481ae08745Sheppo if (cookie[idx].addr & MMU_PAGEOFFSET || 49491ae08745Sheppo cookie[idx].size & MMU_PAGEOFFSET) { 49501ae08745Sheppo DWARN(ldcp->id, 49511ae08745Sheppo "ldc_mem_map: cookie addr/size not page aligned, " 49521ae08745Sheppo "0x%llx\n", cookie[idx].addr); 49531ae08745Sheppo mutex_exit(&ldcp->lock); 49541ae08745Sheppo mutex_exit(&mhdl->lock); 49551ae08745Sheppo return (EINVAL); 49561ae08745Sheppo } 49571ae08745Sheppo exp_size += cookie[idx].size; 49581ae08745Sheppo } 49591ae08745Sheppo npages = (exp_size >> pg_shift); 49601ae08745Sheppo 49611ae08745Sheppo /* Allocate memseg structure */ 49621ae08745Sheppo memseg = mhdl->memseg = kmem_zalloc(sizeof (ldc_memseg_t), KM_SLEEP); 49631ae08745Sheppo 49641ae08745Sheppo /* Allocate memory to store all pages and cookies */ 49651ae08745Sheppo memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP); 49661ae08745Sheppo memseg->cookies = 49671ae08745Sheppo kmem_zalloc((sizeof (ldc_mem_cookie_t) * ccount), KM_SLEEP); 49681ae08745Sheppo 49691ae08745Sheppo D2(ldcp->id, "ldc_mem_map: (0x%llx) processing 0x%llx pages\n", 49701ae08745Sheppo ldcp->id, npages); 49711ae08745Sheppo 49721ae08745Sheppo /* Check to see if the client is requesting direct or shadow map */ 49731ae08745Sheppo if (mtype == LDC_SHADOW_MAP) { 49741ae08745Sheppo if (*vaddr == NULL) { 49751ae08745Sheppo shadow_base = 49761ae08745Sheppo contig_mem_alloc_align(exp_size, PAGESIZE); 49771ae08745Sheppo if (shadow_base == NULL) { 49781ae08745Sheppo cmn_err(CE_WARN, "ldc_mem_map: shadow memory " 49791ae08745Sheppo "allocation failed\n"); 49801ae08745Sheppo kmem_free(memseg->cookies, 49811ae08745Sheppo (sizeof (ldc_mem_cookie_t) * ccount)); 49821ae08745Sheppo kmem_free(memseg->pages, 49831ae08745Sheppo (sizeof (ldc_page_t) * npages)); 49841ae08745Sheppo kmem_free(memseg, sizeof (ldc_memseg_t)); 49851ae08745Sheppo mutex_exit(&ldcp->lock); 49861ae08745Sheppo mutex_exit(&mhdl->lock); 49871ae08745Sheppo return (ENOMEM); 49881ae08745Sheppo } 49891ae08745Sheppo 49901ae08745Sheppo bzero(shadow_base, exp_size); 49911ae08745Sheppo mhdl->myshadow = B_TRUE; 49921ae08745Sheppo 49931ae08745Sheppo D1(ldcp->id, "ldc_mem_map: (0x%llx) allocated " 49941ae08745Sheppo "shadow page va=0x%llx\n", ldcp->id, shadow_base); 49951ae08745Sheppo } else { 49961ae08745Sheppo /* 49971ae08745Sheppo * Use client supplied memory for shadow_base 49981ae08745Sheppo * WARNING: assuming that client mem is >= exp_size 49991ae08745Sheppo */ 50001ae08745Sheppo shadow_base = *vaddr; 50011ae08745Sheppo } 50021ae08745Sheppo } else if (mtype == LDC_DIRECT_MAP) { 50031ae08745Sheppo /* FUTURE: Do a direct map by calling into HV */ 50041ae08745Sheppo _NOTE(EMPTY) 50051ae08745Sheppo } 50061ae08745Sheppo 50071ae08745Sheppo /* Save all page and cookie information */ 50081ae08745Sheppo for (i = 0, tmpaddr = shadow_base; i < npages; i++) { 50091ae08745Sheppo memseg->pages[i].raddr = va_to_pa(tmpaddr); 50101ae08745Sheppo memseg->pages[i].size = pg_size; 50111ae08745Sheppo memseg->pages[i].index = 0; 50121ae08745Sheppo memseg->pages[i].offset = 0; 50131ae08745Sheppo memseg->pages[i].mte = NULL; 50141ae08745Sheppo tmpaddr += pg_size; 50151ae08745Sheppo } 50161ae08745Sheppo for (i = 0; i < ccount; i++) { 50171ae08745Sheppo memseg->cookies[i].addr = cookie[i].addr; 50181ae08745Sheppo memseg->cookies[i].size = cookie[i].size; 50191ae08745Sheppo } 50201ae08745Sheppo 50211ae08745Sheppo /* update memseg_t */ 50221ae08745Sheppo memseg->vaddr = shadow_base; 50231ae08745Sheppo memseg->raddr = memseg->pages[0].raddr; 50241ae08745Sheppo memseg->size = exp_size; 50251ae08745Sheppo memseg->npages = npages; 50261ae08745Sheppo memseg->ncookies = ccount; 50271ae08745Sheppo memseg->next_cookie = 0; 50281ae08745Sheppo 50291ae08745Sheppo /* memory handle = mapped */ 50301ae08745Sheppo mhdl->mtype = mtype; 50311ae08745Sheppo mhdl->perm = 0; 50321ae08745Sheppo mhdl->status = LDC_MAPPED; 50331ae08745Sheppo 50341ae08745Sheppo D1(ldcp->id, "ldc_mem_map: (0x%llx) mapped 0x%llx, ra=0x%llx, " 50351ae08745Sheppo "va=0x%llx, pgs=0x%llx cookies=0x%llx\n", 50361ae08745Sheppo ldcp->id, mhdl, memseg->raddr, memseg->vaddr, 50371ae08745Sheppo memseg->npages, memseg->ncookies); 50381ae08745Sheppo 50391ae08745Sheppo if (raddr) 50401ae08745Sheppo *raddr = (caddr_t)memseg->raddr; 50411ae08745Sheppo if (vaddr) 50421ae08745Sheppo *vaddr = memseg->vaddr; 50431ae08745Sheppo 50441ae08745Sheppo mutex_exit(&ldcp->lock); 50451ae08745Sheppo mutex_exit(&mhdl->lock); 50461ae08745Sheppo return (0); 50471ae08745Sheppo } 50481ae08745Sheppo 50491ae08745Sheppo /* 50501ae08745Sheppo * Unmap a memory segment. Free shadow memory (if any). 50511ae08745Sheppo */ 50521ae08745Sheppo int 50531ae08745Sheppo ldc_mem_unmap(ldc_mem_handle_t mhandle) 50541ae08745Sheppo { 50551ae08745Sheppo ldc_mhdl_t *mhdl = (ldc_mhdl_t *)mhandle; 50561ae08745Sheppo ldc_chan_t *ldcp; 50571ae08745Sheppo ldc_memseg_t *memseg; 50581ae08745Sheppo 50591ae08745Sheppo if (mhdl == 0 || mhdl->status != LDC_MAPPED) { 50601ae08745Sheppo DWARN(DBG_ALL_LDCS, 50611ae08745Sheppo "ldc_mem_unmap: (0x%llx) handle is not mapped\n", 50621ae08745Sheppo mhandle); 50631ae08745Sheppo return (EINVAL); 50641ae08745Sheppo } 50651ae08745Sheppo 50661ae08745Sheppo mutex_enter(&mhdl->lock); 50671ae08745Sheppo 50681ae08745Sheppo ldcp = mhdl->ldcp; 50691ae08745Sheppo memseg = mhdl->memseg; 50701ae08745Sheppo 50711ae08745Sheppo D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapping handle 0x%llx\n", 50721ae08745Sheppo ldcp->id, mhdl); 50731ae08745Sheppo 50741ae08745Sheppo /* if we allocated shadow memory - free it */ 50751ae08745Sheppo if (mhdl->mtype == LDC_SHADOW_MAP && mhdl->myshadow) { 50761ae08745Sheppo contig_mem_free(memseg->vaddr, memseg->size); 50771ae08745Sheppo } 50781ae08745Sheppo 50791ae08745Sheppo /* free the allocated memseg and page structures */ 50801ae08745Sheppo kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages)); 50811ae08745Sheppo kmem_free(memseg->cookies, 50821ae08745Sheppo (sizeof (ldc_mem_cookie_t) * memseg->ncookies)); 50831ae08745Sheppo kmem_free(memseg, sizeof (ldc_memseg_t)); 50841ae08745Sheppo 50851ae08745Sheppo /* uninitialize the memory handle */ 50861ae08745Sheppo mhdl->memseg = NULL; 50871ae08745Sheppo mhdl->status = LDC_UNBOUND; 50881ae08745Sheppo 50891ae08745Sheppo D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapped handle 0x%llx\n", 50901ae08745Sheppo ldcp->id, mhdl); 50911ae08745Sheppo 50921ae08745Sheppo mutex_exit(&mhdl->lock); 50931ae08745Sheppo return (0); 50941ae08745Sheppo } 50951ae08745Sheppo 50961ae08745Sheppo /* 50971ae08745Sheppo * Internal entry point for LDC mapped memory entry consistency 50981ae08745Sheppo * semantics. Acquire copies the contents of the remote memory 50991ae08745Sheppo * into the local shadow copy. The release operation copies the local 51001ae08745Sheppo * contents into the remote memory. The offset and size specify the 51011ae08745Sheppo * bounds for the memory range being synchronized. 51021ae08745Sheppo */ 51031ae08745Sheppo static int 51041ae08745Sheppo i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, uint8_t direction, 51051ae08745Sheppo uint64_t offset, size_t size) 51061ae08745Sheppo { 51071ae08745Sheppo int err; 51081ae08745Sheppo ldc_mhdl_t *mhdl; 51091ae08745Sheppo ldc_chan_t *ldcp; 51101ae08745Sheppo ldc_memseg_t *memseg; 51111ae08745Sheppo caddr_t local_vaddr; 51121ae08745Sheppo size_t copy_size; 51131ae08745Sheppo 51141ae08745Sheppo if (mhandle == NULL) { 51151ae08745Sheppo DWARN(DBG_ALL_LDCS, 51161ae08745Sheppo "i_ldc_mem_acquire_release: invalid memory handle\n"); 51171ae08745Sheppo return (EINVAL); 51181ae08745Sheppo } 51191ae08745Sheppo mhdl = (ldc_mhdl_t *)mhandle; 51201ae08745Sheppo 51211ae08745Sheppo mutex_enter(&mhdl->lock); 51221ae08745Sheppo 51231ae08745Sheppo if (mhdl->status != LDC_MAPPED || mhdl->ldcp == NULL) { 51241ae08745Sheppo DWARN(DBG_ALL_LDCS, 51251ae08745Sheppo "i_ldc_mem_acquire_release: not mapped memory\n"); 51261ae08745Sheppo mutex_exit(&mhdl->lock); 51271ae08745Sheppo return (EINVAL); 51281ae08745Sheppo } 51291ae08745Sheppo 51301ae08745Sheppo if (offset >= mhdl->memseg->size || 51311ae08745Sheppo (offset + size) > mhdl->memseg->size) { 51321ae08745Sheppo DWARN(DBG_ALL_LDCS, 51331ae08745Sheppo "i_ldc_mem_acquire_release: memory out of range\n"); 51341ae08745Sheppo mutex_exit(&mhdl->lock); 51351ae08745Sheppo return (EINVAL); 51361ae08745Sheppo } 51371ae08745Sheppo 51381ae08745Sheppo /* get the channel handle and memory segment */ 51391ae08745Sheppo ldcp = mhdl->ldcp; 51401ae08745Sheppo memseg = mhdl->memseg; 51411ae08745Sheppo 51421ae08745Sheppo if (mhdl->mtype == LDC_SHADOW_MAP) { 51431ae08745Sheppo 51441ae08745Sheppo local_vaddr = memseg->vaddr + offset; 51451ae08745Sheppo copy_size = size; 51461ae08745Sheppo 51471ae08745Sheppo /* copy to/from remote from/to local memory */ 51481ae08745Sheppo err = ldc_mem_copy((ldc_handle_t)ldcp, local_vaddr, offset, 51491ae08745Sheppo ©_size, memseg->cookies, memseg->ncookies, 51501ae08745Sheppo direction); 51511ae08745Sheppo if (err || copy_size != size) { 51521ae08745Sheppo cmn_err(CE_WARN, 51531ae08745Sheppo "i_ldc_mem_acquire_release: copy failed\n"); 51541ae08745Sheppo mutex_exit(&mhdl->lock); 51551ae08745Sheppo return (err); 51561ae08745Sheppo } 51571ae08745Sheppo } 51581ae08745Sheppo 51591ae08745Sheppo mutex_exit(&mhdl->lock); 51601ae08745Sheppo 51611ae08745Sheppo return (0); 51621ae08745Sheppo } 51631ae08745Sheppo 51641ae08745Sheppo /* 51651ae08745Sheppo * Ensure that the contents in the remote memory seg are consistent 51661ae08745Sheppo * with the contents if of local segment 51671ae08745Sheppo */ 51681ae08745Sheppo int 51691ae08745Sheppo ldc_mem_acquire(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size) 51701ae08745Sheppo { 51711ae08745Sheppo return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_IN, offset, size)); 51721ae08745Sheppo } 51731ae08745Sheppo 51741ae08745Sheppo 51751ae08745Sheppo /* 51761ae08745Sheppo * Ensure that the contents in the local memory seg are consistent 51771ae08745Sheppo * with the contents if of remote segment 51781ae08745Sheppo */ 51791ae08745Sheppo int 51801ae08745Sheppo ldc_mem_release(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size) 51811ae08745Sheppo { 51821ae08745Sheppo return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_OUT, offset, size)); 51831ae08745Sheppo } 51841ae08745Sheppo 51851ae08745Sheppo /* 51861ae08745Sheppo * Allocate a descriptor ring. The size of each each descriptor 51871ae08745Sheppo * must be 8-byte aligned and the entire ring should be a multiple 51881ae08745Sheppo * of MMU_PAGESIZE. 51891ae08745Sheppo */ 51901ae08745Sheppo int 51911ae08745Sheppo ldc_mem_dring_create(uint32_t len, uint32_t dsize, ldc_dring_handle_t *dhandle) 51921ae08745Sheppo { 51931ae08745Sheppo ldc_dring_t *dringp; 51941ae08745Sheppo size_t size = (dsize * len); 51951ae08745Sheppo 51961ae08745Sheppo D1(DBG_ALL_LDCS, "ldc_mem_dring_create: len=0x%x, size=0x%x\n", 51971ae08745Sheppo len, dsize); 51981ae08745Sheppo 51991ae08745Sheppo if (dhandle == NULL) { 52001ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid dhandle\n"); 52011ae08745Sheppo return (EINVAL); 52021ae08745Sheppo } 52031ae08745Sheppo 52041ae08745Sheppo if (len == 0) { 52051ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid length\n"); 52061ae08745Sheppo return (EINVAL); 52071ae08745Sheppo } 52081ae08745Sheppo 52091ae08745Sheppo /* descriptor size should be 8-byte aligned */ 52101ae08745Sheppo if (dsize == 0 || (dsize & 0x7)) { 52111ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid size\n"); 52121ae08745Sheppo return (EINVAL); 52131ae08745Sheppo } 52141ae08745Sheppo 52151ae08745Sheppo *dhandle = 0; 52161ae08745Sheppo 52171ae08745Sheppo /* Allocate a desc ring structure */ 52181ae08745Sheppo dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP); 52191ae08745Sheppo 52201ae08745Sheppo /* Initialize dring */ 52211ae08745Sheppo dringp->length = len; 52221ae08745Sheppo dringp->dsize = dsize; 52231ae08745Sheppo 52241ae08745Sheppo /* round off to multiple of pagesize */ 52251ae08745Sheppo dringp->size = (size & MMU_PAGEMASK); 52261ae08745Sheppo if (size & MMU_PAGEOFFSET) 52271ae08745Sheppo dringp->size += MMU_PAGESIZE; 52281ae08745Sheppo 52291ae08745Sheppo dringp->status = LDC_UNBOUND; 52301ae08745Sheppo 52311ae08745Sheppo /* allocate descriptor ring memory */ 52321ae08745Sheppo dringp->base = contig_mem_alloc_align(dringp->size, PAGESIZE); 52331ae08745Sheppo if (dringp->base == NULL) { 52341ae08745Sheppo cmn_err(CE_WARN, 52351ae08745Sheppo "ldc_mem_dring_create: unable to alloc desc\n"); 52361ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 52371ae08745Sheppo return (ENOMEM); 52381ae08745Sheppo } 52391ae08745Sheppo 52401ae08745Sheppo bzero(dringp->base, dringp->size); 52411ae08745Sheppo 52421ae08745Sheppo /* initialize the desc ring lock */ 52431ae08745Sheppo mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL); 52441ae08745Sheppo 52451ae08745Sheppo /* Add descriptor ring to the head of global list */ 52461ae08745Sheppo mutex_enter(&ldcssp->lock); 52471ae08745Sheppo dringp->next = ldcssp->dring_list; 52481ae08745Sheppo ldcssp->dring_list = dringp; 52491ae08745Sheppo mutex_exit(&ldcssp->lock); 52501ae08745Sheppo 52511ae08745Sheppo *dhandle = (ldc_dring_handle_t)dringp; 52521ae08745Sheppo 52531ae08745Sheppo D1(DBG_ALL_LDCS, "ldc_mem_dring_create: dring allocated\n"); 52541ae08745Sheppo 52551ae08745Sheppo return (0); 52561ae08745Sheppo } 52571ae08745Sheppo 52581ae08745Sheppo 52591ae08745Sheppo /* 52601ae08745Sheppo * Destroy a descriptor ring. 52611ae08745Sheppo */ 52621ae08745Sheppo int 52631ae08745Sheppo ldc_mem_dring_destroy(ldc_dring_handle_t dhandle) 52641ae08745Sheppo { 52651ae08745Sheppo ldc_dring_t *dringp; 52661ae08745Sheppo ldc_dring_t *tmp_dringp; 52671ae08745Sheppo 52681ae08745Sheppo D1(DBG_ALL_LDCS, "ldc_mem_dring_destroy: entered\n"); 52691ae08745Sheppo 52701ae08745Sheppo if (dhandle == NULL) { 52711ae08745Sheppo DWARN(DBG_ALL_LDCS, 52721ae08745Sheppo "ldc_mem_dring_destroy: invalid desc ring handle\n"); 52731ae08745Sheppo return (EINVAL); 52741ae08745Sheppo } 52751ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 52761ae08745Sheppo 52771ae08745Sheppo if (dringp->status == LDC_BOUND) { 52781ae08745Sheppo DWARN(DBG_ALL_LDCS, 52791ae08745Sheppo "ldc_mem_dring_destroy: desc ring is bound\n"); 52801ae08745Sheppo return (EACCES); 52811ae08745Sheppo } 52821ae08745Sheppo 52831ae08745Sheppo mutex_enter(&dringp->lock); 52841ae08745Sheppo mutex_enter(&ldcssp->lock); 52851ae08745Sheppo 52861ae08745Sheppo /* remove from linked list - if not bound */ 52871ae08745Sheppo tmp_dringp = ldcssp->dring_list; 52881ae08745Sheppo if (tmp_dringp == dringp) { 52891ae08745Sheppo ldcssp->dring_list = dringp->next; 52901ae08745Sheppo dringp->next = NULL; 52911ae08745Sheppo 52921ae08745Sheppo } else { 52931ae08745Sheppo while (tmp_dringp != NULL) { 52941ae08745Sheppo if (tmp_dringp->next == dringp) { 52951ae08745Sheppo tmp_dringp->next = dringp->next; 52961ae08745Sheppo dringp->next = NULL; 52971ae08745Sheppo break; 52981ae08745Sheppo } 52991ae08745Sheppo tmp_dringp = tmp_dringp->next; 53001ae08745Sheppo } 53011ae08745Sheppo if (tmp_dringp == NULL) { 53021ae08745Sheppo DWARN(DBG_ALL_LDCS, 53031ae08745Sheppo "ldc_mem_dring_destroy: invalid descriptor\n"); 53041ae08745Sheppo mutex_exit(&ldcssp->lock); 53051ae08745Sheppo mutex_exit(&dringp->lock); 53061ae08745Sheppo return (EINVAL); 53071ae08745Sheppo } 53081ae08745Sheppo } 53091ae08745Sheppo 53101ae08745Sheppo mutex_exit(&ldcssp->lock); 53111ae08745Sheppo 53121ae08745Sheppo /* free the descriptor ring */ 53131ae08745Sheppo contig_mem_free((caddr_t)dringp->base, dringp->size); 53141ae08745Sheppo 53151ae08745Sheppo mutex_exit(&dringp->lock); 53161ae08745Sheppo 53171ae08745Sheppo /* destroy dring lock */ 53181ae08745Sheppo mutex_destroy(&dringp->lock); 53191ae08745Sheppo 53201ae08745Sheppo /* free desc ring object */ 53211ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 53221ae08745Sheppo 53231ae08745Sheppo return (0); 53241ae08745Sheppo } 53251ae08745Sheppo 53261ae08745Sheppo /* 53271ae08745Sheppo * Bind a previously allocated dring to a channel. The channel should 53281ae08745Sheppo * be OPEN in order to bind the ring to the channel. Returns back a 53291ae08745Sheppo * descriptor ring cookie. The descriptor ring is exported for remote 53301ae08745Sheppo * access by the client at the other end of the channel. An entry for 53311ae08745Sheppo * dring pages is stored in map table (via call to ldc_mem_bind_handle). 53321ae08745Sheppo */ 53331ae08745Sheppo int 53341ae08745Sheppo ldc_mem_dring_bind(ldc_handle_t handle, ldc_dring_handle_t dhandle, 53351ae08745Sheppo uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount) 53361ae08745Sheppo { 53371ae08745Sheppo int err; 53381ae08745Sheppo ldc_chan_t *ldcp; 53391ae08745Sheppo ldc_dring_t *dringp; 53401ae08745Sheppo ldc_mem_handle_t mhandle; 53411ae08745Sheppo 53421ae08745Sheppo /* check to see if channel is initalized */ 53431ae08745Sheppo if (handle == NULL) { 53441ae08745Sheppo DWARN(DBG_ALL_LDCS, 53451ae08745Sheppo "ldc_mem_dring_bind: invalid channel handle\n"); 53461ae08745Sheppo return (EINVAL); 53471ae08745Sheppo } 53481ae08745Sheppo ldcp = (ldc_chan_t *)handle; 53491ae08745Sheppo 53501ae08745Sheppo if (dhandle == NULL) { 53511ae08745Sheppo DWARN(DBG_ALL_LDCS, 53521ae08745Sheppo "ldc_mem_dring_bind: invalid desc ring handle\n"); 53531ae08745Sheppo return (EINVAL); 53541ae08745Sheppo } 53551ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 53561ae08745Sheppo 53571ae08745Sheppo if (cookie == NULL) { 53581ae08745Sheppo DWARN(ldcp->id, 53591ae08745Sheppo "ldc_mem_dring_bind: invalid cookie arg\n"); 53601ae08745Sheppo return (EINVAL); 53611ae08745Sheppo } 53621ae08745Sheppo 53631ae08745Sheppo mutex_enter(&dringp->lock); 53641ae08745Sheppo 53651ae08745Sheppo if (dringp->status == LDC_BOUND) { 53661ae08745Sheppo DWARN(DBG_ALL_LDCS, 53671ae08745Sheppo "ldc_mem_dring_bind: (0x%llx) descriptor ring is bound\n", 53681ae08745Sheppo ldcp->id); 53691ae08745Sheppo mutex_exit(&dringp->lock); 53701ae08745Sheppo return (EINVAL); 53711ae08745Sheppo } 53721ae08745Sheppo 53731ae08745Sheppo if ((perm & LDC_MEM_RW) == 0) { 53741ae08745Sheppo DWARN(DBG_ALL_LDCS, 53751ae08745Sheppo "ldc_mem_dring_bind: invalid permissions\n"); 53761ae08745Sheppo mutex_exit(&dringp->lock); 53771ae08745Sheppo return (EINVAL); 53781ae08745Sheppo } 53791ae08745Sheppo 53801ae08745Sheppo if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) { 53811ae08745Sheppo DWARN(DBG_ALL_LDCS, "ldc_mem_dring_bind: invalid type\n"); 53821ae08745Sheppo mutex_exit(&dringp->lock); 53831ae08745Sheppo return (EINVAL); 53841ae08745Sheppo } 53851ae08745Sheppo 53861ae08745Sheppo dringp->ldcp = ldcp; 53871ae08745Sheppo 53881ae08745Sheppo /* create an memory handle */ 53891ae08745Sheppo err = ldc_mem_alloc_handle(handle, &mhandle); 53901ae08745Sheppo if (err || mhandle == NULL) { 53911ae08745Sheppo DWARN(DBG_ALL_LDCS, 53921ae08745Sheppo "ldc_mem_dring_bind: (0x%llx) error allocating mhandle\n", 53931ae08745Sheppo ldcp->id); 53941ae08745Sheppo mutex_exit(&dringp->lock); 53951ae08745Sheppo return (err); 53961ae08745Sheppo } 53971ae08745Sheppo dringp->mhdl = mhandle; 53981ae08745Sheppo 53991ae08745Sheppo /* bind the descriptor ring to channel */ 54001ae08745Sheppo err = ldc_mem_bind_handle(mhandle, dringp->base, dringp->size, 54011ae08745Sheppo mtype, perm, cookie, ccount); 54021ae08745Sheppo if (err) { 54031ae08745Sheppo DWARN(ldcp->id, 54041ae08745Sheppo "ldc_mem_dring_bind: (0x%llx) error binding mhandle\n", 54051ae08745Sheppo ldcp->id); 54061ae08745Sheppo mutex_exit(&dringp->lock); 54071ae08745Sheppo return (err); 54081ae08745Sheppo } 54091ae08745Sheppo 54101ae08745Sheppo /* 54111ae08745Sheppo * For now return error if we get more than one cookie 54121ae08745Sheppo * FUTURE: Return multiple cookies .. 54131ae08745Sheppo */ 54141ae08745Sheppo if (*ccount > 1) { 54151ae08745Sheppo (void) ldc_mem_unbind_handle(mhandle); 54161ae08745Sheppo (void) ldc_mem_free_handle(mhandle); 54171ae08745Sheppo 54181ae08745Sheppo dringp->ldcp = NULL; 54191ae08745Sheppo dringp->mhdl = NULL; 54201ae08745Sheppo *ccount = 0; 54211ae08745Sheppo 54221ae08745Sheppo mutex_exit(&dringp->lock); 54231ae08745Sheppo return (EAGAIN); 54241ae08745Sheppo } 54251ae08745Sheppo 54261ae08745Sheppo /* Add descriptor ring to channel's exported dring list */ 54271ae08745Sheppo mutex_enter(&ldcp->exp_dlist_lock); 54281ae08745Sheppo dringp->ch_next = ldcp->exp_dring_list; 54291ae08745Sheppo ldcp->exp_dring_list = dringp; 54301ae08745Sheppo mutex_exit(&ldcp->exp_dlist_lock); 54311ae08745Sheppo 54321ae08745Sheppo dringp->status = LDC_BOUND; 54331ae08745Sheppo 54341ae08745Sheppo mutex_exit(&dringp->lock); 54351ae08745Sheppo 54361ae08745Sheppo return (0); 54371ae08745Sheppo } 54381ae08745Sheppo 54391ae08745Sheppo /* 54401ae08745Sheppo * Return the next cookie associated with the specified dring handle 54411ae08745Sheppo */ 54421ae08745Sheppo int 54431ae08745Sheppo ldc_mem_dring_nextcookie(ldc_dring_handle_t dhandle, ldc_mem_cookie_t *cookie) 54441ae08745Sheppo { 54451ae08745Sheppo int rv = 0; 54461ae08745Sheppo ldc_dring_t *dringp; 54471ae08745Sheppo ldc_chan_t *ldcp; 54481ae08745Sheppo 54491ae08745Sheppo if (dhandle == NULL) { 54501ae08745Sheppo DWARN(DBG_ALL_LDCS, 54511ae08745Sheppo "ldc_mem_dring_nextcookie: invalid desc ring handle\n"); 54521ae08745Sheppo return (EINVAL); 54531ae08745Sheppo } 54541ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 54551ae08745Sheppo mutex_enter(&dringp->lock); 54561ae08745Sheppo 54571ae08745Sheppo if (dringp->status != LDC_BOUND) { 54581ae08745Sheppo DWARN(DBG_ALL_LDCS, 54591ae08745Sheppo "ldc_mem_dring_nextcookie: descriptor ring 0x%llx " 54601ae08745Sheppo "is not bound\n", dringp); 54611ae08745Sheppo mutex_exit(&dringp->lock); 54621ae08745Sheppo return (EINVAL); 54631ae08745Sheppo } 54641ae08745Sheppo 54651ae08745Sheppo ldcp = dringp->ldcp; 54661ae08745Sheppo 54671ae08745Sheppo if (cookie == NULL) { 54681ae08745Sheppo DWARN(ldcp->id, 54691ae08745Sheppo "ldc_mem_dring_nextcookie:(0x%llx) invalid cookie arg\n", 54701ae08745Sheppo ldcp->id); 54711ae08745Sheppo mutex_exit(&dringp->lock); 54721ae08745Sheppo return (EINVAL); 54731ae08745Sheppo } 54741ae08745Sheppo 54751ae08745Sheppo rv = ldc_mem_nextcookie((ldc_mem_handle_t)dringp->mhdl, cookie); 54761ae08745Sheppo mutex_exit(&dringp->lock); 54771ae08745Sheppo 54781ae08745Sheppo return (rv); 54791ae08745Sheppo } 54801ae08745Sheppo /* 54811ae08745Sheppo * Unbind a previously bound dring from a channel. 54821ae08745Sheppo */ 54831ae08745Sheppo int 54841ae08745Sheppo ldc_mem_dring_unbind(ldc_dring_handle_t dhandle) 54851ae08745Sheppo { 54861ae08745Sheppo ldc_dring_t *dringp; 54871ae08745Sheppo ldc_dring_t *tmp_dringp; 54881ae08745Sheppo ldc_chan_t *ldcp; 54891ae08745Sheppo 54901ae08745Sheppo if (dhandle == NULL) { 54911ae08745Sheppo DWARN(DBG_ALL_LDCS, 54921ae08745Sheppo "ldc_mem_dring_unbind: invalid desc ring handle\n"); 54931ae08745Sheppo return (EINVAL); 54941ae08745Sheppo } 54951ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 54961ae08745Sheppo 54971ae08745Sheppo mutex_enter(&dringp->lock); 54981ae08745Sheppo 54991ae08745Sheppo if (dringp->status == LDC_UNBOUND) { 55001ae08745Sheppo DWARN(DBG_ALL_LDCS, 55011ae08745Sheppo "ldc_mem_dring_bind: descriptor ring 0x%llx is unbound\n", 55021ae08745Sheppo dringp); 55031ae08745Sheppo mutex_exit(&dringp->lock); 55041ae08745Sheppo return (EINVAL); 55051ae08745Sheppo } 55061ae08745Sheppo ldcp = dringp->ldcp; 55071ae08745Sheppo 55081ae08745Sheppo mutex_enter(&ldcp->exp_dlist_lock); 55091ae08745Sheppo 55101ae08745Sheppo tmp_dringp = ldcp->exp_dring_list; 55111ae08745Sheppo if (tmp_dringp == dringp) { 55121ae08745Sheppo ldcp->exp_dring_list = dringp->ch_next; 55131ae08745Sheppo dringp->ch_next = NULL; 55141ae08745Sheppo 55151ae08745Sheppo } else { 55161ae08745Sheppo while (tmp_dringp != NULL) { 55171ae08745Sheppo if (tmp_dringp->ch_next == dringp) { 55181ae08745Sheppo tmp_dringp->ch_next = dringp->ch_next; 55191ae08745Sheppo dringp->ch_next = NULL; 55201ae08745Sheppo break; 55211ae08745Sheppo } 55221ae08745Sheppo tmp_dringp = tmp_dringp->ch_next; 55231ae08745Sheppo } 55241ae08745Sheppo if (tmp_dringp == NULL) { 55251ae08745Sheppo DWARN(DBG_ALL_LDCS, 55261ae08745Sheppo "ldc_mem_dring_unbind: invalid descriptor\n"); 55271ae08745Sheppo mutex_exit(&ldcp->exp_dlist_lock); 55281ae08745Sheppo mutex_exit(&dringp->lock); 55291ae08745Sheppo return (EINVAL); 55301ae08745Sheppo } 55311ae08745Sheppo } 55321ae08745Sheppo 55331ae08745Sheppo mutex_exit(&ldcp->exp_dlist_lock); 55341ae08745Sheppo 55351ae08745Sheppo (void) ldc_mem_unbind_handle((ldc_mem_handle_t)dringp->mhdl); 55361ae08745Sheppo (void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl); 55371ae08745Sheppo 55381ae08745Sheppo dringp->ldcp = NULL; 55391ae08745Sheppo dringp->mhdl = NULL; 55401ae08745Sheppo dringp->status = LDC_UNBOUND; 55411ae08745Sheppo 55421ae08745Sheppo mutex_exit(&dringp->lock); 55431ae08745Sheppo 55441ae08745Sheppo return (0); 55451ae08745Sheppo } 55461ae08745Sheppo 55471ae08745Sheppo /* 55481ae08745Sheppo * Get information about the dring. The base address of the descriptor 55491ae08745Sheppo * ring along with the type and permission are returned back. 55501ae08745Sheppo */ 55511ae08745Sheppo int 55521ae08745Sheppo ldc_mem_dring_info(ldc_dring_handle_t dhandle, ldc_mem_info_t *minfo) 55531ae08745Sheppo { 55541ae08745Sheppo ldc_dring_t *dringp; 55551ae08745Sheppo int rv; 55561ae08745Sheppo 55571ae08745Sheppo if (dhandle == NULL) { 55581ae08745Sheppo DWARN(DBG_ALL_LDCS, 55591ae08745Sheppo "ldc_mem_dring_info: invalid desc ring handle\n"); 55601ae08745Sheppo return (EINVAL); 55611ae08745Sheppo } 55621ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 55631ae08745Sheppo 55641ae08745Sheppo mutex_enter(&dringp->lock); 55651ae08745Sheppo 55661ae08745Sheppo if (dringp->mhdl) { 55671ae08745Sheppo rv = ldc_mem_info(dringp->mhdl, minfo); 55681ae08745Sheppo if (rv) { 55691ae08745Sheppo DWARN(DBG_ALL_LDCS, 55701ae08745Sheppo "ldc_mem_dring_info: error reading mem info\n"); 55711ae08745Sheppo mutex_exit(&dringp->lock); 55721ae08745Sheppo return (rv); 55731ae08745Sheppo } 55741ae08745Sheppo } else { 55751ae08745Sheppo minfo->vaddr = dringp->base; 55761ae08745Sheppo minfo->raddr = NULL; 55771ae08745Sheppo minfo->status = dringp->status; 55781ae08745Sheppo } 55791ae08745Sheppo 55801ae08745Sheppo mutex_exit(&dringp->lock); 55811ae08745Sheppo 55821ae08745Sheppo return (0); 55831ae08745Sheppo } 55841ae08745Sheppo 55851ae08745Sheppo /* 55861ae08745Sheppo * Map an exported descriptor ring into the local address space. If the 55871ae08745Sheppo * descriptor ring was exported for direct map access, a HV call is made 55881ae08745Sheppo * to allocate a RA range. If the map is done via a shadow copy, local 55891ae08745Sheppo * shadow memory is allocated. 55901ae08745Sheppo */ 55911ae08745Sheppo int 55921ae08745Sheppo ldc_mem_dring_map(ldc_handle_t handle, ldc_mem_cookie_t *cookie, 55931ae08745Sheppo uint32_t ccount, uint32_t len, uint32_t dsize, uint8_t mtype, 55941ae08745Sheppo ldc_dring_handle_t *dhandle) 55951ae08745Sheppo { 55961ae08745Sheppo int err; 55971ae08745Sheppo ldc_chan_t *ldcp = (ldc_chan_t *)handle; 55981ae08745Sheppo ldc_mem_handle_t mhandle; 55991ae08745Sheppo ldc_dring_t *dringp; 56001ae08745Sheppo size_t dring_size; 56011ae08745Sheppo 56021ae08745Sheppo if (dhandle == NULL) { 56031ae08745Sheppo DWARN(DBG_ALL_LDCS, 56041ae08745Sheppo "ldc_mem_dring_map: invalid dhandle\n"); 56051ae08745Sheppo return (EINVAL); 56061ae08745Sheppo } 56071ae08745Sheppo 56081ae08745Sheppo /* check to see if channel is initalized */ 56091ae08745Sheppo if (handle == NULL) { 56101ae08745Sheppo DWARN(DBG_ALL_LDCS, 56111ae08745Sheppo "ldc_mem_dring_map: invalid channel handle\n"); 56121ae08745Sheppo return (EINVAL); 56131ae08745Sheppo } 56141ae08745Sheppo ldcp = (ldc_chan_t *)handle; 56151ae08745Sheppo 56161ae08745Sheppo if (cookie == NULL) { 56171ae08745Sheppo DWARN(ldcp->id, 56181ae08745Sheppo "ldc_mem_dring_map: (0x%llx) invalid cookie\n", 56191ae08745Sheppo ldcp->id); 56201ae08745Sheppo return (EINVAL); 56211ae08745Sheppo } 56221ae08745Sheppo 56231ae08745Sheppo /* FUTURE: For now we support only one cookie per dring */ 56241ae08745Sheppo ASSERT(ccount == 1); 56251ae08745Sheppo 56261ae08745Sheppo if (cookie->size < (dsize * len)) { 56271ae08745Sheppo DWARN(ldcp->id, 56281ae08745Sheppo "ldc_mem_dring_map: (0x%llx) invalid dsize/len\n", 56291ae08745Sheppo ldcp->id); 56301ae08745Sheppo return (EINVAL); 56311ae08745Sheppo } 56321ae08745Sheppo 56331ae08745Sheppo *dhandle = 0; 56341ae08745Sheppo 56351ae08745Sheppo /* Allocate an dring structure */ 56361ae08745Sheppo dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP); 56371ae08745Sheppo 56381ae08745Sheppo D1(ldcp->id, 56391ae08745Sheppo "ldc_mem_dring_map: 0x%x,0x%x,0x%x,0x%llx,0x%llx\n", 56401ae08745Sheppo mtype, len, dsize, cookie->addr, cookie->size); 56411ae08745Sheppo 56421ae08745Sheppo /* Initialize dring */ 56431ae08745Sheppo dringp->length = len; 56441ae08745Sheppo dringp->dsize = dsize; 56451ae08745Sheppo 56461ae08745Sheppo /* round of to multiple of page size */ 56471ae08745Sheppo dring_size = len * dsize; 56481ae08745Sheppo dringp->size = (dring_size & MMU_PAGEMASK); 56491ae08745Sheppo if (dring_size & MMU_PAGEOFFSET) 56501ae08745Sheppo dringp->size += MMU_PAGESIZE; 56511ae08745Sheppo 56521ae08745Sheppo dringp->ldcp = ldcp; 56531ae08745Sheppo 56541ae08745Sheppo /* create an memory handle */ 56551ae08745Sheppo err = ldc_mem_alloc_handle(handle, &mhandle); 56561ae08745Sheppo if (err || mhandle == NULL) { 56571ae08745Sheppo DWARN(DBG_ALL_LDCS, 56581ae08745Sheppo "ldc_mem_dring_map: cannot alloc hdl err=%d\n", 56591ae08745Sheppo err); 56601ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 56611ae08745Sheppo return (ENOMEM); 56621ae08745Sheppo } 56631ae08745Sheppo 56641ae08745Sheppo dringp->mhdl = mhandle; 56651ae08745Sheppo dringp->base = NULL; 56661ae08745Sheppo 56671ae08745Sheppo /* map the dring into local memory */ 56681ae08745Sheppo err = ldc_mem_map(mhandle, cookie, ccount, mtype, 56691ae08745Sheppo &(dringp->base), NULL); 56701ae08745Sheppo if (err || dringp->base == NULL) { 56711ae08745Sheppo cmn_err(CE_WARN, 56721ae08745Sheppo "ldc_mem_dring_map: cannot map desc ring err=%d\n", err); 56731ae08745Sheppo (void) ldc_mem_free_handle(mhandle); 56741ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 56751ae08745Sheppo return (ENOMEM); 56761ae08745Sheppo } 56771ae08745Sheppo 56781ae08745Sheppo /* initialize the desc ring lock */ 56791ae08745Sheppo mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL); 56801ae08745Sheppo 56811ae08745Sheppo /* Add descriptor ring to channel's imported dring list */ 56821ae08745Sheppo mutex_enter(&ldcp->imp_dlist_lock); 56831ae08745Sheppo dringp->ch_next = ldcp->imp_dring_list; 56841ae08745Sheppo ldcp->imp_dring_list = dringp; 56851ae08745Sheppo mutex_exit(&ldcp->imp_dlist_lock); 56861ae08745Sheppo 56871ae08745Sheppo dringp->status = LDC_MAPPED; 56881ae08745Sheppo 56891ae08745Sheppo *dhandle = (ldc_dring_handle_t)dringp; 56901ae08745Sheppo 56911ae08745Sheppo return (0); 56921ae08745Sheppo } 56931ae08745Sheppo 56941ae08745Sheppo /* 56951ae08745Sheppo * Unmap a descriptor ring. Free shadow memory (if any). 56961ae08745Sheppo */ 56971ae08745Sheppo int 56981ae08745Sheppo ldc_mem_dring_unmap(ldc_dring_handle_t dhandle) 56991ae08745Sheppo { 57001ae08745Sheppo ldc_dring_t *dringp; 57011ae08745Sheppo ldc_dring_t *tmp_dringp; 57021ae08745Sheppo ldc_chan_t *ldcp; 57031ae08745Sheppo 57041ae08745Sheppo if (dhandle == NULL) { 57051ae08745Sheppo DWARN(DBG_ALL_LDCS, 57061ae08745Sheppo "ldc_mem_dring_unmap: invalid desc ring handle\n"); 57071ae08745Sheppo return (EINVAL); 57081ae08745Sheppo } 57091ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 57101ae08745Sheppo 57111ae08745Sheppo if (dringp->status != LDC_MAPPED) { 57121ae08745Sheppo DWARN(DBG_ALL_LDCS, 57131ae08745Sheppo "ldc_mem_dring_unmap: not a mapped desc ring\n"); 57141ae08745Sheppo return (EINVAL); 57151ae08745Sheppo } 57161ae08745Sheppo 57171ae08745Sheppo mutex_enter(&dringp->lock); 57181ae08745Sheppo 57191ae08745Sheppo ldcp = dringp->ldcp; 57201ae08745Sheppo 57211ae08745Sheppo mutex_enter(&ldcp->imp_dlist_lock); 57221ae08745Sheppo 57231ae08745Sheppo /* find and unlink the desc ring from channel import list */ 57241ae08745Sheppo tmp_dringp = ldcp->imp_dring_list; 57251ae08745Sheppo if (tmp_dringp == dringp) { 57261ae08745Sheppo ldcp->imp_dring_list = dringp->ch_next; 57271ae08745Sheppo dringp->ch_next = NULL; 57281ae08745Sheppo 57291ae08745Sheppo } else { 57301ae08745Sheppo while (tmp_dringp != NULL) { 57311ae08745Sheppo if (tmp_dringp->ch_next == dringp) { 57321ae08745Sheppo tmp_dringp->ch_next = dringp->ch_next; 57331ae08745Sheppo dringp->ch_next = NULL; 57341ae08745Sheppo break; 57351ae08745Sheppo } 57361ae08745Sheppo tmp_dringp = tmp_dringp->ch_next; 57371ae08745Sheppo } 57381ae08745Sheppo if (tmp_dringp == NULL) { 57391ae08745Sheppo DWARN(DBG_ALL_LDCS, 57401ae08745Sheppo "ldc_mem_dring_unmap: invalid descriptor\n"); 57411ae08745Sheppo mutex_exit(&ldcp->imp_dlist_lock); 57421ae08745Sheppo mutex_exit(&dringp->lock); 57431ae08745Sheppo return (EINVAL); 57441ae08745Sheppo } 57451ae08745Sheppo } 57461ae08745Sheppo 57471ae08745Sheppo mutex_exit(&ldcp->imp_dlist_lock); 57481ae08745Sheppo 57491ae08745Sheppo /* do a LDC memory handle unmap and free */ 57501ae08745Sheppo (void) ldc_mem_unmap(dringp->mhdl); 57511ae08745Sheppo (void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl); 57521ae08745Sheppo 57531ae08745Sheppo dringp->status = 0; 57541ae08745Sheppo dringp->ldcp = NULL; 57551ae08745Sheppo 57561ae08745Sheppo mutex_exit(&dringp->lock); 57571ae08745Sheppo 57581ae08745Sheppo /* destroy dring lock */ 57591ae08745Sheppo mutex_destroy(&dringp->lock); 57601ae08745Sheppo 57611ae08745Sheppo /* free desc ring object */ 57621ae08745Sheppo kmem_free(dringp, sizeof (ldc_dring_t)); 57631ae08745Sheppo 57641ae08745Sheppo return (0); 57651ae08745Sheppo } 57661ae08745Sheppo 57671ae08745Sheppo /* 57681ae08745Sheppo * Internal entry point for descriptor ring access entry consistency 57691ae08745Sheppo * semantics. Acquire copies the contents of the remote descriptor ring 57701ae08745Sheppo * into the local shadow copy. The release operation copies the local 57711ae08745Sheppo * contents into the remote dring. The start and end locations specify 57721ae08745Sheppo * bounds for the entries being synchronized. 57731ae08745Sheppo */ 57741ae08745Sheppo static int 57751ae08745Sheppo i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle, 57761ae08745Sheppo uint8_t direction, uint64_t start, uint64_t end) 57771ae08745Sheppo { 57781ae08745Sheppo int err; 57791ae08745Sheppo ldc_dring_t *dringp; 57801ae08745Sheppo ldc_chan_t *ldcp; 57811ae08745Sheppo uint64_t soff; 57821ae08745Sheppo size_t copy_size; 57831ae08745Sheppo 57841ae08745Sheppo if (dhandle == NULL) { 57851ae08745Sheppo DWARN(DBG_ALL_LDCS, 57861ae08745Sheppo "i_ldc_dring_acquire_release: invalid desc ring handle\n"); 57871ae08745Sheppo return (EINVAL); 57881ae08745Sheppo } 57891ae08745Sheppo dringp = (ldc_dring_t *)dhandle; 57901ae08745Sheppo mutex_enter(&dringp->lock); 57911ae08745Sheppo 57921ae08745Sheppo if (dringp->status != LDC_MAPPED || dringp->ldcp == NULL) { 57931ae08745Sheppo DWARN(DBG_ALL_LDCS, 57941ae08745Sheppo "i_ldc_dring_acquire_release: not a mapped desc ring\n"); 57951ae08745Sheppo mutex_exit(&dringp->lock); 57961ae08745Sheppo return (EINVAL); 57971ae08745Sheppo } 57981ae08745Sheppo 57991ae08745Sheppo if (start >= dringp->length || end >= dringp->length) { 58001ae08745Sheppo DWARN(DBG_ALL_LDCS, 58011ae08745Sheppo "i_ldc_dring_acquire_release: index out of range\n"); 58021ae08745Sheppo mutex_exit(&dringp->lock); 58031ae08745Sheppo return (EINVAL); 58041ae08745Sheppo } 58051ae08745Sheppo 58061ae08745Sheppo /* get the channel handle */ 58071ae08745Sheppo ldcp = dringp->ldcp; 58081ae08745Sheppo 58091ae08745Sheppo copy_size = (start <= end) ? (((end - start) + 1) * dringp->dsize) : 58101ae08745Sheppo ((dringp->length - start) * dringp->dsize); 58111ae08745Sheppo 58121ae08745Sheppo /* Calculate the relative offset for the first desc */ 58131ae08745Sheppo soff = (start * dringp->dsize); 58141ae08745Sheppo 58151ae08745Sheppo /* copy to/from remote from/to local memory */ 58161ae08745Sheppo D1(ldcp->id, "i_ldc_dring_acquire_release: c1 off=0x%llx sz=0x%llx\n", 58171ae08745Sheppo soff, copy_size); 58181ae08745Sheppo err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl, 58191ae08745Sheppo direction, soff, copy_size); 58201ae08745Sheppo if (err) { 58211ae08745Sheppo DWARN(ldcp->id, 58221ae08745Sheppo "i_ldc_dring_acquire_release: copy failed\n"); 58231ae08745Sheppo mutex_exit(&dringp->lock); 58241ae08745Sheppo return (err); 58251ae08745Sheppo } 58261ae08745Sheppo 58271ae08745Sheppo /* do the balance */ 58281ae08745Sheppo if (start > end) { 58291ae08745Sheppo copy_size = ((end + 1) * dringp->dsize); 58301ae08745Sheppo soff = 0; 58311ae08745Sheppo 58321ae08745Sheppo /* copy to/from remote from/to local memory */ 58331ae08745Sheppo D1(ldcp->id, "i_ldc_dring_acquire_release: c2 " 58341ae08745Sheppo "off=0x%llx sz=0x%llx\n", soff, copy_size); 58351ae08745Sheppo err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl, 58361ae08745Sheppo direction, soff, copy_size); 58371ae08745Sheppo if (err) { 58381ae08745Sheppo DWARN(ldcp->id, 58391ae08745Sheppo "i_ldc_dring_acquire_release: copy failed\n"); 58401ae08745Sheppo mutex_exit(&dringp->lock); 58411ae08745Sheppo return (err); 58421ae08745Sheppo } 58431ae08745Sheppo } 58441ae08745Sheppo 58451ae08745Sheppo mutex_exit(&dringp->lock); 58461ae08745Sheppo 58471ae08745Sheppo return (0); 58481ae08745Sheppo } 58491ae08745Sheppo 58501ae08745Sheppo /* 58511ae08745Sheppo * Ensure that the contents in the local dring are consistent 58521ae08745Sheppo * with the contents if of remote dring 58531ae08745Sheppo */ 58541ae08745Sheppo int 58551ae08745Sheppo ldc_mem_dring_acquire(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end) 58561ae08745Sheppo { 58571ae08745Sheppo return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_IN, start, end)); 58581ae08745Sheppo } 58591ae08745Sheppo 58601ae08745Sheppo /* 58611ae08745Sheppo * Ensure that the contents in the remote dring are consistent 58621ae08745Sheppo * with the contents if of local dring 58631ae08745Sheppo */ 58641ae08745Sheppo int 58651ae08745Sheppo ldc_mem_dring_release(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end) 58661ae08745Sheppo { 58671ae08745Sheppo return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_OUT, start, end)); 58681ae08745Sheppo } 58691ae08745Sheppo 58701ae08745Sheppo 58711ae08745Sheppo /* ------------------------------------------------------------------------- */ 5872