19e39c5baSBill Taylor /*
29e39c5baSBill Taylor * CDDL HEADER START
39e39c5baSBill Taylor *
49e39c5baSBill Taylor * The contents of this file are subject to the terms of the
59e39c5baSBill Taylor * Common Development and Distribution License (the "License").
69e39c5baSBill Taylor * You may not use this file except in compliance with the License.
79e39c5baSBill Taylor *
89e39c5baSBill Taylor * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
99e39c5baSBill Taylor * or http://www.opensolaris.org/os/licensing.
109e39c5baSBill Taylor * See the License for the specific language governing permissions
119e39c5baSBill Taylor * and limitations under the License.
129e39c5baSBill Taylor *
139e39c5baSBill Taylor * When distributing Covered Code, include this CDDL HEADER in each
149e39c5baSBill Taylor * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
159e39c5baSBill Taylor * If applicable, add the following below this CDDL HEADER, with the
169e39c5baSBill Taylor * fields enclosed by brackets "[]" replaced with your own identifying
179e39c5baSBill Taylor * information: Portions Copyright [yyyy] [name of copyright owner]
189e39c5baSBill Taylor *
199e39c5baSBill Taylor * CDDL HEADER END
209e39c5baSBill Taylor */
219e39c5baSBill Taylor
229e39c5baSBill Taylor /*
2317a2b317SBill Taylor * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
249e39c5baSBill Taylor */
259e39c5baSBill Taylor
269e39c5baSBill Taylor /*
279e39c5baSBill Taylor * tavor_mr.c
289e39c5baSBill Taylor * Tavor Memory Region/Window Routines
299e39c5baSBill Taylor *
309e39c5baSBill Taylor * Implements all the routines necessary to provide the requisite memory
319e39c5baSBill Taylor * registration verbs. These include operations like RegisterMemRegion(),
329e39c5baSBill Taylor * DeregisterMemRegion(), ReregisterMemRegion, RegisterSharedMemRegion,
339e39c5baSBill Taylor * etc., that affect Memory Regions. It also includes the verbs that
349e39c5baSBill Taylor * affect Memory Windows, including AllocMemWindow(), FreeMemWindow(),
359e39c5baSBill Taylor * and QueryMemWindow().
369e39c5baSBill Taylor */
379e39c5baSBill Taylor
389e39c5baSBill Taylor #include <sys/types.h>
399e39c5baSBill Taylor #include <sys/conf.h>
409e39c5baSBill Taylor #include <sys/ddi.h>
419e39c5baSBill Taylor #include <sys/sunddi.h>
429e39c5baSBill Taylor #include <sys/modctl.h>
439e39c5baSBill Taylor #include <sys/esunddi.h>
449e39c5baSBill Taylor
459e39c5baSBill Taylor #include <sys/ib/adapters/tavor/tavor.h>
469e39c5baSBill Taylor
479e39c5baSBill Taylor
489e39c5baSBill Taylor /*
499e39c5baSBill Taylor * Used by tavor_mr_keycalc() below to fill in the "unconstrained" portion
509e39c5baSBill Taylor * of Tavor memory keys (LKeys and RKeys)
519e39c5baSBill Taylor */
529e39c5baSBill Taylor static uint_t tavor_debug_memkey_cnt = 0x00000000;
539e39c5baSBill Taylor
549e39c5baSBill Taylor static int tavor_mr_common_reg(tavor_state_t *state, tavor_pdhdl_t pd,
559e39c5baSBill Taylor tavor_bind_info_t *bind, tavor_mrhdl_t *mrhdl, tavor_mr_options_t *op);
569e39c5baSBill Taylor static int tavor_mr_common_rereg(tavor_state_t *state, tavor_mrhdl_t mr,
579e39c5baSBill Taylor tavor_pdhdl_t pd, tavor_bind_info_t *bind, tavor_mrhdl_t *mrhdl_new,
589e39c5baSBill Taylor tavor_mr_options_t *op);
599e39c5baSBill Taylor static int tavor_mr_rereg_xlat_helper(tavor_state_t *state, tavor_mrhdl_t mr,
609e39c5baSBill Taylor tavor_bind_info_t *bind, tavor_mr_options_t *op, uint64_t *mtt_addr,
619e39c5baSBill Taylor uint_t sleep, uint_t *dereg_level);
629e39c5baSBill Taylor static uint64_t tavor_mr_nummtt_needed(tavor_state_t *state,
639e39c5baSBill Taylor tavor_bind_info_t *bind, uint_t *mtt_pgsize);
649e39c5baSBill Taylor static int tavor_mr_mem_bind(tavor_state_t *state, tavor_bind_info_t *bind,
659e39c5baSBill Taylor ddi_dma_handle_t dmahdl, uint_t sleep);
669e39c5baSBill Taylor static void tavor_mr_mem_unbind(tavor_state_t *state,
679e39c5baSBill Taylor tavor_bind_info_t *bind);
689e39c5baSBill Taylor static int tavor_mr_fast_mtt_write(tavor_rsrc_t *mtt, tavor_bind_info_t *bind,
699e39c5baSBill Taylor uint32_t mtt_pgsize_bits);
709e39c5baSBill Taylor static int tavor_mtt_refcnt_inc(tavor_rsrc_t *rsrc);
719e39c5baSBill Taylor static int tavor_mtt_refcnt_dec(tavor_rsrc_t *rsrc);
729e39c5baSBill Taylor
739e39c5baSBill Taylor /*
749e39c5baSBill Taylor * The Tavor umem_lockmemory() callback ops. When userland memory is
759e39c5baSBill Taylor * registered, these callback ops are specified. The tavor_umap_umemlock_cb()
769e39c5baSBill Taylor * callback will be called whenever the memory for the corresponding
779e39c5baSBill Taylor * ddi_umem_cookie_t is being freed.
789e39c5baSBill Taylor */
799e39c5baSBill Taylor static struct umem_callback_ops tavor_umem_cbops = {
809e39c5baSBill Taylor UMEM_CALLBACK_VERSION,
819e39c5baSBill Taylor tavor_umap_umemlock_cb,
829e39c5baSBill Taylor };
839e39c5baSBill Taylor
849e39c5baSBill Taylor
859e39c5baSBill Taylor /*
869e39c5baSBill Taylor * tavor_mr_register()
879e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
889e39c5baSBill Taylor */
899e39c5baSBill Taylor int
tavor_mr_register(tavor_state_t * state,tavor_pdhdl_t pd,ibt_mr_attr_t * mr_attr,tavor_mrhdl_t * mrhdl,tavor_mr_options_t * op)909e39c5baSBill Taylor tavor_mr_register(tavor_state_t *state, tavor_pdhdl_t pd,
919e39c5baSBill Taylor ibt_mr_attr_t *mr_attr, tavor_mrhdl_t *mrhdl, tavor_mr_options_t *op)
929e39c5baSBill Taylor {
939e39c5baSBill Taylor tavor_bind_info_t bind;
949e39c5baSBill Taylor int status;
959e39c5baSBill Taylor
969e39c5baSBill Taylor /*
979e39c5baSBill Taylor * Fill in the "bind" struct. This struct provides the majority
989e39c5baSBill Taylor * of the information that will be used to distinguish between an
999e39c5baSBill Taylor * "addr" binding (as is the case here) and a "buf" binding (see
1009e39c5baSBill Taylor * below). The "bind" struct is later passed to tavor_mr_mem_bind()
1019e39c5baSBill Taylor * which does most of the "heavy lifting" for the Tavor memory
1029e39c5baSBill Taylor * registration routines.
1039e39c5baSBill Taylor */
1049e39c5baSBill Taylor bind.bi_type = TAVOR_BINDHDL_VADDR;
1059e39c5baSBill Taylor bind.bi_addr = mr_attr->mr_vaddr;
1069e39c5baSBill Taylor bind.bi_len = mr_attr->mr_len;
1079e39c5baSBill Taylor bind.bi_as = mr_attr->mr_as;
1089e39c5baSBill Taylor bind.bi_flags = mr_attr->mr_flags;
1099e39c5baSBill Taylor status = tavor_mr_common_reg(state, pd, &bind, mrhdl, op);
1109e39c5baSBill Taylor
111*2570281cSToomas Soome return (status);
1129e39c5baSBill Taylor }
1139e39c5baSBill Taylor
1149e39c5baSBill Taylor
1159e39c5baSBill Taylor /*
1169e39c5baSBill Taylor * tavor_mr_register_buf()
1179e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
1189e39c5baSBill Taylor */
1199e39c5baSBill Taylor int
tavor_mr_register_buf(tavor_state_t * state,tavor_pdhdl_t pd,ibt_smr_attr_t * mr_attr,struct buf * buf,tavor_mrhdl_t * mrhdl,tavor_mr_options_t * op)1209e39c5baSBill Taylor tavor_mr_register_buf(tavor_state_t *state, tavor_pdhdl_t pd,
1219e39c5baSBill Taylor ibt_smr_attr_t *mr_attr, struct buf *buf, tavor_mrhdl_t *mrhdl,
1229e39c5baSBill Taylor tavor_mr_options_t *op)
1239e39c5baSBill Taylor {
1249e39c5baSBill Taylor tavor_bind_info_t bind;
1259e39c5baSBill Taylor int status;
1269e39c5baSBill Taylor
1279e39c5baSBill Taylor /*
1289e39c5baSBill Taylor * Fill in the "bind" struct. This struct provides the majority
1299e39c5baSBill Taylor * of the information that will be used to distinguish between an
1309e39c5baSBill Taylor * "addr" binding (see above) and a "buf" binding (as is the case
1319e39c5baSBill Taylor * here). The "bind" struct is later passed to tavor_mr_mem_bind()
1329e39c5baSBill Taylor * which does most of the "heavy lifting" for the Tavor memory
1339e39c5baSBill Taylor * registration routines. Note: We have chosen to provide
1349e39c5baSBill Taylor * "b_un.b_addr" as the IB address (when the IBT_MR_PHYS_IOVA flag is
1359e39c5baSBill Taylor * not set). It is not critical what value we choose here as it need
1369e39c5baSBill Taylor * only be unique for the given RKey (which will happen by default),
1379e39c5baSBill Taylor * so the choice here is somewhat arbitrary.
1389e39c5baSBill Taylor */
1399e39c5baSBill Taylor bind.bi_type = TAVOR_BINDHDL_BUF;
1409e39c5baSBill Taylor bind.bi_buf = buf;
1419e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_PHYS_IOVA) {
1429e39c5baSBill Taylor bind.bi_addr = mr_attr->mr_vaddr;
1439e39c5baSBill Taylor } else {
1449e39c5baSBill Taylor bind.bi_addr = (uint64_t)(uintptr_t)buf->b_un.b_addr;
1459e39c5baSBill Taylor }
1469e39c5baSBill Taylor bind.bi_as = NULL;
1479e39c5baSBill Taylor bind.bi_len = (uint64_t)buf->b_bcount;
1489e39c5baSBill Taylor bind.bi_flags = mr_attr->mr_flags;
1499e39c5baSBill Taylor status = tavor_mr_common_reg(state, pd, &bind, mrhdl, op);
1509e39c5baSBill Taylor
151*2570281cSToomas Soome return (status);
1529e39c5baSBill Taylor }
1539e39c5baSBill Taylor
1549e39c5baSBill Taylor
1559e39c5baSBill Taylor /*
1569e39c5baSBill Taylor * tavor_mr_register_shared()
1579e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
1589e39c5baSBill Taylor */
1599e39c5baSBill Taylor int
tavor_mr_register_shared(tavor_state_t * state,tavor_mrhdl_t mrhdl,tavor_pdhdl_t pd,ibt_smr_attr_t * mr_attr,tavor_mrhdl_t * mrhdl_new)1609e39c5baSBill Taylor tavor_mr_register_shared(tavor_state_t *state, tavor_mrhdl_t mrhdl,
1619e39c5baSBill Taylor tavor_pdhdl_t pd, ibt_smr_attr_t *mr_attr, tavor_mrhdl_t *mrhdl_new)
1629e39c5baSBill Taylor {
1639e39c5baSBill Taylor tavor_rsrc_pool_info_t *rsrc_pool;
1649e39c5baSBill Taylor tavor_rsrc_t *mpt, *mtt, *rsrc;
1659e39c5baSBill Taylor tavor_umap_db_entry_t *umapdb;
1669e39c5baSBill Taylor tavor_hw_mpt_t mpt_entry;
1679e39c5baSBill Taylor tavor_mrhdl_t mr;
1689e39c5baSBill Taylor tavor_bind_info_t *bind;
1699e39c5baSBill Taylor ddi_umem_cookie_t umem_cookie;
1709e39c5baSBill Taylor size_t umem_len;
1719e39c5baSBill Taylor caddr_t umem_addr;
1729e39c5baSBill Taylor uint64_t mtt_addr, mtt_ddrbaseaddr, pgsize_msk;
1739e39c5baSBill Taylor uint_t sleep, mr_is_umem;
1749e39c5baSBill Taylor int status, umem_flags;
1759e39c5baSBill Taylor
1769e39c5baSBill Taylor /*
1779e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the
1789e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt
1799e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep).
1809e39c5baSBill Taylor */
1819e39c5baSBill Taylor sleep = (mr_attr->mr_flags & IBT_MR_NOSLEEP) ? TAVOR_NOSLEEP :
1829e39c5baSBill Taylor TAVOR_SLEEP;
1839e39c5baSBill Taylor if ((sleep == TAVOR_SLEEP) &&
1849e39c5baSBill Taylor (sleep != TAVOR_SLEEPFLAG_FOR_CONTEXT())) {
1859e39c5baSBill Taylor goto mrshared_fail;
1869e39c5baSBill Taylor }
1879e39c5baSBill Taylor
1889e39c5baSBill Taylor /* Increment the reference count on the protection domain (PD) */
1899e39c5baSBill Taylor tavor_pd_refcnt_inc(pd);
1909e39c5baSBill Taylor
1919e39c5baSBill Taylor /*
1929e39c5baSBill Taylor * Allocate an MPT entry. This will be filled in with all the
1939e39c5baSBill Taylor * necessary parameters to define the shared memory region.
1949e39c5baSBill Taylor * Specifically, it will be made to reference the currently existing
1959e39c5baSBill Taylor * MTT entries and ownership of the MPT will be passed to the hardware
1969e39c5baSBill Taylor * in the last step below. If we fail here, we must undo the
1979e39c5baSBill Taylor * protection domain reference count.
1989e39c5baSBill Taylor */
1999e39c5baSBill Taylor status = tavor_rsrc_alloc(state, TAVOR_MPT, 1, sleep, &mpt);
2009e39c5baSBill Taylor if (status != DDI_SUCCESS) {
2019e39c5baSBill Taylor goto mrshared_fail1;
2029e39c5baSBill Taylor }
2039e39c5baSBill Taylor
2049e39c5baSBill Taylor /*
2059e39c5baSBill Taylor * Allocate the software structure for tracking the shared memory
2069e39c5baSBill Taylor * region (i.e. the Tavor Memory Region handle). If we fail here, we
2079e39c5baSBill Taylor * must undo the protection domain reference count and the previous
2089e39c5baSBill Taylor * resource allocation.
2099e39c5baSBill Taylor */
2109e39c5baSBill Taylor status = tavor_rsrc_alloc(state, TAVOR_MRHDL, 1, sleep, &rsrc);
2119e39c5baSBill Taylor if (status != DDI_SUCCESS) {
2129e39c5baSBill Taylor goto mrshared_fail2;
2139e39c5baSBill Taylor }
2149e39c5baSBill Taylor mr = (tavor_mrhdl_t)rsrc->tr_addr;
2159e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
2169e39c5baSBill Taylor
2179e39c5baSBill Taylor /*
2189e39c5baSBill Taylor * Setup and validate the memory region access flags. This means
2199e39c5baSBill Taylor * translating the IBTF's enable flags into the access flags that
2209e39c5baSBill Taylor * will be used in later operations.
2219e39c5baSBill Taylor */
2229e39c5baSBill Taylor mr->mr_accflag = 0;
2239e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_WINDOW_BIND)
2249e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_WINDOW_BIND;
2259e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_LOCAL_WRITE)
2269e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_LOCAL_WRITE;
2279e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)
2289e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_READ;
2299e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE)
2309e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_WRITE;
2319e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)
2329e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC;
2339e39c5baSBill Taylor
2349e39c5baSBill Taylor /*
2359e39c5baSBill Taylor * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed
2369e39c5baSBill Taylor * from a certain number of "constrained" bits (the least significant
2379e39c5baSBill Taylor * bits) and some number of "unconstrained" bits. The constrained
2389e39c5baSBill Taylor * bits must be set to the index of the entry in the MPT table, but
2399e39c5baSBill Taylor * the unconstrained bits can be set to any value we wish. Note:
2409e39c5baSBill Taylor * if no remote access is required, then the RKey value is not filled
2419e39c5baSBill Taylor * in. Otherwise both Rkey and LKey are given the same value.
2429e39c5baSBill Taylor */
2439e39c5baSBill Taylor tavor_mr_keycalc(state, mpt->tr_indx, &mr->mr_lkey);
2449e39c5baSBill Taylor if ((mr->mr_accflag & IBT_MR_REMOTE_READ) ||
2459e39c5baSBill Taylor (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ||
2469e39c5baSBill Taylor (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC)) {
2479e39c5baSBill Taylor mr->mr_rkey = mr->mr_lkey;
2489e39c5baSBill Taylor }
2499e39c5baSBill Taylor
2509e39c5baSBill Taylor /* Grab the MR lock for the current memory region */
2519e39c5baSBill Taylor mutex_enter(&mrhdl->mr_lock);
2529e39c5baSBill Taylor
2539e39c5baSBill Taylor /*
2549e39c5baSBill Taylor * Check here to see if the memory region has already been partially
2559e39c5baSBill Taylor * deregistered as a result of a tavor_umap_umemlock_cb() callback.
2569e39c5baSBill Taylor * If so, this is an error, return failure.
2579e39c5baSBill Taylor */
2589e39c5baSBill Taylor if ((mrhdl->mr_is_umem) && (mrhdl->mr_umemcookie == NULL)) {
2599e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock);
2609e39c5baSBill Taylor goto mrshared_fail3;
2619e39c5baSBill Taylor }
2629e39c5baSBill Taylor
2639e39c5baSBill Taylor /*
2649e39c5baSBill Taylor * Determine if the original memory was from userland and, if so, pin
2659e39c5baSBill Taylor * the pages (again) with umem_lockmemory(). This will guarantee a
2669e39c5baSBill Taylor * separate callback for each of this shared region's MR handles.
2679e39c5baSBill Taylor * If this is userland memory, then allocate an entry in the
2689e39c5baSBill Taylor * "userland resources database". This will later be added to
2699e39c5baSBill Taylor * the database (after all further memory registration operations are
2709e39c5baSBill Taylor * successful). If we fail here, we must undo all the above setup.
2719e39c5baSBill Taylor */
2729e39c5baSBill Taylor mr_is_umem = mrhdl->mr_is_umem;
2739e39c5baSBill Taylor if (mr_is_umem) {
2749e39c5baSBill Taylor umem_len = ptob(btopr(mrhdl->mr_bindinfo.bi_len +
2759e39c5baSBill Taylor ((uintptr_t)mrhdl->mr_bindinfo.bi_addr & PAGEOFFSET)));
2769e39c5baSBill Taylor umem_addr = (caddr_t)((uintptr_t)mrhdl->mr_bindinfo.bi_addr &
2779e39c5baSBill Taylor ~PAGEOFFSET);
2789e39c5baSBill Taylor umem_flags = (DDI_UMEMLOCK_WRITE | DDI_UMEMLOCK_READ |
2799e39c5baSBill Taylor DDI_UMEMLOCK_LONGTERM);
2809e39c5baSBill Taylor status = umem_lockmemory(umem_addr, umem_len, umem_flags,
281d863b343SBill Taylor &umem_cookie, &tavor_umem_cbops, NULL);
2829e39c5baSBill Taylor if (status != 0) {
2839e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock);
2849e39c5baSBill Taylor goto mrshared_fail3;
2859e39c5baSBill Taylor }
2869e39c5baSBill Taylor
2879e39c5baSBill Taylor umapdb = tavor_umap_db_alloc(state->ts_instance,
2889e39c5baSBill Taylor (uint64_t)(uintptr_t)umem_cookie, MLNX_UMAP_MRMEM_RSRC,
2899e39c5baSBill Taylor (uint64_t)(uintptr_t)rsrc);
2909e39c5baSBill Taylor if (umapdb == NULL) {
2919e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock);
2929e39c5baSBill Taylor goto mrshared_fail4;
2939e39c5baSBill Taylor }
2949e39c5baSBill Taylor }
2959e39c5baSBill Taylor
2969e39c5baSBill Taylor /*
2979e39c5baSBill Taylor * Copy the MTT resource pointer (and additional parameters) from
2989e39c5baSBill Taylor * the original Tavor Memory Region handle. Note: this is normally
2999e39c5baSBill Taylor * where the tavor_mr_mem_bind() routine would be called, but because
3009e39c5baSBill Taylor * we already have bound and filled-in MTT entries it is simply a
3019e39c5baSBill Taylor * matter here of managing the MTT reference count and grabbing the
3029e39c5baSBill Taylor * address of the MTT table entries (for filling in the shared region's
3039e39c5baSBill Taylor * MPT entry).
3049e39c5baSBill Taylor */
3059e39c5baSBill Taylor mr->mr_mttrsrcp = mrhdl->mr_mttrsrcp;
3069e39c5baSBill Taylor mr->mr_logmttpgsz = mrhdl->mr_logmttpgsz;
3079e39c5baSBill Taylor mr->mr_bindinfo = mrhdl->mr_bindinfo;
3089e39c5baSBill Taylor mr->mr_mttrefcntp = mrhdl->mr_mttrefcntp;
3099e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock);
3109e39c5baSBill Taylor bind = &mr->mr_bindinfo;
3119e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
3129e39c5baSBill Taylor mtt = mr->mr_mttrsrcp;
3139e39c5baSBill Taylor
3149e39c5baSBill Taylor /*
3159e39c5baSBill Taylor * Increment the MTT reference count (to reflect the fact that
3169e39c5baSBill Taylor * the MTT is now shared)
3179e39c5baSBill Taylor */
3189e39c5baSBill Taylor (void) tavor_mtt_refcnt_inc(mr->mr_mttrefcntp);
3199e39c5baSBill Taylor
3209e39c5baSBill Taylor /*
3219e39c5baSBill Taylor * Update the new "bind" virtual address. Do some extra work here
3229e39c5baSBill Taylor * to ensure proper alignment. That is, make sure that the page
3239e39c5baSBill Taylor * offset for the beginning of the old range is the same as the
3249e39c5baSBill Taylor * offset for this new mapping
3259e39c5baSBill Taylor */
3269e39c5baSBill Taylor pgsize_msk = (((uint64_t)1 << mr->mr_logmttpgsz) - 1);
3279e39c5baSBill Taylor bind->bi_addr = ((mr_attr->mr_vaddr & ~pgsize_msk) |
3289e39c5baSBill Taylor (mr->mr_bindinfo.bi_addr & pgsize_msk));
3299e39c5baSBill Taylor
3309e39c5baSBill Taylor /*
3319e39c5baSBill Taylor * Get the base address for the MTT table. This will be necessary
3329e39c5baSBill Taylor * in the next step when we are setting up the MPT entry.
3339e39c5baSBill Taylor */
3349e39c5baSBill Taylor rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
3359e39c5baSBill Taylor mtt_ddrbaseaddr = (uint64_t)(uintptr_t)rsrc_pool->rsrc_ddr_offset;
3369e39c5baSBill Taylor
3379e39c5baSBill Taylor /*
3389e39c5baSBill Taylor * Fill in the MPT entry. This is the final step before passing
3399e39c5baSBill Taylor * ownership of the MPT entry to the Tavor hardware. We use all of
3409e39c5baSBill Taylor * the information collected/calculated above to fill in the
3419e39c5baSBill Taylor * requisite portions of the MPT.
3429e39c5baSBill Taylor */
3439e39c5baSBill Taylor bzero(&mpt_entry, sizeof (tavor_hw_mpt_t));
3449e39c5baSBill Taylor mpt_entry.m_io = TAVOR_MEM_CYCLE_GENERATE;
3459e39c5baSBill Taylor mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0;
3469e39c5baSBill Taylor mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
3479e39c5baSBill Taylor mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
3489e39c5baSBill Taylor mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
3499e39c5baSBill Taylor mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
3509e39c5baSBill Taylor mpt_entry.lr = 1;
3519e39c5baSBill Taylor mpt_entry.reg_win = TAVOR_MPT_IS_REGION;
3529e39c5baSBill Taylor mpt_entry.page_sz = mr->mr_logmttpgsz - 0xC;
3539e39c5baSBill Taylor mpt_entry.mem_key = mr->mr_lkey;
3549e39c5baSBill Taylor mpt_entry.pd = pd->pd_pdnum;
3559e39c5baSBill Taylor mpt_entry.start_addr = bind->bi_addr;
3569e39c5baSBill Taylor mpt_entry.reg_win_len = bind->bi_len;
3579e39c5baSBill Taylor mpt_entry.win_cnt_limit = TAVOR_UNLIMITED_WIN_BIND;
3589e39c5baSBill Taylor mtt_addr = mtt_ddrbaseaddr + (mtt->tr_indx << TAVOR_MTT_SIZE_SHIFT);
3599e39c5baSBill Taylor mpt_entry.mttseg_addr_h = mtt_addr >> 32;
3609e39c5baSBill Taylor mpt_entry.mttseg_addr_l = mtt_addr >> 6;
3619e39c5baSBill Taylor
3629e39c5baSBill Taylor /*
3639e39c5baSBill Taylor * Write the MPT entry to hardware. Lastly, we pass ownership of
3649e39c5baSBill Taylor * the entry to the hardware. Note: in general, this operation
3659e39c5baSBill Taylor * shouldn't fail. But if it does, we have to undo everything we've
3669e39c5baSBill Taylor * done above before returning error.
3679e39c5baSBill Taylor */
3689e39c5baSBill Taylor status = tavor_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
3699e39c5baSBill Taylor sizeof (tavor_hw_mpt_t), mpt->tr_indx, sleep);
3709e39c5baSBill Taylor if (status != TAVOR_CMD_SUCCESS) {
3719e39c5baSBill Taylor cmn_err(CE_CONT, "Tavor: SW2HW_MPT command failed: %08x\n",
3729e39c5baSBill Taylor status);
3739e39c5baSBill Taylor goto mrshared_fail5;
3749e39c5baSBill Taylor }
3759e39c5baSBill Taylor
3769e39c5baSBill Taylor /*
3779e39c5baSBill Taylor * Fill in the rest of the Tavor Memory Region handle. Having
3789e39c5baSBill Taylor * successfully transferred ownership of the MPT, we can update the
3799e39c5baSBill Taylor * following fields for use in further operations on the MR.
3809e39c5baSBill Taylor */
3819e39c5baSBill Taylor mr->mr_mptrsrcp = mpt;
3829e39c5baSBill Taylor mr->mr_mttrsrcp = mtt;
3839e39c5baSBill Taylor mr->mr_pdhdl = pd;
3849e39c5baSBill Taylor mr->mr_rsrcp = rsrc;
3859e39c5baSBill Taylor mr->mr_is_umem = mr_is_umem;
3869e39c5baSBill Taylor mr->mr_umemcookie = (mr_is_umem != 0) ? umem_cookie : NULL;
3879e39c5baSBill Taylor mr->mr_umem_cbfunc = NULL;
3889e39c5baSBill Taylor mr->mr_umem_cbarg1 = NULL;
3899e39c5baSBill Taylor mr->mr_umem_cbarg2 = NULL;
3909e39c5baSBill Taylor
3919e39c5baSBill Taylor /*
3929e39c5baSBill Taylor * If this is userland memory, then we need to insert the previously
3939e39c5baSBill Taylor * allocated entry into the "userland resources database". This will
3949e39c5baSBill Taylor * allow for later coordination between the tavor_umap_umemlock_cb()
3959e39c5baSBill Taylor * callback and tavor_mr_deregister().
3969e39c5baSBill Taylor */
3979e39c5baSBill Taylor if (mr_is_umem) {
3989e39c5baSBill Taylor tavor_umap_db_add(umapdb);
3999e39c5baSBill Taylor }
4009e39c5baSBill Taylor
4019e39c5baSBill Taylor *mrhdl_new = mr;
4029e39c5baSBill Taylor
4039e39c5baSBill Taylor return (DDI_SUCCESS);
4049e39c5baSBill Taylor
4059e39c5baSBill Taylor /*
4069e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine
4079e39c5baSBill Taylor */
4089e39c5baSBill Taylor mrshared_fail5:
4099e39c5baSBill Taylor (void) tavor_mtt_refcnt_dec(mr->mr_mttrefcntp);
4109e39c5baSBill Taylor if (mr_is_umem) {
4119e39c5baSBill Taylor tavor_umap_db_free(umapdb);
4129e39c5baSBill Taylor }
4139e39c5baSBill Taylor mrshared_fail4:
4149e39c5baSBill Taylor if (mr_is_umem) {
4159e39c5baSBill Taylor ddi_umem_unlock(umem_cookie);
4169e39c5baSBill Taylor }
4179e39c5baSBill Taylor mrshared_fail3:
4189e39c5baSBill Taylor tavor_rsrc_free(state, &rsrc);
4199e39c5baSBill Taylor mrshared_fail2:
4209e39c5baSBill Taylor tavor_rsrc_free(state, &mpt);
4219e39c5baSBill Taylor mrshared_fail1:
4229e39c5baSBill Taylor tavor_pd_refcnt_dec(pd);
4239e39c5baSBill Taylor mrshared_fail:
4249e39c5baSBill Taylor return (status);
4259e39c5baSBill Taylor }
4269e39c5baSBill Taylor
4279e39c5baSBill Taylor
4289e39c5baSBill Taylor /*
4299e39c5baSBill Taylor * tavor_mr_deregister()
4309e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
4319e39c5baSBill Taylor */
4329e39c5baSBill Taylor /* ARGSUSED */
4339e39c5baSBill Taylor int
tavor_mr_deregister(tavor_state_t * state,tavor_mrhdl_t * mrhdl,uint_t level,uint_t sleep)4349e39c5baSBill Taylor tavor_mr_deregister(tavor_state_t *state, tavor_mrhdl_t *mrhdl, uint_t level,
4359e39c5baSBill Taylor uint_t sleep)
4369e39c5baSBill Taylor {
4379e39c5baSBill Taylor tavor_rsrc_t *mpt, *mtt, *rsrc, *mtt_refcnt;
4389e39c5baSBill Taylor tavor_umap_db_entry_t *umapdb;
4399e39c5baSBill Taylor tavor_pdhdl_t pd;
4409e39c5baSBill Taylor tavor_mrhdl_t mr;
4419e39c5baSBill Taylor tavor_bind_info_t *bind;
4429e39c5baSBill Taylor uint64_t value;
4439e39c5baSBill Taylor int status, shared_mtt;
4449e39c5baSBill Taylor
4459e39c5baSBill Taylor /*
4469e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the
4479e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt
4489e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep).
4499e39c5baSBill Taylor */
4509e39c5baSBill Taylor if ((sleep == TAVOR_SLEEP) &&
4519e39c5baSBill Taylor (sleep != TAVOR_SLEEPFLAG_FOR_CONTEXT())) {
4529e39c5baSBill Taylor return (status);
4539e39c5baSBill Taylor }
4549e39c5baSBill Taylor
4559e39c5baSBill Taylor /*
4569e39c5baSBill Taylor * Pull all the necessary information from the Tavor Memory Region
4579e39c5baSBill Taylor * handle. This is necessary here because the resource for the
4589e39c5baSBill Taylor * MR handle is going to be freed up as part of the this
4599e39c5baSBill Taylor * deregistration
4609e39c5baSBill Taylor */
4619e39c5baSBill Taylor mr = *mrhdl;
4629e39c5baSBill Taylor mutex_enter(&mr->mr_lock);
4639e39c5baSBill Taylor mpt = mr->mr_mptrsrcp;
4649e39c5baSBill Taylor mtt = mr->mr_mttrsrcp;
4659e39c5baSBill Taylor mtt_refcnt = mr->mr_mttrefcntp;
4669e39c5baSBill Taylor rsrc = mr->mr_rsrcp;
4679e39c5baSBill Taylor pd = mr->mr_pdhdl;
4689e39c5baSBill Taylor bind = &mr->mr_bindinfo;
4699e39c5baSBill Taylor
4709e39c5baSBill Taylor /*
4719e39c5baSBill Taylor * Check here to see if the memory region has already been partially
4729e39c5baSBill Taylor * deregistered as a result of the tavor_umap_umemlock_cb() callback.
4739e39c5baSBill Taylor * If so, then jump to the end and free the remaining resources.
4749e39c5baSBill Taylor */
4759e39c5baSBill Taylor if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
4769e39c5baSBill Taylor goto mrdereg_finish_cleanup;
4779e39c5baSBill Taylor }
4789e39c5baSBill Taylor
4799e39c5baSBill Taylor /*
4809e39c5baSBill Taylor * We must drop the "mr_lock" here to ensure that both SLEEP and
4819e39c5baSBill Taylor * NOSLEEP calls into the firmware work as expected. Also, if two
4829e39c5baSBill Taylor * threads are attemping to access this MR (via de-register,
4839e39c5baSBill Taylor * re-register, or otherwise), then we allow the firmware to enforce
4849e39c5baSBill Taylor * the checking, that only one deregister is valid.
4859e39c5baSBill Taylor */
4869e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
4879e39c5baSBill Taylor
4889e39c5baSBill Taylor /*
4899e39c5baSBill Taylor * Reclaim MPT entry from hardware (if necessary). Since the
4909e39c5baSBill Taylor * tavor_mr_deregister() routine is used in the memory region
4919e39c5baSBill Taylor * reregistration process as well, it is possible that we will
4929e39c5baSBill Taylor * not always wish to reclaim ownership of the MPT. Check the
4939e39c5baSBill Taylor * "level" arg and, if necessary, attempt to reclaim it. If
4949e39c5baSBill Taylor * the ownership transfer fails for any reason, we check to see
4959e39c5baSBill Taylor * what command status was returned from the hardware. The only
4969e39c5baSBill Taylor * "expected" error status is the one that indicates an attempt to
4979e39c5baSBill Taylor * deregister a memory region that has memory windows bound to it
4989e39c5baSBill Taylor */
4999e39c5baSBill Taylor if (level >= TAVOR_MR_DEREG_ALL) {
5009e39c5baSBill Taylor status = tavor_cmn_ownership_cmd_post(state, HW2SW_MPT,
5019e39c5baSBill Taylor NULL, 0, mpt->tr_indx, sleep);
5029e39c5baSBill Taylor if (status != TAVOR_CMD_SUCCESS) {
5039e39c5baSBill Taylor if (status == TAVOR_CMD_REG_BOUND) {
5049e39c5baSBill Taylor return (IBT_MR_IN_USE);
5059e39c5baSBill Taylor } else {
5069e39c5baSBill Taylor cmn_err(CE_CONT, "Tavor: HW2SW_MPT command "
5079e39c5baSBill Taylor "failed: %08x\n", status);
5089e39c5baSBill Taylor return (IBT_INVALID_PARAM);
5099e39c5baSBill Taylor }
5109e39c5baSBill Taylor }
5119e39c5baSBill Taylor }
5129e39c5baSBill Taylor
5139e39c5baSBill Taylor /*
5149e39c5baSBill Taylor * Re-grab the mr_lock here. Since further access to the protected
5159e39c5baSBill Taylor * 'mr' structure is needed, and we would have returned previously for
5169e39c5baSBill Taylor * the multiple deregistration case, we can safely grab the lock here.
5179e39c5baSBill Taylor */
5189e39c5baSBill Taylor mutex_enter(&mr->mr_lock);
5199e39c5baSBill Taylor
5209e39c5baSBill Taylor /*
5219e39c5baSBill Taylor * If the memory had come from userland, then we do a lookup in the
5229e39c5baSBill Taylor * "userland resources database". On success, we free the entry, call
5239e39c5baSBill Taylor * ddi_umem_unlock(), and continue the cleanup. On failure (which is
5249e39c5baSBill Taylor * an indication that the umem_lockmemory() callback has called
5259e39c5baSBill Taylor * tavor_mr_deregister()), we call ddi_umem_unlock() and invalidate
5269e39c5baSBill Taylor * the "mr_umemcookie" field in the MR handle (this will be used
5279e39c5baSBill Taylor * later to detect that only partial cleaup still remains to be done
5289e39c5baSBill Taylor * on the MR handle).
5299e39c5baSBill Taylor */
5309e39c5baSBill Taylor if (mr->mr_is_umem) {
5319e39c5baSBill Taylor status = tavor_umap_db_find(state->ts_instance,
5329e39c5baSBill Taylor (uint64_t)(uintptr_t)mr->mr_umemcookie,
5339e39c5baSBill Taylor MLNX_UMAP_MRMEM_RSRC, &value, TAVOR_UMAP_DB_REMOVE,
5349e39c5baSBill Taylor &umapdb);
5359e39c5baSBill Taylor if (status == DDI_SUCCESS) {
5369e39c5baSBill Taylor tavor_umap_db_free(umapdb);
5379e39c5baSBill Taylor ddi_umem_unlock(mr->mr_umemcookie);
5389e39c5baSBill Taylor } else {
5399e39c5baSBill Taylor ddi_umem_unlock(mr->mr_umemcookie);
5409e39c5baSBill Taylor mr->mr_umemcookie = NULL;
5419e39c5baSBill Taylor }
5429e39c5baSBill Taylor }
5439e39c5baSBill Taylor
54417a2b317SBill Taylor /* mtt_refcnt is NULL in the case of tavor_dma_mr_register() */
54517a2b317SBill Taylor if (mtt_refcnt != NULL) {
5469e39c5baSBill Taylor /*
5479e39c5baSBill Taylor * Decrement the MTT reference count. Since the MTT resource
5489e39c5baSBill Taylor * may be shared between multiple memory regions (as a result
5499e39c5baSBill Taylor * of a "RegisterSharedMR" verb) it is important that we not
55017a2b317SBill Taylor * free up or unbind resources prematurely. If it's not shared
55117a2b317SBill Taylor * (as indicated by the return status), then free the resource.
5529e39c5baSBill Taylor */
5539e39c5baSBill Taylor shared_mtt = tavor_mtt_refcnt_dec(mtt_refcnt);
5549e39c5baSBill Taylor if (!shared_mtt) {
5559e39c5baSBill Taylor tavor_rsrc_free(state, &mtt_refcnt);
5569e39c5baSBill Taylor }
5579e39c5baSBill Taylor
5589e39c5baSBill Taylor /*
55917a2b317SBill Taylor * Free up the MTT entries and unbind the memory. Here,
56017a2b317SBill Taylor * as above, we attempt to free these resources only if
56117a2b317SBill Taylor * it is appropriate to do so.
5629e39c5baSBill Taylor */
5639e39c5baSBill Taylor if (!shared_mtt) {
5649e39c5baSBill Taylor if (level >= TAVOR_MR_DEREG_NO_HW2SW_MPT) {
5659e39c5baSBill Taylor tavor_mr_mem_unbind(state, bind);
5669e39c5baSBill Taylor }
5679e39c5baSBill Taylor tavor_rsrc_free(state, &mtt);
5689e39c5baSBill Taylor }
56917a2b317SBill Taylor }
5709e39c5baSBill Taylor
5719e39c5baSBill Taylor /*
5729e39c5baSBill Taylor * If the MR handle has been invalidated, then drop the
5739e39c5baSBill Taylor * lock and return success. Note: This only happens because
5749e39c5baSBill Taylor * the umem_lockmemory() callback has been triggered. The
5759e39c5baSBill Taylor * cleanup here is partial, and further cleanup (in a
5769e39c5baSBill Taylor * subsequent tavor_mr_deregister() call) will be necessary.
5779e39c5baSBill Taylor */
5789e39c5baSBill Taylor if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
5799e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
5809e39c5baSBill Taylor return (DDI_SUCCESS);
5819e39c5baSBill Taylor }
5829e39c5baSBill Taylor
5839e39c5baSBill Taylor mrdereg_finish_cleanup:
5849e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
5859e39c5baSBill Taylor
5869e39c5baSBill Taylor /* Free the Tavor Memory Region handle */
5879e39c5baSBill Taylor tavor_rsrc_free(state, &rsrc);
5889e39c5baSBill Taylor
5899e39c5baSBill Taylor /* Free up the MPT entry resource */
5909e39c5baSBill Taylor tavor_rsrc_free(state, &mpt);
5919e39c5baSBill Taylor
5929e39c5baSBill Taylor /* Decrement the reference count on the protection domain (PD) */
5939e39c5baSBill Taylor tavor_pd_refcnt_dec(pd);
5949e39c5baSBill Taylor
5959e39c5baSBill Taylor /* Set the mrhdl pointer to NULL and return success */
5969e39c5baSBill Taylor *mrhdl = NULL;
5979e39c5baSBill Taylor
5989e39c5baSBill Taylor return (DDI_SUCCESS);
5999e39c5baSBill Taylor }
6009e39c5baSBill Taylor
6019e39c5baSBill Taylor
6029e39c5baSBill Taylor /*
6039e39c5baSBill Taylor * tavor_mr_query()
6049e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
6059e39c5baSBill Taylor */
6069e39c5baSBill Taylor /* ARGSUSED */
6079e39c5baSBill Taylor int
tavor_mr_query(tavor_state_t * state,tavor_mrhdl_t mr,ibt_mr_query_attr_t * attr)6089e39c5baSBill Taylor tavor_mr_query(tavor_state_t *state, tavor_mrhdl_t mr,
6099e39c5baSBill Taylor ibt_mr_query_attr_t *attr)
6109e39c5baSBill Taylor {
6119e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr))
6129e39c5baSBill Taylor
6139e39c5baSBill Taylor mutex_enter(&mr->mr_lock);
6149e39c5baSBill Taylor
6159e39c5baSBill Taylor /*
6169e39c5baSBill Taylor * Check here to see if the memory region has already been partially
6179e39c5baSBill Taylor * deregistered as a result of a tavor_umap_umemlock_cb() callback.
6189e39c5baSBill Taylor * If so, this is an error, return failure.
6199e39c5baSBill Taylor */
6209e39c5baSBill Taylor if ((mr->mr_is_umem) && (mr->mr_umemcookie == NULL)) {
6219e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
6229e39c5baSBill Taylor return (IBT_MR_HDL_INVALID);
6239e39c5baSBill Taylor }
6249e39c5baSBill Taylor
6259e39c5baSBill Taylor /* Fill in the queried attributes */
6269e39c5baSBill Taylor attr->mr_attr_flags = mr->mr_accflag;
6279e39c5baSBill Taylor attr->mr_pd = (ibt_pd_hdl_t)mr->mr_pdhdl;
6289e39c5baSBill Taylor
6299e39c5baSBill Taylor /* Fill in the "local" attributes */
6309e39c5baSBill Taylor attr->mr_lkey = (ibt_lkey_t)mr->mr_lkey;
6319e39c5baSBill Taylor attr->mr_lbounds.pb_addr = (ib_vaddr_t)mr->mr_bindinfo.bi_addr;
6329e39c5baSBill Taylor attr->mr_lbounds.pb_len = (size_t)mr->mr_bindinfo.bi_len;
6339e39c5baSBill Taylor
6349e39c5baSBill Taylor /*
6359e39c5baSBill Taylor * Fill in the "remote" attributes (if necessary). Note: the
6369e39c5baSBill Taylor * remote attributes are only valid if the memory region has one
6379e39c5baSBill Taylor * or more of the remote access flags set.
6389e39c5baSBill Taylor */
6399e39c5baSBill Taylor if ((mr->mr_accflag & IBT_MR_REMOTE_READ) ||
6409e39c5baSBill Taylor (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ||
6419e39c5baSBill Taylor (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC)) {
6429e39c5baSBill Taylor attr->mr_rkey = (ibt_rkey_t)mr->mr_rkey;
6439e39c5baSBill Taylor attr->mr_rbounds.pb_addr = (ib_vaddr_t)mr->mr_bindinfo.bi_addr;
6449e39c5baSBill Taylor attr->mr_rbounds.pb_len = (size_t)mr->mr_bindinfo.bi_len;
6459e39c5baSBill Taylor }
6469e39c5baSBill Taylor
6479e39c5baSBill Taylor /*
6489e39c5baSBill Taylor * If region is mapped for streaming (i.e. noncoherent), then set sync
6499e39c5baSBill Taylor * is required
6509e39c5baSBill Taylor */
6519e39c5baSBill Taylor attr->mr_sync_required = (mr->mr_bindinfo.bi_flags &
6529e39c5baSBill Taylor IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;
6539e39c5baSBill Taylor
6549e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
6559e39c5baSBill Taylor return (DDI_SUCCESS);
6569e39c5baSBill Taylor }
6579e39c5baSBill Taylor
6589e39c5baSBill Taylor
6599e39c5baSBill Taylor /*
6609e39c5baSBill Taylor * tavor_mr_reregister()
6619e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
6629e39c5baSBill Taylor */
6639e39c5baSBill Taylor int
tavor_mr_reregister(tavor_state_t * state,tavor_mrhdl_t mr,tavor_pdhdl_t pd,ibt_mr_attr_t * mr_attr,tavor_mrhdl_t * mrhdl_new,tavor_mr_options_t * op)6649e39c5baSBill Taylor tavor_mr_reregister(tavor_state_t *state, tavor_mrhdl_t mr,
6659e39c5baSBill Taylor tavor_pdhdl_t pd, ibt_mr_attr_t *mr_attr, tavor_mrhdl_t *mrhdl_new,
6669e39c5baSBill Taylor tavor_mr_options_t *op)
6679e39c5baSBill Taylor {
6689e39c5baSBill Taylor tavor_bind_info_t bind;
6699e39c5baSBill Taylor int status;
6709e39c5baSBill Taylor
6719e39c5baSBill Taylor /*
6729e39c5baSBill Taylor * Fill in the "bind" struct. This struct provides the majority
6739e39c5baSBill Taylor * of the information that will be used to distinguish between an
6749e39c5baSBill Taylor * "addr" binding (as is the case here) and a "buf" binding (see
6759e39c5baSBill Taylor * below). The "bind" struct is later passed to tavor_mr_mem_bind()
6769e39c5baSBill Taylor * which does most of the "heavy lifting" for the Tavor memory
6779e39c5baSBill Taylor * registration (and reregistration) routines.
6789e39c5baSBill Taylor */
6799e39c5baSBill Taylor bind.bi_type = TAVOR_BINDHDL_VADDR;
6809e39c5baSBill Taylor bind.bi_addr = mr_attr->mr_vaddr;
6819e39c5baSBill Taylor bind.bi_len = mr_attr->mr_len;
6829e39c5baSBill Taylor bind.bi_as = mr_attr->mr_as;
6839e39c5baSBill Taylor bind.bi_flags = mr_attr->mr_flags;
6849e39c5baSBill Taylor status = tavor_mr_common_rereg(state, mr, pd, &bind, mrhdl_new, op);
6859e39c5baSBill Taylor
686*2570281cSToomas Soome return (status);
6879e39c5baSBill Taylor }
6889e39c5baSBill Taylor
6899e39c5baSBill Taylor
6909e39c5baSBill Taylor /*
6919e39c5baSBill Taylor * tavor_mr_reregister_buf()
6929e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
6939e39c5baSBill Taylor */
6949e39c5baSBill Taylor int
tavor_mr_reregister_buf(tavor_state_t * state,tavor_mrhdl_t mr,tavor_pdhdl_t pd,ibt_smr_attr_t * mr_attr,struct buf * buf,tavor_mrhdl_t * mrhdl_new,tavor_mr_options_t * op)6959e39c5baSBill Taylor tavor_mr_reregister_buf(tavor_state_t *state, tavor_mrhdl_t mr,
6969e39c5baSBill Taylor tavor_pdhdl_t pd, ibt_smr_attr_t *mr_attr, struct buf *buf,
6979e39c5baSBill Taylor tavor_mrhdl_t *mrhdl_new, tavor_mr_options_t *op)
6989e39c5baSBill Taylor {
6999e39c5baSBill Taylor tavor_bind_info_t bind;
7009e39c5baSBill Taylor int status;
7019e39c5baSBill Taylor
7029e39c5baSBill Taylor /*
7039e39c5baSBill Taylor * Fill in the "bind" struct. This struct provides the majority
7049e39c5baSBill Taylor * of the information that will be used to distinguish between an
7059e39c5baSBill Taylor * "addr" binding (see above) and a "buf" binding (as is the case
7069e39c5baSBill Taylor * here). The "bind" struct is later passed to tavor_mr_mem_bind()
7079e39c5baSBill Taylor * which does most of the "heavy lifting" for the Tavor memory
7089e39c5baSBill Taylor * registration routines. Note: We have chosen to provide
7099e39c5baSBill Taylor * "b_un.b_addr" as the IB address (when the IBT_MR_PHYS_IOVA flag is
7109e39c5baSBill Taylor * not set). It is not critical what value we choose here as it need
7119e39c5baSBill Taylor * only be unique for the given RKey (which will happen by default),
7129e39c5baSBill Taylor * so the choice here is somewhat arbitrary.
7139e39c5baSBill Taylor */
7149e39c5baSBill Taylor bind.bi_type = TAVOR_BINDHDL_BUF;
7159e39c5baSBill Taylor bind.bi_buf = buf;
7169e39c5baSBill Taylor if (mr_attr->mr_flags & IBT_MR_PHYS_IOVA) {
7179e39c5baSBill Taylor bind.bi_addr = mr_attr->mr_vaddr;
7189e39c5baSBill Taylor } else {
7199e39c5baSBill Taylor bind.bi_addr = (uint64_t)(uintptr_t)buf->b_un.b_addr;
7209e39c5baSBill Taylor }
7219e39c5baSBill Taylor bind.bi_len = (uint64_t)buf->b_bcount;
7229e39c5baSBill Taylor bind.bi_flags = mr_attr->mr_flags;
7239e39c5baSBill Taylor bind.bi_as = NULL;
7249e39c5baSBill Taylor status = tavor_mr_common_rereg(state, mr, pd, &bind, mrhdl_new, op);
7259e39c5baSBill Taylor
726*2570281cSToomas Soome return (status);
7279e39c5baSBill Taylor }
7289e39c5baSBill Taylor
7299e39c5baSBill Taylor
7309e39c5baSBill Taylor /*
7319e39c5baSBill Taylor * tavor_mr_sync()
7329e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
7339e39c5baSBill Taylor */
7349e39c5baSBill Taylor /* ARGSUSED */
7359e39c5baSBill Taylor int
tavor_mr_sync(tavor_state_t * state,ibt_mr_sync_t * mr_segs,size_t num_segs)7369e39c5baSBill Taylor tavor_mr_sync(tavor_state_t *state, ibt_mr_sync_t *mr_segs, size_t num_segs)
7379e39c5baSBill Taylor {
7389e39c5baSBill Taylor tavor_mrhdl_t mrhdl;
7399e39c5baSBill Taylor uint64_t seg_vaddr, seg_len, seg_end;
7409e39c5baSBill Taylor uint64_t mr_start, mr_end;
7419e39c5baSBill Taylor uint_t type;
7429e39c5baSBill Taylor int status, i;
7439e39c5baSBill Taylor
7449e39c5baSBill Taylor /* Process each of the ibt_mr_sync_t's */
7459e39c5baSBill Taylor for (i = 0; i < num_segs; i++) {
7469e39c5baSBill Taylor mrhdl = (tavor_mrhdl_t)mr_segs[i].ms_handle;
7479e39c5baSBill Taylor
7489e39c5baSBill Taylor /* Check for valid memory region handle */
7499e39c5baSBill Taylor if (mrhdl == NULL) {
7509e39c5baSBill Taylor goto mrsync_fail;
7519e39c5baSBill Taylor }
7529e39c5baSBill Taylor
7539e39c5baSBill Taylor mutex_enter(&mrhdl->mr_lock);
7549e39c5baSBill Taylor
7559e39c5baSBill Taylor /*
7569e39c5baSBill Taylor * Check here to see if the memory region has already been
7579e39c5baSBill Taylor * partially deregistered as a result of a
7589e39c5baSBill Taylor * tavor_umap_umemlock_cb() callback. If so, this is an
7599e39c5baSBill Taylor * error, return failure.
7609e39c5baSBill Taylor */
7619e39c5baSBill Taylor if ((mrhdl->mr_is_umem) && (mrhdl->mr_umemcookie == NULL)) {
7629e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock);
7639e39c5baSBill Taylor goto mrsync_fail;
7649e39c5baSBill Taylor }
7659e39c5baSBill Taylor
7669e39c5baSBill Taylor /* Check for valid bounds on sync request */
7679e39c5baSBill Taylor seg_vaddr = mr_segs[i].ms_vaddr;
7689e39c5baSBill Taylor seg_len = mr_segs[i].ms_len;
7699e39c5baSBill Taylor seg_end = seg_vaddr + seg_len - 1;
7709e39c5baSBill Taylor mr_start = mrhdl->mr_bindinfo.bi_addr;
7719e39c5baSBill Taylor mr_end = mr_start + mrhdl->mr_bindinfo.bi_len - 1;
7729e39c5baSBill Taylor if ((seg_vaddr < mr_start) || (seg_vaddr > mr_end)) {
7739e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock);
7749e39c5baSBill Taylor goto mrsync_fail;
7759e39c5baSBill Taylor }
7769e39c5baSBill Taylor if ((seg_end < mr_start) || (seg_end > mr_end)) {
7779e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock);
7789e39c5baSBill Taylor goto mrsync_fail;
7799e39c5baSBill Taylor }
7809e39c5baSBill Taylor
7819e39c5baSBill Taylor /* Determine what type (i.e. direction) for sync */
7829e39c5baSBill Taylor if (mr_segs[i].ms_flags & IBT_SYNC_READ) {
7839e39c5baSBill Taylor type = DDI_DMA_SYNC_FORDEV;
7849e39c5baSBill Taylor } else if (mr_segs[i].ms_flags & IBT_SYNC_WRITE) {
7859e39c5baSBill Taylor type = DDI_DMA_SYNC_FORCPU;
7869e39c5baSBill Taylor } else {
7879e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock);
7889e39c5baSBill Taylor goto mrsync_fail;
7899e39c5baSBill Taylor }
7909e39c5baSBill Taylor
7919e39c5baSBill Taylor (void) ddi_dma_sync(mrhdl->mr_bindinfo.bi_dmahdl,
7929e39c5baSBill Taylor (off_t)(seg_vaddr - mr_start), (size_t)seg_len, type);
7939e39c5baSBill Taylor mutex_exit(&mrhdl->mr_lock);
7949e39c5baSBill Taylor }
7959e39c5baSBill Taylor
7969e39c5baSBill Taylor return (DDI_SUCCESS);
7979e39c5baSBill Taylor
7989e39c5baSBill Taylor mrsync_fail:
7999e39c5baSBill Taylor return (status);
8009e39c5baSBill Taylor }
8019e39c5baSBill Taylor
8029e39c5baSBill Taylor
8039e39c5baSBill Taylor /*
8049e39c5baSBill Taylor * tavor_mw_alloc()
8059e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
8069e39c5baSBill Taylor */
8079e39c5baSBill Taylor int
tavor_mw_alloc(tavor_state_t * state,tavor_pdhdl_t pd,ibt_mw_flags_t flags,tavor_mwhdl_t * mwhdl)8089e39c5baSBill Taylor tavor_mw_alloc(tavor_state_t *state, tavor_pdhdl_t pd, ibt_mw_flags_t flags,
8099e39c5baSBill Taylor tavor_mwhdl_t *mwhdl)
8109e39c5baSBill Taylor {
8119e39c5baSBill Taylor tavor_rsrc_t *mpt, *rsrc;
8129e39c5baSBill Taylor tavor_hw_mpt_t mpt_entry;
8139e39c5baSBill Taylor tavor_mwhdl_t mw;
8149e39c5baSBill Taylor uint_t sleep;
8159e39c5baSBill Taylor int status;
8169e39c5baSBill Taylor
8179e39c5baSBill Taylor /*
8189e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the
8199e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt
8209e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep).
8219e39c5baSBill Taylor */
8229e39c5baSBill Taylor sleep = (flags & IBT_MW_NOSLEEP) ? TAVOR_NOSLEEP : TAVOR_SLEEP;
8239e39c5baSBill Taylor if ((sleep == TAVOR_SLEEP) &&
8249e39c5baSBill Taylor (sleep != TAVOR_SLEEPFLAG_FOR_CONTEXT())) {
8259e39c5baSBill Taylor goto mwalloc_fail;
8269e39c5baSBill Taylor }
8279e39c5baSBill Taylor
8289e39c5baSBill Taylor /* Increment the reference count on the protection domain (PD) */
8299e39c5baSBill Taylor tavor_pd_refcnt_inc(pd);
8309e39c5baSBill Taylor
8319e39c5baSBill Taylor /*
8329e39c5baSBill Taylor * Allocate an MPT entry (for use as a memory window). Since the
8339e39c5baSBill Taylor * Tavor hardware uses the MPT entry for memory regions and for
8349e39c5baSBill Taylor * memory windows, we will fill in this MPT with all the necessary
8359e39c5baSBill Taylor * parameters for the memory window. And then (just as we do for
8369e39c5baSBill Taylor * memory regions) ownership will be passed to the hardware in the
8379e39c5baSBill Taylor * final step below. If we fail here, we must undo the protection
8389e39c5baSBill Taylor * domain reference count.
8399e39c5baSBill Taylor */
8409e39c5baSBill Taylor status = tavor_rsrc_alloc(state, TAVOR_MPT, 1, sleep, &mpt);
8419e39c5baSBill Taylor if (status != DDI_SUCCESS) {
8429e39c5baSBill Taylor goto mwalloc_fail1;
8439e39c5baSBill Taylor }
8449e39c5baSBill Taylor
8459e39c5baSBill Taylor /*
8469e39c5baSBill Taylor * Allocate the software structure for tracking the memory window (i.e.
8479e39c5baSBill Taylor * the Tavor Memory Window handle). Note: This is actually the same
8489e39c5baSBill Taylor * software structure used for tracking memory regions, but since many
8499e39c5baSBill Taylor * of the same properties are needed, only a single structure is
8509e39c5baSBill Taylor * necessary. If we fail here, we must undo the protection domain
8519e39c5baSBill Taylor * reference count and the previous resource allocation.
8529e39c5baSBill Taylor */
8539e39c5baSBill Taylor status = tavor_rsrc_alloc(state, TAVOR_MRHDL, 1, sleep, &rsrc);
8549e39c5baSBill Taylor if (status != DDI_SUCCESS) {
8559e39c5baSBill Taylor goto mwalloc_fail2;
8569e39c5baSBill Taylor }
8579e39c5baSBill Taylor mw = (tavor_mwhdl_t)rsrc->tr_addr;
8589e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mw))
8599e39c5baSBill Taylor
8609e39c5baSBill Taylor /*
8619e39c5baSBill Taylor * Calculate an "unbound" RKey from MPT index. In much the same way
8629e39c5baSBill Taylor * as we do for memory regions (above), this key is constructed from
8639e39c5baSBill Taylor * a "constrained" (which depends on the MPT index) and an
8649e39c5baSBill Taylor * "unconstrained" portion (which may be arbitrarily chosen).
8659e39c5baSBill Taylor */
8669e39c5baSBill Taylor tavor_mr_keycalc(state, mpt->tr_indx, &mw->mr_rkey);
8679e39c5baSBill Taylor
8689e39c5baSBill Taylor /*
8699e39c5baSBill Taylor * Fill in the MPT entry. This is the final step before passing
8709e39c5baSBill Taylor * ownership of the MPT entry to the Tavor hardware. We use all of
8719e39c5baSBill Taylor * the information collected/calculated above to fill in the
8729e39c5baSBill Taylor * requisite portions of the MPT. Note: fewer entries in the MPT
8739e39c5baSBill Taylor * entry are necessary to allocate a memory window.
8749e39c5baSBill Taylor */
8759e39c5baSBill Taylor bzero(&mpt_entry, sizeof (tavor_hw_mpt_t));
8769e39c5baSBill Taylor mpt_entry.reg_win = TAVOR_MPT_IS_WINDOW;
8779e39c5baSBill Taylor mpt_entry.mem_key = mw->mr_rkey;
8789e39c5baSBill Taylor mpt_entry.pd = pd->pd_pdnum;
8799e39c5baSBill Taylor
8809e39c5baSBill Taylor /*
8819e39c5baSBill Taylor * Write the MPT entry to hardware. Lastly, we pass ownership of
8829e39c5baSBill Taylor * the entry to the hardware. Note: in general, this operation
8839e39c5baSBill Taylor * shouldn't fail. But if it does, we have to undo everything we've
8849e39c5baSBill Taylor * done above before returning error.
8859e39c5baSBill Taylor */
8869e39c5baSBill Taylor status = tavor_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
8879e39c5baSBill Taylor sizeof (tavor_hw_mpt_t), mpt->tr_indx, sleep);
8889e39c5baSBill Taylor if (status != TAVOR_CMD_SUCCESS) {
8899e39c5baSBill Taylor cmn_err(CE_CONT, "Tavor: SW2HW_MPT command failed: %08x\n",
8909e39c5baSBill Taylor status);
8919e39c5baSBill Taylor goto mwalloc_fail3;
8929e39c5baSBill Taylor }
8939e39c5baSBill Taylor
8949e39c5baSBill Taylor /*
8959e39c5baSBill Taylor * Fill in the rest of the Tavor Memory Window handle. Having
8969e39c5baSBill Taylor * successfully transferred ownership of the MPT, we can update the
8979e39c5baSBill Taylor * following fields for use in further operations on the MW.
8989e39c5baSBill Taylor */
8999e39c5baSBill Taylor mw->mr_mptrsrcp = mpt;
9009e39c5baSBill Taylor mw->mr_pdhdl = pd;
9019e39c5baSBill Taylor mw->mr_rsrcp = rsrc;
9029e39c5baSBill Taylor *mwhdl = mw;
9039e39c5baSBill Taylor
9049e39c5baSBill Taylor return (DDI_SUCCESS);
9059e39c5baSBill Taylor
9069e39c5baSBill Taylor mwalloc_fail3:
9079e39c5baSBill Taylor tavor_rsrc_free(state, &rsrc);
9089e39c5baSBill Taylor mwalloc_fail2:
9099e39c5baSBill Taylor tavor_rsrc_free(state, &mpt);
9109e39c5baSBill Taylor mwalloc_fail1:
9119e39c5baSBill Taylor tavor_pd_refcnt_dec(pd);
9129e39c5baSBill Taylor mwalloc_fail:
9139e39c5baSBill Taylor return (status);
9149e39c5baSBill Taylor }
9159e39c5baSBill Taylor
9169e39c5baSBill Taylor
9179e39c5baSBill Taylor /*
9189e39c5baSBill Taylor * tavor_mw_free()
9199e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
9209e39c5baSBill Taylor */
9219e39c5baSBill Taylor int
tavor_mw_free(tavor_state_t * state,tavor_mwhdl_t * mwhdl,uint_t sleep)9229e39c5baSBill Taylor tavor_mw_free(tavor_state_t *state, tavor_mwhdl_t *mwhdl, uint_t sleep)
9239e39c5baSBill Taylor {
9249e39c5baSBill Taylor tavor_rsrc_t *mpt, *rsrc;
9259e39c5baSBill Taylor tavor_mwhdl_t mw;
9269e39c5baSBill Taylor int status;
9279e39c5baSBill Taylor tavor_pdhdl_t pd;
9289e39c5baSBill Taylor
9299e39c5baSBill Taylor /*
9309e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the
9319e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt
9329e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep).
9339e39c5baSBill Taylor */
9349e39c5baSBill Taylor if ((sleep == TAVOR_SLEEP) &&
9359e39c5baSBill Taylor (sleep != TAVOR_SLEEPFLAG_FOR_CONTEXT())) {
9369e39c5baSBill Taylor return (status);
9379e39c5baSBill Taylor }
9389e39c5baSBill Taylor
9399e39c5baSBill Taylor /*
9409e39c5baSBill Taylor * Pull all the necessary information from the Tavor Memory Window
9419e39c5baSBill Taylor * handle. This is necessary here because the resource for the
9429e39c5baSBill Taylor * MW handle is going to be freed up as part of the this operation.
9439e39c5baSBill Taylor */
9449e39c5baSBill Taylor mw = *mwhdl;
9459e39c5baSBill Taylor mutex_enter(&mw->mr_lock);
9469e39c5baSBill Taylor mpt = mw->mr_mptrsrcp;
9479e39c5baSBill Taylor rsrc = mw->mr_rsrcp;
9489e39c5baSBill Taylor pd = mw->mr_pdhdl;
9499e39c5baSBill Taylor mutex_exit(&mw->mr_lock);
9509e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mw))
9519e39c5baSBill Taylor
9529e39c5baSBill Taylor /*
9539e39c5baSBill Taylor * Reclaim the MPT entry from hardware. Note: in general, it is
9549e39c5baSBill Taylor * unexpected for this operation to return an error.
9559e39c5baSBill Taylor */
9569e39c5baSBill Taylor status = tavor_cmn_ownership_cmd_post(state, HW2SW_MPT, NULL,
9579e39c5baSBill Taylor 0, mpt->tr_indx, sleep);
9589e39c5baSBill Taylor if (status != TAVOR_CMD_SUCCESS) {
9599e39c5baSBill Taylor cmn_err(CE_CONT, "Tavor: HW2SW_MPT command failed: %08x\n",
9609e39c5baSBill Taylor status);
9619e39c5baSBill Taylor return (IBT_INVALID_PARAM);
9629e39c5baSBill Taylor }
9639e39c5baSBill Taylor
9649e39c5baSBill Taylor /* Free the Tavor Memory Window handle */
9659e39c5baSBill Taylor tavor_rsrc_free(state, &rsrc);
9669e39c5baSBill Taylor
9679e39c5baSBill Taylor /* Free up the MPT entry resource */
9689e39c5baSBill Taylor tavor_rsrc_free(state, &mpt);
9699e39c5baSBill Taylor
9709e39c5baSBill Taylor /* Decrement the reference count on the protection domain (PD) */
9719e39c5baSBill Taylor tavor_pd_refcnt_dec(pd);
9729e39c5baSBill Taylor
9739e39c5baSBill Taylor /* Set the mwhdl pointer to NULL and return success */
9749e39c5baSBill Taylor *mwhdl = NULL;
9759e39c5baSBill Taylor
9769e39c5baSBill Taylor return (DDI_SUCCESS);
9779e39c5baSBill Taylor }
9789e39c5baSBill Taylor
9799e39c5baSBill Taylor
9809e39c5baSBill Taylor /*
9819e39c5baSBill Taylor * tavor_mr_keycalc()
9829e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
9839e39c5baSBill Taylor */
9849e39c5baSBill Taylor void
tavor_mr_keycalc(tavor_state_t * state,uint32_t indx,uint32_t * key)9859e39c5baSBill Taylor tavor_mr_keycalc(tavor_state_t *state, uint32_t indx, uint32_t *key)
9869e39c5baSBill Taylor {
9879e39c5baSBill Taylor uint32_t tmp, log_num_mpt;
9889e39c5baSBill Taylor
9899e39c5baSBill Taylor /*
9909e39c5baSBill Taylor * Generate a simple key from counter. Note: We increment this
9919e39c5baSBill Taylor * static variable _intentionally_ without any kind of mutex around
9929e39c5baSBill Taylor * it. First, single-threading all operations through a single lock
9939e39c5baSBill Taylor * would be a bad idea (from a performance point-of-view). Second,
9949e39c5baSBill Taylor * the upper "unconstrained" bits don't really have to be unique
9959e39c5baSBill Taylor * because the lower bits are guaranteed to be (although we do make a
9969e39c5baSBill Taylor * best effort to ensure that they are). Third, the window for the
9979e39c5baSBill Taylor * race (where both threads read and update the counter at the same
9989e39c5baSBill Taylor * time) is incredibly small.
9999e39c5baSBill Taylor * And, lastly, we'd like to make this into a "random" key XXX
10009e39c5baSBill Taylor */
10019e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(tavor_debug_memkey_cnt))
10029e39c5baSBill Taylor log_num_mpt = state->ts_cfg_profile->cp_log_num_mpt;
10039e39c5baSBill Taylor tmp = (tavor_debug_memkey_cnt++) << log_num_mpt;
10049e39c5baSBill Taylor *key = tmp | indx;
10059e39c5baSBill Taylor }
10069e39c5baSBill Taylor
10079e39c5baSBill Taylor
10089e39c5baSBill Taylor /*
10099e39c5baSBill Taylor * tavor_mr_common_reg()
10109e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
10119e39c5baSBill Taylor */
10129e39c5baSBill Taylor static int
tavor_mr_common_reg(tavor_state_t * state,tavor_pdhdl_t pd,tavor_bind_info_t * bind,tavor_mrhdl_t * mrhdl,tavor_mr_options_t * op)10139e39c5baSBill Taylor tavor_mr_common_reg(tavor_state_t *state, tavor_pdhdl_t pd,
10149e39c5baSBill Taylor tavor_bind_info_t *bind, tavor_mrhdl_t *mrhdl, tavor_mr_options_t *op)
10159e39c5baSBill Taylor {
10169e39c5baSBill Taylor tavor_rsrc_pool_info_t *rsrc_pool;
10179e39c5baSBill Taylor tavor_rsrc_t *mpt, *mtt, *rsrc, *mtt_refcnt;
10189e39c5baSBill Taylor tavor_umap_db_entry_t *umapdb;
10199e39c5baSBill Taylor tavor_sw_refcnt_t *swrc_tmp;
10209e39c5baSBill Taylor tavor_hw_mpt_t mpt_entry;
10219e39c5baSBill Taylor tavor_mrhdl_t mr;
10229e39c5baSBill Taylor ibt_mr_flags_t flags;
10239e39c5baSBill Taylor tavor_bind_info_t *bh;
10249e39c5baSBill Taylor ddi_dma_handle_t bind_dmahdl;
10259e39c5baSBill Taylor ddi_umem_cookie_t umem_cookie;
10269e39c5baSBill Taylor size_t umem_len;
10279e39c5baSBill Taylor caddr_t umem_addr;
10289e39c5baSBill Taylor uint64_t mtt_addr, mtt_ddrbaseaddr, max_sz;
10299e39c5baSBill Taylor uint_t sleep, mtt_pgsize_bits, bind_type, mr_is_umem;
10309e39c5baSBill Taylor int status, umem_flags, bind_override_addr;
10319e39c5baSBill Taylor
10329e39c5baSBill Taylor /*
10339e39c5baSBill Taylor * Check the "options" flag. Currently this flag tells the driver
10349e39c5baSBill Taylor * whether or not the region should be bound normally (i.e. with
10359e39c5baSBill Taylor * entries written into the PCI IOMMU), whether it should be
10369e39c5baSBill Taylor * registered to bypass the IOMMU, and whether or not the resulting
10379e39c5baSBill Taylor * address should be "zero-based" (to aid the alignment restrictions
10389e39c5baSBill Taylor * for QPs).
10399e39c5baSBill Taylor */
10409e39c5baSBill Taylor if (op == NULL) {
10419e39c5baSBill Taylor bind_type = TAVOR_BINDMEM_NORMAL;
10429e39c5baSBill Taylor bind_dmahdl = NULL;
10439e39c5baSBill Taylor bind_override_addr = 0;
10449e39c5baSBill Taylor } else {
10459e39c5baSBill Taylor bind_type = op->mro_bind_type;
10469e39c5baSBill Taylor bind_dmahdl = op->mro_bind_dmahdl;
10479e39c5baSBill Taylor bind_override_addr = op->mro_bind_override_addr;
10489e39c5baSBill Taylor }
10499e39c5baSBill Taylor
10509e39c5baSBill Taylor /* Extract the flags field from the tavor_bind_info_t */
10519e39c5baSBill Taylor flags = bind->bi_flags;
10529e39c5baSBill Taylor
10539e39c5baSBill Taylor /*
10549e39c5baSBill Taylor * Check for invalid length. Check is the length is zero or if the
10559e39c5baSBill Taylor * length is larger than the maximum configured value. Return error
10569e39c5baSBill Taylor * if it is.
10579e39c5baSBill Taylor */
10589e39c5baSBill Taylor max_sz = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_mrw_sz);
10599e39c5baSBill Taylor if ((bind->bi_len == 0) || (bind->bi_len > max_sz)) {
10609e39c5baSBill Taylor goto mrcommon_fail;
10619e39c5baSBill Taylor }
10629e39c5baSBill Taylor
10639e39c5baSBill Taylor /*
10649e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the
10659e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt
10669e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep).
10679e39c5baSBill Taylor */
10689e39c5baSBill Taylor sleep = (flags & IBT_MR_NOSLEEP) ? TAVOR_NOSLEEP: TAVOR_SLEEP;
10699e39c5baSBill Taylor if ((sleep == TAVOR_SLEEP) &&
10709e39c5baSBill Taylor (sleep != TAVOR_SLEEPFLAG_FOR_CONTEXT())) {
10719e39c5baSBill Taylor goto mrcommon_fail;
10729e39c5baSBill Taylor }
10739e39c5baSBill Taylor
10749e39c5baSBill Taylor /*
10759e39c5baSBill Taylor * Get the base address for the MTT table. This will be necessary
10769e39c5baSBill Taylor * below when we are setting up the MPT entry.
10779e39c5baSBill Taylor */
10789e39c5baSBill Taylor rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
10799e39c5baSBill Taylor mtt_ddrbaseaddr = (uint64_t)(uintptr_t)rsrc_pool->rsrc_ddr_offset;
10809e39c5baSBill Taylor
10819e39c5baSBill Taylor /* Increment the reference count on the protection domain (PD) */
10829e39c5baSBill Taylor tavor_pd_refcnt_inc(pd);
10839e39c5baSBill Taylor
10849e39c5baSBill Taylor /*
10859e39c5baSBill Taylor * Allocate an MPT entry. This will be filled in with all the
10869e39c5baSBill Taylor * necessary parameters to define the memory region. And then
10879e39c5baSBill Taylor * ownership will be passed to the hardware in the final step
10889e39c5baSBill Taylor * below. If we fail here, we must undo the protection domain
10899e39c5baSBill Taylor * reference count.
10909e39c5baSBill Taylor */
10919e39c5baSBill Taylor status = tavor_rsrc_alloc(state, TAVOR_MPT, 1, sleep, &mpt);
10929e39c5baSBill Taylor if (status != DDI_SUCCESS) {
10939e39c5baSBill Taylor goto mrcommon_fail1;
10949e39c5baSBill Taylor }
10959e39c5baSBill Taylor
10969e39c5baSBill Taylor /*
10979e39c5baSBill Taylor * Allocate the software structure for tracking the memory region (i.e.
10989e39c5baSBill Taylor * the Tavor Memory Region handle). If we fail here, we must undo
10999e39c5baSBill Taylor * the protection domain reference count and the previous resource
11009e39c5baSBill Taylor * allocation.
11019e39c5baSBill Taylor */
11029e39c5baSBill Taylor status = tavor_rsrc_alloc(state, TAVOR_MRHDL, 1, sleep, &rsrc);
11039e39c5baSBill Taylor if (status != DDI_SUCCESS) {
11049e39c5baSBill Taylor goto mrcommon_fail2;
11059e39c5baSBill Taylor }
11069e39c5baSBill Taylor mr = (tavor_mrhdl_t)rsrc->tr_addr;
11079e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
11089e39c5baSBill Taylor
11099e39c5baSBill Taylor /*
11109e39c5baSBill Taylor * Setup and validate the memory region access flags. This means
11119e39c5baSBill Taylor * translating the IBTF's enable flags into the access flags that
11129e39c5baSBill Taylor * will be used in later operations.
11139e39c5baSBill Taylor */
11149e39c5baSBill Taylor mr->mr_accflag = 0;
11159e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_WINDOW_BIND)
11169e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_WINDOW_BIND;
11179e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_LOCAL_WRITE)
11189e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_LOCAL_WRITE;
11199e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_READ)
11209e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_READ;
11219e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_WRITE)
11229e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_WRITE;
11239e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_ATOMIC)
11249e39c5baSBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC;
11259e39c5baSBill Taylor
11269e39c5baSBill Taylor /*
11279e39c5baSBill Taylor * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed
11289e39c5baSBill Taylor * from a certain number of "constrained" bits (the least significant
11299e39c5baSBill Taylor * bits) and some number of "unconstrained" bits. The constrained
11309e39c5baSBill Taylor * bits must be set to the index of the entry in the MPT table, but
11319e39c5baSBill Taylor * the unconstrained bits can be set to any value we wish. Note:
11329e39c5baSBill Taylor * if no remote access is required, then the RKey value is not filled
11339e39c5baSBill Taylor * in. Otherwise both Rkey and LKey are given the same value.
11349e39c5baSBill Taylor */
11359e39c5baSBill Taylor tavor_mr_keycalc(state, mpt->tr_indx, &mr->mr_lkey);
11369e39c5baSBill Taylor if ((mr->mr_accflag & IBT_MR_REMOTE_READ) ||
11379e39c5baSBill Taylor (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ||
11389e39c5baSBill Taylor (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC)) {
11399e39c5baSBill Taylor mr->mr_rkey = mr->mr_lkey;
11409e39c5baSBill Taylor }
11419e39c5baSBill Taylor
11429e39c5baSBill Taylor /*
11439e39c5baSBill Taylor * Determine if the memory is from userland and pin the pages
11449e39c5baSBill Taylor * with umem_lockmemory() if necessary.
11459e39c5baSBill Taylor * Then, if this is userland memory, allocate an entry in the
11469e39c5baSBill Taylor * "userland resources database". This will later be added to
11479e39c5baSBill Taylor * the database (after all further memory registration operations are
11489e39c5baSBill Taylor * successful). If we fail here, we must undo the reference counts
11499e39c5baSBill Taylor * and the previous resource allocations.
11509e39c5baSBill Taylor */
11519e39c5baSBill Taylor mr_is_umem = (((bind->bi_as != NULL) && (bind->bi_as != &kas)) ? 1 : 0);
11529e39c5baSBill Taylor if (mr_is_umem) {
11539e39c5baSBill Taylor umem_len = ptob(btopr(bind->bi_len +
11549e39c5baSBill Taylor ((uintptr_t)bind->bi_addr & PAGEOFFSET)));
11559e39c5baSBill Taylor umem_addr = (caddr_t)((uintptr_t)bind->bi_addr & ~PAGEOFFSET);
11569e39c5baSBill Taylor umem_flags = (DDI_UMEMLOCK_WRITE | DDI_UMEMLOCK_READ |
11579e39c5baSBill Taylor DDI_UMEMLOCK_LONGTERM);
11589e39c5baSBill Taylor status = umem_lockmemory(umem_addr, umem_len, umem_flags,
1159d863b343SBill Taylor &umem_cookie, &tavor_umem_cbops, NULL);
11609e39c5baSBill Taylor if (status != 0) {
11619e39c5baSBill Taylor goto mrcommon_fail3;
11629e39c5baSBill Taylor }
11639e39c5baSBill Taylor
11649e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
11659e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind->bi_buf))
11669e39c5baSBill Taylor
11679e39c5baSBill Taylor bind->bi_buf = ddi_umem_iosetup(umem_cookie, 0, umem_len,
11689e39c5baSBill Taylor B_WRITE, 0, 0, NULL, DDI_UMEM_SLEEP);
11699e39c5baSBill Taylor if (bind->bi_buf == NULL) {
11709e39c5baSBill Taylor goto mrcommon_fail3;
11719e39c5baSBill Taylor }
11729e39c5baSBill Taylor bind->bi_type = TAVOR_BINDHDL_UBUF;
11739e39c5baSBill Taylor bind->bi_buf->b_flags |= B_READ;
11749e39c5baSBill Taylor
11759e39c5baSBill Taylor _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*bind->bi_buf))
11769e39c5baSBill Taylor _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*bind))
11779e39c5baSBill Taylor
11789e39c5baSBill Taylor umapdb = tavor_umap_db_alloc(state->ts_instance,
11799e39c5baSBill Taylor (uint64_t)(uintptr_t)umem_cookie, MLNX_UMAP_MRMEM_RSRC,
11809e39c5baSBill Taylor (uint64_t)(uintptr_t)rsrc);
11819e39c5baSBill Taylor if (umapdb == NULL) {
11829e39c5baSBill Taylor goto mrcommon_fail4;
11839e39c5baSBill Taylor }
11849e39c5baSBill Taylor }
11859e39c5baSBill Taylor
11869e39c5baSBill Taylor /*
11879e39c5baSBill Taylor * Setup the bindinfo for the mtt bind call
11889e39c5baSBill Taylor */
11899e39c5baSBill Taylor bh = &mr->mr_bindinfo;
11909e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bh))
11919e39c5baSBill Taylor bcopy(bind, bh, sizeof (tavor_bind_info_t));
11929e39c5baSBill Taylor bh->bi_bypass = bind_type;
11939e39c5baSBill Taylor status = tavor_mr_mtt_bind(state, bh, bind_dmahdl, &mtt,
11949e39c5baSBill Taylor &mtt_pgsize_bits);
11959e39c5baSBill Taylor if (status != DDI_SUCCESS) {
1196949b58c7SBill Taylor /*
1197949b58c7SBill Taylor * When mtt_bind fails, freerbuf has already been done,
1198949b58c7SBill Taylor * so make sure not to call it again.
1199949b58c7SBill Taylor */
1200949b58c7SBill Taylor bind->bi_type = bh->bi_type;
12019e39c5baSBill Taylor goto mrcommon_fail5;
12029e39c5baSBill Taylor }
12039e39c5baSBill Taylor mr->mr_logmttpgsz = mtt_pgsize_bits;
12049e39c5baSBill Taylor
12059e39c5baSBill Taylor /*
12069e39c5baSBill Taylor * Allocate MTT reference count (to track shared memory regions).
12079e39c5baSBill Taylor * This reference count resource may never be used on the given
12089e39c5baSBill Taylor * memory region, but if it is ever later registered as "shared"
12099e39c5baSBill Taylor * memory region then this resource will be necessary. If we fail
12109e39c5baSBill Taylor * here, we do pretty much the same as above to clean up.
12119e39c5baSBill Taylor */
12129e39c5baSBill Taylor status = tavor_rsrc_alloc(state, TAVOR_REFCNT, 1, sleep,
12139e39c5baSBill Taylor &mtt_refcnt);
12149e39c5baSBill Taylor if (status != DDI_SUCCESS) {
12159e39c5baSBill Taylor goto mrcommon_fail6;
12169e39c5baSBill Taylor }
12179e39c5baSBill Taylor mr->mr_mttrefcntp = mtt_refcnt;
12189e39c5baSBill Taylor swrc_tmp = (tavor_sw_refcnt_t *)mtt_refcnt->tr_addr;
12199e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*swrc_tmp))
12209e39c5baSBill Taylor TAVOR_MTT_REFCNT_INIT(swrc_tmp);
12219e39c5baSBill Taylor
12229e39c5baSBill Taylor /*
12239e39c5baSBill Taylor * Fill in the MPT entry. This is the final step before passing
12249e39c5baSBill Taylor * ownership of the MPT entry to the Tavor hardware. We use all of
12259e39c5baSBill Taylor * the information collected/calculated above to fill in the
12269e39c5baSBill Taylor * requisite portions of the MPT.
12279e39c5baSBill Taylor */
12289e39c5baSBill Taylor bzero(&mpt_entry, sizeof (tavor_hw_mpt_t));
12299e39c5baSBill Taylor mpt_entry.m_io = TAVOR_MEM_CYCLE_GENERATE;
12309e39c5baSBill Taylor mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0;
12319e39c5baSBill Taylor mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
12329e39c5baSBill Taylor mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
12339e39c5baSBill Taylor mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
12349e39c5baSBill Taylor mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
12359e39c5baSBill Taylor mpt_entry.lr = 1;
12369e39c5baSBill Taylor mpt_entry.reg_win = TAVOR_MPT_IS_REGION;
12379e39c5baSBill Taylor mpt_entry.page_sz = mr->mr_logmttpgsz - 0xC;
12389e39c5baSBill Taylor mpt_entry.mem_key = mr->mr_lkey;
12399e39c5baSBill Taylor mpt_entry.pd = pd->pd_pdnum;
12409e39c5baSBill Taylor if (bind_override_addr == 0) {
12419e39c5baSBill Taylor mpt_entry.start_addr = bh->bi_addr;
12429e39c5baSBill Taylor } else {
12439e39c5baSBill Taylor bh->bi_addr = bh->bi_addr & ((1 << mr->mr_logmttpgsz) - 1);
12449e39c5baSBill Taylor mpt_entry.start_addr = bh->bi_addr;
12459e39c5baSBill Taylor }
12469e39c5baSBill Taylor mpt_entry.reg_win_len = bh->bi_len;
12479e39c5baSBill Taylor mpt_entry.win_cnt_limit = TAVOR_UNLIMITED_WIN_BIND;
12489e39c5baSBill Taylor mtt_addr = mtt_ddrbaseaddr + (mtt->tr_indx << TAVOR_MTT_SIZE_SHIFT);
12499e39c5baSBill Taylor mpt_entry.mttseg_addr_h = mtt_addr >> 32;
12509e39c5baSBill Taylor mpt_entry.mttseg_addr_l = mtt_addr >> 6;
12519e39c5baSBill Taylor
12529e39c5baSBill Taylor /*
12539e39c5baSBill Taylor * Write the MPT entry to hardware. Lastly, we pass ownership of
12549e39c5baSBill Taylor * the entry to the hardware. Note: in general, this operation
12559e39c5baSBill Taylor * shouldn't fail. But if it does, we have to undo everything we've
12569e39c5baSBill Taylor * done above before returning error.
12579e39c5baSBill Taylor */
12589e39c5baSBill Taylor status = tavor_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
12599e39c5baSBill Taylor sizeof (tavor_hw_mpt_t), mpt->tr_indx, sleep);
12609e39c5baSBill Taylor if (status != TAVOR_CMD_SUCCESS) {
12619e39c5baSBill Taylor cmn_err(CE_CONT, "Tavor: SW2HW_MPT command failed: %08x\n",
12629e39c5baSBill Taylor status);
12639e39c5baSBill Taylor goto mrcommon_fail7;
12649e39c5baSBill Taylor }
12659e39c5baSBill Taylor
12669e39c5baSBill Taylor /*
12679e39c5baSBill Taylor * Fill in the rest of the Tavor Memory Region handle. Having
12689e39c5baSBill Taylor * successfully transferred ownership of the MPT, we can update the
12699e39c5baSBill Taylor * following fields for use in further operations on the MR.
12709e39c5baSBill Taylor */
12719e39c5baSBill Taylor mr->mr_mptrsrcp = mpt;
12729e39c5baSBill Taylor mr->mr_mttrsrcp = mtt;
12739e39c5baSBill Taylor mr->mr_pdhdl = pd;
12749e39c5baSBill Taylor mr->mr_rsrcp = rsrc;
12759e39c5baSBill Taylor mr->mr_is_umem = mr_is_umem;
12769e39c5baSBill Taylor mr->mr_umemcookie = (mr_is_umem != 0) ? umem_cookie : NULL;
12779e39c5baSBill Taylor mr->mr_umem_cbfunc = NULL;
12789e39c5baSBill Taylor mr->mr_umem_cbarg1 = NULL;
12799e39c5baSBill Taylor mr->mr_umem_cbarg2 = NULL;
12809e39c5baSBill Taylor
12819e39c5baSBill Taylor /*
12829e39c5baSBill Taylor * If this is userland memory, then we need to insert the previously
12839e39c5baSBill Taylor * allocated entry into the "userland resources database". This will
12849e39c5baSBill Taylor * allow for later coordination between the tavor_umap_umemlock_cb()
12859e39c5baSBill Taylor * callback and tavor_mr_deregister().
12869e39c5baSBill Taylor */
12879e39c5baSBill Taylor if (mr_is_umem) {
12889e39c5baSBill Taylor tavor_umap_db_add(umapdb);
12899e39c5baSBill Taylor }
12909e39c5baSBill Taylor
12919e39c5baSBill Taylor *mrhdl = mr;
12929e39c5baSBill Taylor
12939e39c5baSBill Taylor return (DDI_SUCCESS);
12949e39c5baSBill Taylor
12959e39c5baSBill Taylor /*
12969e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine
12979e39c5baSBill Taylor */
12989e39c5baSBill Taylor mrcommon_fail7:
12999e39c5baSBill Taylor tavor_rsrc_free(state, &mtt_refcnt);
13009e39c5baSBill Taylor mrcommon_fail6:
13019e39c5baSBill Taylor tavor_rsrc_free(state, &mtt);
13029e39c5baSBill Taylor tavor_mr_mem_unbind(state, bh);
130336e3c695SPramod Gunjikar bind->bi_type = bh->bi_type;
13049e39c5baSBill Taylor mrcommon_fail5:
13059e39c5baSBill Taylor if (mr_is_umem) {
13069e39c5baSBill Taylor tavor_umap_db_free(umapdb);
13079e39c5baSBill Taylor }
13089e39c5baSBill Taylor mrcommon_fail4:
13099e39c5baSBill Taylor if (mr_is_umem) {
13109e39c5baSBill Taylor /*
13119e39c5baSBill Taylor * Free up the memory ddi_umem_iosetup() allocates
13129e39c5baSBill Taylor * internally.
13139e39c5baSBill Taylor */
13149e39c5baSBill Taylor if (bind->bi_type == TAVOR_BINDHDL_UBUF) {
13159e39c5baSBill Taylor freerbuf(bind->bi_buf);
13169e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
13179e39c5baSBill Taylor bind->bi_type = TAVOR_BINDHDL_NONE;
13189e39c5baSBill Taylor _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*bind))
13199e39c5baSBill Taylor }
13209e39c5baSBill Taylor ddi_umem_unlock(umem_cookie);
13219e39c5baSBill Taylor }
13229e39c5baSBill Taylor mrcommon_fail3:
13239e39c5baSBill Taylor tavor_rsrc_free(state, &rsrc);
13249e39c5baSBill Taylor mrcommon_fail2:
13259e39c5baSBill Taylor tavor_rsrc_free(state, &mpt);
13269e39c5baSBill Taylor mrcommon_fail1:
13279e39c5baSBill Taylor tavor_pd_refcnt_dec(pd);
13289e39c5baSBill Taylor mrcommon_fail:
13299e39c5baSBill Taylor return (status);
13309e39c5baSBill Taylor }
13319e39c5baSBill Taylor
133217a2b317SBill Taylor int
tavor_dma_mr_register(tavor_state_t * state,tavor_pdhdl_t pd,ibt_dmr_attr_t * mr_attr,tavor_mrhdl_t * mrhdl)133317a2b317SBill Taylor tavor_dma_mr_register(tavor_state_t *state, tavor_pdhdl_t pd,
133417a2b317SBill Taylor ibt_dmr_attr_t *mr_attr, tavor_mrhdl_t *mrhdl)
133517a2b317SBill Taylor {
133617a2b317SBill Taylor tavor_rsrc_t *mpt, *rsrc;
133717a2b317SBill Taylor tavor_hw_mpt_t mpt_entry;
133817a2b317SBill Taylor tavor_mrhdl_t mr;
133917a2b317SBill Taylor ibt_mr_flags_t flags;
134017a2b317SBill Taylor uint_t sleep;
134117a2b317SBill Taylor int status;
134217a2b317SBill Taylor
134317a2b317SBill Taylor /* Extract the flags field */
134417a2b317SBill Taylor flags = mr_attr->dmr_flags;
134517a2b317SBill Taylor
134617a2b317SBill Taylor /*
134717a2b317SBill Taylor * Check the sleep flag. Ensure that it is consistent with the
134817a2b317SBill Taylor * current thread context (i.e. if we are currently in the interrupt
134917a2b317SBill Taylor * context, then we shouldn't be attempting to sleep).
135017a2b317SBill Taylor */
135117a2b317SBill Taylor sleep = (flags & IBT_MR_NOSLEEP) ? TAVOR_NOSLEEP: TAVOR_SLEEP;
135217a2b317SBill Taylor if ((sleep == TAVOR_SLEEP) &&
135317a2b317SBill Taylor (sleep != TAVOR_SLEEPFLAG_FOR_CONTEXT())) {
135417a2b317SBill Taylor status = IBT_INVALID_PARAM;
135517a2b317SBill Taylor goto mrcommon_fail;
135617a2b317SBill Taylor }
135717a2b317SBill Taylor
135817a2b317SBill Taylor /* Increment the reference count on the protection domain (PD) */
135917a2b317SBill Taylor tavor_pd_refcnt_inc(pd);
136017a2b317SBill Taylor
136117a2b317SBill Taylor /*
136217a2b317SBill Taylor * Allocate an MPT entry. This will be filled in with all the
136317a2b317SBill Taylor * necessary parameters to define the memory region. And then
136417a2b317SBill Taylor * ownership will be passed to the hardware in the final step
136517a2b317SBill Taylor * below. If we fail here, we must undo the protection domain
136617a2b317SBill Taylor * reference count.
136717a2b317SBill Taylor */
136817a2b317SBill Taylor status = tavor_rsrc_alloc(state, TAVOR_MPT, 1, sleep, &mpt);
136917a2b317SBill Taylor if (status != DDI_SUCCESS) {
137017a2b317SBill Taylor status = IBT_INSUFF_RESOURCE;
137117a2b317SBill Taylor goto mrcommon_fail1;
137217a2b317SBill Taylor }
137317a2b317SBill Taylor
137417a2b317SBill Taylor /*
137517a2b317SBill Taylor * Allocate the software structure for tracking the memory region (i.e.
137617a2b317SBill Taylor * the Tavor Memory Region handle). If we fail here, we must undo
137717a2b317SBill Taylor * the protection domain reference count and the previous resource
137817a2b317SBill Taylor * allocation.
137917a2b317SBill Taylor */
138017a2b317SBill Taylor status = tavor_rsrc_alloc(state, TAVOR_MRHDL, 1, sleep, &rsrc);
138117a2b317SBill Taylor if (status != DDI_SUCCESS) {
138217a2b317SBill Taylor status = IBT_INSUFF_RESOURCE;
138317a2b317SBill Taylor goto mrcommon_fail2;
138417a2b317SBill Taylor }
138517a2b317SBill Taylor mr = (tavor_mrhdl_t)rsrc->tr_addr;
138617a2b317SBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
138717a2b317SBill Taylor bzero(mr, sizeof (*mr));
138817a2b317SBill Taylor
138917a2b317SBill Taylor /*
139017a2b317SBill Taylor * Setup and validate the memory region access flags. This means
139117a2b317SBill Taylor * translating the IBTF's enable flags into the access flags that
139217a2b317SBill Taylor * will be used in later operations.
139317a2b317SBill Taylor */
139417a2b317SBill Taylor mr->mr_accflag = 0;
139517a2b317SBill Taylor if (flags & IBT_MR_ENABLE_WINDOW_BIND)
139617a2b317SBill Taylor mr->mr_accflag |= IBT_MR_WINDOW_BIND;
139717a2b317SBill Taylor if (flags & IBT_MR_ENABLE_LOCAL_WRITE)
139817a2b317SBill Taylor mr->mr_accflag |= IBT_MR_LOCAL_WRITE;
139917a2b317SBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_READ)
140017a2b317SBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_READ;
140117a2b317SBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_WRITE)
140217a2b317SBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_WRITE;
140317a2b317SBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_ATOMIC)
140417a2b317SBill Taylor mr->mr_accflag |= IBT_MR_REMOTE_ATOMIC;
140517a2b317SBill Taylor
140617a2b317SBill Taylor /*
140717a2b317SBill Taylor * Calculate keys (Lkey, Rkey) from MPT index. Each key is formed
140817a2b317SBill Taylor * from a certain number of "constrained" bits (the least significant
140917a2b317SBill Taylor * bits) and some number of "unconstrained" bits. The constrained
141017a2b317SBill Taylor * bits must be set to the index of the entry in the MPT table, but
141117a2b317SBill Taylor * the unconstrained bits can be set to any value we wish. Note:
141217a2b317SBill Taylor * if no remote access is required, then the RKey value is not filled
141317a2b317SBill Taylor * in. Otherwise both Rkey and LKey are given the same value.
141417a2b317SBill Taylor */
141517a2b317SBill Taylor tavor_mr_keycalc(state, mpt->tr_indx, &mr->mr_lkey);
141617a2b317SBill Taylor if ((mr->mr_accflag & IBT_MR_REMOTE_READ) ||
141717a2b317SBill Taylor (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ||
141817a2b317SBill Taylor (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC)) {
141917a2b317SBill Taylor mr->mr_rkey = mr->mr_lkey;
142017a2b317SBill Taylor }
142117a2b317SBill Taylor
142217a2b317SBill Taylor /*
142317a2b317SBill Taylor * Fill in the MPT entry. This is the final step before passing
142417a2b317SBill Taylor * ownership of the MPT entry to the Tavor hardware. We use all of
142517a2b317SBill Taylor * the information collected/calculated above to fill in the
142617a2b317SBill Taylor * requisite portions of the MPT.
142717a2b317SBill Taylor */
142817a2b317SBill Taylor bzero(&mpt_entry, sizeof (tavor_hw_mpt_t));
142917a2b317SBill Taylor
143017a2b317SBill Taylor mpt_entry.m_io = TAVOR_MEM_CYCLE_GENERATE;
143117a2b317SBill Taylor mpt_entry.en_bind = (mr->mr_accflag & IBT_MR_WINDOW_BIND) ? 1 : 0;
143217a2b317SBill Taylor mpt_entry.atomic = (mr->mr_accflag & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
143317a2b317SBill Taylor mpt_entry.rw = (mr->mr_accflag & IBT_MR_REMOTE_WRITE) ? 1 : 0;
143417a2b317SBill Taylor mpt_entry.rr = (mr->mr_accflag & IBT_MR_REMOTE_READ) ? 1 : 0;
143517a2b317SBill Taylor mpt_entry.lw = (mr->mr_accflag & IBT_MR_LOCAL_WRITE) ? 1 : 0;
143617a2b317SBill Taylor mpt_entry.lr = 1;
143717a2b317SBill Taylor mpt_entry.phys_addr = 1; /* critical bit for this */
143817a2b317SBill Taylor mpt_entry.reg_win = TAVOR_MPT_IS_REGION;
143917a2b317SBill Taylor
144017a2b317SBill Taylor mpt_entry.page_sz = mr->mr_logmttpgsz - 0xC;
144117a2b317SBill Taylor mpt_entry.mem_key = mr->mr_lkey;
144217a2b317SBill Taylor mpt_entry.pd = pd->pd_pdnum;
144317a2b317SBill Taylor mpt_entry.win_cnt_limit = TAVOR_UNLIMITED_WIN_BIND;
144417a2b317SBill Taylor
144517a2b317SBill Taylor mpt_entry.start_addr = mr_attr->dmr_paddr;
144617a2b317SBill Taylor mpt_entry.reg_win_len = mr_attr->dmr_len;
144717a2b317SBill Taylor
144817a2b317SBill Taylor mpt_entry.mttseg_addr_h = 0;
144917a2b317SBill Taylor mpt_entry.mttseg_addr_l = 0;
145017a2b317SBill Taylor
145117a2b317SBill Taylor /*
145217a2b317SBill Taylor * Write the MPT entry to hardware. Lastly, we pass ownership of
145317a2b317SBill Taylor * the entry to the hardware if needed. Note: in general, this
145417a2b317SBill Taylor * operation shouldn't fail. But if it does, we have to undo
145517a2b317SBill Taylor * everything we've done above before returning error.
145617a2b317SBill Taylor *
145717a2b317SBill Taylor * For Tavor, this routine (which is common to the contexts) will only
145817a2b317SBill Taylor * set the ownership if needed - the process of passing the context
145917a2b317SBill Taylor * itself to HW will take care of setting up the MPT (based on type
146017a2b317SBill Taylor * and index).
146117a2b317SBill Taylor */
146217a2b317SBill Taylor
146317a2b317SBill Taylor status = tavor_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
146417a2b317SBill Taylor sizeof (tavor_hw_mpt_t), mpt->tr_indx, sleep);
146517a2b317SBill Taylor if (status != TAVOR_CMD_SUCCESS) {
146617a2b317SBill Taylor cmn_err(CE_CONT, "Tavor: SW2HW_MPT command failed: %08x\n",
146717a2b317SBill Taylor status);
146817a2b317SBill Taylor status = ibc_get_ci_failure(0);
146917a2b317SBill Taylor goto mrcommon_fail7;
147017a2b317SBill Taylor }
147117a2b317SBill Taylor
147217a2b317SBill Taylor /*
147317a2b317SBill Taylor * Fill in the rest of the Tavor Memory Region handle. Having
147417a2b317SBill Taylor * successfully transferred ownership of the MPT, we can update the
147517a2b317SBill Taylor * following fields for use in further operations on the MR.
147617a2b317SBill Taylor */
147717a2b317SBill Taylor mr->mr_mptrsrcp = mpt;
147817a2b317SBill Taylor mr->mr_mttrsrcp = NULL;
147917a2b317SBill Taylor mr->mr_pdhdl = pd;
148017a2b317SBill Taylor mr->mr_rsrcp = rsrc;
148117a2b317SBill Taylor mr->mr_is_umem = 0;
148217a2b317SBill Taylor mr->mr_umemcookie = NULL;
148317a2b317SBill Taylor mr->mr_umem_cbfunc = NULL;
148417a2b317SBill Taylor mr->mr_umem_cbarg1 = NULL;
148517a2b317SBill Taylor mr->mr_umem_cbarg2 = NULL;
148617a2b317SBill Taylor
148717a2b317SBill Taylor *mrhdl = mr;
148817a2b317SBill Taylor
148917a2b317SBill Taylor return (DDI_SUCCESS);
149017a2b317SBill Taylor
149117a2b317SBill Taylor /*
149217a2b317SBill Taylor * The following is cleanup for all possible failure cases in this routine
149317a2b317SBill Taylor */
149417a2b317SBill Taylor mrcommon_fail7:
149517a2b317SBill Taylor tavor_rsrc_free(state, &rsrc);
149617a2b317SBill Taylor mrcommon_fail2:
149717a2b317SBill Taylor tavor_rsrc_free(state, &mpt);
149817a2b317SBill Taylor mrcommon_fail1:
149917a2b317SBill Taylor tavor_pd_refcnt_dec(pd);
150017a2b317SBill Taylor mrcommon_fail:
150117a2b317SBill Taylor return (status);
150217a2b317SBill Taylor }
150317a2b317SBill Taylor
15049e39c5baSBill Taylor /*
15059e39c5baSBill Taylor * tavor_mr_mtt_bind()
15069e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
15079e39c5baSBill Taylor */
15089e39c5baSBill Taylor int
tavor_mr_mtt_bind(tavor_state_t * state,tavor_bind_info_t * bind,ddi_dma_handle_t bind_dmahdl,tavor_rsrc_t ** mtt,uint_t * mtt_pgsize_bits)15099e39c5baSBill Taylor tavor_mr_mtt_bind(tavor_state_t *state, tavor_bind_info_t *bind,
15109e39c5baSBill Taylor ddi_dma_handle_t bind_dmahdl, tavor_rsrc_t **mtt, uint_t *mtt_pgsize_bits)
15119e39c5baSBill Taylor {
15129e39c5baSBill Taylor uint64_t nummtt;
15139e39c5baSBill Taylor uint_t sleep;
15149e39c5baSBill Taylor int status;
15159e39c5baSBill Taylor
15169e39c5baSBill Taylor /*
15179e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the
15189e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt
15199e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep).
15209e39c5baSBill Taylor */
15219e39c5baSBill Taylor sleep = (bind->bi_flags & IBT_MR_NOSLEEP) ? TAVOR_NOSLEEP: TAVOR_SLEEP;
15229e39c5baSBill Taylor if ((sleep == TAVOR_SLEEP) &&
15239e39c5baSBill Taylor (sleep != TAVOR_SLEEPFLAG_FOR_CONTEXT())) {
15249e39c5baSBill Taylor goto mrmttbind_fail;
15259e39c5baSBill Taylor }
15269e39c5baSBill Taylor
15279e39c5baSBill Taylor /*
15289e39c5baSBill Taylor * Bind the memory and determine the mapped addresses. This is
15299e39c5baSBill Taylor * the first of two routines that do all the "heavy lifting" for
15309e39c5baSBill Taylor * the Tavor memory registration routines. The tavor_mr_mem_bind()
15319e39c5baSBill Taylor * routine takes the "bind" struct with all its fields filled
15329e39c5baSBill Taylor * in and returns a list of DMA cookies (for the PCI mapped addresses
15339e39c5baSBill Taylor * corresponding to the specified address region) which are used by
15349e39c5baSBill Taylor * the tavor_mr_fast_mtt_write() routine below. If we fail here, we
15359e39c5baSBill Taylor * must undo all the previous resource allocation (and PD reference
15369e39c5baSBill Taylor * count).
15379e39c5baSBill Taylor */
15389e39c5baSBill Taylor status = tavor_mr_mem_bind(state, bind, bind_dmahdl, sleep);
15399e39c5baSBill Taylor if (status != DDI_SUCCESS) {
15409e39c5baSBill Taylor goto mrmttbind_fail;
15419e39c5baSBill Taylor }
15429e39c5baSBill Taylor
15439e39c5baSBill Taylor /*
15449e39c5baSBill Taylor * Determine number of pages spanned. This routine uses the
15459e39c5baSBill Taylor * information in the "bind" struct to determine the required
15469e39c5baSBill Taylor * number of MTT entries needed (and returns the suggested page size -
15479e39c5baSBill Taylor * as a "power-of-2" - for each MTT entry).
15489e39c5baSBill Taylor */
15499e39c5baSBill Taylor nummtt = tavor_mr_nummtt_needed(state, bind, mtt_pgsize_bits);
15509e39c5baSBill Taylor
15519e39c5baSBill Taylor /*
15529e39c5baSBill Taylor * Allocate the MTT entries. Use the calculations performed above to
15539e39c5baSBill Taylor * allocate the required number of MTT entries. Note: MTT entries are
15549e39c5baSBill Taylor * allocated in "MTT segments" which consist of complete cachelines
15559e39c5baSBill Taylor * (i.e. 8 entries, 16 entries, etc.) So the TAVOR_NUMMTT_TO_MTTSEG()
15569e39c5baSBill Taylor * macro is used to do the proper conversion. If we fail here, we
15579e39c5baSBill Taylor * must not only undo all the previous resource allocation (and PD
15589e39c5baSBill Taylor * reference count), but we must also unbind the memory.
15599e39c5baSBill Taylor */
15609e39c5baSBill Taylor status = tavor_rsrc_alloc(state, TAVOR_MTT,
15619e39c5baSBill Taylor TAVOR_NUMMTT_TO_MTTSEG(nummtt), sleep, mtt);
15629e39c5baSBill Taylor if (status != DDI_SUCCESS) {
15639e39c5baSBill Taylor goto mrmttbind_fail2;
15649e39c5baSBill Taylor }
15659e39c5baSBill Taylor
15669e39c5baSBill Taylor /*
15679e39c5baSBill Taylor * Write the mapped addresses into the MTT entries. This is part two
15689e39c5baSBill Taylor * of the "heavy lifting" routines that we talked about above. Note:
15699e39c5baSBill Taylor * we pass the suggested page size from the earlier operation here.
15709e39c5baSBill Taylor * And if we fail here, we again do pretty much the same huge clean up.
15719e39c5baSBill Taylor */
15729e39c5baSBill Taylor status = tavor_mr_fast_mtt_write(*mtt, bind, *mtt_pgsize_bits);
15739e39c5baSBill Taylor if (status != DDI_SUCCESS) {
15749e39c5baSBill Taylor goto mrmttbind_fail3;
15759e39c5baSBill Taylor }
15769e39c5baSBill Taylor return (DDI_SUCCESS);
15779e39c5baSBill Taylor
15789e39c5baSBill Taylor /*
15799e39c5baSBill Taylor * The following is cleanup for all possible failure cases in this routine
15809e39c5baSBill Taylor */
15819e39c5baSBill Taylor mrmttbind_fail3:
15829e39c5baSBill Taylor tavor_rsrc_free(state, mtt);
15839e39c5baSBill Taylor mrmttbind_fail2:
15849e39c5baSBill Taylor tavor_mr_mem_unbind(state, bind);
15859e39c5baSBill Taylor mrmttbind_fail:
15869e39c5baSBill Taylor return (status);
15879e39c5baSBill Taylor }
15889e39c5baSBill Taylor
15899e39c5baSBill Taylor
15909e39c5baSBill Taylor /*
15919e39c5baSBill Taylor * tavor_mr_mtt_unbind()
15929e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
15939e39c5baSBill Taylor */
15949e39c5baSBill Taylor int
tavor_mr_mtt_unbind(tavor_state_t * state,tavor_bind_info_t * bind,tavor_rsrc_t * mtt)15959e39c5baSBill Taylor tavor_mr_mtt_unbind(tavor_state_t *state, tavor_bind_info_t *bind,
15969e39c5baSBill Taylor tavor_rsrc_t *mtt)
15979e39c5baSBill Taylor {
15989e39c5baSBill Taylor /*
15999e39c5baSBill Taylor * Free up the MTT entries and unbind the memory. Here, as above, we
16009e39c5baSBill Taylor * attempt to free these resources only if it is appropriate to do so.
16019e39c5baSBill Taylor */
16029e39c5baSBill Taylor tavor_mr_mem_unbind(state, bind);
16039e39c5baSBill Taylor tavor_rsrc_free(state, &mtt);
16049e39c5baSBill Taylor
16059e39c5baSBill Taylor return (DDI_SUCCESS);
16069e39c5baSBill Taylor }
16079e39c5baSBill Taylor
16089e39c5baSBill Taylor
16099e39c5baSBill Taylor /*
16109e39c5baSBill Taylor * tavor_mr_common_rereg()
16119e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
16129e39c5baSBill Taylor */
16139e39c5baSBill Taylor static int
tavor_mr_common_rereg(tavor_state_t * state,tavor_mrhdl_t mr,tavor_pdhdl_t pd,tavor_bind_info_t * bind,tavor_mrhdl_t * mrhdl_new,tavor_mr_options_t * op)16149e39c5baSBill Taylor tavor_mr_common_rereg(tavor_state_t *state, tavor_mrhdl_t mr,
16159e39c5baSBill Taylor tavor_pdhdl_t pd, tavor_bind_info_t *bind, tavor_mrhdl_t *mrhdl_new,
16169e39c5baSBill Taylor tavor_mr_options_t *op)
16179e39c5baSBill Taylor {
16189e39c5baSBill Taylor tavor_rsrc_t *mpt;
16199e39c5baSBill Taylor ibt_mr_attr_flags_t acc_flags_to_use;
16209e39c5baSBill Taylor ibt_mr_flags_t flags;
16219e39c5baSBill Taylor tavor_pdhdl_t pd_to_use;
16229e39c5baSBill Taylor tavor_hw_mpt_t mpt_entry;
16239e39c5baSBill Taylor uint64_t mtt_addr_to_use, vaddr_to_use, len_to_use;
16249e39c5baSBill Taylor uint_t sleep, dereg_level;
16259e39c5baSBill Taylor int status;
16269e39c5baSBill Taylor
16279e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
16289e39c5baSBill Taylor
16299e39c5baSBill Taylor /*
16309e39c5baSBill Taylor * Check here to see if the memory region corresponds to a userland
16319e39c5baSBill Taylor * mapping. Reregistration of userland memory regions is not
16329e39c5baSBill Taylor * currently supported. Return failure. XXX
16339e39c5baSBill Taylor */
16349e39c5baSBill Taylor if (mr->mr_is_umem) {
16359e39c5baSBill Taylor goto mrrereg_fail;
16369e39c5baSBill Taylor }
16379e39c5baSBill Taylor
16389e39c5baSBill Taylor mutex_enter(&mr->mr_lock);
16399e39c5baSBill Taylor
16409e39c5baSBill Taylor /* Pull MPT resource pointer from the Tavor Memory Region handle */
16419e39c5baSBill Taylor mpt = mr->mr_mptrsrcp;
16429e39c5baSBill Taylor
16439e39c5baSBill Taylor /* Extract the flags field from the tavor_bind_info_t */
16449e39c5baSBill Taylor flags = bind->bi_flags;
16459e39c5baSBill Taylor
16469e39c5baSBill Taylor /*
16479e39c5baSBill Taylor * Check the sleep flag. Ensure that it is consistent with the
16489e39c5baSBill Taylor * current thread context (i.e. if we are currently in the interrupt
16499e39c5baSBill Taylor * context, then we shouldn't be attempting to sleep).
16509e39c5baSBill Taylor */
16519e39c5baSBill Taylor sleep = (flags & IBT_MR_NOSLEEP) ? TAVOR_NOSLEEP: TAVOR_SLEEP;
16529e39c5baSBill Taylor if ((sleep == TAVOR_SLEEP) &&
16539e39c5baSBill Taylor (sleep != TAVOR_SLEEPFLAG_FOR_CONTEXT())) {
16549e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
16559e39c5baSBill Taylor goto mrrereg_fail;
16569e39c5baSBill Taylor }
16579e39c5baSBill Taylor
16589e39c5baSBill Taylor /*
16599e39c5baSBill Taylor * First step is to temporarily invalidate the MPT entry. This
16609e39c5baSBill Taylor * regains ownership from the hardware, and gives us the opportunity
16619e39c5baSBill Taylor * to modify the entry. Note: The HW2SW_MPT command returns the
16629e39c5baSBill Taylor * current MPT entry contents. These are saved away here because
16639e39c5baSBill Taylor * they will be reused in a later step below. If the region has
16649e39c5baSBill Taylor * bound memory windows that we fail returning an "in use" error code.
16659e39c5baSBill Taylor * Otherwise, this is an unexpected error and we deregister the
16669e39c5baSBill Taylor * memory region and return error.
16679e39c5baSBill Taylor *
16689e39c5baSBill Taylor * We use TAVOR_CMD_NOSLEEP_SPIN here always because we must protect
16699e39c5baSBill Taylor * against holding the lock around this rereg call in all contexts.
16709e39c5baSBill Taylor */
16719e39c5baSBill Taylor status = tavor_cmn_ownership_cmd_post(state, HW2SW_MPT, &mpt_entry,
16729e39c5baSBill Taylor sizeof (tavor_hw_mpt_t), mpt->tr_indx, TAVOR_CMD_NOSLEEP_SPIN);
16739e39c5baSBill Taylor if (status != TAVOR_CMD_SUCCESS) {
16749e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
16759e39c5baSBill Taylor if (status == TAVOR_CMD_REG_BOUND) {
16769e39c5baSBill Taylor return (IBT_MR_IN_USE);
16779e39c5baSBill Taylor } else {
16789e39c5baSBill Taylor cmn_err(CE_CONT, "Tavor: HW2SW_MPT command failed: "
16799e39c5baSBill Taylor "%08x\n", status);
16809e39c5baSBill Taylor
16819e39c5baSBill Taylor /*
16829e39c5baSBill Taylor * Call deregister and ensure that all current
16839e39c5baSBill Taylor * resources get freed up
16849e39c5baSBill Taylor */
16859e39c5baSBill Taylor if (tavor_mr_deregister(state, &mr,
16869e39c5baSBill Taylor TAVOR_MR_DEREG_ALL, sleep) != DDI_SUCCESS) {
16879e39c5baSBill Taylor TAVOR_WARNING(state, "failed to deregister "
16889e39c5baSBill Taylor "memory region");
16899e39c5baSBill Taylor }
16909e39c5baSBill Taylor return (ibc_get_ci_failure(0));
16919e39c5baSBill Taylor }
16929e39c5baSBill Taylor }
16939e39c5baSBill Taylor
16949e39c5baSBill Taylor /*
16959e39c5baSBill Taylor * If we're changing the protection domain, then validate the new one
16969e39c5baSBill Taylor */
16979e39c5baSBill Taylor if (flags & IBT_MR_CHANGE_PD) {
16989e39c5baSBill Taylor
16999e39c5baSBill Taylor /* Check for valid PD handle pointer */
17009e39c5baSBill Taylor if (pd == NULL) {
17019e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
17029e39c5baSBill Taylor /*
17039e39c5baSBill Taylor * Call deregister and ensure that all current
17049e39c5baSBill Taylor * resources get properly freed up. Unnecessary
17059e39c5baSBill Taylor * here to attempt to regain software ownership
17069e39c5baSBill Taylor * of the MPT entry as that has already been
17079e39c5baSBill Taylor * done above.
17089e39c5baSBill Taylor */
17099e39c5baSBill Taylor if (tavor_mr_deregister(state, &mr,
17109e39c5baSBill Taylor TAVOR_MR_DEREG_NO_HW2SW_MPT, sleep) !=
17119e39c5baSBill Taylor DDI_SUCCESS) {
17129e39c5baSBill Taylor TAVOR_WARNING(state, "failed to deregister "
17139e39c5baSBill Taylor "memory region");
17149e39c5baSBill Taylor }
17159e39c5baSBill Taylor goto mrrereg_fail;
17169e39c5baSBill Taylor }
17179e39c5baSBill Taylor
17189e39c5baSBill Taylor /* Use the new PD handle in all operations below */
17199e39c5baSBill Taylor pd_to_use = pd;
17209e39c5baSBill Taylor
17219e39c5baSBill Taylor } else {
17229e39c5baSBill Taylor /* Use the current PD handle in all operations below */
17239e39c5baSBill Taylor pd_to_use = mr->mr_pdhdl;
17249e39c5baSBill Taylor }
17259e39c5baSBill Taylor
17269e39c5baSBill Taylor /*
17279e39c5baSBill Taylor * If we're changing access permissions, then validate the new ones
17289e39c5baSBill Taylor */
17299e39c5baSBill Taylor if (flags & IBT_MR_CHANGE_ACCESS) {
17309e39c5baSBill Taylor /*
17319e39c5baSBill Taylor * Validate the access flags. Both remote write and remote
17329e39c5baSBill Taylor * atomic require the local write flag to be set
17339e39c5baSBill Taylor */
17349e39c5baSBill Taylor if (((flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
17359e39c5baSBill Taylor (flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
17369e39c5baSBill Taylor !(flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
17379e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
17389e39c5baSBill Taylor /*
17399e39c5baSBill Taylor * Call deregister and ensure that all current
17409e39c5baSBill Taylor * resources get properly freed up. Unnecessary
17419e39c5baSBill Taylor * here to attempt to regain software ownership
17429e39c5baSBill Taylor * of the MPT entry as that has already been
17439e39c5baSBill Taylor * done above.
17449e39c5baSBill Taylor */
17459e39c5baSBill Taylor if (tavor_mr_deregister(state, &mr,
17469e39c5baSBill Taylor TAVOR_MR_DEREG_NO_HW2SW_MPT, sleep) !=
17479e39c5baSBill Taylor DDI_SUCCESS) {
17489e39c5baSBill Taylor TAVOR_WARNING(state, "failed to deregister "
17499e39c5baSBill Taylor "memory region");
17509e39c5baSBill Taylor }
17519e39c5baSBill Taylor goto mrrereg_fail;
17529e39c5baSBill Taylor }
17539e39c5baSBill Taylor
17549e39c5baSBill Taylor /*
17559e39c5baSBill Taylor * Setup and validate the memory region access flags. This
17569e39c5baSBill Taylor * means translating the IBTF's enable flags into the access
17579e39c5baSBill Taylor * flags that will be used in later operations.
17589e39c5baSBill Taylor */
17599e39c5baSBill Taylor acc_flags_to_use = 0;
17609e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_WINDOW_BIND)
17619e39c5baSBill Taylor acc_flags_to_use |= IBT_MR_WINDOW_BIND;
17629e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_LOCAL_WRITE)
17639e39c5baSBill Taylor acc_flags_to_use |= IBT_MR_LOCAL_WRITE;
17649e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_READ)
17659e39c5baSBill Taylor acc_flags_to_use |= IBT_MR_REMOTE_READ;
17669e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_WRITE)
17679e39c5baSBill Taylor acc_flags_to_use |= IBT_MR_REMOTE_WRITE;
17689e39c5baSBill Taylor if (flags & IBT_MR_ENABLE_REMOTE_ATOMIC)
17699e39c5baSBill Taylor acc_flags_to_use |= IBT_MR_REMOTE_ATOMIC;
17709e39c5baSBill Taylor
17719e39c5baSBill Taylor } else {
17729e39c5baSBill Taylor acc_flags_to_use = mr->mr_accflag;
17739e39c5baSBill Taylor }
17749e39c5baSBill Taylor
17759e39c5baSBill Taylor /*
17769e39c5baSBill Taylor * If we're modifying the translation, then figure out whether
17779e39c5baSBill Taylor * we can reuse the current MTT resources. This means calling
17789e39c5baSBill Taylor * tavor_mr_rereg_xlat_helper() which does most of the heavy lifting
17799e39c5baSBill Taylor * for the reregistration. If the current memory region contains
17809e39c5baSBill Taylor * sufficient MTT entries for the new regions, then it will be
17819e39c5baSBill Taylor * reused and filled in. Otherwise, new entries will be allocated,
17829e39c5baSBill Taylor * the old ones will be freed, and the new entries will be filled
17839e39c5baSBill Taylor * in. Note: If we're not modifying the translation, then we
17849e39c5baSBill Taylor * should already have all the information we need to update the MPT.
17859e39c5baSBill Taylor * Also note: If tavor_mr_rereg_xlat_helper() fails, it will return
17869e39c5baSBill Taylor * a "dereg_level" which is the level of cleanup that needs to be
17879e39c5baSBill Taylor * passed to tavor_mr_deregister() to finish the cleanup.
17889e39c5baSBill Taylor */
17899e39c5baSBill Taylor if (flags & IBT_MR_CHANGE_TRANSLATION) {
17909e39c5baSBill Taylor status = tavor_mr_rereg_xlat_helper(state, mr, bind, op,
17919e39c5baSBill Taylor &mtt_addr_to_use, sleep, &dereg_level);
17929e39c5baSBill Taylor if (status != DDI_SUCCESS) {
17939e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
17949e39c5baSBill Taylor /*
17959e39c5baSBill Taylor * Call deregister and ensure that all resources get
17969e39c5baSBill Taylor * properly freed up.
17979e39c5baSBill Taylor */
17989e39c5baSBill Taylor if (tavor_mr_deregister(state, &mr, dereg_level,
17999e39c5baSBill Taylor sleep) != DDI_SUCCESS) {
18009e39c5baSBill Taylor TAVOR_WARNING(state, "failed to deregister "
18019e39c5baSBill Taylor "memory region");
18029e39c5baSBill Taylor }
18039e39c5baSBill Taylor
18049e39c5baSBill Taylor goto mrrereg_fail;
18059e39c5baSBill Taylor }
18069e39c5baSBill Taylor vaddr_to_use = mr->mr_bindinfo.bi_addr;
18079e39c5baSBill Taylor len_to_use = mr->mr_bindinfo.bi_len;
18089e39c5baSBill Taylor } else {
18099e39c5baSBill Taylor mtt_addr_to_use = (((uint64_t)mpt_entry.mttseg_addr_h << 32) |
18109e39c5baSBill Taylor ((uint64_t)mpt_entry.mttseg_addr_l << 6));
18119e39c5baSBill Taylor vaddr_to_use = mr->mr_bindinfo.bi_addr;
18129e39c5baSBill Taylor len_to_use = mr->mr_bindinfo.bi_len;
18139e39c5baSBill Taylor }
18149e39c5baSBill Taylor
18159e39c5baSBill Taylor /*
18169e39c5baSBill Taylor * Calculate new keys (Lkey, Rkey) from MPT index. Just like they were
18179e39c5baSBill Taylor * when the region was first registered, each key is formed from
18189e39c5baSBill Taylor * "constrained" bits and "unconstrained" bits. Note: If no remote
18199e39c5baSBill Taylor * access is required, then the RKey value is not filled in. Otherwise
18209e39c5baSBill Taylor * both Rkey and LKey are given the same value.
18219e39c5baSBill Taylor */
18229e39c5baSBill Taylor tavor_mr_keycalc(state, mpt->tr_indx, &mr->mr_lkey);
18239e39c5baSBill Taylor if ((acc_flags_to_use & IBT_MR_REMOTE_READ) ||
18249e39c5baSBill Taylor (acc_flags_to_use & IBT_MR_REMOTE_WRITE) ||
18259e39c5baSBill Taylor (acc_flags_to_use & IBT_MR_REMOTE_ATOMIC)) {
18269e39c5baSBill Taylor mr->mr_rkey = mr->mr_lkey;
18279e39c5baSBill Taylor }
18289e39c5baSBill Taylor
18299e39c5baSBill Taylor /*
18309e39c5baSBill Taylor * Update the MPT entry with the new information. Some of this
18319e39c5baSBill Taylor * information is retained from the previous operation, some of
18329e39c5baSBill Taylor * it is new based on request.
18339e39c5baSBill Taylor */
18349e39c5baSBill Taylor mpt_entry.en_bind = (acc_flags_to_use & IBT_MR_WINDOW_BIND) ? 1 : 0;
18359e39c5baSBill Taylor mpt_entry.atomic = (acc_flags_to_use & IBT_MR_REMOTE_ATOMIC) ? 1 : 0;
18369e39c5baSBill Taylor mpt_entry.rw = (acc_flags_to_use & IBT_MR_REMOTE_WRITE) ? 1 : 0;
18379e39c5baSBill Taylor mpt_entry.rr = (acc_flags_to_use & IBT_MR_REMOTE_READ) ? 1 : 0;
18389e39c5baSBill Taylor mpt_entry.lw = (acc_flags_to_use & IBT_MR_LOCAL_WRITE) ? 1 : 0;
18399e39c5baSBill Taylor mpt_entry.page_sz = mr->mr_logmttpgsz - 0xC;
18409e39c5baSBill Taylor mpt_entry.mem_key = mr->mr_lkey;
18419e39c5baSBill Taylor mpt_entry.pd = pd_to_use->pd_pdnum;
18429e39c5baSBill Taylor mpt_entry.start_addr = vaddr_to_use;
18439e39c5baSBill Taylor mpt_entry.reg_win_len = len_to_use;
18449e39c5baSBill Taylor mpt_entry.mttseg_addr_h = mtt_addr_to_use >> 32;
18459e39c5baSBill Taylor mpt_entry.mttseg_addr_l = mtt_addr_to_use >> 6;
18469e39c5baSBill Taylor
18479e39c5baSBill Taylor /*
18489e39c5baSBill Taylor * Write the updated MPT entry to hardware
18499e39c5baSBill Taylor *
18509e39c5baSBill Taylor * We use TAVOR_CMD_NOSLEEP_SPIN here always because we must protect
18519e39c5baSBill Taylor * against holding the lock around this rereg call in all contexts.
18529e39c5baSBill Taylor */
18539e39c5baSBill Taylor status = tavor_cmn_ownership_cmd_post(state, SW2HW_MPT, &mpt_entry,
18549e39c5baSBill Taylor sizeof (tavor_hw_mpt_t), mpt->tr_indx, TAVOR_CMD_NOSLEEP_SPIN);
18559e39c5baSBill Taylor if (status != TAVOR_CMD_SUCCESS) {
18569e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
18579e39c5baSBill Taylor cmn_err(CE_CONT, "Tavor: SW2HW_MPT command failed: %08x\n",
18589e39c5baSBill Taylor status);
18599e39c5baSBill Taylor /*
18609e39c5baSBill Taylor * Call deregister and ensure that all current resources get
18619e39c5baSBill Taylor * properly freed up. Unnecessary here to attempt to regain
18629e39c5baSBill Taylor * software ownership of the MPT entry as that has already
18639e39c5baSBill Taylor * been done above.
18649e39c5baSBill Taylor */
18659e39c5baSBill Taylor if (tavor_mr_deregister(state, &mr,
18669e39c5baSBill Taylor TAVOR_MR_DEREG_NO_HW2SW_MPT, sleep) != DDI_SUCCESS) {
18679e39c5baSBill Taylor TAVOR_WARNING(state, "failed to deregister memory "
18689e39c5baSBill Taylor "region");
18699e39c5baSBill Taylor }
18709e39c5baSBill Taylor return (ibc_get_ci_failure(0));
18719e39c5baSBill Taylor }
18729e39c5baSBill Taylor
18739e39c5baSBill Taylor /*
18749e39c5baSBill Taylor * If we're changing PD, then update their reference counts now.
18759e39c5baSBill Taylor * This means decrementing the reference count on the old PD and
18769e39c5baSBill Taylor * incrementing the reference count on the new PD.
18779e39c5baSBill Taylor */
18789e39c5baSBill Taylor if (flags & IBT_MR_CHANGE_PD) {
18799e39c5baSBill Taylor tavor_pd_refcnt_dec(mr->mr_pdhdl);
18809e39c5baSBill Taylor tavor_pd_refcnt_inc(pd);
18819e39c5baSBill Taylor }
18829e39c5baSBill Taylor
18839e39c5baSBill Taylor /*
18849e39c5baSBill Taylor * Update the contents of the Tavor Memory Region handle to reflect
18859e39c5baSBill Taylor * what has been changed.
18869e39c5baSBill Taylor */
18879e39c5baSBill Taylor mr->mr_pdhdl = pd_to_use;
18889e39c5baSBill Taylor mr->mr_accflag = acc_flags_to_use;
18899e39c5baSBill Taylor mr->mr_is_umem = 0;
18909e39c5baSBill Taylor mr->mr_umemcookie = NULL;
18919e39c5baSBill Taylor
18929e39c5baSBill Taylor /* New MR handle is same as the old */
18939e39c5baSBill Taylor *mrhdl_new = mr;
18949e39c5baSBill Taylor mutex_exit(&mr->mr_lock);
18959e39c5baSBill Taylor
18969e39c5baSBill Taylor return (DDI_SUCCESS);
18979e39c5baSBill Taylor
18989e39c5baSBill Taylor mrrereg_fail:
18999e39c5baSBill Taylor return (status);
19009e39c5baSBill Taylor }
19019e39c5baSBill Taylor
19029e39c5baSBill Taylor
19039e39c5baSBill Taylor /*
19049e39c5baSBill Taylor * tavor_mr_rereg_xlat_helper
19059e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
19069e39c5baSBill Taylor * Note: This routine expects the "mr_lock" to be held when it
19079e39c5baSBill Taylor * is called. Upon returning failure, this routine passes information
19089e39c5baSBill Taylor * about what "dereg_level" should be passed to tavor_mr_deregister().
19099e39c5baSBill Taylor */
19109e39c5baSBill Taylor static int
tavor_mr_rereg_xlat_helper(tavor_state_t * state,tavor_mrhdl_t mr,tavor_bind_info_t * bind,tavor_mr_options_t * op,uint64_t * mtt_addr,uint_t sleep,uint_t * dereg_level)19119e39c5baSBill Taylor tavor_mr_rereg_xlat_helper(tavor_state_t *state, tavor_mrhdl_t mr,
19129e39c5baSBill Taylor tavor_bind_info_t *bind, tavor_mr_options_t *op, uint64_t *mtt_addr,
19139e39c5baSBill Taylor uint_t sleep, uint_t *dereg_level)
19149e39c5baSBill Taylor {
19159e39c5baSBill Taylor tavor_rsrc_pool_info_t *rsrc_pool;
19169e39c5baSBill Taylor tavor_rsrc_t *mtt, *mtt_refcnt;
19179e39c5baSBill Taylor tavor_sw_refcnt_t *swrc_old, *swrc_new;
19189e39c5baSBill Taylor ddi_dma_handle_t dmahdl;
19199e39c5baSBill Taylor uint64_t nummtt_needed, nummtt_in_currrsrc, max_sz;
19209e39c5baSBill Taylor uint64_t mtt_ddrbaseaddr;
19219e39c5baSBill Taylor uint_t mtt_pgsize_bits, bind_type, reuse_dmahdl;
19229e39c5baSBill Taylor int status;
19239e39c5baSBill Taylor
19249e39c5baSBill Taylor ASSERT(MUTEX_HELD(&mr->mr_lock));
19259e39c5baSBill Taylor
19269e39c5baSBill Taylor /*
19279e39c5baSBill Taylor * Check the "options" flag. Currently this flag tells the driver
19289e39c5baSBill Taylor * whether or not the region should be bound normally (i.e. with
19299e39c5baSBill Taylor * entries written into the PCI IOMMU) or whether it should be
19309e39c5baSBill Taylor * registered to bypass the IOMMU.
19319e39c5baSBill Taylor */
19329e39c5baSBill Taylor if (op == NULL) {
19339e39c5baSBill Taylor bind_type = TAVOR_BINDMEM_NORMAL;
19349e39c5baSBill Taylor } else {
19359e39c5baSBill Taylor bind_type = op->mro_bind_type;
19369e39c5baSBill Taylor }
19379e39c5baSBill Taylor
19389e39c5baSBill Taylor /*
19399e39c5baSBill Taylor * Check for invalid length. Check is the length is zero or if the
19409e39c5baSBill Taylor * length is larger than the maximum configured value. Return error
19419e39c5baSBill Taylor * if it is.
19429e39c5baSBill Taylor */
19439e39c5baSBill Taylor max_sz = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_mrw_sz);
19449e39c5baSBill Taylor if ((bind->bi_len == 0) || (bind->bi_len > max_sz)) {
19459e39c5baSBill Taylor /*
19469e39c5baSBill Taylor * Deregister will be called upon returning failure from this
19479e39c5baSBill Taylor * routine. This will ensure that all current resources get
19489e39c5baSBill Taylor * properly freed up. Unnecessary to attempt to regain
19499e39c5baSBill Taylor * software ownership of the MPT entry as that has already
19509e39c5baSBill Taylor * been done above (in tavor_mr_reregister())
19519e39c5baSBill Taylor */
19529e39c5baSBill Taylor *dereg_level = TAVOR_MR_DEREG_NO_HW2SW_MPT;
19539e39c5baSBill Taylor
19549e39c5baSBill Taylor goto mrrereghelp_fail;
19559e39c5baSBill Taylor }
19569e39c5baSBill Taylor
19579e39c5baSBill Taylor /*
19589e39c5baSBill Taylor * Determine the number of pages necessary for new region and the
19599e39c5baSBill Taylor * number of pages supported by the current MTT resources
19609e39c5baSBill Taylor */
19619e39c5baSBill Taylor nummtt_needed = tavor_mr_nummtt_needed(state, bind, &mtt_pgsize_bits);
19629e39c5baSBill Taylor nummtt_in_currrsrc = mr->mr_mttrsrcp->tr_len >> TAVOR_MTT_SIZE_SHIFT;
19639e39c5baSBill Taylor
19649e39c5baSBill Taylor /*
19659e39c5baSBill Taylor * Depending on whether we have enough pages or not, the next step is
19669e39c5baSBill Taylor * to fill in a set of MTT entries that reflect the new mapping. In
19679e39c5baSBill Taylor * the first case below, we already have enough entries. This means
19689e39c5baSBill Taylor * we need to unbind the memory from the previous mapping, bind the
19699e39c5baSBill Taylor * memory for the new mapping, write the new MTT entries, and update
19709e39c5baSBill Taylor * the mr to reflect the changes.
19719e39c5baSBill Taylor * In the second case below, we do not have enough entries in the
19729e39c5baSBill Taylor * current mapping. So, in this case, we need not only to unbind the
19739e39c5baSBill Taylor * current mapping, but we need to free up the MTT resources associated
19749e39c5baSBill Taylor * with that mapping. After we've successfully done that, we continue
19759e39c5baSBill Taylor * by binding the new memory, allocating new MTT entries, writing the
19769e39c5baSBill Taylor * new MTT entries, and updating the mr to reflect the changes.
19779e39c5baSBill Taylor */
19789e39c5baSBill Taylor
19799e39c5baSBill Taylor /*
19809e39c5baSBill Taylor * If this region is being shared (i.e. MTT refcount != 1), then we
19819e39c5baSBill Taylor * can't reuse the current MTT resources regardless of their size.
19829e39c5baSBill Taylor * Instead we'll need to alloc new ones (below) just as if there
19839e39c5baSBill Taylor * hadn't been enough room in the current entries.
19849e39c5baSBill Taylor */
19859e39c5baSBill Taylor swrc_old = (tavor_sw_refcnt_t *)mr->mr_mttrefcntp->tr_addr;
19869e39c5baSBill Taylor if (TAVOR_MTT_IS_NOT_SHARED(swrc_old) &&
19879e39c5baSBill Taylor (nummtt_needed <= nummtt_in_currrsrc)) {
19889e39c5baSBill Taylor
19899e39c5baSBill Taylor /*
19909e39c5baSBill Taylor * Unbind the old mapping for this memory region, but retain
19919e39c5baSBill Taylor * the ddi_dma_handle_t (if possible) for reuse in the bind
19929e39c5baSBill Taylor * operation below. Note: If original memory region was
19939e39c5baSBill Taylor * bound for IOMMU bypass and the new region can not use
19949e39c5baSBill Taylor * bypass, then a new DMA handle will be necessary.
19959e39c5baSBill Taylor */
19969e39c5baSBill Taylor if (TAVOR_MR_REUSE_DMAHDL(mr, bind->bi_flags)) {
19979e39c5baSBill Taylor mr->mr_bindinfo.bi_free_dmahdl = 0;
19989e39c5baSBill Taylor tavor_mr_mem_unbind(state, &mr->mr_bindinfo);
19999e39c5baSBill Taylor dmahdl = mr->mr_bindinfo.bi_dmahdl;
20009e39c5baSBill Taylor reuse_dmahdl = 1;
20019e39c5baSBill Taylor } else {
20029e39c5baSBill Taylor tavor_mr_mem_unbind(state, &mr->mr_bindinfo);
20039e39c5baSBill Taylor dmahdl = NULL;
20049e39c5baSBill Taylor reuse_dmahdl = 0;
20059e39c5baSBill Taylor }
20069e39c5baSBill Taylor
20079e39c5baSBill Taylor /*
20089e39c5baSBill Taylor * Bind the new memory and determine the mapped addresses.
20099e39c5baSBill Taylor * As described, this routine and tavor_mr_fast_mtt_write()
20109e39c5baSBill Taylor * do the majority of the work for the memory registration
20119e39c5baSBill Taylor * operations. Note: When we successfully finish the binding,
20129e39c5baSBill Taylor * we will set the "bi_free_dmahdl" flag to indicate that
20139e39c5baSBill Taylor * even though we may have reused the ddi_dma_handle_t we do
20149e39c5baSBill Taylor * wish it to be freed up at some later time. Note also that
20159e39c5baSBill Taylor * if we fail, we may need to cleanup the ddi_dma_handle_t.
20169e39c5baSBill Taylor */
20179e39c5baSBill Taylor bind->bi_bypass = bind_type;
20189e39c5baSBill Taylor status = tavor_mr_mem_bind(state, bind, dmahdl, sleep);
20199e39c5baSBill Taylor if (status != DDI_SUCCESS) {
20209e39c5baSBill Taylor if (reuse_dmahdl) {
20219e39c5baSBill Taylor ddi_dma_free_handle(&dmahdl);
20229e39c5baSBill Taylor }
20239e39c5baSBill Taylor
20249e39c5baSBill Taylor /*
20259e39c5baSBill Taylor * Deregister will be called upon returning failure
20269e39c5baSBill Taylor * from this routine. This will ensure that all
20279e39c5baSBill Taylor * current resources get properly freed up.
20289e39c5baSBill Taylor * Unnecessary to attempt to regain software ownership
20299e39c5baSBill Taylor * of the MPT entry as that has already been done
20309e39c5baSBill Taylor * above (in tavor_mr_reregister()). Also unnecessary
20319e39c5baSBill Taylor * to attempt to unbind the memory.
20329e39c5baSBill Taylor */
20339e39c5baSBill Taylor *dereg_level = TAVOR_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
20349e39c5baSBill Taylor
20359e39c5baSBill Taylor goto mrrereghelp_fail;
20369e39c5baSBill Taylor }
20379e39c5baSBill Taylor if (reuse_dmahdl) {
20389e39c5baSBill Taylor bind->bi_free_dmahdl = 1;
20399e39c5baSBill Taylor }
20409e39c5baSBill Taylor
20419e39c5baSBill Taylor /*
20429e39c5baSBill Taylor * Using the new mapping, but reusing the current MTT
20439e39c5baSBill Taylor * resources, write the updated entries to MTT
20449e39c5baSBill Taylor */
20459e39c5baSBill Taylor mtt = mr->mr_mttrsrcp;
20469e39c5baSBill Taylor status = tavor_mr_fast_mtt_write(mtt, bind, mtt_pgsize_bits);
20479e39c5baSBill Taylor if (status != DDI_SUCCESS) {
20489e39c5baSBill Taylor /*
20499e39c5baSBill Taylor * Deregister will be called upon returning failure
20509e39c5baSBill Taylor * from this routine. This will ensure that all
20519e39c5baSBill Taylor * current resources get properly freed up.
20529e39c5baSBill Taylor * Unnecessary to attempt to regain software ownership
20539e39c5baSBill Taylor * of the MPT entry as that has already been done
20549e39c5baSBill Taylor * above (in tavor_mr_reregister()). Also unnecessary
20559e39c5baSBill Taylor * to attempt to unbind the memory.
20569e39c5baSBill Taylor *
20579e39c5baSBill Taylor * But we do need to unbind the newly bound memory
20589e39c5baSBill Taylor * before returning.
20599e39c5baSBill Taylor */
20609e39c5baSBill Taylor tavor_mr_mem_unbind(state, bind);
20619e39c5baSBill Taylor *dereg_level = TAVOR_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
20629e39c5baSBill Taylor
20639e39c5baSBill Taylor goto mrrereghelp_fail;
20649e39c5baSBill Taylor }
20659e39c5baSBill Taylor
20669e39c5baSBill Taylor /* Put the updated information into the Mem Region handle */
20679e39c5baSBill Taylor mr->mr_bindinfo = *bind;
20689e39c5baSBill Taylor mr->mr_logmttpgsz = mtt_pgsize_bits;
20699e39c5baSBill Taylor
20709e39c5baSBill Taylor } else {
20719e39c5baSBill Taylor /*
20729e39c5baSBill Taylor * Check if the memory region MTT is shared by any other MRs.
20739e39c5baSBill Taylor * Since the resource may be shared between multiple memory
20749e39c5baSBill Taylor * regions (as a result of a "RegisterSharedMR()" verb) it is
20759e39c5baSBill Taylor * important that we not unbind any resources prematurely.
20769e39c5baSBill Taylor */
20779e39c5baSBill Taylor if (!TAVOR_MTT_IS_SHARED(swrc_old)) {
20789e39c5baSBill Taylor /*
20799e39c5baSBill Taylor * Unbind the old mapping for this memory region, but
20809e39c5baSBill Taylor * retain the ddi_dma_handle_t for reuse in the bind
20819e39c5baSBill Taylor * operation below. Note: This can only be done here
20829e39c5baSBill Taylor * because the region being reregistered is not
20839e39c5baSBill Taylor * currently shared. Also if original memory region
20849e39c5baSBill Taylor * was bound for IOMMU bypass and the new region can
20859e39c5baSBill Taylor * not use bypass, then a new DMA handle will be
20869e39c5baSBill Taylor * necessary.
20879e39c5baSBill Taylor */
20889e39c5baSBill Taylor if (TAVOR_MR_REUSE_DMAHDL(mr, bind->bi_flags)) {
20899e39c5baSBill Taylor mr->mr_bindinfo.bi_free_dmahdl = 0;
20909e39c5baSBill Taylor tavor_mr_mem_unbind(state, &mr->mr_bindinfo);
20919e39c5baSBill Taylor dmahdl = mr->mr_bindinfo.bi_dmahdl;
20929e39c5baSBill Taylor reuse_dmahdl = 1;
20939e39c5baSBill Taylor } else {
20949e39c5baSBill Taylor tavor_mr_mem_unbind(state, &mr->mr_bindinfo);
20959e39c5baSBill Taylor dmahdl = NULL;
20969e39c5baSBill Taylor reuse_dmahdl = 0;
20979e39c5baSBill Taylor }
20989e39c5baSBill Taylor } else {
20999e39c5baSBill Taylor dmahdl = NULL;
21009e39c5baSBill Taylor reuse_dmahdl = 0;
21019e39c5baSBill Taylor }
21029e39c5baSBill Taylor
21039e39c5baSBill Taylor /*
21049e39c5baSBill Taylor * Bind the new memory and determine the mapped addresses.
21059e39c5baSBill Taylor * As described, this routine and tavor_mr_fast_mtt_write()
21069e39c5baSBill Taylor * do the majority of the work for the memory registration
21079e39c5baSBill Taylor * operations. Note: When we successfully finish the binding,
21089e39c5baSBill Taylor * we will set the "bi_free_dmahdl" flag to indicate that
21099e39c5baSBill Taylor * even though we may have reused the ddi_dma_handle_t we do
21109e39c5baSBill Taylor * wish it to be freed up at some later time. Note also that
21119e39c5baSBill Taylor * if we fail, we may need to cleanup the ddi_dma_handle_t.
21129e39c5baSBill Taylor */
21139e39c5baSBill Taylor bind->bi_bypass = bind_type;
21149e39c5baSBill Taylor status = tavor_mr_mem_bind(state, bind, dmahdl, sleep);
21159e39c5baSBill Taylor if (status != DDI_SUCCESS) {
21169e39c5baSBill Taylor if (reuse_dmahdl) {
21179e39c5baSBill Taylor ddi_dma_free_handle(&dmahdl);
21189e39c5baSBill Taylor }
21199e39c5baSBill Taylor
21209e39c5baSBill Taylor /*
21219e39c5baSBill Taylor * Deregister will be called upon returning failure
21229e39c5baSBill Taylor * from this routine. This will ensure that all
21239e39c5baSBill Taylor * current resources get properly freed up.
21249e39c5baSBill Taylor * Unnecessary to attempt to regain software ownership
21259e39c5baSBill Taylor * of the MPT entry as that has already been done
21269e39c5baSBill Taylor * above (in tavor_mr_reregister()). Also unnecessary
21279e39c5baSBill Taylor * to attempt to unbind the memory.
21289e39c5baSBill Taylor */
21299e39c5baSBill Taylor *dereg_level = TAVOR_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
21309e39c5baSBill Taylor
21319e39c5baSBill Taylor goto mrrereghelp_fail;
21329e39c5baSBill Taylor }
21339e39c5baSBill Taylor if (reuse_dmahdl) {
21349e39c5baSBill Taylor bind->bi_free_dmahdl = 1;
21359e39c5baSBill Taylor }
21369e39c5baSBill Taylor
21379e39c5baSBill Taylor /*
21389e39c5baSBill Taylor * Allocate the new MTT entries resource
21399e39c5baSBill Taylor */
21409e39c5baSBill Taylor status = tavor_rsrc_alloc(state, TAVOR_MTT,
21419e39c5baSBill Taylor TAVOR_NUMMTT_TO_MTTSEG(nummtt_needed), sleep, &mtt);
21429e39c5baSBill Taylor if (status != DDI_SUCCESS) {
21439e39c5baSBill Taylor /*
21449e39c5baSBill Taylor * Deregister will be called upon returning failure
21459e39c5baSBill Taylor * from this routine. This will ensure that all
21469e39c5baSBill Taylor * current resources get properly freed up.
21479e39c5baSBill Taylor * Unnecessary to attempt to regain software ownership
21489e39c5baSBill Taylor * of the MPT entry as that has already been done
21499e39c5baSBill Taylor * above (in tavor_mr_reregister()). Also unnecessary
21509e39c5baSBill Taylor * to attempt to unbind the memory.
21519e39c5baSBill Taylor *
21529e39c5baSBill Taylor * But we do need to unbind the newly bound memory
21539e39c5baSBill Taylor * before returning.
21549e39c5baSBill Taylor */
21559e39c5baSBill Taylor tavor_mr_mem_unbind(state, bind);
21569e39c5baSBill Taylor *dereg_level = TAVOR_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
21579e39c5baSBill Taylor
21589e39c5baSBill Taylor goto mrrereghelp_fail;
21599e39c5baSBill Taylor }
21609e39c5baSBill Taylor
21619e39c5baSBill Taylor /*
21629e39c5baSBill Taylor * Allocate MTT reference count (to track shared memory
21639e39c5baSBill Taylor * regions). As mentioned elsewhere above, this reference
21649e39c5baSBill Taylor * count resource may never be used on the given memory region,
21659e39c5baSBill Taylor * but if it is ever later registered as a "shared" memory
21669e39c5baSBill Taylor * region then this resource will be necessary. Note: This
21679e39c5baSBill Taylor * is only necessary here if the existing memory region is
21689e39c5baSBill Taylor * already being shared (because otherwise we already have
21699e39c5baSBill Taylor * a useable reference count resource).
21709e39c5baSBill Taylor */
21719e39c5baSBill Taylor if (TAVOR_MTT_IS_SHARED(swrc_old)) {
21729e39c5baSBill Taylor status = tavor_rsrc_alloc(state, TAVOR_REFCNT, 1,
21739e39c5baSBill Taylor sleep, &mtt_refcnt);
21749e39c5baSBill Taylor if (status != DDI_SUCCESS) {
21759e39c5baSBill Taylor /*
21769e39c5baSBill Taylor * Deregister will be called upon returning
21779e39c5baSBill Taylor * failure from this routine. This will ensure
21789e39c5baSBill Taylor * that all current resources get properly
21799e39c5baSBill Taylor * freed up. Unnecessary to attempt to regain
21809e39c5baSBill Taylor * software ownership of the MPT entry as that
21819e39c5baSBill Taylor * has already been done above (in
21829e39c5baSBill Taylor * tavor_mr_reregister()). Also unnecessary
21839e39c5baSBill Taylor * to attempt to unbind the memory.
21849e39c5baSBill Taylor *
21859e39c5baSBill Taylor * But we need to unbind the newly bound
21869e39c5baSBill Taylor * memory and free up the newly allocated MTT
21879e39c5baSBill Taylor * entries before returning.
21889e39c5baSBill Taylor */
21899e39c5baSBill Taylor tavor_mr_mem_unbind(state, bind);
21909e39c5baSBill Taylor tavor_rsrc_free(state, &mtt);
21919e39c5baSBill Taylor *dereg_level =
21929e39c5baSBill Taylor TAVOR_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
21939e39c5baSBill Taylor
21949e39c5baSBill Taylor goto mrrereghelp_fail;
21959e39c5baSBill Taylor }
21969e39c5baSBill Taylor swrc_new = (tavor_sw_refcnt_t *)mtt_refcnt->tr_addr;
21979e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*swrc_new))
21989e39c5baSBill Taylor TAVOR_MTT_REFCNT_INIT(swrc_new);
21999e39c5baSBill Taylor } else {
22009e39c5baSBill Taylor mtt_refcnt = mr->mr_mttrefcntp;
22019e39c5baSBill Taylor }
22029e39c5baSBill Taylor
22039e39c5baSBill Taylor /*
22049e39c5baSBill Taylor * Using the new mapping and the new MTT resources, write the
22059e39c5baSBill Taylor * updated entries to MTT
22069e39c5baSBill Taylor */
22079e39c5baSBill Taylor status = tavor_mr_fast_mtt_write(mtt, bind, mtt_pgsize_bits);
22089e39c5baSBill Taylor if (status != DDI_SUCCESS) {
22099e39c5baSBill Taylor /*
22109e39c5baSBill Taylor * Deregister will be called upon returning failure
22119e39c5baSBill Taylor * from this routine. This will ensure that all
22129e39c5baSBill Taylor * current resources get properly freed up.
22139e39c5baSBill Taylor * Unnecessary to attempt to regain software ownership
22149e39c5baSBill Taylor * of the MPT entry as that has already been done
22159e39c5baSBill Taylor * above (in tavor_mr_reregister()). Also unnecessary
22169e39c5baSBill Taylor * to attempt to unbind the memory.
22179e39c5baSBill Taylor *
22189e39c5baSBill Taylor * But we need to unbind the newly bound memory,
22199e39c5baSBill Taylor * free up the newly allocated MTT entries, and
22209e39c5baSBill Taylor * (possibly) free the new MTT reference count
22219e39c5baSBill Taylor * resource before returning.
22229e39c5baSBill Taylor */
22239e39c5baSBill Taylor if (TAVOR_MTT_IS_SHARED(swrc_old)) {
22249e39c5baSBill Taylor tavor_rsrc_free(state, &mtt_refcnt);
22259e39c5baSBill Taylor }
22269e39c5baSBill Taylor tavor_mr_mem_unbind(state, bind);
22279e39c5baSBill Taylor tavor_rsrc_free(state, &mtt);
22289e39c5baSBill Taylor *dereg_level = TAVOR_MR_DEREG_NO_HW2SW_MPT_OR_UNBIND;
22299e39c5baSBill Taylor
22309e39c5baSBill Taylor goto mrrereghelp_fail;
22319e39c5baSBill Taylor }
22329e39c5baSBill Taylor
22339e39c5baSBill Taylor /*
22349e39c5baSBill Taylor * Check if the memory region MTT is shared by any other MRs.
22359e39c5baSBill Taylor * Since the resource may be shared between multiple memory
22369e39c5baSBill Taylor * regions (as a result of a "RegisterSharedMR()" verb) it is
22379e39c5baSBill Taylor * important that we not free up any resources prematurely.
22389e39c5baSBill Taylor */
22399e39c5baSBill Taylor if (TAVOR_MTT_IS_SHARED(swrc_old)) {
22409e39c5baSBill Taylor /* Decrement MTT reference count for "old" region */
22419e39c5baSBill Taylor (void) tavor_mtt_refcnt_dec(mr->mr_mttrefcntp);
22429e39c5baSBill Taylor } else {
22439e39c5baSBill Taylor /* Free up the old MTT entries resource */
22449e39c5baSBill Taylor tavor_rsrc_free(state, &mr->mr_mttrsrcp);
22459e39c5baSBill Taylor }
22469e39c5baSBill Taylor
22479e39c5baSBill Taylor /* Put the updated information into the mrhdl */
22489e39c5baSBill Taylor mr->mr_bindinfo = *bind;
22499e39c5baSBill Taylor mr->mr_logmttpgsz = mtt_pgsize_bits;
22509e39c5baSBill Taylor mr->mr_mttrsrcp = mtt;
22519e39c5baSBill Taylor mr->mr_mttrefcntp = mtt_refcnt;
22529e39c5baSBill Taylor }
22539e39c5baSBill Taylor
22549e39c5baSBill Taylor /*
22559e39c5baSBill Taylor * Calculate and return the updated MTT address (in the DDR address
22569e39c5baSBill Taylor * space). This will be used by the caller (tavor_mr_reregister) in
22579e39c5baSBill Taylor * the updated MPT entry
22589e39c5baSBill Taylor */
22599e39c5baSBill Taylor rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
22609e39c5baSBill Taylor mtt_ddrbaseaddr = (uint64_t)(uintptr_t)rsrc_pool->rsrc_ddr_offset;
22619e39c5baSBill Taylor *mtt_addr = mtt_ddrbaseaddr + (mtt->tr_indx <<
22629e39c5baSBill Taylor TAVOR_MTT_SIZE_SHIFT);
22639e39c5baSBill Taylor
22649e39c5baSBill Taylor return (DDI_SUCCESS);
22659e39c5baSBill Taylor
22669e39c5baSBill Taylor mrrereghelp_fail:
22679e39c5baSBill Taylor return (status);
22689e39c5baSBill Taylor }
22699e39c5baSBill Taylor
22709e39c5baSBill Taylor
22719e39c5baSBill Taylor /*
22729e39c5baSBill Taylor * tavor_mr_nummtt_needed()
22739e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
22749e39c5baSBill Taylor */
22759e39c5baSBill Taylor /* ARGSUSED */
22769e39c5baSBill Taylor static uint64_t
tavor_mr_nummtt_needed(tavor_state_t * state,tavor_bind_info_t * bind,uint_t * mtt_pgsize_bits)22779e39c5baSBill Taylor tavor_mr_nummtt_needed(tavor_state_t *state, tavor_bind_info_t *bind,
22789e39c5baSBill Taylor uint_t *mtt_pgsize_bits)
22799e39c5baSBill Taylor {
22809e39c5baSBill Taylor uint64_t pg_offset_mask;
22819e39c5baSBill Taylor uint64_t pg_offset, tmp_length;
22829e39c5baSBill Taylor
22839e39c5baSBill Taylor /*
22849e39c5baSBill Taylor * For now we specify the page size as 8Kb (the default page size for
22859e39c5baSBill Taylor * the sun4u architecture), or 4Kb for x86. Figure out optimal page
22869e39c5baSBill Taylor * size by examining the dmacookies XXX
22879e39c5baSBill Taylor */
22889e39c5baSBill Taylor *mtt_pgsize_bits = PAGESHIFT;
22899e39c5baSBill Taylor
22909e39c5baSBill Taylor pg_offset_mask = ((uint64_t)1 << *mtt_pgsize_bits) - 1;
22919e39c5baSBill Taylor pg_offset = bind->bi_addr & pg_offset_mask;
22929e39c5baSBill Taylor tmp_length = pg_offset + (bind->bi_len - 1);
22939e39c5baSBill Taylor return ((tmp_length >> *mtt_pgsize_bits) + 1);
22949e39c5baSBill Taylor }
22959e39c5baSBill Taylor
22969e39c5baSBill Taylor
22979e39c5baSBill Taylor /*
22989e39c5baSBill Taylor * tavor_mr_mem_bind()
22999e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
23009e39c5baSBill Taylor */
23019e39c5baSBill Taylor static int
tavor_mr_mem_bind(tavor_state_t * state,tavor_bind_info_t * bind,ddi_dma_handle_t dmahdl,uint_t sleep)23029e39c5baSBill Taylor tavor_mr_mem_bind(tavor_state_t *state, tavor_bind_info_t *bind,
23039e39c5baSBill Taylor ddi_dma_handle_t dmahdl, uint_t sleep)
23049e39c5baSBill Taylor {
23059e39c5baSBill Taylor ddi_dma_attr_t dma_attr;
23069e39c5baSBill Taylor int (*callback)(caddr_t);
23079e39c5baSBill Taylor uint_t dma_xfer_mode;
23089e39c5baSBill Taylor int status;
23099e39c5baSBill Taylor
23109e39c5baSBill Taylor /* bi_type must be set to a meaningful value to get a bind handle */
23119e39c5baSBill Taylor ASSERT(bind->bi_type == TAVOR_BINDHDL_VADDR ||
23129e39c5baSBill Taylor bind->bi_type == TAVOR_BINDHDL_BUF ||
23139e39c5baSBill Taylor bind->bi_type == TAVOR_BINDHDL_UBUF);
23149e39c5baSBill Taylor
23159e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
23169e39c5baSBill Taylor
23179e39c5baSBill Taylor /* Set the callback flag appropriately */
23189e39c5baSBill Taylor callback = (sleep == TAVOR_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
23199e39c5baSBill Taylor
23209e39c5baSBill Taylor /* Determine whether to map STREAMING or CONSISTENT */
23219e39c5baSBill Taylor dma_xfer_mode = (bind->bi_flags & IBT_MR_NONCOHERENT) ?
23229e39c5baSBill Taylor DDI_DMA_STREAMING : DDI_DMA_CONSISTENT;
23239e39c5baSBill Taylor
23249e39c5baSBill Taylor /*
23259e39c5baSBill Taylor * Initialize many of the default DMA attributes. Then, if we're
23269e39c5baSBill Taylor * bypassing the IOMMU, set the DDI_DMA_FORCE_PHYSICAL flag.
23279e39c5baSBill Taylor */
23289e39c5baSBill Taylor if (dmahdl == NULL) {
23299e39c5baSBill Taylor tavor_dma_attr_init(&dma_attr);
23309e39c5baSBill Taylor #ifdef __sparc
23319e39c5baSBill Taylor /*
23329e39c5baSBill Taylor * First, disable streaming and switch to consistent if
23339e39c5baSBill Taylor * configured to do so and IOMMU BYPASS is enabled.
23349e39c5baSBill Taylor */
23359e39c5baSBill Taylor if (state->ts_cfg_profile->cp_disable_streaming_on_bypass &&
23369e39c5baSBill Taylor dma_xfer_mode == DDI_DMA_STREAMING &&
23379e39c5baSBill Taylor bind->bi_bypass == TAVOR_BINDMEM_BYPASS) {
23389e39c5baSBill Taylor dma_xfer_mode = DDI_DMA_CONSISTENT;
23399e39c5baSBill Taylor }
23409e39c5baSBill Taylor
23419e39c5baSBill Taylor /*
23429e39c5baSBill Taylor * Then, if streaming is still specified, then "bypass" is not
23439e39c5baSBill Taylor * allowed.
23449e39c5baSBill Taylor */
23459e39c5baSBill Taylor if ((dma_xfer_mode == DDI_DMA_CONSISTENT) &&
23469e39c5baSBill Taylor (bind->bi_bypass == TAVOR_BINDMEM_BYPASS)) {
23479e39c5baSBill Taylor dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;
23489e39c5baSBill Taylor }
23499e39c5baSBill Taylor #endif
23509e39c5baSBill Taylor /* Allocate a DMA handle for the binding */
23519e39c5baSBill Taylor status = ddi_dma_alloc_handle(state->ts_dip, &dma_attr,
23529e39c5baSBill Taylor callback, NULL, &bind->bi_dmahdl);
23539e39c5baSBill Taylor if (status != DDI_SUCCESS) {
23549e39c5baSBill Taylor return (status);
23559e39c5baSBill Taylor }
23569e39c5baSBill Taylor bind->bi_free_dmahdl = 1;
23579e39c5baSBill Taylor
23589e39c5baSBill Taylor } else {
23599e39c5baSBill Taylor bind->bi_dmahdl = dmahdl;
23609e39c5baSBill Taylor bind->bi_free_dmahdl = 0;
23619e39c5baSBill Taylor }
23629e39c5baSBill Taylor
23639e39c5baSBill Taylor /*
23649e39c5baSBill Taylor * Bind the memory to get the PCI mapped addresses. The decision
23659e39c5baSBill Taylor * to call ddi_dma_addr_bind_handle() or ddi_dma_buf_bind_handle()
23669e39c5baSBill Taylor * is determined by the "bi_type" flag. Note: if the bind operation
23679e39c5baSBill Taylor * fails then we have to free up the DMA handle and return error.
23689e39c5baSBill Taylor */
23699e39c5baSBill Taylor if (bind->bi_type == TAVOR_BINDHDL_VADDR) {
23709e39c5baSBill Taylor status = ddi_dma_addr_bind_handle(bind->bi_dmahdl, NULL,
23719e39c5baSBill Taylor (caddr_t)(uintptr_t)bind->bi_addr, bind->bi_len,
23729e39c5baSBill Taylor (DDI_DMA_RDWR | dma_xfer_mode), callback, NULL,
23739e39c5baSBill Taylor &bind->bi_dmacookie, &bind->bi_cookiecnt);
23749e39c5baSBill Taylor } else { /* TAVOR_BINDHDL_BUF || TAVOR_BINDHDL_UBUF */
23759e39c5baSBill Taylor status = ddi_dma_buf_bind_handle(bind->bi_dmahdl,
23769e39c5baSBill Taylor bind->bi_buf, (DDI_DMA_RDWR | dma_xfer_mode), callback,
23779e39c5baSBill Taylor NULL, &bind->bi_dmacookie, &bind->bi_cookiecnt);
23789e39c5baSBill Taylor }
23799e39c5baSBill Taylor
23809e39c5baSBill Taylor if (status != DDI_DMA_MAPPED) {
23819e39c5baSBill Taylor if (bind->bi_free_dmahdl != 0) {
23829e39c5baSBill Taylor ddi_dma_free_handle(&bind->bi_dmahdl);
23839e39c5baSBill Taylor }
23849e39c5baSBill Taylor return (status);
23859e39c5baSBill Taylor }
23869e39c5baSBill Taylor
23879e39c5baSBill Taylor return (DDI_SUCCESS);
23889e39c5baSBill Taylor }
23899e39c5baSBill Taylor
23909e39c5baSBill Taylor
23919e39c5baSBill Taylor /*
23929e39c5baSBill Taylor * tavor_mr_mem_unbind()
23939e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
23949e39c5baSBill Taylor */
23959e39c5baSBill Taylor static void
tavor_mr_mem_unbind(tavor_state_t * state,tavor_bind_info_t * bind)23969e39c5baSBill Taylor tavor_mr_mem_unbind(tavor_state_t *state, tavor_bind_info_t *bind)
23979e39c5baSBill Taylor {
23989e39c5baSBill Taylor int status;
23999e39c5baSBill Taylor
24009e39c5baSBill Taylor /*
24019e39c5baSBill Taylor * In case of TAVOR_BINDHDL_UBUF, the memory bi_buf points to
24029e39c5baSBill Taylor * is actually allocated by ddi_umem_iosetup() internally, then
24039e39c5baSBill Taylor * it's required to free it here. Reset bi_type to TAVOR_BINDHDL_NONE
24049e39c5baSBill Taylor * not to free it again later.
24059e39c5baSBill Taylor */
24069e39c5baSBill Taylor _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*bind))
24079e39c5baSBill Taylor if (bind->bi_type == TAVOR_BINDHDL_UBUF) {
24089e39c5baSBill Taylor freerbuf(bind->bi_buf);
24099e39c5baSBill Taylor bind->bi_type = TAVOR_BINDHDL_NONE;
24109e39c5baSBill Taylor }
24119e39c5baSBill Taylor _NOTE(NOW_VISIBLE_TO_OTHER_THREADS(*bind))
24129e39c5baSBill Taylor
24139e39c5baSBill Taylor /*
24149e39c5baSBill Taylor * Unbind the DMA memory for the region
24159e39c5baSBill Taylor *
24169e39c5baSBill Taylor * Note: The only way ddi_dma_unbind_handle() currently
24179e39c5baSBill Taylor * can return an error is if the handle passed in is invalid.
24189e39c5baSBill Taylor * Since this should never happen, we choose to return void
24199e39c5baSBill Taylor * from this function! If this does return an error, however,
24209e39c5baSBill Taylor * then we print a warning message to the console.
24219e39c5baSBill Taylor */
24229e39c5baSBill Taylor status = ddi_dma_unbind_handle(bind->bi_dmahdl);
24239e39c5baSBill Taylor if (status != DDI_SUCCESS) {
24249e39c5baSBill Taylor TAVOR_WARNING(state, "failed to unbind DMA mapping");
24259e39c5baSBill Taylor return;
24269e39c5baSBill Taylor }
24279e39c5baSBill Taylor
24289e39c5baSBill Taylor /* Free up the DMA handle */
24299e39c5baSBill Taylor if (bind->bi_free_dmahdl != 0) {
24309e39c5baSBill Taylor ddi_dma_free_handle(&bind->bi_dmahdl);
24319e39c5baSBill Taylor }
24329e39c5baSBill Taylor }
24339e39c5baSBill Taylor
24349e39c5baSBill Taylor
24359e39c5baSBill Taylor /*
24369e39c5baSBill Taylor * tavor_mr_fast_mtt_write()
24379e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
24389e39c5baSBill Taylor */
24399e39c5baSBill Taylor static int
tavor_mr_fast_mtt_write(tavor_rsrc_t * mtt,tavor_bind_info_t * bind,uint32_t mtt_pgsize_bits)24409e39c5baSBill Taylor tavor_mr_fast_mtt_write(tavor_rsrc_t *mtt, tavor_bind_info_t *bind,
24419e39c5baSBill Taylor uint32_t mtt_pgsize_bits)
24429e39c5baSBill Taylor {
24439e39c5baSBill Taylor ddi_dma_cookie_t dmacookie;
24449e39c5baSBill Taylor uint_t cookie_cnt;
24459e39c5baSBill Taylor uint64_t *mtt_table;
24469e39c5baSBill Taylor uint64_t mtt_entry;
24479e39c5baSBill Taylor uint64_t addr, endaddr;
24489e39c5baSBill Taylor uint64_t pagesize;
24499e39c5baSBill Taylor int i;
24509e39c5baSBill Taylor
24519e39c5baSBill Taylor /* Calculate page size from the suggested value passed in */
24529e39c5baSBill Taylor pagesize = ((uint64_t)1 << mtt_pgsize_bits);
24539e39c5baSBill Taylor
24549e39c5baSBill Taylor /*
24559e39c5baSBill Taylor * Walk the "cookie list" and fill in the MTT table entries
24569e39c5baSBill Taylor */
24579e39c5baSBill Taylor i = 0;
24589e39c5baSBill Taylor mtt_table = (uint64_t *)mtt->tr_addr;
24599e39c5baSBill Taylor dmacookie = bind->bi_dmacookie;
24609e39c5baSBill Taylor cookie_cnt = bind->bi_cookiecnt;
24619e39c5baSBill Taylor while (cookie_cnt-- > 0) {
24629e39c5baSBill Taylor addr = dmacookie.dmac_laddress;
24639e39c5baSBill Taylor endaddr = addr + (dmacookie.dmac_size - 1);
24649e39c5baSBill Taylor addr = addr & ~((uint64_t)pagesize - 1);
24659e39c5baSBill Taylor while (addr <= endaddr) {
24669e39c5baSBill Taylor /*
24679e39c5baSBill Taylor * Fill in the mapped addresses (calculated above) and
24689e39c5baSBill Taylor * set TAVOR_MTT_ENTRY_PRESET flag for each MTT entry.
24699e39c5baSBill Taylor */
24709e39c5baSBill Taylor mtt_entry = addr | TAVOR_MTT_ENTRY_PRESET;
24719e39c5baSBill Taylor ddi_put64(mtt->tr_acchdl, &mtt_table[i], mtt_entry);
24729e39c5baSBill Taylor addr += pagesize;
24739e39c5baSBill Taylor i++;
24749e39c5baSBill Taylor
24759e39c5baSBill Taylor if (addr == 0) {
24769e39c5baSBill Taylor static int do_once = 1;
24779e39c5baSBill Taylor _NOTE(SCHEME_PROTECTS_DATA("safe sharing",
24789e39c5baSBill Taylor do_once))
24799e39c5baSBill Taylor if (do_once) {
24809e39c5baSBill Taylor do_once = 0;
24819e39c5baSBill Taylor cmn_err(CE_NOTE, "probable error in "
24829e39c5baSBill Taylor "dma_cookie address from caller\n");
24839e39c5baSBill Taylor }
24849e39c5baSBill Taylor break;
24859e39c5baSBill Taylor }
24869e39c5baSBill Taylor }
24879e39c5baSBill Taylor
24889e39c5baSBill Taylor /*
24899e39c5baSBill Taylor * When we've reached the end of the current DMA cookie,
24909e39c5baSBill Taylor * jump to the next cookie (if there are more)
24919e39c5baSBill Taylor */
24929e39c5baSBill Taylor if (cookie_cnt != 0) {
24939e39c5baSBill Taylor ddi_dma_nextcookie(bind->bi_dmahdl, &dmacookie);
24949e39c5baSBill Taylor }
24959e39c5baSBill Taylor }
24969e39c5baSBill Taylor
24979e39c5baSBill Taylor return (DDI_SUCCESS);
24989e39c5baSBill Taylor }
24999e39c5baSBill Taylor
25009e39c5baSBill Taylor /*
25019e39c5baSBill Taylor * tavor_mtt_refcnt_inc()
25029e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
25039e39c5baSBill Taylor */
25049e39c5baSBill Taylor static int
tavor_mtt_refcnt_inc(tavor_rsrc_t * rsrc)25059e39c5baSBill Taylor tavor_mtt_refcnt_inc(tavor_rsrc_t *rsrc)
25069e39c5baSBill Taylor {
25079e39c5baSBill Taylor tavor_sw_refcnt_t *rc;
25089e39c5baSBill Taylor uint32_t cnt;
25099e39c5baSBill Taylor
25109e39c5baSBill Taylor rc = (tavor_sw_refcnt_t *)rsrc->tr_addr;
25119e39c5baSBill Taylor
25129e39c5baSBill Taylor /* Increment the MTT's reference count */
25139e39c5baSBill Taylor mutex_enter(&rc->swrc_lock);
25149e39c5baSBill Taylor cnt = rc->swrc_refcnt++;
25159e39c5baSBill Taylor mutex_exit(&rc->swrc_lock);
25169e39c5baSBill Taylor
25179e39c5baSBill Taylor return (cnt);
25189e39c5baSBill Taylor }
25199e39c5baSBill Taylor
25209e39c5baSBill Taylor
25219e39c5baSBill Taylor /*
25229e39c5baSBill Taylor * tavor_mtt_refcnt_dec()
25239e39c5baSBill Taylor * Context: Can be called from interrupt or base context.
25249e39c5baSBill Taylor */
25259e39c5baSBill Taylor static int
tavor_mtt_refcnt_dec(tavor_rsrc_t * rsrc)25269e39c5baSBill Taylor tavor_mtt_refcnt_dec(tavor_rsrc_t *rsrc)
25279e39c5baSBill Taylor {
25289e39c5baSBill Taylor tavor_sw_refcnt_t *rc;
25299e39c5baSBill Taylor uint32_t cnt;
25309e39c5baSBill Taylor
25319e39c5baSBill Taylor rc = (tavor_sw_refcnt_t *)rsrc->tr_addr;
25329e39c5baSBill Taylor
25339e39c5baSBill Taylor /* Decrement the MTT's reference count */
25349e39c5baSBill Taylor mutex_enter(&rc->swrc_lock);
25359e39c5baSBill Taylor cnt = --rc->swrc_refcnt;
25369e39c5baSBill Taylor mutex_exit(&rc->swrc_lock);
25379e39c5baSBill Taylor
25389e39c5baSBill Taylor return (cnt);
25399e39c5baSBill Taylor }
2540