xref: /illumos-gate/usr/src/lib/udapl/udapl_tavor/common/dapl_rmr_bind.c (revision fb4cdc19f4c1650a26fa4b7f4faf5a036225c7d4)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2002-2003, Network Appliance, Inc. All rights reserved.
24  */
25 
26 /*
27  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
28  * Use is subject to license terms.
29  */
30 
31 /*
32  *
33  * MODULE: dapl_rmr_bind.c
34  *
35  * PURPOSE: Memory management
36  * Description: Interfaces in this file are completely described in
37  *		the DAPL 1.1 API, Chapter 6, section 6
38  *
39  * $Id: dapl_rmr_bind.c,v 1.14 2003/07/11 18:23:31 jlentini Exp $
40  */
41 
42 #include "dapl.h"
43 #include "dapl_rmr_util.h"
44 #include "dapl_ep_util.h"
45 #include "dapl_cookie.h"
46 #include "dapl_adapter_util.h"
47 
48 /*
49  *
50  * Function Prototypes
51  *
52  */
53 
54 static DAT_RETURN
55 dapli_rmr_bind_fuse(
56     IN  DAPL_RMR		*rmr,
57     IN  const DAT_LMR_TRIPLET 	*lmr_triplet,
58     IN  DAT_MEM_PRIV_FLAGS 	mem_priv,
59     IN  DAPL_EP 		*ep,
60     IN  DAT_RMR_COOKIE 		user_cookie,
61     IN  DAT_COMPLETION_FLAGS	completion_flags,
62     OUT DAT_RMR_CONTEXT 	*rmr_context);
63 
64 static DAT_RETURN
65 dapli_rmr_bind_unfuse(
66     IN  DAPL_RMR		*rmr,
67     IN  const DAT_LMR_TRIPLET 	*lmr_triplet,
68     IN  DAPL_EP 		*ep,
69     IN  DAT_RMR_COOKIE 		user_cookie,
70     IN  DAT_COMPLETION_FLAGS 	completion_flags);
71 
72 
73 /*
74  *
75  * Function Definitions
76  *
77  */
78 
79 static DAT_RETURN
dapli_rmr_bind_fuse(IN DAPL_RMR * rmr,IN const DAT_LMR_TRIPLET * lmr_triplet,IN DAT_MEM_PRIV_FLAGS mem_priv,IN DAPL_EP * ep_ptr,IN DAT_RMR_COOKIE user_cookie,IN DAT_COMPLETION_FLAGS completion_flags,OUT DAT_RMR_CONTEXT * rmr_context)80 dapli_rmr_bind_fuse(
81     IN  DAPL_RMR		*rmr,
82     IN  const DAT_LMR_TRIPLET* 	lmr_triplet,
83     IN  DAT_MEM_PRIV_FLAGS 	mem_priv,
84     IN  DAPL_EP 		*ep_ptr,
85     IN  DAT_RMR_COOKIE 		user_cookie,
86     IN  DAT_COMPLETION_FLAGS	completion_flags,
87     OUT DAT_RMR_CONTEXT 	*rmr_context)
88 {
89 	DAPL_LMR 			*lmr;
90 	DAPL_COOKIE			*cookie;
91 	DAT_RETURN 			dat_status;
92 
93 	dat_status = dapls_hash_search(
94 	    rmr->header.owner_ia->hca_ptr->lmr_hash_table,
95 	    lmr_triplet->lmr_context,
96 	    (DAPL_HASH_DATA *) &lmr);
97 	if (DAT_SUCCESS != dat_status) {
98 		dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG2);
99 		goto bail;
100 	}
101 
102 	/*
103 	 * if the ep in unconnected return an error. IB requires that the
104 	 * QP be connected to change a memory window binding since:
105 	 *
106 	 * - memory window bind operations are WQEs placed on a QP's
107 	 *   send queue
108 	 *
109 	 * - QP's only process WQEs on the send queue when the QP is in
110 	 *   the RTS state
111 	 */
112 	if (DAT_EP_STATE_CONNECTED != ep_ptr->param.ep_state) {
113 		dat_status = DAT_ERROR(DAT_INVALID_STATE,
114 		    dapls_ep_state_subtype(ep_ptr));
115 		goto bail;
116 	}
117 
118 	if (DAT_FALSE == dapl_mr_bounds_check(
119 	    dapl_mr_get_address(lmr->param.region_desc, lmr->param.mem_type),
120 	    lmr->param.length,
121 	    lmr_triplet->virtual_address,
122 	    lmr_triplet->segment_length)) {
123 		dat_status = DAT_ERROR(DAT_INVALID_PARAMETER,
124 		    DAT_INVALID_ARG2);
125 		goto bail;
126 	}
127 
128 	/* If the LMR, RMR, and EP are not in the same PZ, there is an error */
129 	if ((ep_ptr->param.pz_handle != lmr->param.pz_handle) ||
130 	    (ep_ptr->param.pz_handle != rmr->param.pz_handle)) {
131 		dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG4);
132 		goto bail;
133 	}
134 
135 	if (!dapl_rmr_validate_completion_flag(DAT_COMPLETION_SUPPRESS_FLAG,
136 	    ep_ptr->param.ep_attr.request_completion_flags, completion_flags) ||
137 	    !dapl_rmr_validate_completion_flag(DAT_COMPLETION_UNSIGNALLED_FLAG,
138 	    ep_ptr->param.ep_attr.request_completion_flags, completion_flags) ||
139 	    !dapl_rmr_validate_completion_flag(
140 	    DAT_COMPLETION_BARRIER_FENCE_FLAG,
141 	    ep_ptr->param.ep_attr.request_completion_flags, completion_flags)) {
142 		dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG4);
143 		goto bail;
144 	}
145 
146 	dat_status = dapls_rmr_cookie_alloc(&ep_ptr->req_buffer,
147 	    rmr, user_cookie, &cookie);
148 	if (DAT_SUCCESS != dat_status) {
149 		goto bail;
150 	}
151 
152 	dat_status = dapls_ib_mw_bind(rmr,
153 	    lmr_triplet->lmr_context,
154 	    ep_ptr,
155 	    cookie,
156 	    lmr_triplet->virtual_address,
157 	    lmr_triplet->segment_length,
158 	    mem_priv,
159 	    completion_flags);
160 	if (DAT_SUCCESS != dat_status) {
161 		dapls_cookie_dealloc(&ep_ptr->req_buffer, cookie);
162 		goto bail;
163 	}
164 
165 	dapl_os_atomic_inc(&lmr->lmr_ref_count);
166 
167 	/* if the RMR was previously bound */
168 	if (NULL != rmr->lmr) {
169 		dapl_os_atomic_dec(&rmr->lmr->lmr_ref_count);
170 	}
171 
172 	rmr->param.mem_priv = mem_priv;
173 	rmr->param.lmr_triplet = *lmr_triplet;
174 	rmr->ep = ep_ptr;
175 	rmr->lmr = lmr;
176 
177 	dapl_os_atomic_inc(&ep_ptr->req_count);
178 
179 	if (NULL != rmr_context) { *rmr_context = rmr->param.rmr_context; }
180 bail:
181 	return (dat_status);
182 }
183 
184 
185 static DAT_RETURN
dapli_rmr_bind_unfuse(IN DAPL_RMR * rmr,IN const DAT_LMR_TRIPLET * lmr_triplet,IN DAPL_EP * ep_ptr,IN DAT_RMR_COOKIE user_cookie,IN DAT_COMPLETION_FLAGS completion_flags)186 dapli_rmr_bind_unfuse(
187     IN  DAPL_RMR		*rmr,
188     IN  const DAT_LMR_TRIPLET 	*lmr_triplet,
189     IN  DAPL_EP 		*ep_ptr,
190     IN  DAT_RMR_COOKIE 		user_cookie,
191     IN  DAT_COMPLETION_FLAGS 	completion_flags)
192 {
193 	DAPL_COOKIE			*cookie;
194 	DAT_RETURN 			dat_status;
195 
196 	dat_status = DAT_SUCCESS;
197 	/*
198 	 * if the ep in unconnected return an error. IB requires that the
199 	 * QP be connected to change a memory window binding since:
200 	 *
201 	 * - memory window bind operations are WQEs placed on a QP's
202 	 *   send queue
203 	 *
204 	 * - QP's only process WQEs on the send queue when the QP is in
205 	 *   the RTS state
206 	 */
207 	if (DAT_EP_STATE_CONNECTED != ep_ptr->param.ep_state) {
208 		dat_status = DAT_ERROR(DAT_INVALID_STATE,
209 		    dapls_ep_state_subtype(ep_ptr));
210 		goto bail1;
211 	}
212 
213 	/* If the RMR and EP are not in the same PZ, there is an error */
214 	if (ep_ptr->param.pz_handle != rmr->param.pz_handle) {
215 		dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG2);
216 		goto bail1;
217 	}
218 
219 	if (!dapl_rmr_validate_completion_flag(DAT_COMPLETION_SUPPRESS_FLAG,
220 	    ep_ptr->param.ep_attr.request_completion_flags, completion_flags) ||
221 	    !dapl_rmr_validate_completion_flag(DAT_COMPLETION_UNSIGNALLED_FLAG,
222 	    ep_ptr->param.ep_attr.request_completion_flags, completion_flags) ||
223 	    !dapl_rmr_validate_completion_flag(
224 	    DAT_COMPLETION_BARRIER_FENCE_FLAG,
225 	    ep_ptr->param.ep_attr.request_completion_flags, completion_flags)) {
226 		dat_status = DAT_ERROR(DAT_INVALID_PARAMETER, DAT_INVALID_ARG2);
227 		goto bail1;
228 	}
229 
230 	dat_status = dapls_rmr_cookie_alloc(&ep_ptr->req_buffer, rmr,
231 	    user_cookie, &cookie);
232 	if (DAT_SUCCESS != dat_status) {
233 		goto bail1;
234 	}
235 
236 
237 	dat_status = dapls_ib_mw_unbind(rmr, lmr_triplet->lmr_context,
238 	    ep_ptr, cookie, completion_flags);
239 
240 	if (DAT_SUCCESS != dat_status) {
241 		dapls_cookie_dealloc(&ep_ptr->req_buffer, cookie);
242 		goto bail1;
243 	}
244 
245 	/* if the RMR was previously bound */
246 	if (NULL != rmr->lmr) {
247 		dapl_os_atomic_dec(&rmr->lmr->lmr_ref_count);
248 	}
249 
250 	rmr->param.mem_priv = DAT_MEM_PRIV_NONE_FLAG;
251 	rmr->param.lmr_triplet.lmr_context = 0;
252 	rmr->param.lmr_triplet.virtual_address = 0;
253 	rmr->param.lmr_triplet.segment_length = 0;
254 	rmr->ep = ep_ptr;
255 	rmr->lmr = NULL;
256 
257 	dapl_os_atomic_inc(&ep_ptr->req_count);
258 
259 bail1:
260 	return (dat_status);
261 }
262 
263 
264 /*
265  * dapl_rmr_bind
266  *
267  * DAPL Requirements Version xxx, 6.6.4.4
268  *
269  * Bind the RMR to the specified memory region within the LMR and
270  * provide a new rmr_context value.
271  *
272  * Input:
273  * Output:
274  */
275 DAT_RETURN
dapl_rmr_bind(IN DAT_RMR_HANDLE rmr_handle,IN const DAT_LMR_TRIPLET * lmr_triplet,IN DAT_MEM_PRIV_FLAGS mem_priv,IN DAT_EP_HANDLE ep_handle,IN DAT_RMR_COOKIE user_cookie,IN DAT_COMPLETION_FLAGS completion_flags,OUT DAT_RMR_CONTEXT * rmr_context)276 dapl_rmr_bind(
277 	IN	DAT_RMR_HANDLE		rmr_handle,
278 	IN	const DAT_LMR_TRIPLET	*lmr_triplet,
279 	IN	DAT_MEM_PRIV_FLAGS	mem_priv,
280 	IN	DAT_EP_HANDLE		ep_handle,
281 	IN	DAT_RMR_COOKIE		user_cookie,
282 	IN	DAT_COMPLETION_FLAGS 	completion_flags,
283 	OUT	DAT_RMR_CONTEXT		*rmr_context)
284 {
285 	DAPL_RMR				*rmr;
286 	DAPL_EP 				*ep_ptr;
287 
288 	if (DAPL_BAD_HANDLE(rmr_handle, DAPL_MAGIC_RMR)) {
289 		return (DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_RMR));
290 	}
291 	if (DAPL_BAD_HANDLE(ep_handle, DAPL_MAGIC_EP)) {
292 		return (DAT_ERROR(DAT_INVALID_HANDLE, DAT_INVALID_HANDLE_EP));
293 	}
294 
295 	rmr = (DAPL_RMR *) rmr_handle;
296 	ep_ptr = (DAPL_EP *) ep_handle;
297 
298 	/* if the rmr should be bound */
299 	if (0 != lmr_triplet->segment_length) {
300 		return (dapli_rmr_bind_fuse(rmr,
301 		    lmr_triplet,
302 		    mem_priv,
303 		    ep_ptr,
304 		    user_cookie,
305 		    completion_flags,
306 		    rmr_context));
307 	} else { /* the rmr should be unbound */
308 		return (dapli_rmr_bind_unfuse(rmr,
309 		    lmr_triplet,
310 		    ep_ptr,
311 		    user_cookie,
312 		    completion_flags));
313 	}
314 }
315